mulmocast 2.8.0 → 2.9.0

package/lib/actions/image_agents.d.ts

@@ -24,6 +24,9 @@ type ImagePreprocessAgentReturnValue = {
     };
     firstFrameImagePath?: string;
     lastFrameImagePath?: string;
+    firstFrameIsBeatImage?: boolean;
+    lastFrameIsBeatImage?: boolean;
+    frameFillColor?: string;
     movieReferenceImages?: {
         imagePath: string;
         referenceType: "ASSET" | "STYLE";
@@ -54,6 +57,7 @@ type ImagePluginPreprocessAgentResponse = ImagePreprocessAgentResponseBase & {
     markdown: string;
     html: string;
 };
+export declare const conformFrameImageToCanvas: (context: MulmoStudioContext, imageName: string, imagePath: string, fillColor: string) => Promise<string>;
 type ImagePreprocessAgentResponse = ImagePreprocessAgentResponseBase | ImageHtmlPreprocessAgentResponse | ImagePluginPreprocessAgentResponse | ImageOnlyMoviePreprocessAgentResponse | ImageGenearalPreprocessAgentResponse;
 export declare const imagePreprocessAgent: (namedInputs: {
     context: MulmoStudioContext;
@@ -61,6 +65,23 @@ export declare const imagePreprocessAgent: (namedInputs: {
     index: number;
     imageRefs?: Record<string, string>;
 }) => Promise<ImagePreprocessAgentResponse>;
+export type BeatFramePreprocessorInput = {
+    imagePath?: string;
+    lastFrameImagePath?: string;
+    firstFrameIsBeatImage?: boolean;
+    lastFrameIsBeatImage?: boolean;
+    frameFillColor?: string;
+    referenceImageForMovie?: string;
+};
+export declare const beatFrameResolverAgent: (namedInputs: {
+    context: MulmoStudioContext;
+    beat: MulmoBeat;
+    index: number;
+    preprocessor?: BeatFramePreprocessorInput;
+}) => Promise<{
+    lastFrameImagePath: string | undefined;
+    referenceImageForMovie: string | undefined;
+}>;
 export declare const imagePluginAgent: (namedInputs: {
     context: MulmoStudioContext;
     beat: MulmoBeat;

package/lib/actions/image_agents.js

@@ -1,6 +1,11 @@
+import fs from "fs";
+import { createHash } from "crypto";
 import { GraphAILogger } from "graphai";
+import { BEAT_IMAGE_SENTINEL, ImageMediaType, beatImageTypesWithoutOwnStill } from "../types/schema.js";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods, MulmoBeatMethods, MulmoMediaSourceMethods } from "../methods/index.js";
-import { getBeatPngImagePath, getBeatMoviePaths, getBeatAnimatedVideoPath, getAudioFilePath, getGroupedAudioFilePath } from "../utils/file.js";
+import { getBeatPngImagePath, getBeatMoviePaths, getBeatAnimatedVideoPath, getAudioFilePath, getGroupedAudioFilePath, getReferenceImagePath, } from "../utils/file.js";
+import { ffmpegGetImageDimensions, padImageToCanvas } from "../utils/ffmpeg_utils.js";
+import { beatImageSentinelError, bothFramesTarget, missingBeatImageTarget, unsupportedImageTypeTarget, missingFirstFrameTarget } from "../utils/error_cause.js";
 import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
 import { renderHTMLToImage } from "../utils/html_render.js";
 import { beatId } from "../utils/utils.js";
@@ -11,6 +16,121 @@ const htmlStyle = (context, beat) => {
         textSlideStyle: MulmoPresentationStyleMethods.getTextSlideStyle(context.presentationStyle, beat),
     };
 };
+// Image-to-video models expect first/last frame images that match the video canvas.
+// Generated reference images come back at the provider's fixed sizes (e.g. gpt-image emits
+// 1536x1024), so aspect-fit pad them to the canvas before movie generation.
+const conformingInFlight = new Map();
+// Aspect-matching sources never get a destPath, so without this memo every beat
+// (and every rerun) would ffprobe them again. Keyed by path + mtime.
+const imageDimensionsCache = new Map();
+const getImageDimensionsCached = async (imagePath, mtimeMs) => {
+    const key = `${imagePath}:${mtimeMs}`;
+    const cached = imageDimensionsCache.get(key);
+    if (cached) {
+        return cached;
+    }
+    const dimensions = await ffmpegGetImageDimensions(imagePath);
+    imageDimensionsCache.set(key, dimensions);
+    return dimensions;
+};
+export const conformFrameImageToCanvas = async (context, imageName, imagePath, fillColor) => {
+    if (!fs.existsSync(imagePath)) {
+        return imagePath; // mock agents / dry runs
+    }
+    const canvasSize = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
+    // The cache key includes the source path and fill color: the same ref name can resolve
+    // to different sources in different beats, and a color change must invalidate the cache.
+    const digest = createHash("sha256").update(`${imagePath}|${fillColor}`).digest("hex").slice(0, 8);
+    const destPath = getReferenceImagePath(context, `${imageName}_fit_${canvasSize.width}x${canvasSize.height}_${digest}`, "png");
+    const inFlight = conformingInFlight.get(destPath);
+    if (inFlight) {
+        return inFlight;
+    }
+    const promise = (async () => {
+        const sourceMtimeMs = fs.statSync(imagePath).mtimeMs;
+        if (fs.existsSync(destPath) && fs.statSync(destPath).mtimeMs >= sourceMtimeMs) {
+            return destPath;
+        }
+        const { width, height } = await getImageDimensionsCached(imagePath, sourceMtimeMs);
+        if (Math.abs(width / height - canvasSize.width / canvasSize.height) < 0.01) {
+            return imagePath;
+        }
+        GraphAILogger.info(`conformFrameImageToCanvas: padding ${imageName} (${width}x${height}) to ${canvasSize.width}x${canvasSize.height}`);
+        try {
+            await padImageToCanvas(imagePath, destPath, canvasSize.width, canvasSize.height, fillColor);
+        }
+        catch (error) {
+            fs.rmSync(destPath, { force: true }); // don't let a partial file poison the mtime cache
+            throw error;
+        }
+        return destPath;
+    })();
+    conformingInFlight.set(destPath, promise);
+    try {
+        return await promise;
+    }
+    finally {
+        conformingInFlight.delete(destPath);
+    }
+};
+const validateBeatImageSentinel = (movieParams, beat, index) => {
+    if (movieParams.firstFrameImageName === BEAT_IMAGE_SENTINEL && movieParams.lastFrameImageName === BEAT_IMAGE_SENTINEL) {
+        throw new Error(`${BEAT_IMAGE_SENTINEL} cannot be used for both firstFrameImageName and lastFrameImageName. beat: ${index}`, {
+            cause: beatImageSentinelError(index, bothFramesTarget),
+        });
+    }
+    if (!beat.imagePrompt && !beat.image) {
+        throw new Error(`${BEAT_IMAGE_SENTINEL} requires the beat to have an imagePrompt or image. beat: ${index}`, {
+            cause: beatImageSentinelError(index, missingBeatImageTarget),
+        });
+    }
+    if (beat.image && beatImageTypesWithoutOwnStill.includes(beat.image.type)) {
+        throw new Error(`${BEAT_IMAGE_SENTINEL} cannot be used with image type "${beat.image.type}" (the beat image is not its own still). beat: ${index}`, {
+            cause: beatImageSentinelError(index, unsupportedImageTypeTarget),
+        });
+    }
+    // The implicit first frame (the raw beat image) is not conformed to the canvas, so pairing
+    // it with a conformed $beatImage last frame would feed mismatched frame sizes to strict
+    // image-to-video models (e.g. wan-2.2-i2v-fast). Require an explicit, conformed first frame.
+    if (movieParams.lastFrameImageName === BEAT_IMAGE_SENTINEL && !movieParams.firstFrameImageName) {
+        throw new Error(`${BEAT_IMAGE_SENTINEL} as lastFrameImageName requires firstFrameImageName. beat: ${index}`, {
+            cause: beatImageSentinelError(index, missingFirstFrameTarget),
+        });
+    }
+};
+// Resolve firstFrameImageName/lastFrameImageName. Named refs conform to the canvas here;
+// the $beatImage sentinel only sets flags (the beat image does not exist yet at preprocess
+// time) and is resolved by beatFrameResolverAgent after image generation.
+const applyFrameImageParams = async (returnValue, context, beat, index, movieParams, imageRefs) => {
+    const frameFillColor = movieParams.frameFillColor ?? "black";
+    // A style-level $beatImage merges into every beat; beats that generate no movie
+    // (no moviePrompt) ignore the sentinel instead of failing validation.
+    const sentinelActive = Boolean(beat.moviePrompt);
+    const firstFrameImageName = !sentinelActive && movieParams.firstFrameImageName === BEAT_IMAGE_SENTINEL ? undefined : movieParams.firstFrameImageName;
+    const lastFrameImageName = !sentinelActive && movieParams.lastFrameImageName === BEAT_IMAGE_SENTINEL ? undefined : movieParams.lastFrameImageName;
+    if (firstFrameImageName === BEAT_IMAGE_SENTINEL || lastFrameImageName === BEAT_IMAGE_SENTINEL) {
+        validateBeatImageSentinel(movieParams, beat, index);
+        returnValue.frameFillColor = frameFillColor;
+    }
+    if (firstFrameImageName === BEAT_IMAGE_SENTINEL) {
+        returnValue.firstFrameIsBeatImage = true;
+    }
+    else if (firstFrameImageName && imageRefs) {
+        const firstFramePath = imageRefs[firstFrameImageName];
+        if (firstFramePath) {
+            returnValue.firstFrameImagePath = await conformFrameImageToCanvas(context, firstFrameImageName, firstFramePath, frameFillColor);
+        }
+    }
+    if (lastFrameImageName === BEAT_IMAGE_SENTINEL) {
+        returnValue.lastFrameIsBeatImage = true;
+    }
+    else if (lastFrameImageName && imageRefs) {
+        const lastFramePath = imageRefs[lastFrameImageName];
+        if (lastFramePath) {
+            returnValue.lastFrameImagePath = await conformFrameImageToCanvas(context, lastFrameImageName, lastFramePath, frameFillColor);
+        }
+    }
+};
 export const imagePreprocessAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs } = namedInputs;
     const studioBeat = context.studio.beats[index];
@@ -28,7 +148,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
         movieFile: beat.moviePrompt ? moviePaths.movieFile : undefined,
         beatDuration: beat.duration ?? studioBeat?.duration,
     };
-    const isMovie = Boolean(beat.moviePrompt || beat?.image?.type === "movie");
+    const isMovie = Boolean(beat.moviePrompt || beat?.image?.type === ImageMediaType.Movie);
     if (beat.soundEffectPrompt) {
         if (isMovie) {
             returnValue.soundEffectAgentInfo = MulmoPresentationStyleMethods.getSoundEffectAgentInfo(context.presentationStyle, beat);
@@ -66,21 +186,12 @@ export const imagePreprocessAgent = async (namedInputs) => {
         }
     }
     returnValue.movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle, beat);
-    // Resolve movie reference images from imageRefs
-    const movieParams = beat.movieParams ?? context.presentationStyle.movieParams;
-    if (movieParams?.firstFrameImageName && imageRefs) {
-        const firstFramePath = imageRefs[movieParams.firstFrameImageName];
-        if (firstFramePath) {
-            returnValue.firstFrameImagePath = firstFramePath;
-        }
-    }
-    if (movieParams?.lastFrameImageName && imageRefs) {
-        const lastFramePath = imageRefs[movieParams.lastFrameImageName];
-        if (lastFramePath) {
-            returnValue.lastFrameImagePath = lastFramePath;
-        }
-    }
-    if (movieParams?.referenceImages && imageRefs) {
+    // Resolve movie reference images from imageRefs.
+    // Shallow-merge like getMovieAgentInfo: beat-level fields override style-level fields
+    // individually, so a beat setting only lastFrameImageName keeps the global frameFillColor.
+    const movieParams = { ...context.presentationStyle.movieParams, ...beat.movieParams };
+    await applyFrameImageParams(returnValue, context, beat, index, movieParams, imageRefs);
+    if (movieParams.referenceImages && imageRefs) {
         returnValue.movieReferenceImages = movieParams.referenceImages
             .map((ref) => {
             const refPath = imageRefs[ref.imageName];
@@ -93,7 +204,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
         const pluginPath = plugin.path({ beat, context, imagePath, ...htmlStyle(context, beat) });
         const markdown = plugin.markdown ? plugin.markdown({ beat, context, imagePath, ...htmlStyle(context, beat) }) : undefined;
         const html = plugin.html ? await plugin.html({ beat, context, imagePath, ...htmlStyle(context, beat) }) : undefined;
-        const isTypeMovie = beat.image.type === "movie";
+        const isTypeMovie = beat.image.type === ImageMediaType.Movie;
         const isAnimatedHtml = MulmoBeatMethods.isAnimatedHtmlTailwind(beat);
         // animation and moviePrompt cannot be used together
         if (isAnimatedHtml && beat.moviePrompt) {
@@ -140,6 +251,26 @@ export const imagePreprocessAgent = async (namedInputs) => {
     const movieFirstFramePath = returnValue.firstFrameImagePath ?? imagePath;
     return { ...returnValue, imagePath, referenceImageForMovie: movieFirstFramePath, imageAgentInfo, prompt, referenceImages };
 };
+// Resolves $beatImage frame references after the beat's image exists. imagePreprocessAgent runs
+// before image generation, so it can only flag the sentinel; this agent runs after
+// imageGenerator/imagePlugin and conforms the now-existing beat image to the canvas.
+// movieGenerator consumes only referenceImageForMovie (the movie's first frame / i2v input)
+// and lastFrameImagePath.
+export const beatFrameResolverAgent = async (namedInputs) => {
+    const { context, beat, index, preprocessor } = namedInputs;
+    if (preprocessor?.imagePath && (preprocessor.firstFrameIsBeatImage || preprocessor.lastFrameIsBeatImage)) {
+        const name = `${beatId(beat.id, index)}_beatImage`;
+        const conformed = await conformFrameImageToCanvas(context, name, preprocessor.imagePath, preprocessor.frameFillColor ?? "black");
+        return {
+            lastFrameImagePath: preprocessor.lastFrameIsBeatImage ? conformed : preprocessor.lastFrameImagePath,
+            referenceImageForMovie: preprocessor.firstFrameIsBeatImage ? conformed : preprocessor.referenceImageForMovie,
+        };
+    }
+    return {
+        lastFrameImagePath: preprocessor?.lastFrameImagePath,
+        referenceImageForMovie: preprocessor?.referenceImageForMovie,
+    };
+};
 export const imagePluginAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs, movieRefs } = namedInputs;
     const { imagePath } = getBeatPngImagePath(context, index);

package/lib/actions/image_references.d.ts

@@ -1,10 +1,13 @@
 import { MulmoStudioContext, MulmoBeat, MulmoImagePromptMedia } from "../types/index.js";
+import type { ResolvedImageReference } from "../methods/mulmo_image_prompt_media.js";
+export type { ResolvedImageReference } from "../methods/mulmo_image_prompt_media.js";
 export declare const generateReferenceImage: (inputs: {
     context: MulmoStudioContext;
     key: string;
     index: number;
     image: MulmoImagePromptMedia;
     referenceImagePath?: string;
+    references?: ResolvedImageReference[];
     force?: boolean;
 }) => Promise<string>;
 export type MediaRefs = {

package/lib/actions/image_references.js

@@ -1,10 +1,36 @@
-import { GraphAI, GraphAILogger } from "graphai";
+import { GraphAI, GraphAILogger, graphDataLatestVersion } from "graphai";
 import { getReferenceImagePath } from "../utils/file.js";
 import { imageGraphOption } from "./graph_option.js";
 import { createUsageCallback } from "../utils/usage_callback.js";
-import { MulmoPresentationStyleMethods, MulmoMediaSourceMethods } from "../methods/index.js";
+import { MulmoPresentationStyleMethods, MulmoMediaSourceMethods, MulmoImagePromptMediaMethods } from "../methods/index.js";
+import { ImageMediaType, } from "../types/index.js";
 import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent, imageReplicateAgent, movieGenAIAgent, movieReplicateAgent } from "../agents/index.js";
 import { agentGenerationError, imageReferenceAction, imageFileTarget, movieFileTarget } from "../utils/error_cause.js";
+// Resolve the normalized reference list to local file paths (with labels), in reference order.
+// imageKey is the imageRefs key of the image being generated (used for warnings and url
+// download file naming). A named reference that is missing from imageRefs warns and is
+// skipped (existing behavior).
+const resolveImageReferences = async (context, imageKey, image, imageRefs) => {
+    const imageReferences = MulmoImagePromptMediaMethods.getReferences(image);
+    const resolved = await Promise.all(imageReferences.map(async (ref, i) => {
+        if (ref.name !== undefined) {
+            const path = imageRefs?.[ref.name];
+            if (!path) {
+                GraphAILogger.warn(`imagePrompt "${imageKey}": reference "${ref.name}" not found in imageRefs — generating without this reference`);
+                return undefined;
+            }
+            return { path, label: ref.label };
+        }
+        if (ref.source !== undefined) {
+            // The legacy referenceImage keeps the bare imageKey so url downloads reuse their historical cache file name.
+            const sourceKey = ref.legacy ? imageKey : `${imageKey}_ref${i}`;
+            return { path: await MulmoMediaSourceMethods.imageReference(ref.source, context, sourceKey), label: ref.label };
+        }
+        return undefined;
+    }));
+    const references = resolved.filter((ref) => ref !== undefined);
+    return references.length > 0 ? references : undefined;
+};
 // public api
 // Application may call this function directly to generate reference image.
 export const generateReferenceImage = async (inputs) => {
@@ -12,11 +38,13 @@ export const generateReferenceImage = async (inputs) => {
     const imagePath = getReferenceImagePath(context, key, "png");
     // generate image
     const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle);
-    const prompt = `${image.prompt}\n${imageAgentInfo.imageParams.style || ""}`;
+    const references = MulmoImagePromptMediaMethods.limitReferences(inputs.references ?? (referenceImagePath ? [{ path: referenceImagePath }] : undefined), imageAgentInfo, key);
+    const preamble = MulmoImagePromptMediaMethods.buildReferencePreamble((references ?? []).map((ref) => ref.label));
+    const prompt = `${preamble}${image.prompt}\n${imageAgentInfo.imageParams.style || ""}`;
     GraphAILogger.info(`Generating reference image for ${key}: ${prompt}`);
-    const referenceImages = referenceImagePath ? [referenceImagePath] : undefined;
+    const referenceImages = references ? references.map((ref) => ref.path) : undefined;
     const image_graph_data = {
-        version: 0.5,
+        version: graphDataLatestVersion,
         nodes: {
             imageGenerator: {
                 agent: imageAgentInfo.agent,
@@ -62,33 +90,30 @@ export const getMediaRefs = async (context) => {
     }
     const imageRefs = {};
     const movieRefs = {};
-    // Stage 1: resolve non-referencing entries (image, imagePrompt without referenceImageName, movie)
+    // Stage 1: resolve non-referencing entries (image, imagePrompt without named references, movie)
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, index) => {
         const image = images[key];
-        if (image.type === "imagePrompt" && !image.referenceImageName) {
-            const refPath = image.referenceImage ? await MulmoMediaSourceMethods.imageReference(image.referenceImage, context, key) : undefined;
-            imageRefs[key] = await generateReferenceImage({ context, key, index, image, referenceImagePath: refPath, force: false });
+        if (image.type === ImageMediaType.ImagePrompt && !MulmoImagePromptMediaMethods.hasNamedReference(image)) {
+            const references = await resolveImageReferences(context, key, image);
+            imageRefs[key] = await generateReferenceImage({ context, key, index, image, references, force: false });
         }
-        else if (image.type === "image") {
+        else if (image.type === ImageMediaType.Image) {
             imageRefs[key] = await MulmoMediaSourceMethods.imageReference(image.source, context, key);
         }
-        else if (image.type === "movie") {
+        else if (image.type === ImageMediaType.Movie) {
             movieRefs[key] = await resolveMovieReference(image, context, key);
         }
     }));
-    // Stage 2: resolve imagePrompt with referenceImageName (depends on Stage 1 results)
+    // Stage 2: resolve imagePrompt with named references (depends on Stage 1 results)
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, index) => {
         const image = images[key];
-        if (image.type === "imagePrompt" && image.referenceImageName) {
-            const refPath = imageRefs[image.referenceImageName];
-            if (!refPath) {
-                GraphAILogger.warn(`imagePrompt "${key}": referenceImageName "${image.referenceImageName}" not found in imageRefs — generating without reference`);
-            }
-            imageRefs[key] = await generateReferenceImage({ context, key, index, image, referenceImagePath: refPath, force: false });
+        if (image.type === ImageMediaType.ImagePrompt && MulmoImagePromptMediaMethods.hasNamedReference(image)) {
+            const references = await resolveImageReferences(context, key, image, imageRefs);
+            imageRefs[key] = await generateReferenceImage({ context, key, index, image, references, force: false });
         }
     }));
     return { imageRefs, movieRefs };
@@ -102,7 +127,7 @@ const generateReferenceMovie = async (inputs) => {
     const movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle);
     GraphAILogger.info(`Generating reference movie for ${key}: ${moviePrompt.prompt}`);
     const movie_graph_data = {
-        version: 0.5,
+        version: graphDataLatestVersion,
         nodes: {
             movieGenerator: {
                 agent: movieAgentInfo.agent,
@@ -145,45 +170,42 @@ const generateReferenceMovie = async (inputs) => {
 const resolveLocalRefs = async (context, images, beatIndex, globalImageRefs) => {
     const localImageRefs = {};
     const localMovieRefs = {};
-    // Stage 1: image, imagePrompt (without referenceImageName), movie (parallel)
+    // Stage 1: image, imagePrompt (without named references), movie (parallel)
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, i) => {
         const entry = images[key];
-        if (entry.type === "imagePrompt" && !entry.referenceImageName) {
-            const refPath = entry.referenceImage ? await MulmoMediaSourceMethods.imageReference(entry.referenceImage, context, key) : undefined;
+        if (entry.type === ImageMediaType.ImagePrompt && !MulmoImagePromptMediaMethods.hasNamedReference(entry)) {
+            const references = await resolveImageReferences(context, key, entry);
             localImageRefs[key] = await generateReferenceImage({
                 context,
                 key,
                 index: beatIndex * 100 + i,
                 image: entry,
-                referenceImagePath: refPath,
+                references,
             });
         }
-        else if (entry.type === "image") {
+        else if (entry.type === ImageMediaType.Image) {
             localImageRefs[key] = await MulmoMediaSourceMethods.imageReference(entry.source, context, key);
         }
-        else if (entry.type === "movie") {
+        else if (entry.type === ImageMediaType.Movie) {
             localMovieRefs[key] = await resolveMovieReference(entry, context, key);
         }
     }));
-    // Stage 2: imagePrompt with referenceImageName (depends on Stage 1)
+    // Stage 2: imagePrompt with named references (depends on Stage 1)
     const combinedImageRefsForImagePrompt = { ...globalImageRefs, ...localImageRefs };
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, i) => {
         const entry = images[key];
-        if (entry.type === "imagePrompt" && entry.referenceImageName) {
-            const refPath = combinedImageRefsForImagePrompt[entry.referenceImageName];
-            if (!refPath) {
-                GraphAILogger.warn(`imagePrompt "${key}": referenceImageName "${entry.referenceImageName}" not found — generating without reference`);
-            }
+        if (entry.type === ImageMediaType.ImagePrompt && MulmoImagePromptMediaMethods.hasNamedReference(entry)) {
+            const references = await resolveImageReferences(context, key, entry, combinedImageRefsForImagePrompt);
             localImageRefs[key] = await generateReferenceImage({
                 context,
                 key,
                 index: beatIndex * 100 + i,
                 image: entry,
-                referenceImagePath: refPath,
+                references,
             });
         }
     }));
@@ -193,7 +215,7 @@ const resolveLocalRefs = async (context, images, beatIndex, globalImageRefs) =>
         .sort()
         .map(async (key, i) => {
         const entry = images[key];
-        if (entry.type === "moviePrompt") {
+        if (entry.type === ImageMediaType.MoviePrompt) {
             const refImagePath = entry.imageName ? combinedImageRefs[entry.imageName] : undefined;
             localMovieRefs[key] = await generateReferenceMovie({
                 context,

package/lib/actions/images.d.ts

@@ -72,6 +72,9 @@ export declare const beat_graph_data: {
                 };
                 firstFrameImagePath?: string;
                 lastFrameImagePath?: string;
+                firstFrameIsBeatImage?: boolean;
+                lastFrameIsBeatImage?: boolean;
+                frameFillColor?: string;
                 movieReferenceImages?: {
                     imagePath: string;
                     referenceType: "ASSET" | "STYLE";
@@ -103,6 +106,9 @@ export declare const beat_graph_data: {
                 };
                 firstFrameImagePath?: string;
                 lastFrameImagePath?: string;
+                firstFrameIsBeatImage?: boolean;
+                lastFrameIsBeatImage?: boolean;
+                frameFillColor?: string;
                 movieReferenceImages?: {
                     imagePath: string;
                     referenceType: "ASSET" | "STYLE";
@@ -145,6 +151,9 @@ export declare const beat_graph_data: {
                 };
                 firstFrameImagePath?: string;
                 lastFrameImagePath?: string;
+                firstFrameIsBeatImage?: boolean;
+                lastFrameIsBeatImage?: boolean;
+                frameFillColor?: string;
                 movieReferenceImages?: {
                     imagePath: string;
                     referenceType: "ASSET" | "STYLE";
@@ -179,6 +188,9 @@ export declare const beat_graph_data: {
                 };
                 firstFrameImagePath?: string;
                 lastFrameImagePath?: string;
+                firstFrameIsBeatImage?: boolean;
+                lastFrameIsBeatImage?: boolean;
+                frameFillColor?: string;
                 movieReferenceImages?: {
                     imagePath: string;
                     referenceType: "ASSET" | "STYLE";
@@ -298,6 +310,24 @@ export declare const beat_graph_data: {
             };
             defaultValue: {};
         };
+        frameResolver: {
+            agent: (namedInputs: {
+                context: MulmoStudioContext;
+                beat: import("../types/type.js").MulmoBeat;
+                index: number;
+                preprocessor?: import("./image_agents.js").BeatFramePreprocessorInput;
+            }) => Promise<{
+                lastFrameImagePath: string | undefined;
+                referenceImageForMovie: string | undefined;
+            }>;
+            inputs: {
+                onComplete: string[];
+                context: string;
+                beat: string;
+                index: string;
+                preprocessor: string;
+            };
+        };
         movieGenerator: {
             if: string;
             agent: string;

package/lib/actions/images.js

@@ -13,7 +13,7 @@ import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
 import { audioCheckerError } from "../utils/error_cause.js";
 import { extractImageFromMovie, ffmpegGetMediaDuration, trimMusic } from "../utils/ffmpeg_utils.js";
 import { getMediaRefs, resolveBeatLocalRefs } from "./image_references.js";
-import { imagePreprocessAgent, imagePluginAgent, htmlImageGeneratorAgent } from "./image_agents.js";
+import { imagePreprocessAgent, imagePluginAgent, htmlImageGeneratorAgent, beatFrameResolverAgent } from "./image_agents.js";
 import { imageGraphOption } from "./graph_option.js";
 import { createUsageCallback } from "../utils/usage_callback.js";
 const vanillaAgents = vanilla.default ?? vanilla;
@@ -169,15 +169,27 @@ export const beat_graph_data = {
             },
             defaultValue: {},
         },
+        // Resolves $beatImage frame references once the beat image exists (after imageGenerator/imagePlugin);
+        // passes the preprocessor's frame paths through unchanged for non-sentinel beats.
+        frameResolver: {
+            agent: beatFrameResolverAgent,
+            inputs: {
+                onComplete: [":imageGenerator", ":imagePlugin"],
+                context: ":context",
+                beat: ":beat",
+                index: ":__mapIndex",
+                preprocessor: ":preprocessor",
+            },
+        },
         movieGenerator: {
             if: ":beat.moviePrompt",
             agent: ":preprocessor.movieAgentInfo.agent",
             inputs: {
                 media: "movie",
-                onComplete: [":imageGenerator", ":imagePlugin"], // to wait for imageGenerator to finish
+                onComplete: [":frameResolver"], // frameResolver itself waits for imageGenerator/imagePlugin
                 prompt: ":beat.moviePrompt",
-                imagePath: ":preprocessor.referenceImageForMovie",
-                lastFrameImagePath: ":preprocessor.lastFrameImagePath",
+                imagePath: ":frameResolver.referenceImageForMovie",
+                lastFrameImagePath: ":frameResolver.lastFrameImagePath",
                 referenceImages: ":preprocessor.movieReferenceImages",
                 movieFile: ":preprocessor.movieFile", // for google genai agent
                 cache: {

package/lib/methods/index.d.ts

@@ -1,5 +1,6 @@
 export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
+export * from "./mulmo_image_prompt_media.js";
 export * from "./mulmo_beat_node.js";
 export * from "./mulmo_script.js";

package/lib/methods/index.js

@@ -1,6 +1,7 @@
 export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
+export * from "./mulmo_image_prompt_media.js";
 // Node-only MulmoBeatMethods (includes getPlugin). The browser-safe subset lives in
 // ./mulmo_beat.js and is re-exported by ../index.common.js for mulmocast/browser.
 export * from "./mulmo_beat_node.js";

package/lib/methods/mulmo_image_prompt_media.d.ts

@@ -0,0 +1,14 @@
+import { MulmoImagePromptMedia, MulmoImageReference, Text2ImageAgentInfo } from "../types/index.js";
+export type MulmoNormalizedImageReference = MulmoImageReference & {
+    legacy?: boolean;
+};
+export type ResolvedImageReference = {
+    path: string;
+    label?: string;
+};
+export declare const MulmoImagePromptMediaMethods: {
+    getReferences(image: MulmoImagePromptMedia): MulmoNormalizedImageReference[];
+    hasNamedReference(image: MulmoImagePromptMedia): boolean;
+    buildReferencePreamble(labels: (string | undefined)[]): string;
+    limitReferences(references: ResolvedImageReference[] | undefined, imageAgentInfo: Text2ImageAgentInfo, imageKey: string): ResolvedImageReference[] | undefined;
+};

package/lib/methods/mulmo_image_prompt_media.js

@@ -0,0 +1,45 @@
+import { GraphAILogger } from "graphai";
+import { getMaxImageReferenceImages } from "../types/provider2agent.js";
+export const MulmoImagePromptMediaMethods = {
+    // Normalized reference list: legacy single-ref fields first, then the references array.
+    // When both legacy fields are set, referenceImageName wins and referenceImage is ignored,
+    // matching the pre-references resolution behavior.
+    getReferences(image) {
+        const references = [];
+        if (image.referenceImageName) {
+            references.push({ name: image.referenceImageName, legacy: true });
+        }
+        else if (image.referenceImage) {
+            references.push({ source: image.referenceImage, legacy: true });
+        }
+        references.push(...(image.references ?? []));
+        return references;
+    },
+    // Entries with a named reference must resolve after stage 1 (which generates/loads the named images).
+    hasNamedReference(image) {
+        return this.getReferences(image).some((ref) => ref.name !== undefined);
+    },
+    // "Reference image 1 shows <label>. ... Use these exact designs.\n"
+    // Empty when no reference has a label, keeping label-free prompts byte-identical to before.
+    buildReferencePreamble(labels) {
+        const sentences = labels.map((label, index) => (label ? `Reference image ${index + 1} shows ${label}.` : undefined)).filter((s) => s !== undefined);
+        if (sentences.length === 0) {
+            return "";
+        }
+        return `${sentences.join(" ")} Use these exact designs.\n`;
+    },
+    // Truncate to the provider/model limit, keeping the first N deterministically.
+    // imageKey is the imageRefs key of the image being generated (used for the warning).
+    limitReferences(references, imageAgentInfo, imageKey) {
+        if (!references) {
+            return undefined;
+        }
+        const { provider, model } = imageAgentInfo.imageParams;
+        const max = getMaxImageReferenceImages(provider, model ?? "");
+        if (max !== undefined && references.length > max) {
+            GraphAILogger.warn(`imagePrompt "${imageKey}": ${references.length} reference images exceed the ${provider} limit of ${max} — keeping the first ${max}`);
+            return references.slice(0, max);
+        }
+        return references;
+    },
+};

package/lib/methods/mulmo_presentation_style.d.ts

@@ -194,6 +194,7 @@ export declare const MulmoPresentationStyleMethods: {
             vertexai_location?: string | undefined;
             firstFrameImageName?: string | undefined;
             lastFrameImageName?: string | undefined;
+            frameFillColor?: string | undefined;
             referenceImages?: {
                 imageName: string;
                 referenceType: "ASSET" | "STYLE";