mulmocast 2.8.1 → 2.9.0

package/lib/actions/image_agents.d.ts

@@ -24,6 +24,9 @@ type ImagePreprocessAgentReturnValue = {
     };
     firstFrameImagePath?: string;
     lastFrameImagePath?: string;
+    firstFrameIsBeatImage?: boolean;
+    lastFrameIsBeatImage?: boolean;
+    frameFillColor?: string;
     movieReferenceImages?: {
         imagePath: string;
         referenceType: "ASSET" | "STYLE";
@@ -62,6 +65,23 @@ export declare const imagePreprocessAgent: (namedInputs: {
     index: number;
     imageRefs?: Record<string, string>;
 }) => Promise<ImagePreprocessAgentResponse>;
+export type BeatFramePreprocessorInput = {
+    imagePath?: string;
+    lastFrameImagePath?: string;
+    firstFrameIsBeatImage?: boolean;
+    lastFrameIsBeatImage?: boolean;
+    frameFillColor?: string;
+    referenceImageForMovie?: string;
+};
+export declare const beatFrameResolverAgent: (namedInputs: {
+    context: MulmoStudioContext;
+    beat: MulmoBeat;
+    index: number;
+    preprocessor?: BeatFramePreprocessorInput;
+}) => Promise<{
+    lastFrameImagePath: string | undefined;
+    referenceImageForMovie: string | undefined;
+}>;
 export declare const imagePluginAgent: (namedInputs: {
     context: MulmoStudioContext;
     beat: MulmoBeat;

package/lib/actions/image_agents.js

@@ -1,9 +1,11 @@
 import fs from "fs";
 import { createHash } from "crypto";
 import { GraphAILogger } from "graphai";
+import { BEAT_IMAGE_SENTINEL, ImageMediaType, beatImageTypesWithoutOwnStill } from "../types/schema.js";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods, MulmoBeatMethods, MulmoMediaSourceMethods } from "../methods/index.js";
 import { getBeatPngImagePath, getBeatMoviePaths, getBeatAnimatedVideoPath, getAudioFilePath, getGroupedAudioFilePath, getReferenceImagePath, } from "../utils/file.js";
 import { ffmpegGetImageDimensions, padImageToCanvas } from "../utils/ffmpeg_utils.js";
+import { beatImageSentinelError, bothFramesTarget, missingBeatImageTarget, unsupportedImageTypeTarget, missingFirstFrameTarget } from "../utils/error_cause.js";
 import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
 import { renderHTMLToImage } from "../utils/html_render.js";
 import { beatId } from "../utils/utils.js";
@@ -71,6 +73,64 @@ export const conformFrameImageToCanvas = async (context, imageName, imagePath, f
         conformingInFlight.delete(destPath);
     }
 };
+const validateBeatImageSentinel = (movieParams, beat, index) => {
+    if (movieParams.firstFrameImageName === BEAT_IMAGE_SENTINEL && movieParams.lastFrameImageName === BEAT_IMAGE_SENTINEL) {
+        throw new Error(`${BEAT_IMAGE_SENTINEL} cannot be used for both firstFrameImageName and lastFrameImageName. beat: ${index}`, {
+            cause: beatImageSentinelError(index, bothFramesTarget),
+        });
+    }
+    if (!beat.imagePrompt && !beat.image) {
+        throw new Error(`${BEAT_IMAGE_SENTINEL} requires the beat to have an imagePrompt or image. beat: ${index}`, {
+            cause: beatImageSentinelError(index, missingBeatImageTarget),
+        });
+    }
+    if (beat.image && beatImageTypesWithoutOwnStill.includes(beat.image.type)) {
+        throw new Error(`${BEAT_IMAGE_SENTINEL} cannot be used with image type "${beat.image.type}" (the beat image is not its own still). beat: ${index}`, {
+            cause: beatImageSentinelError(index, unsupportedImageTypeTarget),
+        });
+    }
+    // The implicit first frame (the raw beat image) is not conformed to the canvas, so pairing
+    // it with a conformed $beatImage last frame would feed mismatched frame sizes to strict
+    // image-to-video models (e.g. wan-2.2-i2v-fast). Require an explicit, conformed first frame.
+    if (movieParams.lastFrameImageName === BEAT_IMAGE_SENTINEL && !movieParams.firstFrameImageName) {
+        throw new Error(`${BEAT_IMAGE_SENTINEL} as lastFrameImageName requires firstFrameImageName. beat: ${index}`, {
+            cause: beatImageSentinelError(index, missingFirstFrameTarget),
+        });
+    }
+};
+// Resolve firstFrameImageName/lastFrameImageName. Named refs conform to the canvas here;
+// the $beatImage sentinel only sets flags (the beat image does not exist yet at preprocess
+// time) and is resolved by beatFrameResolverAgent after image generation.
+const applyFrameImageParams = async (returnValue, context, beat, index, movieParams, imageRefs) => {
+    const frameFillColor = movieParams.frameFillColor ?? "black";
+    // A style-level $beatImage merges into every beat; beats that generate no movie
+    // (no moviePrompt) ignore the sentinel instead of failing validation.
+    const sentinelActive = Boolean(beat.moviePrompt);
+    const firstFrameImageName = !sentinelActive && movieParams.firstFrameImageName === BEAT_IMAGE_SENTINEL ? undefined : movieParams.firstFrameImageName;
+    const lastFrameImageName = !sentinelActive && movieParams.lastFrameImageName === BEAT_IMAGE_SENTINEL ? undefined : movieParams.lastFrameImageName;
+    if (firstFrameImageName === BEAT_IMAGE_SENTINEL || lastFrameImageName === BEAT_IMAGE_SENTINEL) {
+        validateBeatImageSentinel(movieParams, beat, index);
+        returnValue.frameFillColor = frameFillColor;
+    }
+    if (firstFrameImageName === BEAT_IMAGE_SENTINEL) {
+        returnValue.firstFrameIsBeatImage = true;
+    }
+    else if (firstFrameImageName && imageRefs) {
+        const firstFramePath = imageRefs[firstFrameImageName];
+        if (firstFramePath) {
+            returnValue.firstFrameImagePath = await conformFrameImageToCanvas(context, firstFrameImageName, firstFramePath, frameFillColor);
+        }
+    }
+    if (lastFrameImageName === BEAT_IMAGE_SENTINEL) {
+        returnValue.lastFrameIsBeatImage = true;
+    }
+    else if (lastFrameImageName && imageRefs) {
+        const lastFramePath = imageRefs[lastFrameImageName];
+        if (lastFramePath) {
+            returnValue.lastFrameImagePath = await conformFrameImageToCanvas(context, lastFrameImageName, lastFramePath, frameFillColor);
+        }
+    }
+};
 export const imagePreprocessAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs } = namedInputs;
     const studioBeat = context.studio.beats[index];
@@ -88,7 +148,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
         movieFile: beat.moviePrompt ? moviePaths.movieFile : undefined,
         beatDuration: beat.duration ?? studioBeat?.duration,
     };
-    const isMovie = Boolean(beat.moviePrompt || beat?.image?.type === "movie");
+    const isMovie = Boolean(beat.moviePrompt || beat?.image?.type === ImageMediaType.Movie);
     if (beat.soundEffectPrompt) {
         if (isMovie) {
             returnValue.soundEffectAgentInfo = MulmoPresentationStyleMethods.getSoundEffectAgentInfo(context.presentationStyle, beat);
@@ -130,19 +190,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
     // Shallow-merge like getMovieAgentInfo: beat-level fields override style-level fields
     // individually, so a beat setting only lastFrameImageName keeps the global frameFillColor.
     const movieParams = { ...context.presentationStyle.movieParams, ...beat.movieParams };
-    const frameFillColor = movieParams.frameFillColor ?? "black";
-    if (movieParams.firstFrameImageName && imageRefs) {
-        const firstFramePath = imageRefs[movieParams.firstFrameImageName];
-        if (firstFramePath) {
-            returnValue.firstFrameImagePath = await conformFrameImageToCanvas(context, movieParams.firstFrameImageName, firstFramePath, frameFillColor);
-        }
-    }
-    if (movieParams.lastFrameImageName && imageRefs) {
-        const lastFramePath = imageRefs[movieParams.lastFrameImageName];
-        if (lastFramePath) {
-            returnValue.lastFrameImagePath = await conformFrameImageToCanvas(context, movieParams.lastFrameImageName, lastFramePath, frameFillColor);
-        }
-    }
+    await applyFrameImageParams(returnValue, context, beat, index, movieParams, imageRefs);
     if (movieParams.referenceImages && imageRefs) {
         returnValue.movieReferenceImages = movieParams.referenceImages
             .map((ref) => {
@@ -156,7 +204,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
         const pluginPath = plugin.path({ beat, context, imagePath, ...htmlStyle(context, beat) });
         const markdown = plugin.markdown ? plugin.markdown({ beat, context, imagePath, ...htmlStyle(context, beat) }) : undefined;
         const html = plugin.html ? await plugin.html({ beat, context, imagePath, ...htmlStyle(context, beat) }) : undefined;
-        const isTypeMovie = beat.image.type === "movie";
+        const isTypeMovie = beat.image.type === ImageMediaType.Movie;
         const isAnimatedHtml = MulmoBeatMethods.isAnimatedHtmlTailwind(beat);
         // animation and moviePrompt cannot be used together
         if (isAnimatedHtml && beat.moviePrompt) {
@@ -203,6 +251,26 @@ export const imagePreprocessAgent = async (namedInputs) => {
     const movieFirstFramePath = returnValue.firstFrameImagePath ?? imagePath;
     return { ...returnValue, imagePath, referenceImageForMovie: movieFirstFramePath, imageAgentInfo, prompt, referenceImages };
 };
+// Resolves $beatImage frame references after the beat's image exists. imagePreprocessAgent runs
+// before image generation, so it can only flag the sentinel; this agent runs after
+// imageGenerator/imagePlugin and conforms the now-existing beat image to the canvas.
+// movieGenerator consumes only referenceImageForMovie (the movie's first frame / i2v input)
+// and lastFrameImagePath.
+export const beatFrameResolverAgent = async (namedInputs) => {
+    const { context, beat, index, preprocessor } = namedInputs;
+    if (preprocessor?.imagePath && (preprocessor.firstFrameIsBeatImage || preprocessor.lastFrameIsBeatImage)) {
+        const name = `${beatId(beat.id, index)}_beatImage`;
+        const conformed = await conformFrameImageToCanvas(context, name, preprocessor.imagePath, preprocessor.frameFillColor ?? "black");
+        return {
+            lastFrameImagePath: preprocessor.lastFrameIsBeatImage ? conformed : preprocessor.lastFrameImagePath,
+            referenceImageForMovie: preprocessor.firstFrameIsBeatImage ? conformed : preprocessor.referenceImageForMovie,
+        };
+    }
+    return {
+        lastFrameImagePath: preprocessor?.lastFrameImagePath,
+        referenceImageForMovie: preprocessor?.referenceImageForMovie,
+    };
+};
 export const imagePluginAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs, movieRefs } = namedInputs;
     const { imagePath } = getBeatPngImagePath(context, index);

package/lib/actions/image_references.d.ts

@@ -1,10 +1,13 @@
 import { MulmoStudioContext, MulmoBeat, MulmoImagePromptMedia } from "../types/index.js";
+import type { ResolvedImageReference } from "../methods/mulmo_image_prompt_media.js";
+export type { ResolvedImageReference } from "../methods/mulmo_image_prompt_media.js";
 export declare const generateReferenceImage: (inputs: {
     context: MulmoStudioContext;
     key: string;
     index: number;
     image: MulmoImagePromptMedia;
     referenceImagePath?: string;
+    references?: ResolvedImageReference[];
     force?: boolean;
 }) => Promise<string>;
 export type MediaRefs = {

package/lib/actions/image_references.js

@@ -1,10 +1,36 @@
-import { GraphAI, GraphAILogger } from "graphai";
+import { GraphAI, GraphAILogger, graphDataLatestVersion } from "graphai";
 import { getReferenceImagePath } from "../utils/file.js";
 import { imageGraphOption } from "./graph_option.js";
 import { createUsageCallback } from "../utils/usage_callback.js";
-import { MulmoPresentationStyleMethods, MulmoMediaSourceMethods } from "../methods/index.js";
+import { MulmoPresentationStyleMethods, MulmoMediaSourceMethods, MulmoImagePromptMediaMethods } from "../methods/index.js";
+import { ImageMediaType, } from "../types/index.js";
 import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent, imageReplicateAgent, movieGenAIAgent, movieReplicateAgent } from "../agents/index.js";
 import { agentGenerationError, imageReferenceAction, imageFileTarget, movieFileTarget } from "../utils/error_cause.js";
+// Resolve the normalized reference list to local file paths (with labels), in reference order.
+// imageKey is the imageRefs key of the image being generated (used for warnings and url
+// download file naming). A named reference that is missing from imageRefs warns and is
+// skipped (existing behavior).
+const resolveImageReferences = async (context, imageKey, image, imageRefs) => {
+    const imageReferences = MulmoImagePromptMediaMethods.getReferences(image);
+    const resolved = await Promise.all(imageReferences.map(async (ref, i) => {
+        if (ref.name !== undefined) {
+            const path = imageRefs?.[ref.name];
+            if (!path) {
+                GraphAILogger.warn(`imagePrompt "${imageKey}": reference "${ref.name}" not found in imageRefs — generating without this reference`);
+                return undefined;
+            }
+            return { path, label: ref.label };
+        }
+        if (ref.source !== undefined) {
+            // The legacy referenceImage keeps the bare imageKey so url downloads reuse their historical cache file name.
+            const sourceKey = ref.legacy ? imageKey : `${imageKey}_ref${i}`;
+            return { path: await MulmoMediaSourceMethods.imageReference(ref.source, context, sourceKey), label: ref.label };
+        }
+        return undefined;
+    }));
+    const references = resolved.filter((ref) => ref !== undefined);
+    return references.length > 0 ? references : undefined;
+};
 // public api
 // Application may call this function directly to generate reference image.
 export const generateReferenceImage = async (inputs) => {
@@ -12,11 +38,13 @@ export const generateReferenceImage = async (inputs) => {
     const imagePath = getReferenceImagePath(context, key, "png");
     // generate image
     const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle);
-    const prompt = `${image.prompt}\n${imageAgentInfo.imageParams.style || ""}`;
+    const references = MulmoImagePromptMediaMethods.limitReferences(inputs.references ?? (referenceImagePath ? [{ path: referenceImagePath }] : undefined), imageAgentInfo, key);
+    const preamble = MulmoImagePromptMediaMethods.buildReferencePreamble((references ?? []).map((ref) => ref.label));
+    const prompt = `${preamble}${image.prompt}\n${imageAgentInfo.imageParams.style || ""}`;
     GraphAILogger.info(`Generating reference image for ${key}: ${prompt}`);
-    const referenceImages = referenceImagePath ? [referenceImagePath] : undefined;
+    const referenceImages = references ? references.map((ref) => ref.path) : undefined;
     const image_graph_data = {
-        version: 0.5,
+        version: graphDataLatestVersion,
         nodes: {
             imageGenerator: {
                 agent: imageAgentInfo.agent,
@@ -62,33 +90,30 @@ export const getMediaRefs = async (context) => {
     }
     const imageRefs = {};
     const movieRefs = {};
-    // Stage 1: resolve non-referencing entries (image, imagePrompt without referenceImageName, movie)
+    // Stage 1: resolve non-referencing entries (image, imagePrompt without named references, movie)
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, index) => {
         const image = images[key];
-        if (image.type === "imagePrompt" && !image.referenceImageName) {
-            const refPath = image.referenceImage ? await MulmoMediaSourceMethods.imageReference(image.referenceImage, context, key) : undefined;
-            imageRefs[key] = await generateReferenceImage({ context, key, index, image, referenceImagePath: refPath, force: false });
+        if (image.type === ImageMediaType.ImagePrompt && !MulmoImagePromptMediaMethods.hasNamedReference(image)) {
+            const references = await resolveImageReferences(context, key, image);
+            imageRefs[key] = await generateReferenceImage({ context, key, index, image, references, force: false });
         }
-        else if (image.type === "image") {
+        else if (image.type === ImageMediaType.Image) {
             imageRefs[key] = await MulmoMediaSourceMethods.imageReference(image.source, context, key);
         }
-        else if (image.type === "movie") {
+        else if (image.type === ImageMediaType.Movie) {
             movieRefs[key] = await resolveMovieReference(image, context, key);
         }
     }));
-    // Stage 2: resolve imagePrompt with referenceImageName (depends on Stage 1 results)
+    // Stage 2: resolve imagePrompt with named references (depends on Stage 1 results)
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, index) => {
         const image = images[key];
-        if (image.type === "imagePrompt" && image.referenceImageName) {
-            const refPath = imageRefs[image.referenceImageName];
-            if (!refPath) {
-                GraphAILogger.warn(`imagePrompt "${key}": referenceImageName "${image.referenceImageName}" not found in imageRefs — generating without reference`);
-            }
-            imageRefs[key] = await generateReferenceImage({ context, key, index, image, referenceImagePath: refPath, force: false });
+        if (image.type === ImageMediaType.ImagePrompt && MulmoImagePromptMediaMethods.hasNamedReference(image)) {
+            const references = await resolveImageReferences(context, key, image, imageRefs);
+            imageRefs[key] = await generateReferenceImage({ context, key, index, image, references, force: false });
         }
     }));
     return { imageRefs, movieRefs };
@@ -102,7 +127,7 @@ const generateReferenceMovie = async (inputs) => {
     const movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle);
     GraphAILogger.info(`Generating reference movie for ${key}: ${moviePrompt.prompt}`);
     const movie_graph_data = {
-        version: 0.5,
+        version: graphDataLatestVersion,
         nodes: {
             movieGenerator: {
                 agent: movieAgentInfo.agent,
@@ -145,45 +170,42 @@ const generateReferenceMovie = async (inputs) => {
 const resolveLocalRefs = async (context, images, beatIndex, globalImageRefs) => {
     const localImageRefs = {};
     const localMovieRefs = {};
-    // Stage 1: image, imagePrompt (without referenceImageName), movie (parallel)
+    // Stage 1: image, imagePrompt (without named references), movie (parallel)
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, i) => {
         const entry = images[key];
-        if (entry.type === "imagePrompt" && !entry.referenceImageName) {
-            const refPath = entry.referenceImage ? await MulmoMediaSourceMethods.imageReference(entry.referenceImage, context, key) : undefined;
+        if (entry.type === ImageMediaType.ImagePrompt && !MulmoImagePromptMediaMethods.hasNamedReference(entry)) {
+            const references = await resolveImageReferences(context, key, entry);
             localImageRefs[key] = await generateReferenceImage({
                 context,
                 key,
                 index: beatIndex * 100 + i,
                 image: entry,
-                referenceImagePath: refPath,
+                references,
             });
         }
-        else if (entry.type === "image") {
+        else if (entry.type === ImageMediaType.Image) {
             localImageRefs[key] = await MulmoMediaSourceMethods.imageReference(entry.source, context, key);
         }
-        else if (entry.type === "movie") {
+        else if (entry.type === ImageMediaType.Movie) {
             localMovieRefs[key] = await resolveMovieReference(entry, context, key);
         }
     }));
-    // Stage 2: imagePrompt with referenceImageName (depends on Stage 1)
+    // Stage 2: imagePrompt with named references (depends on Stage 1)
     const combinedImageRefsForImagePrompt = { ...globalImageRefs, ...localImageRefs };
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, i) => {
         const entry = images[key];
-        if (entry.type === "imagePrompt" && entry.referenceImageName) {
-            const refPath = combinedImageRefsForImagePrompt[entry.referenceImageName];
-            if (!refPath) {
-                GraphAILogger.warn(`imagePrompt "${key}": referenceImageName "${entry.referenceImageName}" not found — generating without reference`);
-            }
+        if (entry.type === ImageMediaType.ImagePrompt && MulmoImagePromptMediaMethods.hasNamedReference(entry)) {
+            const references = await resolveImageReferences(context, key, entry, combinedImageRefsForImagePrompt);
             localImageRefs[key] = await generateReferenceImage({
                 context,
                 key,
                 index: beatIndex * 100 + i,
                 image: entry,
-                referenceImagePath: refPath,
+                references,
             });
         }
     }));
@@ -193,7 +215,7 @@ const resolveLocalRefs = async (context, images, beatIndex, globalImageRefs) =>
         .sort()
         .map(async (key, i) => {
         const entry = images[key];
-        if (entry.type === "moviePrompt") {
+        if (entry.type === ImageMediaType.MoviePrompt) {
             const refImagePath = entry.imageName ? combinedImageRefs[entry.imageName] : undefined;
             localMovieRefs[key] = await generateReferenceMovie({
                 context,

package/lib/actions/images.d.ts

@@ -72,6 +72,9 @@ export declare const beat_graph_data: {
                 };
                 firstFrameImagePath?: string;
                 lastFrameImagePath?: string;
+                firstFrameIsBeatImage?: boolean;
+                lastFrameIsBeatImage?: boolean;
+                frameFillColor?: string;
                 movieReferenceImages?: {
                     imagePath: string;
                     referenceType: "ASSET" | "STYLE";
@@ -103,6 +106,9 @@ export declare const beat_graph_data: {
                 };
                 firstFrameImagePath?: string;
                 lastFrameImagePath?: string;
+                firstFrameIsBeatImage?: boolean;
+                lastFrameIsBeatImage?: boolean;
+                frameFillColor?: string;
                 movieReferenceImages?: {
                     imagePath: string;
                     referenceType: "ASSET" | "STYLE";
@@ -145,6 +151,9 @@ export declare const beat_graph_data: {
                 };
                 firstFrameImagePath?: string;
                 lastFrameImagePath?: string;
+                firstFrameIsBeatImage?: boolean;
+                lastFrameIsBeatImage?: boolean;
+                frameFillColor?: string;
                 movieReferenceImages?: {
                     imagePath: string;
                     referenceType: "ASSET" | "STYLE";
@@ -179,6 +188,9 @@ export declare const beat_graph_data: {
                 };
                 firstFrameImagePath?: string;
                 lastFrameImagePath?: string;
+                firstFrameIsBeatImage?: boolean;
+                lastFrameIsBeatImage?: boolean;
+                frameFillColor?: string;
                 movieReferenceImages?: {
                     imagePath: string;
                     referenceType: "ASSET" | "STYLE";
@@ -298,6 +310,24 @@ export declare const beat_graph_data: {
             };
             defaultValue: {};
         };
+        frameResolver: {
+            agent: (namedInputs: {
+                context: MulmoStudioContext;
+                beat: import("../types/type.js").MulmoBeat;
+                index: number;
+                preprocessor?: import("./image_agents.js").BeatFramePreprocessorInput;
+            }) => Promise<{
+                lastFrameImagePath: string | undefined;
+                referenceImageForMovie: string | undefined;
+            }>;
+            inputs: {
+                onComplete: string[];
+                context: string;
+                beat: string;
+                index: string;
+                preprocessor: string;
+            };
+        };
         movieGenerator: {
             if: string;
             agent: string;

package/lib/actions/images.js

@@ -13,7 +13,7 @@ import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
 import { audioCheckerError } from "../utils/error_cause.js";
 import { extractImageFromMovie, ffmpegGetMediaDuration, trimMusic } from "../utils/ffmpeg_utils.js";
 import { getMediaRefs, resolveBeatLocalRefs } from "./image_references.js";
-import { imagePreprocessAgent, imagePluginAgent, htmlImageGeneratorAgent } from "./image_agents.js";
+import { imagePreprocessAgent, imagePluginAgent, htmlImageGeneratorAgent, beatFrameResolverAgent } from "./image_agents.js";
 import { imageGraphOption } from "./graph_option.js";
 import { createUsageCallback } from "../utils/usage_callback.js";
 const vanillaAgents = vanilla.default ?? vanilla;
@@ -169,15 +169,27 @@ export const beat_graph_data = {
             },
             defaultValue: {},
         },
+        // Resolves $beatImage frame references once the beat image exists (after imageGenerator/imagePlugin);
+        // passes the preprocessor's frame paths through unchanged for non-sentinel beats.
+        frameResolver: {
+            agent: beatFrameResolverAgent,
+            inputs: {
+                onComplete: [":imageGenerator", ":imagePlugin"],
+                context: ":context",
+                beat: ":beat",
+                index: ":__mapIndex",
+                preprocessor: ":preprocessor",
+            },
+        },
         movieGenerator: {
             if: ":beat.moviePrompt",
             agent: ":preprocessor.movieAgentInfo.agent",
             inputs: {
                 media: "movie",
-                onComplete: [":imageGenerator", ":imagePlugin"], // to wait for imageGenerator to finish
+                onComplete: [":frameResolver"], // frameResolver itself waits for imageGenerator/imagePlugin
                 prompt: ":beat.moviePrompt",
-                imagePath: ":preprocessor.referenceImageForMovie",
-                lastFrameImagePath: ":preprocessor.lastFrameImagePath",
+                imagePath: ":frameResolver.referenceImageForMovie",
+                lastFrameImagePath: ":frameResolver.lastFrameImagePath",
                 referenceImages: ":preprocessor.movieReferenceImages",
                 movieFile: ":preprocessor.movieFile", // for google genai agent
                 cache: {

package/lib/methods/index.d.ts

@@ -1,5 +1,6 @@
 export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
+export * from "./mulmo_image_prompt_media.js";
 export * from "./mulmo_beat_node.js";
 export * from "./mulmo_script.js";

package/lib/methods/index.js

@@ -1,6 +1,7 @@
 export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
+export * from "./mulmo_image_prompt_media.js";
 // Node-only MulmoBeatMethods (includes getPlugin). The browser-safe subset lives in
 // ./mulmo_beat.js and is re-exported by ../index.common.js for mulmocast/browser.
 export * from "./mulmo_beat_node.js";

package/lib/methods/mulmo_image_prompt_media.d.ts

@@ -0,0 +1,14 @@
+import { MulmoImagePromptMedia, MulmoImageReference, Text2ImageAgentInfo } from "../types/index.js";
+export type MulmoNormalizedImageReference = MulmoImageReference & {
+    legacy?: boolean;
+};
+export type ResolvedImageReference = {
+    path: string;
+    label?: string;
+};
+export declare const MulmoImagePromptMediaMethods: {
+    getReferences(image: MulmoImagePromptMedia): MulmoNormalizedImageReference[];
+    hasNamedReference(image: MulmoImagePromptMedia): boolean;
+    buildReferencePreamble(labels: (string | undefined)[]): string;
+    limitReferences(references: ResolvedImageReference[] | undefined, imageAgentInfo: Text2ImageAgentInfo, imageKey: string): ResolvedImageReference[] | undefined;
+};

package/lib/methods/mulmo_image_prompt_media.js

@@ -0,0 +1,45 @@
+import { GraphAILogger } from "graphai";
+import { getMaxImageReferenceImages } from "../types/provider2agent.js";
+export const MulmoImagePromptMediaMethods = {
+    // Normalized reference list: legacy single-ref fields first, then the references array.
+    // When both legacy fields are set, referenceImageName wins and referenceImage is ignored,
+    // matching the pre-references resolution behavior.
+    getReferences(image) {
+        const references = [];
+        if (image.referenceImageName) {
+            references.push({ name: image.referenceImageName, legacy: true });
+        }
+        else if (image.referenceImage) {
+            references.push({ source: image.referenceImage, legacy: true });
+        }
+        references.push(...(image.references ?? []));
+        return references;
+    },
+    // Entries with a named reference must resolve after stage 1 (which generates/loads the named images).
+    hasNamedReference(image) {
+        return this.getReferences(image).some((ref) => ref.name !== undefined);
+    },
+    // "Reference image 1 shows <label>. ... Use these exact designs.\n"
+    // Empty when no reference has a label, keeping label-free prompts byte-identical to before.
+    buildReferencePreamble(labels) {
+        const sentences = labels.map((label, index) => (label ? `Reference image ${index + 1} shows ${label}.` : undefined)).filter((s) => s !== undefined);
+        if (sentences.length === 0) {
+            return "";
+        }
+        return `${sentences.join(" ")} Use these exact designs.\n`;
+    },
+    // Truncate to the provider/model limit, keeping the first N deterministically.
+    // imageKey is the imageRefs key of the image being generated (used for the warning).
+    limitReferences(references, imageAgentInfo, imageKey) {
+        if (!references) {
+            return undefined;
+        }
+        const { provider, model } = imageAgentInfo.imageParams;
+        const max = getMaxImageReferenceImages(provider, model ?? "");
+        if (max !== undefined && references.length > max) {
+            GraphAILogger.warn(`imagePrompt "${imageKey}": ${references.length} reference images exceed the ${provider} limit of ${max} — keeping the first ${max}`);
+            return references.slice(0, max);
+        }
+        return references;
+    },
+};

package/lib/types/provider2agent.d.ts

@@ -64,6 +64,7 @@ export declare const provider2ImageAgent: {
         models: string[];
         keyName: string;
         baseURLKeyName: string;
+        maxReferenceImages: number;
     };
     google: {
         agentName: string;
@@ -74,6 +75,9 @@ export declare const provider2ImageAgent: {
     replicate: {
         agentName: string;
         defaultModel: string;
+        imageModelParams: Record<string, {
+            maxReferenceImages?: number;
+        }>;
         models: string[];
         keyName: string;
     };
@@ -213,6 +217,17 @@ export declare const defaultProviders: {
 export declare const llm: (keyof typeof provider2LLMAgent)[];
 export type LLM = keyof typeof provider2LLMAgent;
 export declare const htmlLLMProvider: string[];
+export declare const ProviderName: {
+    readonly OpenAI: "openai";
+    readonly Google: "google";
+    readonly Gemini: "gemini";
+    readonly Anthropic: "anthropic";
+    readonly Replicate: "replicate";
+    readonly ElevenLabs: "elevenlabs";
+    readonly Mock: "mock";
+};
+export declare const isImageProvider: (provider: string | undefined) => provider is keyof typeof provider2ImageAgent;
+export declare const getMaxImageReferenceImages: (provider: string | undefined, model: string) => number | undefined;
 export declare const getModelAudio: (provider: keyof typeof provider2MovieAgent, model: string) => MovieAudioSpec | undefined;
 export declare const getModelDuration: (provider: keyof typeof provider2MovieAgent, model: string, movieDuration?: number) => number | undefined;
 export type ModelPricing = {

package/lib/types/provider2agent.js

@@ -64,6 +64,10 @@ export const deprecatedGoogleImageModelHints = {
 // See https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image
 // and https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-1-flash-image
 export const vertexAIGlobalOnlyImageModels = new Set(["gemini-3-pro-image-preview", "gemini-3.1-flash-image-preview"]);
+// Per-model reference image limits (image_input array). Only verified entries; unlisted models are not truncated.
+const replicateImageModelParams = {
+    "bytedance/seedream-4": { maxReferenceImages: 10 },
+};
 export const provider2ImageAgent = {
     openai: {
         agentName: "imageOpenaiAgent",
@@ -71,6 +75,7 @@ export const provider2ImageAgent = {
         models: [...gptImages],
         keyName: "OPENAI_API_KEY",
         baseURLKeyName: "OPENAI_BASE_URL",
+        maxReferenceImages: 16, // images.edit accepts up to 16 input images
     },
     google: {
         agentName: "imageGenAIAgent",
@@ -81,6 +86,7 @@ export const provider2ImageAgent = {
     replicate: {
         agentName: "imageReplicateAgent",
         defaultModel: "bytedance/seedream-4",
+        imageModelParams: replicateImageModelParams,
         models: [
             "bytedance/seedream-4",
             "qwen/qwen-image",
@@ -527,6 +533,31 @@ export const defaultProviders = {
 };
 export const llm = Object.keys(provider2LLMAgent);
 export const htmlLLMProvider = ["openai", "anthropic", "mock"];
+// Provider name literals. Compare via these identifiers (e.g. provider === ProviderName.Replicate)
+// instead of raw strings.
+export const ProviderName = {
+    OpenAI: "openai",
+    Google: "google",
+    Gemini: "gemini",
+    Anthropic: "anthropic",
+    Replicate: "replicate",
+    ElevenLabs: "elevenlabs",
+    Mock: "mock",
+};
+export const isImageProvider = (provider) => {
+    return provider !== undefined && provider in provider2ImageAgent;
+};
+// Max reference images an image model accepts as input; undefined means no known limit (no truncation).
+export const getMaxImageReferenceImages = (provider, model) => {
+    if (!isImageProvider(provider)) {
+        return undefined;
+    }
+    if (provider === ProviderName.Replicate) {
+        return provider2ImageAgent.replicate.imageModelParams[model]?.maxReferenceImages;
+    }
+    const agentInfo = provider2ImageAgent[provider];
+    return "maxReferenceImages" in agentInfo ? agentInfo.maxReferenceImages : undefined;
+};
 export const getModelAudio = (provider, model) => {
     const modelParams = provider2MovieAgent[provider]?.modelParams;
     return modelParams?.[model]?.audio;

package/lib/types/schema.d.ts

@@ -263,6 +263,15 @@ export declare const mulmoMarkdownMediaSchema: z.ZodObject<{
         opacity: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>]>>>;
 }, z.core.$strict>;
+export declare const ImageMediaType: {
+    readonly Image: "image";
+    readonly ImagePrompt: "imagePrompt";
+    readonly Movie: "movie";
+    readonly MoviePrompt: "moviePrompt";
+    readonly Beat: "beat";
+    readonly VoiceOver: "voice_over";
+};
+export declare const beatImageTypesWithoutOwnStill: readonly string[];
 export declare const mulmoImageMediaSchema: z.ZodObject<{
     type: z.ZodLiteral<"image">;
     source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -4135,6 +4144,21 @@ export declare const mulmoAudioAssetSchema: z.ZodUnion<readonly [z.ZodObject<{
     type: z.ZodLiteral<"midi">;
     source: z.ZodString;
 }, z.core.$strict>]>;
+export declare const BEAT_IMAGE_SENTINEL = "$beatImage";
+export declare const mulmoImageReferenceSchema: z.ZodObject<{
+    name: z.ZodOptional<z.ZodString>;
+    source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"url">;
+        url: z.ZodURL;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"base64">;
+        data: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"path">;
+        path: z.ZodString;
+    }, z.core.$strict>], "kind">>;
+    label: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
 export declare const mulmoImagePromptMediaSchema: z.ZodObject<{
     type: z.ZodLiteral<"imagePrompt">;
     prompt: z.ZodString;
@@ -4153,6 +4177,20 @@ export declare const mulmoImagePromptMediaSchema: z.ZodObject<{
         kind: z.ZodLiteral<"path">;
         path: z.ZodString;
     }, z.core.$strict>], "kind">>;
+    references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodOptional<z.ZodString>;
+        source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"url">;
+            url: z.ZodURL;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"base64">;
+            data: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"path">;
+            path: z.ZodString;
+        }, z.core.$strict>], "kind">>;
+        label: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>>;
 }, z.core.$strict>;
 export declare const mulmoImageParamsImagesValueSchema: z.ZodUnion<readonly [z.ZodObject<{
     type: z.ZodLiteral<"image">;
@@ -4184,6 +4222,20 @@ export declare const mulmoImageParamsImagesValueSchema: z.ZodUnion<readonly [z.Z
         kind: z.ZodLiteral<"path">;
         path: z.ZodString;
     }, z.core.$strict>], "kind">>;
+    references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodOptional<z.ZodString>;
+        source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"url">;
+            url: z.ZodURL;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"base64">;
+            data: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"path">;
+            path: z.ZodString;
+        }, z.core.$strict>], "kind">>;
+        label: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>>;
 }, z.core.$strict>, z.ZodObject<{
     type: z.ZodLiteral<"movie">;
     source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -4231,6 +4283,20 @@ export declare const mulmoImageParamsImagesSchema: z.ZodRecord<z.ZodString, z.Zo
         kind: z.ZodLiteral<"path">;
         path: z.ZodString;
     }, z.core.$strict>], "kind">>;
+    references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodOptional<z.ZodString>;
+        source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"url">;
+            url: z.ZodURL;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"base64">;
+            data: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"path">;
+            path: z.ZodString;
+        }, z.core.$strict>], "kind">>;
+        label: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>>;
 }, z.core.$strict>, z.ZodObject<{
     type: z.ZodLiteral<"movie">;
     source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -4330,6 +4396,20 @@ export declare const mulmoImageParamsSchema: z.ZodObject<{
             kind: z.ZodLiteral<"path">;
             path: z.ZodString;
         }, z.core.$strict>], "kind">>;
+        references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodOptional<z.ZodString>;
+            source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"url">;
+                url: z.ZodURL;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"base64">;
+                data: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"path">;
+                path: z.ZodString;
+            }, z.core.$strict>], "kind">>;
+            label: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>>;
     }, z.core.$strict>, z.ZodObject<{
         type: z.ZodLiteral<"movie">;
         source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -8800,6 +8880,20 @@ export declare const mulmoBeatSchema: z.ZodObject<{
             kind: z.ZodLiteral<"path">;
             path: z.ZodString;
         }, z.core.$strict>], "kind">>;
+        references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodOptional<z.ZodString>;
+            source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"url">;
+                url: z.ZodURL;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"base64">;
+                data: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"path">;
+                path: z.ZodString;
+            }, z.core.$strict>], "kind">>;
+            label: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>>;
     }, z.core.$strict>, z.ZodObject<{
         type: z.ZodLiteral<"movie">;
         source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -8942,6 +9036,20 @@ export declare const mulmoPresentationStyleSchema: z.ZodObject<{
                 kind: z.ZodLiteral<"path">;
                 path: z.ZodString;
             }, z.core.$strict>], "kind">>;
+            references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                name: z.ZodOptional<z.ZodString>;
+                source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"url">;
+                    url: z.ZodURL;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"base64">;
+                    data: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"path">;
+                    path: z.ZodString;
+                }, z.core.$strict>], "kind">>;
+                label: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
         }, z.core.$strict>, z.ZodObject<{
             type: z.ZodLiteral<"movie">;
             source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -9455,6 +9563,20 @@ export declare const mulmoScriptSchema: z.ZodObject<{
                 kind: z.ZodLiteral<"path">;
                 path: z.ZodString;
             }, z.core.$strict>], "kind">>;
+            references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                name: z.ZodOptional<z.ZodString>;
+                source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"url">;
+                    url: z.ZodURL;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"base64">;
+                    data: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"path">;
+                    path: z.ZodString;
+                }, z.core.$strict>], "kind">>;
+                label: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
         }, z.core.$strict>, z.ZodObject<{
             type: z.ZodLiteral<"movie">;
             source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -13913,6 +14035,20 @@ export declare const mulmoScriptSchema: z.ZodObject<{
                 kind: z.ZodLiteral<"path">;
                 path: z.ZodString;
             }, z.core.$strict>], "kind">>;
+            references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                name: z.ZodOptional<z.ZodString>;
+                source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"url">;
+                    url: z.ZodURL;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"base64">;
+                    data: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"path">;
+                    path: z.ZodString;
+                }, z.core.$strict>], "kind">>;
+                label: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
         }, z.core.$strict>, z.ZodObject<{
             type: z.ZodLiteral<"movie">;
             source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -14131,6 +14267,20 @@ export declare const mulmoStudioSchema: z.ZodObject<{
                     kind: z.ZodLiteral<"path">;
                     path: z.ZodString;
                 }, z.core.$strict>], "kind">>;
+                references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    name: z.ZodOptional<z.ZodString>;
+                    source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                        kind: z.ZodLiteral<"url">;
+                        url: z.ZodURL;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"base64">;
+                        data: z.ZodString;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"path">;
+                        path: z.ZodString;
+                    }, z.core.$strict>], "kind">>;
+                    label: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>>;
             }, z.core.$strict>, z.ZodObject<{
                 type: z.ZodLiteral<"movie">;
                 source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -18589,6 +18739,20 @@ export declare const mulmoStudioSchema: z.ZodObject<{
                     kind: z.ZodLiteral<"path">;
                     path: z.ZodString;
                 }, z.core.$strict>], "kind">>;
+                references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    name: z.ZodOptional<z.ZodString>;
+                    source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                        kind: z.ZodLiteral<"url">;
+                        url: z.ZodURL;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"base64">;
+                        data: z.ZodString;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"path">;
+                        path: z.ZodString;
+                    }, z.core.$strict>], "kind">>;
+                    label: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>>;
             }, z.core.$strict>, z.ZodObject<{
                 type: z.ZodLiteral<"movie">;
                 source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -18742,6 +18906,20 @@ export declare const mulmoPromptTemplateSchema: z.ZodObject<{
                     kind: z.ZodLiteral<"path">;
                     path: z.ZodString;
                 }, z.core.$strict>], "kind">>;
+                references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    name: z.ZodOptional<z.ZodString>;
+                    source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                        kind: z.ZodLiteral<"url">;
+                        url: z.ZodURL;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"base64">;
+                        data: z.ZodString;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"path">;
+                        path: z.ZodString;
+                    }, z.core.$strict>], "kind">>;
+                    label: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>>;
             }, z.core.$strict>, z.ZodObject<{
                 type: z.ZodLiteral<"movie">;
                 source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -19249,6 +19427,20 @@ export declare const mulmoPromptTemplateFileSchema: z.ZodObject<{
                     kind: z.ZodLiteral<"path">;
                     path: z.ZodString;
                 }, z.core.$strict>], "kind">>;
+                references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    name: z.ZodOptional<z.ZodString>;
+                    source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                        kind: z.ZodLiteral<"url">;
+                        url: z.ZodURL;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"base64">;
+                        data: z.ZodString;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"path">;
+                        path: z.ZodString;
+                    }, z.core.$strict>], "kind">>;
+                    label: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>>;
             }, z.core.$strict>, z.ZodObject<{
                 type: z.ZodLiteral<"movie">;
                 source: z.ZodDiscriminatedUnion<[z.ZodObject<{

package/lib/types/schema.js

@@ -130,9 +130,22 @@ const mulmoPdfMediaSchema = z
     source: mediaSourceSchema,
 })
     .strict();
+// Media type literals shared by imageParams.images entries and beat images. Compare via
+// these identifiers (e.g. entry.type === ImageMediaType.ImagePrompt) instead of raw strings.
+export const ImageMediaType = {
+    Image: "image",
+    ImagePrompt: "imagePrompt",
+    Movie: "movie",
+    MoviePrompt: "moviePrompt",
+    Beat: "beat",
+    VoiceOver: "voice_over",
+};
+// Beat image types whose still derives from the movie or another beat — they cannot supply
+// their own still for the $beatImage frame sentinel.
+export const beatImageTypesWithoutOwnStill = [ImageMediaType.Movie, ImageMediaType.Beat, ImageMediaType.VoiceOver];
 export const mulmoImageMediaSchema = z
     .object({
-    type: z.literal("image"),
+    type: z.literal(ImageMediaType.Image),
     source: mediaSourceSchema,
 })
     .strict();
@@ -144,13 +157,13 @@ const mulmoSvgMediaSchema = z
     .strict();
 export const mulmoMovieMediaSchema = z
     .object({
-    type: z.literal("movie"),
+    type: z.literal(ImageMediaType.Movie),
     source: mediaSourceSchema,
 })
     .strict();
 export const mulmoMoviePromptMediaSchema = z
     .object({
-    type: z.literal("moviePrompt"),
+    type: z.literal(ImageMediaType.MoviePrompt),
     prompt: z.string().min(1),
     imageName: z.string().optional().describe("Reference an imageRefs key to use as image-to-video input"),
 })
@@ -292,13 +305,13 @@ export const mulmoHtmlTailwindMediaSchema = z
     .strict();
 export const mulmoBeatReferenceMediaSchema = z
     .object({
-    type: z.literal("beat"),
+    type: z.literal(ImageMediaType.Beat),
     id: z.string().optional().describe("Specifies the beat to reference."),
 })
     .strict();
 export const mulmoVoiceOverMediaSchema = z
     .object({
-    type: z.literal("voice_over"),
+    type: z.literal(ImageMediaType.VoiceOver),
     startAt: z.number().optional().describe("The time to start the voice over the video in seconds."),
 })
     .strict();
@@ -339,13 +352,26 @@ const mulmoMidiMediaSchema = z
     .strict();
 export const mulmoAudioAssetSchema = z.union([mulmoAudioMediaSchema, mulmoMidiMediaSchema]);
 const imageIdSchema = z.string();
+// Sentinel for movieParams.firstFrameImageName / lastFrameImageName: "this beat's own generated image".
+export const BEAT_IMAGE_SENTINEL = "$beatImage";
+// imageRefs keys may not start with "$": that prefix is reserved for sentinels like $beatImage.
+const imageRefKeySchema = imageIdSchema.refine((key) => !key.startsWith("$"), { message: 'image keys must not start with the reserved prefix "$"' });
+export const mulmoImageReferenceSchema = z
+    .object({
+    name: imageIdSchema.optional().describe("An imageRefs key to use as a reference image"),
+    source: mediaSourceSchema.optional().describe("Direct source (path/url) to use as a reference image"),
+    label: z.string().optional().describe("Role of this reference (e.g. 'the firefighter Travis Kane'), injected into the prompt as a preamble"),
+})
+    .strict()
+    .refine((ref) => (ref.name === undefined) !== (ref.source === undefined), { message: "specify exactly one of name or source" });
 export const mulmoImagePromptMediaSchema = z
     .object({
-    type: z.literal("imagePrompt"),
+    type: z.literal(ImageMediaType.ImagePrompt),
     prompt: z.string().min(1),
     canvasSize: z.object({ width: z.number(), height: z.number() }).strict().optional(),
-    referenceImageName: imageIdSchema.optional().describe("Reference another imageRefs key as input for image generation"),
-    referenceImage: mediaSourceSchema.optional().describe("Direct source (path/url/base64) as reference image for generation"),
+    referenceImageName: imageIdSchema.optional().describe("Reference another imageRefs key as input for image generation. Sugar for references[0].name"),
+    referenceImage: mediaSourceSchema.optional().describe("Direct source (path/url/base64) as reference image for generation. Sugar for references[0].source"),
+    references: z.array(mulmoImageReferenceSchema).optional().describe("Reference images (by imageRefs key or direct source) with optional role labels"),
 })
     .strict();
 export const mulmoImageParamsImagesValueSchema = z.union([
@@ -354,7 +380,7 @@ export const mulmoImageParamsImagesValueSchema = z.union([
     mulmoMovieMediaSchema,
     mulmoMoviePromptMediaSchema,
 ]);
-export const mulmoImageParamsImagesSchema = z.record(imageIdSchema, mulmoImageParamsImagesValueSchema);
+export const mulmoImageParamsImagesSchema = z.record(imageRefKeySchema, mulmoImageParamsImagesValueSchema);
 export const mulmoFillOptionSchema = z
     .object({
     style: z.enum(["aspectFit", "aspectFill"]).optional().default("aspectFit"),
@@ -500,8 +526,12 @@ export const mulmoMovieParamsSchema = z.object({
     filters: z.array(mulmoVideoFilterSchema).optional(), // for movie.ts
     vertexai_project: z.string().optional(), // Google Cloud Project ID for Vertex AI
     vertexai_location: z.string().optional(), // Vertex AI location (default: us-central1)
-    firstFrameImageName: imageIdSchema.optional().describe("Reference an imageRefs key for the first frame (image-to-video input)"),
-    lastFrameImageName: imageIdSchema.optional().describe("Reference an imageRefs key for the last frame (image-to-video interpolation)"),
+    firstFrameImageName: imageIdSchema
+        .optional()
+        .describe("Reference an imageRefs key for the first frame (image-to-video input), or '$beatImage' for the beat's own generated image"),
+    lastFrameImageName: imageIdSchema
+        .optional()
+        .describe("Reference an imageRefs key for the last frame (image-to-video interpolation), or '$beatImage' for the beat's own generated image"),
     frameFillColor: z
         .string()
         .regex(/^(#|0x)?[0-9a-fA-F]{6}([0-9a-fA-F]{2})?$|^[a-zA-Z]+$/, "must be a hex color ('#RRGGBB[AA]', '0x' or no prefix) or a color name")
@@ -540,7 +570,7 @@ export const mulmoBeatSchema = z
     images: mulmoImageParamsImagesSchema
         .optional()
         .describe("Beat-local media references. Same schema as imageParams.images. Merged with global refs (local takes precedence)."),
-    imageNames: z.array(imageIdSchema).optional(), // list of image names to use for image generation. The default is all images in the imageParams.images.
+    imageNames: z.array(imageRefKeySchema).optional(), // list of image names to use for image generation. The default is all images in the imageParams.images.
     imagePrompt: z.string().optional(),
     moviePrompt: z.string().optional(),
     soundEffectPrompt: z.string().optional(),

package/lib/types/type.d.ts

@@ -1,6 +1,6 @@
 import { type CallbackFunction } from "graphai";
 import type { UsageCollectorAPI } from "./usage.js";
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoMovieMediaSchema, mulmoMoviePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoImageReferenceSchema, mulmoMovieMediaSchema, mulmoMoviePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "./const.js";
 import type { LLM } from "./provider2agent.js";
 import { z } from "zod";
@@ -49,6 +49,7 @@ export type MulmoGoogleImageModel = z.infer<typeof mulmoGoogleImageModelSchema>;
 export type MulmoGoogleMovieModel = z.infer<typeof mulmoGoogleMovieModelSchema>;
 export type MulmoReplicateMovieModel = z.infer<typeof mulmoReplicateMovieModelSchema>;
 export type MulmoImagePromptMedia = z.infer<typeof mulmoImagePromptMediaSchema>;
+export type MulmoImageReference = z.infer<typeof mulmoImageReferenceSchema>;
 export type MulmoMoviePromptMedia = z.infer<typeof mulmoMoviePromptMediaSchema>;
 export type MulmoMarkdownLayout = z.infer<typeof markdownLayoutSchema>;
 export type MulmoRow2 = z.infer<typeof row2Schema>;

package/lib/utils/context.d.ts

@@ -103,6 +103,20 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                     kind: "path";
                     path: string;
                 } | undefined;
+                references?: {
+                    name?: string | undefined;
+                    source?: {
+                        kind: "url";
+                        url: string;
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        kind: "path";
+                        path: string;
+                    } | undefined;
+                    label?: string | undefined;
+                }[] | undefined;
             }> | undefined;
             backgroundImage?: string | {
                 source: {
@@ -2404,6 +2418,20 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                     kind: "path";
                     path: string;
                 } | undefined;
+                references?: {
+                    name?: string | undefined;
+                    source?: {
+                        kind: "url";
+                        url: string;
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        kind: "path";
+                        path: string;
+                    } | undefined;
+                    label?: string | undefined;
+                }[] | undefined;
             }> | undefined;
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
@@ -2641,6 +2669,20 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         kind: "path";
                         path: string;
                     } | undefined;
+                    references?: {
+                        name?: string | undefined;
+                        source?: {
+                            kind: "url";
+                            url: string;
+                        } | {
+                            kind: "base64";
+                            data: string;
+                        } | {
+                            kind: "path";
+                            path: string;
+                        } | undefined;
+                        label?: string | undefined;
+                    }[] | undefined;
                 }> | undefined;
                 backgroundImage?: string | {
                     source: {
@@ -4942,6 +4984,20 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         kind: "path";
                         path: string;
                     } | undefined;
+                    references?: {
+                        name?: string | undefined;
+                        source?: {
+                            kind: "url";
+                            url: string;
+                        } | {
+                            kind: "base64";
+                            data: string;
+                        } | {
+                            kind: "path";
+                            path: string;
+                        } | undefined;
+                        label?: string | undefined;
+                    }[] | undefined;
                 }> | undefined;
                 imageNames?: string[] | undefined;
                 imagePrompt?: string | undefined;
@@ -5186,6 +5242,20 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     kind: "path";
                     path: string;
                 } | undefined;
+                references?: {
+                    name?: string | undefined;
+                    source?: {
+                        kind: "url";
+                        url: string;
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        kind: "path";
+                        path: string;
+                    } | undefined;
+                    label?: string | undefined;
+                }[] | undefined;
             }> | undefined;
             backgroundImage?: string | {
                 source: {

package/lib/utils/error_cause.d.ts

@@ -55,6 +55,7 @@ export declare const apiRateLimitErrorType = "apiRateLimit";
 export declare const apiKeyMissingType = "apiKeyMissing";
 export declare const invalidResponseType = "invalidResponse";
 export declare const voiceLimitReachedType = "voice_limit_reached";
+export declare const invalidFrameSentinelType = "invalidFrameSentinel";
 export declare const movieAction = "movie";
 export declare const imageAction = "images";
 export declare const audioAction = "audio";
@@ -69,6 +70,10 @@ export declare const multiLingualFileTarget = "multiLingualFile";
 export declare const videoSourceTarget = "videoSource";
 export declare const audioSourceTarget = "audioSource";
 export declare const codeTextTarget = "codeText";
+export declare const bothFramesTarget = "bothFrames";
+export declare const missingBeatImageTarget = "missingBeatImage";
+export declare const unsupportedImageTypeTarget = "unsupportedImageType";
+export declare const missingFirstFrameTarget = "missingFirstFrame";
 export declare const agentFileNotExistError: (agentName: string, action: string, target: string, fileName: string, beatIndex?: number) => {
     beatIndex?: number | undefined;
     type: string;
@@ -115,6 +120,13 @@ export declare const invalidAudioSourceError: (beatIndex: number) => {
     agentName: string;
     beatIndex: number;
 };
+export declare const beatImageSentinelError: (beatIndex: number, target: string) => {
+    type: string;
+    action: string;
+    target: string;
+    agentName: string;
+    beatIndex: number;
+};
 export declare const downLoadReferenceImageError: (key: string, url: string) => {
     type: string;
     action: string;

package/lib/utils/error_cause.js

@@ -56,6 +56,7 @@ export const apiRateLimitErrorType = "apiRateLimit";
 export const apiKeyMissingType = "apiKeyMissing";
 export const invalidResponseType = "invalidResponse";
 export const voiceLimitReachedType = "voice_limit_reached";
+export const invalidFrameSentinelType = "invalidFrameSentinel";
 // Actions
 export const movieAction = "movie";
 export const imageAction = "images";
@@ -72,6 +73,10 @@ export const multiLingualFileTarget = "multiLingualFile";
 export const videoSourceTarget = "videoSource";
 export const audioSourceTarget = "audioSource";
 export const codeTextTarget = "codeText";
+export const bothFramesTarget = "bothFrames";
+export const missingBeatImageTarget = "missingBeatImage";
+export const unsupportedImageTypeTarget = "unsupportedImageType";
+export const missingFirstFrameTarget = "missingFirstFrame";
 // Agent File Not Exist Errors
 export const agentFileNotExistError = (agentName, action, target, fileName, beatIndex) => {
     return {
@@ -111,6 +116,16 @@ export const invalidAudioSourceError = (beatIndex) => {
         beatIndex,
     };
 };
+// $beatImage frame sentinel misuse (invalid combination or beat cannot supply its own image)
+export const beatImageSentinelError = (beatIndex, target) => {
+    return {
+        type: invalidFrameSentinelType,
+        action: imageAction,
+        target,
+        agentName: "imagePreprocessAgent",
+        beatIndex,
+    };
+};
 // 404
 export const downLoadReferenceImageError = (key, url) => {
     return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.8.1",
+  "version": "2.9.0",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -89,7 +89,7 @@
   "homepage": "https://github.com/receptron/mulmocast-cli#readme",
   "dependencies": {
     "@google-cloud/text-to-speech": "^6.4.1",
-    "@google/genai": "^2.10.0",
+    "@google/genai": "^2.12.0",
     "@graphai/anthropic_agent": "^2.0.12",
     "@graphai/browserless_agent": "^2.0.3",
     "@graphai/gemini_agent": "^2.0.5",
@@ -102,7 +102,7 @@
     "@inquirer/input": "^5.1.2",
     "@inquirer/select": "^5.2.1",
     "@modelcontextprotocol/sdk": "^1.29.0",
-    "@modernized/fluent-ffmpeg": "^1.0.1",
+    "@modernized/fluent-ffmpeg": "^1.0.2",
     "@mozilla/readability": "^0.6.0",
     "@mulmocast/deck": "^1.1.0",
     "@tavily/core": "^0.7.6",