mulmocast 2.8.1 → 2.9.1

package/lib/actions/image_agents.d.ts

@@ -24,6 +24,9 @@ type ImagePreprocessAgentReturnValue = {
     };
     firstFrameImagePath?: string;
     lastFrameImagePath?: string;
+    firstFrameIsBeatImage?: boolean;
+    lastFrameIsBeatImage?: boolean;
+    frameFillColor?: string;
     movieReferenceImages?: {
         imagePath: string;
         referenceType: "ASSET" | "STYLE";
@@ -62,6 +65,23 @@ export declare const imagePreprocessAgent: (namedInputs: {
     index: number;
     imageRefs?: Record<string, string>;
 }) => Promise<ImagePreprocessAgentResponse>;
+export type BeatFramePreprocessorInput = {
+    imagePath?: string;
+    lastFrameImagePath?: string;
+    firstFrameIsBeatImage?: boolean;
+    lastFrameIsBeatImage?: boolean;
+    frameFillColor?: string;
+    referenceImageForMovie?: string;
+};
+export declare const beatFrameResolverAgent: (namedInputs: {
+    context: MulmoStudioContext;
+    beat: MulmoBeat;
+    index: number;
+    preprocessor?: BeatFramePreprocessorInput;
+}) => Promise<{
+    lastFrameImagePath: string | undefined;
+    referenceImageForMovie: string | undefined;
+}>;
 export declare const imagePluginAgent: (namedInputs: {
     context: MulmoStudioContext;
     beat: MulmoBeat;

package/lib/actions/image_agents.js

@@ -1,9 +1,11 @@
 import fs from "fs";
 import { createHash } from "crypto";
 import { GraphAILogger } from "graphai";
+import { BEAT_IMAGE_SENTINEL, ImageMediaType, beatImageTypesWithoutOwnStill } from "../types/schema.js";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods, MulmoBeatMethods, MulmoMediaSourceMethods } from "../methods/index.js";
 import { getBeatPngImagePath, getBeatMoviePaths, getBeatAnimatedVideoPath, getAudioFilePath, getGroupedAudioFilePath, getReferenceImagePath, } from "../utils/file.js";
 import { ffmpegGetImageDimensions, padImageToCanvas } from "../utils/ffmpeg_utils.js";
+import { beatImageSentinelError, bothFramesTarget, missingBeatImageTarget, unsupportedImageTypeTarget, missingFirstFrameTarget } from "../utils/error_cause.js";
 import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
 import { renderHTMLToImage } from "../utils/html_render.js";
 import { beatId } from "../utils/utils.js";
@@ -71,6 +73,64 @@ export const conformFrameImageToCanvas = async (context, imageName, imagePath, f
         conformingInFlight.delete(destPath);
     }
 };
+const validateBeatImageSentinel = (movieParams, beat, index) => {
+    if (movieParams.firstFrameImageName === BEAT_IMAGE_SENTINEL && movieParams.lastFrameImageName === BEAT_IMAGE_SENTINEL) {
+        throw new Error(`${BEAT_IMAGE_SENTINEL} cannot be used for both firstFrameImageName and lastFrameImageName. beat: ${index}`, {
+            cause: beatImageSentinelError(index, bothFramesTarget),
+        });
+    }
+    if (!beat.imagePrompt && !beat.image) {
+        throw new Error(`${BEAT_IMAGE_SENTINEL} requires the beat to have an imagePrompt or image. beat: ${index}`, {
+            cause: beatImageSentinelError(index, missingBeatImageTarget),
+        });
+    }
+    if (beat.image && beatImageTypesWithoutOwnStill.includes(beat.image.type)) {
+        throw new Error(`${BEAT_IMAGE_SENTINEL} cannot be used with image type "${beat.image.type}" (the beat image is not its own still). beat: ${index}`, {
+            cause: beatImageSentinelError(index, unsupportedImageTypeTarget),
+        });
+    }
+    // The implicit first frame (the raw beat image) is not conformed to the canvas, so pairing
+    // it with a conformed $beatImage last frame would feed mismatched frame sizes to strict
+    // image-to-video models (e.g. wan-2.2-i2v-fast). Require an explicit, conformed first frame.
+    if (movieParams.lastFrameImageName === BEAT_IMAGE_SENTINEL && !movieParams.firstFrameImageName) {
+        throw new Error(`${BEAT_IMAGE_SENTINEL} as lastFrameImageName requires firstFrameImageName. beat: ${index}`, {
+            cause: beatImageSentinelError(index, missingFirstFrameTarget),
+        });
+    }
+};
+// Resolve firstFrameImageName/lastFrameImageName. Named refs conform to the canvas here;
+// the $beatImage sentinel only sets flags (the beat image does not exist yet at preprocess
+// time) and is resolved by beatFrameResolverAgent after image generation.
+const applyFrameImageParams = async (returnValue, context, beat, index, movieParams, imageRefs) => {
+    const frameFillColor = movieParams.frameFillColor ?? "black";
+    // A style-level $beatImage merges into every beat; beats that generate no movie
+    // (no moviePrompt) ignore the sentinel instead of failing validation.
+    const sentinelActive = Boolean(beat.moviePrompt);
+    const firstFrameImageName = !sentinelActive && movieParams.firstFrameImageName === BEAT_IMAGE_SENTINEL ? undefined : movieParams.firstFrameImageName;
+    const lastFrameImageName = !sentinelActive && movieParams.lastFrameImageName === BEAT_IMAGE_SENTINEL ? undefined : movieParams.lastFrameImageName;
+    if (firstFrameImageName === BEAT_IMAGE_SENTINEL || lastFrameImageName === BEAT_IMAGE_SENTINEL) {
+        validateBeatImageSentinel(movieParams, beat, index);
+        returnValue.frameFillColor = frameFillColor;
+    }
+    if (firstFrameImageName === BEAT_IMAGE_SENTINEL) {
+        returnValue.firstFrameIsBeatImage = true;
+    }
+    else if (firstFrameImageName && imageRefs) {
+        const firstFramePath = imageRefs[firstFrameImageName];
+        if (firstFramePath) {
+            returnValue.firstFrameImagePath = await conformFrameImageToCanvas(context, firstFrameImageName, firstFramePath, frameFillColor);
+        }
+    }
+    if (lastFrameImageName === BEAT_IMAGE_SENTINEL) {
+        returnValue.lastFrameIsBeatImage = true;
+    }
+    else if (lastFrameImageName && imageRefs) {
+        const lastFramePath = imageRefs[lastFrameImageName];
+        if (lastFramePath) {
+            returnValue.lastFrameImagePath = await conformFrameImageToCanvas(context, lastFrameImageName, lastFramePath, frameFillColor);
+        }
+    }
+};
 export const imagePreprocessAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs } = namedInputs;
     const studioBeat = context.studio.beats[index];
@@ -88,7 +148,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
         movieFile: beat.moviePrompt ? moviePaths.movieFile : undefined,
         beatDuration: beat.duration ?? studioBeat?.duration,
     };
-    const isMovie = Boolean(beat.moviePrompt || beat?.image?.type === "movie");
+    const isMovie = Boolean(beat.moviePrompt || beat?.image?.type === ImageMediaType.Movie);
     if (beat.soundEffectPrompt) {
         if (isMovie) {
             returnValue.soundEffectAgentInfo = MulmoPresentationStyleMethods.getSoundEffectAgentInfo(context.presentationStyle, beat);
@@ -130,19 +190,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
     // Shallow-merge like getMovieAgentInfo: beat-level fields override style-level fields
     // individually, so a beat setting only lastFrameImageName keeps the global frameFillColor.
     const movieParams = { ...context.presentationStyle.movieParams, ...beat.movieParams };
-    const frameFillColor = movieParams.frameFillColor ?? "black";
-    if (movieParams.firstFrameImageName && imageRefs) {
-        const firstFramePath = imageRefs[movieParams.firstFrameImageName];
-        if (firstFramePath) {
-            returnValue.firstFrameImagePath = await conformFrameImageToCanvas(context, movieParams.firstFrameImageName, firstFramePath, frameFillColor);
-        }
-    }
-    if (movieParams.lastFrameImageName && imageRefs) {
-        const lastFramePath = imageRefs[movieParams.lastFrameImageName];
-        if (lastFramePath) {
-            returnValue.lastFrameImagePath = await conformFrameImageToCanvas(context, movieParams.lastFrameImageName, lastFramePath, frameFillColor);
-        }
-    }
+    await applyFrameImageParams(returnValue, context, beat, index, movieParams, imageRefs);
     if (movieParams.referenceImages && imageRefs) {
         returnValue.movieReferenceImages = movieParams.referenceImages
             .map((ref) => {
@@ -156,7 +204,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
         const pluginPath = plugin.path({ beat, context, imagePath, ...htmlStyle(context, beat) });
         const markdown = plugin.markdown ? plugin.markdown({ beat, context, imagePath, ...htmlStyle(context, beat) }) : undefined;
         const html = plugin.html ? await plugin.html({ beat, context, imagePath, ...htmlStyle(context, beat) }) : undefined;
-        const isTypeMovie = beat.image.type === "movie";
+        const isTypeMovie = beat.image.type === ImageMediaType.Movie;
         const isAnimatedHtml = MulmoBeatMethods.isAnimatedHtmlTailwind(beat);
         // animation and moviePrompt cannot be used together
         if (isAnimatedHtml && beat.moviePrompt) {
@@ -203,6 +251,26 @@ export const imagePreprocessAgent = async (namedInputs) => {
     const movieFirstFramePath = returnValue.firstFrameImagePath ?? imagePath;
     return { ...returnValue, imagePath, referenceImageForMovie: movieFirstFramePath, imageAgentInfo, prompt, referenceImages };
 };
+// Resolves $beatImage frame references after the beat's image exists. imagePreprocessAgent runs
+// before image generation, so it can only flag the sentinel; this agent runs after
+// imageGenerator/imagePlugin and conforms the now-existing beat image to the canvas.
+// movieGenerator consumes only referenceImageForMovie (the movie's first frame / i2v input)
+// and lastFrameImagePath.
+export const beatFrameResolverAgent = async (namedInputs) => {
+    const { context, beat, index, preprocessor } = namedInputs;
+    if (preprocessor?.imagePath && (preprocessor.firstFrameIsBeatImage || preprocessor.lastFrameIsBeatImage)) {
+        const name = `${beatId(beat.id, index)}_beatImage`;
+        const conformed = await conformFrameImageToCanvas(context, name, preprocessor.imagePath, preprocessor.frameFillColor ?? "black");
+        return {
+            lastFrameImagePath: preprocessor.lastFrameIsBeatImage ? conformed : preprocessor.lastFrameImagePath,
+            referenceImageForMovie: preprocessor.firstFrameIsBeatImage ? conformed : preprocessor.referenceImageForMovie,
+        };
+    }
+    return {
+        lastFrameImagePath: preprocessor?.lastFrameImagePath,
+        referenceImageForMovie: preprocessor?.referenceImageForMovie,
+    };
+};
 export const imagePluginAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs, movieRefs } = namedInputs;
     const { imagePath } = getBeatPngImagePath(context, index);

package/lib/actions/image_references.d.ts

@@ -1,10 +1,13 @@
 import { MulmoStudioContext, MulmoBeat, MulmoImagePromptMedia } from "../types/index.js";
+import type { ResolvedImageReference } from "../methods/mulmo_image_prompt_media.js";
+export type { ResolvedImageReference } from "../methods/mulmo_image_prompt_media.js";
 export declare const generateReferenceImage: (inputs: {
     context: MulmoStudioContext;
     key: string;
     index: number;
     image: MulmoImagePromptMedia;
     referenceImagePath?: string;
+    references?: ResolvedImageReference[];
     force?: boolean;
 }) => Promise<string>;
 export type MediaRefs = {

package/lib/actions/image_references.js

@@ -1,10 +1,36 @@
-import { GraphAI, GraphAILogger } from "graphai";
+import { GraphAI, GraphAILogger, graphDataLatestVersion } from "graphai";
 import { getReferenceImagePath } from "../utils/file.js";
 import { imageGraphOption } from "./graph_option.js";
 import { createUsageCallback } from "../utils/usage_callback.js";
-import { MulmoPresentationStyleMethods, MulmoMediaSourceMethods } from "../methods/index.js";
+import { MulmoPresentationStyleMethods, MulmoMediaSourceMethods, MulmoImagePromptMediaMethods } from "../methods/index.js";
+import { ImageMediaType, } from "../types/index.js";
 import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent, imageReplicateAgent, movieGenAIAgent, movieReplicateAgent } from "../agents/index.js";
 import { agentGenerationError, imageReferenceAction, imageFileTarget, movieFileTarget } from "../utils/error_cause.js";
+// Resolve the normalized reference list to local file paths (with labels), in reference order.
+// imageKey is the imageRefs key of the image being generated (used for warnings and url
+// download file naming). A named reference that is missing from imageRefs warns and is
+// skipped (existing behavior).
+const resolveImageReferences = async (context, imageKey, image, imageRefs) => {
+    const imageReferences = MulmoImagePromptMediaMethods.getReferences(image);
+    const resolved = await Promise.all(imageReferences.map(async (ref, i) => {
+        if (ref.name !== undefined) {
+            const path = imageRefs?.[ref.name];
+            if (!path) {
+                GraphAILogger.warn(`imagePrompt "${imageKey}": reference "${ref.name}" not found in imageRefs — generating without this reference`);
+                return undefined;
+            }
+            return { path, label: ref.label };
+        }
+        if (ref.source !== undefined) {
+            // The legacy referenceImage keeps the bare imageKey so url downloads reuse their historical cache file name.
+            const sourceKey = ref.legacy ? imageKey : `${imageKey}_ref${i}`;
+            return { path: await MulmoMediaSourceMethods.imageReference(ref.source, context, sourceKey), label: ref.label };
+        }
+        return undefined;
+    }));
+    const references = resolved.filter((ref) => ref !== undefined);
+    return references.length > 0 ? references : undefined;
+};
 // public api
 // Application may call this function directly to generate reference image.
 export const generateReferenceImage = async (inputs) => {
@@ -12,11 +38,13 @@ export const generateReferenceImage = async (inputs) => {
     const imagePath = getReferenceImagePath(context, key, "png");
     // generate image
     const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle);
-    const prompt = `${image.prompt}\n${imageAgentInfo.imageParams.style || ""}`;
+    const references = MulmoImagePromptMediaMethods.limitReferences(inputs.references ?? (referenceImagePath ? [{ path: referenceImagePath }] : undefined), imageAgentInfo, key);
+    const preamble = MulmoImagePromptMediaMethods.buildReferencePreamble((references ?? []).map((ref) => ref.label));
+    const prompt = `${preamble}${image.prompt}\n${imageAgentInfo.imageParams.style || ""}`;
     GraphAILogger.info(`Generating reference image for ${key}: ${prompt}`);
-    const referenceImages = referenceImagePath ? [referenceImagePath] : undefined;
+    const referenceImages = references ? references.map((ref) => ref.path) : undefined;
     const image_graph_data = {
-        version: 0.5,
+        version: graphDataLatestVersion,
         nodes: {
             imageGenerator: {
                 agent: imageAgentInfo.agent,
@@ -62,33 +90,30 @@ export const getMediaRefs = async (context) => {
     }
     const imageRefs = {};
     const movieRefs = {};
-    // Stage 1: resolve non-referencing entries (image, imagePrompt without referenceImageName, movie)
+    // Stage 1: resolve non-referencing entries (image, imagePrompt without named references, movie)
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, index) => {
         const image = images[key];
-        if (image.type === "imagePrompt" && !image.referenceImageName) {
-            const refPath = image.referenceImage ? await MulmoMediaSourceMethods.imageReference(image.referenceImage, context, key) : undefined;
-            imageRefs[key] = await generateReferenceImage({ context, key, index, image, referenceImagePath: refPath, force: false });
+        if (image.type === ImageMediaType.ImagePrompt && !MulmoImagePromptMediaMethods.hasNamedReference(image)) {
+            const references = await resolveImageReferences(context, key, image);
+            imageRefs[key] = await generateReferenceImage({ context, key, index, image, references, force: false });
         }
-        else if (image.type === "image") {
+        else if (image.type === ImageMediaType.Image) {
             imageRefs[key] = await MulmoMediaSourceMethods.imageReference(image.source, context, key);
         }
-        else if (image.type === "movie") {
+        else if (image.type === ImageMediaType.Movie) {
             movieRefs[key] = await resolveMovieReference(image, context, key);
         }
     }));
-    // Stage 2: resolve imagePrompt with referenceImageName (depends on Stage 1 results)
+    // Stage 2: resolve imagePrompt with named references (depends on Stage 1 results)
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, index) => {
         const image = images[key];
-        if (image.type === "imagePrompt" && image.referenceImageName) {
-            const refPath = imageRefs[image.referenceImageName];
-            if (!refPath) {
-                GraphAILogger.warn(`imagePrompt "${key}": referenceImageName "${image.referenceImageName}" not found in imageRefs — generating without reference`);
-            }
-            imageRefs[key] = await generateReferenceImage({ context, key, index, image, referenceImagePath: refPath, force: false });
+        if (image.type === ImageMediaType.ImagePrompt && MulmoImagePromptMediaMethods.hasNamedReference(image)) {
+            const references = await resolveImageReferences(context, key, image, imageRefs);
+            imageRefs[key] = await generateReferenceImage({ context, key, index, image, references, force: false });
         }
     }));
     return { imageRefs, movieRefs };
@@ -102,7 +127,7 @@ const generateReferenceMovie = async (inputs) => {
     const movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle);
     GraphAILogger.info(`Generating reference movie for ${key}: ${moviePrompt.prompt}`);
     const movie_graph_data = {
-        version: 0.5,
+        version: graphDataLatestVersion,
         nodes: {
             movieGenerator: {
                 agent: movieAgentInfo.agent,
@@ -145,45 +170,42 @@ const generateReferenceMovie = async (inputs) => {
 const resolveLocalRefs = async (context, images, beatIndex, globalImageRefs) => {
     const localImageRefs = {};
     const localMovieRefs = {};
-    // Stage 1: image, imagePrompt (without referenceImageName), movie (parallel)
+    // Stage 1: image, imagePrompt (without named references), movie (parallel)
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, i) => {
         const entry = images[key];
-        if (entry.type === "imagePrompt" && !entry.referenceImageName) {
-            const refPath = entry.referenceImage ? await MulmoMediaSourceMethods.imageReference(entry.referenceImage, context, key) : undefined;
+        if (entry.type === ImageMediaType.ImagePrompt && !MulmoImagePromptMediaMethods.hasNamedReference(entry)) {
+            const references = await resolveImageReferences(context, key, entry);
             localImageRefs[key] = await generateReferenceImage({
                 context,
                 key,
                 index: beatIndex * 100 + i,
                 image: entry,
-                referenceImagePath: refPath,
+                references,
             });
         }
-        else if (entry.type === "image") {
+        else if (entry.type === ImageMediaType.Image) {
             localImageRefs[key] = await MulmoMediaSourceMethods.imageReference(entry.source, context, key);
         }
-        else if (entry.type === "movie") {
+        else if (entry.type === ImageMediaType.Movie) {
             localMovieRefs[key] = await resolveMovieReference(entry, context, key);
         }
     }));
-    // Stage 2: imagePrompt with referenceImageName (depends on Stage 1)
+    // Stage 2: imagePrompt with named references (depends on Stage 1)
     const combinedImageRefsForImagePrompt = { ...globalImageRefs, ...localImageRefs };
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, i) => {
         const entry = images[key];
-        if (entry.type === "imagePrompt" && entry.referenceImageName) {
-            const refPath = combinedImageRefsForImagePrompt[entry.referenceImageName];
-            if (!refPath) {
-                GraphAILogger.warn(`imagePrompt "${key}": referenceImageName "${entry.referenceImageName}" not found — generating without reference`);
-            }
+        if (entry.type === ImageMediaType.ImagePrompt && MulmoImagePromptMediaMethods.hasNamedReference(entry)) {
+            const references = await resolveImageReferences(context, key, entry, combinedImageRefsForImagePrompt);
             localImageRefs[key] = await generateReferenceImage({
                 context,
                 key,
                 index: beatIndex * 100 + i,
                 image: entry,
-                referenceImagePath: refPath,
+                references,
             });
         }
     }));
@@ -193,7 +215,7 @@ const resolveLocalRefs = async (context, images, beatIndex, globalImageRefs) =>
         .sort()
         .map(async (key, i) => {
         const entry = images[key];
-        if (entry.type === "moviePrompt") {
+        if (entry.type === ImageMediaType.MoviePrompt) {
             const refImagePath = entry.imageName ? combinedImageRefs[entry.imageName] : undefined;
             localMovieRefs[key] = await generateReferenceMovie({
                 context,

package/lib/actions/images.d.ts

@@ -72,6 +72,9 @@ export declare const beat_graph_data: {
                 };
                 firstFrameImagePath?: string;
                 lastFrameImagePath?: string;
+                firstFrameIsBeatImage?: boolean;
+                lastFrameIsBeatImage?: boolean;
+                frameFillColor?: string;
                 movieReferenceImages?: {
                     imagePath: string;
                     referenceType: "ASSET" | "STYLE";
@@ -103,6 +106,9 @@ export declare const beat_graph_data: {
                 };
                 firstFrameImagePath?: string;
                 lastFrameImagePath?: string;
+                firstFrameIsBeatImage?: boolean;
+                lastFrameIsBeatImage?: boolean;
+                frameFillColor?: string;
                 movieReferenceImages?: {
                     imagePath: string;
                     referenceType: "ASSET" | "STYLE";
@@ -145,6 +151,9 @@ export declare const beat_graph_data: {
                 };
                 firstFrameImagePath?: string;
                 lastFrameImagePath?: string;
+                firstFrameIsBeatImage?: boolean;
+                lastFrameIsBeatImage?: boolean;
+                frameFillColor?: string;
                 movieReferenceImages?: {
                     imagePath: string;
                     referenceType: "ASSET" | "STYLE";
@@ -179,6 +188,9 @@ export declare const beat_graph_data: {
                 };
                 firstFrameImagePath?: string;
                 lastFrameImagePath?: string;
+                firstFrameIsBeatImage?: boolean;
+                lastFrameIsBeatImage?: boolean;
+                frameFillColor?: string;
                 movieReferenceImages?: {
                     imagePath: string;
                     referenceType: "ASSET" | "STYLE";
@@ -298,6 +310,24 @@ export declare const beat_graph_data: {
             };
             defaultValue: {};
         };
+        frameResolver: {
+            agent: (namedInputs: {
+                context: MulmoStudioContext;
+                beat: import("../types/type.js").MulmoBeat;
+                index: number;
+                preprocessor?: import("./image_agents.js").BeatFramePreprocessorInput;
+            }) => Promise<{
+                lastFrameImagePath: string | undefined;
+                referenceImageForMovie: string | undefined;
+            }>;
+            inputs: {
+                onComplete: string[];
+                context: string;
+                beat: string;
+                index: string;
+                preprocessor: string;
+            };
+        };
         movieGenerator: {
             if: string;
             agent: string;

package/lib/actions/images.js

@@ -13,7 +13,7 @@ import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
 import { audioCheckerError } from "../utils/error_cause.js";
 import { extractImageFromMovie, ffmpegGetMediaDuration, trimMusic } from "../utils/ffmpeg_utils.js";
 import { getMediaRefs, resolveBeatLocalRefs } from "./image_references.js";
-import { imagePreprocessAgent, imagePluginAgent, htmlImageGeneratorAgent } from "./image_agents.js";
+import { imagePreprocessAgent, imagePluginAgent, htmlImageGeneratorAgent, beatFrameResolverAgent } from "./image_agents.js";
 import { imageGraphOption } from "./graph_option.js";
 import { createUsageCallback } from "../utils/usage_callback.js";
 const vanillaAgents = vanilla.default ?? vanilla;
@@ -169,15 +169,27 @@ export const beat_graph_data = {
             },
             defaultValue: {},
         },
+        // Resolves $beatImage frame references once the beat image exists (after imageGenerator/imagePlugin);
+        // passes the preprocessor's frame paths through unchanged for non-sentinel beats.
+        frameResolver: {
+            agent: beatFrameResolverAgent,
+            inputs: {
+                onComplete: [":imageGenerator", ":imagePlugin"],
+                context: ":context",
+                beat: ":beat",
+                index: ":__mapIndex",
+                preprocessor: ":preprocessor",
+            },
+        },
         movieGenerator: {
             if: ":beat.moviePrompt",
             agent: ":preprocessor.movieAgentInfo.agent",
             inputs: {
                 media: "movie",
-                onComplete: [":imageGenerator", ":imagePlugin"], // to wait for imageGenerator to finish
+                onComplete: [":frameResolver"], // frameResolver itself waits for imageGenerator/imagePlugin
                 prompt: ":beat.moviePrompt",
-                imagePath: ":preprocessor.referenceImageForMovie",
-                lastFrameImagePath: ":preprocessor.lastFrameImagePath",
+                imagePath: ":frameResolver.referenceImageForMovie",
+                lastFrameImagePath: ":frameResolver.lastFrameImagePath",
                 referenceImages: ":preprocessor.movieReferenceImages",
                 movieFile: ":preprocessor.movieFile", // for google genai agent
                 cache: {

package/lib/actions/movie.d.ts

@@ -1,4 +1,4 @@
-import { MulmoStudioContext, MulmoBeat, MulmoTransition, MulmoCanvasDimension, MulmoFillOption, MulmoVideoFilter } from "../types/index.js";
+import { MulmoStudioContext, MulmoTransition, MulmoCanvasDimension, MulmoFillOption, MulmoVideoFilter } from "../types/index.js";
 import { FfmpegContext } from "../utils/ffmpeg_utils.js";
 type VideoId = string | undefined;
 export declare const getVideoPart: (inputIndex: number, isMovie: boolean, duration: number, canvasInfo: MulmoCanvasDimension, fillOption: MulmoFillOption, speed: number, filters?: MulmoVideoFilter[], frameCount?: number) => {
@@ -11,15 +11,8 @@ export declare const getAudioPart: (inputIndex: number, duration: number, delay:
 };
 export declare const getOutOverlayCoords: (transitionType: string, d: number, t: number) => string;
 export declare const getInOverlayCoords: (transitionType: string, d: number, t: number) => string;
-export declare const getNeedFirstFrame: (context: MulmoStudioContext) => boolean[];
-export declare const getNeedLastFrame: (context: MulmoStudioContext) => boolean[];
-export declare const resolveMovieVolume: (beat: MulmoBeat, context: MulmoStudioContext) => number;
 export declare const isExplicitMixMode: (context: MulmoStudioContext) => boolean;
 export declare const mixAudiosFromMovieBeats: (ffmpegContext: FfmpegContext, artifactAudioId: string, audioIdsFromMovieBeats: string[], context: MulmoStudioContext) => string;
-export declare const getExtraPadding: (context: MulmoStudioContext, index: number) => number;
-export declare const getFillOption: (context: MulmoStudioContext, beat: MulmoBeat) => {
-    style: "aspectFit" | "aspectFill";
-};
 export declare const getTransitionVideoId: (transition: MulmoTransition, videoIdsForBeats: VideoId[], index: number) => {
     videoId: string;
     nextVideoId: undefined;
@@ -28,7 +21,7 @@ export declare const getTransitionVideoId: (transition: MulmoTransition, videoId
     videoId: string;
     nextVideoId: string;
     beatIndex: number;
-};
+} | null;
 export declare const getConcatVideoFilter: (concatVideoId: string, videoIdsForBeats: VideoId[]) => string;
 export declare const getTransitionFrameDurations: (context: MulmoStudioContext, index: number) => {
     firstDuration: number;