mulmocast 2.8.1 → 2.9.1

package/lib/actions/movie.js

@@ -1,6 +1,5 @@
 import { GraphAILogger, assert } from "graphai";
-import { mulmoFillOptionSchema, } from "../types/index.js";
-import { MulmoPresentationStyleMethods } from "../methods/index.js";
+import { MulmoPresentationStyleMethods, MulmoBeatMethods } from "../methods/index.js";
 import { getAudioArtifactFilePath, getOutputVideoFilePath, writingMessage, isFile } from "../utils/file.js";
 import { createVideoFileError, createVideoSourceError } from "../utils/error_cause.js";
 import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextPushFormattedAudio, FfmpegContextGenerateOutput, } from "../utils/ffmpeg_utils.js";
@@ -9,7 +8,14 @@ import { convertVideoFilterToFFmpeg } from "../utils/video_filter.js";
 // const isMac = process.platform === "darwin";
 const videoCodec = "libx264"; // "h264_videotoolbox" (macOS only) is too noisy
 const VIDEO_FPS = 30;
-const DEFAULT_DUCKING_RATIO = 0.3;
+// videoIdsForBeats holds undefined for voice_over beats; walk back to the previous rendered beat's videoId.
+const getPrevVideoId = (videoIdsForBeats, index) => {
+    const offset = videoIdsForBeats
+        .slice(0, Math.max(index, 0))
+        .reverse()
+        .findIndex((videoId) => videoId !== undefined);
+    return offset < 0 ? undefined : videoIdsForBeats[index - 1 - offset];
+};
 export const getVideoPart = (inputIndex, isMovie, duration, canvasInfo, fillOption, speed, filters, frameCount) => {
     const videoId = `v${inputIndex}`;
     const videoFilters = [];
@@ -149,109 +155,104 @@ export const getInOverlayCoords = (transitionType, d, t) => {
     }
     throw new Error(`Unknown transition type: ${transitionType}`);
 };
-const addTransitionEffects = (ffmpegContext, captionedVideoId, context, transitionVideoIds, beatTimestamps, videoIdsForBeats) => {
-    if (transitionVideoIds.length === 0) {
-        return captionedVideoId;
+// Each of these returns the id of the video to keep building on, or undefined when the transition
+// cannot be applied (in which case the caller keeps the untouched video).
+const addFadeTransition = ({ ffmpegContext, prevVideoId, transitionVideoId, outputVideoId, startAt, duration }) => {
+    // Fade out the previous beat's last frame
+    const processedVideoId = `${transitionVideoId}_f`;
+    ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,fade=t=out:d=${duration}:alpha=1,setpts=PTS-STARTPTS+${startAt}/TB[${processedVideoId}]`);
+    ffmpegContext.filterComplex.push(`[${prevVideoId}][${processedVideoId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
+    return outputVideoId;
+};
+const addSlideoutTransition = ({ ffmpegContext, transitionType, prevVideoId, transitionVideoId, outputVideoId, startAt, duration }) => {
+    // Slideout: previous beat's last frame slides out
+    const processedVideoId = `${transitionVideoId}_f`;
+    ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${processedVideoId}]`);
+    ffmpegContext.filterComplex.push(`[${prevVideoId}][${processedVideoId}]overlay=${getOutOverlayCoords(transitionType, duration, startAt)}:enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
+    return outputVideoId;
+};
+const addSlideinTransition = ({ ffmpegContext, transitionType, prevVideoId, nextVideoId, prevLastFrame, outputVideoId, startAt, duration, }) => {
+    // Slidein: this beat's first frame slides in over the previous beat's last frame
+    if (!nextVideoId || !prevLastFrame) {
+        return undefined; // Cannot apply slidein without the first frame or the previous shot
+    }
+    // Prepare background (last frame of previous beat)
+    const backgroundVideoId = `${prevLastFrame}_bg`;
+    ffmpegContext.filterComplex.push(`[${prevLastFrame}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${backgroundVideoId}]`);
+    // Prepare sliding frame (first frame of this beat)
+    const slideinFrameId = `${nextVideoId}_f`;
+    ffmpegContext.filterComplex.push(`[${nextVideoId}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${slideinFrameId}]`);
+    // First overlay: put background on top of concat video
+    const bgOutputId = `${prevLastFrame}_bg_o`;
+    ffmpegContext.filterComplex.push(`[${prevVideoId}][${backgroundVideoId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${bgOutputId}]`);
+    // Second overlay: slide in the new frame on top of background
+    ffmpegContext.filterComplex.push(`[${bgOutputId}][${slideinFrameId}]overlay=${getInOverlayCoords(transitionType, duration, startAt)}:enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
+    return outputVideoId;
+};
+const addWipeTransition = ({ ffmpegContext, transitionType, prevVideoId, transitionVideoId, nextVideoId, outputVideoId, startAt, duration, }) => {
+    // Wipe transition: use xfade filter between previous beat's last frame and this beat's first frame
+    if (!nextVideoId) {
+        return undefined; // Cannot apply wipe without first frame
+    }
+    // For wipe transitions, use offset=0 so transition starts immediately and completes within duration
+    // This ensures 0-100% wipe happens exactly during the transition duration
+    const xfadeOffset = 0;
+    // Apply xfade with offset=0 for complete 0-100% transition
+    // Both input videos should be at least duration seconds long (ensured by static frame generation)
+    const xfadeOutputId = `${transitionVideoId}_xfade`;
+    ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuv420p[${transitionVideoId}_fmt]`);
+    ffmpegContext.filterComplex.push(`[${nextVideoId}]format=yuv420p[${nextVideoId}_fmt]`);
+    ffmpegContext.filterComplex.push(`[${transitionVideoId}_fmt][${nextVideoId}_fmt]xfade=transition=${transitionType}:duration=${duration}:offset=${xfadeOffset}[${xfadeOutputId}]`);
+    // Set PTS for overlay timing
+    const xfadeTimedId = `${xfadeOutputId}_t`;
+    ffmpegContext.filterComplex.push(`[${xfadeOutputId}]setpts=PTS-STARTPTS+${startAt}/TB[${xfadeTimedId}]`);
+    // Overlay the xfade result on the concat video
+    ffmpegContext.filterComplex.push(`[${prevVideoId}][${xfadeTimedId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
+    return outputVideoId;
+};
+const addTransitionEffect = (transitionContext) => {
+    const { transitionType } = transitionContext;
+    if (transitionType === "fade") {
+        return addFadeTransition(transitionContext);
+    }
+    if (transitionType.startsWith("slideout_")) {
+        return addSlideoutTransition(transitionContext);
+    }
+    if (transitionType.startsWith("slidein_")) {
+        return addSlideinTransition(transitionContext);
     }
+    if (transitionType.startsWith("wipe")) {
+        return addWipeTransition(transitionContext);
+    }
+    throw new Error(`Unknown transition type: ${transitionType}`);
+};
+const addTransitionEffects = (ffmpegContext, captionedVideoId, context, transitionVideoIds, beatTimestamps, videoIdsForBeats) => {
     return transitionVideoIds.reduce((prevVideoId, { videoId: transitionVideoId, nextVideoId, beatIndex }) => {
         const beat = context.studio.script.beats[beatIndex];
         const transition = MulmoPresentationStyleMethods.getMovieTransition(context, beat);
         if (!transition) {
             return prevVideoId; // Skip if no transition is defined
         }
-        // Transition happens at the start of this beat
-        const startAt = beatTimestamps[beatIndex] - 0.05; // 0.05 is to avoid flickering
         // Limit transition duration to be no longer than either beat's duration
-        const prevBeatDuration = context.studio.beats[beatIndex - 1].duration ?? 1;
-        const currentBeatDuration = context.studio.beats[beatIndex].duration ?? 1;
-        const duration = getClampedTransitionDuration(transition.duration, prevBeatDuration, currentBeatDuration);
-        const outputVideoId = `trans_${beatIndex}_o`;
-        const processedVideoId = `${transitionVideoId}_f`;
-        if (transition.type === "fade") {
-            // Fade out the previous beat's last frame
-            ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,fade=t=out:d=${duration}:alpha=1,setpts=PTS-STARTPTS+${startAt}/TB[${processedVideoId}]`);
-            ffmpegContext.filterComplex.push(`[${prevVideoId}][${processedVideoId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
-        }
-        else if (transition.type.startsWith("slideout_")) {
-            // Slideout: previous beat's last frame slides out
-            ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${processedVideoId}]`);
-            ffmpegContext.filterComplex.push(`[${prevVideoId}][${processedVideoId}]overlay=${getOutOverlayCoords(transition.type, duration, startAt)}:enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
-        }
-        else if (transition.type.startsWith("slidein_")) {
-            // Slidein: this beat's first frame slides in over the previous beat's last frame
-            if (!nextVideoId) {
-                // Cannot apply slidein without first frame
-                return prevVideoId;
-            }
-            // Get previous beat's last frame for background
-            const prevVideoSourceId = videoIdsForBeats[beatIndex - 1];
-            // Both movie and image beats now have _last
-            const prevLastFrame = `${prevVideoSourceId}_last`;
-            // Prepare background (last frame of previous beat)
-            const backgroundVideoId = `${prevLastFrame}_bg`;
-            ffmpegContext.filterComplex.push(`[${prevLastFrame}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${backgroundVideoId}]`);
-            // Prepare sliding frame (first frame of this beat)
-            const slideinFrameId = `${nextVideoId}_f`;
-            ffmpegContext.filterComplex.push(`[${nextVideoId}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${slideinFrameId}]`);
-            // First overlay: put background on top of concat video
-            const bgOutputId = `${prevLastFrame}_bg_o`;
-            ffmpegContext.filterComplex.push(`[${prevVideoId}][${backgroundVideoId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${bgOutputId}]`);
-            // Second overlay: slide in the new frame on top of background
-            ffmpegContext.filterComplex.push(`[${bgOutputId}][${slideinFrameId}]overlay=${getInOverlayCoords(transition.type, duration, startAt)}:enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
-        }
-        else if (transition.type.startsWith("wipe")) {
-            // Wipe transition: use xfade filter between previous beat's last frame and this beat's first frame
-            if (!nextVideoId) {
-                // Cannot apply wipe without first frame
-                return prevVideoId;
-            }
-            // For wipe transitions, use offset=0 so transition starts immediately and completes within duration
-            // This ensures 0-100% wipe happens exactly during the transition duration
-            const xfadeOffset = 0;
-            // Apply xfade with offset=0 for complete 0-100% transition
-            // Both input videos should be at least duration seconds long (ensured by static frame generation)
-            const xfadeOutputId = `${transitionVideoId}_xfade`;
-            ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuv420p[${transitionVideoId}_fmt]`);
-            ffmpegContext.filterComplex.push(`[${nextVideoId}]format=yuv420p[${nextVideoId}_fmt]`);
-            ffmpegContext.filterComplex.push(`[${transitionVideoId}_fmt][${nextVideoId}_fmt]xfade=transition=${transition.type}:duration=${duration}:offset=${xfadeOffset}[${xfadeOutputId}]`);
-            // Set PTS for overlay timing
-            const xfadeTimedId = `${xfadeOutputId}_t`;
-            ffmpegContext.filterComplex.push(`[${xfadeOutputId}]setpts=PTS-STARTPTS+${startAt}/TB[${xfadeTimedId}]`);
-            // Overlay the xfade result on the concat video
-            ffmpegContext.filterComplex.push(`[${prevVideoId}][${xfadeTimedId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
-        }
-        else {
-            throw new Error(`Unknown transition type: ${transition.type}`);
-        }
-        return outputVideoId;
+        const prevBeatIndex = MulmoBeatMethods.getPrevRenderedBeatIndex(context.studio.script.beats, beatIndex);
+        const duration = getClampedTransitionDuration(transition.duration, MulmoStudioContextMethods.getSegmentDuration(context, prevBeatIndex), MulmoStudioContextMethods.getSegmentDuration(context, beatIndex));
+        // The last frame of the previous rendered beat (voice_over beats have no video of their own).
+        // Both movie and image beats now have _last.
+        const prevVideoSourceId = getPrevVideoId(videoIdsForBeats, beatIndex);
+        return (addTransitionEffect({
+            ffmpegContext,
+            transitionType: transition.type,
+            prevVideoId,
+            transitionVideoId,
+            nextVideoId,
+            prevLastFrame: prevVideoSourceId ? `${prevVideoSourceId}_last` : undefined,
+            outputVideoId: `trans_${beatIndex}_o`,
+            // Transition happens at the start of this beat. 0.05 is to avoid flickering.
+            startAt: beatTimestamps[beatIndex] - 0.05,
+            duration,
+        }) ?? prevVideoId);
     }, captionedVideoId);
 };
-export const getNeedFirstFrame = (context) => {
-    return context.studio.script.beats.map((beat, index) => {
-        if (index === 0)
-            return false; // First beat cannot have transition
-        const transition = MulmoPresentationStyleMethods.getMovieTransition(context, beat);
-        return (transition?.type.startsWith("slidein_") || transition?.type.startsWith("wipe")) ?? false;
-    });
-};
-export const getNeedLastFrame = (context) => {
-    return context.studio.script.beats.map((beat, index) => {
-        if (index === context.studio.script.beats.length - 1)
-            return false; // Last beat doesn't need _last
-        const nextTransition = MulmoPresentationStyleMethods.getMovieTransition(context, context.studio.script.beats[index + 1]);
-        return nextTransition !== null; // Any transition on next beat requires this beat's last frame
-    });
-};
-export const resolveMovieVolume = (beat, context) => {
-    const baseMovieVolume = beat.audioParams?.movieVolume ?? context.presentationStyle.audioParams.movieVolume ?? 1.0;
-    const ducking = context.presentationStyle.audioParams.ducking;
-    const hasSpeech = !!beat.text && !context.presentationStyle.audioParams.suppressSpeech;
-    if (ducking && hasSpeech) {
-        const ratio = ducking.ratio ?? DEFAULT_DUCKING_RATIO;
-        return baseMovieVolume * ratio;
-    }
-    return baseMovieVolume;
-};
 export const isExplicitMixMode = (context) => {
     const audioParams = context.presentationStyle.audioParams;
     const duckingRequested = audioParams.ducking !== undefined;
@@ -285,40 +286,27 @@ export const mixAudiosFromMovieBeats = (ffmpegContext, artifactAudioId, audioIds
     }
     return artifactAudioId;
 };
-export const getExtraPadding = (context, index) => {
-    // We need to consider only intro and outro padding because the other paddings were already added to the beat.duration
-    if (index === 0) {
-        return MulmoStudioContextMethods.getIntroPadding(context);
-    }
-    else if (index === context.studio.beats.length - 1) {
-        return context.presentationStyle.audioParams.outroPadding;
-    }
-    return 0;
-};
-export const getFillOption = (context, beat) => {
-    // Get fillOption from merged imageParams (global + beat-specific)
-    const globalFillOption = context.presentationStyle.movieParams?.fillOption;
-    const beatFillOption = beat.movieParams?.fillOption;
-    const defaultFillOption = mulmoFillOptionSchema.parse({}); // let the schema infer the default value
-    return { ...defaultFillOption, ...globalFillOption, ...beatFillOption };
-};
 export const getTransitionVideoId = (transition, videoIdsForBeats, index) => {
+    // Every transition needs the previous *rendered* beat (voice_over beats have no video of their own).
+    const prevVideoSourceId = getPrevVideoId(videoIdsForBeats, index);
+    const currentVideoId = videoIdsForBeats[index];
+    if (!prevVideoSourceId || !currentVideoId) {
+        return null; // Skip the transition instead of referencing a filtergraph pad which does not exist.
+    }
     if (transition.type === "fade" || transition.type.startsWith("slideout_")) {
-        // Use previous beat's last frame. TODO: support voice-over
-        const prevVideoSourceId = videoIdsForBeats[index - 1];
+        // Use previous beat's last frame.
         // Both movie and image beats now have _last
         const frameId = `${prevVideoSourceId}_last`;
         return { videoId: frameId, nextVideoId: undefined, beatIndex: index };
     }
     if (transition.type.startsWith("wipe")) {
         // Wipe needs both previous beat's last frame and this beat's first frame
-        const prevVideoSourceId = videoIdsForBeats[index - 1];
         const prevLastFrame = `${prevVideoSourceId}_last`;
-        const nextFirstFrame = `${videoIdsForBeats[index]}_first`;
+        const nextFirstFrame = `${currentVideoId}_first`;
         return { videoId: prevLastFrame, nextVideoId: nextFirstFrame, beatIndex: index };
     }
     // Use this beat's first frame. slidein_ case
-    return { videoId: "", nextVideoId: `${videoIdsForBeats[index]}_first`, beatIndex: index };
+    return { videoId: "", nextVideoId: `${currentVideoId}_first`, beatIndex: index };
 };
 export const getConcatVideoFilter = (concatVideoId, videoIdsForBeats) => {
     const videoIds = videoIdsForBeats.filter((id) => id !== undefined); // filter out voice-over beats
@@ -336,14 +324,17 @@ export const getTransitionFrameDurations = (context, index) => {
     const getTransitionDuration = (transition, prevBeatIndex, currentBeatIndex) => {
         if (!transition || prevBeatIndex < 0 || currentBeatIndex >= beats.length)
             return 0;
-        const prevBeatDuration = beats[prevBeatIndex].duration ?? 1;
-        const currentBeatDuration = beats[currentBeatIndex].duration ?? 1;
+        const prevBeatDuration = MulmoStudioContextMethods.getSegmentDuration(context, prevBeatIndex);
+        const currentBeatDuration = MulmoStudioContextMethods.getSegmentDuration(context, currentBeatIndex);
         return getClampedTransitionDuration(transition.duration, prevBeatDuration, currentBeatDuration);
     };
+    // Transitions are resolved against the neighboring *rendered* beats (voice_over beats have no video).
+    const prevIndex = MulmoBeatMethods.getPrevRenderedBeatIndex(scriptBeats, index);
+    const nextIndex = MulmoBeatMethods.getNextRenderedBeatIndex(scriptBeats, index);
     const currentTransition = MulmoPresentationStyleMethods.getMovieTransition(context, scriptBeats[index]);
-    const firstDuration = index > 0 ? getTransitionDuration(currentTransition, index - 1, index) : 0;
-    const nextTransition = index < scriptBeats.length - 1 ? MulmoPresentationStyleMethods.getMovieTransition(context, scriptBeats[index + 1]) : null;
-    const lastDuration = getTransitionDuration(nextTransition, index, index + 1);
+    const firstDuration = prevIndex >= 0 ? getTransitionDuration(currentTransition, prevIndex, index) : 0;
+    const nextTransition = nextIndex >= 0 ? MulmoPresentationStyleMethods.getMovieTransition(context, scriptBeats[nextIndex]) : null;
+    const lastDuration = nextIndex >= 0 ? getTransitionDuration(nextTransition, index, nextIndex) : 0;
     return {
         firstDuration: Math.max(firstDuration, minFrame),
         lastDuration: Math.max(lastDuration, minFrame),
@@ -388,12 +379,26 @@ export const addSplitAndExtractFrames = (ffmpegContext, videoId, firstDuration,
 const findMissingIndex = (context) => {
     return context.studio.beats.findIndex((studioBeat, index) => {
         const beat = context.studio.script.beats[index];
-        if (beat.image?.type === "voice_over") {
+        if (MulmoBeatMethods.isVoiceOver(beat)) {
             return false; // Voice-over does not have either imageFile or movieFile.
         }
         return !studioBeat.imageFile && !studioBeat.movieFile;
     });
 };
+// Extract the stills the transitions around this beat consume, and return the transition this beat
+// starts (null when it has none, or when there is no previous shot to transition from).
+const addTransitionAssets = (ffmpegContext, context, index, videoIdsForBeats, { isMovie, canvasInfo, needFirst, needLast }) => {
+    const videoId = videoIdsForBeats[index];
+    if (needFirst || needLast) {
+        const { firstDuration, lastDuration } = getTransitionFrameDurations(context, index);
+        addSplitAndExtractFrames(ffmpegContext, videoId, firstDuration, lastDuration, isMovie, needFirst, needLast, canvasInfo);
+    }
+    const transition = MulmoPresentationStyleMethods.getMovieTransition(context, context.studio.script.beats[index]);
+    if (!transition || index === 0) {
+        return null;
+    }
+    return getTransitionVideoId(transition, videoIdsForBeats, index);
+};
 export const createVideo = async (audioArtifactFilePath, outputVideoPath, context, isTest = false) => {
     const caption = MulmoStudioContextMethods.getCaption(context);
     const start = performance.now();
@@ -410,20 +415,22 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
     const transitionVideoIds = [];
     const beatTimestamps = [];
     // Check which beats need _first (for slidein transition on this beat)
-    const needsFirstFrame = getNeedFirstFrame(context);
+    const needsFirstFrame = MulmoPresentationStyleMethods.getNeedFirstFrame(context);
     // Check which beats need _last (for any transition on next beat - they all need previous beat's last frame)
-    const needsLastFrame = getNeedLastFrame(context);
+    const needsLastFrame = MulmoPresentationStyleMethods.getNeedLastFrame(context);
     let cumulativeFrames = 0;
     context.studio.beats.reduce((timestamp, studioBeat, index) => {
         const beat = context.studio.script.beats[index];
-        if (beat.image?.type === "voice_over") {
+        if (MulmoBeatMethods.isVoiceOver(beat)) {
             videoIdsForBeats.push(undefined);
             beatTimestamps.push(timestamp);
-            return timestamp; // Skip voice-over beats.
+            // Skip voice-over beats: their video is covered by the owner beat's segment, but the
+            // audio timeline still advances, so keep the timestamp in sync with the audio artifact.
+            return timestamp + MulmoStudioContextMethods.getBeatDuration(context, index);
         }
         const sourceFile = isTest ? "/test/dummy.mp4" : validateBeatSource(studioBeat, index);
-        // The movie duration is bigger in case of voice-over.
-        const duration = Math.max(studioBeat.duration + getExtraPadding(context, index), studioBeat.movieDuration ?? 0);
+        const voiceOverDuration = MulmoStudioContextMethods.getVoiceOverGroupDuration(context, index);
+        const duration = MulmoStudioContextMethods.getSegmentDuration(context, index);
         // Use cumulative frame tracking to prevent audio-video drift from frame quantization.
         // trim=duration=X rounds up to the next frame boundary (~0.03s per beat at 30fps),
         // causing cumulative drift. Instead, compute exact frame counts per beat.
@@ -431,29 +438,23 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
         const frameCount = targetEndFrame - cumulativeFrames;
         cumulativeFrames = targetEndFrame;
         const inputIndex = FfmpegContextAddInput(ffmpegContext, sourceFile);
-        const isMovie = !!(studioBeat.lipSyncFile ||
-            studioBeat.movieFile ||
-            MulmoPresentationStyleMethods.getImageType(context.presentationStyle, beat) === "movie");
+        const isMovie = MulmoStudioContextMethods.isMovieBeat(context, studioBeat, beat);
         const speed = beat.movieParams?.speed ?? 1.0;
         const filters = beat.movieParams?.filters;
-        const { videoId, videoPart } = getVideoPart(inputIndex, isMovie, duration, canvasInfo, getFillOption(context, beat), speed, filters, frameCount);
+        const { videoId, videoPart } = getVideoPart(inputIndex, isMovie, duration, canvasInfo, MulmoPresentationStyleMethods.getFillOption(context, beat), speed, filters, frameCount);
         ffmpegContext.filterComplex.push(videoPart);
-        // for transition
-        const needFirst = needsFirstFrame[index]; // This beat has slidein
-        const needLast = needsLastFrame[index]; // Next beat has transition
         videoIdsForBeats.push(videoId);
-        if (needFirst || needLast) {
-            const { firstDuration, lastDuration } = getTransitionFrameDurations(context, index);
-            addSplitAndExtractFrames(ffmpegContext, videoId, firstDuration, lastDuration, isMovie, needFirst, needLast, canvasInfo);
-        }
-        // Record transition info if this beat has a transition
-        const transition = MulmoPresentationStyleMethods.getMovieTransition(context, beat);
-        if (transition && index > 0) {
-            const transitionVideoId = getTransitionVideoId(transition, videoIdsForBeats, index);
+        const transitionVideoId = addTransitionAssets(ffmpegContext, context, index, videoIdsForBeats, {
+            isMovie,
+            canvasInfo,
+            needFirst: needsFirstFrame[index], // This beat has slidein
+            needLast: needsLastFrame[index], // Next beat has transition
+        });
+        if (transitionVideoId) {
             transitionVideoIds.push(transitionVideoId);
         }
         // NOTE: We don't support audio if the speed is not 1.0.
-        const movieVolume = resolveMovieVolume(beat, context);
+        const movieVolume = MulmoPresentationStyleMethods.getMovieVolume(context, beat);
         if (studioBeat.hasMovieAudio && movieVolume > 0.0 && speed === 1.0) {
             // TODO: Handle a special case where it has lipSyncFile AND hasMovieAudio is on (the source file has an audio, such as sound effect).
             const { audioId, audioPart } = getAudioPart(inputIndex, duration, timestamp, movieVolume);
@@ -461,7 +462,8 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
             ffmpegContext.filterComplex.push(audioPart);
         }
         beatTimestamps.push(timestamp);
-        return timestamp + duration;
+        // The trailing voice_over beats advance the timestamp themselves.
+        return timestamp + duration - voiceOverDuration;
     }, 0);
     assert(videoIdsForBeats.length === context.studio.beats.length, "videoIds.length !== studio.beats.length");
     assert(beatTimestamps.length === context.studio.beats.length, "beatTimestamps.length !== studio.beats.length");

package/lib/methods/index.d.ts

@@ -1,5 +1,6 @@
 export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
+export * from "./mulmo_image_prompt_media.js";
 export * from "./mulmo_beat_node.js";
 export * from "./mulmo_script.js";

package/lib/methods/index.js

@@ -1,6 +1,7 @@
 export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
+export * from "./mulmo_image_prompt_media.js";
 // Node-only MulmoBeatMethods (includes getPlugin). The browser-safe subset lives in
 // ./mulmo_beat.js and is re-exported by ../index.common.js for mulmocast/browser.
 export * from "./mulmo_beat_node.js";

package/lib/methods/mulmo_beat.d.ts

@@ -8,6 +8,9 @@ export declare const MulmoBeatMethods: {
     isAnimationObject: (animation: unknown) => animation is AnimationConfig;
     isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
     isMovieMode: (animation: unknown) => boolean;
+    isVoiceOver: (beat?: MulmoBeat) => boolean;
+    getPrevRenderedBeatIndex(beats: MulmoBeat[], index: number): number;
+    getNextRenderedBeatIndex(beats: MulmoBeat[], index: number): number;
     getHtmlPrompt(beat: MulmoBeat): string | undefined;
     getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
 };

package/lib/methods/mulmo_beat.js

@@ -1,3 +1,4 @@
+import { ImageMediaType } from "../types/index.js";
 /** Type guard: checks if animation value is an object config like { fps: 30 } */
 const isAnimationObject = (animation) => {
     return typeof animation === "object" && animation !== null && !Array.isArray(animation);
@@ -17,11 +18,27 @@ const isAnimatedHtmlTailwind = (beat) => {
     const animation = beat.image.animation;
     return isAnimationEnabled(animation);
 };
+// voice_over beats share the preceding beat's shot, and contribute no video segment of their own.
+const isVoiceOver = (beat) => beat?.image?.type === ImageMediaType.VoiceOver;
 export const MulmoBeatMethods = {
     isAnimationEnabled,
     isAnimationObject,
     isAnimatedHtmlTailwind,
     isMovieMode,
+    isVoiceOver,
+    // Index of the closest preceding beat which is actually rendered as a video segment (-1 if none).
+    getPrevRenderedBeatIndex(beats, index) {
+        const offset = beats
+            .slice(0, Math.max(index, 0))
+            .reverse()
+            .findIndex((beat) => !isVoiceOver(beat));
+        return offset < 0 ? -1 : index - 1 - offset;
+    },
+    // Index of the closest following beat which is actually rendered as a video segment (-1 if none).
+    getNextRenderedBeatIndex(beats, index) {
+        const offset = beats.slice(index + 1).findIndex((beat) => !isVoiceOver(beat));
+        return offset < 0 ? -1 : index + 1 + offset;
+    },
     getHtmlPrompt(beat) {
         if (beat?.htmlPrompt?.data) {
             return beat.htmlPrompt.prompt + "\n\n[data]\n" + JSON.stringify(beat.htmlPrompt.data, null, 2);

package/lib/methods/mulmo_beat_node.d.ts

@@ -17,6 +17,9 @@ export declare const MulmoBeatMethods: {
     };
     isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
     isMovieMode: (animation: unknown) => boolean;
+    isVoiceOver: (beat?: MulmoBeat) => boolean;
+    getPrevRenderedBeatIndex(beats: MulmoBeat[], index: number): number;
+    getNextRenderedBeatIndex(beats: MulmoBeat[], index: number): number;
     getHtmlPrompt(beat: MulmoBeat): string | undefined;
     getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
 };

package/lib/methods/mulmo_image_prompt_media.d.ts

@@ -0,0 +1,14 @@
+import { MulmoImagePromptMedia, MulmoImageReference, Text2ImageAgentInfo } from "../types/index.js";
+export type MulmoNormalizedImageReference = MulmoImageReference & {
+    legacy?: boolean;
+};
+export type ResolvedImageReference = {
+    path: string;
+    label?: string;
+};
+export declare const MulmoImagePromptMediaMethods: {
+    getReferences(image: MulmoImagePromptMedia): MulmoNormalizedImageReference[];
+    hasNamedReference(image: MulmoImagePromptMedia): boolean;
+    buildReferencePreamble(labels: (string | undefined)[]): string;
+    limitReferences(references: ResolvedImageReference[] | undefined, imageAgentInfo: Text2ImageAgentInfo, imageKey: string): ResolvedImageReference[] | undefined;
+};

package/lib/methods/mulmo_image_prompt_media.js

@@ -0,0 +1,45 @@
+import { GraphAILogger } from "graphai";
+import { getMaxImageReferenceImages } from "../types/provider2agent.js";
+export const MulmoImagePromptMediaMethods = {
+    // Normalized reference list: legacy single-ref fields first, then the references array.
+    // When both legacy fields are set, referenceImageName wins and referenceImage is ignored,
+    // matching the pre-references resolution behavior.
+    getReferences(image) {
+        const references = [];
+        if (image.referenceImageName) {
+            references.push({ name: image.referenceImageName, legacy: true });
+        }
+        else if (image.referenceImage) {
+            references.push({ source: image.referenceImage, legacy: true });
+        }
+        references.push(...(image.references ?? []));
+        return references;
+    },
+    // Entries with a named reference must resolve after stage 1 (which generates/loads the named images).
+    hasNamedReference(image) {
+        return this.getReferences(image).some((ref) => ref.name !== undefined);
+    },
+    // "Reference image 1 shows <label>. ... Use these exact designs.\n"
+    // Empty when no reference has a label, keeping label-free prompts byte-identical to before.
+    buildReferencePreamble(labels) {
+        const sentences = labels.map((label, index) => (label ? `Reference image ${index + 1} shows ${label}.` : undefined)).filter((s) => s !== undefined);
+        if (sentences.length === 0) {
+            return "";
+        }
+        return `${sentences.join(" ")} Use these exact designs.\n`;
+    },
+    // Truncate to the provider/model limit, keeping the first N deterministically.
+    // imageKey is the imageRefs key of the image being generated (used for the warning).
+    limitReferences(references, imageAgentInfo, imageKey) {
+        if (!references) {
+            return undefined;
+        }
+        const { provider, model } = imageAgentInfo.imageParams;
+        const max = getMaxImageReferenceImages(provider, model ?? "");
+        if (max !== undefined && references.length > max) {
+            GraphAILogger.warn(`imagePrompt "${imageKey}": ${references.length} reference images exceed the ${provider} limit of ${max} — keeping the first ${max}`);
+            return references.slice(0, max);
+        }
+        return references;
+    },
+};

package/lib/methods/mulmo_presentation_style.d.ts

@@ -4,7 +4,7 @@
  * Works in both Node.js and modern browsers.
  */
 import type { SlideTheme } from "@mulmocast/deck";
-import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider, MulmoStudioContext, MulmoTransition } from "../types/index.js";
+import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider, MulmoStudioContext, MulmoTransition, MulmoFillOption } from "../types/index.js";
 export declare const MulmoPresentationStyleMethods: {
     getCanvasSize(presentationStyle: MulmoPresentationStyle): MulmoCanvasDimension;
     getAllSpeechProviders(presentationStyle: MulmoPresentationStyle): Set<Text2SpeechProvider>;
@@ -30,6 +30,10 @@ export declare const MulmoPresentationStyleMethods: {
     getSpeakerData(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat, lang: string | undefined): SpeakerData;
     getSpeaker(context: MulmoStudioContext, beat: MulmoBeat, targetLang: string | undefined): SpeakerData;
     getMovieTransition(context: MulmoStudioContext, beat: MulmoBeat): MulmoTransition | null;
+    getNeedFirstFrame(context: MulmoStudioContext): boolean[];
+    getNeedLastFrame(context: MulmoStudioContext): boolean[];
+    getFillOption(context: MulmoStudioContext, beat: MulmoBeat): MulmoFillOption;
+    getMovieVolume(context: MulmoStudioContext, beat: MulmoBeat): number;
     getText2ImageProvider(provider: Text2ImageProvider | undefined): Text2ImageProvider;
     getImageAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): Text2ImageAgentInfo;
     getMovieAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): {

package/lib/methods/mulmo_presentation_style.js

@@ -6,7 +6,8 @@
 import { isNull } from "graphai";
 import { userAssert } from "../utils/utils.js";
 import { slideThemes } from "../data/slideThemes.js";
-import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, mulmoTransitionSchema, } from "../types/schema.js";
+import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, mulmoTransitionSchema, mulmoFillOptionSchema, } from "../types/schema.js";
+import { MulmoBeatMethods } from "./mulmo_beat.js";
 import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, getModelAudio, AUDIO_MODE_ALWAYS, } from "../types/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
@@ -22,6 +23,13 @@ const defaultTextSlideStyles = [
     "td, th { padding: 8px }",
     "tr:nth-child(even) { background-color: #eee }",
 ];
+const DEFAULT_DUCKING_RATIO = 0.3;
+// Both frame requirements answer the same question for each *rendered* beat (voice_over beats have
+// no video segment, so they never supply a frame): does a transition need a still from this beat?
+const mapRenderedBeats = (context, needsFrame) => {
+    const beats = context.studio.script.beats;
+    return beats.map((beat, index) => (MulmoBeatMethods.isVoiceOver(beat) ? false : needsFrame(beats, index)));
+};
 export const MulmoPresentationStyleMethods = {
     getCanvasSize(presentationStyle) {
         return mulmoCanvasDimensionSchema.parse(presentationStyle.canvasSize);
@@ -95,6 +103,40 @@ export const MulmoPresentationStyleMethods = {
             return null;
         return mulmoTransitionSchema.parse(transitionData);
     },
+    getNeedFirstFrame(context) {
+        // This beat slides/wipes in over the previous shot, so it supplies its own first frame.
+        return mapRenderedBeats(context, (beats, index) => {
+            if (MulmoBeatMethods.getPrevRenderedBeatIndex(beats, index) < 0)
+                return false; // No previous shot: the transition is skipped
+            const transition = MulmoPresentationStyleMethods.getMovieTransition(context, beats[index]);
+            return (transition?.type.startsWith("slidein_") || transition?.type.startsWith("wipe")) ?? false;
+        });
+    },
+    getNeedLastFrame(context) {
+        // Any transition consumes the last frame of the previous rendered beat, skipping voice_over beats.
+        return mapRenderedBeats(context, (beats, index) => {
+            const nextIndex = MulmoBeatMethods.getNextRenderedBeatIndex(beats, index);
+            if (nextIndex < 0)
+                return false; // Last rendered beat doesn't need _last
+            return MulmoPresentationStyleMethods.getMovieTransition(context, beats[nextIndex]) !== null;
+        });
+    },
+    getFillOption(context, beat) {
+        const globalFillOption = context.presentationStyle.movieParams?.fillOption;
+        const beatFillOption = beat.movieParams?.fillOption;
+        const defaultFillOption = mulmoFillOptionSchema.parse({}); // let the schema infer the default value
+        return { ...defaultFillOption, ...globalFillOption, ...beatFillOption };
+    },
+    getMovieVolume(context, beat) {
+        const baseMovieVolume = beat.audioParams?.movieVolume ?? context.presentationStyle.audioParams.movieVolume ?? 1.0;
+        const ducking = context.presentationStyle.audioParams.ducking;
+        const hasSpeech = !!beat.text && !context.presentationStyle.audioParams.suppressSpeech;
+        if (ducking && hasSpeech) {
+            const ratio = ducking.ratio ?? DEFAULT_DUCKING_RATIO;
+            return baseMovieVolume * ratio;
+        }
+        return baseMovieVolume;
+    },
     /* NOTE: This method is not used.
     getTTSModel(context: MulmoStudioContext, beat: MulmoBeat): string | undefined {
       const speaker = MulmoPresentationStyleMethods.getSpeaker(context, beat);