mulmocast 2.9.0 → 2.9.1

package/lib/actions/movie.d.ts

@@ -1,4 +1,4 @@
-import { MulmoStudioContext, MulmoBeat, MulmoTransition, MulmoCanvasDimension, MulmoFillOption, MulmoVideoFilter } from "../types/index.js";
+import { MulmoStudioContext, MulmoTransition, MulmoCanvasDimension, MulmoFillOption, MulmoVideoFilter } from "../types/index.js";
 import { FfmpegContext } from "../utils/ffmpeg_utils.js";
 type VideoId = string | undefined;
 export declare const getVideoPart: (inputIndex: number, isMovie: boolean, duration: number, canvasInfo: MulmoCanvasDimension, fillOption: MulmoFillOption, speed: number, filters?: MulmoVideoFilter[], frameCount?: number) => {
@@ -11,15 +11,8 @@ export declare const getAudioPart: (inputIndex: number, duration: number, delay:
 };
 export declare const getOutOverlayCoords: (transitionType: string, d: number, t: number) => string;
 export declare const getInOverlayCoords: (transitionType: string, d: number, t: number) => string;
-export declare const getNeedFirstFrame: (context: MulmoStudioContext) => boolean[];
-export declare const getNeedLastFrame: (context: MulmoStudioContext) => boolean[];
-export declare const resolveMovieVolume: (beat: MulmoBeat, context: MulmoStudioContext) => number;
 export declare const isExplicitMixMode: (context: MulmoStudioContext) => boolean;
 export declare const mixAudiosFromMovieBeats: (ffmpegContext: FfmpegContext, artifactAudioId: string, audioIdsFromMovieBeats: string[], context: MulmoStudioContext) => string;
-export declare const getExtraPadding: (context: MulmoStudioContext, index: number) => number;
-export declare const getFillOption: (context: MulmoStudioContext, beat: MulmoBeat) => {
-    style: "aspectFit" | "aspectFill";
-};
 export declare const getTransitionVideoId: (transition: MulmoTransition, videoIdsForBeats: VideoId[], index: number) => {
     videoId: string;
     nextVideoId: undefined;
@@ -28,7 +21,7 @@ export declare const getTransitionVideoId: (transition: MulmoTransition, videoId
     videoId: string;
     nextVideoId: string;
     beatIndex: number;
-};
+} | null;
 export declare const getConcatVideoFilter: (concatVideoId: string, videoIdsForBeats: VideoId[]) => string;
 export declare const getTransitionFrameDurations: (context: MulmoStudioContext, index: number) => {
     firstDuration: number;

package/lib/actions/movie.js

@@ -1,6 +1,5 @@
 import { GraphAILogger, assert } from "graphai";
-import { mulmoFillOptionSchema, } from "../types/index.js";
-import { MulmoPresentationStyleMethods } from "../methods/index.js";
+import { MulmoPresentationStyleMethods, MulmoBeatMethods } from "../methods/index.js";
 import { getAudioArtifactFilePath, getOutputVideoFilePath, writingMessage, isFile } from "../utils/file.js";
 import { createVideoFileError, createVideoSourceError } from "../utils/error_cause.js";
 import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextPushFormattedAudio, FfmpegContextGenerateOutput, } from "../utils/ffmpeg_utils.js";
@@ -9,7 +8,14 @@ import { convertVideoFilterToFFmpeg } from "../utils/video_filter.js";
 // const isMac = process.platform === "darwin";
 const videoCodec = "libx264"; // "h264_videotoolbox" (macOS only) is too noisy
 const VIDEO_FPS = 30;
-const DEFAULT_DUCKING_RATIO = 0.3;
+// videoIdsForBeats holds undefined for voice_over beats; walk back to the previous rendered beat's videoId.
+const getPrevVideoId = (videoIdsForBeats, index) => {
+    const offset = videoIdsForBeats
+        .slice(0, Math.max(index, 0))
+        .reverse()
+        .findIndex((videoId) => videoId !== undefined);
+    return offset < 0 ? undefined : videoIdsForBeats[index - 1 - offset];
+};
 export const getVideoPart = (inputIndex, isMovie, duration, canvasInfo, fillOption, speed, filters, frameCount) => {
     const videoId = `v${inputIndex}`;
     const videoFilters = [];
@@ -149,109 +155,104 @@ export const getInOverlayCoords = (transitionType, d, t) => {
     }
     throw new Error(`Unknown transition type: ${transitionType}`);
 };
-const addTransitionEffects = (ffmpegContext, captionedVideoId, context, transitionVideoIds, beatTimestamps, videoIdsForBeats) => {
-    if (transitionVideoIds.length === 0) {
-        return captionedVideoId;
+// Each of these returns the id of the video to keep building on, or undefined when the transition
+// cannot be applied (in which case the caller keeps the untouched video).
+const addFadeTransition = ({ ffmpegContext, prevVideoId, transitionVideoId, outputVideoId, startAt, duration }) => {
+    // Fade out the previous beat's last frame
+    const processedVideoId = `${transitionVideoId}_f`;
+    ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,fade=t=out:d=${duration}:alpha=1,setpts=PTS-STARTPTS+${startAt}/TB[${processedVideoId}]`);
+    ffmpegContext.filterComplex.push(`[${prevVideoId}][${processedVideoId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
+    return outputVideoId;
+};
+const addSlideoutTransition = ({ ffmpegContext, transitionType, prevVideoId, transitionVideoId, outputVideoId, startAt, duration }) => {
+    // Slideout: previous beat's last frame slides out
+    const processedVideoId = `${transitionVideoId}_f`;
+    ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${processedVideoId}]`);
+    ffmpegContext.filterComplex.push(`[${prevVideoId}][${processedVideoId}]overlay=${getOutOverlayCoords(transitionType, duration, startAt)}:enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
+    return outputVideoId;
+};
+const addSlideinTransition = ({ ffmpegContext, transitionType, prevVideoId, nextVideoId, prevLastFrame, outputVideoId, startAt, duration, }) => {
+    // Slidein: this beat's first frame slides in over the previous beat's last frame
+    if (!nextVideoId || !prevLastFrame) {
+        return undefined; // Cannot apply slidein without the first frame or the previous shot
+    }
+    // Prepare background (last frame of previous beat)
+    const backgroundVideoId = `${prevLastFrame}_bg`;
+    ffmpegContext.filterComplex.push(`[${prevLastFrame}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${backgroundVideoId}]`);
+    // Prepare sliding frame (first frame of this beat)
+    const slideinFrameId = `${nextVideoId}_f`;
+    ffmpegContext.filterComplex.push(`[${nextVideoId}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${slideinFrameId}]`);
+    // First overlay: put background on top of concat video
+    const bgOutputId = `${prevLastFrame}_bg_o`;
+    ffmpegContext.filterComplex.push(`[${prevVideoId}][${backgroundVideoId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${bgOutputId}]`);
+    // Second overlay: slide in the new frame on top of background
+    ffmpegContext.filterComplex.push(`[${bgOutputId}][${slideinFrameId}]overlay=${getInOverlayCoords(transitionType, duration, startAt)}:enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
+    return outputVideoId;
+};
+const addWipeTransition = ({ ffmpegContext, transitionType, prevVideoId, transitionVideoId, nextVideoId, outputVideoId, startAt, duration, }) => {
+    // Wipe transition: use xfade filter between previous beat's last frame and this beat's first frame
+    if (!nextVideoId) {
+        return undefined; // Cannot apply wipe without first frame
+    }
+    // For wipe transitions, use offset=0 so transition starts immediately and completes within duration
+    // This ensures 0-100% wipe happens exactly during the transition duration
+    const xfadeOffset = 0;
+    // Apply xfade with offset=0 for complete 0-100% transition
+    // Both input videos should be at least duration seconds long (ensured by static frame generation)
+    const xfadeOutputId = `${transitionVideoId}_xfade`;
+    ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuv420p[${transitionVideoId}_fmt]`);
+    ffmpegContext.filterComplex.push(`[${nextVideoId}]format=yuv420p[${nextVideoId}_fmt]`);
+    ffmpegContext.filterComplex.push(`[${transitionVideoId}_fmt][${nextVideoId}_fmt]xfade=transition=${transitionType}:duration=${duration}:offset=${xfadeOffset}[${xfadeOutputId}]`);
+    // Set PTS for overlay timing
+    const xfadeTimedId = `${xfadeOutputId}_t`;
+    ffmpegContext.filterComplex.push(`[${xfadeOutputId}]setpts=PTS-STARTPTS+${startAt}/TB[${xfadeTimedId}]`);
+    // Overlay the xfade result on the concat video
+    ffmpegContext.filterComplex.push(`[${prevVideoId}][${xfadeTimedId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
+    return outputVideoId;
+};
+const addTransitionEffect = (transitionContext) => {
+    const { transitionType } = transitionContext;
+    if (transitionType === "fade") {
+        return addFadeTransition(transitionContext);
+    }
+    if (transitionType.startsWith("slideout_")) {
+        return addSlideoutTransition(transitionContext);
+    }
+    if (transitionType.startsWith("slidein_")) {
+        return addSlideinTransition(transitionContext);
     }
+    if (transitionType.startsWith("wipe")) {
+        return addWipeTransition(transitionContext);
+    }
+    throw new Error(`Unknown transition type: ${transitionType}`);
+};
+const addTransitionEffects = (ffmpegContext, captionedVideoId, context, transitionVideoIds, beatTimestamps, videoIdsForBeats) => {
     return transitionVideoIds.reduce((prevVideoId, { videoId: transitionVideoId, nextVideoId, beatIndex }) => {
         const beat = context.studio.script.beats[beatIndex];
         const transition = MulmoPresentationStyleMethods.getMovieTransition(context, beat);
         if (!transition) {
             return prevVideoId; // Skip if no transition is defined
         }
-        // Transition happens at the start of this beat
-        const startAt = beatTimestamps[beatIndex] - 0.05; // 0.05 is to avoid flickering
         // Limit transition duration to be no longer than either beat's duration
-        const prevBeatDuration = context.studio.beats[beatIndex - 1].duration ?? 1;
-        const currentBeatDuration = context.studio.beats[beatIndex].duration ?? 1;
-        const duration = getClampedTransitionDuration(transition.duration, prevBeatDuration, currentBeatDuration);
-        const outputVideoId = `trans_${beatIndex}_o`;
-        const processedVideoId = `${transitionVideoId}_f`;
-        if (transition.type === "fade") {
-            // Fade out the previous beat's last frame
-            ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,fade=t=out:d=${duration}:alpha=1,setpts=PTS-STARTPTS+${startAt}/TB[${processedVideoId}]`);
-            ffmpegContext.filterComplex.push(`[${prevVideoId}][${processedVideoId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
-        }
-        else if (transition.type.startsWith("slideout_")) {
-            // Slideout: previous beat's last frame slides out
-            ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${processedVideoId}]`);
-            ffmpegContext.filterComplex.push(`[${prevVideoId}][${processedVideoId}]overlay=${getOutOverlayCoords(transition.type, duration, startAt)}:enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
-        }
-        else if (transition.type.startsWith("slidein_")) {
-            // Slidein: this beat's first frame slides in over the previous beat's last frame
-            if (!nextVideoId) {
-                // Cannot apply slidein without first frame
-                return prevVideoId;
-            }
-            // Get previous beat's last frame for background
-            const prevVideoSourceId = videoIdsForBeats[beatIndex - 1];
-            // Both movie and image beats now have _last
-            const prevLastFrame = `${prevVideoSourceId}_last`;
-            // Prepare background (last frame of previous beat)
-            const backgroundVideoId = `${prevLastFrame}_bg`;
-            ffmpegContext.filterComplex.push(`[${prevLastFrame}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${backgroundVideoId}]`);
-            // Prepare sliding frame (first frame of this beat)
-            const slideinFrameId = `${nextVideoId}_f`;
-            ffmpegContext.filterComplex.push(`[${nextVideoId}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${slideinFrameId}]`);
-            // First overlay: put background on top of concat video
-            const bgOutputId = `${prevLastFrame}_bg_o`;
-            ffmpegContext.filterComplex.push(`[${prevVideoId}][${backgroundVideoId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${bgOutputId}]`);
-            // Second overlay: slide in the new frame on top of background
-            ffmpegContext.filterComplex.push(`[${bgOutputId}][${slideinFrameId}]overlay=${getInOverlayCoords(transition.type, duration, startAt)}:enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
-        }
-        else if (transition.type.startsWith("wipe")) {
-            // Wipe transition: use xfade filter between previous beat's last frame and this beat's first frame
-            if (!nextVideoId) {
-                // Cannot apply wipe without first frame
-                return prevVideoId;
-            }
-            // For wipe transitions, use offset=0 so transition starts immediately and completes within duration
-            // This ensures 0-100% wipe happens exactly during the transition duration
-            const xfadeOffset = 0;
-            // Apply xfade with offset=0 for complete 0-100% transition
-            // Both input videos should be at least duration seconds long (ensured by static frame generation)
-            const xfadeOutputId = `${transitionVideoId}_xfade`;
-            ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuv420p[${transitionVideoId}_fmt]`);
-            ffmpegContext.filterComplex.push(`[${nextVideoId}]format=yuv420p[${nextVideoId}_fmt]`);
-            ffmpegContext.filterComplex.push(`[${transitionVideoId}_fmt][${nextVideoId}_fmt]xfade=transition=${transition.type}:duration=${duration}:offset=${xfadeOffset}[${xfadeOutputId}]`);
-            // Set PTS for overlay timing
-            const xfadeTimedId = `${xfadeOutputId}_t`;
-            ffmpegContext.filterComplex.push(`[${xfadeOutputId}]setpts=PTS-STARTPTS+${startAt}/TB[${xfadeTimedId}]`);
-            // Overlay the xfade result on the concat video
-            ffmpegContext.filterComplex.push(`[${prevVideoId}][${xfadeTimedId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
-        }
-        else {
-            throw new Error(`Unknown transition type: ${transition.type}`);
-        }
-        return outputVideoId;
+        const prevBeatIndex = MulmoBeatMethods.getPrevRenderedBeatIndex(context.studio.script.beats, beatIndex);
+        const duration = getClampedTransitionDuration(transition.duration, MulmoStudioContextMethods.getSegmentDuration(context, prevBeatIndex), MulmoStudioContextMethods.getSegmentDuration(context, beatIndex));
+        // The last frame of the previous rendered beat (voice_over beats have no video of their own).
+        // Both movie and image beats now have _last.
+        const prevVideoSourceId = getPrevVideoId(videoIdsForBeats, beatIndex);
+        return (addTransitionEffect({
+            ffmpegContext,
+            transitionType: transition.type,
+            prevVideoId,
+            transitionVideoId,
+            nextVideoId,
+            prevLastFrame: prevVideoSourceId ? `${prevVideoSourceId}_last` : undefined,
+            outputVideoId: `trans_${beatIndex}_o`,
+            // Transition happens at the start of this beat. 0.05 is to avoid flickering.
+            startAt: beatTimestamps[beatIndex] - 0.05,
+            duration,
+        }) ?? prevVideoId);
     }, captionedVideoId);
 };
-export const getNeedFirstFrame = (context) => {
-    return context.studio.script.beats.map((beat, index) => {
-        if (index === 0)
-            return false; // First beat cannot have transition
-        const transition = MulmoPresentationStyleMethods.getMovieTransition(context, beat);
-        return (transition?.type.startsWith("slidein_") || transition?.type.startsWith("wipe")) ?? false;
-    });
-};
-export const getNeedLastFrame = (context) => {
-    return context.studio.script.beats.map((beat, index) => {
-        if (index === context.studio.script.beats.length - 1)
-            return false; // Last beat doesn't need _last
-        const nextTransition = MulmoPresentationStyleMethods.getMovieTransition(context, context.studio.script.beats[index + 1]);
-        return nextTransition !== null; // Any transition on next beat requires this beat's last frame
-    });
-};
-export const resolveMovieVolume = (beat, context) => {
-    const baseMovieVolume = beat.audioParams?.movieVolume ?? context.presentationStyle.audioParams.movieVolume ?? 1.0;
-    const ducking = context.presentationStyle.audioParams.ducking;
-    const hasSpeech = !!beat.text && !context.presentationStyle.audioParams.suppressSpeech;
-    if (ducking && hasSpeech) {
-        const ratio = ducking.ratio ?? DEFAULT_DUCKING_RATIO;
-        return baseMovieVolume * ratio;
-    }
-    return baseMovieVolume;
-};
 export const isExplicitMixMode = (context) => {
     const audioParams = context.presentationStyle.audioParams;
     const duckingRequested = audioParams.ducking !== undefined;
@@ -285,40 +286,27 @@ export const mixAudiosFromMovieBeats = (ffmpegContext, artifactAudioId, audioIds
     }
     return artifactAudioId;
 };
-export const getExtraPadding = (context, index) => {
-    // We need to consider only intro and outro padding because the other paddings were already added to the beat.duration
-    if (index === 0) {
-        return MulmoStudioContextMethods.getIntroPadding(context);
-    }
-    else if (index === context.studio.beats.length - 1) {
-        return context.presentationStyle.audioParams.outroPadding;
-    }
-    return 0;
-};
-export const getFillOption = (context, beat) => {
-    // Get fillOption from merged imageParams (global + beat-specific)
-    const globalFillOption = context.presentationStyle.movieParams?.fillOption;
-    const beatFillOption = beat.movieParams?.fillOption;
-    const defaultFillOption = mulmoFillOptionSchema.parse({}); // let the schema infer the default value
-    return { ...defaultFillOption, ...globalFillOption, ...beatFillOption };
-};
 export const getTransitionVideoId = (transition, videoIdsForBeats, index) => {
+    // Every transition needs the previous *rendered* beat (voice_over beats have no video of their own).
+    const prevVideoSourceId = getPrevVideoId(videoIdsForBeats, index);
+    const currentVideoId = videoIdsForBeats[index];
+    if (!prevVideoSourceId || !currentVideoId) {
+        return null; // Skip the transition instead of referencing a filtergraph pad which does not exist.
+    }
     if (transition.type === "fade" || transition.type.startsWith("slideout_")) {
-        // Use previous beat's last frame. TODO: support voice-over
-        const prevVideoSourceId = videoIdsForBeats[index - 1];
+        // Use previous beat's last frame.
         // Both movie and image beats now have _last
         const frameId = `${prevVideoSourceId}_last`;
         return { videoId: frameId, nextVideoId: undefined, beatIndex: index };
     }
     if (transition.type.startsWith("wipe")) {
         // Wipe needs both previous beat's last frame and this beat's first frame
-        const prevVideoSourceId = videoIdsForBeats[index - 1];
         const prevLastFrame = `${prevVideoSourceId}_last`;
-        const nextFirstFrame = `${videoIdsForBeats[index]}_first`;
+        const nextFirstFrame = `${currentVideoId}_first`;
         return { videoId: prevLastFrame, nextVideoId: nextFirstFrame, beatIndex: index };
     }
     // Use this beat's first frame. slidein_ case
-    return { videoId: "", nextVideoId: `${videoIdsForBeats[index]}_first`, beatIndex: index };
+    return { videoId: "", nextVideoId: `${currentVideoId}_first`, beatIndex: index };
 };
 export const getConcatVideoFilter = (concatVideoId, videoIdsForBeats) => {
     const videoIds = videoIdsForBeats.filter((id) => id !== undefined); // filter out voice-over beats
@@ -336,14 +324,17 @@ export const getTransitionFrameDurations = (context, index) => {
     const getTransitionDuration = (transition, prevBeatIndex, currentBeatIndex) => {
         if (!transition || prevBeatIndex < 0 || currentBeatIndex >= beats.length)
             return 0;
-        const prevBeatDuration = beats[prevBeatIndex].duration ?? 1;
-        const currentBeatDuration = beats[currentBeatIndex].duration ?? 1;
+        const prevBeatDuration = MulmoStudioContextMethods.getSegmentDuration(context, prevBeatIndex);
+        const currentBeatDuration = MulmoStudioContextMethods.getSegmentDuration(context, currentBeatIndex);
         return getClampedTransitionDuration(transition.duration, prevBeatDuration, currentBeatDuration);
     };
+    // Transitions are resolved against the neighboring *rendered* beats (voice_over beats have no video).
+    const prevIndex = MulmoBeatMethods.getPrevRenderedBeatIndex(scriptBeats, index);
+    const nextIndex = MulmoBeatMethods.getNextRenderedBeatIndex(scriptBeats, index);
     const currentTransition = MulmoPresentationStyleMethods.getMovieTransition(context, scriptBeats[index]);
-    const firstDuration = index > 0 ? getTransitionDuration(currentTransition, index - 1, index) : 0;
-    const nextTransition = index < scriptBeats.length - 1 ? MulmoPresentationStyleMethods.getMovieTransition(context, scriptBeats[index + 1]) : null;
-    const lastDuration = getTransitionDuration(nextTransition, index, index + 1);
+    const firstDuration = prevIndex >= 0 ? getTransitionDuration(currentTransition, prevIndex, index) : 0;
+    const nextTransition = nextIndex >= 0 ? MulmoPresentationStyleMethods.getMovieTransition(context, scriptBeats[nextIndex]) : null;
+    const lastDuration = nextIndex >= 0 ? getTransitionDuration(nextTransition, index, nextIndex) : 0;
     return {
         firstDuration: Math.max(firstDuration, minFrame),
         lastDuration: Math.max(lastDuration, minFrame),
@@ -388,12 +379,26 @@ export const addSplitAndExtractFrames = (ffmpegContext, videoId, firstDuration,
 const findMissingIndex = (context) => {
     return context.studio.beats.findIndex((studioBeat, index) => {
         const beat = context.studio.script.beats[index];
-        if (beat.image?.type === "voice_over") {
+        if (MulmoBeatMethods.isVoiceOver(beat)) {
             return false; // Voice-over does not have either imageFile or movieFile.
         }
         return !studioBeat.imageFile && !studioBeat.movieFile;
     });
 };
+// Extract the stills the transitions around this beat consume, and return the transition this beat
+// starts (null when it has none, or when there is no previous shot to transition from).
+const addTransitionAssets = (ffmpegContext, context, index, videoIdsForBeats, { isMovie, canvasInfo, needFirst, needLast }) => {
+    const videoId = videoIdsForBeats[index];
+    if (needFirst || needLast) {
+        const { firstDuration, lastDuration } = getTransitionFrameDurations(context, index);
+        addSplitAndExtractFrames(ffmpegContext, videoId, firstDuration, lastDuration, isMovie, needFirst, needLast, canvasInfo);
+    }
+    const transition = MulmoPresentationStyleMethods.getMovieTransition(context, context.studio.script.beats[index]);
+    if (!transition || index === 0) {
+        return null;
+    }
+    return getTransitionVideoId(transition, videoIdsForBeats, index);
+};
 export const createVideo = async (audioArtifactFilePath, outputVideoPath, context, isTest = false) => {
     const caption = MulmoStudioContextMethods.getCaption(context);
     const start = performance.now();
@@ -410,20 +415,22 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
     const transitionVideoIds = [];
     const beatTimestamps = [];
     // Check which beats need _first (for slidein transition on this beat)
-    const needsFirstFrame = getNeedFirstFrame(context);
+    const needsFirstFrame = MulmoPresentationStyleMethods.getNeedFirstFrame(context);
     // Check which beats need _last (for any transition on next beat - they all need previous beat's last frame)
-    const needsLastFrame = getNeedLastFrame(context);
+    const needsLastFrame = MulmoPresentationStyleMethods.getNeedLastFrame(context);
     let cumulativeFrames = 0;
     context.studio.beats.reduce((timestamp, studioBeat, index) => {
         const beat = context.studio.script.beats[index];
-        if (beat.image?.type === "voice_over") {
+        if (MulmoBeatMethods.isVoiceOver(beat)) {
             videoIdsForBeats.push(undefined);
             beatTimestamps.push(timestamp);
-            return timestamp; // Skip voice-over beats.
+            // Skip voice-over beats: their video is covered by the owner beat's segment, but the
+            // audio timeline still advances, so keep the timestamp in sync with the audio artifact.
+            return timestamp + MulmoStudioContextMethods.getBeatDuration(context, index);
         }
         const sourceFile = isTest ? "/test/dummy.mp4" : validateBeatSource(studioBeat, index);
-        // The movie duration is bigger in case of voice-over.
-        const duration = Math.max(studioBeat.duration + getExtraPadding(context, index), studioBeat.movieDuration ?? 0);
+        const voiceOverDuration = MulmoStudioContextMethods.getVoiceOverGroupDuration(context, index);
+        const duration = MulmoStudioContextMethods.getSegmentDuration(context, index);
         // Use cumulative frame tracking to prevent audio-video drift from frame quantization.
         // trim=duration=X rounds up to the next frame boundary (~0.03s per beat at 30fps),
         // causing cumulative drift. Instead, compute exact frame counts per beat.
@@ -431,29 +438,23 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
         const frameCount = targetEndFrame - cumulativeFrames;
         cumulativeFrames = targetEndFrame;
         const inputIndex = FfmpegContextAddInput(ffmpegContext, sourceFile);
-        const isMovie = !!(studioBeat.lipSyncFile ||
-            studioBeat.movieFile ||
-            MulmoPresentationStyleMethods.getImageType(context.presentationStyle, beat) === "movie");
+        const isMovie = MulmoStudioContextMethods.isMovieBeat(context, studioBeat, beat);
         const speed = beat.movieParams?.speed ?? 1.0;
         const filters = beat.movieParams?.filters;
-        const { videoId, videoPart } = getVideoPart(inputIndex, isMovie, duration, canvasInfo, getFillOption(context, beat), speed, filters, frameCount);
+        const { videoId, videoPart } = getVideoPart(inputIndex, isMovie, duration, canvasInfo, MulmoPresentationStyleMethods.getFillOption(context, beat), speed, filters, frameCount);
         ffmpegContext.filterComplex.push(videoPart);
-        // for transition
-        const needFirst = needsFirstFrame[index]; // This beat has slidein
-        const needLast = needsLastFrame[index]; // Next beat has transition
         videoIdsForBeats.push(videoId);
-        if (needFirst || needLast) {
-            const { firstDuration, lastDuration } = getTransitionFrameDurations(context, index);
-            addSplitAndExtractFrames(ffmpegContext, videoId, firstDuration, lastDuration, isMovie, needFirst, needLast, canvasInfo);
-        }
-        // Record transition info if this beat has a transition
-        const transition = MulmoPresentationStyleMethods.getMovieTransition(context, beat);
-        if (transition && index > 0) {
-            const transitionVideoId = getTransitionVideoId(transition, videoIdsForBeats, index);
+        const transitionVideoId = addTransitionAssets(ffmpegContext, context, index, videoIdsForBeats, {
+            isMovie,
+            canvasInfo,
+            needFirst: needsFirstFrame[index], // This beat has slidein
+            needLast: needsLastFrame[index], // Next beat has transition
+        });
+        if (transitionVideoId) {
             transitionVideoIds.push(transitionVideoId);
         }
         // NOTE: We don't support audio if the speed is not 1.0.
-        const movieVolume = resolveMovieVolume(beat, context);
+        const movieVolume = MulmoPresentationStyleMethods.getMovieVolume(context, beat);
         if (studioBeat.hasMovieAudio && movieVolume > 0.0 && speed === 1.0) {
             // TODO: Handle a special case where it has lipSyncFile AND hasMovieAudio is on (the source file has an audio, such as sound effect).
             const { audioId, audioPart } = getAudioPart(inputIndex, duration, timestamp, movieVolume);
@@ -461,7 +462,8 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
             ffmpegContext.filterComplex.push(audioPart);
         }
         beatTimestamps.push(timestamp);
-        return timestamp + duration;
+        // The trailing voice_over beats advance the timestamp themselves.
+        return timestamp + duration - voiceOverDuration;
     }, 0);
     assert(videoIdsForBeats.length === context.studio.beats.length, "videoIds.length !== studio.beats.length");
     assert(beatTimestamps.length === context.studio.beats.length, "beatTimestamps.length !== studio.beats.length");

package/lib/methods/mulmo_beat.d.ts

@@ -8,6 +8,9 @@ export declare const MulmoBeatMethods: {
     isAnimationObject: (animation: unknown) => animation is AnimationConfig;
     isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
     isMovieMode: (animation: unknown) => boolean;
+    isVoiceOver: (beat?: MulmoBeat) => boolean;
+    getPrevRenderedBeatIndex(beats: MulmoBeat[], index: number): number;
+    getNextRenderedBeatIndex(beats: MulmoBeat[], index: number): number;
     getHtmlPrompt(beat: MulmoBeat): string | undefined;
     getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
 };

package/lib/methods/mulmo_beat.js

@@ -1,3 +1,4 @@
+import { ImageMediaType } from "../types/index.js";
 /** Type guard: checks if animation value is an object config like { fps: 30 } */
 const isAnimationObject = (animation) => {
     return typeof animation === "object" && animation !== null && !Array.isArray(animation);
@@ -17,11 +18,27 @@ const isAnimatedHtmlTailwind = (beat) => {
     const animation = beat.image.animation;
     return isAnimationEnabled(animation);
 };
+// voice_over beats share the preceding beat's shot, and contribute no video segment of their own.
+const isVoiceOver = (beat) => beat?.image?.type === ImageMediaType.VoiceOver;
 export const MulmoBeatMethods = {
     isAnimationEnabled,
     isAnimationObject,
     isAnimatedHtmlTailwind,
     isMovieMode,
+    isVoiceOver,
+    // Index of the closest preceding beat which is actually rendered as a video segment (-1 if none).
+    getPrevRenderedBeatIndex(beats, index) {
+        const offset = beats
+            .slice(0, Math.max(index, 0))
+            .reverse()
+            .findIndex((beat) => !isVoiceOver(beat));
+        return offset < 0 ? -1 : index - 1 - offset;
+    },
+    // Index of the closest following beat which is actually rendered as a video segment (-1 if none).
+    getNextRenderedBeatIndex(beats, index) {
+        const offset = beats.slice(index + 1).findIndex((beat) => !isVoiceOver(beat));
+        return offset < 0 ? -1 : index + 1 + offset;
+    },
     getHtmlPrompt(beat) {
         if (beat?.htmlPrompt?.data) {
             return beat.htmlPrompt.prompt + "\n\n[data]\n" + JSON.stringify(beat.htmlPrompt.data, null, 2);

package/lib/methods/mulmo_beat_node.d.ts

@@ -17,6 +17,9 @@ export declare const MulmoBeatMethods: {
     };
     isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
     isMovieMode: (animation: unknown) => boolean;
+    isVoiceOver: (beat?: MulmoBeat) => boolean;
+    getPrevRenderedBeatIndex(beats: MulmoBeat[], index: number): number;
+    getNextRenderedBeatIndex(beats: MulmoBeat[], index: number): number;
     getHtmlPrompt(beat: MulmoBeat): string | undefined;
     getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
 };

package/lib/methods/mulmo_presentation_style.d.ts

@@ -4,7 +4,7 @@
  * Works in both Node.js and modern browsers.
  */
 import type { SlideTheme } from "@mulmocast/deck";
-import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider, MulmoStudioContext, MulmoTransition } from "../types/index.js";
+import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider, MulmoStudioContext, MulmoTransition, MulmoFillOption } from "../types/index.js";
 export declare const MulmoPresentationStyleMethods: {
     getCanvasSize(presentationStyle: MulmoPresentationStyle): MulmoCanvasDimension;
     getAllSpeechProviders(presentationStyle: MulmoPresentationStyle): Set<Text2SpeechProvider>;
@@ -30,6 +30,10 @@ export declare const MulmoPresentationStyleMethods: {
     getSpeakerData(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat, lang: string | undefined): SpeakerData;
     getSpeaker(context: MulmoStudioContext, beat: MulmoBeat, targetLang: string | undefined): SpeakerData;
     getMovieTransition(context: MulmoStudioContext, beat: MulmoBeat): MulmoTransition | null;
+    getNeedFirstFrame(context: MulmoStudioContext): boolean[];
+    getNeedLastFrame(context: MulmoStudioContext): boolean[];
+    getFillOption(context: MulmoStudioContext, beat: MulmoBeat): MulmoFillOption;
+    getMovieVolume(context: MulmoStudioContext, beat: MulmoBeat): number;
     getText2ImageProvider(provider: Text2ImageProvider | undefined): Text2ImageProvider;
     getImageAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): Text2ImageAgentInfo;
     getMovieAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): {

package/lib/methods/mulmo_presentation_style.js

@@ -6,7 +6,8 @@
 import { isNull } from "graphai";
 import { userAssert } from "../utils/utils.js";
 import { slideThemes } from "../data/slideThemes.js";
-import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, mulmoTransitionSchema, } from "../types/schema.js";
+import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, mulmoTransitionSchema, mulmoFillOptionSchema, } from "../types/schema.js";
+import { MulmoBeatMethods } from "./mulmo_beat.js";
 import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, getModelAudio, AUDIO_MODE_ALWAYS, } from "../types/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
@@ -22,6 +23,13 @@ const defaultTextSlideStyles = [
     "td, th { padding: 8px }",
     "tr:nth-child(even) { background-color: #eee }",
 ];
+const DEFAULT_DUCKING_RATIO = 0.3;
+// Both frame requirements answer the same question for each *rendered* beat (voice_over beats have
+// no video segment, so they never supply a frame): does a transition need a still from this beat?
+const mapRenderedBeats = (context, needsFrame) => {
+    const beats = context.studio.script.beats;
+    return beats.map((beat, index) => (MulmoBeatMethods.isVoiceOver(beat) ? false : needsFrame(beats, index)));
+};
 export const MulmoPresentationStyleMethods = {
     getCanvasSize(presentationStyle) {
         return mulmoCanvasDimensionSchema.parse(presentationStyle.canvasSize);
@@ -95,6 +103,40 @@ export const MulmoPresentationStyleMethods = {
             return null;
         return mulmoTransitionSchema.parse(transitionData);
     },
+    getNeedFirstFrame(context) {
+        // This beat slides/wipes in over the previous shot, so it supplies its own first frame.
+        return mapRenderedBeats(context, (beats, index) => {
+            if (MulmoBeatMethods.getPrevRenderedBeatIndex(beats, index) < 0)
+                return false; // No previous shot: the transition is skipped
+            const transition = MulmoPresentationStyleMethods.getMovieTransition(context, beats[index]);
+            return (transition?.type.startsWith("slidein_") || transition?.type.startsWith("wipe")) ?? false;
+        });
+    },
+    getNeedLastFrame(context) {
+        // Any transition consumes the last frame of the previous rendered beat, skipping voice_over beats.
+        return mapRenderedBeats(context, (beats, index) => {
+            const nextIndex = MulmoBeatMethods.getNextRenderedBeatIndex(beats, index);
+            if (nextIndex < 0)
+                return false; // Last rendered beat doesn't need _last
+            return MulmoPresentationStyleMethods.getMovieTransition(context, beats[nextIndex]) !== null;
+        });
+    },
+    getFillOption(context, beat) {
+        const globalFillOption = context.presentationStyle.movieParams?.fillOption;
+        const beatFillOption = beat.movieParams?.fillOption;
+        const defaultFillOption = mulmoFillOptionSchema.parse({}); // let the schema infer the default value
+        return { ...defaultFillOption, ...globalFillOption, ...beatFillOption };
+    },
+    getMovieVolume(context, beat) {
+        const baseMovieVolume = beat.audioParams?.movieVolume ?? context.presentationStyle.audioParams.movieVolume ?? 1.0;
+        const ducking = context.presentationStyle.audioParams.ducking;
+        const hasSpeech = !!beat.text && !context.presentationStyle.audioParams.suppressSpeech;
+        if (ducking && hasSpeech) {
+            const ratio = ducking.ratio ?? DEFAULT_DUCKING_RATIO;
+            return baseMovieVolume * ratio;
+        }
+        return baseMovieVolume;
+    },
     /* NOTE: This method is not used.
     getTTSModel(context: MulmoStudioContext, beat: MulmoBeat): string | undefined {
       const speaker = MulmoPresentationStyleMethods.getSpeaker(context, beat);

package/lib/methods/mulmo_studio_context.d.ts

@@ -3,7 +3,7 @@
  * (No Node.js built-ins like fs, path, dotenv, etc.)
  * Works in both Node.js and modern browsers.
  */
-import { BeatSessionType, MulmoStudioContext, SessionProgressCallback, SessionType, MulmoBeat, SpeechOptions } from "../types/index.js";
+import { BeatSessionType, MulmoStudioContext, SessionProgressCallback, SessionType, MulmoBeat, MulmoStudioBeat, SpeechOptions } from "../types/index.js";
 import { provider2TTSAgent } from "../types/provider2agent.js";
 export declare const addSessionProgressCallback: (cb: SessionProgressCallback) => void;
 export declare const removeSessionProgressCallback: (cb: SessionProgressCallback) => void;
@@ -18,6 +18,11 @@ export declare const MulmoStudioContextMethods: {
     setBeatSessionState(context: MulmoStudioContext, sessionType: BeatSessionType | undefined, index: number, id: string | undefined, value: boolean): void;
     needTranslate(context: MulmoStudioContext, includeCaption?: boolean): boolean | "" | undefined;
     getIntroPadding(context: MulmoStudioContext): number;
+    getExtraPadding(context: MulmoStudioContext, index: number): number;
+    getBeatDuration(context: MulmoStudioContext, index: number): number;
+    getVoiceOverGroupDuration(context: MulmoStudioContext, index: number): number;
+    isMovieBeat(context: MulmoStudioContext, studioBeat: MulmoStudioBeat, beat: MulmoBeat): boolean;
+    getSegmentDuration(context: MulmoStudioContext, index: number): number;
     getAudioParam(context: MulmoStudioContext, beat: MulmoBeat, lang?: string): {
         provider: keyof typeof provider2TTSAgent;
         voiceId: string;

package/lib/methods/mulmo_studio_context.js

@@ -3,10 +3,11 @@
  * (No Node.js built-ins like fs, path, dotenv, etc.)
  * Works in both Node.js and modern browsers.
  */
-import { text2SpeechProviderSchema, } from "../types/index.js";
+import { ImageMediaType, text2SpeechProviderSchema, } from "../types/index.js";
 import { beatId } from "../utils/utils.js";
 import { GraphAILogger } from "graphai";
 import { MulmoPresentationStyleMethods } from "./mulmo_presentation_style.js";
+import { MulmoBeatMethods } from "./mulmo_beat.js";
 const sessionProgressCallbacks = new Set();
 export const addSessionProgressCallback = (cb) => {
     sessionProgressCallbacks.add(cb);
@@ -92,6 +93,40 @@ export const MulmoStudioContextMethods = {
         }
         return context.presentationStyle.audioParams.introPadding;
     },
+    // Intro/outro padding is the only padding not already folded into studio.beats[].duration.
+    getExtraPadding(context, index) {
+        if (index === 0) {
+            return MulmoStudioContextMethods.getIntroPadding(context);
+        }
+        else if (index === context.studio.beats.length - 1) {
+            return context.presentationStyle.audioParams.outroPadding;
+        }
+        return 0;
+    },
+    // The duration this beat occupies on the audio timeline.
+    getBeatDuration(context, index) {
+        return (context.studio.beats[index]?.duration ?? 0) + MulmoStudioContextMethods.getExtraPadding(context, index);
+    },
+    // The total duration of the voice_over beats which follow (and share the shot of) this beat.
+    getVoiceOverGroupDuration(context, index) {
+        const trailingBeats = context.studio.script.beats.slice(index + 1);
+        const groupEnd = trailingBeats.findIndex((beat) => !MulmoBeatMethods.isVoiceOver(beat));
+        const groupSize = groupEnd < 0 ? trailingBeats.length : groupEnd;
+        return trailingBeats.slice(0, groupSize).reduce((total, _, offset) => total + MulmoStudioContextMethods.getBeatDuration(context, index + 1 + offset), 0);
+    },
+    // Whether this beat's source is a movie (as opposed to a still which has to be looped).
+    isMovieBeat(context, studioBeat, beat) {
+        return !!(studioBeat.lipSyncFile ||
+            studioBeat.movieFile ||
+            MulmoPresentationStyleMethods.getImageType(context.presentationStyle, beat) === ImageMediaType.Movie);
+    },
+    // How long this beat's video segment is: its own duration plus the trailing voice_over beats which
+    // share its shot (they have no video segment of their own), never shorter than its movie. This is
+    // the single rule for a segment's length -- createVideo builds from it and transitions clamp against it.
+    getSegmentDuration(context, index) {
+        const ownDuration = MulmoStudioContextMethods.getBeatDuration(context, index) + MulmoStudioContextMethods.getVoiceOverGroupDuration(context, index);
+        return Math.max(ownDuration, context.studio.beats[index]?.movieDuration ?? 0);
+    },
     getAudioParam(context, beat, lang) {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(context, beat, lang);
         const speechOptions = { ...speaker.speechOptions, ...beat.speechOptions };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.9.0",
+  "version": "2.9.1",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -25,7 +25,7 @@
   },
   "resolutions": {
     "minimatch": "^10.2.5",
-    "tar": "7.5.20",
+    "tar": "7.5.22",
     "yauzl": "^3.4.0"
   },
   "bin": {
@@ -89,7 +89,7 @@
   "homepage": "https://github.com/receptron/mulmocast-cli#readme",
   "dependencies": {
     "@google-cloud/text-to-speech": "^6.4.1",
-    "@google/genai": "^2.12.0",
+    "@google/genai": "^2.13.0",
     "@graphai/anthropic_agent": "^2.0.12",
     "@graphai/browserless_agent": "^2.0.3",
     "@graphai/gemini_agent": "^2.0.5",
@@ -107,12 +107,12 @@
     "@mulmocast/deck": "^1.1.0",
     "@tavily/core": "^0.7.6",
     "archiver": "^8.0.0",
-    "clipboardy": "^5.3.1",
+    "clipboardy": "^5.3.2",
     "dotenv": "^17.4.2",
     "graphai": "^2.0.18",
     "js-tiktoken": "^1.0.21",
     "jsdom": "^29.1.1",
-    "marked": "^18.0.6",
+    "marked": "^18.0.7",
     "mulmocast-vision": "^1.0.12",
     "ora": "^9.4.1",
     "puppeteer": "^25.3.0",
@@ -128,16 +128,16 @@
     "@types/jsdom": "^28.0.3",
     "@types/yargs": "^17.0.35",
     "cross-env": "^10.1.0",
-    "eslint": "^10.7.0",
+    "eslint": "^10.8.0",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-import": "^2.32.0",
     "eslint-plugin-prettier": "^5.5.6",
     "eslint-plugin-sonarjs": "^4.2.0",
     "globals": "^17.7.0",
-    "prettier": "^3.9.5",
+    "prettier": "^3.9.6",
     "tsx": "^4.23.1",
     "typescript": "6.0.3",
-    "typescript-eslint": "^8.64.0"
+    "typescript-eslint": "^8.65.0"
   },
   "engines": {
     "node": ">=22.0.0"

package/scripts/test/test_voice_over_transition_no_audio.json

@@ -0,0 +1,71 @@
+{
+  "$mulmocast": {
+    "version": "1.1"
+  },
+  "lang": "en",
+  "title": "VoiceOver Transition Test",
+  "movieParams": {
+    "transition": {
+      "type": "fade",
+      "duration": 1.0
+    }
+  },
+  "beats": [
+    {
+      "speaker": "Presenter",
+      "text": "",
+      "duration": 3.0,
+      "image": {
+        "type": "image",
+        "source": {
+          "kind": "path",
+          "path": "../../assets/images/mulmocast_credit.png"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "",
+      "duration": 2.0,
+      "image": {
+        "type": "voice_over"
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "",
+      "duration": 3.0,
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Fade after a voice_over"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "",
+      "duration": 2.0,
+      "image": {
+        "type": "voice_over"
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "",
+      "duration": 3.0,
+      "movieParams": {
+        "transition": {
+          "type": "wipeleft",
+          "duration": 1.0
+        }
+      },
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Wipe after a voice_over"
+        }
+      }
+    }
+  ]
+}