mulmocast 2.9.0 → 2.9.2

package/lib/actions/movie.d.ts

@@ -1,4 +1,4 @@
-import { MulmoStudioContext, MulmoBeat, MulmoTransition, MulmoCanvasDimension, MulmoFillOption, MulmoVideoFilter } from "../types/index.js";
+import { MulmoStudioContext, MulmoTransition, MulmoCanvasDimension, MulmoFillOption, MulmoVideoFilter } from "../types/index.js";
 import { FfmpegContext } from "../utils/ffmpeg_utils.js";
 type VideoId = string | undefined;
 export declare const getVideoPart: (inputIndex: number, isMovie: boolean, duration: number, canvasInfo: MulmoCanvasDimension, fillOption: MulmoFillOption, speed: number, filters?: MulmoVideoFilter[], frameCount?: number) => {
@@ -11,15 +11,8 @@ export declare const getAudioPart: (inputIndex: number, duration: number, delay:
 };
 export declare const getOutOverlayCoords: (transitionType: string, d: number, t: number) => string;
 export declare const getInOverlayCoords: (transitionType: string, d: number, t: number) => string;
-export declare const getNeedFirstFrame: (context: MulmoStudioContext) => boolean[];
-export declare const getNeedLastFrame: (context: MulmoStudioContext) => boolean[];
-export declare const resolveMovieVolume: (beat: MulmoBeat, context: MulmoStudioContext) => number;
 export declare const isExplicitMixMode: (context: MulmoStudioContext) => boolean;
 export declare const mixAudiosFromMovieBeats: (ffmpegContext: FfmpegContext, artifactAudioId: string, audioIdsFromMovieBeats: string[], context: MulmoStudioContext) => string;
-export declare const getExtraPadding: (context: MulmoStudioContext, index: number) => number;
-export declare const getFillOption: (context: MulmoStudioContext, beat: MulmoBeat) => {
-    style: "aspectFit" | "aspectFill";
-};
 export declare const getTransitionVideoId: (transition: MulmoTransition, videoIdsForBeats: VideoId[], index: number) => {
     videoId: string;
     nextVideoId: undefined;
@@ -28,7 +21,7 @@ export declare const getTransitionVideoId: (transition: MulmoTransition, videoId
     videoId: string;
     nextVideoId: string;
     beatIndex: number;
-};
+} | null;
 export declare const getConcatVideoFilter: (concatVideoId: string, videoIdsForBeats: VideoId[]) => string;
 export declare const getTransitionFrameDurations: (context: MulmoStudioContext, index: number) => {
     firstDuration: number;

package/lib/actions/movie.js

@@ -1,6 +1,5 @@
 import { GraphAILogger, assert } from "graphai";
-import { mulmoFillOptionSchema, } from "../types/index.js";
-import { MulmoPresentationStyleMethods } from "../methods/index.js";
+import { MulmoPresentationStyleMethods, MulmoBeatMethods } from "../methods/index.js";
 import { getAudioArtifactFilePath, getOutputVideoFilePath, writingMessage, isFile } from "../utils/file.js";
 import { createVideoFileError, createVideoSourceError } from "../utils/error_cause.js";
 import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextPushFormattedAudio, FfmpegContextGenerateOutput, } from "../utils/ffmpeg_utils.js";
@@ -9,7 +8,14 @@ import { convertVideoFilterToFFmpeg } from "../utils/video_filter.js";
 // const isMac = process.platform === "darwin";
 const videoCodec = "libx264"; // "h264_videotoolbox" (macOS only) is too noisy
 const VIDEO_FPS = 30;
-const DEFAULT_DUCKING_RATIO = 0.3;
+// videoIdsForBeats holds undefined for voice_over beats; walk back to the previous rendered beat's videoId.
+const getPrevVideoId = (videoIdsForBeats, index) => {
+    const offset = videoIdsForBeats
+        .slice(0, Math.max(index, 0))
+        .reverse()
+        .findIndex((videoId) => videoId !== undefined);
+    return offset < 0 ? undefined : videoIdsForBeats[index - 1 - offset];
+};
 export const getVideoPart = (inputIndex, isMovie, duration, canvasInfo, fillOption, speed, filters, frameCount) => {
     const videoId = `v${inputIndex}`;
     const videoFilters = [];
@@ -149,109 +155,104 @@ export const getInOverlayCoords = (transitionType, d, t) => {
     }
     throw new Error(`Unknown transition type: ${transitionType}`);
 };
-const addTransitionEffects = (ffmpegContext, captionedVideoId, context, transitionVideoIds, beatTimestamps, videoIdsForBeats) => {
-    if (transitionVideoIds.length === 0) {
-        return captionedVideoId;
+// Each of these returns the id of the video to keep building on, or undefined when the transition
+// cannot be applied (in which case the caller keeps the untouched video).
+const addFadeTransition = ({ ffmpegContext, prevVideoId, transitionVideoId, outputVideoId, startAt, duration }) => {
+    // Fade out the previous beat's last frame
+    const processedVideoId = `${transitionVideoId}_f`;
+    ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,fade=t=out:d=${duration}:alpha=1,setpts=PTS-STARTPTS+${startAt}/TB[${processedVideoId}]`);
+    ffmpegContext.filterComplex.push(`[${prevVideoId}][${processedVideoId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
+    return outputVideoId;
+};
+const addSlideoutTransition = ({ ffmpegContext, transitionType, prevVideoId, transitionVideoId, outputVideoId, startAt, duration }) => {
+    // Slideout: previous beat's last frame slides out
+    const processedVideoId = `${transitionVideoId}_f`;
+    ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${processedVideoId}]`);
+    ffmpegContext.filterComplex.push(`[${prevVideoId}][${processedVideoId}]overlay=${getOutOverlayCoords(transitionType, duration, startAt)}:enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
+    return outputVideoId;
+};
+const addSlideinTransition = ({ ffmpegContext, transitionType, prevVideoId, nextVideoId, prevLastFrame, outputVideoId, startAt, duration, }) => {
+    // Slidein: this beat's first frame slides in over the previous beat's last frame
+    if (!nextVideoId || !prevLastFrame) {
+        return undefined; // Cannot apply slidein without the first frame or the previous shot
+    }
+    // Prepare background (last frame of previous beat)
+    const backgroundVideoId = `${prevLastFrame}_bg`;
+    ffmpegContext.filterComplex.push(`[${prevLastFrame}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${backgroundVideoId}]`);
+    // Prepare sliding frame (first frame of this beat)
+    const slideinFrameId = `${nextVideoId}_f`;
+    ffmpegContext.filterComplex.push(`[${nextVideoId}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${slideinFrameId}]`);
+    // First overlay: put background on top of concat video
+    const bgOutputId = `${prevLastFrame}_bg_o`;
+    ffmpegContext.filterComplex.push(`[${prevVideoId}][${backgroundVideoId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${bgOutputId}]`);
+    // Second overlay: slide in the new frame on top of background
+    ffmpegContext.filterComplex.push(`[${bgOutputId}][${slideinFrameId}]overlay=${getInOverlayCoords(transitionType, duration, startAt)}:enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
+    return outputVideoId;
+};
+const addWipeTransition = ({ ffmpegContext, transitionType, prevVideoId, transitionVideoId, nextVideoId, outputVideoId, startAt, duration, }) => {
+    // Wipe transition: use xfade filter between previous beat's last frame and this beat's first frame
+    if (!nextVideoId) {
+        return undefined; // Cannot apply wipe without first frame
+    }
+    // For wipe transitions, use offset=0 so transition starts immediately and completes within duration
+    // This ensures 0-100% wipe happens exactly during the transition duration
+    const xfadeOffset = 0;
+    // Apply xfade with offset=0 for complete 0-100% transition
+    // Both input videos should be at least duration seconds long (ensured by static frame generation)
+    const xfadeOutputId = `${transitionVideoId}_xfade`;
+    ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuv420p[${transitionVideoId}_fmt]`);
+    ffmpegContext.filterComplex.push(`[${nextVideoId}]format=yuv420p[${nextVideoId}_fmt]`);
+    ffmpegContext.filterComplex.push(`[${transitionVideoId}_fmt][${nextVideoId}_fmt]xfade=transition=${transitionType}:duration=${duration}:offset=${xfadeOffset}[${xfadeOutputId}]`);
+    // Set PTS for overlay timing
+    const xfadeTimedId = `${xfadeOutputId}_t`;
+    ffmpegContext.filterComplex.push(`[${xfadeOutputId}]setpts=PTS-STARTPTS+${startAt}/TB[${xfadeTimedId}]`);
+    // Overlay the xfade result on the concat video
+    ffmpegContext.filterComplex.push(`[${prevVideoId}][${xfadeTimedId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
+    return outputVideoId;
+};
+const addTransitionEffect = (transitionContext) => {
+    const { transitionType } = transitionContext;
+    if (transitionType === "fade") {
+        return addFadeTransition(transitionContext);
+    }
+    if (transitionType.startsWith("slideout_")) {
+        return addSlideoutTransition(transitionContext);
+    }
+    if (transitionType.startsWith("slidein_")) {
+        return addSlideinTransition(transitionContext);
     }
+    if (transitionType.startsWith("wipe")) {
+        return addWipeTransition(transitionContext);
+    }
+    throw new Error(`Unknown transition type: ${transitionType}`);
+};
+const addTransitionEffects = (ffmpegContext, captionedVideoId, context, transitionVideoIds, beatTimestamps, videoIdsForBeats) => {
     return transitionVideoIds.reduce((prevVideoId, { videoId: transitionVideoId, nextVideoId, beatIndex }) => {
         const beat = context.studio.script.beats[beatIndex];
         const transition = MulmoPresentationStyleMethods.getMovieTransition(context, beat);
         if (!transition) {
             return prevVideoId; // Skip if no transition is defined
         }
-        // Transition happens at the start of this beat
-        const startAt = beatTimestamps[beatIndex] - 0.05; // 0.05 is to avoid flickering
         // Limit transition duration to be no longer than either beat's duration
-        const prevBeatDuration = context.studio.beats[beatIndex - 1].duration ?? 1;
-        const currentBeatDuration = context.studio.beats[beatIndex].duration ?? 1;
-        const duration = getClampedTransitionDuration(transition.duration, prevBeatDuration, currentBeatDuration);
-        const outputVideoId = `trans_${beatIndex}_o`;
-        const processedVideoId = `${transitionVideoId}_f`;
-        if (transition.type === "fade") {
-            // Fade out the previous beat's last frame
-            ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,fade=t=out:d=${duration}:alpha=1,setpts=PTS-STARTPTS+${startAt}/TB[${processedVideoId}]`);
-            ffmpegContext.filterComplex.push(`[${prevVideoId}][${processedVideoId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
-        }
-        else if (transition.type.startsWith("slideout_")) {
-            // Slideout: previous beat's last frame slides out
-            ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${processedVideoId}]`);
-            ffmpegContext.filterComplex.push(`[${prevVideoId}][${processedVideoId}]overlay=${getOutOverlayCoords(transition.type, duration, startAt)}:enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
-        }
-        else if (transition.type.startsWith("slidein_")) {
-            // Slidein: this beat's first frame slides in over the previous beat's last frame
-            if (!nextVideoId) {
-                // Cannot apply slidein without first frame
-                return prevVideoId;
-            }
-            // Get previous beat's last frame for background
-            const prevVideoSourceId = videoIdsForBeats[beatIndex - 1];
-            // Both movie and image beats now have _last
-            const prevLastFrame = `${prevVideoSourceId}_last`;
-            // Prepare background (last frame of previous beat)
-            const backgroundVideoId = `${prevLastFrame}_bg`;
-            ffmpegContext.filterComplex.push(`[${prevLastFrame}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${backgroundVideoId}]`);
-            // Prepare sliding frame (first frame of this beat)
-            const slideinFrameId = `${nextVideoId}_f`;
-            ffmpegContext.filterComplex.push(`[${nextVideoId}]format=yuva420p,setpts=PTS-STARTPTS+${startAt}/TB[${slideinFrameId}]`);
-            // First overlay: put background on top of concat video
-            const bgOutputId = `${prevLastFrame}_bg_o`;
-            ffmpegContext.filterComplex.push(`[${prevVideoId}][${backgroundVideoId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${bgOutputId}]`);
-            // Second overlay: slide in the new frame on top of background
-            ffmpegContext.filterComplex.push(`[${bgOutputId}][${slideinFrameId}]overlay=${getInOverlayCoords(transition.type, duration, startAt)}:enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
-        }
-        else if (transition.type.startsWith("wipe")) {
-            // Wipe transition: use xfade filter between previous beat's last frame and this beat's first frame
-            if (!nextVideoId) {
-                // Cannot apply wipe without first frame
-                return prevVideoId;
-            }
-            // For wipe transitions, use offset=0 so transition starts immediately and completes within duration
-            // This ensures 0-100% wipe happens exactly during the transition duration
-            const xfadeOffset = 0;
-            // Apply xfade with offset=0 for complete 0-100% transition
-            // Both input videos should be at least duration seconds long (ensured by static frame generation)
-            const xfadeOutputId = `${transitionVideoId}_xfade`;
-            ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuv420p[${transitionVideoId}_fmt]`);
-            ffmpegContext.filterComplex.push(`[${nextVideoId}]format=yuv420p[${nextVideoId}_fmt]`);
-            ffmpegContext.filterComplex.push(`[${transitionVideoId}_fmt][${nextVideoId}_fmt]xfade=transition=${transition.type}:duration=${duration}:offset=${xfadeOffset}[${xfadeOutputId}]`);
-            // Set PTS for overlay timing
-            const xfadeTimedId = `${xfadeOutputId}_t`;
-            ffmpegContext.filterComplex.push(`[${xfadeOutputId}]setpts=PTS-STARTPTS+${startAt}/TB[${xfadeTimedId}]`);
-            // Overlay the xfade result on the concat video
-            ffmpegContext.filterComplex.push(`[${prevVideoId}][${xfadeTimedId}]overlay=enable='between(t,${startAt},${startAt + duration})'[${outputVideoId}]`);
-        }
-        else {
-            throw new Error(`Unknown transition type: ${transition.type}`);
-        }
-        return outputVideoId;
+        const prevBeatIndex = MulmoBeatMethods.getPrevRenderedBeatIndex(context.studio.script.beats, beatIndex);
+        const duration = getClampedTransitionDuration(transition.duration, MulmoStudioContextMethods.getSegmentDuration(context, prevBeatIndex), MulmoStudioContextMethods.getSegmentDuration(context, beatIndex));
+        // The last frame of the previous rendered beat (voice_over beats have no video of their own).
+        // Both movie and image beats now have _last.
+        const prevVideoSourceId = getPrevVideoId(videoIdsForBeats, beatIndex);
+        return (addTransitionEffect({
+            ffmpegContext,
+            transitionType: transition.type,
+            prevVideoId,
+            transitionVideoId,
+            nextVideoId,
+            prevLastFrame: prevVideoSourceId ? `${prevVideoSourceId}_last` : undefined,
+            outputVideoId: `trans_${beatIndex}_o`,
+            // Transition happens at the start of this beat. 0.05 is to avoid flickering.
+            startAt: beatTimestamps[beatIndex] - 0.05,
+            duration,
+        }) ?? prevVideoId);
     }, captionedVideoId);
 };
-export const getNeedFirstFrame = (context) => {
-    return context.studio.script.beats.map((beat, index) => {
-        if (index === 0)
-            return false; // First beat cannot have transition
-        const transition = MulmoPresentationStyleMethods.getMovieTransition(context, beat);
-        return (transition?.type.startsWith("slidein_") || transition?.type.startsWith("wipe")) ?? false;
-    });
-};
-export const getNeedLastFrame = (context) => {
-    return context.studio.script.beats.map((beat, index) => {
-        if (index === context.studio.script.beats.length - 1)
-            return false; // Last beat doesn't need _last
-        const nextTransition = MulmoPresentationStyleMethods.getMovieTransition(context, context.studio.script.beats[index + 1]);
-        return nextTransition !== null; // Any transition on next beat requires this beat's last frame
-    });
-};
-export const resolveMovieVolume = (beat, context) => {
-    const baseMovieVolume = beat.audioParams?.movieVolume ?? context.presentationStyle.audioParams.movieVolume ?? 1.0;
-    const ducking = context.presentationStyle.audioParams.ducking;
-    const hasSpeech = !!beat.text && !context.presentationStyle.audioParams.suppressSpeech;
-    if (ducking && hasSpeech) {
-        const ratio = ducking.ratio ?? DEFAULT_DUCKING_RATIO;
-        return baseMovieVolume * ratio;
-    }
-    return baseMovieVolume;
-};
 export const isExplicitMixMode = (context) => {
     const audioParams = context.presentationStyle.audioParams;
     const duckingRequested = audioParams.ducking !== undefined;
@@ -285,40 +286,27 @@ export const mixAudiosFromMovieBeats = (ffmpegContext, artifactAudioId, audioIds
     }
     return artifactAudioId;
 };
-export const getExtraPadding = (context, index) => {
-    // We need to consider only intro and outro padding because the other paddings were already added to the beat.duration
-    if (index === 0) {
-        return MulmoStudioContextMethods.getIntroPadding(context);
-    }
-    else if (index === context.studio.beats.length - 1) {
-        return context.presentationStyle.audioParams.outroPadding;
-    }
-    return 0;
-};
-export const getFillOption = (context, beat) => {
-    // Get fillOption from merged imageParams (global + beat-specific)
-    const globalFillOption = context.presentationStyle.movieParams?.fillOption;
-    const beatFillOption = beat.movieParams?.fillOption;
-    const defaultFillOption = mulmoFillOptionSchema.parse({}); // let the schema infer the default value
-    return { ...defaultFillOption, ...globalFillOption, ...beatFillOption };
-};
 export const getTransitionVideoId = (transition, videoIdsForBeats, index) => {
+    // Every transition needs the previous *rendered* beat (voice_over beats have no video of their own).
+    const prevVideoSourceId = getPrevVideoId(videoIdsForBeats, index);
+    const currentVideoId = videoIdsForBeats[index];
+    if (!prevVideoSourceId || !currentVideoId) {
+        return null; // Skip the transition instead of referencing a filtergraph pad which does not exist.
+    }
     if (transition.type === "fade" || transition.type.startsWith("slideout_")) {
-        // Use previous beat's last frame. TODO: support voice-over
-        const prevVideoSourceId = videoIdsForBeats[index - 1];
+        // Use previous beat's last frame.
         // Both movie and image beats now have _last
         const frameId = `${prevVideoSourceId}_last`;
         return { videoId: frameId, nextVideoId: undefined, beatIndex: index };
     }
     if (transition.type.startsWith("wipe")) {
         // Wipe needs both previous beat's last frame and this beat's first frame
-        const prevVideoSourceId = videoIdsForBeats[index - 1];
         const prevLastFrame = `${prevVideoSourceId}_last`;
-        const nextFirstFrame = `${videoIdsForBeats[index]}_first`;
+        const nextFirstFrame = `${currentVideoId}_first`;
         return { videoId: prevLastFrame, nextVideoId: nextFirstFrame, beatIndex: index };
     }
     // Use this beat's first frame. slidein_ case
-    return { videoId: "", nextVideoId: `${videoIdsForBeats[index]}_first`, beatIndex: index };
+    return { videoId: "", nextVideoId: `${currentVideoId}_first`, beatIndex: index };
 };
 export const getConcatVideoFilter = (concatVideoId, videoIdsForBeats) => {
     const videoIds = videoIdsForBeats.filter((id) => id !== undefined); // filter out voice-over beats
@@ -336,14 +324,17 @@ export const getTransitionFrameDurations = (context, index) => {
     const getTransitionDuration = (transition, prevBeatIndex, currentBeatIndex) => {
         if (!transition || prevBeatIndex < 0 || currentBeatIndex >= beats.length)
             return 0;
-        const prevBeatDuration = beats[prevBeatIndex].duration ?? 1;
-        const currentBeatDuration = beats[currentBeatIndex].duration ?? 1;
+        const prevBeatDuration = MulmoStudioContextMethods.getSegmentDuration(context, prevBeatIndex);
+        const currentBeatDuration = MulmoStudioContextMethods.getSegmentDuration(context, currentBeatIndex);
         return getClampedTransitionDuration(transition.duration, prevBeatDuration, currentBeatDuration);
     };
+    // Transitions are resolved against the neighboring *rendered* beats (voice_over beats have no video).
+    const prevIndex = MulmoBeatMethods.getPrevRenderedBeatIndex(scriptBeats, index);
+    const nextIndex = MulmoBeatMethods.getNextRenderedBeatIndex(scriptBeats, index);
     const currentTransition = MulmoPresentationStyleMethods.getMovieTransition(context, scriptBeats[index]);
-    const firstDuration = index > 0 ? getTransitionDuration(currentTransition, index - 1, index) : 0;
-    const nextTransition = index < scriptBeats.length - 1 ? MulmoPresentationStyleMethods.getMovieTransition(context, scriptBeats[index + 1]) : null;
-    const lastDuration = getTransitionDuration(nextTransition, index, index + 1);
+    const firstDuration = prevIndex >= 0 ? getTransitionDuration(currentTransition, prevIndex, index) : 0;
+    const nextTransition = nextIndex >= 0 ? MulmoPresentationStyleMethods.getMovieTransition(context, scriptBeats[nextIndex]) : null;
+    const lastDuration = nextIndex >= 0 ? getTransitionDuration(nextTransition, index, nextIndex) : 0;
     return {
         firstDuration: Math.max(firstDuration, minFrame),
         lastDuration: Math.max(lastDuration, minFrame),
@@ -388,12 +379,26 @@ export const addSplitAndExtractFrames = (ffmpegContext, videoId, firstDuration,
 const findMissingIndex = (context) => {
     return context.studio.beats.findIndex((studioBeat, index) => {
         const beat = context.studio.script.beats[index];
-        if (beat.image?.type === "voice_over") {
+        if (MulmoBeatMethods.isVoiceOver(beat)) {
             return false; // Voice-over does not have either imageFile or movieFile.
         }
         return !studioBeat.imageFile && !studioBeat.movieFile;
     });
 };
+// Extract the stills the transitions around this beat consume, and return the transition this beat
+// starts (null when it has none, or when there is no previous shot to transition from).
+const addTransitionAssets = (ffmpegContext, context, index, videoIdsForBeats, { isMovie, canvasInfo, needFirst, needLast }) => {
+    const videoId = videoIdsForBeats[index];
+    if (needFirst || needLast) {
+        const { firstDuration, lastDuration } = getTransitionFrameDurations(context, index);
+        addSplitAndExtractFrames(ffmpegContext, videoId, firstDuration, lastDuration, isMovie, needFirst, needLast, canvasInfo);
+    }
+    const transition = MulmoPresentationStyleMethods.getMovieTransition(context, context.studio.script.beats[index]);
+    if (!transition || index === 0) {
+        return null;
+    }
+    return getTransitionVideoId(transition, videoIdsForBeats, index);
+};
 export const createVideo = async (audioArtifactFilePath, outputVideoPath, context, isTest = false) => {
     const caption = MulmoStudioContextMethods.getCaption(context);
     const start = performance.now();
@@ -410,20 +415,22 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
     const transitionVideoIds = [];
     const beatTimestamps = [];
     // Check which beats need _first (for slidein transition on this beat)
-    const needsFirstFrame = getNeedFirstFrame(context);
+    const needsFirstFrame = MulmoPresentationStyleMethods.getNeedFirstFrame(context);
     // Check which beats need _last (for any transition on next beat - they all need previous beat's last frame)
-    const needsLastFrame = getNeedLastFrame(context);
+    const needsLastFrame = MulmoPresentationStyleMethods.getNeedLastFrame(context);
     let cumulativeFrames = 0;
     context.studio.beats.reduce((timestamp, studioBeat, index) => {
         const beat = context.studio.script.beats[index];
-        if (beat.image?.type === "voice_over") {
+        if (MulmoBeatMethods.isVoiceOver(beat)) {
             videoIdsForBeats.push(undefined);
             beatTimestamps.push(timestamp);
-            return timestamp; // Skip voice-over beats.
+            // Skip voice-over beats: their video is covered by the owner beat's segment, but the
+            // audio timeline still advances, so keep the timestamp in sync with the audio artifact.
+            return timestamp + MulmoStudioContextMethods.getBeatDuration(context, index);
         }
         const sourceFile = isTest ? "/test/dummy.mp4" : validateBeatSource(studioBeat, index);
-        // The movie duration is bigger in case of voice-over.
-        const duration = Math.max(studioBeat.duration + getExtraPadding(context, index), studioBeat.movieDuration ?? 0);
+        const voiceOverDuration = MulmoStudioContextMethods.getVoiceOverGroupDuration(context, index);
+        const duration = MulmoStudioContextMethods.getSegmentDuration(context, index);
         // Use cumulative frame tracking to prevent audio-video drift from frame quantization.
         // trim=duration=X rounds up to the next frame boundary (~0.03s per beat at 30fps),
         // causing cumulative drift. Instead, compute exact frame counts per beat.
@@ -431,29 +438,23 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
         const frameCount = targetEndFrame - cumulativeFrames;
         cumulativeFrames = targetEndFrame;
         const inputIndex = FfmpegContextAddInput(ffmpegContext, sourceFile);
-        const isMovie = !!(studioBeat.lipSyncFile ||
-            studioBeat.movieFile ||
-            MulmoPresentationStyleMethods.getImageType(context.presentationStyle, beat) === "movie");
+        const isMovie = MulmoStudioContextMethods.isMovieBeat(context, studioBeat, beat);
         const speed = beat.movieParams?.speed ?? 1.0;
         const filters = beat.movieParams?.filters;
-        const { videoId, videoPart } = getVideoPart(inputIndex, isMovie, duration, canvasInfo, getFillOption(context, beat), speed, filters, frameCount);
+        const { videoId, videoPart } = getVideoPart(inputIndex, isMovie, duration, canvasInfo, MulmoPresentationStyleMethods.getFillOption(context, beat), speed, filters, frameCount);
         ffmpegContext.filterComplex.push(videoPart);
-        // for transition
-        const needFirst = needsFirstFrame[index]; // This beat has slidein
-        const needLast = needsLastFrame[index]; // Next beat has transition
         videoIdsForBeats.push(videoId);
-        if (needFirst || needLast) {
-            const { firstDuration, lastDuration } = getTransitionFrameDurations(context, index);
-            addSplitAndExtractFrames(ffmpegContext, videoId, firstDuration, lastDuration, isMovie, needFirst, needLast, canvasInfo);
-        }
-        // Record transition info if this beat has a transition
-        const transition = MulmoPresentationStyleMethods.getMovieTransition(context, beat);
-        if (transition && index > 0) {
-            const transitionVideoId = getTransitionVideoId(transition, videoIdsForBeats, index);
+        const transitionVideoId = addTransitionAssets(ffmpegContext, context, index, videoIdsForBeats, {
+            isMovie,
+            canvasInfo,
+            needFirst: needsFirstFrame[index], // This beat has slidein
+            needLast: needsLastFrame[index], // Next beat has transition
+        });
+        if (transitionVideoId) {
             transitionVideoIds.push(transitionVideoId);
         }
         // NOTE: We don't support audio if the speed is not 1.0.
-        const movieVolume = resolveMovieVolume(beat, context);
+        const movieVolume = MulmoPresentationStyleMethods.getMovieVolume(context, beat);
         if (studioBeat.hasMovieAudio && movieVolume > 0.0 && speed === 1.0) {
             // TODO: Handle a special case where it has lipSyncFile AND hasMovieAudio is on (the source file has an audio, such as sound effect).
             const { audioId, audioPart } = getAudioPart(inputIndex, duration, timestamp, movieVolume);
@@ -461,7 +462,8 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
             ffmpegContext.filterComplex.push(audioPart);
         }
         beatTimestamps.push(timestamp);
-        return timestamp + duration;
+        // The trailing voice_over beats advance the timestamp themselves.
+        return timestamp + duration - voiceOverDuration;
     }, 0);
     assert(videoIdsForBeats.length === context.studio.beats.length, "videoIds.length !== studio.beats.length");
     assert(beatTimestamps.length === context.studio.beats.length, "beatTimestamps.length !== studio.beats.length");

package/lib/agents/image_google_agent.d.ts

@@ -0,0 +1,5 @@
+import type { AgentFunction, AgentFunctionInfo } from "graphai";
+import type { AgentBufferResult, ImageAgentInputs, ImageAgentParams, GoogleImageAgentConfig } from "../types/agent.js";
+export declare const imageGoogleAgent: AgentFunction<ImageAgentParams, AgentBufferResult, ImageAgentInputs, GoogleImageAgentConfig>;
+declare const imageGoogleAgentInfo: AgentFunctionInfo;
+export default imageGoogleAgentInfo;

package/lib/agents/image_google_agent.js

@@ -0,0 +1,86 @@
+import { GraphAILogger } from "graphai";
+import { getAspectRatio } from "./movie_google_agent.js";
+import { provider2ImageAgent } from "../utils/provider2agent.js";
+async function generateImage(projectId, model, token, prompt, aspectRatio) {
+    const GOOGLE_IMAGEN_ENDPOINT = `https://us-central1-aiplatform.googleapis.com/v1/projects/${projectId}/locations/us-central1/publishers/google/models/${model}:predict`;
+    try {
+        // Prepare the payload for the API request
+        const payload = {
+            instances: [
+                {
+                    prompt,
+                },
+            ],
+            parameters: {
+                sampleCount: 1,
+                aspectRatio,
+                safetySetting: "block_only_high",
+            },
+        };
+        // Make the API call using fetch
+        const response = await fetch(GOOGLE_IMAGEN_ENDPOINT, {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${token}`,
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify(payload),
+        });
+        if (!response.ok) {
+            throw new Error(`Error: ${response.status} - ${response.statusText}`);
+        }
+        const responseData = await response.json();
+        // Parse and return the generated image URL or data
+        const predictions = responseData.predictions;
+        if (predictions && predictions.length > 0) {
+            const base64Image = predictions[0].bytesBase64Encoded;
+            if (base64Image) {
+                return Buffer.from(base64Image, "base64"); // Decode the base64 image to a buffer
+            }
+            else {
+                throw new Error("No base64-encoded image data returned from the API.");
+            }
+        }
+        else {
+            // console.log(response);
+            GraphAILogger.info("No predictions returned from the API.", responseData, prompt);
+            return undefined;
+        }
+    }
+    catch (error) {
+        GraphAILogger.info("Error generating image:", error);
+        throw error;
+    }
+}
+export const imageGoogleAgent = async ({ namedInputs, params, config, }) => {
+    const { prompt } = namedInputs;
+    const aspectRatio = getAspectRatio(params.canvasSize);
+    const model = params.model ?? provider2ImageAgent["google"].defaultModel;
+    const projectId = config?.projectId;
+    const token = config?.token;
+    try {
+        const buffer = await generateImage(projectId, model, token, prompt, aspectRatio);
+        if (buffer) {
+            return { buffer };
+        }
+        throw new Error("ERROR: geneateImage returned undefined");
+    }
+    catch (error) {
+        GraphAILogger.info("Failed to generate image:", error);
+        throw error;
+    }
+};
+const imageGoogleAgentInfo = {
+    name: "imageGoogleAgent",
+    agent: imageGoogleAgent,
+    mock: imageGoogleAgent,
+    samples: [],
+    description: "Google Image agent",
+    category: ["image"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    // source: "https://github.com/receptron/mulmocast-cli/blob/main/src/agents/image_google_agent.ts",
+    license: "MIT",
+    environmentVariables: [],
+};
+export default imageGoogleAgentInfo;

package/lib/agents/movie_google_agent.d.ts

@@ -0,0 +1,9 @@
+import type { AgentFunction, AgentFunctionInfo } from "graphai";
+import type { AgentBufferResult, GoogleImageAgentConfig, GoogleMovieAgentParams, MovieAgentInputs } from "../types/agent.js";
+export declare const getAspectRatio: (canvasSize: {
+    width: number;
+    height: number;
+}) => string;
+export declare const movieGoogleAgent: AgentFunction<GoogleMovieAgentParams, AgentBufferResult, MovieAgentInputs, GoogleImageAgentConfig>;
+declare const movieGoogleAgentInfo: AgentFunctionInfo;
+export default movieGoogleAgentInfo;