mulmocast 2.17.0 → 2.19.0

package/lib/agents/add_bgm_agent.d.ts

@@ -1,13 +1,25 @@
 import type { AgentFunctionInfo } from "graphai";
 import { MulmoStudioContext } from "../types/index.js";
+import { FfmpegContext } from "../utils/ffmpeg_utils.js";
 export declare const resolveAddBgmMixParams: (audioParams: MulmoStudioContext["presentationStyle"]["audioParams"]) => {
     useExplicitMix: boolean;
     voiceVolume: number;
 };
-export declare const resolveAddBgmFilterConfig: (useExplicitMix: boolean) => {
+export declare const resolveAddBgmFilterConfig: (useExplicitMix: boolean, mixedInputId?: string) => {
     amixNormalize: string;
     mixedOutputId: string;
     limiterFilter: string | undefined;
 };
+export type SoundEffectPlacement = {
+    file: string;
+    startAt: number;
+    volume: number;
+    duration?: number;
+    loop: boolean;
+};
+export declare const getSoundEffectPlacements: (context: MulmoStudioContext) => SoundEffectPlacement[];
+export declare const soundEffectGain: (useExplicitMix: boolean) => 0.5 | 1;
+export declare const buildSoundEffectFilters: (inputIds: number[], placements: SoundEffectPlacement[], gain: number, outputId: string) => string[];
+export declare const addSoundEffects: (ffmpegContext: FfmpegContext, context: MulmoStudioContext) => "mixed" | "mixed_sfx";
 declare const addBGMAgentInfo: AgentFunctionInfo;
 export default addBGMAgentInfo;

package/lib/agents/add_bgm_agent.js

@@ -1,7 +1,10 @@
 import { GraphAILogger } from "graphai";
 import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextGenerateOutput, ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
+import { MulmoMediaSourceMethods } from "../methods/mulmo_media_source.js";
 import { isFile } from "../utils/file.js";
+import { userAssert } from "../utils/utils.js";
+import { getJingleFilePath } from "../utils/jinglescript.js";
 import { agentGenerationError, agentFileNotExistError, audioAction, audioFileTarget } from "../utils/error_cause.js";
 export const resolveAddBgmMixParams = (audioParams) => {
     const useExplicitMix = audioParams.ttsVolume !== undefined;
@@ -11,14 +14,82 @@ export const resolveAddBgmMixParams = (audioParams) => {
         voiceVolume: audioParams.audioVolume * ttsVolume,
     };
 };
-export const resolveAddBgmFilterConfig = (useExplicitMix) => {
+export const resolveAddBgmFilterConfig = (useExplicitMix, mixedInputId = "mixed") => {
     const amixNormalize = useExplicitMix ? ":normalize=0" : "";
     return {
         amixNormalize,
-        mixedOutputId: useExplicitMix ? "mixed_limited" : "mixed",
-        limiterFilter: useExplicitMix ? "[mixed]alimiter=limit=0.95:attack=5:release=50[mixed_limited]" : undefined,
+        mixedOutputId: useExplicitMix ? "mixed_limited" : mixedInputId,
+        limiterFilter: useExplicitMix ? `[${mixedInputId}]alimiter=limit=0.95:attack=5:release=50[mixed_limited]` : undefined,
     };
 };
+// Absolute placement of every beat's sound effects on the final audio timeline. startAt counts from
+// when the beat appears on screen, the same clock as its html animation (for the first beat, that is
+// introPadding before its narration starts).
+// The context must come from combineAudioFilesAgent, which sets studio.beats[].startAt.
+export const getSoundEffectPlacements = (context) => {
+    return context.studio.script.beats.flatMap((beat, index) => (beat.soundEffects ?? []).map((soundEffect, seIndex) => {
+        const beatStartAt = context.studio.beats[index]?.startAt;
+        userAssert(beatStartAt !== undefined, `soundEffects: startAt of beat ${index} is not computed yet`);
+        const { source } = soundEffect;
+        const file = source.kind === "jinglescript" ? getJingleFilePath(source.score, context) : MulmoMediaSourceMethods.resolve(source, context);
+        userAssert(!!file, `soundEffects: unsupported source at beat ${index}, effect ${seIndex} (use url or path)`);
+        const startAt = soundEffect.startAt ?? 0;
+        const loop = soundEffect.loop ?? false;
+        // A looping effect without duration plays until the beat leaves the screen.
+        const duration = soundEffect.duration ?? (loop ? MulmoStudioContextMethods.getBeatScreenDuration(context, index) - startAt : undefined);
+        userAssert(duration === undefined || duration > 0, `soundEffects: beat ${index}, effect ${seIndex} starts after the end of the beat`);
+        return {
+            file,
+            startAt: MulmoStudioContextMethods.getBeatScreenStartAt(context, index) + startAt,
+            volume: soundEffect.volume ?? 1.0,
+            ...(duration !== undefined ? { duration } : {}),
+            loop,
+        };
+    }));
+};
+// In legacy mode, amix (normalize=1) halves the voice and the music. Sound effects are mixed in
+// afterwards with normalize=0, so apply the same factor to keep volume 1.0 at the gain of the narration at its
+// default volume. audioVolume / ttsVolume deliberately do not scale sound effects.
+export const soundEffectGain = (useExplicitMix) => (useExplicitMix ? 1.0 : 0.5);
+const soundEffectFadeOut = 0.05; // seconds, avoids a click where the effect is cut
+const getSoundEffectTrim = (duration) => {
+    const fade = Math.min(soundEffectFadeOut, duration);
+    return `atrim=duration=${duration}, afade=t=out:st=${duration - fade}:d=${fade}, `;
+};
+export const buildSoundEffectFilters = (inputIds, placements, gain, outputId) => {
+    const ids = placements.map((_, index) => `se_${index}`);
+    const filters = placements.map((placement, index) => {
+        const delayMs = Math.round(placement.startAt * 1000);
+        const trim = placement.duration !== undefined ? getSoundEffectTrim(placement.duration) : "";
+        return `[${inputIds[index]}:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo, ${trim}volume=${placement.volume * gain}, adelay=${delayMs}|${delayMs}[${ids[index]}]`;
+    });
+    if (ids.length === 1) {
+        return [...filters, `[${ids[0]}]anull[${outputId}]`];
+    }
+    const inputs = ids.map((id) => "[" + id + "]").join("");
+    return [...filters, `${inputs}amix=inputs=${ids.length}:duration=longest:normalize=0[${outputId}]`];
+};
+// Adds the beats' sound effects to ffmpegContext and mixes them onto [mixed].
+// Returns the id of the resulting stream ("mixed" when there are none).
+export const addSoundEffects = (ffmpegContext, context) => {
+    const { useExplicitMix } = resolveAddBgmMixParams(context.presentationStyle.audioParams);
+    const soundEffects = getSoundEffectPlacements(context);
+    if (soundEffects.length === 0) {
+        return "mixed";
+    }
+    soundEffects.forEach(({ file }) => {
+        if (!/^https?:\/\//.test(file) && !isFile(file)) {
+            throw new Error(`AddBGMAgent soundEffect file not exist: ${file}`, {
+                cause: agentFileNotExistError("addBGMAgent", audioAction, audioFileTarget, file),
+            });
+        }
+    });
+    GraphAILogger.log("soundEffects:", soundEffects);
+    const inputIds = soundEffects.map(({ file, loop }) => FfmpegContextAddInput(ffmpegContext, file, loop ? ["-stream_loop", "-1"] : undefined));
+    ffmpegContext.filterComplex.push(...buildSoundEffectFilters(inputIds, soundEffects, soundEffectGain(useExplicitMix), "sfx"));
+    ffmpegContext.filterComplex.push("[mixed][sfx]amix=inputs=2:duration=first:normalize=0[mixed_sfx]");
+    return "mixed_sfx";
+};
 const addBGMAgent = async ({ namedInputs, params, }) => {
     const { voiceFile, outputFile, context } = namedInputs;
     const { musicFile } = params;
@@ -44,8 +115,10 @@ const addBGMAgent = async ({ namedInputs, params, }) => {
     const { useExplicitMix, voiceVolume } = resolveAddBgmMixParams(audioParams);
     ffmpegContext.filterComplex.push(`[${musicInputIndex}:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo, volume=${audioParams.bgmVolume}[music]`);
     ffmpegContext.filterComplex.push(`[${voiceInputIndex}:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo, volume=${voiceVolume}, adelay=${introPadding * 1000}|${introPadding * 1000}[voice]`);
-    const { amixNormalize, mixedOutputId, limiterFilter } = resolveAddBgmFilterConfig(useExplicitMix);
+    const { amixNormalize } = resolveAddBgmFilterConfig(useExplicitMix);
     ffmpegContext.filterComplex.push(`[music][voice]amix=inputs=2:duration=longest${amixNormalize}[mixed]`);
+    const mixedInputId = addSoundEffects(ffmpegContext, context);
+    const { mixedOutputId, limiterFilter } = resolveAddBgmFilterConfig(useExplicitMix, mixedInputId);
     if (limiterFilter) {
         ffmpegContext.filterComplex.push(limiterFilter);
     }

package/lib/agents/movie_genai_agent.d.ts

@@ -1,5 +1,29 @@
 import type { AgentFunction, AgentFunctionInfo } from "graphai";
 import type { AgentBufferResult, GenAIImageAgentConfig, GoogleMovieAgentParams, MovieAgentInputs } from "../types/agent.js";
+import { type GeminiOmniImage, type GeminiOmniVideoRequest } from "../utils/gemini_omni_video.js";
+type GeminiOmniInteraction = {
+    id?: string;
+    output_video?: {
+        data?: string;
+        mime_type?: string;
+    };
+};
+export type GeminiOmniVideoClient = {
+    interactions: {
+        create: (params: GeminiOmniVideoRequest, options?: {
+            timeout?: number;
+        }) => Promise<GeminiOmniInteraction>;
+    };
+};
+export type GeminiOmniVideoInput = {
+    model: string;
+    prompt: string;
+    aspectRatio: string;
+    requestedSec: number;
+    firstFrame?: GeminiOmniImage;
+    lastFrame?: GeminiOmniImage;
+};
+export declare const generateGeminiOmniVideo: (ai: GeminiOmniVideoClient, input: GeminiOmniVideoInput) => Promise<Buffer>;
 export declare const movieGenAIAgent: AgentFunction<GoogleMovieAgentParams, AgentBufferResult, MovieAgentInputs, GenAIImageAgentConfig>;
 declare const movieGenAIAgentInfo: AgentFunctionInfo;
 export default movieGenAIAgentInfo;

package/lib/agents/movie_genai_agent.js

@@ -5,11 +5,14 @@ import { apiKeyMissingError, agentGenerationError, agentInvalidResponseError, im
 import { getAspectRatio } from "../utils/utils.js";
 import { ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
 import { ASPECT_RATIOS } from "../types/const.js";
-import { getModelDuration, provider2MovieAgent, AUDIO_MODE_NEVER, AUDIO_MODE_ALWAYS, defaultMovieModel, veoExtensionCount, VEO_EXTENSION_MODEL, VEO_SEGMENT_SEC, } from "../types/provider2agent.js";
+import { getModelDuration, provider2MovieAgent, AUDIO_MODE_NEVER, AUDIO_MODE_ALWAYS, defaultMovieModel, unsupportedGoogleMovieModelMessage, } from "../types/provider2agent.js";
+import { GEMINI_OMNI_MAX_TOTAL_SEC, buildGeminiOmniVideoRequest, geminiOmniAspectRatio, geminiOmniExtensionPrompt, geminiOmniSegments, isGeminiOmniVideoModel, videoFromInteraction, } from "../utils/gemini_omni_video.js";
 // Per-request timeout so a stalled GenAI video API call rejects instead of hanging.
 const GENAI_REQUEST_TIMEOUT_MS = 120_000;
 // Wall-clock cap on the long-running video operation poll loop (Veo runs minutes).
 const VIDEO_POLL_TIMEOUT_MS = 1_200_000;
+// A Gemini Omni call answers synchronously with the whole video (10s at 720p took about 45s).
+const GEMINI_OMNI_REQUEST_TIMEOUT_MS = 600_000;
 const pollUntilDone = async (ai, operation) => {
     const response = { operation };
     const deadline = Date.now() + VIDEO_POLL_TIMEOUT_MS;
@@ -78,47 +81,53 @@ const downloadVideo = async (ai, video, movieFile, isVertexAI, model) => {
     const usage = predictSec !== undefined ? { provider: "google", model, predictSec } : undefined;
     return { saved: movieFile, usage };
 };
-const createVeo31Payload = (model, prompt, aspectRatio, source) => ({
-    model,
-    prompt,
-    config: {
-        aspectRatio,
-        resolution: "720p",
-        numberOfVideos: 1,
-    },
-    ...source,
-});
-const generateExtendedVideo = async (ai, model, prompt, aspectRatio, imagePath, requestedDuration, movieFile, isVertexAI) => {
-    const extensionsNeeded = veoExtensionCount(requestedDuration);
-    GraphAILogger.info(`Veo 3.1 video extension: ${extensionsNeeded} extensions needed for ${requestedDuration}s target`);
-    const generateIteration = async (iteration, accumulatedDuration, previousVideo) => {
-        const isInitial = iteration === 0;
-        const remainingDuration = requestedDuration - accumulatedDuration;
-        const extensionDuration = isInitial ? VEO_SEGMENT_SEC : (getModelDuration("google", model, remainingDuration) ?? VEO_SEGMENT_SEC);
-        const getSource = () => {
-            if (isInitial)
-                return imagePath ? { image: loadImageAsBase64(imagePath) } : undefined;
-            return previousVideo?.uri ? { video: { uri: previousVideo.uri } } : undefined;
-        };
-        const payload = createVeo31Payload(model, prompt, aspectRatio, getSource());
-        GraphAILogger.info(isInitial ? "Generating initial 8s video..." : `Extending video: iteration ${iteration}/${extensionsNeeded} (+${extensionDuration}s)...`);
-        const operation = await ai.models.generateVideos(payload);
-        const response = await pollUntilDone(ai, operation);
-        const video = getVideoFromResponse(response, iteration);
-        const totalDuration = accumulatedDuration + extensionDuration;
-        GraphAILogger.info(`Video ${isInitial ? "generated" : "extended"}: ~${totalDuration}s total`);
-        return { video, duration: totalDuration };
-    };
-    const result = await Array.from({ length: extensionsNeeded + 1 }).reduce(async (prev, _, index) => {
-        const { video, duration } = await prev;
-        return generateIteration(index, duration, video);
-    }, Promise.resolve({ video: undefined, duration: 0 }));
-    if (!result.video) {
-        throw new Error("Failed to generate extended video", {
-            cause: agentInvalidResponseError("movieGenAIAgent", imageAction, movieFileTarget),
+const isOmniImage = (frame) => frame !== undefined;
+const nextOmniRequest = (input, durationSec, previous, frames) => {
+    if (!previous)
+        return buildGeminiOmniVideoRequest({ ...input, durationSec, frames });
+    if (!previous.id)
+        throw new Error("Gemini Omni returned no interaction id to extend");
+    return buildGeminiOmniVideoRequest({ ...input, prompt: geminiOmniExtensionPrompt(input.prompt), durationSec, previousInteractionId: previous.id });
+};
+// Each call after the first extends the previous interaction's video and answers with the whole video so far.
+export const generateGeminiOmniVideo = async (ai, input) => {
+    const segments = geminiOmniSegments(input.requestedSec);
+    const frames = [input.firstFrame, segments.length === 1 ? input.lastFrame : undefined].filter(isOmniImage);
+    const last = await segments.reduce(async (previousPromise, durationSec) => {
+        const request = nextOmniRequest(input, durationSec, await previousPromise, frames);
+        return ai.interactions.create(request, { timeout: GEMINI_OMNI_REQUEST_TIMEOUT_MS });
+    }, Promise.resolve(undefined));
+    return videoFromInteraction(last?.output_video);
+};
+const omniImage = (imagePath) => {
+    const { imageBytes, mimeType } = loadImageAsBase64(imagePath);
+    return { data: imageBytes, mime_type: mimeType };
+};
+const omniFrames = (input, { imagePath, lastFrameImagePath, referenceImages }) => {
+    if (referenceImages && referenceImages.length > 0) {
+        GraphAILogger.warn(`movieGenAIAgent: model ${input.model} does not support referenceImages — ignoring`);
+    }
+    if (lastFrameImagePath && !imagePath) {
+        GraphAILogger.warn(`movieGenAIAgent: lastFrame requires a first frame image (imagePrompt or firstFrameImageName) — ignoring lastFrameImageName`);
+    }
+    else if (lastFrameImagePath && geminiOmniSegments(input.requestedSec).length > 1) {
+        GraphAILogger.warn(`movieGenAIAgent: lastFrame applies to a single ${input.model} segment — ignoring lastFrameImageName for a ${input.requestedSec}s beat`);
+    }
+    const lastFrame = lastFrameImagePath && imagePath ? omniImage(lastFrameImagePath) : undefined;
+    return { firstFrame: imagePath ? omniImage(imagePath) : undefined, lastFrame };
+};
+const generateOmniVideoFile = async (ai, isVertexAI, input, sources, movieFile) => {
+    if (isVertexAI) {
+        throw new Error(`Model ${input.model} is supported on the Gemini API only; use veo-3.1-generate-001 with Vertex AI.`, {
+            cause: agentGenerationError("movieGenAIAgent", imageAction, unsupportedModelTarget),
         });
     }
-    return downloadVideo(ai, result.video, movieFile, isVertexAI, model);
+    if (input.requestedSec > GEMINI_OMNI_MAX_TOTAL_SEC) {
+        GraphAILogger.warn(`movieGenAIAgent: ${input.model} makes at most ${GEMINI_OMNI_MAX_TOTAL_SEC}s — the ${input.requestedSec}s beat gets a shorter video`);
+    }
+    writeFileSync(movieFile, await generateGeminiOmniVideo(ai, { ...input, ...omniFrames(input, sources) }));
+    const predictSec = await probeDurationSec(movieFile);
+    return { saved: movieFile, usage: predictSec !== undefined ? { provider: "google", model: input.model, predictSec } : undefined };
 };
 const generateStandardVideo = async (ai, model, prompt, aspectRatio, imagePath, lastFrameImagePath, referenceImages, duration, movieFile, isVertexAI) => {
     const capabilities = provider2MovieAgent.google.modelParams[model];
@@ -173,6 +182,10 @@ export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
     const model = params.model ?? defaultMovieModel("google", params);
     const apiKey = config?.apiKey;
     try {
+        const unsupportedMessage = unsupportedGoogleMovieModelMessage(model);
+        if (unsupportedMessage) {
+            throw new Error(unsupportedMessage, { cause: agentGenerationError("movieGenAIAgent", imageAction, unsupportedModelTarget) });
+        }
         const requestedDuration = params.duration ?? 8;
         const duration = getModelDuration("google", model, requestedDuration);
         if (duration === undefined) {
@@ -208,9 +221,9 @@ export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
                 }
                 return new GoogleGenAI({ apiKey, httpOptions: { timeout: GENAI_REQUEST_TIMEOUT_MS } });
             })();
-        // Veo 3.1: Video extension mode for videos longer than 8s
-        if (model === VEO_EXTENSION_MODEL && requestedDuration > VEO_SEGMENT_SEC && params.canvasSize) {
-            return generateExtendedVideo(ai, model, prompt, aspectRatio, imagePath, requestedDuration, movieFile, isVertexAI);
+        if (isGeminiOmniVideoModel(model)) {
+            const omniInput = { model, prompt, aspectRatio: geminiOmniAspectRatio(params.canvasSize), requestedSec: requestedDuration };
+            return generateOmniVideoFile(ai, isVertexAI, omniInput, { imagePath, lastFrameImagePath, referenceImages }, movieFile);
         }
         // Standard mode
         return generateStandardVideo(ai, model, prompt, aspectRatio, imagePath, lastFrameImagePath, referenceImages, duration, movieFile, isVertexAI);

package/lib/cli/commands/bundle/builder.d.ts

@@ -11,6 +11,8 @@ export declare const builder: (yargs: Argv) => Argv<{
     g: boolean;
 } & {
     backup: boolean;
+} & {
+    "strict-network": boolean;
 } & {
     p: string | undefined;
 } & {

package/lib/cli/commands/html/builder.d.ts

@@ -11,6 +11,8 @@ export declare const builder: (yargs: Argv) => Argv<{
     g: boolean;
 } & {
     backup: boolean;
+} & {
+    "strict-network": boolean;
 } & {
     p: string | undefined;
 } & {

package/lib/cli/commands/markdown/builder.d.ts

@@ -11,6 +11,8 @@ export declare const builder: (yargs: Argv) => Argv<{
     g: boolean;
 } & {
     backup: boolean;
+} & {
+    "strict-network": boolean;
 } & {
     p: string | undefined;
 } & {

package/lib/cli/commands/viewer/builder.d.ts

@@ -11,6 +11,8 @@ export declare const builder: (yargs: Argv) => Argv<{
     g: boolean;
 } & {
     backup: boolean;
+} & {
+    "strict-network": boolean;
 } & {
     p: string | undefined;
 } & {

package/lib/cli/common.d.ts

@@ -16,6 +16,8 @@ export declare const commonOptions: (yargs: Argv) => Argv<{
     g: boolean;
 } & {
     backup: boolean;
+} & {
+    "strict-network": boolean;
 } & {
     p: string | undefined;
 } & {

package/lib/cli/common.js

@@ -49,6 +49,11 @@ export const commonOptions = (yargs) => {
         describe: "create backup media file",
         type: "boolean",
         default: false,
+    })
+        .option("strict-network", {
+        describe: "Let HTML renderers reach only local files and the CDNs mulmocast's templates use (also MULMO_STRICT_NETWORK=1)",
+        type: "boolean",
+        default: false,
     })
         .option("p", {
         alias: "presentationStyle",

package/lib/cli/helpers.js

@@ -10,6 +10,7 @@ import { translate } from "../actions/translate.js";
 import { initializeContextFromFiles } from "../utils/context.js";
 import { estimateUsage, actionEstimateProcesses } from "../utils/estimate_usage.js";
 import { formatUsageEstimates } from "../utils/estimate_usage_format.js";
+import { strictNetworkFromEnv } from "../utils/render_network_policy.js";
 export const runTranslateIfNeeded = async (context, includeCaption = false) => {
     if (MulmoStudioContextMethods.needTranslate(context, includeCaption)) {
         GraphAILogger.log("run translate");
@@ -151,5 +152,6 @@ export const initializeContext = async (argv, raiseError = false) => {
         grouped: Boolean(argv.g),
     });
     setGraphAILogger(Boolean(argv.v), { files });
-    return await initializeContextFromFiles(files, raiseError, Boolean(argv.f), Boolean(argv.backup), argv.c, argv.l);
+    const context = await initializeContextFromFiles(files, raiseError, Boolean(argv.f), Boolean(argv.backup), argv.c, argv.l);
+    return context && { ...context, strictNetwork: Boolean(argv.strictNetwork) || strictNetworkFromEnv(process.env) };
 };

package/lib/methods/mulmo_studio_context.d.ts

@@ -12,6 +12,10 @@ export declare const MulmoStudioContextMethods: {
     getImageDirPath(context: MulmoStudioContext): string;
     getImageProjectDirPath(context: MulmoStudioContext): string;
     getOutDirPath(context: MulmoStudioContext): string;
+    getRenderNetworkOptions(context: MulmoStudioContext): {
+        strictNetwork: boolean;
+        allowedFileRoots: string[];
+    };
     getFileName(context: MulmoStudioContext): string;
     getCaption(context: MulmoStudioContext): string | undefined;
     setSessionState(context: MulmoStudioContext, sessionType: SessionType, value: boolean, result?: boolean): void;
@@ -19,6 +23,8 @@ export declare const MulmoStudioContextMethods: {
     needTranslate(context: MulmoStudioContext, includeCaption?: boolean): boolean | "" | undefined;
     getIntroPadding(context: MulmoStudioContext): number;
     getExtraPadding(context: MulmoStudioContext, index: number): number;
+    getBeatScreenStartAt(context: MulmoStudioContext, index: number): number;
+    getBeatScreenDuration(context: MulmoStudioContext, index: number): number;
     getBeatDuration(context: MulmoStudioContext, index: number): number;
     getVoiceOverGroupDuration(context: MulmoStudioContext, index: number): number;
     isMovieBeat(context: MulmoStudioContext, studioBeat: MulmoStudioBeat, beat: MulmoBeat): boolean;

package/lib/methods/mulmo_studio_context.js

@@ -53,6 +53,13 @@ export const MulmoStudioContextMethods = {
     getOutDirPath(context) {
         return context.fileDirs.outDirPath;
     },
+    // In strict network mode a renderer may read files only from the script's folder and mulmocast's output folders.
+    // A script loaded by URL has no local folder: its mulmoFileDirPath is the parent of the base directory.
+    getRenderNetworkOptions(context) {
+        const { mulmoFileDirPath, outDirPath, imageDirPath, audioDirPath, isHttpPath } = context.fileDirs;
+        const scriptRoots = isHttpPath ? [] : [mulmoFileDirPath];
+        return { strictNetwork: Boolean(context.strictNetwork), allowedFileRoots: [...scriptRoots, outDirPath, imageDirPath, audioDirPath] };
+    },
     getFileName(context) {
         return context.studio.filename;
     },
@@ -103,6 +110,25 @@ export const MulmoStudioContextMethods = {
         }
         return 0;
     },
+    // When this beat appears on screen, in seconds from the start of the movie. The first beat's segment
+    // starts at 0 and absorbs the intro padding; every later beat starts when its narration does.
+    getBeatScreenStartAt(context, index) {
+        if (index === 0) {
+            return 0;
+        }
+        return MulmoStudioContextMethods.getIntroPadding(context) + (context.studio.beats[index]?.startAt ?? 0);
+    },
+    // How long this beat stays on screen from getBeatScreenStartAt: its whole segment, including the
+    // voice_over beats that share its shot. A voice_over beat itself is counted for its own slice only.
+    // The audio ends outroPadding after the last beat; getExtraPadding gives a single-beat script only
+    // the intro padding, so its outro is added here.
+    getBeatScreenDuration(context, index) {
+        if (MulmoBeatMethods.isVoiceOver(context.studio.script.beats[index])) {
+            return MulmoStudioContextMethods.getBeatDuration(context, index);
+        }
+        const singleBeatOutro = context.studio.beats.length === 1 ? context.presentationStyle.audioParams.outroPadding : 0;
+        return MulmoStudioContextMethods.getSegmentDuration(context, index) + singleBeatOutro;
+    },
     // The duration this beat occupies on the audio timeline.
     getBeatDuration(context, index) {
         return (context.studio.beats[index]?.duration ?? 0) + MulmoStudioContextMethods.getExtraPadding(context, index);

package/lib/types/provider2agent.d.ts

@@ -233,13 +233,10 @@ export declare const ProviderName: {
 export declare const isImageProvider: (provider: string | undefined) => provider is keyof typeof provider2ImageAgent;
 export declare const getMaxImageReferenceImages: (provider: string | undefined, model: string) => number | undefined;
 export declare const getModelAudio: (provider: keyof typeof provider2MovieAgent, model: string) => MovieAudioSpec | undefined;
+export declare const unsupportedGoogleMovieModelMessage: (model: string) => string | null;
 export declare const defaultMovieModel: (provider: keyof typeof provider2MovieAgent, movieParams?: {
     vertexai_project?: string;
 }) => string;
-export declare const VEO_EXTENSION_MODEL = "veo-3.1-generate-preview";
-export declare const VEO_SEGMENT_SEC = 8;
-export declare const veoExtensionCount: (requestedSec: number) => number;
-export declare const veoExtendedSeconds: (requestedSec: number) => number;
 export declare const getModelDuration: (provider: keyof typeof provider2MovieAgent, model: string, movieDuration?: number) => number | undefined;
 export type ModelPricing = {
     unit: "tokens" | "chars" | "seconds" | "images";

package/lib/types/provider2agent.js

@@ -383,20 +383,21 @@ export const provider2MovieAgent = {
     },
     google: {
         agentName: "movieGenAIAgent",
-        // veo-2.0 and veo-3.0 were shut down on 2026-06-30; the Gemini API now lists only the Veo 3.1 previews.
-        defaultModel: "veo-3.1-generate-preview",
-        models: ["veo-3.1-generate-preview", "veo-3.1-lite-generate-preview", "veo-3.1-generate-001"],
+        // The Veo 3.1 previews shut down on the Gemini API on 2026-10-22 (removed here ahead of that); Gemini Omni is their successor.
+        defaultModel: "gemini-omni-1.1-flash",
+        models: ["gemini-omni-1.1-flash", "veo-3.1-generate-001"],
         keyName: "GEMINI_API_KEY",
         modelParams: {
-            "veo-3.1-lite-generate-preview": {
-                durations: [4, 6, 8],
+            // Gemini API only. Longer beats are extended, up to 40 seconds (see gemini_omni_video.ts).
+            "gemini-omni-1.1-flash": {
+                durations: [3, 4, 5, 6, 7, 8, 9, 10],
                 supportsDuration: true,
                 supportsLastFrame: true,
                 supportsReferenceImages: false,
                 supportsPersonGeneration: false,
                 audio: { mode: AUDIO_MODE_ALWAYS },
             },
-            // Vertex AI only: the GA name of Veo 3.1 there, where the preview IDs are retired.
+            // Vertex AI only: the GA name of Veo 3.1 there.
             "veo-3.1-generate-001": {
                 durations: [4, 6, 8],
                 supportsDuration: true,
@@ -405,14 +406,6 @@ export const provider2MovieAgent = {
                 supportsPersonGeneration: false,
                 audio: { mode: AUDIO_MODE_ALWAYS },
             },
-            "veo-3.1-generate-preview": {
-                durations: [4, 6, 8],
-                supportsDuration: true,
-                supportsLastFrame: true,
-                supportsReferenceImages: true,
-                supportsPersonGeneration: false,
-                audio: { mode: AUDIO_MODE_ALWAYS },
-            },
         },
     },
     mock: {
@@ -585,15 +578,26 @@ export const getModelAudio = (provider, model) => {
     return modelParams?.[model]?.audio;
 };
 const GOOGLE_VERTEX_DEFAULT_MOVIE_MODEL = "veo-3.1-generate-001";
-// The default model is shared by the Gemini API and Vertex AI, but Vertex retired the Veo 3.1 preview IDs.
+const removedGoogleMovieModelHint = "Its Gemini API shutdown date is 2026-10-22, so mulmocast no longer offers it. Use 'gemini-omni-1.1-flash' on the Gemini API, or 'veo-3.1-generate-001' with movieParams.vertexai_project on Vertex AI (needed for referenceImages).";
+const removedGoogleMovieModelHints = {
+    "veo-3.1-generate-preview": removedGoogleMovieModelHint,
+    "veo-3.1-lite-generate-preview": removedGoogleMovieModelHint,
+};
+// An unknown model fails here with a hint, not with a TypeError in the duration lookup.
+export const unsupportedGoogleMovieModelMessage = (model) => {
+    if (Object.hasOwn(provider2MovieAgent.google.modelParams, model))
+        return null;
+    const hint = Object.hasOwn(removedGoogleMovieModelHints, model)
+        ? removedGoogleMovieModelHints[model]
+        : `Supported models: ${provider2MovieAgent.google.models.join(", ")}.`;
+    return `Google movie model "${model}" is not supported. ${hint}`;
+};
+// Gemini Omni is a Gemini API model here, so Vertex AI keeps Veo 3.1 as its default.
 export const defaultMovieModel = (provider, movieParams) => provider === "google" && movieParams?.vertexai_project ? GOOGLE_VERTEX_DEFAULT_MOVIE_MODEL : provider2MovieAgent[provider].defaultModel;
-// Veo 3.1 (preview) builds clips longer than one segment by extending an 8-second first segment 8 seconds at a time.
-export const VEO_EXTENSION_MODEL = "veo-3.1-generate-preview";
-export const VEO_SEGMENT_SEC = 8;
-export const veoExtensionCount = (requestedSec) => Math.ceil((requestedSec - VEO_SEGMENT_SEC) / VEO_SEGMENT_SEC);
-export const veoExtendedSeconds = (requestedSec) => VEO_SEGMENT_SEC * (1 + veoExtensionCount(requestedSec));
 export const getModelDuration = (provider, model, movieDuration) => {
     const modelParams = provider2MovieAgent[provider]?.modelParams;
+    if (!modelParams || !Object.hasOwn(modelParams, model))
+        return undefined;
     const { durations } = modelParams[model];
     if (durations && movieDuration) {
         const largerDurations = durations.filter((d) => d >= movieDuration);
@@ -656,12 +660,11 @@ export const modelPricing = {
         "gemini-2.5-flash-image": { unit: "images", inputPerMTokensUSD: 0.3, perImageUSD: 0.039, asOf: "2026-07-03" },
         // $0.0336 per 1K-resolution image
         "gemini-3.1-flash-lite-image": { unit: "images", inputPerMTokensUSD: 0.25, perImageUSD: 0.0336, asOf: "2026-10-07" },
-        // Veo per second of generated video (720p). veo-2.0-generate-001 and veo-3.0-generate-001
-        // were shut down on 2026-06-30, so they intentionally have no price.
-        "veo-3.1-generate-preview": { unit: "seconds", perSecUSD: 0.4, asOf: "2026-07-03" },
+        // Video per generated second at 720p. Shut-down models (veo-2.0, veo-3.0, the Veo 3.1 previews) intentionally have no price.
+        // Gemini Omni bills video output tokens at $17.50/1M, 5,792 tokens per second of 720p video.
+        "gemini-omni-1.1-flash": { unit: "seconds", perSecUSD: 0.1014, asOf: "2026-10-07" },
         // Vertex AI: https://cloud.google.com/vertex-ai/generative-ai/pricing (video + audio, 720p/1080p)
         "veo-3.1-generate-001": { unit: "seconds", perSecUSD: 0.4, asOf: "2026-10-07" },
-        "veo-3.1-lite-generate-preview": { unit: "seconds", perSecUSD: 0.05, asOf: "2026-07-03" },
         // Google Cloud Text-to-Speech per 1M characters, keyed by voice tier.
         // https://cloud.google.com/text-to-speech/pricing
         "tts-standard": { unit: "chars", perMCharsUSD: 4, asOf: "2026-07-03" },