mulmocast 2.8.1 → 2.9.1

package/lib/methods/mulmo_studio_context.d.ts

@@ -3,7 +3,7 @@
  * (No Node.js built-ins like fs, path, dotenv, etc.)
  * Works in both Node.js and modern browsers.
  */
-import { BeatSessionType, MulmoStudioContext, SessionProgressCallback, SessionType, MulmoBeat, SpeechOptions } from "../types/index.js";
+import { BeatSessionType, MulmoStudioContext, SessionProgressCallback, SessionType, MulmoBeat, MulmoStudioBeat, SpeechOptions } from "../types/index.js";
 import { provider2TTSAgent } from "../types/provider2agent.js";
 export declare const addSessionProgressCallback: (cb: SessionProgressCallback) => void;
 export declare const removeSessionProgressCallback: (cb: SessionProgressCallback) => void;
@@ -18,6 +18,11 @@ export declare const MulmoStudioContextMethods: {
     setBeatSessionState(context: MulmoStudioContext, sessionType: BeatSessionType | undefined, index: number, id: string | undefined, value: boolean): void;
     needTranslate(context: MulmoStudioContext, includeCaption?: boolean): boolean | "" | undefined;
     getIntroPadding(context: MulmoStudioContext): number;
+    getExtraPadding(context: MulmoStudioContext, index: number): number;
+    getBeatDuration(context: MulmoStudioContext, index: number): number;
+    getVoiceOverGroupDuration(context: MulmoStudioContext, index: number): number;
+    isMovieBeat(context: MulmoStudioContext, studioBeat: MulmoStudioBeat, beat: MulmoBeat): boolean;
+    getSegmentDuration(context: MulmoStudioContext, index: number): number;
     getAudioParam(context: MulmoStudioContext, beat: MulmoBeat, lang?: string): {
         provider: keyof typeof provider2TTSAgent;
         voiceId: string;

package/lib/methods/mulmo_studio_context.js

@@ -3,10 +3,11 @@
  * (No Node.js built-ins like fs, path, dotenv, etc.)
  * Works in both Node.js and modern browsers.
  */
-import { text2SpeechProviderSchema, } from "../types/index.js";
+import { ImageMediaType, text2SpeechProviderSchema, } from "../types/index.js";
 import { beatId } from "../utils/utils.js";
 import { GraphAILogger } from "graphai";
 import { MulmoPresentationStyleMethods } from "./mulmo_presentation_style.js";
+import { MulmoBeatMethods } from "./mulmo_beat.js";
 const sessionProgressCallbacks = new Set();
 export const addSessionProgressCallback = (cb) => {
     sessionProgressCallbacks.add(cb);
@@ -92,6 +93,40 @@ export const MulmoStudioContextMethods = {
         }
         return context.presentationStyle.audioParams.introPadding;
     },
+    // Intro/outro padding is the only padding not already folded into studio.beats[].duration.
+    getExtraPadding(context, index) {
+        if (index === 0) {
+            return MulmoStudioContextMethods.getIntroPadding(context);
+        }
+        else if (index === context.studio.beats.length - 1) {
+            return context.presentationStyle.audioParams.outroPadding;
+        }
+        return 0;
+    },
+    // The duration this beat occupies on the audio timeline.
+    getBeatDuration(context, index) {
+        return (context.studio.beats[index]?.duration ?? 0) + MulmoStudioContextMethods.getExtraPadding(context, index);
+    },
+    // The total duration of the voice_over beats which follow (and share the shot of) this beat.
+    getVoiceOverGroupDuration(context, index) {
+        const trailingBeats = context.studio.script.beats.slice(index + 1);
+        const groupEnd = trailingBeats.findIndex((beat) => !MulmoBeatMethods.isVoiceOver(beat));
+        const groupSize = groupEnd < 0 ? trailingBeats.length : groupEnd;
+        return trailingBeats.slice(0, groupSize).reduce((total, _, offset) => total + MulmoStudioContextMethods.getBeatDuration(context, index + 1 + offset), 0);
+    },
+    // Whether this beat's source is a movie (as opposed to a still which has to be looped).
+    isMovieBeat(context, studioBeat, beat) {
+        return !!(studioBeat.lipSyncFile ||
+            studioBeat.movieFile ||
+            MulmoPresentationStyleMethods.getImageType(context.presentationStyle, beat) === ImageMediaType.Movie);
+    },
+    // How long this beat's video segment is: its own duration plus the trailing voice_over beats which
+    // share its shot (they have no video segment of their own), never shorter than its movie. This is
+    // the single rule for a segment's length -- createVideo builds from it and transitions clamp against it.
+    getSegmentDuration(context, index) {
+        const ownDuration = MulmoStudioContextMethods.getBeatDuration(context, index) + MulmoStudioContextMethods.getVoiceOverGroupDuration(context, index);
+        return Math.max(ownDuration, context.studio.beats[index]?.movieDuration ?? 0);
+    },
     getAudioParam(context, beat, lang) {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(context, beat, lang);
         const speechOptions = { ...speaker.speechOptions, ...beat.speechOptions };

package/lib/types/provider2agent.d.ts

@@ -64,6 +64,7 @@ export declare const provider2ImageAgent: {
         models: string[];
         keyName: string;
         baseURLKeyName: string;
+        maxReferenceImages: number;
     };
     google: {
         agentName: string;
@@ -74,6 +75,9 @@ export declare const provider2ImageAgent: {
     replicate: {
         agentName: string;
         defaultModel: string;
+        imageModelParams: Record<string, {
+            maxReferenceImages?: number;
+        }>;
         models: string[];
         keyName: string;
     };
@@ -213,6 +217,17 @@ export declare const defaultProviders: {
 export declare const llm: (keyof typeof provider2LLMAgent)[];
 export type LLM = keyof typeof provider2LLMAgent;
 export declare const htmlLLMProvider: string[];
+export declare const ProviderName: {
+    readonly OpenAI: "openai";
+    readonly Google: "google";
+    readonly Gemini: "gemini";
+    readonly Anthropic: "anthropic";
+    readonly Replicate: "replicate";
+    readonly ElevenLabs: "elevenlabs";
+    readonly Mock: "mock";
+};
+export declare const isImageProvider: (provider: string | undefined) => provider is keyof typeof provider2ImageAgent;
+export declare const getMaxImageReferenceImages: (provider: string | undefined, model: string) => number | undefined;
 export declare const getModelAudio: (provider: keyof typeof provider2MovieAgent, model: string) => MovieAudioSpec | undefined;
 export declare const getModelDuration: (provider: keyof typeof provider2MovieAgent, model: string, movieDuration?: number) => number | undefined;
 export type ModelPricing = {

package/lib/types/provider2agent.js

@@ -64,6 +64,10 @@ export const deprecatedGoogleImageModelHints = {
 // See https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image
 // and https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-1-flash-image
 export const vertexAIGlobalOnlyImageModels = new Set(["gemini-3-pro-image-preview", "gemini-3.1-flash-image-preview"]);
+// Per-model reference image limits (image_input array). Only verified entries; unlisted models are not truncated.
+const replicateImageModelParams = {
+    "bytedance/seedream-4": { maxReferenceImages: 10 },
+};
 export const provider2ImageAgent = {
     openai: {
         agentName: "imageOpenaiAgent",
@@ -71,6 +75,7 @@ export const provider2ImageAgent = {
         models: [...gptImages],
         keyName: "OPENAI_API_KEY",
         baseURLKeyName: "OPENAI_BASE_URL",
+        maxReferenceImages: 16, // images.edit accepts up to 16 input images
     },
     google: {
         agentName: "imageGenAIAgent",
@@ -81,6 +86,7 @@ export const provider2ImageAgent = {
     replicate: {
         agentName: "imageReplicateAgent",
         defaultModel: "bytedance/seedream-4",
+        imageModelParams: replicateImageModelParams,
         models: [
             "bytedance/seedream-4",
             "qwen/qwen-image",
@@ -527,6 +533,31 @@ export const defaultProviders = {
 };
 export const llm = Object.keys(provider2LLMAgent);
 export const htmlLLMProvider = ["openai", "anthropic", "mock"];
+// Provider name literals. Compare via these identifiers (e.g. provider === ProviderName.Replicate)
+// instead of raw strings.
+export const ProviderName = {
+    OpenAI: "openai",
+    Google: "google",
+    Gemini: "gemini",
+    Anthropic: "anthropic",
+    Replicate: "replicate",
+    ElevenLabs: "elevenlabs",
+    Mock: "mock",
+};
+export const isImageProvider = (provider) => {
+    return provider !== undefined && provider in provider2ImageAgent;
+};
+// Max reference images an image model accepts as input; undefined means no known limit (no truncation).
+export const getMaxImageReferenceImages = (provider, model) => {
+    if (!isImageProvider(provider)) {
+        return undefined;
+    }
+    if (provider === ProviderName.Replicate) {
+        return provider2ImageAgent.replicate.imageModelParams[model]?.maxReferenceImages;
+    }
+    const agentInfo = provider2ImageAgent[provider];
+    return "maxReferenceImages" in agentInfo ? agentInfo.maxReferenceImages : undefined;
+};
 export const getModelAudio = (provider, model) => {
     const modelParams = provider2MovieAgent[provider]?.modelParams;
     return modelParams?.[model]?.audio;

package/lib/types/schema.d.ts

@@ -263,6 +263,15 @@ export declare const mulmoMarkdownMediaSchema: z.ZodObject<{
         opacity: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>]>>>;
 }, z.core.$strict>;
+export declare const ImageMediaType: {
+    readonly Image: "image";
+    readonly ImagePrompt: "imagePrompt";
+    readonly Movie: "movie";
+    readonly MoviePrompt: "moviePrompt";
+    readonly Beat: "beat";
+    readonly VoiceOver: "voice_over";
+};
+export declare const beatImageTypesWithoutOwnStill: readonly string[];
 export declare const mulmoImageMediaSchema: z.ZodObject<{
     type: z.ZodLiteral<"image">;
     source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -4135,6 +4144,21 @@ export declare const mulmoAudioAssetSchema: z.ZodUnion<readonly [z.ZodObject<{
     type: z.ZodLiteral<"midi">;
     source: z.ZodString;
 }, z.core.$strict>]>;
+export declare const BEAT_IMAGE_SENTINEL = "$beatImage";
+export declare const mulmoImageReferenceSchema: z.ZodObject<{
+    name: z.ZodOptional<z.ZodString>;
+    source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"url">;
+        url: z.ZodURL;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"base64">;
+        data: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"path">;
+        path: z.ZodString;
+    }, z.core.$strict>], "kind">>;
+    label: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
 export declare const mulmoImagePromptMediaSchema: z.ZodObject<{
     type: z.ZodLiteral<"imagePrompt">;
     prompt: z.ZodString;
@@ -4153,6 +4177,20 @@ export declare const mulmoImagePromptMediaSchema: z.ZodObject<{
         kind: z.ZodLiteral<"path">;
         path: z.ZodString;
     }, z.core.$strict>], "kind">>;
+    references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodOptional<z.ZodString>;
+        source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"url">;
+            url: z.ZodURL;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"base64">;
+            data: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"path">;
+            path: z.ZodString;
+        }, z.core.$strict>], "kind">>;
+        label: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>>;
 }, z.core.$strict>;
 export declare const mulmoImageParamsImagesValueSchema: z.ZodUnion<readonly [z.ZodObject<{
     type: z.ZodLiteral<"image">;
@@ -4184,6 +4222,20 @@ export declare const mulmoImageParamsImagesValueSchema: z.ZodUnion<readonly [z.Z
         kind: z.ZodLiteral<"path">;
         path: z.ZodString;
     }, z.core.$strict>], "kind">>;
+    references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodOptional<z.ZodString>;
+        source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"url">;
+            url: z.ZodURL;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"base64">;
+            data: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"path">;
+            path: z.ZodString;
+        }, z.core.$strict>], "kind">>;
+        label: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>>;
 }, z.core.$strict>, z.ZodObject<{
     type: z.ZodLiteral<"movie">;
     source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -4231,6 +4283,20 @@ export declare const mulmoImageParamsImagesSchema: z.ZodRecord<z.ZodString, z.Zo
         kind: z.ZodLiteral<"path">;
         path: z.ZodString;
     }, z.core.$strict>], "kind">>;
+    references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodOptional<z.ZodString>;
+        source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"url">;
+            url: z.ZodURL;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"base64">;
+            data: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"path">;
+            path: z.ZodString;
+        }, z.core.$strict>], "kind">>;
+        label: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>>;
 }, z.core.$strict>, z.ZodObject<{
     type: z.ZodLiteral<"movie">;
     source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -4330,6 +4396,20 @@ export declare const mulmoImageParamsSchema: z.ZodObject<{
             kind: z.ZodLiteral<"path">;
             path: z.ZodString;
         }, z.core.$strict>], "kind">>;
+        references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodOptional<z.ZodString>;
+            source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"url">;
+                url: z.ZodURL;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"base64">;
+                data: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"path">;
+                path: z.ZodString;
+            }, z.core.$strict>], "kind">>;
+            label: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>>;
     }, z.core.$strict>, z.ZodObject<{
         type: z.ZodLiteral<"movie">;
         source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -8800,6 +8880,20 @@ export declare const mulmoBeatSchema: z.ZodObject<{
             kind: z.ZodLiteral<"path">;
             path: z.ZodString;
         }, z.core.$strict>], "kind">>;
+        references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodOptional<z.ZodString>;
+            source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"url">;
+                url: z.ZodURL;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"base64">;
+                data: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"path">;
+                path: z.ZodString;
+            }, z.core.$strict>], "kind">>;
+            label: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>>>;
     }, z.core.$strict>, z.ZodObject<{
         type: z.ZodLiteral<"movie">;
         source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -8942,6 +9036,20 @@ export declare const mulmoPresentationStyleSchema: z.ZodObject<{
                 kind: z.ZodLiteral<"path">;
                 path: z.ZodString;
             }, z.core.$strict>], "kind">>;
+            references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                name: z.ZodOptional<z.ZodString>;
+                source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"url">;
+                    url: z.ZodURL;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"base64">;
+                    data: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"path">;
+                    path: z.ZodString;
+                }, z.core.$strict>], "kind">>;
+                label: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
         }, z.core.$strict>, z.ZodObject<{
             type: z.ZodLiteral<"movie">;
             source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -9455,6 +9563,20 @@ export declare const mulmoScriptSchema: z.ZodObject<{
                 kind: z.ZodLiteral<"path">;
                 path: z.ZodString;
             }, z.core.$strict>], "kind">>;
+            references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                name: z.ZodOptional<z.ZodString>;
+                source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"url">;
+                    url: z.ZodURL;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"base64">;
+                    data: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"path">;
+                    path: z.ZodString;
+                }, z.core.$strict>], "kind">>;
+                label: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
         }, z.core.$strict>, z.ZodObject<{
             type: z.ZodLiteral<"movie">;
             source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -13913,6 +14035,20 @@ export declare const mulmoScriptSchema: z.ZodObject<{
                 kind: z.ZodLiteral<"path">;
                 path: z.ZodString;
             }, z.core.$strict>], "kind">>;
+            references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                name: z.ZodOptional<z.ZodString>;
+                source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"url">;
+                    url: z.ZodURL;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"base64">;
+                    data: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"path">;
+                    path: z.ZodString;
+                }, z.core.$strict>], "kind">>;
+                label: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>>;
         }, z.core.$strict>, z.ZodObject<{
             type: z.ZodLiteral<"movie">;
             source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -14131,6 +14267,20 @@ export declare const mulmoStudioSchema: z.ZodObject<{
                     kind: z.ZodLiteral<"path">;
                     path: z.ZodString;
                 }, z.core.$strict>], "kind">>;
+                references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    name: z.ZodOptional<z.ZodString>;
+                    source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                        kind: z.ZodLiteral<"url">;
+                        url: z.ZodURL;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"base64">;
+                        data: z.ZodString;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"path">;
+                        path: z.ZodString;
+                    }, z.core.$strict>], "kind">>;
+                    label: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>>;
             }, z.core.$strict>, z.ZodObject<{
                 type: z.ZodLiteral<"movie">;
                 source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -18589,6 +18739,20 @@ export declare const mulmoStudioSchema: z.ZodObject<{
                     kind: z.ZodLiteral<"path">;
                     path: z.ZodString;
                 }, z.core.$strict>], "kind">>;
+                references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    name: z.ZodOptional<z.ZodString>;
+                    source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                        kind: z.ZodLiteral<"url">;
+                        url: z.ZodURL;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"base64">;
+                        data: z.ZodString;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"path">;
+                        path: z.ZodString;
+                    }, z.core.$strict>], "kind">>;
+                    label: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>>;
             }, z.core.$strict>, z.ZodObject<{
                 type: z.ZodLiteral<"movie">;
                 source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -18742,6 +18906,20 @@ export declare const mulmoPromptTemplateSchema: z.ZodObject<{
                     kind: z.ZodLiteral<"path">;
                     path: z.ZodString;
                 }, z.core.$strict>], "kind">>;
+                references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    name: z.ZodOptional<z.ZodString>;
+                    source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                        kind: z.ZodLiteral<"url">;
+                        url: z.ZodURL;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"base64">;
+                        data: z.ZodString;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"path">;
+                        path: z.ZodString;
+                    }, z.core.$strict>], "kind">>;
+                    label: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>>;
             }, z.core.$strict>, z.ZodObject<{
                 type: z.ZodLiteral<"movie">;
                 source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -19249,6 +19427,20 @@ export declare const mulmoPromptTemplateFileSchema: z.ZodObject<{
                     kind: z.ZodLiteral<"path">;
                     path: z.ZodString;
                 }, z.core.$strict>], "kind">>;
+                references: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    name: z.ZodOptional<z.ZodString>;
+                    source: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                        kind: z.ZodLiteral<"url">;
+                        url: z.ZodURL;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"base64">;
+                        data: z.ZodString;
+                    }, z.core.$strict>, z.ZodObject<{
+                        kind: z.ZodLiteral<"path">;
+                        path: z.ZodString;
+                    }, z.core.$strict>], "kind">>;
+                    label: z.ZodOptional<z.ZodString>;
+                }, z.core.$strict>>>;
             }, z.core.$strict>, z.ZodObject<{
                 type: z.ZodLiteral<"movie">;
                 source: z.ZodDiscriminatedUnion<[z.ZodObject<{

package/lib/types/schema.js

@@ -130,9 +130,22 @@ const mulmoPdfMediaSchema = z
     source: mediaSourceSchema,
 })
     .strict();
+// Media type literals shared by imageParams.images entries and beat images. Compare via
+// these identifiers (e.g. entry.type === ImageMediaType.ImagePrompt) instead of raw strings.
+export const ImageMediaType = {
+    Image: "image",
+    ImagePrompt: "imagePrompt",
+    Movie: "movie",
+    MoviePrompt: "moviePrompt",
+    Beat: "beat",
+    VoiceOver: "voice_over",
+};
+// Beat image types whose still derives from the movie or another beat — they cannot supply
+// their own still for the $beatImage frame sentinel.
+export const beatImageTypesWithoutOwnStill = [ImageMediaType.Movie, ImageMediaType.Beat, ImageMediaType.VoiceOver];
 export const mulmoImageMediaSchema = z
     .object({
-    type: z.literal("image"),
+    type: z.literal(ImageMediaType.Image),
     source: mediaSourceSchema,
 })
     .strict();
@@ -144,13 +157,13 @@ const mulmoSvgMediaSchema = z
     .strict();
 export const mulmoMovieMediaSchema = z
     .object({
-    type: z.literal("movie"),
+    type: z.literal(ImageMediaType.Movie),
     source: mediaSourceSchema,
 })
     .strict();
 export const mulmoMoviePromptMediaSchema = z
     .object({
-    type: z.literal("moviePrompt"),
+    type: z.literal(ImageMediaType.MoviePrompt),
     prompt: z.string().min(1),
     imageName: z.string().optional().describe("Reference an imageRefs key to use as image-to-video input"),
 })
@@ -292,13 +305,13 @@ export const mulmoHtmlTailwindMediaSchema = z
     .strict();
 export const mulmoBeatReferenceMediaSchema = z
     .object({
-    type: z.literal("beat"),
+    type: z.literal(ImageMediaType.Beat),
     id: z.string().optional().describe("Specifies the beat to reference."),
 })
     .strict();
 export const mulmoVoiceOverMediaSchema = z
     .object({
-    type: z.literal("voice_over"),
+    type: z.literal(ImageMediaType.VoiceOver),
     startAt: z.number().optional().describe("The time to start the voice over the video in seconds."),
 })
     .strict();
@@ -339,13 +352,26 @@ const mulmoMidiMediaSchema = z
     .strict();
 export const mulmoAudioAssetSchema = z.union([mulmoAudioMediaSchema, mulmoMidiMediaSchema]);
 const imageIdSchema = z.string();
+// Sentinel for movieParams.firstFrameImageName / lastFrameImageName: "this beat's own generated image".
+export const BEAT_IMAGE_SENTINEL = "$beatImage";
+// imageRefs keys may not start with "$": that prefix is reserved for sentinels like $beatImage.
+const imageRefKeySchema = imageIdSchema.refine((key) => !key.startsWith("$"), { message: 'image keys must not start with the reserved prefix "$"' });
+export const mulmoImageReferenceSchema = z
+    .object({
+    name: imageIdSchema.optional().describe("An imageRefs key to use as a reference image"),
+    source: mediaSourceSchema.optional().describe("Direct source (path/url) to use as a reference image"),
+    label: z.string().optional().describe("Role of this reference (e.g. 'the firefighter Travis Kane'), injected into the prompt as a preamble"),
+})
+    .strict()
+    .refine((ref) => (ref.name === undefined) !== (ref.source === undefined), { message: "specify exactly one of name or source" });
 export const mulmoImagePromptMediaSchema = z
     .object({
-    type: z.literal("imagePrompt"),
+    type: z.literal(ImageMediaType.ImagePrompt),
     prompt: z.string().min(1),
     canvasSize: z.object({ width: z.number(), height: z.number() }).strict().optional(),
-    referenceImageName: imageIdSchema.optional().describe("Reference another imageRefs key as input for image generation"),
-    referenceImage: mediaSourceSchema.optional().describe("Direct source (path/url/base64) as reference image for generation"),
+    referenceImageName: imageIdSchema.optional().describe("Reference another imageRefs key as input for image generation. Sugar for references[0].name"),
+    referenceImage: mediaSourceSchema.optional().describe("Direct source (path/url/base64) as reference image for generation. Sugar for references[0].source"),
+    references: z.array(mulmoImageReferenceSchema).optional().describe("Reference images (by imageRefs key or direct source) with optional role labels"),
 })
     .strict();
 export const mulmoImageParamsImagesValueSchema = z.union([
@@ -354,7 +380,7 @@ export const mulmoImageParamsImagesValueSchema = z.union([
     mulmoMovieMediaSchema,
     mulmoMoviePromptMediaSchema,
 ]);
-export const mulmoImageParamsImagesSchema = z.record(imageIdSchema, mulmoImageParamsImagesValueSchema);
+export const mulmoImageParamsImagesSchema = z.record(imageRefKeySchema, mulmoImageParamsImagesValueSchema);
 export const mulmoFillOptionSchema = z
     .object({
     style: z.enum(["aspectFit", "aspectFill"]).optional().default("aspectFit"),
@@ -500,8 +526,12 @@ export const mulmoMovieParamsSchema = z.object({
     filters: z.array(mulmoVideoFilterSchema).optional(), // for movie.ts
     vertexai_project: z.string().optional(), // Google Cloud Project ID for Vertex AI
     vertexai_location: z.string().optional(), // Vertex AI location (default: us-central1)
-    firstFrameImageName: imageIdSchema.optional().describe("Reference an imageRefs key for the first frame (image-to-video input)"),
-    lastFrameImageName: imageIdSchema.optional().describe("Reference an imageRefs key for the last frame (image-to-video interpolation)"),
+    firstFrameImageName: imageIdSchema
+        .optional()
+        .describe("Reference an imageRefs key for the first frame (image-to-video input), or '$beatImage' for the beat's own generated image"),
+    lastFrameImageName: imageIdSchema
+        .optional()
+        .describe("Reference an imageRefs key for the last frame (image-to-video interpolation), or '$beatImage' for the beat's own generated image"),
     frameFillColor: z
         .string()
         .regex(/^(#|0x)?[0-9a-fA-F]{6}([0-9a-fA-F]{2})?$|^[a-zA-Z]+$/, "must be a hex color ('#RRGGBB[AA]', '0x' or no prefix) or a color name")
@@ -540,7 +570,7 @@ export const mulmoBeatSchema = z
     images: mulmoImageParamsImagesSchema
         .optional()
         .describe("Beat-local media references. Same schema as imageParams.images. Merged with global refs (local takes precedence)."),
-    imageNames: z.array(imageIdSchema).optional(), // list of image names to use for image generation. The default is all images in the imageParams.images.
+    imageNames: z.array(imageRefKeySchema).optional(), // list of image names to use for image generation. The default is all images in the imageParams.images.
     imagePrompt: z.string().optional(),
     moviePrompt: z.string().optional(),
     soundEffectPrompt: z.string().optional(),

package/lib/types/type.d.ts

@@ -1,6 +1,6 @@
 import { type CallbackFunction } from "graphai";
 import type { UsageCollectorAPI } from "./usage.js";
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoMovieMediaSchema, mulmoMoviePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoImageReferenceSchema, mulmoMovieMediaSchema, mulmoMoviePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "./const.js";
 import type { LLM } from "./provider2agent.js";
 import { z } from "zod";
@@ -49,6 +49,7 @@ export type MulmoGoogleImageModel = z.infer<typeof mulmoGoogleImageModelSchema>;
 export type MulmoGoogleMovieModel = z.infer<typeof mulmoGoogleMovieModelSchema>;
 export type MulmoReplicateMovieModel = z.infer<typeof mulmoReplicateMovieModelSchema>;
 export type MulmoImagePromptMedia = z.infer<typeof mulmoImagePromptMediaSchema>;
+export type MulmoImageReference = z.infer<typeof mulmoImageReferenceSchema>;
 export type MulmoMoviePromptMedia = z.infer<typeof mulmoMoviePromptMediaSchema>;
 export type MulmoMarkdownLayout = z.infer<typeof markdownLayoutSchema>;
 export type MulmoRow2 = z.infer<typeof row2Schema>;