mulmocast 2.7.2 → 2.8.1

package/lib/actions/image_agents.d.ts

@@ -54,6 +54,7 @@ type ImagePluginPreprocessAgentResponse = ImagePreprocessAgentResponseBase & {
     markdown: string;
     html: string;
 };
+export declare const conformFrameImageToCanvas: (context: MulmoStudioContext, imageName: string, imagePath: string, fillColor: string) => Promise<string>;
 type ImagePreprocessAgentResponse = ImagePreprocessAgentResponseBase | ImageHtmlPreprocessAgentResponse | ImagePluginPreprocessAgentResponse | ImageOnlyMoviePreprocessAgentResponse | ImageGenearalPreprocessAgentResponse;
 export declare const imagePreprocessAgent: (namedInputs: {
     context: MulmoStudioContext;

package/lib/actions/image_agents.js

@@ -1,6 +1,9 @@
+import fs from "fs";
+import { createHash } from "crypto";
 import { GraphAILogger } from "graphai";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods, MulmoBeatMethods, MulmoMediaSourceMethods } from "../methods/index.js";
-import { getBeatPngImagePath, getBeatMoviePaths, getBeatAnimatedVideoPath, getAudioFilePath, getGroupedAudioFilePath } from "../utils/file.js";
+import { getBeatPngImagePath, getBeatMoviePaths, getBeatAnimatedVideoPath, getAudioFilePath, getGroupedAudioFilePath, getReferenceImagePath, } from "../utils/file.js";
+import { ffmpegGetImageDimensions, padImageToCanvas } from "../utils/ffmpeg_utils.js";
 import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
 import { renderHTMLToImage } from "../utils/html_render.js";
 import { beatId } from "../utils/utils.js";
@@ -11,6 +14,63 @@ const htmlStyle = (context, beat) => {
         textSlideStyle: MulmoPresentationStyleMethods.getTextSlideStyle(context.presentationStyle, beat),
     };
 };
+// Image-to-video models expect first/last frame images that match the video canvas.
+// Generated reference images come back at the provider's fixed sizes (e.g. gpt-image emits
+// 1536x1024), so aspect-fit pad them to the canvas before movie generation.
+const conformingInFlight = new Map();
+// Aspect-matching sources never get a destPath, so without this memo every beat
+// (and every rerun) would ffprobe them again. Keyed by path + mtime.
+const imageDimensionsCache = new Map();
+const getImageDimensionsCached = async (imagePath, mtimeMs) => {
+    const key = `${imagePath}:${mtimeMs}`;
+    const cached = imageDimensionsCache.get(key);
+    if (cached) {
+        return cached;
+    }
+    const dimensions = await ffmpegGetImageDimensions(imagePath);
+    imageDimensionsCache.set(key, dimensions);
+    return dimensions;
+};
+export const conformFrameImageToCanvas = async (context, imageName, imagePath, fillColor) => {
+    if (!fs.existsSync(imagePath)) {
+        return imagePath; // mock agents / dry runs
+    }
+    const canvasSize = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
+    // The cache key includes the source path and fill color: the same ref name can resolve
+    // to different sources in different beats, and a color change must invalidate the cache.
+    const digest = createHash("sha256").update(`${imagePath}|${fillColor}`).digest("hex").slice(0, 8);
+    const destPath = getReferenceImagePath(context, `${imageName}_fit_${canvasSize.width}x${canvasSize.height}_${digest}`, "png");
+    const inFlight = conformingInFlight.get(destPath);
+    if (inFlight) {
+        return inFlight;
+    }
+    const promise = (async () => {
+        const sourceMtimeMs = fs.statSync(imagePath).mtimeMs;
+        if (fs.existsSync(destPath) && fs.statSync(destPath).mtimeMs >= sourceMtimeMs) {
+            return destPath;
+        }
+        const { width, height } = await getImageDimensionsCached(imagePath, sourceMtimeMs);
+        if (Math.abs(width / height - canvasSize.width / canvasSize.height) < 0.01) {
+            return imagePath;
+        }
+        GraphAILogger.info(`conformFrameImageToCanvas: padding ${imageName} (${width}x${height}) to ${canvasSize.width}x${canvasSize.height}`);
+        try {
+            await padImageToCanvas(imagePath, destPath, canvasSize.width, canvasSize.height, fillColor);
+        }
+        catch (error) {
+            fs.rmSync(destPath, { force: true }); // don't let a partial file poison the mtime cache
+            throw error;
+        }
+        return destPath;
+    })();
+    conformingInFlight.set(destPath, promise);
+    try {
+        return await promise;
+    }
+    finally {
+        conformingInFlight.delete(destPath);
+    }
+};
 export const imagePreprocessAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs } = namedInputs;
     const studioBeat = context.studio.beats[index];
@@ -66,21 +126,24 @@ export const imagePreprocessAgent = async (namedInputs) => {
         }
     }
     returnValue.movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle, beat);
-    // Resolve movie reference images from imageRefs
-    const movieParams = beat.movieParams ?? context.presentationStyle.movieParams;
-    if (movieParams?.firstFrameImageName && imageRefs) {
+    // Resolve movie reference images from imageRefs.
+    // Shallow-merge like getMovieAgentInfo: beat-level fields override style-level fields
+    // individually, so a beat setting only lastFrameImageName keeps the global frameFillColor.
+    const movieParams = { ...context.presentationStyle.movieParams, ...beat.movieParams };
+    const frameFillColor = movieParams.frameFillColor ?? "black";
+    if (movieParams.firstFrameImageName && imageRefs) {
         const firstFramePath = imageRefs[movieParams.firstFrameImageName];
         if (firstFramePath) {
-            returnValue.firstFrameImagePath = firstFramePath;
+            returnValue.firstFrameImagePath = await conformFrameImageToCanvas(context, movieParams.firstFrameImageName, firstFramePath, frameFillColor);
         }
     }
-    if (movieParams?.lastFrameImageName && imageRefs) {
+    if (movieParams.lastFrameImageName && imageRefs) {
         const lastFramePath = imageRefs[movieParams.lastFrameImageName];
         if (lastFramePath) {
-            returnValue.lastFrameImagePath = lastFramePath;
+            returnValue.lastFrameImagePath = await conformFrameImageToCanvas(context, movieParams.lastFrameImageName, lastFramePath, frameFillColor);
         }
     }
-    if (movieParams?.referenceImages && imageRefs) {
+    if (movieParams.referenceImages && imageRefs) {
         returnValue.movieReferenceImages = movieParams.referenceImages
             .map((ref) => {
             const refPath = imageRefs[ref.imageName];

package/lib/agents/combine_audio_files_agent.js

@@ -2,7 +2,7 @@ import { assert, GraphAILogger } from "graphai";
 import { silent60secPath, isFile } from "../utils/file.js";
 import { FfmpegContextInit, FfmpegContextGenerateOutput, FfmpegContextInputFormattedAudio, ffmpegGetMediaDuration, } from "../utils/ffmpeg_utils.js";
 import { MulmoMediaSourceMethods } from "../methods/mulmo_media_source.js";
-import { MulmoBeatMethods } from "../methods/index.js";
+import { MulmoBeatMethods, MulmoPresentationStyleMethods } from "../methods/index.js";
 import { userAssert } from "../utils/utils.js";
 import { getAudioInputIdsError } from "../utils/error_cause.js";
 const getMovieDuration = async (context, beat) => {
@@ -126,7 +126,22 @@ const getVoiceOverGroup = (context, index) => {
 };
 const getSpillOverGroup = (context, mediaDurations, index) => {
     const group = [index];
-    for (let i = index + 1; i < context.studio.beats.length && !mediaDurations[i].hasMedia; i++) {
+    for (let i = index + 1; i < context.studio.beats.length; i++) {
+        const media = mediaDurations[i];
+        // An empty-text beat with only a moviePrompt (no generated audio requested) continues
+        // the preceding narration (spill-over). Generated movies don't exist yet at this point,
+        // so movieDuration === 0; imported movies and animated html_tailwind beats have
+        // movieDuration > 0 and keep their own timeline, ending the group as before.
+        const beat = context.studio.script.beats[i];
+        const isSilentContinuation = !beat.text?.trim() &&
+            media.audioDuration === 0 &&
+            media.movieDuration === 0 &&
+            Boolean(beat.moviePrompt) &&
+            !beat.soundEffectPrompt &&
+            !MulmoPresentationStyleMethods.generatedMovieHasAudio(context.presentationStyle, beat);
+        if (media.hasMedia && !isSilentContinuation) {
+            break;
+        }
         group.push(i);
     }
     return group;

package/lib/agents/movie_replicate_agent.js

@@ -81,11 +81,14 @@ async function generateMovie(model, apiKey, prompt, imagePath, lastFrameImagePat
     }
     // Add generate_audio if the model supports it
     const audio = provider2MovieAgent.replicate.modelParams[model].audio;
+    if (audio.mode === AUDIO_MODE_OPTIONAL) {
+        // Always send the flag: several models (e.g. seedance-2.0's generate_audio, p-video's
+        // save_audio) default it to true, which silently embeds generated audio.
+        // Silent unless explicitly requested.
+        input[audio.param] = generateAudio ?? false;
+    }
     if (generateAudio !== undefined) {
-        if (audio.mode === AUDIO_MODE_OPTIONAL) {
-            input[audio.param] = generateAudio;
-        }
-        else if (audio.mode === AUDIO_MODE_NEVER && generateAudio === true) {
+        if (audio.mode === AUDIO_MODE_NEVER && generateAudio === true) {
             throw new Error(`Model ${model} does not support audio generation`, {
                 cause: agentGenerationError("movieReplicateAgent", imageAction, unsupportedModelTarget),
             });

package/lib/methods/mulmo_presentation_style.d.ts

@@ -194,6 +194,7 @@ export declare const MulmoPresentationStyleMethods: {
             vertexai_location?: string | undefined;
             firstFrameImageName?: string | undefined;
             lastFrameImageName?: string | undefined;
+            frameFillColor?: string | undefined;
             referenceImages?: {
                 imageName: string;
                 referenceType: "ASSET" | "STYLE";
@@ -204,6 +205,7 @@ export declare const MulmoPresentationStyleMethods: {
         };
         keyName: string;
     };
+    generatedMovieHasAudio(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): boolean;
     getSoundEffectAgentInfo(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): {
         agentName: string;
         defaultModel: import("../types/provider2agent.js").ReplicateModel;

package/lib/methods/mulmo_presentation_style.js

@@ -7,7 +7,7 @@ import { isNull } from "graphai";
 import { userAssert } from "../utils/utils.js";
 import { slideThemes } from "../data/slideThemes.js";
 import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, mulmoTransitionSchema, } from "../types/schema.js";
-import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, } from "../types/provider2agent.js";
+import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, getModelAudio, AUDIO_MODE_ALWAYS, } from "../types/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; background-color:#fff; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
@@ -131,6 +131,17 @@ export const MulmoPresentationStyleMethods = {
             keyName: agentInfo.keyName,
         };
     },
+    // True when the movie generated for this beat will carry an audio track:
+    // the model always embeds audio, or audio generation is explicitly requested.
+    generatedMovieHasAudio(presentationStyle, beat) {
+        const movieParams = { ...presentationStyle.movieParams, ...beat.movieParams };
+        if (movieParams?.generateAudio === true) {
+            return true;
+        }
+        const provider = text2MovieProviderSchema.parse(movieParams?.provider ?? defaultProviders.text2movie);
+        const model = movieParams?.model ?? provider2MovieAgent[provider].defaultModel;
+        return getModelAudio(provider, model)?.mode === AUDIO_MODE_ALWAYS;
+    },
     getSoundEffectAgentInfo(presentationStyle, beat) {
         const soundEffectProvider = (beat.soundEffectParams?.provider ??
             presentationStyle.soundEffectParams?.provider ??

package/lib/types/provider2agent.d.ts

@@ -88,7 +88,7 @@ export type ReplicateModel = `${string}/${string}`;
 export declare const AUDIO_MODE_NEVER: "never";
 export declare const AUDIO_MODE_ALWAYS: "always";
 export declare const AUDIO_MODE_OPTIONAL: "optional";
-type MovieAudioSpec = {
+export type MovieAudioSpec = {
     mode: typeof AUDIO_MODE_NEVER;
 } | {
     mode: typeof AUDIO_MODE_ALWAYS;
@@ -213,6 +213,7 @@ export declare const defaultProviders: {
 export declare const llm: (keyof typeof provider2LLMAgent)[];
 export type LLM = keyof typeof provider2LLMAgent;
 export declare const htmlLLMProvider: string[];
+export declare const getModelAudio: (provider: keyof typeof provider2MovieAgent, model: string) => MovieAudioSpec | undefined;
 export declare const getModelDuration: (provider: keyof typeof provider2MovieAgent, model: string, movieDuration?: number) => number | undefined;
 export type ModelPricing = {
     unit: "tokens" | "chars" | "seconds" | "images";

package/lib/types/provider2agent.js

@@ -144,10 +144,12 @@ export const provider2MovieAgent = {
             "minimax/hailuo-2.3",
             "minimax/hailuo-2.3-fast",
             "pixverse/pixverse-v5",
+            "prunaai/p-video",
+            "xai/grok-imagine-video-1.5",
         ],
         modelParams: {
             "bytedance/seedance-1-lite": {
-                durations: [5, 10],
+                durations: [4, 5, 6, 7, 8, 9, 10, 11, 12],
                 start_image: "image",
                 last_image: "last_frame_image",
                 audio: { mode: AUDIO_MODE_NEVER },
@@ -259,10 +261,13 @@ export const provider2MovieAgent = {
                 price_per_sec: 0.12,
             },
             "wan-video/wan-2.2-i2v-fast": {
+                // No duration input: length is num_frames (default 81) / frames_per_second (default 16) ≈ 5s.
                 durations: [5],
                 start_image: "image",
+                start_image_required: true,
+                last_image: "last_image",
                 audio: { mode: AUDIO_MODE_NEVER },
-                price_per_sec: 0.012,
+                price_per_sec: 0.01, // $0.05 per 81-frame 480p video; up to $0.145 at 720p
             },
             "wan-video/wan-2.2-t2v-fast": {
                 durations: [5],
@@ -295,7 +300,7 @@ export const provider2MovieAgent = {
                 last_image: "end_image",
                 reference_images_param: "reference_images",
                 audio: { mode: AUDIO_MODE_OPTIONAL, param: "generate_audio" },
-                price_per_sec: 0.3,
+                price_per_sec: 0.28, // 'pro' (1080p, default); 'standard' $0.168, '4k' $0.42
             },
             "kwaivgi/kling-v3-video": {
                 durations: [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
@@ -335,6 +340,20 @@ export const provider2MovieAgent = {
                 audio: { mode: AUDIO_MODE_NEVER },
                 price_per_sec: 0.12,
             },
+            "prunaai/p-video": {
+                durations: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20],
+                start_image: "image",
+                last_image: "last_frame_image",
+                audio: { mode: AUDIO_MODE_OPTIONAL, param: "save_audio" },
+                price_per_sec: 0.02, // 720p, draft mode off ($0.04 at 1080p)
+            },
+            "xai/grok-imagine-video-1.5": {
+                durations: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
+                start_image: "image",
+                start_image_required: true,
+                audio: { mode: AUDIO_MODE_ALWAYS },
+                price_per_sec: 0.08,
+            },
         },
     },
     google: {
@@ -508,6 +527,10 @@ export const defaultProviders = {
 };
 export const llm = Object.keys(provider2LLMAgent);
 export const htmlLLMProvider = ["openai", "anthropic", "mock"];
+export const getModelAudio = (provider, model) => {
+    const modelParams = provider2MovieAgent[provider]?.modelParams;
+    return modelParams?.[model]?.audio;
+};
 export const getModelDuration = (provider, model, movieDuration) => {
     const modelParams = provider2MovieAgent[provider]?.modelParams;
     const { durations } = modelParams[model];

package/lib/types/schema.d.ts

@@ -4757,6 +4757,7 @@ export declare const mulmoMovieParamsSchema: z.ZodObject<{
     vertexai_location: z.ZodOptional<z.ZodString>;
     firstFrameImageName: z.ZodOptional<z.ZodString>;
     lastFrameImageName: z.ZodOptional<z.ZodString>;
+    frameFillColor: z.ZodOptional<z.ZodString>;
     referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
         imageName: z.ZodString;
         referenceType: z.ZodEnum<{
@@ -8719,6 +8720,7 @@ export declare const mulmoBeatSchema: z.ZodObject<{
         vertexai_location: z.ZodOptional<z.ZodString>;
         firstFrameImageName: z.ZodOptional<z.ZodString>;
         lastFrameImageName: z.ZodOptional<z.ZodString>;
+        frameFillColor: z.ZodOptional<z.ZodString>;
         referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
             imageName: z.ZodString;
             referenceType: z.ZodEnum<{
@@ -9209,6 +9211,7 @@ export declare const mulmoPresentationStyleSchema: z.ZodObject<{
         vertexai_location: z.ZodOptional<z.ZodString>;
         firstFrameImageName: z.ZodOptional<z.ZodString>;
         lastFrameImageName: z.ZodOptional<z.ZodString>;
+        frameFillColor: z.ZodOptional<z.ZodString>;
         referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
             imageName: z.ZodString;
             referenceType: z.ZodEnum<{
@@ -9721,6 +9724,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         vertexai_location: z.ZodOptional<z.ZodString>;
         firstFrameImageName: z.ZodOptional<z.ZodString>;
         lastFrameImageName: z.ZodOptional<z.ZodString>;
+        frameFillColor: z.ZodOptional<z.ZodString>;
         referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
             imageName: z.ZodString;
             referenceType: z.ZodEnum<{
@@ -13829,6 +13833,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
             vertexai_location: z.ZodOptional<z.ZodString>;
             firstFrameImageName: z.ZodOptional<z.ZodString>;
             lastFrameImageName: z.ZodOptional<z.ZodString>;
+            frameFillColor: z.ZodOptional<z.ZodString>;
             referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
                 imageName: z.ZodString;
                 referenceType: z.ZodEnum<{
@@ -14395,6 +14400,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             vertexai_location: z.ZodOptional<z.ZodString>;
             firstFrameImageName: z.ZodOptional<z.ZodString>;
             lastFrameImageName: z.ZodOptional<z.ZodString>;
+            frameFillColor: z.ZodOptional<z.ZodString>;
             referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
                 imageName: z.ZodString;
                 referenceType: z.ZodEnum<{
@@ -18503,6 +18509,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
                 vertexai_location: z.ZodOptional<z.ZodString>;
                 firstFrameImageName: z.ZodOptional<z.ZodString>;
                 lastFrameImageName: z.ZodOptional<z.ZodString>;
+                frameFillColor: z.ZodOptional<z.ZodString>;
                 referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
                     imageName: z.ZodString;
                     referenceType: z.ZodEnum<{
@@ -19004,6 +19011,7 @@ export declare const mulmoPromptTemplateSchema: z.ZodObject<{
             vertexai_location: z.ZodOptional<z.ZodString>;
             firstFrameImageName: z.ZodOptional<z.ZodString>;
             lastFrameImageName: z.ZodOptional<z.ZodString>;
+            frameFillColor: z.ZodOptional<z.ZodString>;
             referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
                 imageName: z.ZodString;
                 referenceType: z.ZodEnum<{
@@ -19510,6 +19518,7 @@ export declare const mulmoPromptTemplateFileSchema: z.ZodObject<{
             vertexai_location: z.ZodOptional<z.ZodString>;
             firstFrameImageName: z.ZodOptional<z.ZodString>;
             lastFrameImageName: z.ZodOptional<z.ZodString>;
+            frameFillColor: z.ZodOptional<z.ZodString>;
             referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
                 imageName: z.ZodString;
                 referenceType: z.ZodEnum<{

package/lib/types/schema.js

@@ -502,6 +502,11 @@ export const mulmoMovieParamsSchema = z.object({
     vertexai_location: z.string().optional(), // Vertex AI location (default: us-central1)
     firstFrameImageName: imageIdSchema.optional().describe("Reference an imageRefs key for the first frame (image-to-video input)"),
     lastFrameImageName: imageIdSchema.optional().describe("Reference an imageRefs key for the last frame (image-to-video interpolation)"),
+    frameFillColor: z
+        .string()
+        .regex(/^(#|0x)?[0-9a-fA-F]{6}([0-9a-fA-F]{2})?$|^[a-zA-Z]+$/, "must be a hex color ('#RRGGBB[AA]', '0x' or no prefix) or a color name")
+        .optional()
+        .describe("Fill color used when padding firstFrame/lastFrame reference images to the canvas aspect ratio (e.g. '#F7F6F4'). Default: black"),
     referenceImages: z
         .array(movieReferenceImageSchema)
         .optional()

package/lib/utils/context.d.ts

@@ -280,6 +280,7 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
             vertexai_location?: string | undefined;
             firstFrameImageName?: string | undefined;
             lastFrameImageName?: string | undefined;
+            frameFillColor?: string | undefined;
             referenceImages?: {
                 imageName: string;
                 referenceType: "ASSET" | "STYLE";
@@ -2315,6 +2316,7 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                 vertexai_location?: string | undefined;
                 firstFrameImageName?: string | undefined;
                 lastFrameImageName?: string | undefined;
+                frameFillColor?: string | undefined;
                 referenceImages?: {
                     imageName: string;
                     referenceType: "ASSET" | "STYLE";
@@ -2816,6 +2818,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 vertexai_location?: string | undefined;
                 firstFrameImageName?: string | undefined;
                 lastFrameImageName?: string | undefined;
+                frameFillColor?: string | undefined;
                 referenceImages?: {
                     imageName: string;
                     referenceType: "ASSET" | "STYLE";
@@ -4851,6 +4854,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     vertexai_location?: string | undefined;
                     firstFrameImageName?: string | undefined;
                     lastFrameImageName?: string | undefined;
+                    frameFillColor?: string | undefined;
                     referenceImages?: {
                         imageName: string;
                         referenceType: "ASSET" | "STYLE";
@@ -5359,6 +5363,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             vertexai_location?: string | undefined;
             firstFrameImageName?: string | undefined;
             lastFrameImageName?: string | undefined;
+            frameFillColor?: string | undefined;
             referenceImages?: {
                 imageName: string;
                 referenceType: "ASSET" | "STYLE";

package/lib/utils/ffmpeg_utils.d.ts

@@ -26,6 +26,11 @@ export declare const ffmpegGetMediaDuration: (filePath: string) => Promise<{
     duration: number;
     hasAudio: boolean;
 }>;
+export declare const ffmpegGetImageDimensions: (filePath: string) => Promise<{
+    width: number;
+    height: number;
+}>;
+export declare const padImageToCanvas: (srcPath: string, destPath: string, width: number, height: number, fillColor: string) => Promise<void>;
 export declare const extractImageFromMovie: (movieFile: string, imagePath: string, useLastFrame?: boolean) => Promise<object>;
 export declare const trimMusic: (inputFile: string, startTime: number, duration: number) => Promise<Buffer>;
 export declare const createSilentAudio: (filePath: string, durationSec: number) => Promise<void>;

package/lib/utils/ffmpeg_utils.js

@@ -117,6 +117,48 @@ export const ffmpegGetMediaDuration = (filePath) => {
         });
     });
 };
+export const ffmpegGetImageDimensions = (filePath) => {
+    return new Promise((resolve, reject) => {
+        ffmpeg.ffprobe(filePath, (err, metadata) => {
+            if (err) {
+                GraphAILogger.error("Error while getting image dimensions:", err);
+                reject(err);
+                return;
+            }
+            const stream = metadata.streams?.find((s) => s.codec_type === "video" && s.width && s.height);
+            if (!stream?.width || !stream?.height) {
+                reject(new Error(`ffmpegGetImageDimensions: no video stream found: ${filePath}`));
+                return;
+            }
+            resolve({ width: stream.width, height: stream.height });
+        });
+    });
+};
+// Aspect-fit the image into width x height, filling the extra space with fillColor (centered).
+export const padImageToCanvas = (srcPath, destPath, width, height, fillColor) => {
+    // Only allow hex colors and color names: fillColor is interpolated into the
+    // filtergraph, where ',' and ':' would allow injecting arbitrary filters.
+    if (!/^(#|0x)?[0-9a-fA-F]{6}([0-9a-fA-F]{2})?$/.test(fillColor) && !/^[a-zA-Z]+$/.test(fillColor)) {
+        return Promise.reject(new Error(`padImageToCanvas: invalid fill color: ${fillColor}`));
+    }
+    // ffmpeg color syntax: hex needs a 0x prefix ('#RRGGBB' and bare 'RRGGBB' both normalize)
+    const color = /^[0-9a-fA-F]{6}([0-9a-fA-F]{2})?$/.test(fillColor) ? `0x${fillColor}` : fillColor.replace(/^#/, "0x");
+    return new Promise((resolve, reject) => {
+        ffmpeg(srcPath)
+            .outputOptions([
+            "-frames:v",
+            "1",
+            "-update",
+            "1",
+            "-vf",
+            `scale=w=${width}:h=${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=${color}`,
+        ])
+            .output(destPath)
+            .on("end", () => resolve())
+            .on("error", (err) => reject(err))
+            .run();
+    });
+};
 export const extractImageFromMovie = (movieFile, imagePath, useLastFrame = false) => {
     return new Promise((resolve, reject) => {
         const command = ffmpeg(movieFile);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.7.2",
+  "version": "2.8.1",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -25,7 +25,7 @@
   },
   "resolutions": {
     "minimatch": "^10.2.5",
-    "tar": "7.5.19",
+    "tar": "7.5.20",
     "yauzl": "^3.4.0"
   },
   "bin": {
@@ -112,8 +112,8 @@
     "graphai": "^2.0.18",
     "js-tiktoken": "^1.0.21",
     "jsdom": "^29.1.1",
-    "marked": "^18.0.5",
-    "mulmocast-vision": "^1.0.11",
+    "marked": "^18.0.6",
+    "mulmocast-vision": "^1.0.12",
     "ora": "^9.4.1",
     "puppeteer": "^25.3.0",
     "replicate": "^1.4.0",
@@ -132,12 +132,12 @@
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-import": "^2.32.0",
     "eslint-plugin-prettier": "^5.5.6",
-    "eslint-plugin-sonarjs": "^4.1.0",
+    "eslint-plugin-sonarjs": "^4.2.0",
     "globals": "^17.7.0",
     "prettier": "^3.9.5",
-    "tsx": "^4.23.0",
+    "tsx": "^4.23.1",
     "typescript": "6.0.3",
-    "typescript-eslint": "^8.63.0"
+    "typescript-eslint": "^8.64.0"
   },
   "engines": {
     "node": ">=22.0.0"