mulmocast 2.8.0 → 2.8.1

package/lib/actions/image_agents.d.ts

@@ -54,6 +54,7 @@ type ImagePluginPreprocessAgentResponse = ImagePreprocessAgentResponseBase & {
     markdown: string;
     html: string;
 };
+export declare const conformFrameImageToCanvas: (context: MulmoStudioContext, imageName: string, imagePath: string, fillColor: string) => Promise<string>;
 type ImagePreprocessAgentResponse = ImagePreprocessAgentResponseBase | ImageHtmlPreprocessAgentResponse | ImagePluginPreprocessAgentResponse | ImageOnlyMoviePreprocessAgentResponse | ImageGenearalPreprocessAgentResponse;
 export declare const imagePreprocessAgent: (namedInputs: {
     context: MulmoStudioContext;

package/lib/actions/image_agents.js

@@ -1,6 +1,9 @@
+import fs from "fs";
+import { createHash } from "crypto";
 import { GraphAILogger } from "graphai";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods, MulmoBeatMethods, MulmoMediaSourceMethods } from "../methods/index.js";
-import { getBeatPngImagePath, getBeatMoviePaths, getBeatAnimatedVideoPath, getAudioFilePath, getGroupedAudioFilePath } from "../utils/file.js";
+import { getBeatPngImagePath, getBeatMoviePaths, getBeatAnimatedVideoPath, getAudioFilePath, getGroupedAudioFilePath, getReferenceImagePath, } from "../utils/file.js";
+import { ffmpegGetImageDimensions, padImageToCanvas } from "../utils/ffmpeg_utils.js";
 import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
 import { renderHTMLToImage } from "../utils/html_render.js";
 import { beatId } from "../utils/utils.js";
@@ -11,6 +14,63 @@ const htmlStyle = (context, beat) => {
         textSlideStyle: MulmoPresentationStyleMethods.getTextSlideStyle(context.presentationStyle, beat),
     };
 };
+// Image-to-video models expect first/last frame images that match the video canvas.
+// Generated reference images come back at the provider's fixed sizes (e.g. gpt-image emits
+// 1536x1024), so aspect-fit pad them to the canvas before movie generation.
+const conformingInFlight = new Map();
+// Aspect-matching sources never get a destPath, so without this memo every beat
+// (and every rerun) would ffprobe them again. Keyed by path + mtime.
+const imageDimensionsCache = new Map();
+const getImageDimensionsCached = async (imagePath, mtimeMs) => {
+    const key = `${imagePath}:${mtimeMs}`;
+    const cached = imageDimensionsCache.get(key);
+    if (cached) {
+        return cached;
+    }
+    const dimensions = await ffmpegGetImageDimensions(imagePath);
+    imageDimensionsCache.set(key, dimensions);
+    return dimensions;
+};
+export const conformFrameImageToCanvas = async (context, imageName, imagePath, fillColor) => {
+    if (!fs.existsSync(imagePath)) {
+        return imagePath; // mock agents / dry runs
+    }
+    const canvasSize = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
+    // The cache key includes the source path and fill color: the same ref name can resolve
+    // to different sources in different beats, and a color change must invalidate the cache.
+    const digest = createHash("sha256").update(`${imagePath}|${fillColor}`).digest("hex").slice(0, 8);
+    const destPath = getReferenceImagePath(context, `${imageName}_fit_${canvasSize.width}x${canvasSize.height}_${digest}`, "png");
+    const inFlight = conformingInFlight.get(destPath);
+    if (inFlight) {
+        return inFlight;
+    }
+    const promise = (async () => {
+        const sourceMtimeMs = fs.statSync(imagePath).mtimeMs;
+        if (fs.existsSync(destPath) && fs.statSync(destPath).mtimeMs >= sourceMtimeMs) {
+            return destPath;
+        }
+        const { width, height } = await getImageDimensionsCached(imagePath, sourceMtimeMs);
+        if (Math.abs(width / height - canvasSize.width / canvasSize.height) < 0.01) {
+            return imagePath;
+        }
+        GraphAILogger.info(`conformFrameImageToCanvas: padding ${imageName} (${width}x${height}) to ${canvasSize.width}x${canvasSize.height}`);
+        try {
+            await padImageToCanvas(imagePath, destPath, canvasSize.width, canvasSize.height, fillColor);
+        }
+        catch (error) {
+            fs.rmSync(destPath, { force: true }); // don't let a partial file poison the mtime cache
+            throw error;
+        }
+        return destPath;
+    })();
+    conformingInFlight.set(destPath, promise);
+    try {
+        return await promise;
+    }
+    finally {
+        conformingInFlight.delete(destPath);
+    }
+};
 export const imagePreprocessAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs } = namedInputs;
     const studioBeat = context.studio.beats[index];
@@ -66,21 +126,24 @@ export const imagePreprocessAgent = async (namedInputs) => {
         }
     }
     returnValue.movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle, beat);
-    // Resolve movie reference images from imageRefs
-    const movieParams = beat.movieParams ?? context.presentationStyle.movieParams;
-    if (movieParams?.firstFrameImageName && imageRefs) {
+    // Resolve movie reference images from imageRefs.
+    // Shallow-merge like getMovieAgentInfo: beat-level fields override style-level fields
+    // individually, so a beat setting only lastFrameImageName keeps the global frameFillColor.
+    const movieParams = { ...context.presentationStyle.movieParams, ...beat.movieParams };
+    const frameFillColor = movieParams.frameFillColor ?? "black";
+    if (movieParams.firstFrameImageName && imageRefs) {
         const firstFramePath = imageRefs[movieParams.firstFrameImageName];
         if (firstFramePath) {
-            returnValue.firstFrameImagePath = firstFramePath;
+            returnValue.firstFrameImagePath = await conformFrameImageToCanvas(context, movieParams.firstFrameImageName, firstFramePath, frameFillColor);
         }
     }
-    if (movieParams?.lastFrameImageName && imageRefs) {
+    if (movieParams.lastFrameImageName && imageRefs) {
         const lastFramePath = imageRefs[movieParams.lastFrameImageName];
         if (lastFramePath) {
-            returnValue.lastFrameImagePath = lastFramePath;
+            returnValue.lastFrameImagePath = await conformFrameImageToCanvas(context, movieParams.lastFrameImageName, lastFramePath, frameFillColor);
         }
     }
-    if (movieParams?.referenceImages && imageRefs) {
+    if (movieParams.referenceImages && imageRefs) {
         returnValue.movieReferenceImages = movieParams.referenceImages
             .map((ref) => {
             const refPath = imageRefs[ref.imageName];

package/lib/methods/mulmo_presentation_style.d.ts

@@ -194,6 +194,7 @@ export declare const MulmoPresentationStyleMethods: {
             vertexai_location?: string | undefined;
             firstFrameImageName?: string | undefined;
             lastFrameImageName?: string | undefined;
+            frameFillColor?: string | undefined;
             referenceImages?: {
                 imageName: string;
                 referenceType: "ASSET" | "STYLE";

package/lib/types/schema.d.ts

@@ -4757,6 +4757,7 @@ export declare const mulmoMovieParamsSchema: z.ZodObject<{
     vertexai_location: z.ZodOptional<z.ZodString>;
     firstFrameImageName: z.ZodOptional<z.ZodString>;
     lastFrameImageName: z.ZodOptional<z.ZodString>;
+    frameFillColor: z.ZodOptional<z.ZodString>;
     referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
         imageName: z.ZodString;
         referenceType: z.ZodEnum<{
@@ -8719,6 +8720,7 @@ export declare const mulmoBeatSchema: z.ZodObject<{
         vertexai_location: z.ZodOptional<z.ZodString>;
         firstFrameImageName: z.ZodOptional<z.ZodString>;
         lastFrameImageName: z.ZodOptional<z.ZodString>;
+        frameFillColor: z.ZodOptional<z.ZodString>;
         referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
             imageName: z.ZodString;
             referenceType: z.ZodEnum<{
@@ -9209,6 +9211,7 @@ export declare const mulmoPresentationStyleSchema: z.ZodObject<{
         vertexai_location: z.ZodOptional<z.ZodString>;
         firstFrameImageName: z.ZodOptional<z.ZodString>;
         lastFrameImageName: z.ZodOptional<z.ZodString>;
+        frameFillColor: z.ZodOptional<z.ZodString>;
         referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
             imageName: z.ZodString;
             referenceType: z.ZodEnum<{
@@ -9721,6 +9724,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         vertexai_location: z.ZodOptional<z.ZodString>;
         firstFrameImageName: z.ZodOptional<z.ZodString>;
         lastFrameImageName: z.ZodOptional<z.ZodString>;
+        frameFillColor: z.ZodOptional<z.ZodString>;
         referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
             imageName: z.ZodString;
             referenceType: z.ZodEnum<{
@@ -13829,6 +13833,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
             vertexai_location: z.ZodOptional<z.ZodString>;
             firstFrameImageName: z.ZodOptional<z.ZodString>;
             lastFrameImageName: z.ZodOptional<z.ZodString>;
+            frameFillColor: z.ZodOptional<z.ZodString>;
             referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
                 imageName: z.ZodString;
                 referenceType: z.ZodEnum<{
@@ -14395,6 +14400,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             vertexai_location: z.ZodOptional<z.ZodString>;
             firstFrameImageName: z.ZodOptional<z.ZodString>;
             lastFrameImageName: z.ZodOptional<z.ZodString>;
+            frameFillColor: z.ZodOptional<z.ZodString>;
             referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
                 imageName: z.ZodString;
                 referenceType: z.ZodEnum<{
@@ -18503,6 +18509,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
                 vertexai_location: z.ZodOptional<z.ZodString>;
                 firstFrameImageName: z.ZodOptional<z.ZodString>;
                 lastFrameImageName: z.ZodOptional<z.ZodString>;
+                frameFillColor: z.ZodOptional<z.ZodString>;
                 referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
                     imageName: z.ZodString;
                     referenceType: z.ZodEnum<{
@@ -19004,6 +19011,7 @@ export declare const mulmoPromptTemplateSchema: z.ZodObject<{
             vertexai_location: z.ZodOptional<z.ZodString>;
             firstFrameImageName: z.ZodOptional<z.ZodString>;
             lastFrameImageName: z.ZodOptional<z.ZodString>;
+            frameFillColor: z.ZodOptional<z.ZodString>;
             referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
                 imageName: z.ZodString;
                 referenceType: z.ZodEnum<{
@@ -19510,6 +19518,7 @@ export declare const mulmoPromptTemplateFileSchema: z.ZodObject<{
             vertexai_location: z.ZodOptional<z.ZodString>;
             firstFrameImageName: z.ZodOptional<z.ZodString>;
             lastFrameImageName: z.ZodOptional<z.ZodString>;
+            frameFillColor: z.ZodOptional<z.ZodString>;
             referenceImages: z.ZodOptional<z.ZodArray<z.ZodObject<{
                 imageName: z.ZodString;
                 referenceType: z.ZodEnum<{

package/lib/types/schema.js

@@ -502,6 +502,11 @@ export const mulmoMovieParamsSchema = z.object({
     vertexai_location: z.string().optional(), // Vertex AI location (default: us-central1)
     firstFrameImageName: imageIdSchema.optional().describe("Reference an imageRefs key for the first frame (image-to-video input)"),
     lastFrameImageName: imageIdSchema.optional().describe("Reference an imageRefs key for the last frame (image-to-video interpolation)"),
+    frameFillColor: z
+        .string()
+        .regex(/^(#|0x)?[0-9a-fA-F]{6}([0-9a-fA-F]{2})?$|^[a-zA-Z]+$/, "must be a hex color ('#RRGGBB[AA]', '0x' or no prefix) or a color name")
+        .optional()
+        .describe("Fill color used when padding firstFrame/lastFrame reference images to the canvas aspect ratio (e.g. '#F7F6F4'). Default: black"),
     referenceImages: z
         .array(movieReferenceImageSchema)
         .optional()

package/lib/utils/context.d.ts

@@ -280,6 +280,7 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
             vertexai_location?: string | undefined;
             firstFrameImageName?: string | undefined;
             lastFrameImageName?: string | undefined;
+            frameFillColor?: string | undefined;
             referenceImages?: {
                 imageName: string;
                 referenceType: "ASSET" | "STYLE";
@@ -2315,6 +2316,7 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                 vertexai_location?: string | undefined;
                 firstFrameImageName?: string | undefined;
                 lastFrameImageName?: string | undefined;
+                frameFillColor?: string | undefined;
                 referenceImages?: {
                     imageName: string;
                     referenceType: "ASSET" | "STYLE";
@@ -2816,6 +2818,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 vertexai_location?: string | undefined;
                 firstFrameImageName?: string | undefined;
                 lastFrameImageName?: string | undefined;
+                frameFillColor?: string | undefined;
                 referenceImages?: {
                     imageName: string;
                     referenceType: "ASSET" | "STYLE";
@@ -4851,6 +4854,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     vertexai_location?: string | undefined;
                     firstFrameImageName?: string | undefined;
                     lastFrameImageName?: string | undefined;
+                    frameFillColor?: string | undefined;
                     referenceImages?: {
                         imageName: string;
                         referenceType: "ASSET" | "STYLE";
@@ -5359,6 +5363,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             vertexai_location?: string | undefined;
             firstFrameImageName?: string | undefined;
             lastFrameImageName?: string | undefined;
+            frameFillColor?: string | undefined;
             referenceImages?: {
                 imageName: string;
                 referenceType: "ASSET" | "STYLE";

package/lib/utils/ffmpeg_utils.d.ts

@@ -26,6 +26,11 @@ export declare const ffmpegGetMediaDuration: (filePath: string) => Promise<{
     duration: number;
     hasAudio: boolean;
 }>;
+export declare const ffmpegGetImageDimensions: (filePath: string) => Promise<{
+    width: number;
+    height: number;
+}>;
+export declare const padImageToCanvas: (srcPath: string, destPath: string, width: number, height: number, fillColor: string) => Promise<void>;
 export declare const extractImageFromMovie: (movieFile: string, imagePath: string, useLastFrame?: boolean) => Promise<object>;
 export declare const trimMusic: (inputFile: string, startTime: number, duration: number) => Promise<Buffer>;
 export declare const createSilentAudio: (filePath: string, durationSec: number) => Promise<void>;

package/lib/utils/ffmpeg_utils.js

@@ -117,6 +117,48 @@ export const ffmpegGetMediaDuration = (filePath) => {
         });
     });
 };
+export const ffmpegGetImageDimensions = (filePath) => {
+    return new Promise((resolve, reject) => {
+        ffmpeg.ffprobe(filePath, (err, metadata) => {
+            if (err) {
+                GraphAILogger.error("Error while getting image dimensions:", err);
+                reject(err);
+                return;
+            }
+            const stream = metadata.streams?.find((s) => s.codec_type === "video" && s.width && s.height);
+            if (!stream?.width || !stream?.height) {
+                reject(new Error(`ffmpegGetImageDimensions: no video stream found: ${filePath}`));
+                return;
+            }
+            resolve({ width: stream.width, height: stream.height });
+        });
+    });
+};
+// Aspect-fit the image into width x height, filling the extra space with fillColor (centered).
+export const padImageToCanvas = (srcPath, destPath, width, height, fillColor) => {
+    // Only allow hex colors and color names: fillColor is interpolated into the
+    // filtergraph, where ',' and ':' would allow injecting arbitrary filters.
+    if (!/^(#|0x)?[0-9a-fA-F]{6}([0-9a-fA-F]{2})?$/.test(fillColor) && !/^[a-zA-Z]+$/.test(fillColor)) {
+        return Promise.reject(new Error(`padImageToCanvas: invalid fill color: ${fillColor}`));
+    }
+    // ffmpeg color syntax: hex needs a 0x prefix ('#RRGGBB' and bare 'RRGGBB' both normalize)
+    const color = /^[0-9a-fA-F]{6}([0-9a-fA-F]{2})?$/.test(fillColor) ? `0x${fillColor}` : fillColor.replace(/^#/, "0x");
+    return new Promise((resolve, reject) => {
+        ffmpeg(srcPath)
+            .outputOptions([
+            "-frames:v",
+            "1",
+            "-update",
+            "1",
+            "-vf",
+            `scale=w=${width}:h=${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=${color}`,
+        ])
+            .output(destPath)
+            .on("end", () => resolve())
+            .on("error", (err) => reject(err))
+            .run();
+    });
+};
 export const extractImageFromMovie = (movieFile, imagePath, useLastFrame = false) => {
     return new Promise((resolve, reject) => {
         const command = ffmpeg(movieFile);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.8.0",
+  "version": "2.8.1",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -25,7 +25,7 @@
   },
   "resolutions": {
     "minimatch": "^10.2.5",
-    "tar": "7.5.19",
+    "tar": "7.5.20",
     "yauzl": "^3.4.0"
   },
   "bin": {
@@ -112,8 +112,8 @@
     "graphai": "^2.0.18",
     "js-tiktoken": "^1.0.21",
     "jsdom": "^29.1.1",
-    "marked": "^18.0.5",
-    "mulmocast-vision": "^1.0.11",
+    "marked": "^18.0.6",
+    "mulmocast-vision": "^1.0.12",
     "ora": "^9.4.1",
     "puppeteer": "^25.3.0",
     "replicate": "^1.4.0",
@@ -132,12 +132,12 @@
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-import": "^2.32.0",
     "eslint-plugin-prettier": "^5.5.6",
-    "eslint-plugin-sonarjs": "^4.1.0",
+    "eslint-plugin-sonarjs": "^4.2.0",
     "globals": "^17.7.0",
     "prettier": "^3.9.5",
-    "tsx": "^4.23.0",
+    "tsx": "^4.23.1",
     "typescript": "6.0.3",
-    "typescript-eslint": "^8.63.0"
+    "typescript-eslint": "^8.64.0"
   },
   "engines": {
     "node": ">=22.0.0"