mulmocast 2.8.0 → 2.9.0

package/lib/utils/context.d.ts

@@ -103,6 +103,20 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                     kind: "path";
                     path: string;
                 } | undefined;
+                references?: {
+                    name?: string | undefined;
+                    source?: {
+                        kind: "url";
+                        url: string;
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        kind: "path";
+                        path: string;
+                    } | undefined;
+                    label?: string | undefined;
+                }[] | undefined;
             }> | undefined;
             backgroundImage?: string | {
                 source: {
@@ -280,6 +294,7 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
             vertexai_location?: string | undefined;
             firstFrameImageName?: string | undefined;
             lastFrameImageName?: string | undefined;
+            frameFillColor?: string | undefined;
             referenceImages?: {
                 imageName: string;
                 referenceType: "ASSET" | "STYLE";
@@ -2315,6 +2330,7 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                 vertexai_location?: string | undefined;
                 firstFrameImageName?: string | undefined;
                 lastFrameImageName?: string | undefined;
+                frameFillColor?: string | undefined;
                 referenceImages?: {
                     imageName: string;
                     referenceType: "ASSET" | "STYLE";
@@ -2402,6 +2418,20 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                     kind: "path";
                     path: string;
                 } | undefined;
+                references?: {
+                    name?: string | undefined;
+                    source?: {
+                        kind: "url";
+                        url: string;
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        kind: "path";
+                        path: string;
+                    } | undefined;
+                    label?: string | undefined;
+                }[] | undefined;
             }> | undefined;
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
@@ -2639,6 +2669,20 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         kind: "path";
                         path: string;
                     } | undefined;
+                    references?: {
+                        name?: string | undefined;
+                        source?: {
+                            kind: "url";
+                            url: string;
+                        } | {
+                            kind: "base64";
+                            data: string;
+                        } | {
+                            kind: "path";
+                            path: string;
+                        } | undefined;
+                        label?: string | undefined;
+                    }[] | undefined;
                 }> | undefined;
                 backgroundImage?: string | {
                     source: {
@@ -2816,6 +2860,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 vertexai_location?: string | undefined;
                 firstFrameImageName?: string | undefined;
                 lastFrameImageName?: string | undefined;
+                frameFillColor?: string | undefined;
                 referenceImages?: {
                     imageName: string;
                     referenceType: "ASSET" | "STYLE";
@@ -4851,6 +4896,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     vertexai_location?: string | undefined;
                     firstFrameImageName?: string | undefined;
                     lastFrameImageName?: string | undefined;
+                    frameFillColor?: string | undefined;
                     referenceImages?: {
                         imageName: string;
                         referenceType: "ASSET" | "STYLE";
@@ -4938,6 +4984,20 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         kind: "path";
                         path: string;
                     } | undefined;
+                    references?: {
+                        name?: string | undefined;
+                        source?: {
+                            kind: "url";
+                            url: string;
+                        } | {
+                            kind: "base64";
+                            data: string;
+                        } | {
+                            kind: "path";
+                            path: string;
+                        } | undefined;
+                        label?: string | undefined;
+                    }[] | undefined;
                 }> | undefined;
                 imageNames?: string[] | undefined;
                 imagePrompt?: string | undefined;
@@ -5182,6 +5242,20 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     kind: "path";
                     path: string;
                 } | undefined;
+                references?: {
+                    name?: string | undefined;
+                    source?: {
+                        kind: "url";
+                        url: string;
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        kind: "path";
+                        path: string;
+                    } | undefined;
+                    label?: string | undefined;
+                }[] | undefined;
             }> | undefined;
             backgroundImage?: string | {
                 source: {
@@ -5359,6 +5433,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             vertexai_location?: string | undefined;
             firstFrameImageName?: string | undefined;
             lastFrameImageName?: string | undefined;
+            frameFillColor?: string | undefined;
             referenceImages?: {
                 imageName: string;
                 referenceType: "ASSET" | "STYLE";

package/lib/utils/error_cause.d.ts

@@ -55,6 +55,7 @@ export declare const apiRateLimitErrorType = "apiRateLimit";
 export declare const apiKeyMissingType = "apiKeyMissing";
 export declare const invalidResponseType = "invalidResponse";
 export declare const voiceLimitReachedType = "voice_limit_reached";
+export declare const invalidFrameSentinelType = "invalidFrameSentinel";
 export declare const movieAction = "movie";
 export declare const imageAction = "images";
 export declare const audioAction = "audio";
@@ -69,6 +70,10 @@ export declare const multiLingualFileTarget = "multiLingualFile";
 export declare const videoSourceTarget = "videoSource";
 export declare const audioSourceTarget = "audioSource";
 export declare const codeTextTarget = "codeText";
+export declare const bothFramesTarget = "bothFrames";
+export declare const missingBeatImageTarget = "missingBeatImage";
+export declare const unsupportedImageTypeTarget = "unsupportedImageType";
+export declare const missingFirstFrameTarget = "missingFirstFrame";
 export declare const agentFileNotExistError: (agentName: string, action: string, target: string, fileName: string, beatIndex?: number) => {
     beatIndex?: number | undefined;
     type: string;
@@ -115,6 +120,13 @@ export declare const invalidAudioSourceError: (beatIndex: number) => {
     agentName: string;
     beatIndex: number;
 };
+export declare const beatImageSentinelError: (beatIndex: number, target: string) => {
+    type: string;
+    action: string;
+    target: string;
+    agentName: string;
+    beatIndex: number;
+};
 export declare const downLoadReferenceImageError: (key: string, url: string) => {
     type: string;
     action: string;

package/lib/utils/error_cause.js

@@ -56,6 +56,7 @@ export const apiRateLimitErrorType = "apiRateLimit";
 export const apiKeyMissingType = "apiKeyMissing";
 export const invalidResponseType = "invalidResponse";
 export const voiceLimitReachedType = "voice_limit_reached";
+export const invalidFrameSentinelType = "invalidFrameSentinel";
 // Actions
 export const movieAction = "movie";
 export const imageAction = "images";
@@ -72,6 +73,10 @@ export const multiLingualFileTarget = "multiLingualFile";
 export const videoSourceTarget = "videoSource";
 export const audioSourceTarget = "audioSource";
 export const codeTextTarget = "codeText";
+export const bothFramesTarget = "bothFrames";
+export const missingBeatImageTarget = "missingBeatImage";
+export const unsupportedImageTypeTarget = "unsupportedImageType";
+export const missingFirstFrameTarget = "missingFirstFrame";
 // Agent File Not Exist Errors
 export const agentFileNotExistError = (agentName, action, target, fileName, beatIndex) => {
     return {
@@ -111,6 +116,16 @@ export const invalidAudioSourceError = (beatIndex) => {
         beatIndex,
     };
 };
+// $beatImage frame sentinel misuse (invalid combination or beat cannot supply its own image)
+export const beatImageSentinelError = (beatIndex, target) => {
+    return {
+        type: invalidFrameSentinelType,
+        action: imageAction,
+        target,
+        agentName: "imagePreprocessAgent",
+        beatIndex,
+    };
+};
 // 404
 export const downLoadReferenceImageError = (key, url) => {
     return {

package/lib/utils/ffmpeg_utils.d.ts

@@ -26,6 +26,11 @@ export declare const ffmpegGetMediaDuration: (filePath: string) => Promise<{
     duration: number;
     hasAudio: boolean;
 }>;
+export declare const ffmpegGetImageDimensions: (filePath: string) => Promise<{
+    width: number;
+    height: number;
+}>;
+export declare const padImageToCanvas: (srcPath: string, destPath: string, width: number, height: number, fillColor: string) => Promise<void>;
 export declare const extractImageFromMovie: (movieFile: string, imagePath: string, useLastFrame?: boolean) => Promise<object>;
 export declare const trimMusic: (inputFile: string, startTime: number, duration: number) => Promise<Buffer>;
 export declare const createSilentAudio: (filePath: string, durationSec: number) => Promise<void>;

package/lib/utils/ffmpeg_utils.js

@@ -117,6 +117,48 @@ export const ffmpegGetMediaDuration = (filePath) => {
         });
     });
 };
+export const ffmpegGetImageDimensions = (filePath) => {
+    return new Promise((resolve, reject) => {
+        ffmpeg.ffprobe(filePath, (err, metadata) => {
+            if (err) {
+                GraphAILogger.error("Error while getting image dimensions:", err);
+                reject(err);
+                return;
+            }
+            const stream = metadata.streams?.find((s) => s.codec_type === "video" && s.width && s.height);
+            if (!stream?.width || !stream?.height) {
+                reject(new Error(`ffmpegGetImageDimensions: no video stream found: ${filePath}`));
+                return;
+            }
+            resolve({ width: stream.width, height: stream.height });
+        });
+    });
+};
+// Aspect-fit the image into width x height, filling the extra space with fillColor (centered).
+export const padImageToCanvas = (srcPath, destPath, width, height, fillColor) => {
+    // Only allow hex colors and color names: fillColor is interpolated into the
+    // filtergraph, where ',' and ':' would allow injecting arbitrary filters.
+    if (!/^(#|0x)?[0-9a-fA-F]{6}([0-9a-fA-F]{2})?$/.test(fillColor) && !/^[a-zA-Z]+$/.test(fillColor)) {
+        return Promise.reject(new Error(`padImageToCanvas: invalid fill color: ${fillColor}`));
+    }
+    // ffmpeg color syntax: hex needs a 0x prefix ('#RRGGBB' and bare 'RRGGBB' both normalize)
+    const color = /^[0-9a-fA-F]{6}([0-9a-fA-F]{2})?$/.test(fillColor) ? `0x${fillColor}` : fillColor.replace(/^#/, "0x");
+    return new Promise((resolve, reject) => {
+        ffmpeg(srcPath)
+            .outputOptions([
+            "-frames:v",
+            "1",
+            "-update",
+            "1",
+            "-vf",
+            `scale=w=${width}:h=${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=${color}`,
+        ])
+            .output(destPath)
+            .on("end", () => resolve())
+            .on("error", (err) => reject(err))
+            .run();
+    });
+};
 export const extractImageFromMovie = (movieFile, imagePath, useLastFrame = false) => {
     return new Promise((resolve, reject) => {
         const command = ffmpeg(movieFile);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.8.0",
+  "version": "2.9.0",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -25,7 +25,7 @@
   },
   "resolutions": {
     "minimatch": "^10.2.5",
-    "tar": "7.5.19",
+    "tar": "7.5.20",
     "yauzl": "^3.4.0"
   },
   "bin": {
@@ -89,7 +89,7 @@
   "homepage": "https://github.com/receptron/mulmocast-cli#readme",
   "dependencies": {
     "@google-cloud/text-to-speech": "^6.4.1",
-    "@google/genai": "^2.10.0",
+    "@google/genai": "^2.12.0",
     "@graphai/anthropic_agent": "^2.0.12",
     "@graphai/browserless_agent": "^2.0.3",
     "@graphai/gemini_agent": "^2.0.5",
@@ -102,7 +102,7 @@
     "@inquirer/input": "^5.1.2",
     "@inquirer/select": "^5.2.1",
     "@modelcontextprotocol/sdk": "^1.29.0",
-    "@modernized/fluent-ffmpeg": "^1.0.1",
+    "@modernized/fluent-ffmpeg": "^1.0.2",
     "@mozilla/readability": "^0.6.0",
     "@mulmocast/deck": "^1.1.0",
     "@tavily/core": "^0.7.6",
@@ -112,8 +112,8 @@
     "graphai": "^2.0.18",
     "js-tiktoken": "^1.0.21",
     "jsdom": "^29.1.1",
-    "marked": "^18.0.5",
-    "mulmocast-vision": "^1.0.11",
+    "marked": "^18.0.6",
+    "mulmocast-vision": "^1.0.12",
     "ora": "^9.4.1",
     "puppeteer": "^25.3.0",
     "replicate": "^1.4.0",
@@ -132,12 +132,12 @@
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-import": "^2.32.0",
     "eslint-plugin-prettier": "^5.5.6",
-    "eslint-plugin-sonarjs": "^4.1.0",
+    "eslint-plugin-sonarjs": "^4.2.0",
     "globals": "^17.7.0",
     "prettier": "^3.9.5",
-    "tsx": "^4.23.0",
+    "tsx": "^4.23.1",
     "typescript": "6.0.3",
-    "typescript-eslint": "^8.63.0"
+    "typescript-eslint": "^8.64.0"
   },
   "engines": {
     "node": ">=22.0.0"