mulmocast 1.2.17 → 1.2.19

package/lib/actions/images.js

@@ -190,7 +190,8 @@ const beat_graph_data = {
                     return { hasMovieAudio: true };
                 }
                 const sourceFile = namedInputs.movieFile || namedInputs.imageFile;
-                if (!sourceFile || !fs.existsSync(sourceFile)) {
+                if (!sourceFile) {
+                    // no need to check if the file exists (ffmpegGetMediaDuration will check it if it is local file)
                     return { hasMovieAudio: false };
                 }
                 const { hasAudio } = await ffmpegGetMediaDuration(sourceFile);

package/lib/actions/translate.js

@@ -68,10 +68,9 @@ const beatGraph = {
         // for cache
         multiLingual: {
             agent: (namedInputs) => {
-                const { multiLinguals, beatIndex, text, beat } = namedInputs;
-                const key = beatId(beat?.id, beatIndex);
+                const { multiLinguals, beatIndex, text } = namedInputs;
                 const cacheKey = hashSHA256(text ?? "");
-                const multiLingual = multiLinguals?.[key];
+                const multiLingual = multiLinguals?.[beatIndex];
                 if (!multiLingual) {
                     return { cacheKey, multiLingualTexts: {} };
                 }
@@ -87,7 +86,6 @@ const beatGraph = {
             },
             inputs: {
                 text: ":beat.text",
-                beat: ":beat",
                 beatIndex: ":__mapIndex",
                 multiLinguals: ":context.multiLingual",
             },
@@ -199,14 +197,17 @@ const localizedTextCacheAgentFilter = async (context, next) => {
     }
     // same language
     if (targetLang === lang) {
+        GraphAILogger.log(`translate: ${beatIndex} same lang`);
         return { text: beat.text };
     }
     // The original text is unchanged and the target language text is present
     if (multiLingual.cacheKey === multiLingual.multiLingualTexts[targetLang]?.cacheKey) {
+        GraphAILogger.log(`translate: ${beatIndex} cache hit`);
         return { text: multiLingual.multiLingualTexts[targetLang].text };
     }
     try {
         MulmoStudioContextMethods.setBeatSessionState(mulmoContext, "multiLingual", beatIndex, beat.id, true);
+        GraphAILogger.log(`translate: ${beatIndex} run`);
         return await next(context);
     }
     finally {

package/lib/agents/index.d.ts

@@ -13,8 +13,9 @@ import ttsGoogleAgent from "./tts_google_agent.js";
 import validateSchemaAgent from "./validate_schema_agent.js";
 import soundEffectReplicateAgent from "./sound_effect_replicate_agent.js";
 import lipSyncReplicateAgent from "./lipsync_replicate_agent.js";
+import puppeteerCrawlerAgent from "./puppeteer_crawler_agent.js";
 import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };

package/lib/agents/index.js

@@ -13,9 +13,10 @@ import ttsGoogleAgent from "./tts_google_agent.js";
 import validateSchemaAgent from "./validate_schema_agent.js";
 import soundEffectReplicateAgent from "./sound_effect_replicate_agent.js";
 import lipSyncReplicateAgent from "./lipsync_replicate_agent.js";
+import puppeteerCrawlerAgent from "./puppeteer_crawler_agent.js";
 import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 // import * as vanilla from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };

package/lib/methods/mulmo_beat.d.ts

@@ -1,6 +1,6 @@
 import { MulmoBeat } from "../types/index.js";
 export declare const MulmoBeatMethods: {
     getHtmlPrompt(beat: MulmoBeat): string | undefined;
-    getPlugin(beat: MulmoBeat): typeof import("../utils/image_plugins/text_slide.js") | typeof import("../utils/image_plugins/markdown.js") | typeof import("../utils/image_plugins/chart.js") | typeof import("../utils/image_plugins/mermaid.js") | typeof import("../utils/image_plugins/html_tailwind.js") | typeof import("../utils/image_plugins/image.js") | typeof import("../utils/image_plugins/movie.js") | typeof import("../utils/image_plugins/beat.js") | typeof import("../utils/image_plugins/voice_over.js");
+    getPlugin(beat: MulmoBeat): typeof import("../utils/image_plugins/text_slide.js") | typeof import("../utils/image_plugins/markdown.js") | typeof import("../utils/image_plugins/chart.js") | typeof import("../utils/image_plugins/mermaid.js") | typeof import("../utils/image_plugins/html_tailwind.js") | typeof import("../utils/image_plugins/image.js") | typeof import("../utils/image_plugins/movie.js") | typeof import("../utils/image_plugins/beat.js") | typeof import("../utils/image_plugins/voice_over.js") | typeof import("../utils/image_plugins/vision.js");
     getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
 };

package/lib/types/schema.d.ts

@@ -465,6 +465,19 @@ export declare const mulmoVoiceOverMediaSchema: z.ZodObject<{
     type: "voice_over";
     startAt?: number | undefined;
 }>;
+export declare const mulmoVisionMediaSchema: z.ZodObject<{
+    type: z.ZodLiteral<"vision">;
+    name: z.ZodString;
+    data: z.ZodRecord<z.ZodString, z.ZodAny>;
+}, "strict", z.ZodTypeAny, {
+    type: "vision";
+    data: Record<string, any>;
+    name: string;
+}, {
+    type: "vision";
+    data: Record<string, any>;
+    name: string;
+}>;
 export declare const mulmoImageAssetSchema: z.ZodUnion<[z.ZodObject<{
     type: z.ZodLiteral<"markdown">;
     markdown: z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>;
@@ -902,6 +915,18 @@ export declare const mulmoImageAssetSchema: z.ZodUnion<[z.ZodObject<{
 }, {
     type: "voice_over";
     startAt?: number | undefined;
+}>, z.ZodObject<{
+    type: z.ZodLiteral<"vision">;
+    name: z.ZodString;
+    data: z.ZodRecord<z.ZodString, z.ZodAny>;
+}, "strict", z.ZodTypeAny, {
+    type: "vision";
+    data: Record<string, any>;
+    name: string;
+}, {
+    type: "vision";
+    data: Record<string, any>;
+    name: string;
 }>]>;
 export declare const mulmoAudioAssetSchema: z.ZodUnion<[z.ZodObject<{
     type: z.ZodLiteral<"audio">;
@@ -1830,6 +1855,18 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     }, {
         type: "voice_over";
         startAt?: number | undefined;
+    }>, z.ZodObject<{
+        type: z.ZodLiteral<"vision">;
+        name: z.ZodString;
+        data: z.ZodRecord<z.ZodString, z.ZodAny>;
+    }, "strict", z.ZodTypeAny, {
+        type: "vision";
+        data: Record<string, any>;
+        name: string;
+    }, {
+        type: "vision";
+        data: Record<string, any>;
+        name: string;
     }>]>>;
     audio: z.ZodOptional<z.ZodUnion<[z.ZodObject<{
         type: z.ZodLiteral<"audio">;
@@ -2263,6 +2300,10 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     } | {
         type: "voice_over";
         startAt?: number | undefined;
+    } | {
+        type: "vision";
+        data: Record<string, any>;
+        name: string;
     } | undefined;
     audio?: {
         type: "audio";
@@ -2462,6 +2503,10 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     } | {
         type: "voice_over";
         startAt?: number | undefined;
+    } | {
+        type: "vision";
+        data: Record<string, any>;
+        name: string;
     } | undefined;
     audio?: {
         type: "audio";
@@ -4310,6 +4355,18 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         }, {
             type: "voice_over";
             startAt?: number | undefined;
+        }>, z.ZodObject<{
+            type: z.ZodLiteral<"vision">;
+            name: z.ZodString;
+            data: z.ZodRecord<z.ZodString, z.ZodAny>;
+        }, "strict", z.ZodTypeAny, {
+            type: "vision";
+            data: Record<string, any>;
+            name: string;
+        }, {
+            type: "vision";
+            data: Record<string, any>;
+            name: string;
         }>]>>;
         audio: z.ZodOptional<z.ZodUnion<[z.ZodObject<{
             type: z.ZodLiteral<"audio">;
@@ -4743,6 +4800,10 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         } | {
             type: "voice_over";
             startAt?: number | undefined;
+        } | {
+            type: "vision";
+            data: Record<string, any>;
+            name: string;
         } | undefined;
         audio?: {
             type: "audio";
@@ -4942,6 +5003,10 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         } | {
             type: "voice_over";
             startAt?: number | undefined;
+        } | {
+            type: "vision";
+            data: Record<string, any>;
+            name: string;
         } | undefined;
         audio?: {
             type: "audio";
@@ -5243,6 +5308,10 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         } | {
             type: "voice_over";
             startAt?: number | undefined;
+        } | {
+            type: "vision";
+            data: Record<string, any>;
+            name: string;
         } | undefined;
         audio?: {
             type: "audio";
@@ -5474,6 +5543,10 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         } | {
             type: "voice_over";
             startAt?: number | undefined;
+        } | {
+            type: "vision";
+            data: Record<string, any>;
+            name: string;
         } | undefined;
         audio?: {
             type: "audio";
@@ -6983,6 +7056,18 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             }, {
                 type: "voice_over";
                 startAt?: number | undefined;
+            }>, z.ZodObject<{
+                type: z.ZodLiteral<"vision">;
+                name: z.ZodString;
+                data: z.ZodRecord<z.ZodString, z.ZodAny>;
+            }, "strict", z.ZodTypeAny, {
+                type: "vision";
+                data: Record<string, any>;
+                name: string;
+            }, {
+                type: "vision";
+                data: Record<string, any>;
+                name: string;
             }>]>>;
             audio: z.ZodOptional<z.ZodUnion<[z.ZodObject<{
                 type: z.ZodLiteral<"audio">;
@@ -7416,6 +7501,10 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             } | {
                 type: "voice_over";
                 startAt?: number | undefined;
+            } | {
+                type: "vision";
+                data: Record<string, any>;
+                name: string;
             } | undefined;
             audio?: {
                 type: "audio";
@@ -7615,6 +7704,10 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             } | {
                 type: "voice_over";
                 startAt?: number | undefined;
+            } | {
+                type: "vision";
+                data: Record<string, any>;
+                name: string;
             } | undefined;
             audio?: {
                 type: "audio";
@@ -7916,6 +8009,10 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             } | {
                 type: "voice_over";
                 startAt?: number | undefined;
+            } | {
+                type: "vision";
+                data: Record<string, any>;
+                name: string;
             } | undefined;
             audio?: {
                 type: "audio";
@@ -8147,6 +8244,10 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             } | {
                 type: "voice_over";
                 startAt?: number | undefined;
+            } | {
+                type: "vision";
+                data: Record<string, any>;
+                name: string;
             } | undefined;
             audio?: {
                 type: "audio";
@@ -8631,6 +8732,10 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             } | {
                 type: "voice_over";
                 startAt?: number | undefined;
+            } | {
+                type: "vision";
+                data: Record<string, any>;
+                name: string;
             } | undefined;
             audio?: {
                 type: "audio";
@@ -8882,6 +8987,10 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             } | {
                 type: "voice_over";
                 startAt?: number | undefined;
+            } | {
+                type: "vision";
+                data: Record<string, any>;
+                name: string;
             } | undefined;
             audio?: {
                 type: "audio";

package/lib/types/schema.js

@@ -130,6 +130,13 @@ export const mulmoVoiceOverMediaSchema = z
     startAt: z.number().optional().describe("The time to start the voice over the video in seconds."),
 })
     .strict();
+export const mulmoVisionMediaSchema = z
+    .object({
+    type: z.literal("vision"),
+    name: z.string(),
+    data: z.record(z.string(), z.any()),
+})
+    .strict();
 export const mulmoImageAssetSchema = z.union([
     mulmoMarkdownMediaSchema,
     mulmoWebMediaSchema,
@@ -143,6 +150,7 @@ export const mulmoImageAssetSchema = z.union([
     mulmoHtmlTailwindMediaSchema,
     mulmoBeatReferenceMediaSchema,
     mulmoVoiceOverMediaSchema,
+    mulmoVisionMediaSchema,
 ]);
 const mulmoAudioMediaSchema = z
     .object({

package/lib/utils/context.d.ts

@@ -219,6 +219,10 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
             } | {
                 type: "voice_over";
                 startAt?: number | undefined;
+            } | {
+                type: "vision";
+                data: Record<string, any>;
+                name: string;
             } | undefined;
             audio?: {
                 type: "audio";
@@ -567,6 +571,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | {
                     type: "voice_over";
                     startAt?: number | undefined;
+                } | {
+                    type: "vision";
+                    data: Record<string, any>;
+                    name: string;
                 } | undefined;
                 audio?: {
                     type: "audio";

package/lib/utils/ffmpeg_utils.js

@@ -64,7 +64,9 @@ export const ffmpegGetMediaDuration = (filePath) => {
     return new Promise((resolve, reject) => {
         // Only check file existence for local paths, not URLs
         if (!filePath.startsWith("http://") && !filePath.startsWith("https://") && !fs.existsSync(filePath)) {
-            reject(new Error(`File not found: ${filePath}`));
+            // NOTE: We don't reject here for scripts/test/test_hello_image.json, which uses mock image agent.
+            // reject(new Error(`File not found: ${filePath}`));
+            resolve({ duration: 0, hasAudio: false });
             return;
         }
         ffmpeg.ffprobe(filePath, (err, metadata) => {

package/lib/utils/image_plugins/index.d.ts

@@ -7,4 +7,5 @@ import * as pluginImage from "./image.js";
 import * as pluginMovie from "./movie.js";
 import * as pluginBeat from "./beat.js";
 import * as pluginVoiceOver from "./voice_over.js";
-export declare const findImagePlugin: (imageType?: string) => typeof pluginTextSlide | typeof pluginMarkdown | typeof pluginChart | typeof pluginMermaid | typeof pluginHtmlTailwind | typeof pluginImage | typeof pluginMovie | typeof pluginBeat | typeof pluginVoiceOver | undefined;
+import * as pluginVision from "./vision.js";
+export declare const findImagePlugin: (imageType?: string) => typeof pluginTextSlide | typeof pluginMarkdown | typeof pluginChart | typeof pluginMermaid | typeof pluginHtmlTailwind | typeof pluginImage | typeof pluginMovie | typeof pluginBeat | typeof pluginVoiceOver | typeof pluginVision | undefined;

package/lib/utils/image_plugins/index.js

@@ -7,7 +7,19 @@ import * as pluginImage from "./image.js";
 import * as pluginMovie from "./movie.js";
 import * as pluginBeat from "./beat.js";
 import * as pluginVoiceOver from "./voice_over.js";
-const imagePlugins = [pluginTextSlide, pluginMarkdown, pluginImage, pluginChart, pluginMermaid, pluginMovie, pluginHtmlTailwind, pluginBeat, pluginVoiceOver];
+import * as pluginVision from "./vision.js";
+const imagePlugins = [
+    pluginTextSlide,
+    pluginMarkdown,
+    pluginImage,
+    pluginChart,
+    pluginMermaid,
+    pluginMovie,
+    pluginHtmlTailwind,
+    pluginBeat,
+    pluginVoiceOver,
+    pluginVision,
+];
 export const findImagePlugin = (imageType) => {
     return imagePlugins.find((plugin) => plugin.imageType === imageType);
 };

package/lib/utils/image_plugins/vision.d.ts

@@ -0,0 +1,4 @@
+import { ImageProcessorParams } from "../../types/index.js";
+export declare const imageType = "vision";
+export declare const process: (params: ImageProcessorParams) => Promise<string | undefined>;
+export declare const path: (params: ImageProcessorParams) => string;

package/lib/utils/image_plugins/vision.js

@@ -0,0 +1,20 @@
+import { parrotingImagePath } from "./utils.js";
+import { htmlPlugin } from "mulmocast-vision";
+export const imageType = "vision";
+const toCreateName = (str) => {
+    return "create" + str.charAt(0).toUpperCase() + str.slice(1);
+};
+const processVision = async (params) => {
+    const { beat, imagePath } = params;
+    if (!beat?.image || beat.image.type !== imageType)
+        return;
+    const handler = new htmlPlugin({ outputDir: "" });
+    await handler[toCreateName(beat.image.name)](beat.image.data, {
+        name: beat.image.name,
+        imageFilePath: imagePath,
+        htmlFilePath: imagePath.replace(/\.png$/, ".html"),
+    });
+    return imagePath;
+};
+export const process = processVision;
+export const path = parrotingImagePath;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "1.2.17",
+  "version": "1.2.19",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -87,6 +87,7 @@
     "graphai": "^2.0.14",
     "jsdom": "^26.1.0",
     "marked": "^16.2.0",
+    "mulmocast-vision": "^0.0.1",
     "ora": "^8.2.0",
     "puppeteer": "^24.17.0",
     "replicate": "^1.1.0",

package/scripts/test/test_vision.json

@@ -0,0 +1,22 @@
+{
+  "$mulmocast": {
+    "version": "1.1"
+  },
+  "lang": "en",
+  "title": "Media Test",
+  "beats": [
+    {
+      "id": "first",
+      "speaker": "Presenter",
+      "text": "This is a local image.",
+      "image": {
+        "type": "vision",
+        "name": "sectionDividerPage",
+        "data": {
+          "heading": "How AI Is Reshaping Referencing",
+          "subheading": "From sources to systems: reliability, traceability, and credit in the age of models"
+        }
+      }
+    }
+  ]
+}