mulmocast 0.1.6 → 0.1.7

package/lib/types/schema.js

@@ -96,7 +96,7 @@ export const mulmoChartMediaSchema = z
     .object({
     type: z.literal("chart"),
     title: z.string(),
-    chartData: z.record(z.any()),
+    chartData: z.record(z.string(), z.any()),
 })
     .strict();
 export const mulmoMermaidMediaSchema = z
@@ -223,12 +223,14 @@ export const htmlPromptParamsSchema = z
     systemPrompt: z.string().default("").optional(),
     prompt: z.string().default(""),
     data: z.any().optional(),
-    images: z.record(z.any()).optional(),
+    images: z.record(z.string(), z.any()).optional(),
 })
     .strict();
+export const text2MovieProviderSchema = z.enum(Object.keys(provider2MovieAgent)).default(defaultProviders.text2movie);
+const defaultSpeaker = "Presenter";
 export const mulmoBeatSchema = z
     .object({
-    speaker: speakerIdSchema.default("Presenter"),
+    speaker: speakerIdSchema.default(defaultSpeaker),
     text: z.string().default("").describe("Text to be spoken. If empty, the audio is not generated."),
     id: z.string().optional().describe("Unique identifier for the beat."),
     description: z.string().optional(),
@@ -239,9 +241,10 @@ export const mulmoBeatSchema = z
     audioParams: beatAudioParamsSchema.optional(), // beat specific parameters
     movieParams: z
         .object({
+        provider: text2MovieProviderSchema.optional(),
         model: z.string().optional(),
-        fillOption: mulmoFillOptionSchema.optional(),
-        speed: z.number().optional().describe("Speed of the video. 1.0 is normal speed. 0.5 is half speed. 2.0 is double speed."),
+        fillOption: mulmoFillOptionSchema.optional(), // for movie.ts
+        speed: z.number().optional().describe("Speed of the video. 1.0 is normal speed. 0.5 is half speed. 2.0 is double speed."), // for movie.ts
     })
         .optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema.optional(),
@@ -275,7 +278,6 @@ export const mulmoSpeechParamsSchema = z
 })
     .strict();
 export const text2HtmlImageProviderSchema = z.enum(htmlLLMProvider).default(defaultProviders.text2Html);
-export const text2MovieProviderSchema = z.enum(Object.keys(provider2MovieAgent)).default(defaultProviders.text2movie);
 // NOTE: This is UI only. (until we figure out how to use it in mulmoMovieParamsSchema)
 export const mulmoGoogleMovieModelSchema = z
     .object({
@@ -296,10 +298,10 @@ export const mulmoTransitionSchema = z.object({
 });
 export const mulmoMovieParamsSchema = z
     .object({
-    provider: text2MovieProviderSchema.optional(),
-    model: z.string().optional(), // default: provider specific
-    transition: mulmoTransitionSchema.optional(),
-    fillOption: mulmoFillOptionSchema.optional(),
+    provider: text2MovieProviderSchema.optional(), // for agent
+    model: z.string().optional(), // default: provider specific. for agent
+    transition: mulmoTransitionSchema.optional(), // for movie.ts
+    fillOption: mulmoFillOptionSchema.optional(), // for movie.ts
 })
     .strict();
 export const mulmoPresentationStyleSchema = z.object({
@@ -307,16 +309,21 @@ export const mulmoPresentationStyleSchema = z.object({
     canvasSize: mulmoCanvasDimensionSchema, // has default value
     speechParams: mulmoSpeechParamsSchema.default({
         speakers: {
-            Presenter: {
+            [defaultSpeaker]: {
                 voiceId: "shimmer",
                 displayName: {
-                    en: "Presenter",
+                    en: defaultSpeaker,
                 },
             },
         },
     }),
-    imageParams: mulmoImageParamsSchema.optional(),
-    movieParams: mulmoMovieParamsSchema.optional(),
+    imageParams: mulmoImageParamsSchema.optional().default({
+        provider: defaultProviders.text2image,
+        images: {},
+    }),
+    movieParams: mulmoMovieParamsSchema.optional().default({
+        provider: defaultProviders.text2movie,
+    }),
     htmlImageParams: mulmoHtmlImageParamsSchema
         .extend({
         provider: text2HtmlImageProviderSchema,

package/lib/types/type.d.ts

@@ -1,6 +1,6 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
-import { LLM } from "../utils/utils.js";
+import type { LLM } from "../utils/provider2agent.js";
 import { z } from "zod";
 export type LANG = z.infer<typeof langSchema>;
 export type MulmoBeat = z.infer<typeof mulmoBeatSchema>;
@@ -36,6 +36,7 @@ export type MulmoGoogleImageModel = z.infer<typeof mulmoGoogleImageModelSchema>;
 export type MulmoGoogleMovieModel = z.infer<typeof mulmoGoogleMovieModelSchema>;
 export type MulmoReplicateMovieModel = z.infer<typeof mulmoReplicateMovieModelSchema>;
 export type MulmoImagePromptMedia = z.infer<typeof mulmoImagePromptMediaSchema>;
+export type MulmoImageAsset = z.infer<typeof mulmoImageAssetSchema>;
 export type MulmoTextSlideMedia = z.infer<typeof mulmoTextSlideMediaSchema>;
 export type MulmoMarkdownMedia = z.infer<typeof mulmoMarkdownMediaSchema>;
 export type MulmoImageMedia = z.infer<typeof mulmoImageMediaSchema>;

package/lib/utils/context.d.ts

@@ -19,6 +19,31 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             captionFile?: string | undefined;
         }[];
         script: {
+            imageParams: {
+                provider: string;
+                model?: string | undefined;
+                style?: string | undefined;
+                moderation?: string | undefined;
+                images?: Record<string, {
+                    type: "image";
+                    source: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
+                } | {
+                    type: "imagePrompt";
+                    prompt: string;
+                }> | undefined;
+            };
             audioParams: {
                 padding: number;
                 introPadding: number;
@@ -41,6 +66,17 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     kind: "path";
                 } | undefined;
             };
+            movieParams: {
+                provider?: string | undefined;
+                model?: string | undefined;
+                fillOption?: {
+                    style: "aspectFit" | "aspectFill";
+                } | undefined;
+                transition?: {
+                    type: "fade" | "slideout_left";
+                    duration: number;
+                } | undefined;
+            };
             $mulmocast: {
                 version: "1.0";
                 credit?: "closing" | undefined;
@@ -228,6 +264,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | undefined;
                 movieParams?: {
                     speed?: number | undefined;
+                    provider?: string | undefined;
                     model?: string | undefined;
                     fillOption?: {
                         style: "aspectFit" | "aspectFill";
@@ -255,42 +292,6 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             lang?: string | undefined;
             title?: string | undefined;
             description?: string | undefined;
-            imageParams?: {
-                provider: string;
-                model?: string | undefined;
-                style?: string | undefined;
-                moderation?: string | undefined;
-                images?: Record<string, {
-                    type: "image";
-                    source: {
-                        url: string;
-                        kind: "url";
-                    } | {
-                        kind: "base64";
-                        data: string;
-                    } | {
-                        text: string;
-                        kind: "text";
-                    } | {
-                        path: string;
-                        kind: "path";
-                    };
-                } | {
-                    type: "imagePrompt";
-                    prompt: string;
-                }> | undefined;
-            } | undefined;
-            movieParams?: {
-                provider?: string | undefined;
-                model?: string | undefined;
-                fillOption?: {
-                    style: "aspectFit" | "aspectFill";
-                } | undefined;
-                transition?: {
-                    type: "fade" | "slideout_left";
-                    duration: number;
-                } | undefined;
-            } | undefined;
             htmlImageParams?: {
                 provider: string;
                 model?: string | undefined;
@@ -336,6 +337,31 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
         };
     };
     presentationStyle: {
+        imageParams: {
+            provider: string;
+            model?: string | undefined;
+            style?: string | undefined;
+            moderation?: string | undefined;
+            images?: Record<string, {
+                type: "image";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "imagePrompt";
+                prompt: string;
+            }> | undefined;
+        };
         audioParams: {
             padding: number;
             introPadding: number;
@@ -358,6 +384,17 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 kind: "path";
             } | undefined;
         };
+        movieParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+            fillOption?: {
+                style: "aspectFit" | "aspectFill";
+            } | undefined;
+            transition?: {
+                type: "fade" | "slideout_left";
+                duration: number;
+            } | undefined;
+        };
         $mulmocast: {
             version: "1.0";
             credit?: "closing" | undefined;
@@ -380,42 +417,6 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             }>;
             model?: string | undefined;
         };
-        imageParams?: {
-            provider: string;
-            model?: string | undefined;
-            style?: string | undefined;
-            moderation?: string | undefined;
-            images?: Record<string, {
-                type: "image";
-                source: {
-                    url: string;
-                    kind: "url";
-                } | {
-                    kind: "base64";
-                    data: string;
-                } | {
-                    text: string;
-                    kind: "text";
-                } | {
-                    path: string;
-                    kind: "path";
-                };
-            } | {
-                type: "imagePrompt";
-                prompt: string;
-            }> | undefined;
-        } | undefined;
-        movieParams?: {
-            provider?: string | undefined;
-            model?: string | undefined;
-            fillOption?: {
-                style: "aspectFit" | "aspectFill";
-            } | undefined;
-            transition?: {
-                type: "fade" | "slideout_left";
-                duration: number;
-            } | undefined;
-        } | undefined;
         htmlImageParams?: {
             provider: string;
             model?: string | undefined;

package/lib/utils/ffmpeg_utils.js

@@ -1,5 +1,6 @@
 import ffmpeg from "fluent-ffmpeg";
 import { GraphAILogger } from "graphai";
+import fs from "fs";
 export const setFfmpegPath = (ffmpegPath) => {
     ffmpeg.setFfmpegPath(ffmpegPath);
 };
@@ -61,6 +62,11 @@ export const FfmpegContextGenerateOutput = (context, output, options = []) => {
 };
 export const ffmpegGetMediaDuration = (filePath) => {
     return new Promise((resolve, reject) => {
+        // Only check file existence for local paths, not URLs
+        if (!filePath.startsWith("http://") && !filePath.startsWith("https://") && !fs.existsSync(filePath)) {
+            reject(new Error(`File not found: ${filePath}`));
+            return;
+        }
         ffmpeg.ffprobe(filePath, (err, metadata) => {
             if (err) {
                 GraphAILogger.info("Error while getting metadata:", err);

package/lib/utils/preprocess.d.ts

@@ -14,6 +14,31 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         captionFile?: string | undefined;
     }[];
     script: {
+        imageParams: {
+            provider: string;
+            model?: string | undefined;
+            style?: string | undefined;
+            moderation?: string | undefined;
+            images?: Record<string, {
+                type: "image";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "imagePrompt";
+                prompt: string;
+            }> | undefined;
+        };
         audioParams: {
             padding: number;
             introPadding: number;
@@ -36,6 +61,17 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 kind: "path";
             } | undefined;
         };
+        movieParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+            fillOption?: {
+                style: "aspectFit" | "aspectFill";
+            } | undefined;
+            transition?: {
+                type: "fade" | "slideout_left";
+                duration: number;
+            } | undefined;
+        };
         $mulmocast: {
             version: "1.0";
             credit?: "closing" | undefined;
@@ -223,6 +259,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | undefined;
             movieParams?: {
                 speed?: number | undefined;
+                provider?: string | undefined;
                 model?: string | undefined;
                 fillOption?: {
                     style: "aspectFit" | "aspectFill";
@@ -250,42 +287,6 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         lang?: string | undefined;
         title?: string | undefined;
         description?: string | undefined;
-        imageParams?: {
-            provider: string;
-            model?: string | undefined;
-            style?: string | undefined;
-            moderation?: string | undefined;
-            images?: Record<string, {
-                type: "image";
-                source: {
-                    url: string;
-                    kind: "url";
-                } | {
-                    kind: "base64";
-                    data: string;
-                } | {
-                    text: string;
-                    kind: "text";
-                } | {
-                    path: string;
-                    kind: "path";
-                };
-            } | {
-                type: "imagePrompt";
-                prompt: string;
-            }> | undefined;
-        } | undefined;
-        movieParams?: {
-            provider?: string | undefined;
-            model?: string | undefined;
-            fillOption?: {
-                style: "aspectFit" | "aspectFill";
-            } | undefined;
-            transition?: {
-                type: "fade" | "slideout_left";
-                duration: number;
-            } | undefined;
-        } | undefined;
         htmlImageParams?: {
             provider: string;
             model?: string | undefined;

package/lib/utils/provider2agent.d.ts

@@ -1,10 +1,3 @@
-export declare const defaultProviders: {
-    tts: string;
-    text2image: string;
-    text2movie: string;
-    text2Html: string;
-    llm: string;
-};
 export declare const provider2TTSAgent: {
     nijivoice: {
         agentName: string;
@@ -42,10 +35,12 @@ export declare const provider2ImageAgent: {
 export declare const provider2MovieAgent: {
     replicate: {
         agentName: string;
+        defaultModel: `${string}/${string}`;
         models: string[];
     };
     google: {
         agentName: string;
+        defaultModel: string;
         models: string[];
     };
 };
@@ -71,6 +66,13 @@ export declare const provider2LLMAgent: {
         readonly max_tokens: 4096;
     };
 };
+export declare const defaultProviders: {
+    tts: keyof typeof provider2TTSAgent;
+    text2image: keyof typeof provider2ImageAgent;
+    text2movie: keyof typeof provider2MovieAgent;
+    text2Html: keyof typeof provider2LLMAgent;
+    llm: keyof typeof provider2LLMAgent;
+};
 export declare const llm: (keyof typeof provider2LLMAgent)[];
 export type LLM = keyof typeof provider2LLMAgent;
 export declare const htmlLLMProvider: string[];

package/lib/utils/provider2agent.js

@@ -1,10 +1,3 @@
-export const defaultProviders = {
-    tts: "openai",
-    text2image: "openai",
-    text2movie: "google",
-    text2Html: "openai",
-    llm: "openai",
-};
 export const provider2TTSAgent = {
     nijivoice: {
         agentName: "ttsNijivoiceAgent",
@@ -44,19 +37,24 @@ export const provider2ImageAgent = {
 export const provider2MovieAgent = {
     replicate: {
         agentName: "movieReplicateAgent",
+        defaultModel: "bytedance/seedance-1-lite",
         models: [
             "bytedance/seedance-1-lite",
             "bytedance/seedance-1-pro",
             "kwaivgi/kling-v1.6-pro",
             "kwaivgi/kling-v2.1",
+            "kwaivgi/kling-v2.1-master",
             "google/veo-2",
             "google/veo-3",
             "google/veo-3-fast",
             "minimax/video-01",
+            "minimax/hailuo-02",
+            "pixverse/pixverse-v4.5",
         ],
     },
     google: {
         agentName: "movieGoogleAgent",
+        defaultModel: "veo-2.0-generate-001",
         models: ["veo-2.0-generate-001"],
     },
 };
@@ -83,5 +81,12 @@ export const provider2LLMAgent = {
         max_tokens: 4096,
     },
 };
+export const defaultProviders = {
+    tts: "openai",
+    text2image: "openai",
+    text2movie: "replicate",
+    text2Html: "openai",
+    llm: "openai",
+};
 export const llm = Object.keys(provider2LLMAgent);
 export const htmlLLMProvider = ["openai", "anthropic"];

package/lib/utils/utils.d.ts

@@ -1,8 +1,6 @@
 import type { ConfigDataDictionary, DefaultConfigData } from "graphai";
 import { MulmoBeat, MulmoStudioMultiLingualData } from "../types/index.js";
-import { llm } from "./provider2agent.js";
 import type { LLM } from "./provider2agent.js";
-export { LLM, llm };
 export declare const llmPair: (_llm?: LLM, _model?: string) => {
     agent: "openAIAgent" | "anthropicAgent" | "geminiAgent" | "groqAgent";
     model: string;
@@ -22,3 +20,4 @@ type CleanableObject = {
     [key: string]: CleanableValue;
 };
 export declare const deepClean: <T extends CleanableValue>(input: T) => T | undefined;
+export {};

package/lib/utils/utils.js

@@ -1,6 +1,5 @@
 import * as crypto from "crypto";
-import { provider2LLMAgent, llm } from "./provider2agent.js";
-export { llm };
+import { provider2LLMAgent } from "./provider2agent.js";
 export const llmPair = (_llm, _model) => {
     const llmKey = _llm ?? "openai";
     const agent = provider2LLMAgent[llmKey]?.agentName ?? provider2LLMAgent.openai.agentName;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -67,7 +67,7 @@
     "@graphai/input_agents": "^1.0.1",
     "@graphai/openai_agent": "^2.0.3",
     "@graphai/stream_agent_filter": "^2.0.2",
-    "@graphai/vanilla": "^2.0.5",
+    "@graphai/vanilla": "^2.0.6",
     "@graphai/vanilla_node_agents": "^2.0.1",
     "@modelcontextprotocol/sdk": "^1.15.1",
     "@tavily/core": "^0.5.9",
@@ -76,11 +76,11 @@
     "dotenv": "^17.2.0",
     "fluent-ffmpeg": "^2.1.3",
     "google-auth-library": "^10.1.0",
-    "graphai": "^2.0.12",
+    "graphai": "^2.0.13",
     "inquirer": "^12.7.0",
-    "marked": "^16.0.0",
+    "marked": "^16.1.1",
     "ora": "^8.2.0",
-    "puppeteer": "^24.12.1",
+    "puppeteer": "^24.14.0",
     "replicate": "^1.0.1",
     "yaml": "^2.8.0",
     "yargs": "^18.0.0",
@@ -94,8 +94,8 @@
     "@types/fluent-ffmpeg": "^2.1.26",
     "@types/yargs": "^17.0.33",
     "eslint": "^9.31.0",
-    "eslint-config-prettier": "^10.1.5",
-    "eslint-plugin-prettier": "^5.5.1",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-prettier": "^5.5.3",
     "eslint-plugin-sonarjs": "^3.0.4",
     "prettier": "^3.6.2",
     "ts-node": "^10.9.2",