mulmocast 0.1.7 → 1.1.1

package/lib/utils/preprocess.d.ts

@@ -1,5 +1,5 @@
-import { MulmoStudio, MulmoScript } from "../types/index.js";
-export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string, videoCaption?: string) => {
+import { MulmoStudio, MulmoScript, MulmoPresentationStyle } from "../types/index.js";
+export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string, videoCaption?: string, presentationStyle?: MulmoPresentationStyle | null) => {
     beats: {
         duration?: number | undefined;
         startAt?: number | undefined;
@@ -11,6 +11,8 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         audioFile?: string | undefined;
         imageFile?: string | undefined;
         movieFile?: string | undefined;
+        soundEffectFile?: string | undefined;
+        lipSyncFile?: string | undefined;
         captionFile?: string | undefined;
     }[];
     script: {
@@ -72,8 +74,12 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 duration: number;
             } | undefined;
         };
+        soundEffectParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        };
         $mulmocast: {
-            version: "1.0";
+            version: "1.1";
             credit?: "closing" | undefined;
         };
         canvasSize: {
@@ -81,10 +87,10 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             height: number;
         };
         speechParams: {
-            provider: string;
             speakers: Record<string, {
                 voiceId: string;
                 displayName?: Record<string, string> | undefined;
+                isDefault?: boolean | undefined;
                 speechOptions?: {
                     speed?: number | undefined;
                     instruction?: string | undefined;
@@ -92,16 +98,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 provider?: string | undefined;
                 model?: string | undefined;
             }>;
-            model?: string | undefined;
         };
         beats: {
             text: string;
-            speaker: string;
-            duration?: number | undefined;
-            speechOptions?: {
-                speed?: number | undefined;
-                instruction?: string | undefined;
-            } | undefined;
             image?: {
                 type: "markdown";
                 markdown: string | string[];
@@ -206,6 +205,11 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 type: "voice_over";
                 startAt?: number | undefined;
             } | undefined;
+            duration?: number | undefined;
+            speechOptions?: {
+                speed?: number | undefined;
+                instruction?: string | undefined;
+            } | undefined;
             id?: string | undefined;
             audio?: {
                 type: "audio";
@@ -227,6 +231,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 source: string;
             } | undefined;
             imagePrompt?: string | undefined;
+            speaker?: string | undefined;
             description?: string | undefined;
             imageParams?: {
                 provider: string;
@@ -265,6 +270,14 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                     style: "aspectFit" | "aspectFill";
                 } | undefined;
             } | undefined;
+            soundEffectParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
+            lipSyncParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
             htmlImageParams?: {
                 model?: string | undefined;
             } | undefined;
@@ -277,16 +290,22 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | undefined;
             imageNames?: string[] | undefined;
             moviePrompt?: string | undefined;
+            soundEffectPrompt?: string | undefined;
             htmlPrompt?: {
                 prompt: string;
                 data?: any;
                 images?: Record<string, any> | undefined;
                 systemPrompt?: string | undefined;
             } | undefined;
+            enableLipSync?: boolean | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;
         description?: string | undefined;
+        lipSyncParams?: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        } | undefined;
         htmlImageParams?: {
             provider: string;
             model?: string | undefined;

package/lib/utils/preprocess.js

@@ -1,5 +1,6 @@
 import { GraphAILogger } from "graphai";
-import { mulmoScriptSchema, mulmoStudioSchema, mulmoCaptionParamsSchema } from "../types/index.js";
+import { mulmoStudioSchema, mulmoCaptionParamsSchema } from "../types/index.js";
+import { MulmoPresentationStyleMethods, MulmoScriptMethods } from "../methods/index.js";
 const rebuildStudio = (currentStudio, mulmoScript, fileName) => {
     const isTest = process.env.NODE_ENV === "test";
     const parsed = isTest && currentStudio ? { data: mulmoStudioSchema.parse(currentStudio), success: true, error: null } : mulmoStudioSchema.safeParse(currentStudio);
@@ -36,15 +37,16 @@ const mulmoCredit = (speaker) => {
         },
     };
 };
-export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName, videoCaption) => {
-    const mulmoScript = _mulmoScript.__test_invalid__ ? _mulmoScript : mulmoScriptSchema.parse(_mulmoScript); // validate and insert default value
+export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName, videoCaption, presentationStyle) => {
+    const mulmoScript = _mulmoScript.__test_invalid__ ? _mulmoScript : MulmoScriptMethods.validate(_mulmoScript); // validate and insert default value
     const studio = rebuildStudio(currentStudio, mulmoScript, fileName);
     // TODO: Move this code out of this function later
     // Addition cloing credit
     if (mulmoScript.$mulmocast.credit === "closing") {
-        mulmoScript.beats.push(mulmoCredit(mulmoScript.beats[0].speaker)); // First speaker
+        const defaultSpeaker = MulmoPresentationStyleMethods.getDefaultSpeaker(presentationStyle ?? studio.script);
+        mulmoScript.beats.push(mulmoCredit(mulmoScript.beats[0].speaker ?? defaultSpeaker)); // First speaker
     }
-    studio.script = mulmoScriptSchema.parse(mulmoScript); // update the script
+    studio.script = MulmoScriptMethods.validate(mulmoScript); // update the script
     studio.beats = studio.script.beats.map((_, index) => studio.beats[index] ?? {});
     if (videoCaption) {
         studio.script.captionParams = mulmoCaptionParamsSchema.parse({

package/lib/utils/provider2agent.d.ts

@@ -32,11 +32,18 @@ export declare const provider2ImageAgent: {
         models: string[];
     };
 };
+export type ReplicateModel = `${string}/${string}`;
 export declare const provider2MovieAgent: {
     replicate: {
         agentName: string;
-        defaultModel: `${string}/${string}`;
+        defaultModel: ReplicateModel;
         models: string[];
+        modelParams: Record<ReplicateModel, {
+            durations: number[];
+            start_image: string | undefined;
+            last_image?: string;
+            price_per_sec: number;
+        }>;
     };
     google: {
         agentName: string;
@@ -44,6 +51,26 @@ export declare const provider2MovieAgent: {
         models: string[];
     };
 };
+export declare const provider2SoundEffectAgent: {
+    replicate: {
+        agentName: string;
+        defaultModel: ReplicateModel;
+        models: ReplicateModel[];
+        modelParams: Record<ReplicateModel, {
+            identifier?: `${string}/${string}:${string}`;
+        }>;
+    };
+};
+export declare const provider2LipSyncAgent: {
+    replicate: {
+        agentName: string;
+        defaultModel: ReplicateModel;
+        models: ReplicateModel[];
+        modelParams: Record<ReplicateModel, {
+            identifier?: `${string}/${string}:${string}`;
+        }>;
+    };
+};
 export declare const provider2LLMAgent: {
     readonly openai: {
         readonly agentName: "openAIAgent";
@@ -72,6 +99,8 @@ export declare const defaultProviders: {
     text2movie: keyof typeof provider2MovieAgent;
     text2Html: keyof typeof provider2LLMAgent;
     llm: keyof typeof provider2LLMAgent;
+    soundEffect: keyof typeof provider2SoundEffectAgent;
+    lipSync: keyof typeof provider2LipSyncAgent;
 };
 export declare const llm: (keyof typeof provider2LLMAgent)[];
 export type LLM = keyof typeof provider2LLMAgent;

package/lib/utils/provider2agent.js

@@ -51,6 +51,66 @@ export const provider2MovieAgent = {
             "minimax/hailuo-02",
             "pixverse/pixverse-v4.5",
         ],
+        modelParams: {
+            "bytedance/seedance-1-lite": {
+                durations: [5, 10],
+                start_image: "image",
+                last_image: "last_frame_image",
+                price_per_sec: 0.036, // in USD
+            },
+            "bytedance/seedance-1-pro": {
+                durations: [5, 10],
+                start_image: "image",
+                last_image: "last_frame_image",
+                price_per_sec: 0.15,
+            },
+            "kwaivgi/kling-v1.6-pro": {
+                durations: [5, 10],
+                start_image: "start_image",
+                price_per_sec: 0.095,
+            },
+            "kwaivgi/kling-v2.1": {
+                durations: [5, 10],
+                start_image: "start_image",
+                price_per_sec: 0.05,
+            },
+            "kwaivgi/kling-v2.1-master": {
+                durations: [5, 10],
+                start_image: "start_image",
+                price_per_sec: 0.28,
+            },
+            "google/veo-2": {
+                durations: [5, 6, 7, 8],
+                start_image: "image",
+                price_per_sec: 0.5,
+            },
+            "google/veo-3": {
+                durations: [8],
+                start_image: undefined,
+                price_per_sec: 0.75,
+            },
+            "google/veo-3-fast": {
+                durations: [8],
+                start_image: undefined,
+                price_per_sec: 0.4,
+            },
+            "minimax/video-01": {
+                durations: [6],
+                start_image: "first_frame_image",
+                price_per_sec: 0.5,
+            },
+            "minimax/hailuo-02": {
+                durations: [6], // NOTE: 10 for only 720p
+                start_image: "first_frame_image",
+                price_per_sec: 0.08,
+            },
+            "pixverse/pixverse-v4.5": {
+                durations: [5, 8],
+                start_image: "image",
+                last_image: "last_frame_image",
+                price_per_sec: 0.12,
+            },
+        },
     },
     google: {
         agentName: "movieGoogleAgent",
@@ -58,6 +118,30 @@ export const provider2MovieAgent = {
         models: ["veo-2.0-generate-001"],
     },
 };
+export const provider2SoundEffectAgent = {
+    replicate: {
+        agentName: "soundEffectReplicateAgent",
+        defaultModel: "zsxkib/mmaudio",
+        models: ["zsxkib/mmaudio"],
+        modelParams: {
+            "zsxkib/mmaudio": {
+                identifier: "zsxkib/mmaudio:62871fb59889b2d7c13777f08deb3b36bdff88f7e1d53a50ad7694548a41b484",
+            },
+        },
+    },
+};
+export const provider2LipSyncAgent = {
+    replicate: {
+        agentName: "lipSyncReplicateAgent",
+        defaultModel: "bytedance/latentsync",
+        models: ["bytedance/latentsync"],
+        modelParams: {
+            "bytedance/latentsync": {
+                identifier: "bytedance/latentsync:637ce1919f807ca20da3a448ddc2743535d2853649574cd52a933120e9b9e293",
+            },
+        },
+    },
+};
 // : Record<LLM, { agent: string; defaultModel: string; max_tokens: number }>
 export const provider2LLMAgent = {
     openai: {
@@ -87,6 +171,8 @@ export const defaultProviders = {
     text2movie: "replicate",
     text2Html: "openai",
     llm: "openai",
+    soundEffect: "replicate",
+    lipSync: "replicate",
 };
 export const llm = Object.keys(provider2LLMAgent);
 export const htmlLLMProvider = ["openai", "anthropic"];

package/lib/utils/utils.js

@@ -73,6 +73,12 @@ export const settings2GraphAIConfig = (settings, env) => {
         ttsElevenlabsAgent: {
             apiKey: getKey("TTS", "ELEVENLABS_API_KEY"),
         },
+        soundEffectReplicateAgent: {
+            apiKey: getKey("SOUND_EFFECT", "REPLICATE_API_TOKEN"),
+        },
+        lipSyncReplicateAgent: {
+            apiKey: getKey("LIPSYNC", "REPLICATE_API_TOKEN"),
+        },
         // TODO
         // browserlessAgent
         // ttsGoogleAgent

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.1.7",
+  "version": "1.1.1",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -12,6 +12,9 @@
     "./browser": {
       "types": "./lib/index.browser.d.ts",
       "default": "./lib/index.browser.js"
+    },
+    "./data": {
+      "default": "./lib/data/index.js"
     }
   },
   "bin": {
@@ -48,6 +51,7 @@
     "latest": "yarn upgrade-interactive  --latest",
     "format": "prettier --write '{src,scripts,assets/templates,assets/styles,draft,ideason,scripts_mag2,proto,test,graphai,output,docs/scripts}/**/*.{ts,json,yaml}'",
     "deep_research": "npx tsx ./src/tools/deep_research.ts",
+    "template": "npx tsx batch/template2tsobject.ts",
     "fake_data": "npx tsx test/fake/sample.ts",
     "mcp_server": "npx tsx ./src/mcp/server.ts"
   },
@@ -73,14 +77,14 @@
     "@tavily/core": "^0.5.9",
     "canvas": "^3.1.2",
     "clipboardy": "^4.0.0",
-    "dotenv": "^17.2.0",
+    "dotenv": "^17.2.1",
     "fluent-ffmpeg": "^2.1.3",
     "google-auth-library": "^10.1.0",
     "graphai": "^2.0.13",
     "inquirer": "^12.7.0",
     "marked": "^16.1.1",
     "ora": "^8.2.0",
-    "puppeteer": "^24.14.0",
+    "puppeteer": "^24.15.0",
     "replicate": "^1.0.1",
     "yaml": "^2.8.0",
     "yargs": "^18.0.0",
@@ -93,7 +97,7 @@
     "@receptron/test_utils": "^2.0.0",
     "@types/fluent-ffmpeg": "^2.1.26",
     "@types/yargs": "^17.0.33",
-    "eslint": "^9.31.0",
+    "eslint": "^9.32.0",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-prettier": "^5.5.3",
     "eslint-plugin-sonarjs": "^3.0.4",

package/scripts/templates/business.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "Sample Title",

package/scripts/templates/children_book.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "桃太郎",

package/scripts/templates/coding.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "Sample Title",

package/scripts/templates/html.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "references": [

package/scripts/templates/image_prompt_only_template.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "[TITLE: Brief, engaging title for the topic]",

package/scripts/templates/image_prompts_template.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "[TITLE: Brief, engaging title for the topic]",

package/scripts/templates/image_refs.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0"
+    "version": "1.1"
   },
   "title": "[TITLE OF THE PRESENTAITON OR STORY]",
   "imageParams": {

package/scripts/templates/movie_prompts_no_text_template.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "[TITLE: Brief, engaging title for the topic]",

package/scripts/templates/movie_prompts_template.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "[TITLE: Brief, engaging title for the topic]",

package/scripts/templates/presentation.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "htmlImageParams": {

package/scripts/templates/sensei_and_taro.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "韓国の戒厳令とその日本への影響",

package/scripts/templates/shorts_template.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0"
+    "version": "1.1"
   },
   "title": "[TITLE: Brief, engaging title for the topic]",
   "lang": "en",

package/scripts/templates/text_only_template.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "[TITLE: Brief, engaging title for the topic]",

package/scripts/templates/voice_over.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0"
+    "version": "1.1"
   },
   "title": "Voice Over Test",
   "captionParams": {