mulmocast 0.1.6 → 1.1.0

package/lib/utils/preprocess.d.ts

@@ -1,5 +1,5 @@
-import { MulmoStudio, MulmoScript } from "../types/index.js";
-export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string, videoCaption?: string) => {
+import { MulmoStudio, MulmoScript, MulmoPresentationStyle } from "../types/index.js";
+export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string, videoCaption?: string, presentationStyle?: MulmoPresentationStyle | null) => {
     beats: {
         duration?: number | undefined;
         startAt?: number | undefined;
@@ -11,9 +11,35 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         audioFile?: string | undefined;
         imageFile?: string | undefined;
         movieFile?: string | undefined;
+        soundEffectFile?: string | undefined;
         captionFile?: string | undefined;
     }[];
     script: {
+        imageParams: {
+            provider: string;
+            model?: string | undefined;
+            style?: string | undefined;
+            moderation?: string | undefined;
+            images?: Record<string, {
+                type: "image";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "imagePrompt";
+                prompt: string;
+            }> | undefined;
+        };
         audioParams: {
             padding: number;
             introPadding: number;
@@ -36,8 +62,23 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 kind: "path";
             } | undefined;
         };
+        movieParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+            fillOption?: {
+                style: "aspectFit" | "aspectFill";
+            } | undefined;
+            transition?: {
+                type: "fade" | "slideout_left";
+                duration: number;
+            } | undefined;
+        };
+        soundEffectParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        };
         $mulmocast: {
-            version: "1.0";
+            version: "1.1";
             credit?: "closing" | undefined;
         };
         canvasSize: {
@@ -45,10 +86,10 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             height: number;
         };
         speechParams: {
-            provider: string;
             speakers: Record<string, {
                 voiceId: string;
                 displayName?: Record<string, string> | undefined;
+                isDefault?: boolean | undefined;
                 speechOptions?: {
                     speed?: number | undefined;
                     instruction?: string | undefined;
@@ -56,16 +97,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 provider?: string | undefined;
                 model?: string | undefined;
             }>;
-            model?: string | undefined;
         };
         beats: {
             text: string;
-            speaker: string;
-            duration?: number | undefined;
-            speechOptions?: {
-                speed?: number | undefined;
-                instruction?: string | undefined;
-            } | undefined;
             image?: {
                 type: "markdown";
                 markdown: string | string[];
@@ -170,6 +204,11 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 type: "voice_over";
                 startAt?: number | undefined;
             } | undefined;
+            duration?: number | undefined;
+            speechOptions?: {
+                speed?: number | undefined;
+                instruction?: string | undefined;
+            } | undefined;
             id?: string | undefined;
             audio?: {
                 type: "audio";
@@ -191,6 +230,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 source: string;
             } | undefined;
             imagePrompt?: string | undefined;
+            speaker?: string | undefined;
             description?: string | undefined;
             imageParams?: {
                 provider: string;
@@ -223,11 +263,16 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | undefined;
             movieParams?: {
                 speed?: number | undefined;
+                provider?: string | undefined;
                 model?: string | undefined;
                 fillOption?: {
                     style: "aspectFit" | "aspectFill";
                 } | undefined;
             } | undefined;
+            soundEffectParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
             htmlImageParams?: {
                 model?: string | undefined;
             } | undefined;
@@ -240,6 +285,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | undefined;
             imageNames?: string[] | undefined;
             moviePrompt?: string | undefined;
+            soundEffectPrompt?: string | undefined;
             htmlPrompt?: {
                 prompt: string;
                 data?: any;
@@ -250,42 +296,6 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         lang?: string | undefined;
         title?: string | undefined;
         description?: string | undefined;
-        imageParams?: {
-            provider: string;
-            model?: string | undefined;
-            style?: string | undefined;
-            moderation?: string | undefined;
-            images?: Record<string, {
-                type: "image";
-                source: {
-                    url: string;
-                    kind: "url";
-                } | {
-                    kind: "base64";
-                    data: string;
-                } | {
-                    text: string;
-                    kind: "text";
-                } | {
-                    path: string;
-                    kind: "path";
-                };
-            } | {
-                type: "imagePrompt";
-                prompt: string;
-            }> | undefined;
-        } | undefined;
-        movieParams?: {
-            provider?: string | undefined;
-            model?: string | undefined;
-            fillOption?: {
-                style: "aspectFit" | "aspectFill";
-            } | undefined;
-            transition?: {
-                type: "fade" | "slideout_left";
-                duration: number;
-            } | undefined;
-        } | undefined;
         htmlImageParams?: {
             provider: string;
             model?: string | undefined;

package/lib/utils/preprocess.js

@@ -1,5 +1,6 @@
 import { GraphAILogger } from "graphai";
-import { mulmoScriptSchema, mulmoStudioSchema, mulmoCaptionParamsSchema } from "../types/index.js";
+import { mulmoStudioSchema, mulmoCaptionParamsSchema } from "../types/index.js";
+import { MulmoPresentationStyleMethods, MulmoScriptMethods } from "../methods/index.js";
 const rebuildStudio = (currentStudio, mulmoScript, fileName) => {
     const isTest = process.env.NODE_ENV === "test";
     const parsed = isTest && currentStudio ? { data: mulmoStudioSchema.parse(currentStudio), success: true, error: null } : mulmoStudioSchema.safeParse(currentStudio);
@@ -36,15 +37,16 @@ const mulmoCredit = (speaker) => {
         },
     };
 };
-export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName, videoCaption) => {
-    const mulmoScript = _mulmoScript.__test_invalid__ ? _mulmoScript : mulmoScriptSchema.parse(_mulmoScript); // validate and insert default value
+export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName, videoCaption, presentationStyle) => {
+    const mulmoScript = _mulmoScript.__test_invalid__ ? _mulmoScript : MulmoScriptMethods.validate(_mulmoScript); // validate and insert default value
     const studio = rebuildStudio(currentStudio, mulmoScript, fileName);
     // TODO: Move this code out of this function later
     // Addition cloing credit
     if (mulmoScript.$mulmocast.credit === "closing") {
-        mulmoScript.beats.push(mulmoCredit(mulmoScript.beats[0].speaker)); // First speaker
+        const defaultSpeaker = MulmoPresentationStyleMethods.getDefaultSpeaker(presentationStyle ?? studio.script);
+        mulmoScript.beats.push(mulmoCredit(mulmoScript.beats[0].speaker ?? defaultSpeaker)); // First speaker
     }
-    studio.script = mulmoScriptSchema.parse(mulmoScript); // update the script
+    studio.script = MulmoScriptMethods.validate(mulmoScript); // update the script
     studio.beats = studio.script.beats.map((_, index) => studio.beats[index] ?? {});
     if (videoCaption) {
         studio.script.captionParams = mulmoCaptionParamsSchema.parse({

package/lib/utils/provider2agent.d.ts

@@ -1,10 +1,3 @@
-export declare const defaultProviders: {
-    tts: string;
-    text2image: string;
-    text2movie: string;
-    text2Html: string;
-    llm: string;
-};
 export declare const provider2TTSAgent: {
     nijivoice: {
         agentName: string;
@@ -39,16 +32,35 @@ export declare const provider2ImageAgent: {
         models: string[];
     };
 };
+export type ReplicateModel = `${string}/${string}`;
 export declare const provider2MovieAgent: {
     replicate: {
         agentName: string;
+        defaultModel: ReplicateModel;
         models: string[];
+        modelParams: Record<ReplicateModel, {
+            durations: number[];
+            start_image: string | undefined;
+            last_image?: string;
+            price_per_sec: number;
+        }>;
     };
     google: {
         agentName: string;
+        defaultModel: string;
         models: string[];
     };
 };
+export declare const provider2SoundEffectAgent: {
+    replicate: {
+        agentName: string;
+        defaultModel: ReplicateModel;
+        models: ReplicateModel[];
+        modelParams: Record<ReplicateModel, {
+            identifier?: `${string}/${string}:${string}`;
+        }>;
+    };
+};
 export declare const provider2LLMAgent: {
     readonly openai: {
         readonly agentName: "openAIAgent";
@@ -71,6 +83,14 @@ export declare const provider2LLMAgent: {
         readonly max_tokens: 4096;
     };
 };
+export declare const defaultProviders: {
+    tts: keyof typeof provider2TTSAgent;
+    text2image: keyof typeof provider2ImageAgent;
+    text2movie: keyof typeof provider2MovieAgent;
+    text2Html: keyof typeof provider2LLMAgent;
+    llm: keyof typeof provider2LLMAgent;
+    soundEffect: keyof typeof provider2SoundEffectAgent;
+};
 export declare const llm: (keyof typeof provider2LLMAgent)[];
 export type LLM = keyof typeof provider2LLMAgent;
 export declare const htmlLLMProvider: string[];

package/lib/utils/provider2agent.js

@@ -1,10 +1,3 @@
-export const defaultProviders = {
-    tts: "openai",
-    text2image: "openai",
-    text2movie: "google",
-    text2Html: "openai",
-    llm: "openai",
-};
 export const provider2TTSAgent = {
     nijivoice: {
         agentName: "ttsNijivoiceAgent",
@@ -44,22 +37,99 @@ export const provider2ImageAgent = {
 export const provider2MovieAgent = {
     replicate: {
         agentName: "movieReplicateAgent",
+        defaultModel: "bytedance/seedance-1-lite",
         models: [
             "bytedance/seedance-1-lite",
             "bytedance/seedance-1-pro",
             "kwaivgi/kling-v1.6-pro",
             "kwaivgi/kling-v2.1",
+            "kwaivgi/kling-v2.1-master",
             "google/veo-2",
             "google/veo-3",
             "google/veo-3-fast",
             "minimax/video-01",
+            "minimax/hailuo-02",
+            "pixverse/pixverse-v4.5",
         ],
+        modelParams: {
+            "bytedance/seedance-1-lite": {
+                durations: [5, 10],
+                start_image: "image",
+                last_image: "last_frame_image",
+                price_per_sec: 0.036, // in USD
+            },
+            "bytedance/seedance-1-pro": {
+                durations: [5, 10],
+                start_image: "image",
+                last_image: "last_frame_image",
+                price_per_sec: 0.15,
+            },
+            "kwaivgi/kling-v1.6-pro": {
+                durations: [5, 10],
+                start_image: "start_image",
+                price_per_sec: 0.095,
+            },
+            "kwaivgi/kling-v2.1": {
+                durations: [5, 10],
+                start_image: "start_image",
+                price_per_sec: 0.05,
+            },
+            "kwaivgi/kling-v2.1-master": {
+                durations: [5, 10],
+                start_image: "start_image",
+                price_per_sec: 0.28,
+            },
+            "google/veo-2": {
+                durations: [5, 6, 7, 8],
+                start_image: "image",
+                price_per_sec: 0.5,
+            },
+            "google/veo-3": {
+                durations: [8],
+                start_image: undefined,
+                price_per_sec: 0.75,
+            },
+            "google/veo-3-fast": {
+                durations: [8],
+                start_image: undefined,
+                price_per_sec: 0.4,
+            },
+            "minimax/video-01": {
+                durations: [6],
+                start_image: "first_frame_image",
+                price_per_sec: 0.5,
+            },
+            "minimax/hailuo-02": {
+                durations: [6], // NOTE: 10 for only 720p
+                start_image: "first_frame_image",
+                price_per_sec: 0.08,
+            },
+            "pixverse/pixverse-v4.5": {
+                durations: [5, 8],
+                start_image: "image",
+                last_image: "last_frame_image",
+                price_per_sec: 0.12,
+            },
+        },
     },
     google: {
         agentName: "movieGoogleAgent",
+        defaultModel: "veo-2.0-generate-001",
         models: ["veo-2.0-generate-001"],
     },
 };
+export const provider2SoundEffectAgent = {
+    replicate: {
+        agentName: "soundEffectReplicateAgent",
+        defaultModel: "zsxkib/mmaudio",
+        models: ["zsxkib/mmaudio"],
+        modelParams: {
+            "zsxkib/mmaudio": {
+                identifier: "zsxkib/mmaudio:62871fb59889b2d7c13777f08deb3b36bdff88f7e1d53a50ad7694548a41b484",
+            },
+        },
+    },
+};
 // : Record<LLM, { agent: string; defaultModel: string; max_tokens: number }>
 export const provider2LLMAgent = {
     openai: {
@@ -83,5 +153,13 @@ export const provider2LLMAgent = {
         max_tokens: 4096,
     },
 };
+export const defaultProviders = {
+    tts: "openai",
+    text2image: "openai",
+    text2movie: "replicate",
+    text2Html: "openai",
+    llm: "openai",
+    soundEffect: "replicate",
+};
 export const llm = Object.keys(provider2LLMAgent);
 export const htmlLLMProvider = ["openai", "anthropic"];

package/lib/utils/utils.d.ts

@@ -1,8 +1,6 @@
 import type { ConfigDataDictionary, DefaultConfigData } from "graphai";
 import { MulmoBeat, MulmoStudioMultiLingualData } from "../types/index.js";
-import { llm } from "./provider2agent.js";
 import type { LLM } from "./provider2agent.js";
-export { LLM, llm };
 export declare const llmPair: (_llm?: LLM, _model?: string) => {
     agent: "openAIAgent" | "anthropicAgent" | "geminiAgent" | "groqAgent";
     model: string;
@@ -22,3 +20,4 @@ type CleanableObject = {
     [key: string]: CleanableValue;
 };
 export declare const deepClean: <T extends CleanableValue>(input: T) => T | undefined;
+export {};

package/lib/utils/utils.js

@@ -1,6 +1,5 @@
 import * as crypto from "crypto";
-import { provider2LLMAgent, llm } from "./provider2agent.js";
-export { llm };
+import { provider2LLMAgent } from "./provider2agent.js";
 export const llmPair = (_llm, _model) => {
     const llmKey = _llm ?? "openai";
     const agent = provider2LLMAgent[llmKey]?.agentName ?? provider2LLMAgent.openai.agentName;
@@ -74,6 +73,9 @@ export const settings2GraphAIConfig = (settings, env) => {
         ttsElevenlabsAgent: {
             apiKey: getKey("TTS", "ELEVENLABS_API_KEY"),
         },
+        soundEffectReplicateAgent: {
+            apiKey: getKey("SOUND_EFFECT", "REPLICATE_API_TOKEN"),
+        },
         // TODO
         // browserlessAgent
         // ttsGoogleAgent

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.1.6",
+  "version": "1.1.0",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -67,7 +67,7 @@
     "@graphai/input_agents": "^1.0.1",
     "@graphai/openai_agent": "^2.0.3",
     "@graphai/stream_agent_filter": "^2.0.2",
-    "@graphai/vanilla": "^2.0.5",
+    "@graphai/vanilla": "^2.0.6",
     "@graphai/vanilla_node_agents": "^2.0.1",
     "@modelcontextprotocol/sdk": "^1.15.1",
     "@tavily/core": "^0.5.9",
@@ -76,11 +76,11 @@
     "dotenv": "^17.2.0",
     "fluent-ffmpeg": "^2.1.3",
     "google-auth-library": "^10.1.0",
-    "graphai": "^2.0.12",
+    "graphai": "^2.0.13",
     "inquirer": "^12.7.0",
-    "marked": "^16.0.0",
+    "marked": "^16.1.1",
     "ora": "^8.2.0",
-    "puppeteer": "^24.12.1",
+    "puppeteer": "^24.14.0",
     "replicate": "^1.0.1",
     "yaml": "^2.8.0",
     "yargs": "^18.0.0",
@@ -94,8 +94,8 @@
     "@types/fluent-ffmpeg": "^2.1.26",
     "@types/yargs": "^17.0.33",
     "eslint": "^9.31.0",
-    "eslint-config-prettier": "^10.1.5",
-    "eslint-plugin-prettier": "^5.5.1",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-prettier": "^5.5.3",
     "eslint-plugin-sonarjs": "^3.0.4",
     "prettier": "^3.6.2",
     "ts-node": "^10.9.2",

package/scripts/templates/presentation.json~

@@ -1,119 +0,0 @@
-{
-  "$mulmocast": {
-    "version": "1.0",
-    "credit": "closing"
-  },
-  "title": "Sample Title",
-  "references": [
-    {
-      "url": "https://www.somegreatwebsite.com/article/123",
-      "title": "Title of the article we are referencing",
-      "type": "article"
-    }
-  ],
-  "lang": "en",
-  "beats": [
-    {
-      "speaker": "Presenter",
-      "text": "Book rooms with locals, not hotels.",
-      "htmlPrompt": {
-        "prompt": "Create a clean and bold title slide featuring the original name 'AirBed & Breakfast' with the tagline: 'Book rooms with locals, not hotels'. Include the old logo and a travel-themed background."
-      }
-    },
-    {
-      "speaker": "Presenter",
-      "text": "Hotels are expensive and detached from local culture.",
-      "htmlPrompt": {
-        "prompt": "Design a slide that lists the main problems with hotels: high prices, lack of personality, and cultural disconnection. Use simple icons for cost, generic room, and traveler."
-      }
-    },
-    {
-      "speaker": "Presenter",
-      "text": "Airbnb lets people rent out their spaces, offering affordable, authentic experiences.",
-      "htmlPrompt": {
-        "prompt": "Illustrate Airbnb's solution: locals listing their spare rooms or homes, travelers booking directly, and both benefiting. Use a flow diagram to represent this."
-      }
-    },
-    {
-      "speaker": "Presenter",
-      "text": "Craigslist and Couchsurfing prove the demand for alternative lodging.",
-      "htmlPrompt": {
-        "prompt": "Show a bar graph comparing Craigslist listings, Couchsurfing users, and early Airbnb listings to demonstrate existing demand and opportunity."
-      }
-    },
-    {
-      "speaker": "Presenter",
-      "text": "A massive market: over 630 million travel bookings annually.",
-      "htmlPrompt": {
-        "prompt": "Visualize the global travel booking market. Use a world map and large numbers to emphasize the size and growth potential."
-      }
-    },
-    {
-      "speaker": "Presenter",
-      "text": "Our platform enables hosts to list once and earn; guests search, book, and review.",
-      "htmlPrompt": {
-        "prompt": "Create a product overview slide showing how the platform works from host listing to guest review. Highlight simplicity and trust."
-      }
-    },
-    {
-      "speaker": "Presenter",
-      "text": "We charge a 10% commission per booking.",
-      "htmlPrompt": {
-        "prompt": "Display a slide with the monetization strategy—Airbnb earns through a 10% commission. Use a simple pie chart or booking flow with fee annotation."
-      }
-    },
-    {
-      "speaker": "Presenter",
-      "text": "Competitors include hotels, Craigslist, Couchsurfing — we offer a better, scalable solution.",
-      "htmlPrompt": {
-        "prompt": "Design a comparison table: price, uniqueness, trust, scalability. Airbnb should clearly stand out on all dimensions."
-      }
-    },
-    {
-      "speaker": "Presenter",
-      "text": "Key advantages: lower price, wide selection, ease of use, host incentives, trusted system.",
-      "htmlPrompt": {
-        "prompt": "List Airbnb’s competitive advantages using 6 icons or badges, one for each feature."
-      }
-    },
-    {
-      "speaker": "Presenter",
-      "text": "Founders: Brian Chesky (Design), Joe Gebbia (Marketing), Nathan Blecharczyk (Engineering).",
-      "htmlPrompt": {
-        "prompt": "Introduce the founding team with photos and brief bios. Highlight their roles and strengths in product, marketing, and tech."
-      }
-    },
-    {
-      "speaker": "Presenter",
-      "text": "Media buzz: featured on TechCrunch and buzz from SXSW.",
-      "htmlPrompt": {
-        "prompt": "Showcase media recognition. Include TechCrunch logo and SXSW quotes or metrics. This adds credibility and traction."
-      }
-    },
-    {
-      "speaker": "Presenter",
-      "text": "Early users love the authentic and affordable experiences.",
-      "htmlPrompt": {
-        "prompt": "Add 2–3 short testimonials from early users (guests and hosts), displayed in speech bubbles with faces or usernames."
-      }
-    },
-    {
-      "speaker": "Presenter",
-      "text": "We’re raising $600K to support 12 months of growth and reach 80K bookings with $2M in revenue.",
-      "htmlPrompt": {
-        "prompt": "Closing investment slide. Include fund usage (pie chart: 40% marketing, 30% product, 30% ops), and KPIs: 80K bookings, $2M revenue.",
-        "data": {
-          "ask": 600000,
-          "duration_months": 12,
-          "target_bookings": 80000,
-          "target_revenue": 2000000,
-          "use_of_funds": {
-            "marketing": 40,
-            "product": 30,
-            "operations": 30
-          }
-        }
-      }
-    }
-  ]
-}