mulmocast 0.1.6 → 1.1.0

package/lib/actions/images.js

@@ -6,7 +6,7 @@ import * as vanilla from "@graphai/vanilla";
 import { openAIAgent } from "@graphai/openai_agent";
 import { anthropicAgent } from "@graphai/anthropic_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-import { imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent } from "../agents/index.js";
+import { imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, soundEffectReplicateAgent } from "../agents/index.js";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
 import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
@@ -23,10 +23,14 @@ const movieAgents = {
     movieGoogleAgent,
     movieReplicateAgent,
 };
+const soundEffectAgents = {
+    soundEffectReplicateAgent,
+};
 const defaultAgents = {
     ...vanillaAgents,
     ...imageAgents,
     ...movieAgents,
+    ...soundEffectAgents,
     mediaMockAgent,
     fileWriteAgent,
     openAIAgent,
@@ -39,7 +43,6 @@ const beat_graph_data = {
     nodes: {
         context: {},
         htmlImageAgentInfo: {},
-        movieAgentInfo: {},
         imageRefs: {},
         beat: {},
         __mapIndex: {},
@@ -134,7 +137,7 @@ const beat_graph_data = {
         },
         movieGenerator: {
             if: ":preprocessor.movieFile",
-            agent: ":movieAgentInfo.agent",
+            agent: ":preprocessor.movieAgentInfo.agent",
             inputs: {
                 onComplete: [":imageGenerator", ":imagePlugin"], // to wait for imageGenerator to finish
                 prompt: ":beat.moviePrompt",
@@ -147,7 +150,7 @@ const beat_graph_data = {
                     mulmoContext: ":context",
                 },
                 params: {
-                    model: ":preprocessor.movieParams.model",
+                    model: ":preprocessor.movieAgentInfo.movieParams.model",
                     duration: ":beat.duration",
                     canvasSize: ":context.presentationStyle.canvasSize",
                 },
@@ -167,28 +170,59 @@ const beat_graph_data = {
             defaultValue: {},
         },
         audioChecker: {
-            if: ":preprocessor.movieFile",
             agent: async (namedInputs) => {
-                const { hasAudio } = await ffmpegGetMediaDuration(namedInputs.movieFile);
+                if (namedInputs.soundEffectFile) {
+                    return { hasMovieAudio: true };
+                }
+                const sourceFile = namedInputs.movieFile || namedInputs.imageFile;
+                if (!sourceFile) {
+                    return { hasMovieAudio: false };
+                }
+                const { hasAudio } = await ffmpegGetMediaDuration(sourceFile);
                 return { hasMovieAudio: hasAudio };
             },
+            inputs: {
+                onComplete: [":movieGenerator", ":htmlImageGenerator", ":soundEffectGenerator"], // to wait for movieGenerator and htmlImageGenerator to finish
+                movieFile: ":preprocessor.movieFile",
+                imageFile: ":preprocessor.imagePath",
+                soundEffectFile: ":preprocessor.soundEffectFile",
+            },
+        },
+        soundEffectGenerator: {
+            if: ":preprocessor.soundEffectPrompt",
+            agent: ":preprocessor.soundEffectAgentInfo.agentName",
             inputs: {
                 onComplete: [":movieGenerator"], // to wait for movieGenerator to finish
+                prompt: ":preprocessor.soundEffectPrompt",
                 movieFile: ":preprocessor.movieFile",
+                soundEffectFile: ":preprocessor.soundEffectFile",
+                params: {
+                    model: ":preprocessor.soundEffectModel",
+                    duration: ":beat.duration",
+                },
+                cache: {
+                    force: [":context.force"],
+                    file: ":preprocessor.soundEffectFile",
+                    index: ":__mapIndex",
+                    sessionType: "soundEffect",
+                    mulmoContext: ":context",
+                },
             },
             defaultValue: {},
         },
         output: {
             agent: "copyAgent",
             inputs: {
-                onComplete: [":imageFromMovie", ":htmlImageGenerator", ":audioChecker"], // to wait for imageFromMovie to finish
+                onComplete: [":imageFromMovie", ":htmlImageGenerator", ":audioChecker", ":soundEffectGenerator"], // to wait for imageFromMovie to finish
                 imageFile: ":preprocessor.imagePath",
                 movieFile: ":preprocessor.movieFile",
+                soundEffectFile: ":preprocessor.soundEffectFile",
                 hasMovieAudio: ":audioChecker.hasMovieAudio",
             },
             output: {
                 imageFile: ".imageFile",
                 movieFile: ".movieFile",
+                soundEffectFile: ".soundEffectFile",
                 hasMovieAudio: ".hasMovieAudio",
             },
             isResult: true,
@@ -201,7 +235,6 @@ const graph_data = {
     nodes: {
         context: {},
         htmlImageAgentInfo: {},
-        movieAgentInfo: {},
         outputStudioFilePath: {},
         imageRefs: {},
         map: {
@@ -210,7 +243,6 @@ const graph_data = {
                 rows: ":context.studio.script.beats",
                 context: ":context",
                 htmlImageAgentInfo: ":htmlImageAgentInfo",
-                movieAgentInfo: ":movieAgentInfo",
                 imageRefs: ":imageRefs",
             },
             isResult: true,
@@ -284,7 +316,7 @@ export const graphOption = async (context, settings) => {
             {
                 name: "fileCacheAgentFilter",
                 agent: fileCacheAgentFilter,
-                nodeIds: ["imageGenerator", "movieGenerator", "htmlImageAgent"],
+                nodeIds: ["imageGenerator", "movieGenerator", "htmlImageAgent", "soundEffectGenerator"],
             },
         ],
         taskManager: new TaskManager(MulmoPresentationStyleMethods.getConcurrency(context.presentationStyle)),
@@ -314,9 +346,6 @@ const prepareGenerateImages = async (context) => {
     const injections = {
         context,
         htmlImageAgentInfo,
-        movieAgentInfo: {
-            agent: MulmoPresentationStyleMethods.getMovieAgent(context.presentationStyle),
-        },
         outputStudioFilePath: getOutputStudioFilePath(outDirPath, fileName),
         imageRefs,
     };

package/lib/actions/movie.js

@@ -162,7 +162,7 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, context) => {
             beatTimestamps.push(timestamp);
             return timestamp; // Skip voice-over beats.
         }
-        const sourceFile = studioBeat.movieFile ?? studioBeat.imageFile;
+        const sourceFile = studioBeat.soundEffectFile ?? studioBeat.movieFile ?? studioBeat.imageFile;
         assert(!!sourceFile, `studioBeat.imageFile or studioBeat.movieFile is not set: index=${index}`);
         assert(!!studioBeat.duration, `studioBeat.duration is not set: index=${index}`);
         const extraPadding = (() => {

package/lib/agents/index.d.ts

@@ -10,8 +10,9 @@ import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
 import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
 import ttsOpenaiAgent from "./tts_openai_agent.js";
 import validateSchemaAgent from "./validate_schema_agent.js";
+import soundEffectReplicateAgent from "./sound_effect_replicate_agent.js";
 import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, soundEffectReplicateAgent, };

package/lib/agents/index.js

@@ -10,9 +10,10 @@ import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
 import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
 import ttsOpenaiAgent from "./tts_openai_agent.js";
 import validateSchemaAgent from "./validate_schema_agent.js";
+import soundEffectReplicateAgent from "./sound_effect_replicate_agent.js";
 import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 // import * as vanilla from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, soundEffectReplicateAgent, };

package/lib/agents/movie_replicate_agent.js

@@ -1,6 +1,7 @@
 import { readFileSync } from "fs";
 import { GraphAILogger } from "graphai";
 import Replicate from "replicate";
+import { provider2MovieAgent } from "../utils/provider2agent.js";
 async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, duration) {
     const replicate = new Replicate({
         auth: apiKey,
@@ -10,6 +11,7 @@ async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, dura
         duration,
         image: undefined,
         start_image: undefined,
+        first_frame_image: undefined,
         aspect_ratio: aspectRatio, // only for bytedance/seedance-1-lite
         // resolution: "720p", // only for bytedance/seedance-1-lite
         // fps: 24, // only for bytedance/seedance-1-lite
@@ -21,15 +23,19 @@ async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, dura
     if (imagePath) {
         const buffer = readFileSync(imagePath);
         const base64Image = `data:image/png;base64,${buffer.toString("base64")}`;
-        if (model === "kwaivgi/kling-v2.1" || model === "kwaivgi/kling-v1.6-pro") {
-            input.start_image = base64Image;
+        const start_image = provider2MovieAgent.replicate.modelParams[model]?.start_image;
+        if (start_image === "first_frame_image" || start_image === "image" || start_image === "start_image") {
+            input[start_image] = base64Image;
+        }
+        else if (start_image === undefined) {
+            throw new Error(`Model ${model} does not support image-to-video generation`);
         }
         else {
             input.image = base64Image;
         }
     }
     try {
-        const output = await replicate.run(model ?? "bytedance/seedance-1-lite", { input });
+        const output = await replicate.run(model, { input });
         // Download the generated video
         if (output && typeof output === "object" && "url" in output) {
             const videoUrl = output.url();
@@ -61,13 +67,20 @@ export const getAspectRatio = (canvasSize) => {
 export const movieReplicateAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, imagePath } = namedInputs;
     const aspectRatio = getAspectRatio(params.canvasSize);
-    const duration = params.duration ?? 5;
+    const model = params.model ?? provider2MovieAgent.replicate.defaultModel;
+    if (!provider2MovieAgent.replicate.modelParams[model]) {
+        throw new Error(`Model ${model} is not supported`);
+    }
+    const duration = params.duration ?? provider2MovieAgent.replicate.modelParams[model].durations[0] ?? 5;
+    if (!provider2MovieAgent.replicate.modelParams[model].durations.includes(duration)) {
+        throw new Error(`Duration ${duration} is not supported for model ${model}. Supported durations: ${provider2MovieAgent.replicate.modelParams[model].durations.join(", ")}`);
+    }
     const apiKey = config?.apiKey;
     if (!apiKey) {
         throw new Error("REPLICATE_API_TOKEN environment variable is required");
     }
     try {
-        const buffer = await generateMovie(params.model, apiKey, prompt, imagePath, aspectRatio, duration);
+        const buffer = await generateMovie(model, apiKey, prompt, imagePath, aspectRatio, duration);
         if (buffer) {
             return { buffer };
         }

package/lib/agents/sound_effect_replicate_agent.d.ts

@@ -0,0 +1,5 @@
+import type { AgentFunction, AgentFunctionInfo } from "graphai";
+import type { AgentBufferResult, SoundEffectAgentInputs, ReplicateSoundEffectAgentParams, ReplicateSoundEffectAgentConfig } from "../types/agent.js";
+export declare const soundEffectReplicateAgent: AgentFunction<ReplicateSoundEffectAgentParams, AgentBufferResult, SoundEffectAgentInputs, ReplicateSoundEffectAgentConfig>;
+declare const soundEffectReplicateAgentInfo: AgentFunctionInfo;
+export default soundEffectReplicateAgentInfo;

package/lib/agents/sound_effect_replicate_agent.js

@@ -0,0 +1,59 @@
+import { readFileSync } from "fs";
+import { GraphAILogger } from "graphai";
+import Replicate from "replicate";
+import { provider2SoundEffectAgent } from "../utils/provider2agent.js";
+export const soundEffectReplicateAgent = async ({ namedInputs, params, config }) => {
+    const { prompt, movieFile } = namedInputs;
+    const apiKey = config?.apiKey;
+    const model = params.model ?? provider2SoundEffectAgent.replicate.defaultModel;
+    if (!apiKey) {
+        throw new Error("REPLICATE_API_TOKEN environment variable is required");
+    }
+    const replicate = new Replicate({
+        auth: apiKey,
+    });
+    const buffer = readFileSync(movieFile);
+    const uri = `data:video/quicktime;base64,${buffer.toString("base64")}`;
+    const input = {
+        video: uri,
+        prompt,
+        duration: params.duration,
+        // seed: -1,
+        // num_steps: 25,
+        // cfg_strength: 4.5,
+        // negative_prompt: "music"
+    };
+    try {
+        const model_identifier = provider2SoundEffectAgent.replicate.modelParams[model]?.identifier ?? model;
+        const output = await replicate.run(model_identifier, {
+            input,
+        });
+        if (output && typeof output === "object" && "url" in output) {
+            const videoUrl = output.url();
+            const videoResponse = await fetch(videoUrl);
+            if (!videoResponse.ok) {
+                throw new Error(`Error downloading video: ${videoResponse.status} - ${videoResponse.statusText}`);
+            }
+            const arrayBuffer = await videoResponse.arrayBuffer();
+            return { buffer: Buffer.from(arrayBuffer) };
+        }
+        return undefined;
+    }
+    catch (error) {
+        GraphAILogger.info("Failed to generate sound effect:", error.message);
+        throw error;
+    }
+};
+const soundEffectReplicateAgentInfo = {
+    name: "soundEffectReplicateAgent",
+    agent: soundEffectReplicateAgent,
+    mock: soundEffectReplicateAgent,
+    samples: [],
+    description: "Replicate Sound Effect agent (movie to movie)",
+    category: ["movie"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    license: "MIT",
+    environmentVariables: ["REPLICATE_API_TOKEN"],
+};
+export default soundEffectReplicateAgentInfo;

package/lib/cli/commands/tool/scripting/builder.js

@@ -1,4 +1,4 @@
-import { llm } from "../../../../utils/utils.js";
+import { llm } from "../../../../utils/provider2agent.js";
 import { getAvailableTemplates } from "../../../../utils/file.js";
 const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
 export const builder = (yargs) => {

package/lib/cli/commands/tool/scripting/handler.d.ts

@@ -1,5 +1,5 @@
 import { ToolCliArgs } from "../../../../types/cli_types.js";
-import { LLM } from "../../../../utils/utils.js";
+import type { LLM } from "../../../../utils/provider2agent.js";
 export declare const handler: (argv: ToolCliArgs<{
     o?: string;
     b?: string;

package/lib/cli/commands/tool/story_to_script/builder.js

@@ -1,5 +1,5 @@
 import { getAvailableTemplates } from "../../../../utils/file.js";
-import { llm } from "../../../../utils/utils.js";
+import { llm } from "../../../../utils/provider2agent.js";
 import { storyToScriptGenerateMode } from "../../../../utils/const.js";
 const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
 export const builder = (yargs) => {

package/lib/cli/commands/tool/story_to_script/handler.d.ts

@@ -1,5 +1,5 @@
 import { ToolCliArgs } from "../../../../types/cli_types.js";
-import { LLM } from "../../../../utils/utils.js";
+import type { LLM } from "../../../../utils/provider2agent.js";
 export declare const handler: (argv: ToolCliArgs<{
     o?: string;
     b?: string;

package/lib/mcp/server.js

@@ -11,7 +11,7 @@ import { audio, images, movie, captions, pdf } from "../actions/index.js";
 import { initializeContext, runTranslateIfNeeded } from "../cli/helpers.js";
 import { outDirName } from "../utils/const.js";
 import { resolveDirPath, mkdir, generateTimestampedFileName } from "../utils/file.js";
-import { mulmoScriptSchema } from "../types/schema.js";
+import { MulmoScriptMethods } from "../methods/index.js";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 // Load MulmoScript JSON Schema from file
@@ -83,7 +83,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         }
         const { cmd, mulmoScript, options = {}, } = args;
         // Validate MulmoScript schema
-        const validatedScript = mulmoScriptSchema.parse(mulmoScript);
+        const validatedScript = MulmoScriptMethods.validate(mulmoScript);
         // Save MulmoScript to output directory
         const filePath = await saveMulmoScriptToOutput(validatedScript);
         // Create argv-like object for CLI compatibility

package/lib/methods/index.d.ts

@@ -3,3 +3,4 @@ export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
 export * from "./mulmo_beat.js";
+export * from "./mulmo_script.js";

package/lib/methods/index.js

@@ -3,3 +3,4 @@ export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
 export * from "./mulmo_beat.js";
+export * from "./mulmo_script.js";

package/lib/methods/mulmo_presentation_style.d.ts

@@ -1,18 +1,37 @@
 import "dotenv/config";
-import { MulmoCanvasDimension, MulmoBeat, SpeechOptions, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider } from "../types/index.js";
+import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider } from "../types/index.js";
 export declare const MulmoPresentationStyleMethods: {
     getCanvasSize(presentationStyle: MulmoPresentationStyle): MulmoCanvasDimension;
-    getSpeechProvider(presentationStyle: MulmoPresentationStyle): Text2SpeechProvider;
     getAllSpeechProviders(presentationStyle: MulmoPresentationStyle): Set<Text2SpeechProvider>;
     getTextSlideStyle(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
-    getSpeechOptions(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SpeechOptions | undefined;
+    getDefaultSpeaker(presentationStyle: MulmoPresentationStyle): string;
     getSpeaker(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SpeakerData;
-    getTTSProvider(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): Text2SpeechProvider;
     getTTSModel(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string | undefined;
-    getVoiceId(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
     getText2ImageProvider(provider: Text2ImageProvider | undefined): Text2ImageProvider;
     getImageAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): Text2ImageAgentInfo;
-    getMovieAgent(presentationStyle: MulmoPresentationStyle): string;
+    getMovieAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): {
+        agent: string;
+        movieParams: {
+            speed?: number | undefined;
+            provider?: string | undefined;
+            model?: string | undefined;
+            fillOption?: {
+                style: "aspectFit" | "aspectFill";
+            } | undefined;
+            transition?: {
+                type: "fade" | "slideout_left";
+                duration: number;
+            } | undefined;
+        };
+    };
+    getSoundEffectAgentInfo(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): {
+        agentName: string;
+        defaultModel: import("../utils/provider2agent.js").ReplicateModel;
+        models: import("../utils/provider2agent.js").ReplicateModel[];
+        modelParams: Record<import("../utils/provider2agent.js").ReplicateModel, {
+            identifier?: `${string}/${string}:${string}`;
+        }>;
+    };
     getConcurrency(presentationStyle: MulmoPresentationStyle): 4 | 16;
     getHtmlImageAgentInfo(presentationStyle: MulmoPresentationStyle): Text2HtmlAgentInfo;
     getImageType(_: MulmoPresentationStyle, beat: MulmoBeat): BeatMediaType;

package/lib/methods/mulmo_presentation_style.js

@@ -1,7 +1,8 @@
 import "dotenv/config";
+import { isNull } from "graphai";
 import { userAssert } from "../utils/utils.js";
-import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema } from "../types/schema.js";
-import { defaultProviders, provider2ImageAgent, provider2MovieAgent, provider2LLMAgent } from "../utils/provider2agent.js";
+import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, } from "../types/schema.js";
+import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2SoundEffectAgent, defaultProviders, } from "../utils/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
@@ -20,14 +21,10 @@ export const MulmoPresentationStyleMethods = {
     getCanvasSize(presentationStyle) {
         return mulmoCanvasDimensionSchema.parse(presentationStyle.canvasSize);
     },
-    getSpeechProvider(presentationStyle) {
-        return text2SpeechProviderSchema.parse(presentationStyle.speechParams?.provider);
-    },
     getAllSpeechProviders(presentationStyle) {
         const providers = new Set();
-        const defaultProvider = this.getSpeechProvider(presentationStyle);
         Object.values(presentationStyle.speechParams.speakers).forEach((speaker) => {
-            const provider = speaker.provider ?? defaultProvider;
+            const provider = text2SpeechProviderSchema.parse(speaker.provider);
             providers.add(provider);
         });
         return providers;
@@ -39,27 +36,27 @@ export const MulmoPresentationStyleMethods = {
         // This code allows us to support both string and array of strings for cssStyles
         return [...defaultTextSlideStyles, ...[styles], ...[extraStyles]].flat().join("\n");
     },
-    getSpeechOptions(presentationStyle, beat) {
-        return { ...presentationStyle.speechParams.speakers[beat.speaker].speechOptions, ...beat.speechOptions };
+    getDefaultSpeaker(presentationStyle) {
+        const speakers = presentationStyle.speechParams.speakers ?? {};
+        const keys = Object.keys(speakers).sort();
+        userAssert(keys.length !== 0, "presentationStyle.speechParams.speakers is not set!!");
+        const defaultSpeaker = keys.find((key) => speakers[key].isDefault);
+        if (!isNull(defaultSpeaker)) {
+            return defaultSpeaker;
+        }
+        return keys[0];
     },
     getSpeaker(presentationStyle, beat) {
         userAssert(!!presentationStyle?.speechParams?.speakers, "presentationStyle.speechParams.speakers is not set!!");
-        userAssert(!!beat?.speaker, "beat.speaker is not set");
-        const speaker = presentationStyle.speechParams.speakers[beat.speaker];
-        userAssert(!!speaker, `speaker is not set: speaker "${beat.speaker}"`);
+        const speakerId = beat?.speaker ?? MulmoPresentationStyleMethods.getDefaultSpeaker(presentationStyle);
+        userAssert(!!speakerId, "beat.speaker and default speaker is not set");
+        const speaker = presentationStyle.speechParams.speakers[speakerId];
+        userAssert(!!speaker, `speaker is not set: speaker "${speakerId}"`);
         return speaker;
     },
-    getTTSProvider(presentationStyle, beat) {
-        const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
-        return speaker.provider ?? presentationStyle.speechParams.provider;
-    },
     getTTSModel(presentationStyle, beat) {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
-        return speaker.model ?? presentationStyle.speechParams.model;
-    },
-    getVoiceId(presentationStyle, beat) {
-        const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
-        return speaker.voiceId;
+        return speaker.model;
     },
     getText2ImageProvider(provider) {
         return text2ImageProviderSchema.parse(provider);
@@ -80,17 +77,23 @@ export const MulmoPresentationStyleMethods = {
             imageParams: { ...defaultImageParams, ...imageParams },
         };
     },
-    // Determine movie agent based on provider
-    getMovieAgent(presentationStyle) {
-        const movieProvider = (presentationStyle.movieParams?.provider ?? defaultProviders.text2movie);
-        return provider2MovieAgent[movieProvider].agentName;
+    getMovieAgentInfo(presentationStyle, beat) {
+        const movieParams = { ...presentationStyle.movieParams, ...beat?.movieParams };
+        const movieProvider = text2MovieProviderSchema.parse(movieParams?.provider);
+        const agentInfo = provider2MovieAgent[movieProvider];
+        return {
+            agent: agentInfo.agentName,
+            movieParams,
+        };
+    },
+    getSoundEffectAgentInfo(presentationStyle, beat) {
+        const soundEffectProvider = (beat.soundEffectParams?.provider ??
+            presentationStyle.soundEffectParams?.provider ??
+            defaultProviders.soundEffect);
+        const agentInfo = provider2SoundEffectAgent[soundEffectProvider];
+        return agentInfo;
     },
     getConcurrency(presentationStyle) {
-        /*
-        if (presentationStyle.movieParams?.provider === "replicate") {
-          return 4;
-        }
-        */
         const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(presentationStyle);
         if (imageAgentInfo.imageParams.provider === "openai") {
             // NOTE: Here are the rate limits of OpenAI's text2image API (1token = 32x32 patch).

package/lib/methods/mulmo_script.d.ts

@@ -0,0 +1,4 @@
+import { MulmoScript } from "../types/index.js";
+export declare const MulmoScriptMethods: {
+    validate(script: any): MulmoScript;
+};

package/lib/methods/mulmo_script.js

@@ -0,0 +1,31 @@
+import { mulmoScriptSchema } from "../types/index.js";
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const validate_1_0 = (script) => {
+    if (script.speechParams?.provider) {
+        if (typeof script.speechParams.speakers === "object") {
+            Object.keys(script.speechParams.speakers).forEach((speakerId) => {
+                const speaker = script.speechParams.speakers[speakerId];
+                if (!speaker.provider) {
+                    speaker.provider = script.speechParams.provider;
+                }
+            });
+        }
+        delete script.speechParams.provider;
+    }
+    return script;
+};
+const validators = [{ from: "1.0", to: "1.1", validator: validate_1_0 }];
+export const MulmoScriptMethods = {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    validate(script) {
+        const validatedScript = validators.reduce((acc, validator) => {
+            if (acc.$mulmocast.version === validator.from) {
+                const validated = validator.validator(acc);
+                validated.$mulmocast.version = validator.to;
+                return validated;
+            }
+            return acc;
+        }, script);
+        return mulmoScriptSchema.parse(validatedScript);
+    },
+};

package/lib/tools/story_to_script.d.ts

@@ -1,5 +1,5 @@
 import { MulmoStoryboard, StoryToScriptGenerateMode } from "../types/index.js";
-import { LLM } from "../utils/utils.js";
+import type { LLM } from "../utils/provider2agent.js";
 export declare const storyToScript: ({ story, beatsPerScene, templateName, outdir, fileName, llm, llmModel, generateMode, }: {
     story: MulmoStoryboard;
     beatsPerScene: number;

package/lib/types/agent.d.ts

@@ -58,8 +58,17 @@ export type ReplicateMovieAgentParams = {
     };
     duration?: number;
 };
+export type ReplicateSoundEffectAgentParams = {
+    model: `${string}/${string}` | undefined;
+    duration?: number;
+};
+export type SoundEffectAgentInputs = AgentPromptInputs & {
+    soundEffectFile: string;
+    movieFile: string;
+};
 export type GoogleMovieAgentConfig = GoogleImageAgentConfig;
 export type ReplicateMovieAgentConfig = AgentConfig;
+export type ReplicateSoundEffectAgentConfig = AgentConfig;
 export type TTSAgentParams = {
     suppressError: boolean;
     voice: string;