mulmocast 0.1.7 → 1.1.0

package/lib/types/schema.js

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { htmlLLMProvider, provider2TTSAgent, provider2ImageAgent, provider2MovieAgent, defaultProviders } from "../utils/provider2agent.js";
+import { htmlLLMProvider, provider2TTSAgent, provider2ImageAgent, provider2MovieAgent, defaultProviders, provider2SoundEffectAgent, } from "../utils/provider2agent.js";
 export const langSchema = z.string();
 const URLStringSchema = z.string().url();
 export const localizedTextSchema = z
@@ -26,6 +26,7 @@ export const speakerDataSchema = z
     .object({
     displayName: z.record(langSchema, z.string()).optional(),
     voiceId: z.string(),
+    isDefault: z.boolean().optional(),
     speechOptions: speechOptionsSchema.optional(),
     provider: text2SpeechProviderSchema.optional(),
     model: z.string().optional().describe("TTS model to use for this speaker"),
@@ -227,10 +228,14 @@ export const htmlPromptParamsSchema = z
 })
     .strict();
 export const text2MovieProviderSchema = z.enum(Object.keys(provider2MovieAgent)).default(defaultProviders.text2movie);
-const defaultSpeaker = "Presenter";
+export const text2SoundEffectProviderSchema = z.enum(Object.keys(provider2SoundEffectAgent)).default(defaultProviders.soundEffect);
+export const mulmoSoundEffectParamsSchema = z.object({
+    provider: text2SoundEffectProviderSchema.optional(),
+    model: z.string().optional(), // default: provider specific
+});
 export const mulmoBeatSchema = z
     .object({
-    speaker: speakerIdSchema.default(defaultSpeaker),
+    speaker: speakerIdSchema.optional(),
     text: z.string().default("").describe("Text to be spoken. If empty, the audio is not generated."),
     id: z.string().optional().describe("Unique identifier for the beat."),
     description: z.string().optional(),
@@ -247,6 +252,7 @@ export const mulmoBeatSchema = z
         speed: z.number().optional().describe("Speed of the video. 1.0 is normal speed. 0.5 is half speed. 2.0 is double speed."), // for movie.ts
     })
         .optional(),
+    soundEffectParams: mulmoSoundEffectParamsSchema.optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema.optional(),
     speechOptions: speechOptionsSchema.optional(),
     textSlideParams: textSlideParamsSchema.optional(),
@@ -254,6 +260,7 @@ export const mulmoBeatSchema = z
     imageNames: z.array(imageIdSchema).optional(), // list of image names to use for image generation. The default is all images in the imageParams.images.
     imagePrompt: z.string().optional(),
     moviePrompt: z.string().optional(),
+    soundEffectPrompt: z.string().optional(),
     htmlPrompt: htmlPromptParamsSchema.optional(),
 })
     .strict();
@@ -266,17 +273,10 @@ export const mulmoCanvasDimensionSchema = z
 // export const voiceMapSchema = z.record(speakerIdSchema, z.string())
 export const mulmoCastCreditSchema = z
     .object({
-    version: z.literal("1.0"),
+    version: z.literal("1.1"),
     credit: z.literal("closing").optional(),
 })
     .strict();
-export const mulmoSpeechParamsSchema = z
-    .object({
-    provider: text2SpeechProviderSchema, // has default value
-    speakers: speakerDictionarySchema,
-    model: z.string().optional().describe("Default TTS model to use"),
-})
-    .strict();
 export const text2HtmlImageProviderSchema = z.enum(htmlLLMProvider).default(defaultProviders.text2Html);
 // NOTE: This is UI only. (until we figure out how to use it in mulmoMovieParamsSchema)
 export const mulmoGoogleMovieModelSchema = z
@@ -304,10 +304,15 @@ export const mulmoMovieParamsSchema = z
     fillOption: mulmoFillOptionSchema.optional(), // for movie.ts
 })
     .strict();
+const defaultSpeaker = "Presenter";
 export const mulmoPresentationStyleSchema = z.object({
     $mulmocast: mulmoCastCreditSchema,
     canvasSize: mulmoCanvasDimensionSchema, // has default value
-    speechParams: mulmoSpeechParamsSchema.default({
+    speechParams: z
+        .object({
+        speakers: speakerDictionarySchema,
+    })
+        .default({
         speakers: {
             [defaultSpeaker]: {
                 voiceId: "shimmer",
@@ -324,6 +329,9 @@ export const mulmoPresentationStyleSchema = z.object({
     movieParams: mulmoMovieParamsSchema.optional().default({
         provider: defaultProviders.text2movie,
     }),
+    soundEffectParams: mulmoSoundEffectParamsSchema.optional().default({
+        provider: defaultProviders.soundEffect,
+    }),
     htmlImageParams: mulmoHtmlImageParamsSchema
         .extend({
         provider: text2HtmlImageProviderSchema,
@@ -372,6 +380,7 @@ export const mulmoStudioBeatSchema = z
     audioFile: z.string().optional(),
     imageFile: z.string().optional(), // path to the image
     movieFile: z.string().optional(), // path to the movie file
+    soundEffectFile: z.string().optional(), // path to the sound effect file
     captionFile: z.string().optional(), // path to the caption image
 })
     .strict();
@@ -396,6 +405,7 @@ export const mulmoSessionStateSchema = z.object({
         caption: z.record(z.number().int(), z.boolean()),
         html: z.record(z.number().int(), z.boolean()),
         imageReference: z.record(z.number().int(), z.boolean()),
+        soundEffect: z.record(z.number().int(), z.boolean()),
     }),
 });
 export const mulmoStudioSchema = z

package/lib/types/type.d.ts

@@ -1,11 +1,10 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
 import type { LLM } from "../utils/provider2agent.js";
 import { z } from "zod";
 export type LANG = z.infer<typeof langSchema>;
 export type MulmoBeat = z.infer<typeof mulmoBeatSchema>;
 export type SpeakerDictonary = z.infer<typeof speakerDictionarySchema>;
-export type MulmoSpeechParams = z.infer<typeof mulmoSpeechParamsSchema>;
 export type SpeechOptions = z.infer<typeof speechOptionsSchema>;
 export type SpeakerData = z.infer<typeof speakerDataSchema>;
 export type MulmoImageParams = z.infer<typeof mulmoImageParamsSchema>;
@@ -92,7 +91,7 @@ export type Text2HtmlAgentInfo = {
 export type BeatMediaType = "movie" | "image";
 export type StoryToScriptGenerateMode = (typeof storyToScriptGenerateMode)[keyof typeof storyToScriptGenerateMode];
 export type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
-export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html" | "imageReference";
+export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html" | "imageReference" | "soundEffect";
 export type SessionProgressEvent = {
     kind: "session";
     sessionType: SessionType;

package/lib/utils/assets.d.ts

@@ -0,0 +1,18 @@
+type BgmAsset = {
+    name: string;
+    title: string;
+    url: string;
+    suno_url: string;
+    date: string;
+    duration: string;
+    account: string;
+    original_license: string;
+    prompt: string;
+    model: string;
+};
+export type BgmAssets = {
+    license: string;
+    bgms: BgmAsset[];
+};
+export declare const bgmAssets: BgmAssets;
+export {};

package/lib/utils/assets.js

@@ -0,0 +1,101 @@
+export const bgmAssets = {
+    license: "Free to distribute as the BMG of media generated by MulmoCast, including commercial use.",
+    bgms: [
+        {
+            name: "story001.mp3",
+            title: "Whispered Melody",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story001.mp3",
+            suno_url: "https://suno.com/s/v6zer50aQJu8Y0qA",
+            date: "2025-06-17",
+            duration: "03:17",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "instrumental, smooth, piano",
+            model: "v4.5 beta",
+        },
+        {
+            name: "story002.mp3",
+            title: "Rise and Shine",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story002.mp3",
+            suno_url: "https://suno.com/s/mJnvyu3UXnkdAPfQ",
+            date: "2025-06-17",
+            duration: "04:04",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "techno, inspiring, piano",
+            model: "v4.5 beta",
+        },
+        {
+            name: "story003.mp3",
+            title: "Chasing the Sunset",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story003.mp3",
+            suno_url: "https://suno.com/s/2zGjMQ9vURJbaMZA",
+            date: "2025-06-17",
+            duration: "02:49",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "piano, inspiring, sunset",
+            model: "v4.5 beta",
+        },
+        {
+            name: "story004.mp3",
+            title: "Whispering Keys",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story004.mp3",
+            suno_url: "https://suno.com/s/0SFoBRsBWsncw6tu",
+            date: "2025-06-17",
+            duration: "04:00",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "Piano, classical, ambient",
+            model: "v4",
+        },
+        {
+            name: "story005.mp3",
+            title: "Whisper of Ivory",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story005.mp3",
+            suno_url: "https://suno.com/s/0SFoBRsBWsncw6tu",
+            date: "2025-06-17",
+            duration: "04:00",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "Piano solo, classical, ambient",
+            model: "v4",
+        },
+        {
+            name: "theme001.mp3",
+            title: "Rise of the Flame",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/theme001.mp3",
+            suno_url: "https://suno.com/s/WhYOf8oJYhBgSKET",
+            date: "2025-06-20",
+            duration: "03:23",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "Olympic Theme, classical, emotional",
+            model: "v4",
+        },
+        {
+            name: "olympic001.mp3",
+            title: "Olympic-style Theme Music",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/olympic001.mp3",
+            suno_url: "https://suno.com/s/32wpnmCrkFVvkTSQ",
+            date: "2025-07-17",
+            duration: "02:54",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "Epic orchestral fanfare in the style of John Williams' Olympic Fanfare and Theme. Bright brass fanfare, soaring strings, powerful percussion, and heroic French horn melodies. Triumphant and majestic mood, suitable for an opening ceremony or national celebration. Emphasize dynamic builds, rich harmonies, and cinematic grandeur.",
+            model: "v4.5+",
+        },
+        {
+            name: "morning001.mp3",
+            title: "Morning Dance",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/morning001.mp3",
+            suno_url: "https://suno.com/s/9MTkutZYqxeyBlwK",
+            date: "2025-07-17",
+            duration: "03:52",
+            account: "@snakajima",
+            original_license: "morning, piano solo, Japanese name, sexy",
+            prompt: "morning, piano solo, Japanese name, sexy",
+            model: "v4.5+",
+        },
+    ],
+};

package/lib/utils/context.d.ts

@@ -16,6 +16,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             audioFile?: string | undefined;
             imageFile?: string | undefined;
             movieFile?: string | undefined;
+            soundEffectFile?: string | undefined;
             captionFile?: string | undefined;
         }[];
         script: {
@@ -77,8 +78,12 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     duration: number;
                 } | undefined;
             };
+            soundEffectParams: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            };
             $mulmocast: {
-                version: "1.0";
+                version: "1.1";
                 credit?: "closing" | undefined;
             };
             canvasSize: {
@@ -86,10 +91,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 height: number;
             };
             speechParams: {
-                provider: string;
                 speakers: Record<string, {
                     voiceId: string;
                     displayName?: Record<string, string> | undefined;
+                    isDefault?: boolean | undefined;
                     speechOptions?: {
                         speed?: number | undefined;
                         instruction?: string | undefined;
@@ -97,16 +102,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     provider?: string | undefined;
                     model?: string | undefined;
                 }>;
-                model?: string | undefined;
             };
             beats: {
                 text: string;
-                speaker: string;
-                duration?: number | undefined;
-                speechOptions?: {
-                    speed?: number | undefined;
-                    instruction?: string | undefined;
-                } | undefined;
                 image?: {
                     type: "markdown";
                     markdown: string | string[];
@@ -211,6 +209,11 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     type: "voice_over";
                     startAt?: number | undefined;
                 } | undefined;
+                duration?: number | undefined;
+                speechOptions?: {
+                    speed?: number | undefined;
+                    instruction?: string | undefined;
+                } | undefined;
                 id?: string | undefined;
                 audio?: {
                     type: "audio";
@@ -232,6 +235,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     source: string;
                 } | undefined;
                 imagePrompt?: string | undefined;
+                speaker?: string | undefined;
                 description?: string | undefined;
                 imageParams?: {
                     provider: string;
@@ -270,6 +274,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         style: "aspectFit" | "aspectFill";
                     } | undefined;
                 } | undefined;
+                soundEffectParams?: {
+                    provider?: string | undefined;
+                    model?: string | undefined;
+                } | undefined;
                 htmlImageParams?: {
                     model?: string | undefined;
                 } | undefined;
@@ -282,6 +290,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | undefined;
                 imageNames?: string[] | undefined;
                 moviePrompt?: string | undefined;
+                soundEffectPrompt?: string | undefined;
                 htmlPrompt?: {
                     prompt: string;
                     data?: any;
@@ -334,6 +343,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             caption: {};
             html: {};
             imageReference: {};
+            soundEffect: {};
         };
     };
     presentationStyle: {
@@ -395,8 +405,12 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 duration: number;
             } | undefined;
         };
+        soundEffectParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        };
         $mulmocast: {
-            version: "1.0";
+            version: "1.1";
             credit?: "closing" | undefined;
         };
         canvasSize: {
@@ -404,10 +418,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             height: number;
         };
         speechParams: {
-            provider: string;
             speakers: Record<string, {
                 voiceId: string;
                 displayName?: Record<string, string> | undefined;
+                isDefault?: boolean | undefined;
                 speechOptions?: {
                     speed?: number | undefined;
                     instruction?: string | undefined;
@@ -415,7 +429,6 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 provider?: string | undefined;
                 model?: string | undefined;
             }>;
-            model?: string | undefined;
         };
         htmlImageParams?: {
             provider: string;

package/lib/utils/context.js

@@ -58,6 +58,7 @@ const initSessionState = () => {
             caption: {},
             html: {},
             imageReference: {},
+            soundEffect: {},
         },
     };
 };
@@ -84,7 +85,7 @@ export const initializeContextFromFiles = async (files, raiseError, force, capti
     const currentStudio = readMulmoScriptFile(outputStudioFilePath);
     try {
         // validate mulmoStudioSchema. skip if __test_invalid__ is true
-        const studio = createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName, caption);
+        const studio = createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName, caption, presentationStyle);
         const multiLingual = getMultiLingual(outputMultilingualFilePath, studio.beats.length);
         return buildContext(studio, files, presentationStyle, multiLingual, force, lang);
     }

package/lib/utils/file.d.ts

@@ -25,7 +25,10 @@ export declare const getAudioArtifactFilePath: (outDirPath: string, fileName: st
 export declare const getOutputVideoFilePath: (outDirPath: string, fileName: string, lang?: string, caption?: string) => string;
 export declare const imageSuffix = "p";
 export declare const getBeatPngImagePath: (context: MulmoStudioContext, index: number) => string;
-export declare const getBeatMoviePath: (context: MulmoStudioContext, index: number) => string;
+export declare const getBeatMoviePaths: (context: MulmoStudioContext, index: number) => {
+    movieFile: string;
+    soundEffectFile: string;
+};
 export declare const getReferenceImagePath: (context: MulmoStudioContext, key: string, extension: string) => string;
 export declare const getCaptionImagePath: (context: MulmoStudioContext, index: number) => string;
 export declare const getOutputPdfFilePath: (outDirPath: string, fileName: string, pdfMode: PDFMode, lang?: string) => string;

package/lib/utils/file.js

@@ -87,13 +87,11 @@ export const getBeatPngImagePath = (context, index) => {
     }
     return `${imageProjectDirPath}/${index}${imageSuffix}.png`;
 };
-export const getBeatMoviePath = (context, index) => {
+export const getBeatMoviePaths = (context, index) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
     const beat = context.studio.script.beats[index]; // beat could be undefined only in a test case.
-    if (beat?.id) {
-        return `${imageProjectDirPath}/${beat.id}.mov`;
-    }
-    return `${imageProjectDirPath}/${index}.mov`;
+    const filename = beat?.id ? `${beat.id}` : `${index}`;
+    return { movieFile: `${imageProjectDirPath}/${filename}.mov`, soundEffectFile: `${imageProjectDirPath}/${filename}_sound.mov` };
 };
 export const getReferenceImagePath = (context, key, extension) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);

package/lib/utils/preprocess.d.ts

@@ -1,5 +1,5 @@
-import { MulmoStudio, MulmoScript } from "../types/index.js";
-export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string, videoCaption?: string) => {
+import { MulmoStudio, MulmoScript, MulmoPresentationStyle } from "../types/index.js";
+export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string, videoCaption?: string, presentationStyle?: MulmoPresentationStyle | null) => {
     beats: {
         duration?: number | undefined;
         startAt?: number | undefined;
@@ -11,6 +11,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         audioFile?: string | undefined;
         imageFile?: string | undefined;
         movieFile?: string | undefined;
+        soundEffectFile?: string | undefined;
         captionFile?: string | undefined;
     }[];
     script: {
@@ -72,8 +73,12 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 duration: number;
             } | undefined;
         };
+        soundEffectParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        };
         $mulmocast: {
-            version: "1.0";
+            version: "1.1";
             credit?: "closing" | undefined;
         };
         canvasSize: {
@@ -81,10 +86,10 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             height: number;
         };
         speechParams: {
-            provider: string;
             speakers: Record<string, {
                 voiceId: string;
                 displayName?: Record<string, string> | undefined;
+                isDefault?: boolean | undefined;
                 speechOptions?: {
                     speed?: number | undefined;
                     instruction?: string | undefined;
@@ -92,16 +97,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 provider?: string | undefined;
                 model?: string | undefined;
             }>;
-            model?: string | undefined;
         };
         beats: {
             text: string;
-            speaker: string;
-            duration?: number | undefined;
-            speechOptions?: {
-                speed?: number | undefined;
-                instruction?: string | undefined;
-            } | undefined;
             image?: {
                 type: "markdown";
                 markdown: string | string[];
@@ -206,6 +204,11 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 type: "voice_over";
                 startAt?: number | undefined;
             } | undefined;
+            duration?: number | undefined;
+            speechOptions?: {
+                speed?: number | undefined;
+                instruction?: string | undefined;
+            } | undefined;
             id?: string | undefined;
             audio?: {
                 type: "audio";
@@ -227,6 +230,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 source: string;
             } | undefined;
             imagePrompt?: string | undefined;
+            speaker?: string | undefined;
             description?: string | undefined;
             imageParams?: {
                 provider: string;
@@ -265,6 +269,10 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                     style: "aspectFit" | "aspectFill";
                 } | undefined;
             } | undefined;
+            soundEffectParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
             htmlImageParams?: {
                 model?: string | undefined;
             } | undefined;
@@ -277,6 +285,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | undefined;
             imageNames?: string[] | undefined;
             moviePrompt?: string | undefined;
+            soundEffectPrompt?: string | undefined;
             htmlPrompt?: {
                 prompt: string;
                 data?: any;

package/lib/utils/preprocess.js

@@ -1,5 +1,6 @@
 import { GraphAILogger } from "graphai";
-import { mulmoScriptSchema, mulmoStudioSchema, mulmoCaptionParamsSchema } from "../types/index.js";
+import { mulmoStudioSchema, mulmoCaptionParamsSchema } from "../types/index.js";
+import { MulmoPresentationStyleMethods, MulmoScriptMethods } from "../methods/index.js";
 const rebuildStudio = (currentStudio, mulmoScript, fileName) => {
     const isTest = process.env.NODE_ENV === "test";
     const parsed = isTest && currentStudio ? { data: mulmoStudioSchema.parse(currentStudio), success: true, error: null } : mulmoStudioSchema.safeParse(currentStudio);
@@ -36,15 +37,16 @@ const mulmoCredit = (speaker) => {
         },
     };
 };
-export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName, videoCaption) => {
-    const mulmoScript = _mulmoScript.__test_invalid__ ? _mulmoScript : mulmoScriptSchema.parse(_mulmoScript); // validate and insert default value
+export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName, videoCaption, presentationStyle) => {
+    const mulmoScript = _mulmoScript.__test_invalid__ ? _mulmoScript : MulmoScriptMethods.validate(_mulmoScript); // validate and insert default value
     const studio = rebuildStudio(currentStudio, mulmoScript, fileName);
     // TODO: Move this code out of this function later
     // Addition cloing credit
     if (mulmoScript.$mulmocast.credit === "closing") {
-        mulmoScript.beats.push(mulmoCredit(mulmoScript.beats[0].speaker)); // First speaker
+        const defaultSpeaker = MulmoPresentationStyleMethods.getDefaultSpeaker(presentationStyle ?? studio.script);
+        mulmoScript.beats.push(mulmoCredit(mulmoScript.beats[0].speaker ?? defaultSpeaker)); // First speaker
     }
-    studio.script = mulmoScriptSchema.parse(mulmoScript); // update the script
+    studio.script = MulmoScriptMethods.validate(mulmoScript); // update the script
     studio.beats = studio.script.beats.map((_, index) => studio.beats[index] ?? {});
     if (videoCaption) {
         studio.script.captionParams = mulmoCaptionParamsSchema.parse({

package/lib/utils/provider2agent.d.ts

@@ -32,11 +32,18 @@ export declare const provider2ImageAgent: {
         models: string[];
     };
 };
+export type ReplicateModel = `${string}/${string}`;
 export declare const provider2MovieAgent: {
     replicate: {
         agentName: string;
-        defaultModel: `${string}/${string}`;
+        defaultModel: ReplicateModel;
         models: string[];
+        modelParams: Record<ReplicateModel, {
+            durations: number[];
+            start_image: string | undefined;
+            last_image?: string;
+            price_per_sec: number;
+        }>;
     };
     google: {
         agentName: string;
@@ -44,6 +51,16 @@ export declare const provider2MovieAgent: {
         models: string[];
     };
 };
+export declare const provider2SoundEffectAgent: {
+    replicate: {
+        agentName: string;
+        defaultModel: ReplicateModel;
+        models: ReplicateModel[];
+        modelParams: Record<ReplicateModel, {
+            identifier?: `${string}/${string}:${string}`;
+        }>;
+    };
+};
 export declare const provider2LLMAgent: {
     readonly openai: {
         readonly agentName: "openAIAgent";
@@ -72,6 +89,7 @@ export declare const defaultProviders: {
     text2movie: keyof typeof provider2MovieAgent;
     text2Html: keyof typeof provider2LLMAgent;
     llm: keyof typeof provider2LLMAgent;
+    soundEffect: keyof typeof provider2SoundEffectAgent;
 };
 export declare const llm: (keyof typeof provider2LLMAgent)[];
 export type LLM = keyof typeof provider2LLMAgent;