mulmocast 0.1.6 → 1.1.0

package/lib/types/schema.js

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { htmlLLMProvider, provider2TTSAgent, provider2ImageAgent, provider2MovieAgent, defaultProviders } from "../utils/provider2agent.js";
+import { htmlLLMProvider, provider2TTSAgent, provider2ImageAgent, provider2MovieAgent, defaultProviders, provider2SoundEffectAgent, } from "../utils/provider2agent.js";
 export const langSchema = z.string();
 const URLStringSchema = z.string().url();
 export const localizedTextSchema = z
@@ -26,6 +26,7 @@ export const speakerDataSchema = z
     .object({
     displayName: z.record(langSchema, z.string()).optional(),
     voiceId: z.string(),
+    isDefault: z.boolean().optional(),
     speechOptions: speechOptionsSchema.optional(),
     provider: text2SpeechProviderSchema.optional(),
     model: z.string().optional().describe("TTS model to use for this speaker"),
@@ -96,7 +97,7 @@ export const mulmoChartMediaSchema = z
     .object({
     type: z.literal("chart"),
     title: z.string(),
-    chartData: z.record(z.any()),
+    chartData: z.record(z.string(), z.any()),
 })
     .strict();
 export const mulmoMermaidMediaSchema = z
@@ -223,12 +224,18 @@ export const htmlPromptParamsSchema = z
     systemPrompt: z.string().default("").optional(),
     prompt: z.string().default(""),
     data: z.any().optional(),
-    images: z.record(z.any()).optional(),
+    images: z.record(z.string(), z.any()).optional(),
 })
     .strict();
+export const text2MovieProviderSchema = z.enum(Object.keys(provider2MovieAgent)).default(defaultProviders.text2movie);
+export const text2SoundEffectProviderSchema = z.enum(Object.keys(provider2SoundEffectAgent)).default(defaultProviders.soundEffect);
+export const mulmoSoundEffectParamsSchema = z.object({
+    provider: text2SoundEffectProviderSchema.optional(),
+    model: z.string().optional(), // default: provider specific
+});
 export const mulmoBeatSchema = z
     .object({
-    speaker: speakerIdSchema.default("Presenter"),
+    speaker: speakerIdSchema.optional(),
     text: z.string().default("").describe("Text to be spoken. If empty, the audio is not generated."),
     id: z.string().optional().describe("Unique identifier for the beat."),
     description: z.string().optional(),
@@ -239,11 +246,13 @@ export const mulmoBeatSchema = z
     audioParams: beatAudioParamsSchema.optional(), // beat specific parameters
     movieParams: z
         .object({
+        provider: text2MovieProviderSchema.optional(),
         model: z.string().optional(),
-        fillOption: mulmoFillOptionSchema.optional(),
-        speed: z.number().optional().describe("Speed of the video. 1.0 is normal speed. 0.5 is half speed. 2.0 is double speed."),
+        fillOption: mulmoFillOptionSchema.optional(), // for movie.ts
+        speed: z.number().optional().describe("Speed of the video. 1.0 is normal speed. 0.5 is half speed. 2.0 is double speed."), // for movie.ts
     })
         .optional(),
+    soundEffectParams: mulmoSoundEffectParamsSchema.optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema.optional(),
     speechOptions: speechOptionsSchema.optional(),
     textSlideParams: textSlideParamsSchema.optional(),
@@ -251,6 +260,7 @@ export const mulmoBeatSchema = z
     imageNames: z.array(imageIdSchema).optional(), // list of image names to use for image generation. The default is all images in the imageParams.images.
     imagePrompt: z.string().optional(),
     moviePrompt: z.string().optional(),
+    soundEffectPrompt: z.string().optional(),
     htmlPrompt: htmlPromptParamsSchema.optional(),
 })
     .strict();
@@ -263,19 +273,11 @@ export const mulmoCanvasDimensionSchema = z
 // export const voiceMapSchema = z.record(speakerIdSchema, z.string())
 export const mulmoCastCreditSchema = z
     .object({
-    version: z.literal("1.0"),
+    version: z.literal("1.1"),
     credit: z.literal("closing").optional(),
 })
     .strict();
-export const mulmoSpeechParamsSchema = z
-    .object({
-    provider: text2SpeechProviderSchema, // has default value
-    speakers: speakerDictionarySchema,
-    model: z.string().optional().describe("Default TTS model to use"),
-})
-    .strict();
 export const text2HtmlImageProviderSchema = z.enum(htmlLLMProvider).default(defaultProviders.text2Html);
-export const text2MovieProviderSchema = z.enum(Object.keys(provider2MovieAgent)).default(defaultProviders.text2movie);
 // NOTE: This is UI only. (until we figure out how to use it in mulmoMovieParamsSchema)
 export const mulmoGoogleMovieModelSchema = z
     .object({
@@ -296,27 +298,40 @@ export const mulmoTransitionSchema = z.object({
 });
 export const mulmoMovieParamsSchema = z
     .object({
-    provider: text2MovieProviderSchema.optional(),
-    model: z.string().optional(), // default: provider specific
-    transition: mulmoTransitionSchema.optional(),
-    fillOption: mulmoFillOptionSchema.optional(),
+    provider: text2MovieProviderSchema.optional(), // for agent
+    model: z.string().optional(), // default: provider specific. for agent
+    transition: mulmoTransitionSchema.optional(), // for movie.ts
+    fillOption: mulmoFillOptionSchema.optional(), // for movie.ts
 })
     .strict();
+const defaultSpeaker = "Presenter";
 export const mulmoPresentationStyleSchema = z.object({
     $mulmocast: mulmoCastCreditSchema,
     canvasSize: mulmoCanvasDimensionSchema, // has default value
-    speechParams: mulmoSpeechParamsSchema.default({
+    speechParams: z
+        .object({
+        speakers: speakerDictionarySchema,
+    })
+        .default({
         speakers: {
-            Presenter: {
+            [defaultSpeaker]: {
                 voiceId: "shimmer",
                 displayName: {
-                    en: "Presenter",
+                    en: defaultSpeaker,
                 },
             },
         },
     }),
-    imageParams: mulmoImageParamsSchema.optional(),
-    movieParams: mulmoMovieParamsSchema.optional(),
+    imageParams: mulmoImageParamsSchema.optional().default({
+        provider: defaultProviders.text2image,
+        images: {},
+    }),
+    movieParams: mulmoMovieParamsSchema.optional().default({
+        provider: defaultProviders.text2movie,
+    }),
+    soundEffectParams: mulmoSoundEffectParamsSchema.optional().default({
+        provider: defaultProviders.soundEffect,
+    }),
     htmlImageParams: mulmoHtmlImageParamsSchema
         .extend({
         provider: text2HtmlImageProviderSchema,
@@ -365,6 +380,7 @@ export const mulmoStudioBeatSchema = z
     audioFile: z.string().optional(),
     imageFile: z.string().optional(), // path to the image
     movieFile: z.string().optional(), // path to the movie file
+    soundEffectFile: z.string().optional(), // path to the sound effect file
     captionFile: z.string().optional(), // path to the caption image
 })
     .strict();
@@ -389,6 +405,7 @@ export const mulmoSessionStateSchema = z.object({
         caption: z.record(z.number().int(), z.boolean()),
         html: z.record(z.number().int(), z.boolean()),
         imageReference: z.record(z.number().int(), z.boolean()),
+        soundEffect: z.record(z.number().int(), z.boolean()),
     }),
 });
 export const mulmoStudioSchema = z

package/lib/types/type.d.ts

@@ -1,11 +1,10 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
-import { LLM } from "../utils/utils.js";
+import type { LLM } from "../utils/provider2agent.js";
 import { z } from "zod";
 export type LANG = z.infer<typeof langSchema>;
 export type MulmoBeat = z.infer<typeof mulmoBeatSchema>;
 export type SpeakerDictonary = z.infer<typeof speakerDictionarySchema>;
-export type MulmoSpeechParams = z.infer<typeof mulmoSpeechParamsSchema>;
 export type SpeechOptions = z.infer<typeof speechOptionsSchema>;
 export type SpeakerData = z.infer<typeof speakerDataSchema>;
 export type MulmoImageParams = z.infer<typeof mulmoImageParamsSchema>;
@@ -36,6 +35,7 @@ export type MulmoGoogleImageModel = z.infer<typeof mulmoGoogleImageModelSchema>;
 export type MulmoGoogleMovieModel = z.infer<typeof mulmoGoogleMovieModelSchema>;
 export type MulmoReplicateMovieModel = z.infer<typeof mulmoReplicateMovieModelSchema>;
 export type MulmoImagePromptMedia = z.infer<typeof mulmoImagePromptMediaSchema>;
+export type MulmoImageAsset = z.infer<typeof mulmoImageAssetSchema>;
 export type MulmoTextSlideMedia = z.infer<typeof mulmoTextSlideMediaSchema>;
 export type MulmoMarkdownMedia = z.infer<typeof mulmoMarkdownMediaSchema>;
 export type MulmoImageMedia = z.infer<typeof mulmoImageMediaSchema>;
@@ -91,7 +91,7 @@ export type Text2HtmlAgentInfo = {
 export type BeatMediaType = "movie" | "image";
 export type StoryToScriptGenerateMode = (typeof storyToScriptGenerateMode)[keyof typeof storyToScriptGenerateMode];
 export type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
-export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html" | "imageReference";
+export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html" | "imageReference" | "soundEffect";
 export type SessionProgressEvent = {
     kind: "session";
     sessionType: SessionType;

package/lib/utils/assets.d.ts

@@ -0,0 +1,18 @@
+type BgmAsset = {
+    name: string;
+    title: string;
+    url: string;
+    suno_url: string;
+    date: string;
+    duration: string;
+    account: string;
+    original_license: string;
+    prompt: string;
+    model: string;
+};
+export type BgmAssets = {
+    license: string;
+    bgms: BgmAsset[];
+};
+export declare const bgmAssets: BgmAssets;
+export {};

package/lib/utils/assets.js

@@ -0,0 +1,101 @@
+export const bgmAssets = {
+    license: "Free to distribute as the BMG of media generated by MulmoCast, including commercial use.",
+    bgms: [
+        {
+            name: "story001.mp3",
+            title: "Whispered Melody",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story001.mp3",
+            suno_url: "https://suno.com/s/v6zer50aQJu8Y0qA",
+            date: "2025-06-17",
+            duration: "03:17",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "instrumental, smooth, piano",
+            model: "v4.5 beta",
+        },
+        {
+            name: "story002.mp3",
+            title: "Rise and Shine",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story002.mp3",
+            suno_url: "https://suno.com/s/mJnvyu3UXnkdAPfQ",
+            date: "2025-06-17",
+            duration: "04:04",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "techno, inspiring, piano",
+            model: "v4.5 beta",
+        },
+        {
+            name: "story003.mp3",
+            title: "Chasing the Sunset",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story003.mp3",
+            suno_url: "https://suno.com/s/2zGjMQ9vURJbaMZA",
+            date: "2025-06-17",
+            duration: "02:49",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "piano, inspiring, sunset",
+            model: "v4.5 beta",
+        },
+        {
+            name: "story004.mp3",
+            title: "Whispering Keys",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story004.mp3",
+            suno_url: "https://suno.com/s/0SFoBRsBWsncw6tu",
+            date: "2025-06-17",
+            duration: "04:00",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "Piano, classical, ambient",
+            model: "v4",
+        },
+        {
+            name: "story005.mp3",
+            title: "Whisper of Ivory",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story005.mp3",
+            suno_url: "https://suno.com/s/0SFoBRsBWsncw6tu",
+            date: "2025-06-17",
+            duration: "04:00",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "Piano solo, classical, ambient",
+            model: "v4",
+        },
+        {
+            name: "theme001.mp3",
+            title: "Rise of the Flame",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/theme001.mp3",
+            suno_url: "https://suno.com/s/WhYOf8oJYhBgSKET",
+            date: "2025-06-20",
+            duration: "03:23",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "Olympic Theme, classical, emotional",
+            model: "v4",
+        },
+        {
+            name: "olympic001.mp3",
+            title: "Olympic-style Theme Music",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/olympic001.mp3",
+            suno_url: "https://suno.com/s/32wpnmCrkFVvkTSQ",
+            date: "2025-07-17",
+            duration: "02:54",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "Epic orchestral fanfare in the style of John Williams' Olympic Fanfare and Theme. Bright brass fanfare, soaring strings, powerful percussion, and heroic French horn melodies. Triumphant and majestic mood, suitable for an opening ceremony or national celebration. Emphasize dynamic builds, rich harmonies, and cinematic grandeur.",
+            model: "v4.5+",
+        },
+        {
+            name: "morning001.mp3",
+            title: "Morning Dance",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/morning001.mp3",
+            suno_url: "https://suno.com/s/9MTkutZYqxeyBlwK",
+            date: "2025-07-17",
+            duration: "03:52",
+            account: "@snakajima",
+            original_license: "morning, piano solo, Japanese name, sexy",
+            prompt: "morning, piano solo, Japanese name, sexy",
+            model: "v4.5+",
+        },
+    ],
+};

package/lib/utils/context.d.ts

@@ -16,9 +16,35 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             audioFile?: string | undefined;
             imageFile?: string | undefined;
             movieFile?: string | undefined;
+            soundEffectFile?: string | undefined;
             captionFile?: string | undefined;
         }[];
         script: {
+            imageParams: {
+                provider: string;
+                model?: string | undefined;
+                style?: string | undefined;
+                moderation?: string | undefined;
+                images?: Record<string, {
+                    type: "image";
+                    source: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
+                } | {
+                    type: "imagePrompt";
+                    prompt: string;
+                }> | undefined;
+            };
             audioParams: {
                 padding: number;
                 introPadding: number;
@@ -41,8 +67,23 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     kind: "path";
                 } | undefined;
             };
+            movieParams: {
+                provider?: string | undefined;
+                model?: string | undefined;
+                fillOption?: {
+                    style: "aspectFit" | "aspectFill";
+                } | undefined;
+                transition?: {
+                    type: "fade" | "slideout_left";
+                    duration: number;
+                } | undefined;
+            };
+            soundEffectParams: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            };
             $mulmocast: {
-                version: "1.0";
+                version: "1.1";
                 credit?: "closing" | undefined;
             };
             canvasSize: {
@@ -50,10 +91,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 height: number;
             };
             speechParams: {
-                provider: string;
                 speakers: Record<string, {
                     voiceId: string;
                     displayName?: Record<string, string> | undefined;
+                    isDefault?: boolean | undefined;
                     speechOptions?: {
                         speed?: number | undefined;
                         instruction?: string | undefined;
@@ -61,16 +102,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     provider?: string | undefined;
                     model?: string | undefined;
                 }>;
-                model?: string | undefined;
             };
             beats: {
                 text: string;
-                speaker: string;
-                duration?: number | undefined;
-                speechOptions?: {
-                    speed?: number | undefined;
-                    instruction?: string | undefined;
-                } | undefined;
                 image?: {
                     type: "markdown";
                     markdown: string | string[];
@@ -175,6 +209,11 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     type: "voice_over";
                     startAt?: number | undefined;
                 } | undefined;
+                duration?: number | undefined;
+                speechOptions?: {
+                    speed?: number | undefined;
+                    instruction?: string | undefined;
+                } | undefined;
                 id?: string | undefined;
                 audio?: {
                     type: "audio";
@@ -196,6 +235,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     source: string;
                 } | undefined;
                 imagePrompt?: string | undefined;
+                speaker?: string | undefined;
                 description?: string | undefined;
                 imageParams?: {
                     provider: string;
@@ -228,11 +268,16 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | undefined;
                 movieParams?: {
                     speed?: number | undefined;
+                    provider?: string | undefined;
                     model?: string | undefined;
                     fillOption?: {
                         style: "aspectFit" | "aspectFill";
                     } | undefined;
                 } | undefined;
+                soundEffectParams?: {
+                    provider?: string | undefined;
+                    model?: string | undefined;
+                } | undefined;
                 htmlImageParams?: {
                     model?: string | undefined;
                 } | undefined;
@@ -245,6 +290,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | undefined;
                 imageNames?: string[] | undefined;
                 moviePrompt?: string | undefined;
+                soundEffectPrompt?: string | undefined;
                 htmlPrompt?: {
                     prompt: string;
                     data?: any;
@@ -255,42 +301,6 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             lang?: string | undefined;
             title?: string | undefined;
             description?: string | undefined;
-            imageParams?: {
-                provider: string;
-                model?: string | undefined;
-                style?: string | undefined;
-                moderation?: string | undefined;
-                images?: Record<string, {
-                    type: "image";
-                    source: {
-                        url: string;
-                        kind: "url";
-                    } | {
-                        kind: "base64";
-                        data: string;
-                    } | {
-                        text: string;
-                        kind: "text";
-                    } | {
-                        path: string;
-                        kind: "path";
-                    };
-                } | {
-                    type: "imagePrompt";
-                    prompt: string;
-                }> | undefined;
-            } | undefined;
-            movieParams?: {
-                provider?: string | undefined;
-                model?: string | undefined;
-                fillOption?: {
-                    style: "aspectFit" | "aspectFill";
-                } | undefined;
-                transition?: {
-                    type: "fade" | "slideout_left";
-                    duration: number;
-                } | undefined;
-            } | undefined;
             htmlImageParams?: {
                 provider: string;
                 model?: string | undefined;
@@ -333,9 +343,35 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             caption: {};
             html: {};
             imageReference: {};
+            soundEffect: {};
         };
     };
     presentationStyle: {
+        imageParams: {
+            provider: string;
+            model?: string | undefined;
+            style?: string | undefined;
+            moderation?: string | undefined;
+            images?: Record<string, {
+                type: "image";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "imagePrompt";
+                prompt: string;
+            }> | undefined;
+        };
         audioParams: {
             padding: number;
             introPadding: number;
@@ -358,8 +394,23 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 kind: "path";
             } | undefined;
         };
+        movieParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+            fillOption?: {
+                style: "aspectFit" | "aspectFill";
+            } | undefined;
+            transition?: {
+                type: "fade" | "slideout_left";
+                duration: number;
+            } | undefined;
+        };
+        soundEffectParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        };
         $mulmocast: {
-            version: "1.0";
+            version: "1.1";
             credit?: "closing" | undefined;
         };
         canvasSize: {
@@ -367,10 +418,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             height: number;
         };
         speechParams: {
-            provider: string;
             speakers: Record<string, {
                 voiceId: string;
                 displayName?: Record<string, string> | undefined;
+                isDefault?: boolean | undefined;
                 speechOptions?: {
                     speed?: number | undefined;
                     instruction?: string | undefined;
@@ -378,44 +429,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 provider?: string | undefined;
                 model?: string | undefined;
             }>;
-            model?: string | undefined;
         };
-        imageParams?: {
-            provider: string;
-            model?: string | undefined;
-            style?: string | undefined;
-            moderation?: string | undefined;
-            images?: Record<string, {
-                type: "image";
-                source: {
-                    url: string;
-                    kind: "url";
-                } | {
-                    kind: "base64";
-                    data: string;
-                } | {
-                    text: string;
-                    kind: "text";
-                } | {
-                    path: string;
-                    kind: "path";
-                };
-            } | {
-                type: "imagePrompt";
-                prompt: string;
-            }> | undefined;
-        } | undefined;
-        movieParams?: {
-            provider?: string | undefined;
-            model?: string | undefined;
-            fillOption?: {
-                style: "aspectFit" | "aspectFill";
-            } | undefined;
-            transition?: {
-                type: "fade" | "slideout_left";
-                duration: number;
-            } | undefined;
-        } | undefined;
         htmlImageParams?: {
             provider: string;
             model?: string | undefined;

package/lib/utils/context.js

@@ -58,6 +58,7 @@ const initSessionState = () => {
             caption: {},
             html: {},
             imageReference: {},
+            soundEffect: {},
         },
     };
 };
@@ -84,7 +85,7 @@ export const initializeContextFromFiles = async (files, raiseError, force, capti
     const currentStudio = readMulmoScriptFile(outputStudioFilePath);
     try {
         // validate mulmoStudioSchema. skip if __test_invalid__ is true
-        const studio = createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName, caption);
+        const studio = createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName, caption, presentationStyle);
         const multiLingual = getMultiLingual(outputMultilingualFilePath, studio.beats.length);
         return buildContext(studio, files, presentationStyle, multiLingual, force, lang);
     }

package/lib/utils/ffmpeg_utils.js

@@ -1,5 +1,6 @@
 import ffmpeg from "fluent-ffmpeg";
 import { GraphAILogger } from "graphai";
+import fs from "fs";
 export const setFfmpegPath = (ffmpegPath) => {
     ffmpeg.setFfmpegPath(ffmpegPath);
 };
@@ -61,6 +62,11 @@ export const FfmpegContextGenerateOutput = (context, output, options = []) => {
 };
 export const ffmpegGetMediaDuration = (filePath) => {
     return new Promise((resolve, reject) => {
+        // Only check file existence for local paths, not URLs
+        if (!filePath.startsWith("http://") && !filePath.startsWith("https://") && !fs.existsSync(filePath)) {
+            reject(new Error(`File not found: ${filePath}`));
+            return;
+        }
         ffmpeg.ffprobe(filePath, (err, metadata) => {
             if (err) {
                 GraphAILogger.info("Error while getting metadata:", err);

package/lib/utils/file.d.ts

@@ -25,7 +25,10 @@ export declare const getAudioArtifactFilePath: (outDirPath: string, fileName: st
 export declare const getOutputVideoFilePath: (outDirPath: string, fileName: string, lang?: string, caption?: string) => string;
 export declare const imageSuffix = "p";
 export declare const getBeatPngImagePath: (context: MulmoStudioContext, index: number) => string;
-export declare const getBeatMoviePath: (context: MulmoStudioContext, index: number) => string;
+export declare const getBeatMoviePaths: (context: MulmoStudioContext, index: number) => {
+    movieFile: string;
+    soundEffectFile: string;
+};
 export declare const getReferenceImagePath: (context: MulmoStudioContext, key: string, extension: string) => string;
 export declare const getCaptionImagePath: (context: MulmoStudioContext, index: number) => string;
 export declare const getOutputPdfFilePath: (outDirPath: string, fileName: string, pdfMode: PDFMode, lang?: string) => string;

package/lib/utils/file.js

@@ -87,13 +87,11 @@ export const getBeatPngImagePath = (context, index) => {
     }
     return `${imageProjectDirPath}/${index}${imageSuffix}.png`;
 };
-export const getBeatMoviePath = (context, index) => {
+export const getBeatMoviePaths = (context, index) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
     const beat = context.studio.script.beats[index]; // beat could be undefined only in a test case.
-    if (beat?.id) {
-        return `${imageProjectDirPath}/${beat.id}.mov`;
-    }
-    return `${imageProjectDirPath}/${index}.mov`;
+    const filename = beat?.id ? `${beat.id}` : `${index}`;
+    return { movieFile: `${imageProjectDirPath}/${filename}.mov`, soundEffectFile: `${imageProjectDirPath}/${filename}_sound.mov` };
 };
 export const getReferenceImagePath = (context, key, extension) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);