mulmocast 0.1.4 → 0.1.5

package/lib/types/schema.js

@@ -28,6 +28,7 @@ export const speakerDataSchema = z
     voiceId: z.string(),
     speechOptions: speechOptionsSchema.optional(),
     provider: text2SpeechProviderSchema.optional(),
+    model: z.string().optional().describe("TTS model to use for this speaker"),
 })
     .strict();
 export const speakerDictionarySchema = z.record(speakerIdSchema, speakerDataSchema);
@@ -130,9 +131,7 @@ export const mulmoImageAssetSchema = z.union([
     mulmoPdfMediaSchema,
     mulmoImageMediaSchema,
     mulmoSvgMediaSchema,
-    mulmoMovieMediaSchema.extend({
-        mixAudio: z.number().default(1.0),
-    }),
+    mulmoMovieMediaSchema,
     mulmoTextSlideMediaSchema,
     mulmoChartMediaSchema,
     mulmoMermaidMediaSchema,
@@ -198,6 +197,7 @@ export const textSlideParamsSchema = z
 export const beatAudioParamsSchema = z
     .object({
     padding: z.number().optional().describe("Padding between beats"), // seconds
+    movieVolume: z.number().default(1.0).describe("Audio volume of the imported or generated movie"),
 })
     .strict();
 export const mulmoHtmlImageParamsSchema = z
@@ -271,6 +271,7 @@ export const mulmoSpeechParamsSchema = z
     .object({
     provider: text2SpeechProviderSchema, // has default value
     speakers: speakerDictionarySchema,
+    model: z.string().optional().describe("Default TTS model to use"),
 })
     .strict();
 export const text2HtmlImageProviderSchema = z.enum(htmlLLMProvider).default(defaultProviders.text2Html);
@@ -360,6 +361,7 @@ export const mulmoStudioBeatSchema = z
     audioDuration: z.number().optional(),
     movieDuration: z.number().optional(),
     silenceDuration: z.number().optional(),
+    hasMovieAudio: z.boolean().optional(),
     audioFile: z.string().optional(),
     imageFile: z.string().optional(), // path to the image
     movieFile: z.string().optional(), // path to the movie file

package/lib/utils/context.d.ts

@@ -12,6 +12,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             audioDuration?: number | undefined;
             movieDuration?: number | undefined;
             silenceDuration?: number | undefined;
+            hasMovieAudio?: boolean | undefined;
             audioFile?: string | undefined;
             imageFile?: string | undefined;
             movieFile?: string | undefined;
@@ -58,7 +59,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         instruction?: string | undefined;
                     } | undefined;
                     provider?: string | undefined;
+                    model?: string | undefined;
                 }>;
+                model?: string | undefined;
             };
             beats: {
                 text: string;
@@ -119,6 +122,21 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         path: string;
                         kind: "path";
                     };
+                } | {
+                    type: "movie";
+                    source: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
                 } | {
                     type: "textSlide";
                     slide: {
@@ -156,22 +174,6 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | {
                     type: "voice_over";
                     startAt?: number | undefined;
-                } | {
-                    type: "movie";
-                    source: {
-                        url: string;
-                        kind: "url";
-                    } | {
-                        kind: "base64";
-                        data: string;
-                    } | {
-                        text: string;
-                        kind: "text";
-                    } | {
-                        path: string;
-                        kind: "path";
-                    };
-                    mixAudio: number;
                 } | undefined;
                 id?: string | undefined;
                 audio?: {
@@ -197,8 +199,8 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 description?: string | undefined;
                 imageParams?: {
                     provider: string;
-                    style?: string | undefined;
                     model?: string | undefined;
+                    style?: string | undefined;
                     moderation?: string | undefined;
                     images?: Record<string, {
                         type: "image";
@@ -221,6 +223,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     }> | undefined;
                 } | undefined;
                 audioParams?: {
+                    movieVolume: number;
                     padding?: number | undefined;
                 } | undefined;
                 movieParams?: {
@@ -254,8 +257,8 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             description?: string | undefined;
             imageParams?: {
                 provider: string;
-                style?: string | undefined;
                 model?: string | undefined;
+                style?: string | undefined;
                 moderation?: string | undefined;
                 images?: Record<string, {
                     type: "image";
@@ -373,12 +376,14 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     instruction?: string | undefined;
                 } | undefined;
                 provider?: string | undefined;
+                model?: string | undefined;
             }>;
+            model?: string | undefined;
         };
         imageParams?: {
             provider: string;
-            style?: string | undefined;
             model?: string | undefined;
+            style?: string | undefined;
             moderation?: string | undefined;
             images?: Record<string, {
                 type: "image";

package/lib/utils/ffmpeg_utils.d.ts

@@ -11,5 +11,8 @@ export declare const FfmpegContextAddInput: (context: FfmpegContext, input: stri
 export declare const FfmpegContextPushFormattedAudio: (context: FfmpegContext, sourceId: string, outputId: string, duration?: number | undefined) => void;
 export declare const FfmpegContextInputFormattedAudio: (context: FfmpegContext, input: string, duration?: number | undefined, inputOptions?: string[]) => string;
 export declare const FfmpegContextGenerateOutput: (context: FfmpegContext, output: string, options?: string[]) => Promise<number>;
-export declare const ffmpegGetMediaDuration: (filePath: string) => Promise<number>;
+export declare const ffmpegGetMediaDuration: (filePath: string) => Promise<{
+    duration: number;
+    hasAudio: boolean;
+}>;
 export declare const extractImageFromMovie: (movieFile: string, imagePath: string) => Promise<object>;

package/lib/utils/ffmpeg_utils.js

@@ -67,7 +67,8 @@ export const ffmpegGetMediaDuration = (filePath) => {
                 reject(err);
             }
             else {
-                resolve(metadata.format.duration);
+                const hasAudio = metadata.streams?.some((stream) => stream.codec_type === "audio") ?? false;
+                resolve({ duration: metadata.format.duration, hasAudio });
             }
         });
     });

package/lib/utils/preprocess.d.ts

@@ -7,6 +7,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         audioDuration?: number | undefined;
         movieDuration?: number | undefined;
         silenceDuration?: number | undefined;
+        hasMovieAudio?: boolean | undefined;
         audioFile?: string | undefined;
         imageFile?: string | undefined;
         movieFile?: string | undefined;
@@ -53,7 +54,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                     instruction?: string | undefined;
                 } | undefined;
                 provider?: string | undefined;
+                model?: string | undefined;
             }>;
+            model?: string | undefined;
         };
         beats: {
             text: string;
@@ -114,6 +117,21 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                     path: string;
                     kind: "path";
                 };
+            } | {
+                type: "movie";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
             } | {
                 type: "textSlide";
                 slide: {
@@ -151,22 +169,6 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | {
                 type: "voice_over";
                 startAt?: number | undefined;
-            } | {
-                type: "movie";
-                source: {
-                    url: string;
-                    kind: "url";
-                } | {
-                    kind: "base64";
-                    data: string;
-                } | {
-                    text: string;
-                    kind: "text";
-                } | {
-                    path: string;
-                    kind: "path";
-                };
-                mixAudio: number;
             } | undefined;
             id?: string | undefined;
             audio?: {
@@ -192,8 +194,8 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             description?: string | undefined;
             imageParams?: {
                 provider: string;
-                style?: string | undefined;
                 model?: string | undefined;
+                style?: string | undefined;
                 moderation?: string | undefined;
                 images?: Record<string, {
                     type: "image";
@@ -216,6 +218,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 }> | undefined;
             } | undefined;
             audioParams?: {
+                movieVolume: number;
                 padding?: number | undefined;
             } | undefined;
             movieParams?: {
@@ -249,8 +252,8 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         description?: string | undefined;
         imageParams?: {
             provider: string;
-            style?: string | undefined;
             model?: string | undefined;
+            style?: string | undefined;
             moderation?: string | undefined;
             images?: Record<string, {
                 type: "image";

package/lib/utils/provider2agent.d.ts

@@ -13,6 +13,8 @@ export declare const provider2TTSAgent: {
     openai: {
         agentName: string;
         hasLimitedConcurrency: boolean;
+        defaultModel: string;
+        defaultVoice: string;
     };
     google: {
         agentName: string;
@@ -21,6 +23,8 @@ export declare const provider2TTSAgent: {
     elevenlabs: {
         agentName: string;
         hasLimitedConcurrency: boolean;
+        defaultModel: string;
+        models: string[];
     };
 };
 export declare const provider2ImageAgent: {

package/lib/utils/provider2agent.js

@@ -13,6 +13,8 @@ export const provider2TTSAgent = {
     openai: {
         agentName: "ttsOpenaiAgent",
         hasLimitedConcurrency: false,
+        defaultModel: "gpt-4o-mini-tts",
+        defaultVoice: "shimmer",
     },
     google: {
         agentName: "ttsGoogleAgent",
@@ -21,6 +23,10 @@ export const provider2TTSAgent = {
     elevenlabs: {
         agentName: "ttsElevenlabsAgent",
         hasLimitedConcurrency: true,
+        defaultModel: "eleven_multilingual_v2",
+        // Models | ElevenLabs Documentation
+        // https://elevenlabs.io/docs/models
+        models: ["eleven_multilingual_v2", "eleven_turbo_v2_5", "eleven_turbo_v2", "eleven_flash_v2_5", "eleven_flash_v2"],
     },
 };
 export const provider2ImageAgent = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "description": "",
   "type": "module",
   "main": "lib/index.js",
@@ -101,7 +101,7 @@
     "ts-node": "^10.9.2",
     "tsx": "^4.20.3",
     "typescript": "^5.7.3",
-    "typescript-eslint": "^8.36.0"
+    "typescript-eslint": "^8.37.0"
   },
   "engines": {
     "node": ">=18.0.0"