mulmocast 1.1.4 → 1.1.5

package/lib/types/schema.js

@@ -32,7 +32,9 @@ export const speakerDataSchema = z
     model: z.string().optional().describe("TTS model to use for this speaker"),
 })
     .strict();
-export const speakerDictionarySchema = z.record(speakerIdSchema, speakerDataSchema);
+export const speakerDictionarySchema = z.record(speakerIdSchema, speakerDataSchema.extend({
+    lang: z.record(langSchema, speakerDataSchema).optional(),
+}));
 export const mediaSourceSchema = z.discriminatedUnion("kind", [
     z.object({ kind: z.literal("url"), url: URLStringSchema }).strict(), // https://example.com/foo.pdf
     z.object({ kind: z.literal("base64"), data: z.string() }).strict(), // base64
@@ -172,6 +174,7 @@ export const mulmoOpenAIImageModelSchema = z
     .object({
     provider: z.literal("openai"),
     model: z.enum(provider2ImageAgent["openai"].models).optional(),
+    quality: z.enum(["low", "medium", "high", "auto"]).optional(),
 })
     .strict();
 // NOTE: This is for UI only. (until we figure out how to use it in mulmoImageParamsSchema)
@@ -185,6 +188,7 @@ export const mulmoImageParamsSchema = z
     .object({
     provider: text2ImageProviderSchema, // has default value
     model: z.string().optional(), // default: provider specific
+    quality: z.string().optional(), // optional image quality (model specific)
     style: z.string().optional(), // optional image style
     moderation: z.string().optional(), // optional image style
     images: mulmoImageParamsImagesSchema.optional(),

package/lib/utils/context.d.ts

@@ -25,6 +25,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 provider: string;
                 model?: string | undefined;
                 style?: string | undefined;
+                quality?: string | undefined;
                 moderation?: string | undefined;
                 images?: Record<string, {
                     type: "image";
@@ -94,6 +95,17 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             speechParams: {
                 speakers: Record<string, {
                     voiceId: string;
+                    lang?: Record<string, {
+                        voiceId: string;
+                        displayName?: Record<string, string> | undefined;
+                        isDefault?: boolean | undefined;
+                        speechOptions?: {
+                            speed?: number | undefined;
+                            instruction?: string | undefined;
+                        } | undefined;
+                        provider?: string | undefined;
+                        model?: string | undefined;
+                    }> | undefined;
                     displayName?: Record<string, string> | undefined;
                     isDefault?: boolean | undefined;
                     speechOptions?: {
@@ -242,6 +254,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     provider: string;
                     model?: string | undefined;
                     style?: string | undefined;
+                    quality?: string | undefined;
                     moderation?: string | undefined;
                     images?: Record<string, {
                         type: "image";
@@ -362,6 +375,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             provider: string;
             model?: string | undefined;
             style?: string | undefined;
+            quality?: string | undefined;
             moderation?: string | undefined;
             images?: Record<string, {
                 type: "image";
@@ -431,6 +445,17 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
         speechParams: {
             speakers: Record<string, {
                 voiceId: string;
+                lang?: Record<string, {
+                    voiceId: string;
+                    displayName?: Record<string, string> | undefined;
+                    isDefault?: boolean | undefined;
+                    speechOptions?: {
+                        speed?: number | undefined;
+                        instruction?: string | undefined;
+                    } | undefined;
+                    provider?: string | undefined;
+                    model?: string | undefined;
+                }> | undefined;
                 displayName?: Record<string, string> | undefined;
                 isDefault?: boolean | undefined;
                 speechOptions?: {

package/lib/utils/file.d.ts

@@ -21,7 +21,7 @@ export declare const getOutputStudioFilePath: (outDirPath: string, fileName: str
 export declare const getOutputMultilingualFilePath: (outDirPath: string, fileName: string) => string;
 export declare const resolveDirPath: (dirPath: string, studioFileName: string) => string;
 export declare const getAudioFilePath: (audioDirPath: string, dirName: string, fileName: string, lang?: string) => string;
-export declare const getAudioArtifactFilePath: (outDirPath: string, fileName: string) => string;
+export declare const getAudioArtifactFilePath: (context: MulmoStudioContext) => string;
 export declare const getOutputVideoFilePath: (outDirPath: string, fileName: string, lang?: string, caption?: string) => string;
 export declare const imageSuffix = "p";
 export declare const getBeatPngImagePath: (context: MulmoStudioContext, index: number) => string;

package/lib/utils/file.js

@@ -72,8 +72,11 @@ export const getAudioFilePath = (audioDirPath, dirName, fileName, lang) => {
     }
     return path.resolve(audioDirPath, dirName, fileName + ".mp3");
 };
-export const getAudioArtifactFilePath = (outDirPath, fileName) => {
-    return path.resolve(outDirPath, fileName + ".mp3");
+export const getAudioArtifactFilePath = (context) => {
+    const suffix = context.lang ? `_${context.lang}` : "";
+    const fileName = MulmoStudioContextMethods.getFileName(context);
+    const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
+    return path.resolve(outDirPath, fileName + suffix + ".mp3");
 };
 export const getOutputVideoFilePath = (outDirPath, fileName, lang, caption) => {
     const suffix = lang ? `_${lang}` : "";

package/lib/utils/preprocess.d.ts

@@ -20,6 +20,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             provider: string;
             model?: string | undefined;
             style?: string | undefined;
+            quality?: string | undefined;
             moderation?: string | undefined;
             images?: Record<string, {
                 type: "image";
@@ -89,6 +90,17 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         speechParams: {
             speakers: Record<string, {
                 voiceId: string;
+                lang?: Record<string, {
+                    voiceId: string;
+                    displayName?: Record<string, string> | undefined;
+                    isDefault?: boolean | undefined;
+                    speechOptions?: {
+                        speed?: number | undefined;
+                        instruction?: string | undefined;
+                    } | undefined;
+                    provider?: string | undefined;
+                    model?: string | undefined;
+                }> | undefined;
                 displayName?: Record<string, string> | undefined;
                 isDefault?: boolean | undefined;
                 speechOptions?: {
@@ -237,6 +249,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 provider: string;
                 model?: string | undefined;
                 style?: string | undefined;
+                quality?: string | undefined;
                 moderation?: string | undefined;
                 images?: Record<string, {
                     type: "image";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "1.1.4",
+  "version": "1.1.5",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -23,6 +23,7 @@
   "files": [
     "./lib",
     "./scripts/templates",
+    "./scripts/test",
     "./assets/audio/silent60sec.mp3",
     "./assets/html/",
     "./assets/templates/"

package/scripts/templates/image_prompt_only_template.ts

@@ -0,0 +1,95 @@
+import {
+  mulmoCastCreditSchema,
+  langSchema,
+}
+export const mulmoPresentationStyleSchema = z.object({
+  $mulmocast: mulmoCastCreditSchema,
+  "lang": "en",
+  title: z.string().default("[TITLE: Brief, engaging title for the topic]")
+  lang: z.string().default("en"),
+  "references": z.array(z.object({
+    "url": "[SOURCE_URL: URL of the source material]",
+    "title": "[SOURCE_TITLE: Title of the referenced article, or paper]",
+      "type": "[SOURCE_TYPE: article, paper]"
+    }
+  ],
+
+  
+  description: z.string().optional(),
+  references: z.array(mulmoReferenceSchema).optional(),
+  beats: z.array(mulmoBeatSchema).min(1),
+
+  // TODO: Delete it later
+  imagePath: z.string().optional(), // for keynote images movie ??
+  canvasSize: mulmoCanvasDimensionSchema, // has default value
+  speechParams: z
+    .object({
+      speakers: speakerDictionarySchema,
+    })
+    .default({
+      speakers: {
+        [defaultSpeaker]: {
+          voiceId: "shimmer",
+          displayName: {
+            en: defaultSpeaker,
+          },
+        },
+      },
+    }),
+  imageParams: mulmoImageParamsSchema.optional().default({
+    provider: defaultProviders.text2image,
+    images: {},
+  }),
+  movieParams: mulmoMovieParamsSchema.optional().default({
+    provider: defaultProviders.text2movie,
+  }),
+  soundEffectParams: mulmoSoundEffectParamsSchema.optional().default({
+    provider: defaultProviders.soundEffect,
+  }),
+  lipSyncParams: mulmoLipSyncParamsSchema.optional(),
+  htmlImageParams: mulmoHtmlImageParamsSchema
+    .extend({
+      provider: text2HtmlImageProviderSchema,
+    })
+    .optional(),
+  // for textSlides
+  textSlideParams: textSlideParamsSchema.optional(),
+  captionParams: mulmoCaptionParamsSchema.optional(),
+  audioParams: audioParamsSchema.default({
+    introPadding: 1.0,
+    padding: 0.3,
+    closingPadding: 0.8,
+    outroPadding: 1.0,
+    bgmVolume: 0.2,
+    audioVolume: 1.0,
+  }),
+});
+
+{
+  "beats": [
+    {
+      "text": "[OPENING_BEAT: Introduce the topic with a hook. Reference the source material and set up why this topic matters. Usually 2-3 sentences that grab attention and provide context.]",
+      "imagePrompt": "[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]"
+    },
+    {
+      "text": "[MAIN_CONCEPT: Define or explain the core concept/idea. This should be the central focus of your narrative. Keep it clear and accessible.]",
+      "imagePrompt": "[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]"
+    },
+    {
+      "text": "[SUPPORTING_DETAIL_1: Additional context, examples, or elaboration that helps illustrate the main concept. This could include how it works, why it's important, or real-world applications.]",
+      "imagePrompt": "[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]"
+    },
+    {
+      "text": "[SUPPORTING_DETAIL_2: Continue with more examples, deeper explanation, or different aspects of the topic if needed.]",
+      "imagePrompt": "[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]"
+    },
+    {
+      "text": "[ADDITIONAL_BEATS: Add more beats as necessary to fully explore the topic. Complex topics may require 6-10+ beats to cover adequately. Each beat should advance the narrative or provide valuable information.]",
+      "imagePrompt": "[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]"
+    },
+    {
+      "text": "[CONCLUSION/IMPACT: Wrap up with the significance, implications, or key takeaway. Help the audience understand why this matters to them.]",
+      "imagePrompt": "[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]"
+    }
+  ]
+}

package/scripts/test/gpt.json

@@ -0,0 +1,32 @@
+{
+  "$mulmocast": {
+    "version": "1.1",
+    "credit": "closing"
+  },
+  "title": "Testing OpenAI's new Image Generation",
+  "description": "Hello",
+  "imageParams": {
+    "model": "gpt-image-1",
+    "style": "<style>Japanese animation with soft watercolor backgrounds, characters with simple rounded faces, large expressive eyes, small nose and mouth, soft jawlines, minimalist facial features, pastel color palette, detailed natural environments, whimsical magical elements, hand-drawn aesthetic, gentle lighting, flowing movement in hair and clothing, nostalgic countryside scenery with fantasy elements."
+  },
+  "speechParams": {
+    "speakers": {
+      "Host": {
+        "voiceId": "shimmer",
+        "displayName": {
+          "en": "Host"
+        }
+      }
+    }
+  },
+  "beats": [
+    {
+      "speaker": "Host",
+      "text": "How are you?",
+      "imagePrompt": "A witch in Harajuku",
+      "imageParams": {
+        "style": "Ukiyoe-style"
+      }
+    }
+  ]
+}

package/scripts/test/mulmo_story.json

@@ -0,0 +1,11 @@
+{
+  "title": "MulmoScript",
+  "scenes": [
+    {
+      "description": "MulmoCast is a multi-modal presentation platform built for the generative AI era. Traditional tools like PowerPoint and Keynote were designed decades ago for human authors. Today, however, large language models (LLMs) are generating content—and they need a native environment optimized for their capabilities.　MulmoCast is that environment.　It empowers AI to automatically create and deliver rich, multi-modal presentations—including slides, videos, podcasts, documents, and comics—using our open presentation language: MulmoScript."
+    },
+    {
+      "description": "　MulmoScript is a JSON-based language that enables LLMs to describe structured, machine-readable presentations. It supports a wide range of elements: bullet points, charts, graphs, images, voiceovers, and videos. Just as HTML unlocked the web, MulmoScript enables interoperability, customization, and ecosystem growth for AI-generated content.　MulmoCast renders this content into any format, giving end users complete flexibility in how they consume it—whether as a slideshow, podcast, video, or document. It also supports multilingual output for both reading and listening."
+    }
+  ]
+}

package/scripts/test/test.json

@@ -0,0 +1,64 @@
+{
+  "$mulmocast": {
+    "version": "1.1",
+    "credit": "closing"
+  },
+  "title": "MASAI: A Modular Future for Software Engineering AI",
+  "description": "Exploring MASAI, a modular approach for AI agents in software engineering that revolutionizes how complex coding issues are tackled.",
+  "references": [
+    {
+      "url": "https://arxiv.org/abs/2406.11638",
+      "title": "MASAI: A Modular Future for Software Engineering AI",
+      "description": "An article on MASAI, a modular approach for AI agents in software engineering."
+    }
+  ],
+  "lang": "ja",
+  "imageParams": {
+    "style": "<style>monochrome"
+  },
+  "speechParams": {
+    "speakers": {
+      "Announcer": {
+        "provider": "nijivoice",
+        "displayName": {
+          "ja": "アナウンサー"
+        },
+        "voiceId": "3708ad43-cace-486c-a4ca-8fe41186e20c",
+        "speechOptions": {
+          "speed": 1.666
+        }
+      },
+      "Student": {
+        "provider": "nijivoice",
+        "displayName": {
+          "ja": "生徒"
+        },
+        "voiceId": "a7619e48-bf6a-4f9f-843f-40485651257f"
+      },
+      "Teacher": {
+        "provider": "nijivoice",
+        "displayName": {
+          "ja": "先生"
+        },
+        "voiceId": "bc06c63f-fef6-43b6-92f7-67f919bd5dae"
+      }
+    }
+  },
+  "beats": [
+    {
+      "speaker": "Announcer",
+      "text": "こんにちは。米国で活躍するエンジニアが新しい技術やビジネスを分かりやすく解説する、中島聡のLife is beautiful。"
+    },
+    {
+      "speaker": "Announcer",
+      "text": "今日は、アメリカで発表された「スターゲート・プロジェクト」に、ついて解説します。",
+      "imagePrompt": "Blue sky, a flock of birds",
+      "imageParams": {
+        "style": "<style>sumie-style"
+      },
+      "speechOptions": {
+        "speed": 0.8
+      }
+    }
+  ]
+}

package/scripts/test/test1.json

@@ -0,0 +1,40 @@
+{
+  "$mulmocast": {
+    "version": "1.1",
+    "credit": "closing"
+  },
+  "title": "The Honey Trap vs. The Frontier: Engineering at Tesla and SpaceX",
+  "description": "We dive into Elon Musk's philosophy on building environments where engineers can truly flourish, contrasting the comfort-focused 'honey trap' with the high-expectation culture at Tesla and SpaceX.",
+  "speechParams": {
+    "speakers": {
+      "Host": {
+        "displayName": {
+          "ja": "司会"
+        },
+        "voiceId": "sage"
+      },
+      "Guest": {
+        "displayName": {
+          "ja": "ゲスト"
+        },
+        "voiceId": "shimmer"
+      }
+    }
+  },
+  "beats": [
+    {
+      "speaker": "Host",
+      "text": "Hello and welcome to another episode of 'life is artificial', where we explore the cutting edge of technology, innovation, and what the future could look like.",
+      "speechOptions": {
+        "instruction": "Voice: Deep and rugged, with a hearty, boisterous quality, like a seasoned sea captain who's seen many voyages.\nTone: Friendly and spirited, with a sense of adventure and enthusiasm, making every detail feel like part of a grand journey.\nDialect: Classic pirate speech with old-timey nautical phrases, dropped 'g's, and exaggerated 'Arrrs' to stay in character."
+      }
+    },
+    {
+      "speaker": "Guest",
+      "text": "こんにちは、ポッドキャスト版、ライフ・イズ・ビューティフルへようこそ。新しいテクノロジーについて分かりやすく語ります",
+      "speechOptions": {
+        "instruction": "voice: 赤ちゃん\nVoice: 高い声で、舌ったらず。甘えっぽく。語尾に’んちゃ’と付ける"
+      }
+    }
+  ]
+}

package/scripts/test/test2.json

@@ -0,0 +1,66 @@
+{
+  "$mulmocast": {
+    "version": "1.1",
+    "credit": "closing"
+  },
+  "title": "AIの進化と私たちの未来 - サム・アルトマン氏の3つの観察",
+  "description": "OpenAIのサム・アルトマン氏による、AIの進化と社会への影響に関する洞察を、高校生向けに分かりやすく解説する対話形式のスクリプト。",
+  "references": [
+    {
+      "url": "https://blog.samaltman.com/three-observations",
+      "title": "Three Observations",
+      "description": "An article on AI's evolution and its impact on society by Sam Altman."
+    }
+  ],
+  "lang": "ja",
+  "imageParams": {
+    "style": "<style>monochrome"
+  },
+  "speechParams": {
+    "provider": "nijivoice",
+    "speakers": {
+      "Announcer": {
+        "displayName": {
+          "ja": "千草朋香"
+        },
+        "voiceId": "3708ad43-cace-486c-a4ca-8fe41186e20c"
+      },
+      "Student": {
+        "displayName": {
+          "ja": "太郎"
+        },
+        "voiceId": "a7619e48-bf6a-4f9f-843f-40485651257f"
+      },
+      "Teacher": {
+        "displayName": {
+          "ja": "山田先生"
+        },
+        "voiceId": "bc06c63f-fef6-43b6-92f7-67f919bd5dae"
+      }
+    }
+  },
+  "beats": [
+    {
+      "speaker": "Announcer",
+      "text": "私たちの目の前で、人工知能の革命が静かに、",
+      "speechOptions": {
+        "speed": 1.5
+      },
+      "imagePrompt": "A futuristic scene depicting the quiet but certain advancement of AI, with digital interfaces and abstract representations of AI technology shaping the future."
+    },
+    {
+      "speaker": "Announcer",
+      "text": "しかし確実に進んでいます。",
+      "speechOptions": {
+        "speed": 1.5
+      }
+    },
+    {
+      "speaker": "Announcer",
+      "text": "オープンエーアイのサム・アルトマン氏が語る「3つの重要な観察」とは何か？",
+      "speechOptions": {
+        "speed": 1.5
+      }
+    }
+  ]
+}

package/scripts/test/test_audio.json

@@ -0,0 +1,151 @@
+{
+  "$mulmocast": {
+    "version": "1.1"
+  },
+  "title": "Media Test",
+  "audioParams": {
+    "introPadding": 0,
+    "padding": 1.0,
+    "closingPadding": 5.0,
+    "outroPadding": 0
+  },
+  "beats": [
+    {
+      "speaker": "Presenter",
+      "text": "This is an opening beat.",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Opening Beat"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "No Audio with Duration 1.0 seconds (default)"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "This is the third beat.",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Third Beat"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "This beat has a custom audio padding of 0.0 seconds.",
+      "audioParams": {
+        "padding": 0.0
+      },
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Custom Audio Padding 0.0 seconds"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "This beat has a custom audio padding of 3.0 seconds.",
+      "audioParams": {
+        "padding": 3.0
+      },
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Custom Audio Padding 3.0 seconds"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "",
+      "duration": 2,
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "No Audio with Duration 2 seconds"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "This is a local movie with audio.",
+      "image": {
+        "type": "movie",
+        "source": {
+          "kind": "url",
+          "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/test/pingpong.mov"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "This is a local movie with 20% audio.",
+      "image": {
+        "type": "movie",
+        "source": {
+          "kind": "url",
+          "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/test/pingpong.mov"
+        }
+      },
+      "audioParams": {
+        "movieVolume": 0.2
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "This is a local movie with no audio.",
+      "image": {
+        "type": "movie",
+        "source": {
+          "kind": "url",
+          "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/test/pingpong.mov"
+        }
+      },
+      "audioParams": {
+        "movieVolume": 0.0
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "This section has longer duration than the audio.",
+      "duration": 10,
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Duration 10 seconds"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "This is the beat before the closing slide.",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Before Closing Slide"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "This is the closing slide.",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Closing Slide"
+        }
+      }
+    }
+  ]
+}