mulmocast 0.1.6 → 0.1.7

package/assets/templates/ani.json

@@ -0,0 +1,48 @@
+{
+  "title": "Presentation with Ani in Japanese",
+  "description": "Template for presentation with Ani in Japanese.",
+  "systemPrompt": "Generate a script for a presentation of the given topic. 言葉づかいは少しツンデレにして。Another AI will generate comic for each beat based on the image prompt of that beat. You don't need to specify the style of the image, just describe the scene. Mention the reference in one of beats, if it exists. Use the JSON below as a template. Create appropriate amount of beats, and make sure the beats are coherent and flow well.",
+  "presentationStyle": {
+    "$mulmocast": {
+      "version": "1.0",
+      "credit": "closing"
+    },
+    "movieParams": {
+      "provider": "replicate",
+      "model": "bytedance/seedance-1-lite"
+    },
+    "speechParams": {
+      "provider": "openai",
+      "speakers": {
+        "Presenter": {
+          "voiceId": "shimmer",
+          "speechOptions": { "instruction": "Speak in a slightly high-pitched, curt tone with sudden flustered shifts—like a tsundere anime girl." }
+        }
+      }
+    },
+    "audioParams": {
+      "bgm": {
+        "kind": "url",
+        "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/morning001.mp3"
+      }
+    },
+    "lang": "en",
+    "canvasSize": {
+      "width": 1024,
+      "height": 1536
+    },
+    "imageParams": {
+      "style": "<style>A highly polished 2D digital illustration in anime and manga style, featuring clean linework, soft shading, vivid colors, and expressive facial detailing. The composition emphasizes clarity and visual impact with a minimalistic background and a strong character focus. The lighting is even and bright, giving the image a crisp and energetic feel, reminiscent of high-quality character art used in Japanese visual novels or mobile games.</style>",
+      "images": {
+        "ani": {
+          "type": "image",
+          "source": {
+            "kind": "url",
+            "url": "https://raw.githubusercontent.com/receptron/mulmocast-media/refs/heads/main/characters/ani.png"
+          }
+        }
+      }
+    }
+  },
+  "scriptName": "image_prompts_template.json"
+}

package/assets/templates/ani_ja.json

@@ -0,0 +1,45 @@
+{
+  "title": "Presentation with Ani",
+  "description": "Template for presentation with Ani.",
+  "systemPrompt": "Generate a Japanese script for a presentation of the given topic. 言葉づかいは少しツンデレにして。Another AI will generate comic for each beat based on the image prompt of that beat. You don't need to specify the style of the image, just describe the scene. Mention the reference in one of beats, if it exists. Use the JSON below as a template. Create appropriate amount of beats, and make sure the beats are coherent and flow well.",
+  "presentationStyle": {
+    "$mulmocast": {
+      "version": "1.0",
+      "credit": "closing"
+    },
+    "movieParams": {
+      "provider": "replicate",
+      "model": "bytedance/seedance-1-lite"
+    },
+    "audioParams": {
+      "bgm": {
+        "kind": "url",
+        "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/morning001.mp3"
+      }
+    },
+    "lang": "ja",
+    "canvasSize": {
+      "width": 1024,
+      "height": 1536
+    },
+    "speechParams": {
+      "provider": "nijivoice",
+      "speakers": {
+        "Presenter": { "voiceId": "9d9ed276-49ee-443a-bc19-26e6136d05f0" }
+      }
+    },
+    "imageParams": {
+      "style": "<style>A highly polished 2D digital illustration in anime and manga style, featuring clean linework, soft shading, vivid colors, and expressive facial detailing. The composition emphasizes clarity and visual impact with a minimalistic background and a strong character focus. The lighting is even and bright, giving the image a crisp and energetic feel, reminiscent of high-quality character art used in Japanese visual novels or mobile games.</style>",
+      "images": {
+        "ani": {
+          "type": "image",
+          "source": {
+            "kind": "url",
+            "url": "https://raw.githubusercontent.com/receptron/mulmocast-media/refs/heads/main/characters/ani.png"
+          }
+        }
+      }
+    }
+  },
+  "scriptName": "image_prompts_template.json"
+}

package/lib/actions/audio.js

@@ -143,6 +143,7 @@ const graph_data = {
         },
         addBGM: {
             agent: "addBGMAgent",
+            unless: ":context.presentationStyle.audioParams.bgmVolume.equal(0)",
             inputs: {
                 wait: ":combineFiles",
                 voiceFile: ":audioCombinedFilePath",
@@ -153,6 +154,7 @@ const graph_data = {
                 },
             },
             isResult: true,
+            defaultValue: {},
         },
         title: {
             agent: "copyAgent",

package/lib/actions/image_agents.d.ts

@@ -44,17 +44,20 @@ export declare const imagePreprocessAgent: (namedInputs: {
 } | {
     imagePath: string;
     imageFromMovie: boolean;
-    movieParams: {
-        speed?: number | undefined;
-        model?: string | undefined;
-        fillOption?: {
-            style: "aspectFit" | "aspectFill";
-        } | undefined;
-        provider?: string | undefined;
-        transition?: {
-            type: "fade" | "slideout_left";
-            duration: number;
-        } | undefined;
+    movieAgentInfo: {
+        agent: string;
+        movieParams: {
+            speed?: number | undefined;
+            provider?: string | undefined;
+            model?: string | undefined;
+            fillOption?: {
+                style: "aspectFit" | "aspectFill";
+            } | undefined;
+            transition?: {
+                type: "fade" | "slideout_left";
+                duration: number;
+            } | undefined;
+        };
     };
     imageParams: {
         provider: string;
@@ -91,17 +94,20 @@ export declare const imagePreprocessAgent: (namedInputs: {
     imageAgentInfo: import("../types/type.js").Text2ImageAgentInfo;
     prompt: string;
     referenceImages: string[];
-    movieParams: {
-        speed?: number | undefined;
-        model?: string | undefined;
-        fillOption?: {
-            style: "aspectFit" | "aspectFill";
-        } | undefined;
-        provider?: string | undefined;
-        transition?: {
-            type: "fade" | "slideout_left";
-            duration: number;
-        } | undefined;
+    movieAgentInfo: {
+        agent: string;
+        movieParams: {
+            speed?: number | undefined;
+            provider?: string | undefined;
+            model?: string | undefined;
+            fillOption?: {
+                style: "aspectFit" | "aspectFill";
+            } | undefined;
+            transition?: {
+                type: "fade" | "slideout_left";
+                duration: number;
+            } | undefined;
+        };
     };
     imageParams: {
         provider: string;

package/lib/actions/image_agents.js

@@ -28,15 +28,15 @@ export const imagePreprocessAgent = async (namedInputs) => {
         // undefined prompt indicates that image generation is not needed
         return { ...returnValue, imagePath: pluginPath, referenceImageForMovie: pluginPath };
     }
-    const movieParams = { ...context.presentationStyle.movieParams, ...beat.movieParams };
-    GraphAILogger.log(`movieParams: ${index}`, movieParams, beat.moviePrompt);
+    const movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle, beat);
+    GraphAILogger.log(`movieParams: ${index}`, movieAgentInfo.movieParams, beat.moviePrompt);
     if (beat.moviePrompt && !beat.imagePrompt) {
-        return { ...returnValue, imagePath, imageFromMovie: true, movieParams }; // no image prompt, only movie prompt
+        return { ...returnValue, imagePath, imageFromMovie: true, movieAgentInfo }; // no image prompt, only movie prompt
     }
     // referenceImages for "edit_image", openai agent.
     const referenceImages = MulmoBeatMethods.getImageReferenceForImageGenerator(beat, imageRefs);
     const prompt = imagePrompt(beat, imageAgentInfo.imageParams.style);
-    return { ...returnValue, imagePath, referenceImageForMovie: imagePath, imageAgentInfo, prompt, referenceImages, movieParams };
+    return { ...returnValue, imagePath, referenceImageForMovie: imagePath, imageAgentInfo, prompt, referenceImages, movieAgentInfo };
 };
 export const imagePluginAgent = async (namedInputs) => {
     const { context, beat, index } = namedInputs;

package/lib/actions/images.js

@@ -39,7 +39,6 @@ const beat_graph_data = {
     nodes: {
         context: {},
         htmlImageAgentInfo: {},
-        movieAgentInfo: {},
         imageRefs: {},
         beat: {},
         __mapIndex: {},
@@ -134,7 +133,7 @@ const beat_graph_data = {
         },
         movieGenerator: {
             if: ":preprocessor.movieFile",
-            agent: ":movieAgentInfo.agent",
+            agent: ":preprocessor.movieAgentInfo.agent",
             inputs: {
                 onComplete: [":imageGenerator", ":imagePlugin"], // to wait for imageGenerator to finish
                 prompt: ":beat.moviePrompt",
@@ -147,7 +146,7 @@ const beat_graph_data = {
                     mulmoContext: ":context",
                 },
                 params: {
-                    model: ":preprocessor.movieParams.model",
+                    model: ":preprocessor.movieAgentInfo.movieParams.model",
                     duration: ":beat.duration",
                     canvasSize: ":context.presentationStyle.canvasSize",
                 },
@@ -167,16 +166,19 @@ const beat_graph_data = {
             defaultValue: {},
         },
         audioChecker: {
-            if: ":preprocessor.movieFile",
             agent: async (namedInputs) => {
-                const { hasAudio } = await ffmpegGetMediaDuration(namedInputs.movieFile);
+                const sourceFile = namedInputs.movieFile || namedInputs.imageFile;
+                if (!sourceFile) {
+                    return { hasMovieAudio: false };
+                }
+                const { hasAudio } = await ffmpegGetMediaDuration(sourceFile);
                 return { hasMovieAudio: hasAudio };
             },
             inputs: {
-                onComplete: [":movieGenerator"], // to wait for movieGenerator to finish
+                onComplete: [":movieGenerator", ":htmlImageGenerator"], // to wait for movieGenerator and htmlImageGenerator to finish
                 movieFile: ":preprocessor.movieFile",
+                imageFile: ":preprocessor.imagePath",
             },
-            defaultValue: {},
         },
         output: {
             agent: "copyAgent",
@@ -201,7 +203,6 @@ const graph_data = {
     nodes: {
         context: {},
         htmlImageAgentInfo: {},
-        movieAgentInfo: {},
         outputStudioFilePath: {},
         imageRefs: {},
         map: {
@@ -210,7 +211,6 @@ const graph_data = {
                 rows: ":context.studio.script.beats",
                 context: ":context",
                 htmlImageAgentInfo: ":htmlImageAgentInfo",
-                movieAgentInfo: ":movieAgentInfo",
                 imageRefs: ":imageRefs",
             },
             isResult: true,
@@ -314,9 +314,6 @@ const prepareGenerateImages = async (context) => {
     const injections = {
         context,
         htmlImageAgentInfo,
-        movieAgentInfo: {
-            agent: MulmoPresentationStyleMethods.getMovieAgent(context.presentationStyle),
-        },
         outputStudioFilePath: getOutputStudioFilePath(outDirPath, fileName),
         imageRefs,
     };

package/lib/agents/movie_replicate_agent.js

@@ -1,6 +1,7 @@
 import { readFileSync } from "fs";
 import { GraphAILogger } from "graphai";
 import Replicate from "replicate";
+import { provider2MovieAgent } from "../utils/provider2agent.js";
 async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, duration) {
     const replicate = new Replicate({
         auth: apiKey,
@@ -21,7 +22,7 @@ async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, dura
     if (imagePath) {
         const buffer = readFileSync(imagePath);
         const base64Image = `data:image/png;base64,${buffer.toString("base64")}`;
-        if (model === "kwaivgi/kling-v2.1" || model === "kwaivgi/kling-v1.6-pro") {
+        if (model === "kwaivgi/kling-v2.1" || model === "kwaivgi/kling-v1.6-pro" || model === "minimax/hailuo-02") {
             input.start_image = base64Image;
         }
         else {
@@ -29,7 +30,7 @@ async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, dura
         }
     }
     try {
-        const output = await replicate.run(model ?? "bytedance/seedance-1-lite", { input });
+        const output = await replicate.run(model ?? provider2MovieAgent.replicate.defaultModel, { input });
         // Download the generated video
         if (output && typeof output === "object" && "url" in output) {
             const videoUrl = output.url();

package/lib/cli/commands/tool/scripting/builder.js

@@ -1,4 +1,4 @@
-import { llm } from "../../../../utils/utils.js";
+import { llm } from "../../../../utils/provider2agent.js";
 import { getAvailableTemplates } from "../../../../utils/file.js";
 const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
 export const builder = (yargs) => {

package/lib/cli/commands/tool/scripting/handler.d.ts

@@ -1,5 +1,5 @@
 import { ToolCliArgs } from "../../../../types/cli_types.js";
-import { LLM } from "../../../../utils/utils.js";
+import type { LLM } from "../../../../utils/provider2agent.js";
 export declare const handler: (argv: ToolCliArgs<{
     o?: string;
     b?: string;

package/lib/cli/commands/tool/story_to_script/builder.js

@@ -1,5 +1,5 @@
 import { getAvailableTemplates } from "../../../../utils/file.js";
-import { llm } from "../../../../utils/utils.js";
+import { llm } from "../../../../utils/provider2agent.js";
 import { storyToScriptGenerateMode } from "../../../../utils/const.js";
 const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
 export const builder = (yargs) => {

package/lib/cli/commands/tool/story_to_script/handler.d.ts

@@ -1,5 +1,5 @@
 import { ToolCliArgs } from "../../../../types/cli_types.js";
-import { LLM } from "../../../../utils/utils.js";
+import type { LLM } from "../../../../utils/provider2agent.js";
 export declare const handler: (argv: ToolCliArgs<{
     o?: string;
     b?: string;

package/lib/methods/mulmo_presentation_style.d.ts

@@ -12,7 +12,21 @@ export declare const MulmoPresentationStyleMethods: {
     getVoiceId(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
     getText2ImageProvider(provider: Text2ImageProvider | undefined): Text2ImageProvider;
     getImageAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): Text2ImageAgentInfo;
-    getMovieAgent(presentationStyle: MulmoPresentationStyle): string;
+    getMovieAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): {
+        agent: string;
+        movieParams: {
+            speed?: number | undefined;
+            provider?: string | undefined;
+            model?: string | undefined;
+            fillOption?: {
+                style: "aspectFit" | "aspectFill";
+            } | undefined;
+            transition?: {
+                type: "fade" | "slideout_left";
+                duration: number;
+            } | undefined;
+        };
+    };
     getConcurrency(presentationStyle: MulmoPresentationStyle): 4 | 16;
     getHtmlImageAgentInfo(presentationStyle: MulmoPresentationStyle): Text2HtmlAgentInfo;
     getImageType(_: MulmoPresentationStyle, beat: MulmoBeat): BeatMediaType;

package/lib/methods/mulmo_presentation_style.js

@@ -1,7 +1,7 @@
 import "dotenv/config";
 import { userAssert } from "../utils/utils.js";
-import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema } from "../types/schema.js";
-import { defaultProviders, provider2ImageAgent, provider2MovieAgent, provider2LLMAgent } from "../utils/provider2agent.js";
+import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, } from "../types/schema.js";
+import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent } from "../utils/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
@@ -80,17 +80,16 @@ export const MulmoPresentationStyleMethods = {
             imageParams: { ...defaultImageParams, ...imageParams },
         };
     },
-    // Determine movie agent based on provider
-    getMovieAgent(presentationStyle) {
-        const movieProvider = (presentationStyle.movieParams?.provider ?? defaultProviders.text2movie);
-        return provider2MovieAgent[movieProvider].agentName;
+    getMovieAgentInfo(presentationStyle, beat) {
+        const movieParams = { ...presentationStyle.movieParams, ...beat?.movieParams };
+        const movieProvider = text2MovieProviderSchema.parse(movieParams?.provider);
+        const agentInfo = provider2MovieAgent[movieProvider];
+        return {
+            agent: agentInfo.agentName,
+            movieParams,
+        };
     },
     getConcurrency(presentationStyle) {
-        /*
-        if (presentationStyle.movieParams?.provider === "replicate") {
-          return 4;
-        }
-        */
         const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(presentationStyle);
         if (imageAgentInfo.imageParams.provider === "openai") {
             // NOTE: Here are the rate limits of OpenAI's text2image API (1token = 32x32 patch).

package/lib/tools/story_to_script.d.ts

@@ -1,5 +1,5 @@
 import { MulmoStoryboard, StoryToScriptGenerateMode } from "../types/index.js";
-import { LLM } from "../utils/utils.js";
+import type { LLM } from "../utils/provider2agent.js";
 export declare const storyToScript: ({ story, beatsPerScene, templateName, outdir, fileName, llm, llmModel, generateMode, }: {
     story: MulmoStoryboard;
     beatsPerScene: number;