mulmocast 1.2.4 → 1.2.6

package/assets/templates/sifi_story.json

@@ -0,0 +1,29 @@
+{
+  "title": "Multi-character Story",
+  "description": "Template for Multi-character Story.",
+  "systemPrompt": "Break the story into multiple beats, and put the story text in 'text' field. Generate image prompt for each character in the imageParams.images. Another AI will generate image for each beat based on its imagePrompt and specified characters in 'imageNames'. You don't need to repeat the image style in those image prompts. Use the JSON below as a template.",
+  "presentationStyle": {
+    "$mulmocast": {
+      "version": "1.1",
+      "credit": "closing"
+    },
+    "canvasSize": {
+      "width": 1536,
+      "height": 1024
+    },
+    "imageParams": {
+      "style": "<style>A dreamy, hyper-detailed anime style that blends photorealistic backgrounds with vibrant, saturated colors. The skies are often filled with luminous clouds, dazzling sunsets, or star-filled nights, rendered with a glowing, almost ethereal quality. Urban landscapes and rural scenery are meticulously illustrated, with attention to tiny details like reflections in puddles, neon lights, or the texture of grass swaying in the wind. Characters are drawn with soft, expressive features, standing out against the breathtaking environments, creating a sense of emotional depth and lyrical atmosphere. The overall mood is cinematic, romantic, and filled with a sense of fleeting beauty and longing.</style>",
+      "images": {
+        "[CHARACTER_1_ID]": {
+          "type": "imagePrompt",
+          "prompt": "[IMAGE PROMPT FOR THIS CHARACTER]"
+        },
+        "[CHARACTER_2_ID]": {
+          "type": "imagePrompt",
+          "prompt": "[IMAGE PROMPT FOR THIS CHARACTER]"
+        }
+      }
+    }
+  },
+  "scriptName": "story_with_characters.json"
+}

package/lib/actions/audio.d.ts

@@ -1,6 +1,5 @@
 import "dotenv/config";
-import type { CallbackFunction } from "graphai";
-import { MulmoStudioContext, MulmoBeat } from "../types/index.js";
+import { MulmoStudioContext, MulmoBeat, PublicAPIArgs } from "../types/index.js";
 export declare const getBeatAudioPath: (text: string, context: MulmoStudioContext, beat: MulmoBeat, lang?: string) => string | undefined;
-export declare const generateBeatAudio: (index: number, context: MulmoStudioContext, settings?: Record<string, string>, callbacks?: CallbackFunction[]) => Promise<void>;
-export declare const audio: (context: MulmoStudioContext, settings?: Record<string, string>, callbacks?: CallbackFunction[]) => Promise<MulmoStudioContext>;
+export declare const generateBeatAudio: (index: number, context: MulmoStudioContext, args?: PublicAPIArgs) => Promise<void>;
+export declare const audio: (context: MulmoStudioContext, args?: PublicAPIArgs) => Promise<MulmoStudioContext>;

package/lib/actions/audio.js

@@ -186,7 +186,8 @@ const audioAgents = {
     addBGMAgent,
     combineAudioFilesAgent,
 };
-export const generateBeatAudio = async (index, context, settings, callbacks) => {
+export const generateBeatAudio = async (index, context, args) => {
+    const { settings, callbacks } = args ?? {};
     try {
         MulmoStudioContextMethods.setSessionState(context, "audio", true);
         const fileName = MulmoStudioContextMethods.getFileName(context);
@@ -214,7 +215,8 @@ export const generateBeatAudio = async (index, context, settings, callbacks) =>
         MulmoStudioContextMethods.setSessionState(context, "audio", false);
     }
 };
-export const audio = async (context, settings, callbacks) => {
+export const audio = async (context, args) => {
+    const { settings, callbacks } = args ?? {};
     try {
         MulmoStudioContextMethods.setSessionState(context, "audio", true);
         const fileName = MulmoStudioContextMethods.getFileName(context);

package/lib/actions/captions.d.ts

@@ -1,3 +1,2 @@
-import { MulmoStudioContext } from "../types/index.js";
-import type { CallbackFunction } from "graphai";
-export declare const captions: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<MulmoStudioContext>;
+import { MulmoStudioContext, PublicAPIArgs } from "../types/index.js";
+export declare const captions: (context: MulmoStudioContext, args?: PublicAPIArgs) => Promise<MulmoStudioContext>;

package/lib/actions/captions.js

@@ -69,7 +69,8 @@ const graph_data = {
         },
     },
 };
-export const captions = async (context, callbacks) => {
+export const captions = async (context, args) => {
+    const { callbacks } = args ?? {};
     if (MulmoStudioContextMethods.getCaption(context)) {
         try {
             MulmoStudioContextMethods.setSessionState(context, "caption", true);

package/lib/actions/image_agents.d.ts

@@ -7,6 +7,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
 }) => Promise<{
     imagePath: string;
     htmlPrompt: string | undefined;
+    htmlImageFile: string;
     htmlPath: string;
     htmlImageSystemPrompt: string;
 } | {
@@ -27,6 +28,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
     audioFile?: string;
     beatDuration?: number;
     htmlPrompt?: undefined;
+    htmlImageFile?: undefined;
     htmlPath?: undefined;
     htmlImageSystemPrompt?: undefined;
 } | {
@@ -62,6 +64,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
     audioFile?: string;
     beatDuration?: number;
     htmlPrompt?: undefined;
+    htmlImageFile?: undefined;
     htmlPath?: undefined;
     htmlImageSystemPrompt?: undefined;
 } | {
@@ -100,6 +103,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
     audioFile?: string;
     beatDuration?: number;
     htmlPrompt?: undefined;
+    htmlImageFile?: undefined;
     htmlPath?: undefined;
     htmlImageSystemPrompt?: undefined;
 }>;

package/lib/actions/image_agents.js

@@ -12,11 +12,11 @@ const htmlStyle = (context, beat) => {
 export const imagePreprocessAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs } = namedInputs;
     const studioBeat = context.studio.beats[index];
-    const imagePath = getBeatPngImagePath(context, index);
+    const { imagePath, htmlImageFile } = getBeatPngImagePath(context, index);
     if (beat.htmlPrompt) {
         const htmlPrompt = MulmoBeatMethods.getHtmlPrompt(beat);
         const htmlPath = imagePath.replace(/\.[^/.]+$/, ".html");
-        return { imagePath, htmlPrompt, htmlPath, htmlImageSystemPrompt: htmlImageSystemPrompt(context.presentationStyle.canvasSize) };
+        return { imagePath, htmlPrompt, htmlImageFile, htmlPath, htmlImageSystemPrompt: htmlImageSystemPrompt(context.presentationStyle.canvasSize) };
     }
     const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle, beat);
     const moviePaths = getBeatMoviePaths(context, index);
@@ -61,7 +61,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
 };
 export const imagePluginAgent = async (namedInputs) => {
     const { context, beat, index } = namedInputs;
-    const imagePath = getBeatPngImagePath(context, index);
+    const { imagePath } = getBeatPngImagePath(context, index);
     const plugin = MulmoBeatMethods.getPlugin(beat);
     try {
         MulmoStudioContextMethods.setBeatSessionState(context, "image", index, beat.id, true);

package/lib/actions/images.d.ts

@@ -1,20 +1,18 @@
-import type { GraphOptions, CallbackFunction } from "graphai";
-import { MulmoStudioContext } from "../types/index.js";
+import type { GraphOptions } from "graphai";
+import { MulmoStudioContext, PublicAPIArgs } from "../types/index.js";
 export declare const graphOption: (context: MulmoStudioContext, settings?: Record<string, string>) => Promise<GraphOptions>;
 type ImageOptions = {
     imageAgents: Record<string, unknown>;
 };
-export declare const images: (context: MulmoStudioContext, args?: {
-    settings?: Record<string, string>;
-    callbacks?: CallbackFunction[];
+export declare const images: (context: MulmoStudioContext, args?: PublicAPIArgs & {
     options?: ImageOptions;
 }) => Promise<MulmoStudioContext>;
 export declare const generateBeatImage: (inputs: {
     index: number;
     context: MulmoStudioContext;
-    settings?: Record<string, string>;
-    callbacks?: CallbackFunction[];
-    forceMovie?: boolean;
-    forceImage?: boolean;
+    args?: PublicAPIArgs & {
+        forceMovie?: boolean;
+        forceImage?: boolean;
+    };
 }) => Promise<void>;
 export {};

package/lib/actions/images.js

@@ -116,7 +116,7 @@ const beat_graph_data = {
             inputs: {
                 htmlText: ":htmlReader.htmlText",
                 canvasSize: ":context.presentationStyle.canvasSize",
-                file: ":preprocessor.imagePath",
+                file: ":preprocessor.htmlImageFile",
             },
         },
         imageGenerator: {
@@ -188,7 +188,7 @@ const beat_graph_data = {
                     return { hasMovieAudio: true };
                 }
                 const sourceFile = namedInputs.movieFile || namedInputs.imageFile;
-                if (!sourceFile) {
+                if (!sourceFile || !fs.existsSync(sourceFile)) {
                     return { hasMovieAudio: false };
                 }
                 const { hasAudio } = await ffmpegGetMediaDuration(sourceFile);
@@ -257,6 +257,7 @@ const beat_graph_data = {
                 soundEffectFile: ":preprocessor.soundEffectFile",
                 lipSyncFile: ":preprocessor.lipSyncFile",
                 hasMovieAudio: ":audioChecker.hasMovieAudio",
+                htmlImageFile: ":preprocessor.htmlImageFile",
             },
             output: {
                 imageFile: ".imageFile",
@@ -264,6 +265,7 @@ const beat_graph_data = {
                 soundEffectFile: ".soundEffectFile",
                 lipSyncFile: ".lipSyncFile",
                 hasMovieAudio: ".hasMovieAudio",
+                htmlImageFile: ".htmlImageFile",
             },
             isResult: true,
         },
@@ -346,8 +348,8 @@ export const graphOption = async (context, settings) => {
             },
         ],
         taskManager: new TaskManager(MulmoPresentationStyleMethods.getConcurrency(context.presentationStyle)),
+        config: settings2GraphAIConfig(settings, process.env),
     };
-    options.config = settings2GraphAIConfig(settings, process.env);
     return options;
 };
 const prepareGenerateImages = async (context) => {
@@ -367,7 +369,8 @@ const prepareGenerateImages = async (context) => {
     };
     return injections;
 };
-const generateImages = async (context, settings, callbacks, options) => {
+const generateImages = async (context, args) => {
+    const { settings, callbacks, options } = args ?? {};
     const optionImageAgents = options?.imageAgents ?? {};
     const injections = await prepareGenerateImages(context);
     const graphaiAgent = {
@@ -388,10 +391,9 @@ const generateImages = async (context, settings, callbacks, options) => {
 };
 // public api
 export const images = async (context, args) => {
-    const { settings, callbacks, options } = args ?? {};
     try {
         MulmoStudioContextMethods.setSessionState(context, "image", true);
-        const newContext = await generateImages(context, settings, callbacks, options);
+        const newContext = await generateImages(context, args);
         MulmoStudioContextMethods.setSessionState(context, "image", false);
         return newContext;
     }
@@ -402,7 +404,8 @@ export const images = async (context, args) => {
 };
 // public api
 export const generateBeatImage = async (inputs) => {
-    const { index, context, settings, callbacks, forceMovie, forceImage } = inputs;
+    const { index, context, args } = inputs;
+    const { settings, callbacks, forceMovie, forceImage } = args ?? {};
     const options = await graphOption(context, settings);
     const injections = await prepareGenerateImages(context);
     const graph = new GraphAI(beat_graph_data, defaultAgents, options);

package/lib/actions/movie.js

@@ -162,7 +162,7 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, context) => {
             beatTimestamps.push(timestamp);
             return timestamp; // Skip voice-over beats.
         }
-        const sourceFile = studioBeat.lipSyncFile ?? studioBeat.soundEffectFile ?? studioBeat.movieFile ?? studioBeat.imageFile;
+        const sourceFile = studioBeat.lipSyncFile ?? studioBeat.soundEffectFile ?? studioBeat.movieFile ?? studioBeat.htmlImageFile ?? studioBeat.imageFile;
         assert(!!sourceFile, `studioBeat.imageFile or studioBeat.movieFile is not set: index=${index}`);
         assert(!!studioBeat.duration, `studioBeat.duration is not set: index=${index}`);
         const extraPadding = (() => {

package/lib/actions/translate.d.ts

@@ -1,6 +1,5 @@
 import "dotenv/config";
-import type { CallbackFunction } from "graphai";
-import type { LANG, MulmoStudioContext } from "../types/index.js";
+import type { LANG, MulmoStudioContext, PublicAPIArgs } from "../types/index.js";
 export declare const translateTextGraph: {
     version: number;
     nodes: {
@@ -50,11 +49,5 @@ export declare const getOutputMultilingualFilePathAndMkdir: (context: MulmoStudi
     outputMultilingualFilePath: string;
     outDirPath: string;
 };
-export declare const translateBeat: (index: number, context: MulmoStudioContext, targetLangs: string[], args?: {
-    settings?: Record<string, string>;
-    callbacks?: CallbackFunction[];
-}) => Promise<void>;
-export declare const translate: (context: MulmoStudioContext, args?: {
-    callbacks?: CallbackFunction[];
-    settings?: Record<string, string>;
-}) => Promise<MulmoStudioContext>;
+export declare const translateBeat: (index: number, context: MulmoStudioContext, targetLangs: string[], args?: PublicAPIArgs) => Promise<void>;
+export declare const translate: (context: MulmoStudioContext, args?: PublicAPIArgs) => Promise<MulmoStudioContext>;

package/lib/agents/lipsync_replicate_agent.js

@@ -1,4 +1,4 @@
-import { readFileSync } from "fs";
+import { readFileSync, existsSync } from "fs";
 import { GraphAILogger } from "graphai";
 import Replicate from "replicate";
 import { provider2LipSyncAgent } from "../utils/provider2agent.js";
@@ -12,11 +12,17 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
     const replicate = new Replicate({
         auth: apiKey,
     });
-    const videoBuffer = movieFile ? readFileSync(movieFile) : undefined;
+    if (!audioFile || !existsSync(audioFile)) {
+        throw new Error(`lipSyncReplicateAgent audioFile not exist: ${audioFile}`);
+    }
     const audioBuffer = readFileSync(audioFile);
+    const videoBuffer = movieFile ? readFileSync(movieFile) : undefined;
     const imageBuffer = imageFile ? readFileSync(imageFile) : undefined;
-    const videoUri = videoBuffer ? `data:video/quicktime;base64,${videoBuffer.toString("base64")}` : undefined;
+    if (!videoBuffer && !imageBuffer) {
+        throw new Error("lipSyncReplicateAgent Either movieFile or imageFile is required");
+    }
     const audioUri = `data:audio/wav;base64,${audioBuffer.toString("base64")}`;
+    const videoUri = videoBuffer ? `data:video/quicktime;base64,${videoBuffer.toString("base64")}` : undefined;
     const imageUri = imageBuffer ? `data:image/png;base64,${imageBuffer.toString("base64")}` : undefined;
     const input = {
         video: undefined,

package/lib/cli/commands/tool/index.js

@@ -2,7 +2,8 @@ import * as scriptingCmd from "./scripting/index.js";
 import * as promptCmd from "./prompt/index.js";
 import * as schemaCmd from "./schema/index.js";
 import * as storyToScriptCmd from "./story_to_script/index.js";
+import * as whisperCmd from "./whisper/index.js";
 export const command = "tool <command>";
 export const desc = "Generate Mulmo script and other tools";
-export const builder = (y) => y.command(scriptingCmd).command(promptCmd).command(schemaCmd).command(storyToScriptCmd).demandCommand().strict();
+export const builder = (y) => y.command(scriptingCmd).command(promptCmd).command(schemaCmd).command(storyToScriptCmd).command(whisperCmd).demandCommand().strict();
 export const handler = (__argv) => { };

package/lib/cli/commands/tool/whisper/builder.d.ts

@@ -0,0 +1,4 @@
+import { Argv } from "yargs";
+export declare const builder: (yargs: Argv) => Argv<{
+    file: string;
+}>;

package/lib/cli/commands/tool/whisper/builder.js

@@ -0,0 +1,7 @@
+export const builder = (yargs) => {
+    return yargs.positional("file", {
+        describe: "File path to process",
+        type: "string",
+        demandOption: true,
+    });
+};

package/lib/cli/commands/tool/whisper/handler.d.ts

@@ -0,0 +1,5 @@
+import "dotenv/config";
+import { ToolCliArgs } from "../../../../types/cli_types.js";
+export declare const handler: (argv: ToolCliArgs<{
+    file: string;
+}>) => Promise<void>;

package/lib/cli/commands/tool/whisper/handler.js

@@ -0,0 +1,100 @@
+import "dotenv/config";
+import { existsSync, createReadStream, writeFileSync, mkdirSync } from "fs";
+import { resolve, basename, extname, join } from "path";
+import OpenAI from "openai";
+import { mulmoScriptSchema } from "../../../../types/index.js";
+import { ffmpegGetMediaDuration } from "../../../../utils/ffmpeg_utils.js";
+import { GraphAILogger } from "graphai";
+const createMulmoScript = (fullPath, beats) => {
+    return mulmoScriptSchema.parse({
+        $mulmocast: {
+            version: "1.1",
+            credit: "closing",
+        },
+        canvasSize: {
+            width: 1536,
+            height: 1024,
+        },
+        lang: "en",
+        title: "Music Video",
+        captionParams: {
+            lang: "en",
+            styles: ["font-size: 64px", "width: 90%", "padding-left: 5%", "padding-right: 5%"],
+        },
+        beats,
+        audioParams: {
+            bgm: {
+                kind: "path",
+                path: fullPath,
+            },
+            padding: 0.0,
+            introPadding: 0.0,
+            closingPadding: 0.0,
+            outroPadding: 0.0,
+            bgmVolume: 1.0,
+            audioVolume: 0.0,
+            suppressSpeech: true,
+        },
+    });
+};
+export const handler = async (argv) => {
+    const { file } = argv;
+    const fullPath = resolve(file);
+    const filename = basename(file, extname(file));
+    if (!existsSync(fullPath)) {
+        GraphAILogger.error(`Error: File '${fullPath}' does not exist.`);
+        process.exit(1);
+    }
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+        GraphAILogger.error("Error: OPENAI_API_KEY environment variable is required");
+        process.exit(1);
+    }
+    try {
+        // Get audio duration using FFmpeg
+        const { duration: audioDuration } = await ffmpegGetMediaDuration(fullPath);
+        GraphAILogger.info(`Audio duration: ${audioDuration.toFixed(2)} seconds`);
+        const openai = new OpenAI({ apiKey });
+        const transcription = await openai.audio.transcriptions.create({
+            file: createReadStream(fullPath),
+            model: "whisper-1",
+            response_format: "verbose_json",
+            timestamp_granularities: ["word", "segment"],
+        });
+        if (transcription.segments) {
+            const starts = transcription.segments.map((segment) => segment.start);
+            starts[0] = 0;
+            starts.push(audioDuration);
+            // Create beats from transcription segments
+            const beats = transcription.segments.map((segment, index) => {
+                const duration = Math.round((starts[index + 1] - starts[index]) * 100) / 100;
+                return {
+                    text: segment.text,
+                    duration,
+                    /*
+                    image: {
+                      type: "textSlide",
+                      slide: {
+                        title: "Place Holder",
+                      },
+                    },
+                    */
+                };
+            });
+            // Create the script with the processed beats
+            const script = createMulmoScript(fullPath, beats);
+            // Save script to output directory
+            const outputDir = "output";
+            if (!existsSync(outputDir)) {
+                mkdirSync(outputDir, { recursive: true });
+            }
+            const outputPath = join(outputDir, `${filename}.json`);
+            writeFileSync(outputPath, JSON.stringify(script, null, 2));
+            GraphAILogger.info(`Script saved to: ${outputPath}`);
+        }
+    }
+    catch (error) {
+        GraphAILogger.error("Error transcribing audio:", error);
+        process.exit(1);
+    }
+};

package/lib/cli/commands/tool/whisper/index.d.ts

@@ -0,0 +1,4 @@
+export declare const command = "whisper <file>";
+export declare const desc = "Process file with whisper";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/commands/tool/whisper/index.js

@@ -0,0 +1,4 @@
+export const command = "whisper <file>";
+export const desc = "Process file with whisper";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/data/promptTemplates.d.ts

@@ -32,6 +32,8 @@ export declare const promptTemplates: ({
                 ani?: undefined;
                 presenter?: undefined;
                 optimus?: undefined;
+                "[CHARACTER_1_ID]"?: undefined;
+                "[CHARACTER_2_ID]"?: undefined;
             };
             style: string;
             provider?: undefined;
@@ -99,6 +101,8 @@ export declare const promptTemplates: ({
                 girl?: undefined;
                 presenter?: undefined;
                 optimus?: undefined;
+                "[CHARACTER_1_ID]"?: undefined;
+                "[CHARACTER_2_ID]"?: undefined;
             };
             style: string;
             provider?: undefined;
@@ -169,6 +173,8 @@ export declare const promptTemplates: ({
                 ani?: undefined;
                 presenter?: undefined;
                 optimus?: undefined;
+                "[CHARACTER_1_ID]"?: undefined;
+                "[CHARACTER_2_ID]"?: undefined;
             };
             provider: string;
             style?: undefined;
@@ -286,6 +292,8 @@ export declare const promptTemplates: ({
                 girl?: undefined;
                 ani?: undefined;
                 optimus?: undefined;
+                "[CHARACTER_1_ID]"?: undefined;
+                "[CHARACTER_2_ID]"?: undefined;
             };
             style: string;
             provider?: undefined;
@@ -356,6 +364,8 @@ export declare const promptTemplates: ({
                 };
                 girl?: undefined;
                 ani?: undefined;
+                "[CHARACTER_1_ID]"?: undefined;
+                "[CHARACTER_2_ID]"?: undefined;
             };
             style: string;
             provider?: undefined;
@@ -503,6 +513,72 @@ export declare const promptTemplates: ({
     scriptName: string;
     systemPrompt: string;
     title: string;
+} | {
+    description: string;
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        audioParams: {
+            audioVolume: number;
+            bgmVolume: number;
+            closingPadding: number;
+            introPadding: number;
+            outroPadding: number;
+            padding: number;
+            suppressSpeech: boolean;
+            bgm?: undefined;
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            images: {
+                "[CHARACTER_1_ID]": {
+                    prompt: string;
+                    type: string;
+                };
+                "[CHARACTER_2_ID]": {
+                    prompt: string;
+                    type: string;
+                };
+                girl?: undefined;
+                ani?: undefined;
+                presenter?: undefined;
+                optimus?: undefined;
+            };
+            style: string;
+            provider?: undefined;
+        };
+        movieParams: {
+            provider: string;
+            model?: undefined;
+        };
+        soundEffectParams: {
+            provider: string;
+        };
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: string;
+                    };
+                    voiceId: string;
+                    lang?: undefined;
+                    speechOptions?: undefined;
+                };
+                Announcer?: undefined;
+                Student?: undefined;
+                Teacher?: undefined;
+            };
+        };
+    };
+    scriptName: string;
+    systemPrompt: string;
+    title: string;
 } | {
     description: string;
     filename: string;

package/lib/data/promptTemplates.js

@@ -749,6 +749,61 @@ export const promptTemplates = [
         systemPrompt: "This script is for YouTube shorts. The first beat should be a hook, which describes the topic. Another AI will generate images for each beat based on the image prompt of that beat. Movie prompts must be written in English.",
         title: "Short movie template",
     },
+    {
+        description: "Template for Multi-character Story.",
+        filename: "sifi_story",
+        presentationStyle: {
+            $mulmocast: {
+                credit: "closing",
+                version: "1.1",
+            },
+            audioParams: {
+                audioVolume: 1,
+                bgmVolume: 0.2,
+                closingPadding: 0.8,
+                introPadding: 1,
+                outroPadding: 1,
+                padding: 0.3,
+                suppressSpeech: false,
+            },
+            canvasSize: {
+                height: 1024,
+                width: 1536,
+            },
+            imageParams: {
+                images: {
+                    "[CHARACTER_1_ID]": {
+                        prompt: "[IMAGE PROMPT FOR THIS CHARACTER]",
+                        type: "imagePrompt",
+                    },
+                    "[CHARACTER_2_ID]": {
+                        prompt: "[IMAGE PROMPT FOR THIS CHARACTER]",
+                        type: "imagePrompt",
+                    },
+                },
+                style: "<style>A dreamy, hyper-detailed anime style that blends photorealistic backgrounds with vibrant, saturated colors. The skies are often filled with luminous clouds, dazzling sunsets, or star-filled nights, rendered with a glowing, almost ethereal quality. Urban landscapes and rural scenery are meticulously illustrated, with attention to tiny details like reflections in puddles, neon lights, or the texture of grass swaying in the wind. Characters are drawn with soft, expressive features, standing out against the breathtaking environments, creating a sense of emotional depth and lyrical atmosphere. The overall mood is cinematic, romantic, and filled with a sense of fleeting beauty and longing.</style>",
+            },
+            movieParams: {
+                provider: "replicate",
+            },
+            soundEffectParams: {
+                provider: "replicate",
+            },
+            speechParams: {
+                speakers: {
+                    Presenter: {
+                        displayName: {
+                            en: "Presenter",
+                        },
+                        voiceId: "shimmer",
+                    },
+                },
+            },
+        },
+        scriptName: "story_with_characters.json",
+        systemPrompt: "Break the story into multiple beats, and put the story text in 'text' field. Generate image prompt for each character in the imageParams.images. Another AI will generate image for each beat based on its imagePrompt and specified characters in 'imageNames'. You don't need to repeat the image style in those image prompts. Use the JSON below as a template.",
+        title: "Multi-character Story",
+    },
     {
         description: "Template for A Movie Trailer.",
         filename: "trailer",

package/lib/data/scriptTemplates.d.ts

@@ -594,6 +594,26 @@ export declare const scriptTemplates: ({
     description?: undefined;
     canvasSize?: undefined;
     captionParams?: undefined;
+} | {
+    $mulmocast: {
+        credit: string;
+        version: string;
+    };
+    beats: {
+        imageNames: string[];
+        imagePrompt: string;
+        text: string;
+    }[];
+    filename: string;
+    lang: string;
+    title: string;
+    references?: undefined;
+    htmlImageParams?: undefined;
+    imageParams?: undefined;
+    movieParams?: undefined;
+    description?: undefined;
+    canvasSize?: undefined;
+    captionParams?: undefined;
 } | {
     $mulmocast: {
         credit: string;

package/lib/data/scriptTemplates.js

@@ -1001,6 +1001,32 @@ export const scriptTemplates = [
         },
         title: "[TITLE: Brief, engaging title for the topic]",
     },
+    {
+        $mulmocast: {
+            credit: "closing",
+            version: "1.1",
+        },
+        beats: [
+            {
+                imageNames: ["[CHARACTER_ID_1]", "[CHARACTER_ID_2]"],
+                imagePrompt: "[IMAGE_PROMPT FOR THIS BEAT with both characters]",
+                text: "[STORY TEXT FOR THIS BEAT FOR THIS BEAT]",
+            },
+            {
+                imageNames: ["[CHARACTER_ID_1]"],
+                imagePrompt: "[IMAGE_PROMPT FOR THIS BEAT with a single character]",
+                text: "[STORY TEXT FOR THIS BEAT FOR THIS BEAT]",
+            },
+            {
+                imageNames: ["[CHARACTER_ID_2]"],
+                imagePrompt: "[IMAGE_PROMPT FOR THIS BEAT with another character]",
+                text: "[STORY TEXT FOR THIS BEAT FOR THIS BEAT]",
+            },
+        ],
+        filename: "story_with_characters",
+        lang: "en",
+        title: "[TITLE: Brief, engaging title for the topic]",
+    },
     {
         $mulmocast: {
             credit: "closing",

package/lib/data/templateDataSet.d.ts

@@ -15,5 +15,6 @@ export declare const templateDataSet: {
     realistic_movie: string;
     sensei_and_taro: string;
     shorts: string;
+    sifi_story: string;
     trailer: string;
 };

package/lib/data/templateDataSet.js

@@ -63,6 +63,10 @@ export const templateDataSet = {
         "```JSON\n" +
         `{"$mulmocast":{"version":"1.1"},"title":"[TITLE: Brief, engaging title for the topic]","lang":"en","references":[{"url":"[SOURCE_URL: URL of the source material]","title":"[SOURCE_TITLE: Title of the referenced article, or paper]","type":"[SOURCE_TYPE: article, paper]"}],"movieParams":{"provider":"google"},"beats":[{"text":"[OPENING_BEAT: Introduce the topic with a hook. Reference the source material and set up why this topic matters. Usually 2-3 sentences that grab attention and provide context.]","imagePrompt":"[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]","moviePrompt":"[MOVIE_PROMPT: A movie prompt for that image.]"},{"text":"[MAIN_CONCEPT: Define or explain the core concept/idea. This should be the central focus of your narrative. Keep it clear and accessible.]","imagePrompt":"[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]","moviePrompt":"[MOVIE_PROMPT: A movie prompt for that image.]"},{"text":"[SUPPORTING_DETAIL_1: Additional context, examples, or elaboration that helps illustrate the main concept. This could include how it works, why it's important, or real-world applications.]","imagePrompt":"[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]","moviePrompt":"[MOVIE_PROMPT: A movie prompt for that image.]"},{"text":"[SUPPORTING_DETAIL_2: Continue with more examples, deeper explanation, or different aspects of the topic if needed.]","imagePrompt":"[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]","moviePrompt":"[MOVIE_PROMPT: A movie prompt for that image.]"},{"text":"[ADDITIONAL_BEATS: Add more beats as necessary to fully explore the topic. Complex topics may require 6-10+ beats to cover adequately. Each beat should advance the narrative or provide valuable information.]","imagePrompt":"[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]","moviePrompt":"[MOVIE_PROMPT: A movie prompt for that image.]"},{"text":"[CONCLUSION/IMPACT: Wrap up with the significance, implications, or key takeaway. Help the audience understand why this matters to them.]","imagePrompt":"[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]","moviePrompt":"[MOVIE_PROMPT: A movie prompt for that image.]"}],"canvasSize":{"width":720,"height":1280},"imageParams":{"style":"<style>Photo realistic, cinematic.</style>"}}\n` +
         "```",
+    sifi_story: "Break the story into multiple beats, and put the story text in 'text' field. Generate image prompt for each character in the imageParams.images. Another AI will generate image for each beat based on its imagePrompt and specified characters in 'imageNames'. You don't need to repeat the image style in those image prompts. Use the JSON below as a template.\n" +
+        "```JSON\n" +
+        '{"$mulmocast":{"version":"1.1","credit":"closing"},"title":"[TITLE: Brief, engaging title for the topic]","lang":"en","beats":[{"text":"[STORY TEXT FOR THIS BEAT FOR THIS BEAT]","imagePrompt":"[IMAGE_PROMPT FOR THIS BEAT with both characters]","imageNames":["[CHARACTER_ID_1]","[CHARACTER_ID_2]"]},{"text":"[STORY TEXT FOR THIS BEAT FOR THIS BEAT]","imagePrompt":"[IMAGE_PROMPT FOR THIS BEAT with a single character]","imageNames":["[CHARACTER_ID_1]"]},{"text":"[STORY TEXT FOR THIS BEAT FOR THIS BEAT]","imagePrompt":"[IMAGE_PROMPT FOR THIS BEAT with another character]","imageNames":["[CHARACTER_ID_2]"]}],"canvasSize":{"width":1536,"height":1024},"imageParams":{"style":"<style>A dreamy, hyper-detailed anime style that blends photorealistic backgrounds with vibrant, saturated colors. The skies are often filled with luminous clouds, dazzling sunsets, or star-filled nights, rendered with a glowing, almost ethereal quality. Urban landscapes and rural scenery are meticulously illustrated, with attention to tiny details like reflections in puddles, neon lights, or the texture of grass swaying in the wind. Characters are drawn with soft, expressive features, standing out against the breathtaking environments, creating a sense of emotional depth and lyrical atmosphere. The overall mood is cinematic, romantic, and filled with a sense of fleeting beauty and longing.</style>","images":{"[CHARACTER_1_ID]":{"type":"imagePrompt","prompt":"[IMAGE PROMPT FOR THIS CHARACTER]"},"[CHARACTER_2_ID]":{"type":"imagePrompt","prompt":"[IMAGE PROMPT FOR THIS CHARACTER]"}}}}\n' +
+        "```",
     trailer: "This script is for a movie trailer. Another AI will generate images for each beat based on the image prompt of that beat. Movie prompts must be written in English.\n" +
         "```JSON\n" +
         '{"$mulmocast":{"version":"1.1"},"title":"[TITLE: Brief, engaging title for the topic]","lang":"en","references":[{"url":"[SOURCE_URL: URL of the source material]","title":"[SOURCE_TITLE: Title of the referenced article, or paper]","type":"[SOURCE_TYPE: article, paper]"}],"movieParams":{"provider":"google"},"beats":[{"duration":5,"imagePrompt":"[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]","moviePrompt":"[MOVIE_PROMPT: A movie prompt for that image.]"},{"duration":5,"imagePrompt":"[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]","moviePrompt":"[MOVIE_PROMPT: A movie prompt for that image.]"},{"duration":5,"imagePrompt":"[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]","moviePrompt":"[MOVIE_PROMPT: A movie prompt for that image.]"},{"duration":5,"imagePrompt":"[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]","moviePrompt":"[MOVIE_PROMPT: A movie prompt for that image.]"},{"duration":5,"imagePrompt":"[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]","moviePrompt":"[MOVIE_PROMPT: A movie prompt for that image.]"},{"duration":5,"imagePrompt":"[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]","moviePrompt":"[MOVIE_PROMPT: A movie prompt for that image.]"}],"canvasSize":{"width":1280,"height":720},"imageParams":{"style":"<style>Photo realistic, cinematic.</style>"},"audioParams":{"padding":0,"introPadding":0,"closingPadding":0,"outroPadding":2.5,"bgm":{"kind":"url","url":"https://raw.githubusercontent.com/receptron/mulmocast-media/refs/heads/main/bgms/trailer_dramatic.mp3"}}}\n' +

package/lib/types/schema.d.ts

@@ -5702,6 +5702,7 @@ export declare const mulmoStudioBeatSchema: z.ZodObject<{
     soundEffectFile: z.ZodOptional<z.ZodString>;
     lipSyncFile: z.ZodOptional<z.ZodString>;
     captionFile: z.ZodOptional<z.ZodString>;
+    htmlImageFile: z.ZodOptional<z.ZodString>;
 }, "strict", z.ZodTypeAny, {
     duration?: number | undefined;
     startAt?: number | undefined;
@@ -5716,6 +5717,7 @@ export declare const mulmoStudioBeatSchema: z.ZodObject<{
     soundEffectFile?: string | undefined;
     lipSyncFile?: string | undefined;
     captionFile?: string | undefined;
+    htmlImageFile?: string | undefined;
 }, {
     duration?: number | undefined;
     startAt?: number | undefined;
@@ -5730,6 +5732,7 @@ export declare const mulmoStudioBeatSchema: z.ZodObject<{
     soundEffectFile?: string | undefined;
     lipSyncFile?: string | undefined;
     captionFile?: string | undefined;
+    htmlImageFile?: string | undefined;
 }>;
 export declare const mulmoStudioMultiLingualDataSchema: z.ZodObject<{
     multiLingualTexts: z.ZodRecord<z.ZodString, z.ZodObject<{
@@ -8325,6 +8328,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
         soundEffectFile: z.ZodOptional<z.ZodString>;
         lipSyncFile: z.ZodOptional<z.ZodString>;
         captionFile: z.ZodOptional<z.ZodString>;
+        htmlImageFile: z.ZodOptional<z.ZodString>;
     }, "strict", z.ZodTypeAny, {
         duration?: number | undefined;
         startAt?: number | undefined;
@@ -8339,6 +8343,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
         soundEffectFile?: string | undefined;
         lipSyncFile?: string | undefined;
         captionFile?: string | undefined;
+        htmlImageFile?: string | undefined;
     }, {
         duration?: number | undefined;
         startAt?: number | undefined;
@@ -8353,6 +8358,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
         soundEffectFile?: string | undefined;
         lipSyncFile?: string | undefined;
         captionFile?: string | undefined;
+        htmlImageFile?: string | undefined;
     }>, "many">;
 }, "strict", z.ZodTypeAny, {
     beats: {
@@ -8369,6 +8375,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
         soundEffectFile?: string | undefined;
         lipSyncFile?: string | undefined;
         captionFile?: string | undefined;
+        htmlImageFile?: string | undefined;
     }[];
     script: {
         lang: string;
@@ -8710,6 +8717,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
         soundEffectFile?: string | undefined;
         lipSyncFile?: string | undefined;
         captionFile?: string | undefined;
+        htmlImageFile?: string | undefined;
     }[];
     script: {
         lang: string;

package/lib/types/schema.js

@@ -396,6 +396,7 @@ export const mulmoStudioBeatSchema = z
     soundEffectFile: z.string().optional(), // path to the sound effect file
     lipSyncFile: z.string().optional(), // path to the lip sync file
     captionFile: z.string().optional(), // path to the caption image
+    htmlImageFile: z.string().optional(), // path to the html image
 })
     .strict();
 export const mulmoStudioMultiLingualDataSchema = z.object({

package/lib/types/type.d.ts

@@ -1,3 +1,4 @@
+import { type CallbackFunction } from "graphai";
 import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
 import type { LLM } from "../utils/provider2agent.js";
@@ -128,3 +129,7 @@ export type InitOptions = {
     c?: string;
     p?: string;
 };
+export type PublicAPIArgs = {
+    settings?: Record<string, string>;
+    callbacks?: CallbackFunction[];
+};

package/lib/utils/context.d.ts

@@ -14,6 +14,7 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
         soundEffectFile?: string | undefined;
         lipSyncFile?: string | undefined;
         captionFile?: string | undefined;
+        htmlImageFile?: string | undefined;
     }[];
     script: {
         lang: string;
@@ -360,6 +361,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             soundEffectFile?: string | undefined;
             lipSyncFile?: string | undefined;
             captionFile?: string | undefined;
+            htmlImageFile?: string | undefined;
         }[];
         script: {
             lang: string;

package/lib/utils/file.d.ts

@@ -24,7 +24,10 @@ export declare const getAudioFilePath: (audioDirPath: string, dirName: string, f
 export declare const getAudioArtifactFilePath: (context: MulmoStudioContext) => string;
 export declare const getOutputVideoFilePath: (outDirPath: string, fileName: string, lang?: string, caption?: string) => string;
 export declare const imageSuffix = "p";
-export declare const getBeatPngImagePath: (context: MulmoStudioContext, index: number) => string;
+export declare const getBeatPngImagePath: (context: MulmoStudioContext, index: number) => {
+    imagePath: string;
+    htmlImageFile: string;
+};
 export declare const getBeatMoviePaths: (context: MulmoStudioContext, index: number) => {
     movieFile: string;
     soundEffectFile: string;

package/lib/utils/file.js

@@ -89,10 +89,10 @@ export const imageSuffix = "p";
 export const getBeatPngImagePath = (context, index) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
     const beat = context.studio.script.beats[index]; // beat could be undefined only in a test case.
-    if (beat?.id) {
-        return `${imageProjectDirPath}/${beat.id}.png`;
-    }
-    return `${imageProjectDirPath}/${index}${imageSuffix}.png`;
+    const filename = beat?.id ? `${beat.id}` : `${index}${imageSuffix}`;
+    const imagePath = `${imageProjectDirPath}/${filename}.png`;
+    const htmlImageFile = `${imageProjectDirPath}/${filename}_html.png`;
+    return { imagePath, htmlImageFile };
 };
 export const getBeatMoviePaths = (context, index) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "1.2.4",
+  "version": "1.2.6",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -50,6 +50,7 @@
     "prompt": "npx tsx ./src/cli/bin.ts tool prompt",
     "schema": "npx tsx ./src/cli/bin.ts tool schema",
     "story_to_script": "npx tsx ./src/cli/bin.ts tool story_to_script",
+    "whisper": "npx tsx ./src/cli/bin.ts tool whisper",
     "latest": "yarn upgrade-interactive  --latest",
     "format": "prettier --write '{src,scripts,assets/templates,assets/styles,draft,ideason,scripts_mag2,proto,test,batch,graphai,output,docs/scripts}/**/*.{ts,json,yaml}'",
     "deep_research": "npx tsx ./src/tools/deep_research.ts",
@@ -66,29 +67,29 @@
   "homepage": "https://github.com/receptron/mulmocast-cli#readme",
   "dependencies": {
     "@google-cloud/text-to-speech": "^6.2.0",
-    "@google/genai": "^1.13.0",
-    "@graphai/anthropic_agent": "^2.0.9",
+    "@google/genai": "^1.15.0",
+    "@graphai/anthropic_agent": "^2.0.11",
     "@graphai/browserless_agent": "^2.0.1",
     "@graphai/gemini_agent": "^2.0.1",
     "@graphai/groq_agent": "^2.0.2",
     "@graphai/input_agents": "^1.0.2",
-    "@graphai/openai_agent": "^2.0.4",
+    "@graphai/openai_agent": "^2.0.7",
     "@graphai/stream_agent_filter": "^2.0.2",
-    "@graphai/vanilla": "^2.0.10",
-    "@graphai/vanilla_node_agents": "^2.0.1",
+    "@graphai/vanilla": "^2.0.12",
+    "@graphai/vanilla_node_agents": "^2.0.4",
     "@inquirer/input": "^4.2.1",
     "@inquirer/select": "^4.3.1",
-    "@modelcontextprotocol/sdk": "^1.15.1",
-    "@tavily/core": "^0.5.9",
-    "canvas": "^3.1.2",
+    "@modelcontextprotocol/sdk": "^1.17.3",
+    "@tavily/core": "^0.5.11",
+    "canvas": "^3.2.0",
     "clipboardy": "^4.0.0",
     "dotenv": "^17.2.1",
     "fluent-ffmpeg": "^2.1.3",
     "graphai": "^2.0.14",
-    "marked": "^16.1.2",
+    "marked": "^16.2.0",
     "ora": "^8.2.0",
-    "puppeteer": "^24.16.2",
-    "replicate": "^1.0.1",
+    "puppeteer": "^24.17.0",
+    "replicate": "^1.1.0",
     "yaml": "^2.8.1",
     "yargs": "^18.0.0",
     "zod": "^3.25.76",
@@ -97,7 +98,7 @@
   "devDependencies": {
     "@anatine/zod-mock": "^3.14.0",
     "@faker-js/faker": "^9.9.0",
-    "@receptron/test_utils": "^2.0.1",
+    "@receptron/test_utils": "^2.0.3",
     "@types/fluent-ffmpeg": "^2.1.26",
     "@types/yargs": "^17.0.33",
     "eslint": "^9.33.0",
@@ -108,7 +109,7 @@
     "ts-node": "^10.9.2",
     "tsx": "^4.20.4",
     "typescript": "^5.9.2",
-    "typescript-eslint": "^8.39.1"
+    "typescript-eslint": "^8.40.0"
   },
   "engines": {
     "node": ">=18.0.0"

package/scripts/templates/story_with_characters.json

@@ -0,0 +1,25 @@
+{
+  "$mulmocast": {
+    "version": "1.1",
+    "credit": "closing"
+  },
+  "title": "[TITLE: Brief, engaging title for the topic]",
+  "lang": "en",
+  "beats": [
+    {
+      "text": "[STORY TEXT FOR THIS BEAT FOR THIS BEAT]",
+      "imagePrompt": "[IMAGE_PROMPT FOR THIS BEAT with both characters]",
+      "imageNames": ["[CHARACTER_ID_1]", "[CHARACTER_ID_2]"]
+    },
+    {
+      "text": "[STORY TEXT FOR THIS BEAT FOR THIS BEAT]",
+      "imagePrompt": "[IMAGE_PROMPT FOR THIS BEAT with a single character]",
+      "imageNames": ["[CHARACTER_ID_1]"]
+    },
+    {
+      "text": "[STORY TEXT FOR THIS BEAT FOR THIS BEAT]",
+      "imagePrompt": "[IMAGE_PROMPT FOR THIS BEAT with another character]",
+      "imageNames": ["[CHARACTER_ID_2]"]
+    }
+  ]
+}