mulmocast 2.1.20 → 2.1.22

package/README.md

@@ -1,5 +1,7 @@
 # MulmoCast: A Multi-Modal Presentation Tool for the AI-Native Era
 
+[![npm version](https://badge.fury.io/js/mulmocast.svg)](https://www.npmjs.com/package/mulmocast)
+
 ## Quick Start Guide
 
 If you want to try our beta version, follow the instruction in the release note below.

package/lib/actions/captions.js

@@ -7,6 +7,105 @@ import { renderHTMLToImage, interpolate } from "../utils/markdown.js";
 import { MulmoStudioContextMethods, MulmoPresentationStyleMethods } from "../methods/index.js";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 const vanillaAgents = agents.default ?? agents;
+const defaultDelimiters = ["。", "？", "！", ".", "?", "!"];
+// Split text by delimiters while keeping delimiters attached to the preceding text
+const splitTextByDelimiters = (text, delimiters) => {
+    if (!text || delimiters.length === 0) {
+        return [text];
+    }
+    const { segments, current } = [...text].reduce((acc, char) => {
+        const newCurrent = acc.current + char;
+        if (delimiters.includes(char)) {
+            const trimmed = newCurrent.trim();
+            return {
+                segments: trimmed ? [...acc.segments, trimmed] : acc.segments,
+                current: "",
+            };
+        }
+        return { ...acc, current: newCurrent };
+    }, { segments: [], current: "" });
+    const finalSegments = current.trim() ? [...segments, current.trim()] : segments;
+    return finalSegments.length > 0 ? finalSegments : [text];
+};
+// Get split texts based on settings
+const getSplitTexts = (text, texts, textSplit) => {
+    // Manual split takes precedence
+    if (texts && texts.length > 0) {
+        return texts;
+    }
+    // No splitting or undefined
+    if (!textSplit || textSplit.type === "none") {
+        return [text];
+    }
+    // Split by delimiters
+    if (textSplit.type === "delimiters") {
+        const delimiters = textSplit.delimiters ?? defaultDelimiters;
+        return splitTextByDelimiters(text, delimiters);
+    }
+    return [text];
+};
+// Calculate timing ratios based on text length
+const calculateTimingRatios = (splitTexts) => {
+    const totalLength = splitTexts.reduce((sum, t) => sum + t.length, 0);
+    if (totalLength === 0) {
+        return splitTexts.map(() => 1 / splitTexts.length);
+    }
+    return splitTexts.map((t) => t.length / totalLength);
+};
+// Convert ratios to cumulative ratios: [0.3, 0.5, 0.2] -> [0, 0.3, 0.8, 1.0]
+const calculateCumulativeRatios = (ratios) => {
+    return ratios.reduce((acc, ratio) => [...acc, acc[acc.length - 1] + ratio], [0]);
+};
+// Generate caption files for a single beat
+const generateBeatCaptions = async (beat, context, index) => {
+    const captionParams = mulmoCaptionParamsSchema.parse({ ...context.studio.script.captionParams, ...beat.captionParams });
+    const canvasSize = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
+    const template = getHTMLFile("caption");
+    if (captionParams.lang && !context.multiLingual?.[index]?.multiLingualTexts?.[captionParams.lang]) {
+        GraphAILogger.warn(`No multiLingual caption found for beat ${index}, lang: ${captionParams.lang}`);
+    }
+    const text = localizedText(beat, context.multiLingual?.[index], captionParams.lang, context.studio.script.lang);
+    // Get beat timing info
+    const studioBeat = context.studio.beats[index];
+    const beatStartAt = studioBeat.startAt ?? 0;
+    const beatDuration = studioBeat.duration ?? 0;
+    const introPadding = MulmoStudioContextMethods.getIntroPadding(context);
+    // Determine split texts based on captionSplit setting
+    const captionSplit = captionParams.captionSplit ?? "none";
+    const splitTexts = captionSplit === "estimate" ? getSplitTexts(text, beat.texts, captionParams.textSplit) : [text];
+    // Calculate timing
+    const cumulativeRatios = calculateCumulativeRatios(calculateTimingRatios(splitTexts));
+    // Generate caption images with absolute timing
+    const captionFiles = await Promise.all(splitTexts.map(async (segmentText, subIndex) => {
+        const imagePath = getCaptionImagePath(context, index, subIndex);
+        const htmlData = interpolate(template, {
+            caption: processLineBreaks(segmentText),
+            width: `${canvasSize.width}`,
+            height: `${canvasSize.height}`,
+            styles: captionParams.styles.join(";\n"),
+        });
+        await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, false, true);
+        return {
+            file: imagePath,
+            startAt: beatStartAt + introPadding + beatDuration * cumulativeRatios[subIndex],
+            endAt: beatStartAt + introPadding + beatDuration * cumulativeRatios[subIndex + 1],
+        };
+    }));
+    return captionFiles;
+};
+// GraphAI agent for caption generation
+const captionGenerationAgent = async (namedInputs) => {
+    const { beat, context, index } = namedInputs;
+    try {
+        MulmoStudioContextMethods.setBeatSessionState(context, "caption", index, beat.id, true);
+        const captionFiles = await generateBeatCaptions(beat, context, index);
+        context.studio.beats[index].captionFiles = captionFiles;
+        return captionFiles;
+    }
+    finally {
+        MulmoStudioContextMethods.setBeatSessionState(context, "caption", index, beat.id, false);
+    }
+};
 export const caption_graph_data = {
     version: 0.5,
     nodes: {
@@ -23,37 +122,8 @@ export const caption_graph_data = {
             graph: {
                 nodes: {
                     generateCaption: {
-                        agent: async (namedInputs) => {
-                            const { beat, context, index } = namedInputs;
-                            try {
-                                MulmoStudioContextMethods.setBeatSessionState(context, "caption", index, beat.id, true);
-                                const captionParams = mulmoCaptionParamsSchema.parse({ ...context.studio.script.captionParams, ...beat.captionParams });
-                                const canvasSize = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
-                                const imagePath = getCaptionImagePath(context, index);
-                                const template = getHTMLFile("caption");
-                                if (captionParams.lang && !context.multiLingual?.[index]?.multiLingualTexts?.[captionParams.lang]) {
-                                    GraphAILogger.warn(`No multiLingual caption found for beat ${index}, lang: ${captionParams.lang}`);
-                                }
-                                const text = localizedText(beat, context.multiLingual?.[index], captionParams.lang, context.studio.script.lang);
-                                const htmlData = interpolate(template, {
-                                    caption: processLineBreaks(text),
-                                    width: `${canvasSize.width}`,
-                                    height: `${canvasSize.height}`,
-                                    styles: captionParams.styles.join(";\n"),
-                                });
-                                await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, false, true);
-                                context.studio.beats[index].captionFile = imagePath;
-                                return imagePath;
-                            }
-                            finally {
-                                MulmoStudioContextMethods.setBeatSessionState(context, "caption", index, beat.id, false);
-                            }
-                        },
-                        inputs: {
-                            beat: ":beat",
-                            context: ":context",
-                            index: ":__mapIndex",
-                        },
+                        agent: captionGenerationAgent,
+                        inputs: { beat: ":beat", context: ":context", index: ":__mapIndex" },
                         isResult: true,
                     },
                 },

package/lib/actions/movie.d.ts

@@ -27,6 +27,10 @@ export declare const getTransitionVideoId: (transition: MulmoTransition, videoId
     beatIndex: number;
 };
 export declare const getConcatVideoFilter: (concatVideoId: string, videoIdsForBeats: VideoId[]) => string;
+export declare const getTransitionFrameDurations: (context: MulmoStudioContext, index: number) => {
+    firstDuration: number;
+    lastDuration: number;
+};
 export declare const validateBeatSource: (studioBeat: MulmoStudioContext["studio"]["beats"][number], index: number) => string;
 export declare const addSplitAndExtractFrames: (ffmpegContext: FfmpegContext, videoId: string, firstDuration: number, lastDuration: number, isMovie: boolean, needFirst: boolean, needLast: boolean, canvasInfo: {
     width: number;

package/lib/actions/movie.js

@@ -86,19 +86,22 @@ const getOutputOption = (audioId, videoId) => {
     ];
 };
 const addCaptions = (ffmpegContext, concatVideoId, context, caption) => {
-    const beatsWithCaptions = context.studio.beats.filter(({ captionFile }) => captionFile);
+    const beatsWithCaptions = context.studio.beats.filter(({ captionFiles }) => captionFiles && captionFiles.length > 0);
     if (caption && beatsWithCaptions.length > 0) {
-        const introPadding = MulmoStudioContextMethods.getIntroPadding(context);
-        return beatsWithCaptions.reduce((prevVideoId, beat, index) => {
-            const { startAt, duration, captionFile } = beat;
-            if (startAt !== undefined && duration !== undefined && captionFile !== undefined) {
-                const captionInputIndex = FfmpegContextAddInput(ffmpegContext, captionFile);
-                const compositeVideoId = `oc${index}`;
-                ffmpegContext.filterComplex.push(`[${prevVideoId}][${captionInputIndex}:v]overlay=format=auto:enable='between(t,${startAt + introPadding},${startAt + duration + introPadding})'[${compositeVideoId}]`);
-                return compositeVideoId;
+        const { videoId } = beatsWithCaptions.reduce((acc, beat) => {
+            const { captionFiles } = beat;
+            if (!captionFiles) {
+                return acc;
             }
-            return prevVideoId;
-        }, concatVideoId);
+            return captionFiles.reduce((innerAcc, captionData) => {
+                const { file, startAt, endAt } = captionData;
+                const captionInputIndex = FfmpegContextAddInput(ffmpegContext, file);
+                const compositeVideoId = `oc${innerAcc.captionIndex}`;
+                ffmpegContext.filterComplex.push(`[${innerAcc.videoId}][${captionInputIndex}:v]overlay=format=auto:enable='between(t,${startAt},${endAt})'[${compositeVideoId}]`);
+                return { videoId: compositeVideoId, captionIndex: innerAcc.captionIndex + 1 };
+            }, acc);
+        }, { videoId: concatVideoId, captionIndex: 0 });
+        return videoId;
     }
     return concatVideoId;
 };
@@ -280,24 +283,21 @@ const getClampedTransitionDuration = (transitionDuration, prevBeatDuration, curr
     const maxDuration = Math.min(prevBeatDuration, currentBeatDuration) * 0.9; // Use 90% to leave some margin
     return Math.min(transitionDuration, maxDuration);
 };
-const getTransitionFrameDurations = (context, index) => {
+export const getTransitionFrameDurations = (context, index) => {
     const minFrame = 1 / 30; // 30fpsを想定。最小1フレーム
     const beats = context.studio.beats;
     const scriptBeats = context.studio.script.beats;
+    const getTransitionDuration = (transition, prevBeatIndex, currentBeatIndex) => {
+        if (!transition || prevBeatIndex < 0 || currentBeatIndex >= beats.length)
+            return 0;
+        const prevBeatDuration = beats[prevBeatIndex].duration ?? 1;
+        const currentBeatDuration = beats[currentBeatIndex].duration ?? 1;
+        return getClampedTransitionDuration(transition.duration, prevBeatDuration, currentBeatDuration);
+    };
     const currentTransition = MulmoPresentationStyleMethods.getMovieTransition(context, scriptBeats[index]);
-    let firstDuration = 0;
-    if (currentTransition && index > 0) {
-        const prevBeatDuration = beats[index - 1].duration ?? 1;
-        const currentBeatDuration = beats[index].duration ?? 1;
-        firstDuration = getClampedTransitionDuration(currentTransition.duration, prevBeatDuration, currentBeatDuration);
-    }
+    const firstDuration = index > 0 ? getTransitionDuration(currentTransition, index - 1, index) : 0;
     const nextTransition = index < scriptBeats.length - 1 ? MulmoPresentationStyleMethods.getMovieTransition(context, scriptBeats[index + 1]) : null;
-    let lastDuration = 0;
-    if (nextTransition) {
-        const prevBeatDuration = beats[index].duration ?? 1;
-        const currentBeatDuration = beats[index + 1].duration ?? 1;
-        lastDuration = getClampedTransitionDuration(nextTransition.duration, prevBeatDuration, currentBeatDuration);
-    }
+    const lastDuration = getTransitionDuration(nextTransition, index, index + 1);
     return {
         firstDuration: Math.max(firstDuration, minFrame),
         lastDuration: Math.max(lastDuration, minFrame),

package/lib/types/schema.d.ts

@@ -190,9 +190,29 @@ export declare const mulmoTextSlideMediaSchema: z.ZodObject<{
         bullets: z.ZodOptional<z.ZodArray<z.ZodString>>;
     }, z.core.$strip>;
 }, z.core.$strict>;
+export declare const captionSplitSchema: z.ZodDefault<z.ZodEnum<{
+    none: "none";
+    estimate: "estimate";
+}>>;
+export declare const textSplitSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    type: z.ZodLiteral<"none">;
+}, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"delimiters">;
+    delimiters: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>], "type">;
 export declare const mulmoCaptionParamsSchema: z.ZodObject<{
     lang: z.ZodOptional<z.ZodString>;
     styles: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+    captionSplit: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
+        none: "none";
+        estimate: "estimate";
+    }>>>;
+    textSplit: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        type: z.ZodLiteral<"none">;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"delimiters">;
+        delimiters: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>], "type">>;
 }, z.core.$strict>;
 export declare const mulmoChartMediaSchema: z.ZodObject<{
     type: z.ZodLiteral<"chart">;
@@ -747,6 +767,7 @@ export declare const mulmoMovieParamsSchema: z.ZodObject<{
 export declare const mulmoBeatSchema: z.ZodObject<{
     speaker: z.ZodOptional<z.ZodString>;
     text: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    texts: z.ZodOptional<z.ZodArray<z.ZodString>>;
     id: z.ZodOptional<z.ZodString>;
     description: z.ZodOptional<z.ZodString>;
     image: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
@@ -1130,6 +1151,16 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     captionParams: z.ZodOptional<z.ZodObject<{
         lang: z.ZodOptional<z.ZodString>;
         styles: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+        captionSplit: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
+            none: "none";
+            estimate: "estimate";
+        }>>>;
+        textSplit: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            type: z.ZodLiteral<"none">;
+        }, z.core.$strip>, z.ZodObject<{
+            type: z.ZodLiteral<"delimiters">;
+            delimiters: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>], "type">>;
     }, z.core.$strict>>;
     imageNames: z.ZodOptional<z.ZodArray<z.ZodString>>;
     imagePrompt: z.ZodOptional<z.ZodString>;
@@ -1485,6 +1516,16 @@ export declare const mulmoPresentationStyleSchema: z.ZodObject<{
     captionParams: z.ZodOptional<z.ZodObject<{
         lang: z.ZodOptional<z.ZodString>;
         styles: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+        captionSplit: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
+            none: "none";
+            estimate: "estimate";
+        }>>>;
+        textSplit: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            type: z.ZodLiteral<"none">;
+        }, z.core.$strip>, z.ZodObject<{
+            type: z.ZodLiteral<"delimiters">;
+            delimiters: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>], "type">>;
     }, z.core.$strict>>;
     audioParams: z.ZodDefault<z.ZodObject<{
         padding: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
@@ -1836,6 +1877,16 @@ export declare const mulmoScriptSchema: z.ZodObject<{
     captionParams: z.ZodOptional<z.ZodObject<{
         lang: z.ZodOptional<z.ZodString>;
         styles: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+        captionSplit: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
+            none: "none";
+            estimate: "estimate";
+        }>>>;
+        textSplit: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            type: z.ZodLiteral<"none">;
+        }, z.core.$strip>, z.ZodObject<{
+            type: z.ZodLiteral<"delimiters">;
+            delimiters: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>], "type">>;
     }, z.core.$strict>>;
     audioParams: z.ZodDefault<z.ZodObject<{
         padding: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
@@ -1874,6 +1925,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
     beats: z.ZodArray<z.ZodObject<{
         speaker: z.ZodOptional<z.ZodString>;
         text: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+        texts: z.ZodOptional<z.ZodArray<z.ZodString>>;
         id: z.ZodOptional<z.ZodString>;
         description: z.ZodOptional<z.ZodString>;
         image: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
@@ -2257,6 +2309,16 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         captionParams: z.ZodOptional<z.ZodObject<{
             lang: z.ZodOptional<z.ZodString>;
             styles: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+            captionSplit: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
+                none: "none";
+                estimate: "estimate";
+            }>>>;
+            textSplit: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                type: z.ZodLiteral<"none">;
+            }, z.core.$strip>, z.ZodObject<{
+                type: z.ZodLiteral<"delimiters">;
+                delimiters: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            }, z.core.$strip>], "type">>;
         }, z.core.$strict>>;
         imageNames: z.ZodOptional<z.ZodArray<z.ZodString>>;
         imagePrompt: z.ZodOptional<z.ZodString>;
@@ -2289,6 +2351,11 @@ export declare const mulmoStudioBeatSchema: z.ZodObject<{
     soundEffectFile: z.ZodOptional<z.ZodString>;
     lipSyncFile: z.ZodOptional<z.ZodString>;
     captionFile: z.ZodOptional<z.ZodString>;
+    captionFiles: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        file: z.ZodString;
+        startAt: z.ZodNumber;
+        endAt: z.ZodNumber;
+    }, z.core.$strip>>>;
     htmlImageFile: z.ZodOptional<z.ZodString>;
     markdown: z.ZodOptional<z.ZodString>;
     html: z.ZodOptional<z.ZodString>;
@@ -2682,6 +2749,16 @@ export declare const mulmoStudioSchema: z.ZodObject<{
         captionParams: z.ZodOptional<z.ZodObject<{
             lang: z.ZodOptional<z.ZodString>;
             styles: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+            captionSplit: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
+                none: "none";
+                estimate: "estimate";
+            }>>>;
+            textSplit: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                type: z.ZodLiteral<"none">;
+            }, z.core.$strip>, z.ZodObject<{
+                type: z.ZodLiteral<"delimiters">;
+                delimiters: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            }, z.core.$strip>], "type">>;
         }, z.core.$strict>>;
         audioParams: z.ZodDefault<z.ZodObject<{
             padding: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
@@ -2720,6 +2797,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
         beats: z.ZodArray<z.ZodObject<{
             speaker: z.ZodOptional<z.ZodString>;
             text: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+            texts: z.ZodOptional<z.ZodArray<z.ZodString>>;
             id: z.ZodOptional<z.ZodString>;
             description: z.ZodOptional<z.ZodString>;
             image: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
@@ -3103,6 +3181,16 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             captionParams: z.ZodOptional<z.ZodObject<{
                 lang: z.ZodOptional<z.ZodString>;
                 styles: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+                captionSplit: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
+                    none: "none";
+                    estimate: "estimate";
+                }>>>;
+                textSplit: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    type: z.ZodLiteral<"none">;
+                }, z.core.$strip>, z.ZodObject<{
+                    type: z.ZodLiteral<"delimiters">;
+                    delimiters: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                }, z.core.$strip>], "type">>;
             }, z.core.$strict>>;
             imageNames: z.ZodOptional<z.ZodArray<z.ZodString>>;
             imagePrompt: z.ZodOptional<z.ZodString>;
@@ -3136,6 +3224,11 @@ export declare const mulmoStudioSchema: z.ZodObject<{
         soundEffectFile: z.ZodOptional<z.ZodString>;
         lipSyncFile: z.ZodOptional<z.ZodString>;
         captionFile: z.ZodOptional<z.ZodString>;
+        captionFiles: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            file: z.ZodString;
+            startAt: z.ZodNumber;
+            endAt: z.ZodNumber;
+        }, z.core.$strip>>>;
         htmlImageFile: z.ZodOptional<z.ZodString>;
         markdown: z.ZodOptional<z.ZodString>;
         html: z.ZodOptional<z.ZodString>;
@@ -3464,6 +3557,16 @@ export declare const mulmoPromptTemplateSchema: z.ZodObject<{
         captionParams: z.ZodOptional<z.ZodObject<{
             lang: z.ZodOptional<z.ZodString>;
             styles: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+            captionSplit: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
+                none: "none";
+                estimate: "estimate";
+            }>>>;
+            textSplit: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                type: z.ZodLiteral<"none">;
+            }, z.core.$strip>, z.ZodObject<{
+                type: z.ZodLiteral<"delimiters">;
+                delimiters: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            }, z.core.$strip>], "type">>;
         }, z.core.$strict>>;
         audioParams: z.ZodDefault<z.ZodObject<{
             padding: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
@@ -3809,6 +3912,16 @@ export declare const mulmoPromptTemplateFileSchema: z.ZodObject<{
         captionParams: z.ZodOptional<z.ZodObject<{
             lang: z.ZodOptional<z.ZodString>;
             styles: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
+            captionSplit: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
+                none: "none";
+                estimate: "estimate";
+            }>>>;
+            textSplit: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                type: z.ZodLiteral<"none">;
+            }, z.core.$strip>, z.ZodObject<{
+                type: z.ZodLiteral<"delimiters">;
+                delimiters: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            }, z.core.$strip>], "type">>;
         }, z.core.$strict>>;
         audioParams: z.ZodDefault<z.ZodObject<{
             padding: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;

package/lib/types/schema.js

@@ -119,10 +119,20 @@ export const mulmoTextSlideMediaSchema = z
     }),
 })
     .strict();
+export const captionSplitSchema = z.enum(["none", "estimate"]).default("none");
+export const textSplitSchema = z.discriminatedUnion("type", [
+    z.object({ type: z.literal("none") }),
+    z.object({
+        type: z.literal("delimiters"),
+        delimiters: z.array(z.string()).optional(), // default: ["。", "？", "！", ".", "?", "!"]
+    }),
+]);
 export const mulmoCaptionParamsSchema = z
     .object({
     lang: langSchema.optional(),
     styles: z.array(z.string()).optional().default([]), // css styles
+    captionSplit: captionSplitSchema.optional(), // how to determine caption timing
+    textSplit: textSplitSchema.optional(), // how to split text into segments (default: none)
 })
     .strict();
 export const mulmoChartMediaSchema = z
@@ -317,6 +327,7 @@ export const mulmoBeatSchema = z
     .object({
     speaker: speakerIdSchema.optional(),
     text: z.string().optional().default("").describe("Text to be spoken. If empty, the audio is not generated."),
+    texts: z.array(z.string()).optional().describe("Manually split texts for captions. Takes precedence over text for caption display."),
     id: z.string().optional().describe("Unique identifier for the beat."),
     description: z.string().optional(),
     image: mulmoImageAssetSchema.optional(),
@@ -442,7 +453,14 @@ export const mulmoStudioBeatSchema = z
     movieFile: z.string().optional(), // path to the movie file
     soundEffectFile: z.string().optional(), // path to the sound effect file
     lipSyncFile: z.string().optional(), // path to the lip sync file
-    captionFile: z.string().optional(), // path to the caption image
+    captionFile: z.string().optional(), // path to the caption image (deprecated, use captionFiles)
+    captionFiles: z
+        .array(z.object({
+        file: z.string(),
+        startAt: z.number(), // absolute start time in seconds
+        endAt: z.number(), // absolute end time in seconds
+    }))
+        .optional(), // split caption images with timing
     htmlImageFile: z.string().optional(), // path to the html image
     markdown: z.string().optional(), // markdown string (alternative to image)
     html: z.string().optional(), // html string (alternative to image)

package/lib/utils/context.d.ts

@@ -247,6 +247,7 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
         beats: {
             text: string;
             speaker?: string | undefined;
+            texts?: string[] | undefined;
             id?: string | undefined;
             description?: string | undefined;
             image?: {
@@ -555,6 +556,13 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
             captionParams?: {
                 styles: string[];
                 lang?: string | undefined;
+                captionSplit?: "none" | "estimate" | undefined;
+                textSplit?: {
+                    type: "none";
+                } | {
+                    type: "delimiters";
+                    delimiters?: string[] | undefined;
+                } | undefined;
             } | undefined;
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
@@ -583,6 +591,13 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
         captionParams?: {
             styles: string[];
             lang?: string | undefined;
+            captionSplit?: "none" | "estimate" | undefined;
+            textSplit?: {
+                type: "none";
+            } | {
+                type: "delimiters";
+                delimiters?: string[] | undefined;
+            } | undefined;
         } | undefined;
         title?: string | undefined;
         description?: string | undefined;
@@ -611,6 +626,11 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
         soundEffectFile?: string | undefined;
         lipSyncFile?: string | undefined;
         captionFile?: string | undefined;
+        captionFiles?: {
+            file: string;
+            startAt: number;
+            endAt: number;
+        }[] | undefined;
         htmlImageFile?: string | undefined;
         markdown?: string | undefined;
         html?: string | undefined;
@@ -867,6 +887,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             beats: {
                 text: string;
                 speaker?: string | undefined;
+                texts?: string[] | undefined;
                 id?: string | undefined;
                 description?: string | undefined;
                 image?: {
@@ -1175,6 +1196,13 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 captionParams?: {
                     styles: string[];
                     lang?: string | undefined;
+                    captionSplit?: "none" | "estimate" | undefined;
+                    textSplit?: {
+                        type: "none";
+                    } | {
+                        type: "delimiters";
+                        delimiters?: string[] | undefined;
+                    } | undefined;
                 } | undefined;
                 imageNames?: string[] | undefined;
                 imagePrompt?: string | undefined;
@@ -1203,6 +1231,13 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             captionParams?: {
                 styles: string[];
                 lang?: string | undefined;
+                captionSplit?: "none" | "estimate" | undefined;
+                textSplit?: {
+                    type: "none";
+                } | {
+                    type: "delimiters";
+                    delimiters?: string[] | undefined;
+                } | undefined;
             } | undefined;
             title?: string | undefined;
             description?: string | undefined;
@@ -1231,6 +1266,11 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             soundEffectFile?: string | undefined;
             lipSyncFile?: string | undefined;
             captionFile?: string | undefined;
+            captionFiles?: {
+                file: string;
+                startAt: number;
+                endAt: number;
+            }[] | undefined;
             htmlImageFile?: string | undefined;
             markdown?: string | undefined;
             html?: string | undefined;
@@ -1504,6 +1544,13 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
         captionParams?: {
             styles: string[];
             lang?: string | undefined;
+            captionSplit?: "none" | "estimate" | undefined;
+            textSplit?: {
+                type: "none";
+            } | {
+                type: "delimiters";
+                delimiters?: string[] | undefined;
+            } | undefined;
         } | undefined;
     };
     sessionState: {

package/lib/utils/file.d.ts

@@ -34,7 +34,7 @@ export declare const getBeatMoviePaths: (context: MulmoStudioContext, index: num
     lipSyncFile: string;
 };
 export declare const getReferenceImagePath: (context: MulmoStudioContext, key: string, extension: string) => string;
-export declare const getCaptionImagePath: (context: MulmoStudioContext, index: number) => string;
+export declare const getCaptionImagePath: (context: MulmoStudioContext, index: number, subIndex?: number) => string;
 export declare const getOutputPdfFilePath: (outDirPath: string, fileName: string, pdfMode: PDFMode, lang?: string) => string;
 export declare const getPromptTemplateFilePath: (promptTemplateName: string) => string;
 export declare const mkdir: (dirPath: string) => void;

package/lib/utils/file.js

@@ -109,8 +109,11 @@ export const getReferenceImagePath = (context, key, extension) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
     return `${imageProjectDirPath}/${key}.${extension}`;
 };
-export const getCaptionImagePath = (context, index) => {
+export const getCaptionImagePath = (context, index, subIndex) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
+    if (subIndex !== undefined) {
+        return `${imageProjectDirPath}/${index}_caption_${subIndex}.png`;
+    }
     return `${imageProjectDirPath}/${index}_caption.png`;
 };
 // pdf

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.1.20",
+  "version": "2.1.22",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",

package/scripts/test/test.json

@@ -23,7 +23,7 @@
         "displayName": {
           "ja": "アナウンサー"
         },
-        "voiceId": "3708ad43-cace-486c-a4ca-8fe41186e20c",
+        "voiceId": "Aoede",
         "speechOptions": {
           "speed": 1.666
         }
@@ -33,14 +33,14 @@
         "displayName": {
           "ja": "生徒"
         },
-        "voiceId": "a7619e48-bf6a-4f9f-843f-40485651257f"
+        "voiceId": "Puck"
       },
       "Teacher": {
         "provider": "gemini",
         "displayName": {
           "ja": "先生"
         },
-        "voiceId": "bc06c63f-fef6-43b6-92f7-67f919bd5dae"
+        "voiceId": "Charon"
       }
     }
   },

package/scripts/test/test2.json

@@ -17,25 +17,27 @@
     "style": "<style>monochrome"
   },
   "speechParams": {
-    "provider": "gemini",
     "speakers": {
       "Announcer": {
         "displayName": {
           "ja": "千草朋香"
         },
-        "voiceId": "3708ad43-cace-486c-a4ca-8fe41186e20c"
+        "provider": "gemini",
+        "voiceId": "Aoede"
       },
       "Student": {
         "displayName": {
           "ja": "太郎"
         },
-        "voiceId": "a7619e48-bf6a-4f9f-843f-40485651257f"
+        "provider": "gemini",
+        "voiceId": "Puck"
       },
       "Teacher": {
         "displayName": {
           "ja": "山田先生"
         },
-        "voiceId": "bc06c63f-fef6-43b6-92f7-67f919bd5dae"
+        "provider": "gemini",
+        "voiceId": "Charon"
       }
     }
   },

package/scripts/test/test_all_tts.json

@@ -22,9 +22,9 @@
         "provider": "elevenlabs",
         "voiceId": "3JDquces8E8bkmvbh6Bc"
       },
-      "Nijivoice": {
-        "provider": "gemini",
-        "voiceId": "231e0170-0ece-4155-be44-231423062f41"
+      "Kotodama": {
+        "provider": "kotodama",
+        "voiceId": "Poporo"
       }
     }
   },
@@ -41,7 +41,7 @@
     },
     {
       "speaker": "Gemini",
-      "text": "こんにちは、テストです。ジェミニ",
+      "text": "こんにちは、テストです。ジェミニです。",
       "image": {
         "type": "textSlide",
         "slide": {
@@ -70,12 +70,12 @@
       }
     },
     {
-      "speaker": "Nijivoice",
-      "text": "こんにちは、テストです。ニジヴォイス",
+      "speaker": "Kotodama",
+      "text": "こんにちは、テストです。コトダマ",
       "image": {
         "type": "textSlide",
         "slide": {
-          "title": "Nijivoice TTS"
+          "title": "Kotodama TTS"
         }
       }
     }

package/scripts/test/test_captions.json

@@ -5,7 +5,12 @@
   "lang": "en",
   "captionParams": {
     "lang": "en",
-    "styles": ["color: yellow"]
+    "styles": ["color: yellow"],
+    "captionSplit": "estimate",
+    "textSplit": {
+      "type": "delimiters",
+      "delimiters": ["。", "！", "？"]
+    }
   },
   "beats": [
     {

package/scripts/test/test_lang.json

@@ -11,7 +11,7 @@
         "lang": {
           "ja": {
             "provider": "gemini",
-            "voiceId": "9d9ed276-49ee-443a-bc19-26e6136d05f0"
+            "voiceId": "Leda"
           }
         }
       }
@@ -70,7 +70,7 @@
       "image": {
         "type": "textSlide",
         "slide": {
-          "title": "Text replacement test for nijivoice"
+          "title": "Text replacement test for Gemini"
         }
       }
     },

package/scripts/test/test_mixed_providers.json

@@ -29,7 +29,7 @@
         }
       },
       "Host": {
-        "voiceId": "3708ad43-cace-486c-a4ca-8fe41186e20c",
+        "voiceId": "Kore",
         "provider": "gemini",
         "displayName": {
           "en": "Japanese Host"
@@ -79,12 +79,12 @@
     },
     {
       "speaker": "Host",
-      "text": "そして私は、日本語音声合成のためのNijivoiceを使用するホストです。",
+      "text": "そして私は、Gemini TTS です。Google's TTS とは別の方法を利用しています。",
       "image": {
         "type": "textSlide",
         "slide": {
           "title": "Mixed Provider Demo",
-          "subtitle": "Nijivoice Speaker (Japanese)"
+          "subtitle": "Gemini Speaker (Japanese)"
         }
       }
     }