mulmocast 0.0.16 → 0.0.17

package/lib/actions/audio.d.ts

@@ -4,4 +4,4 @@ import { MulmoStudioContext, MulmoBeat } from "../types/index.js";
 export declare const getBeatAudioPath: (text: string, context: MulmoStudioContext, beat: MulmoBeat, lang?: string) => string | undefined;
 export declare const audioFilePath: (context: MulmoStudioContext) => string;
 export declare const generateBeatAudio: (index: number, context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;
-export declare const audio: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;
+export declare const audio: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<MulmoStudioContext>;

package/lib/actions/audio.js

@@ -59,7 +59,7 @@ const preprocessor = (namedInputs) => {
     const text = localizedText(beat, multiLingual, lang);
     const { voiceId, provider, speechOptions } = getAudioParam(presentationStyle, beat);
     const audioPath = getBeatAudioPath(text, context, beat, lang);
-    studioBeat.audioFile = audioPath;
+    studioBeat.audioFile = audioPath; // TODO
     const needsTTS = !beat.audio && audioPath !== undefined;
     return {
         ttsAgent: provider_to_agent[provider],
@@ -120,7 +120,7 @@ const graph_data = {
             inputs: {
                 rows: ":context.studio.script.beats",
                 studioBeat: ":context.studio.beats",
-                multiLingual: ":context.studio.multiLingual",
+                multiLingual: ":context.multiLingual",
                 context: ":context",
             },
             params: {
@@ -213,7 +213,7 @@ export const generateBeatAudio = async (index, context, callbacks) => {
         graph.injectValue("__mapIndex", index);
         graph.injectValue("beat", context.studio.script.beats[index]);
         graph.injectValue("studioBeat", context.studio.beats[index]);
-        graph.injectValue("multiLingual", context.studio.multiLingual);
+        graph.injectValue("multiLingual", context.multiLingual);
         graph.injectValue("context", context);
         if (callbacks) {
             callbacks.forEach((callback) => {
@@ -249,10 +249,13 @@ export const audio = async (context, callbacks) => {
                 graph.registerCallback(callback);
             });
         }
-        await graph.run();
+        const result = await graph.run();
         writingMessage(audioCombinedFilePath);
+        MulmoStudioContextMethods.setSessionState(context, "audio", false);
+        return result.combineFiles;
     }
-    finally {
+    catch (__error) {
         MulmoStudioContextMethods.setSessionState(context, "audio", false);
+        throw __error;
     }
 };

package/lib/actions/captions.d.ts

@@ -1,3 +1,3 @@
 import { MulmoStudioContext } from "../types/index.js";
 import type { CallbackFunction } from "graphai";
-export declare const captions: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;
+export declare const captions: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<MulmoStudioContext>;

package/lib/actions/captions.js

@@ -30,7 +30,7 @@ const graph_data = {
                                 const imagePath = `${imageDirPath}/${context.studio.filename}/${index}_caption.png`;
                                 const template = getHTMLFile("caption");
                                 const text = (() => {
-                                    const multiLingual = context.studio.multiLingual;
+                                    const multiLingual = context.multiLingual;
                                     if (caption && multiLingual) {
                                         return multiLingual[index].multiLingualTexts[caption].text;
                                     }
@@ -63,18 +63,21 @@ const graph_data = {
     },
 };
 export const captions = async (context, callbacks) => {
-    try {
-        MulmoStudioContextMethods.setSessionState(context, "caption", true);
-        const graph = new GraphAI(graph_data, { ...vanillaAgents });
-        graph.injectValue("context", context);
-        if (callbacks) {
-            callbacks.forEach((callback) => {
-                graph.registerCallback(callback);
-            });
+    if (context.caption) {
+        try {
+            MulmoStudioContextMethods.setSessionState(context, "caption", true);
+            const graph = new GraphAI(graph_data, { ...vanillaAgents });
+            graph.injectValue("context", context);
+            if (callbacks) {
+                callbacks.forEach((callback) => {
+                    graph.registerCallback(callback);
+                });
+            }
+            await graph.run();
+        }
+        finally {
+            MulmoStudioContextMethods.setSessionState(context, "caption", false);
         }
-        await graph.run();
-    }
-    finally {
-        MulmoStudioContextMethods.setSessionState(context, "caption", false);
     }
+    return context;
 };

package/lib/actions/images.d.ts

@@ -4,8 +4,6 @@ export declare const imagePreprocessAgent: (namedInputs: {
     context: MulmoStudioContext;
     beat: MulmoBeat;
     index: number;
-    suffix: string;
-    imageDirPath: string;
     imageAgentInfo: Text2ImageAgentInfo;
     imageRefs: Record<string, string>;
 }) => Promise<{
@@ -87,5 +85,6 @@ export declare const imagePreprocessAgent: (namedInputs: {
     referenceImage: string;
     prompt: string;
 }>;
-export declare const images: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;
+export declare const getImageRefs: (context: MulmoStudioContext) => Promise<Record<string, string>>;
+export declare const images: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<MulmoStudioContext>;
 export declare const generateBeatImage: (index: number, context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;

package/lib/actions/images.js

@@ -4,7 +4,7 @@ import { GraphAI, GraphAILogger } from "graphai";
 import { TaskManager } from "graphai/lib/task_manager.js";
 import * as agents from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
+import { getOutputStudioFilePath, getBeatPngImagePath, getBeatMoviePath, getReferenceImagePath, mkdir } from "../utils/file.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
 import { imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, mediaMockAgent } from "../agents/index.js";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
@@ -22,12 +22,12 @@ const htmlStyle = (context, beat) => {
     };
 };
 export const imagePreprocessAgent = async (namedInputs) => {
-    const { context, beat, index, suffix, imageDirPath, imageAgentInfo, imageRefs } = namedInputs;
+    const { context, beat, index, imageAgentInfo, imageRefs } = namedInputs;
     const imageParams = { ...imageAgentInfo.imageParams, ...beat.imageParams };
-    const imagePath = `${imageDirPath}/${context.studio.filename}/${index}${suffix}.png`;
+    const imagePath = getBeatPngImagePath(context, index);
     const returnValue = {
         imageParams,
-        movieFile: beat.moviePrompt ? `${imageDirPath}/${context.studio.filename}/${index}.mov` : undefined,
+        movieFile: beat.moviePrompt ? getBeatMoviePath(context, index) : undefined,
     };
     if (beat.image) {
         const plugin = imagePlugins.find((plugin) => plugin.imageType === beat?.image?.type);
@@ -61,7 +61,6 @@ const beat_graph_data = {
     concurrency: 4,
     nodes: {
         context: {},
-        imageDirPath: {},
         imageAgentInfo: {},
         movieAgentInfo: {},
         imageRefs: {},
@@ -73,8 +72,6 @@ const beat_graph_data = {
                 context: ":context",
                 beat: ":beat",
                 index: ":__mapIndex",
-                suffix: "p",
-                imageDirPath: ":imageDirPath",
                 imageAgentInfo: ":imageAgentInfo",
                 imageRefs: ":imageRefs",
             },
@@ -153,7 +150,6 @@ const graph_data = {
     concurrency: 4,
     nodes: {
         context: {},
-        imageDirPath: {},
         imageAgentInfo: {},
         movieAgentInfo: {},
         outputStudioFilePath: {},
@@ -165,7 +161,6 @@ const graph_data = {
                 context: ":context",
                 imageAgentInfo: ":imageAgentInfo",
                 movieAgentInfo: ":movieAgentInfo",
-                imageDirPath: ":imageDirPath",
                 imageRefs: ":imageRefs",
             },
             isResult: true,
@@ -200,7 +195,10 @@ const graph_data = {
                         }
                     }
                 });
-                return { studio };
+                return {
+                    ...context,
+                    studio,
+                };
             },
             inputs: {
                 array: ":map.output",
@@ -262,11 +260,8 @@ const graphOption = async (context) => {
     }
     return options;
 };
-const prepareGenerateImages = async (context) => {
-    const { studio, fileDirs } = context;
-    const { outDirPath, imageDirPath } = fileDirs;
-    mkdir(`${imageDirPath}/${studio.filename}`);
-    const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle, context.dryRun);
+// TODO: unit test
+export const getImageRefs = async (context) => {
     const imageRefs = {};
     const images = context.presentationStyle.imageParams?.images;
     if (images) {
@@ -299,12 +294,21 @@ const prepareGenerateImages = async (context) => {
                         return "png"; // default
                     }
                 })();
-                const imagePath = `${imageDirPath}/${context.studio.filename}/${key}.${extension}`;
+                const imagePath = getReferenceImagePath(context, key, extension);
                 await fs.promises.writeFile(imagePath, buffer);
                 imageRefs[key] = imagePath;
             }
         }));
     }
+    return imageRefs;
+};
+const prepareGenerateImages = async (context) => {
+    const { studio } = context;
+    const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
+    const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
+    mkdir(imageProjectDirPath);
+    const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle, context.dryRun);
+    const imageRefs = await getImageRefs(context);
     GraphAILogger.info(`text2image: provider=${imageAgentInfo.provider} model=${imageAgentInfo.imageParams.model}`);
     const injections = {
         context,
@@ -313,7 +317,6 @@ const prepareGenerateImages = async (context) => {
             agent: context.dryRun ? "mediaMockAgent" : "movieGoogleAgent",
         },
         outputStudioFilePath: getOutputStudioFilePath(outDirPath, studio.filename),
-        imageDirPath,
         imageRefs,
     };
     return injections;
@@ -346,10 +349,13 @@ const generateImages = async (context, callbacks) => {
 export const images = async (context, callbacks) => {
     try {
         MulmoStudioContextMethods.setSessionState(context, "image", true);
-        await generateImages(context, callbacks);
+        const newContext = await generateImages(context, callbacks);
+        MulmoStudioContextMethods.setSessionState(context, "image", false);
+        return newContext;
     }
-    finally {
+    catch (error) {
         MulmoStudioContextMethods.setSessionState(context, "image", false);
+        throw error;
     }
 };
 export const generateBeatImage = async (index, context, callbacks) => {

package/lib/actions/pdf.js

@@ -95,8 +95,7 @@ const getHandoutTemplateData = (isLandscapeImage) => ({
     item_flex: isLandscapeImage ? "flex: 1;" : "",
 });
 const generatePDFHTML = async (context, pdfMode, pdfSize) => {
-    const { studio, lang = "en" } = context;
-    const { multiLingual } = studio;
+    const { studio, multiLingual, lang = "en" } = context;
     const { width: imageWidth, height: imageHeight } = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
     const isLandscapeImage = imageWidth > imageHeight;
     const imagePaths = studio.beats.map((beat) => beat.imageFile);

package/lib/actions/translate.js

@@ -4,7 +4,7 @@ import * as agents from "@graphai/vanilla";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { recursiveSplitJa, replacementsJa, replacePairsJa } from "../utils/string.js";
-import { getOutputStudioFilePath, mkdir, writingMessage } from "../utils/file.js";
+import { getOutputMultilingualFilePath, mkdir, writingMessage } from "../utils/file.js";
 import { translateSystemPrompt, translatePrompts } from "../utils/prompt.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
 const vanillaAgents = agents.default ?? agents;
@@ -14,7 +14,7 @@ const translateGraph = {
         context: {},
         defaultLang: {},
         outDirPath: {},
-        outputStudioFilePath: {},
+        outputMultilingualFilePath: {},
         lang: {
             agent: "stringUpdateTextAgent",
             inputs: {
@@ -27,7 +27,7 @@ const translateGraph = {
             isResult: true,
             agent: "mergeObjectAgent",
             inputs: {
-                items: [":context.studio", { multiLingual: ":beatsMap.mergeMultiLingualData" }],
+                items: [{ multiLingual: ":beatsMap.mergeMultiLingualData" }],
             },
         },
         beatsMap: {
@@ -52,7 +52,7 @@ const translateGraph = {
                         },
                         inputs: {
                             index: ":__mapIndex",
-                            rows: ":context.studio.multiLingual",
+                            rows: ":context.multiLingual",
                         },
                     },
                     preprocessMultiLingual: {
@@ -167,8 +167,8 @@ const translateGraph = {
             // console: { before: true },
             agent: "fileWriteAgent",
             inputs: {
-                file: ":outputStudioFilePath",
-                text: ":mergeStudioResult.toJSON()",
+                file: ":outputMultilingualFilePath",
+                text: ":mergeStudioResult.multiLingual.toJSON()",
             },
         },
     },
@@ -213,7 +213,7 @@ export const translate = async (context, callbacks) => {
         MulmoStudioContextMethods.setSessionState(context, "multiLingual", true);
         const { studio, fileDirs } = context;
         const { outDirPath } = fileDirs;
-        const outputStudioFilePath = getOutputStudioFilePath(outDirPath, studio.filename);
+        const outputMultilingualFilePath = getOutputMultilingualFilePath(outDirPath, studio.filename);
         mkdir(outDirPath);
         assert(!!process.env.OPENAI_API_KEY, "The OPENAI_API_KEY environment variable is missing or empty");
         const graph = new GraphAI(translateGraph, { ...vanillaAgents, fileWriteAgent, openAIAgent }, { agentFilters });
@@ -221,16 +221,16 @@ export const translate = async (context, callbacks) => {
         graph.injectValue("defaultLang", defaultLang);
         graph.injectValue("targetLangs", targetLangs);
         graph.injectValue("outDirPath", outDirPath);
-        graph.injectValue("outputStudioFilePath", outputStudioFilePath);
+        graph.injectValue("outputMultilingualFilePath", outputMultilingualFilePath);
         if (callbacks) {
             callbacks.forEach((callback) => {
                 graph.registerCallback(callback);
             });
         }
         const results = await graph.run();
-        writingMessage(outputStudioFilePath);
+        writingMessage(outputMultilingualFilePath);
         if (results.mergeStudioResult) {
-            context.studio = results.mergeStudioResult;
+            context.multiLingual = results.mergeStudioResult.multiLingual;
         }
     }
     finally {

package/lib/agents/combine_audio_files_agent.js

@@ -1,6 +1,35 @@
-import { GraphAILogger } from "graphai";
+import { assert } from "graphai";
 import { silent60secPath } from "../utils/file.js";
 import { FfmpegContextInit, FfmpegContextGenerateOutput, FfmpegContextInputFormattedAudio, ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
+const getMovieDulation = async (beat) => {
+    if (beat.image?.type === "movie" && (beat.image.source.kind === "url" || beat.image.source.kind === "path")) {
+        const pathOrUrl = beat.image.source.kind === "url" ? beat.image.source.url : beat.image.source.path;
+        return await ffmpegGetMediaDuration(pathOrUrl);
+    }
+    return 0;
+};
+const getPadding = (context, beat, index) => {
+    if (beat.audioParams?.padding !== undefined) {
+        return beat.audioParams.padding;
+    }
+    if (index === context.studio.beats.length - 1) {
+        return 0;
+    }
+    const isClosingGap = index === context.studio.beats.length - 2;
+    return isClosingGap ? context.presentationStyle.audioParams.closingPadding : context.presentationStyle.audioParams.padding;
+};
+const getTotalPadding = (padding, movieDuration, audioDuration, duration, canSpillover = false) => {
+    if (movieDuration > 0) {
+        return padding + (movieDuration - audioDuration);
+    }
+    else if (duration && duration > audioDuration) {
+        return padding + (duration - audioDuration);
+    }
+    else if (canSpillover && duration && audioDuration > duration) {
+        return duration - audioDuration; // negative value to indicate that there is a spill over.
+    }
+    return padding;
+};
 const combineAudioFilesAgent = async ({ namedInputs, }) => {
     const { context, combinedFileName } = namedInputs;
     const ffmpegContext = FfmpegContextInit();
@@ -8,64 +37,81 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
     // We cannot reuse longSilentId. We need to explicitly split it for each beat.
     const silentIds = context.studio.beats.map((_, index) => `[ls_${index}]`);
     ffmpegContext.filterComplex.push(`${longSilentId}asplit=${silentIds.length}${silentIds.join("")}`);
-    const inputIds = (await Promise.all(context.studio.beats.map(async (studioBeat, index) => {
+    // First, get the audio durations of all beats, taking advantage of multi-threading capability of ffmpeg.
+    const mediaDurations = await Promise.all(context.studio.beats.map(async (studioBeat, index) => {
         const beat = context.studio.script.beats[index];
-        const isClosingGap = index === context.studio.beats.length - 2;
-        const movieDuration = await (() => {
-            if (beat.image?.type === "movie" && (beat.image.source.kind === "url" || beat.image.source.kind === "path")) {
-                const pathOrUrl = beat.image.source.kind === "url" ? beat.image.source.url : beat.image.source.path;
-                return ffmpegGetMediaDuration(pathOrUrl);
-            }
-            return 0;
-        })();
+        const movieDuration = await getMovieDulation(beat);
+        const audioDuration = studioBeat.audioFile ? await ffmpegGetMediaDuration(studioBeat.audioFile) : 0;
+        return {
+            movieDuration,
+            audioDuration,
+        };
+    }));
+    const inputIds = [];
+    const beatDurations = [];
+    context.studio.beats.reduce((spillover, studioBeat, index) => {
+        const beat = context.studio.script.beats[index];
+        const { audioDuration, movieDuration } = mediaDurations[index];
+        const paddingId = `[padding_${index}]`;
+        const canSpillover = index < context.studio.beats.length - 1 && mediaDurations[index + 1].movieDuration + mediaDurations[index + 1].audioDuration === 0;
         if (studioBeat.audioFile) {
             const audioId = FfmpegContextInputFormattedAudio(ffmpegContext, studioBeat.audioFile);
-            const padding = (() => {
-                if (beat.audioParams?.padding !== undefined) {
-                    return beat.audioParams.padding;
-                }
-                if (index === context.studio.beats.length - 1) {
-                    return 0;
-                }
-                return isClosingGap ? context.presentationStyle.audioParams.closingPadding : context.presentationStyle.audioParams.padding;
-            })();
-            const audioDuration = await ffmpegGetMediaDuration(studioBeat.audioFile);
-            const totalPadding = await (async () => {
-                if (movieDuration > 0) {
-                    return padding + (movieDuration - audioDuration);
-                }
-                else if (beat.duration && beat.duration > audioDuration) {
-                    return padding + (beat.duration - audioDuration);
-                }
-                return padding;
-            })();
-            studioBeat.duration = audioDuration + totalPadding;
+            // padding is the amount of audio padding specified in the script.
+            const padding = getPadding(context, beat, index);
+            // totalPadding is the amount of audio padding to be added to the audio file.
+            const totalPadding = getTotalPadding(padding, movieDuration, audioDuration, beat.duration, canSpillover);
+            beatDurations.push(audioDuration + totalPadding);
             if (totalPadding > 0) {
                 const silentId = silentIds.pop();
-                ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${totalPadding}[padding_${index}]`);
-                return [audioId, `[padding_${index}]`];
+                ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${totalPadding}${paddingId}`);
+                inputIds.push(audioId, paddingId);
             }
             else {
-                return [audioId];
+                inputIds.push(audioId);
+                if (totalPadding < 0) {
+                    return -totalPadding;
+                }
             }
         }
         else {
             // NOTE: We come here when the text is empty and no audio property is specified.
-            studioBeat.duration = beat.duration ?? (movieDuration > 0 ? movieDuration : 1.0);
+            const beatDuration = (() => {
+                const duration = beat.duration ?? (movieDuration > 0 ? movieDuration : 1.0);
+                if (!canSpillover && duration < spillover) {
+                    return spillover; // We need to consume the spillover here.
+                }
+                return duration;
+            })();
+            beatDurations.push(beatDuration);
+            if (beatDuration <= spillover) {
+                return spillover - beatDuration;
+            }
             const silentId = silentIds.pop();
-            ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${studioBeat.duration}[silent_${index}]`);
-            return [`[silent_${index}]`];
+            ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${beatDuration - spillover}${paddingId}`);
+            inputIds.push(paddingId);
         }
-    }))).flat();
-    silentIds.forEach((silentId) => {
-        GraphAILogger.log(`Using extra silentId: ${silentId}`);
-        ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${0.01}[silent_extra]`);
-        inputIds.push("[silent_extra]");
+        return 0;
+    }, 0);
+    assert(beatDurations.length === context.studio.beats.length, "beatDurations.length !== studio.beats.length");
+    // We need to "consume" extra silentIds.
+    silentIds.forEach((silentId, index) => {
+        const extraId = `[silent_extra_${index}]`;
+        ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${0.01}${extraId}`);
+        inputIds.push(extraId);
     });
+    // Finally, combine all audio files.
     ffmpegContext.filterComplex.push(`${inputIds.join("")}concat=n=${inputIds.length}:v=0:a=1[aout]`);
     await FfmpegContextGenerateOutput(ffmpegContext, combinedFileName, ["-map", "[aout]"]);
+    const result = {
+        studio: {
+            ...context.studio,
+            beats: context.studio.beats.map((studioBeat, index) => ({ ...studioBeat, duration: beatDurations[index] })),
+        },
+    };
+    // context.studio = result.studio; // TODO: removing this breaks test/test_movie.ts
     return {
-        studio: context.studio,
+        ...context,
+        ...result,
     };
 };
 const combineAudioFilesAgentInfo = {

package/lib/cli/commands/movie/handler.js

@@ -6,10 +6,5 @@ export const handler = async (argv) => {
         process.exit(1);
     }
     await runTranslateIfNeeded(context, argv);
-    await audio(context);
-    await images(context);
-    if (context.caption) {
-        await captions(context);
-    }
-    await movie(context);
+    await audio(context).then(images).then(captions).then(movie);
 };

package/lib/cli/helpers.d.ts

@@ -1,4 +1,4 @@
-import type { MulmoScript, MulmoStudioContext, MulmoPresentationStyle } from "../types/type.js";
+import type { MulmoScript, MulmoStudioContext, MulmoPresentationStyle, MulmoStudioMultiLingual } from "../types/type.js";
 import type { CliArgs } from "../types/cli_types.js";
 export declare const setGraphAILogger: (verbose: boolean | undefined, logValues?: Record<string, unknown>) => void;
 export interface FileObject {
@@ -11,6 +11,7 @@ export interface FileObject {
     isHttpPath: boolean;
     fileOrUrl: string;
     outputStudioFilePath: string;
+    outputMultilingualFilePath: string;
     presentationStylePath: string | undefined;
     fileName: string;
 }
@@ -23,6 +24,7 @@ export declare const getFileObject: (args: {
     file: string;
 }) => FileObject;
 export declare const fetchScript: (isHttpPath: boolean, mulmoFilePath: string, fileOrUrl: string) => Promise<MulmoScript | null>;
+export declare const getMultiLingual: (multilingualFilePath: string, beatsLength: number) => MulmoStudioMultiLingual;
 export declare const getPresentationStyle: (presentationStylePath: string | undefined) => MulmoPresentationStyle | null;
 type InitOptions = {
     b?: string;

package/lib/cli/helpers.js

@@ -2,12 +2,12 @@ import { GraphAILogger } from "graphai";
 import fs from "fs";
 import path from "path";
 import clipboardy from "clipboardy";
-import { getBaseDirPath, getFullPath, readMulmoScriptFile, fetchMulmoScriptFile, getOutputStudioFilePath, resolveDirPath, mkdir } from "../utils/file.js";
+import { getBaseDirPath, getFullPath, readMulmoScriptFile, fetchMulmoScriptFile, getOutputStudioFilePath, resolveDirPath, mkdir, getOutputMultilingualFilePath, } from "../utils/file.js";
 import { isHttp } from "../utils/utils.js";
 import { createOrUpdateStudioData } from "../utils/preprocess.js";
 import { outDirName, imageDirName, audioDirName } from "../utils/const.js";
 import { translate } from "../actions/translate.js";
-import { mulmoPresentationStyleSchema } from "../types/schema.js";
+import { mulmoPresentationStyleSchema, mulmoStudioMultiLingualSchema } from "../types/schema.js";
 export const setGraphAILogger = (verbose, logValues) => {
     if (verbose) {
         if (logValues) {
@@ -48,6 +48,7 @@ export const getFileObject = (args) => {
     const imageDirPath = getFullPath(outDirPath, imagedir ?? imageDirName);
     const audioDirPath = getFullPath(outDirPath, audiodir ?? audioDirName);
     const outputStudioFilePath = getOutputStudioFilePath(outDirPath, fileName);
+    const outputMultilingualFilePath = getOutputMultilingualFilePath(outDirPath, fileName);
     const presentationStylePath = presentationStyle ? getFullPath(baseDirPath, presentationStyle) : undefined;
     return {
         baseDirPath,
@@ -59,6 +60,7 @@ export const getFileObject = (args) => {
         isHttpPath,
         fileOrUrl,
         outputStudioFilePath,
+        outputMultilingualFilePath,
         presentationStylePath,
         fileName,
     };
@@ -78,6 +80,13 @@ export const fetchScript = async (isHttpPath, mulmoFilePath, fileOrUrl) => {
     }
     return readMulmoScriptFile(mulmoFilePath, "ERROR: File does not exist " + mulmoFilePath)?.mulmoData ?? null;
 };
+export const getMultiLingual = (multilingualFilePath, beatsLength) => {
+    if (fs.existsSync(multilingualFilePath)) {
+        const jsonData = readMulmoScriptFile(multilingualFilePath, "ERROR: File does not exist " + multilingualFilePath)?.mulmoData ?? null;
+        return mulmoStudioMultiLingualSchema.parse(jsonData);
+    }
+    return [...Array(beatsLength)].map(() => ({ multiLingualTexts: {} }));
+};
 export const getPresentationStyle = (presentationStylePath) => {
     if (presentationStylePath) {
         if (!fs.existsSync(presentationStylePath)) {
@@ -97,7 +106,7 @@ export const initializeContext = async (argv) => {
         presentationStyle: argv.p,
         file: argv.file ?? "",
     });
-    const { fileName, isHttpPath, fileOrUrl, mulmoFilePath, outputStudioFilePath, presentationStylePath } = files;
+    const { fileName, isHttpPath, fileOrUrl, mulmoFilePath, outputStudioFilePath, presentationStylePath, outputMultilingualFilePath } = files;
     setGraphAILogger(argv.v, {
         files,
     });
@@ -106,6 +115,7 @@ export const initializeContext = async (argv) => {
         return null;
     }
     const presentationStyle = getPresentationStyle(presentationStylePath);
+    const multiLingual = getMultiLingual(outputMultilingualFilePath, mulmoScript.beats.length);
     // Create or update MulmoStudio file with MulmoScript
     const currentStudio = readMulmoScriptFile(outputStudioFilePath);
     try {
@@ -136,6 +146,7 @@ export const initializeContext = async (argv) => {
                 },
             },
             presentationStyle: presentationStyle ?? studio.script,
+            multiLingual,
         };
     }
     catch (error) {

package/lib/methods/mulmo_studio_context.d.ts

@@ -17,6 +17,9 @@ export declare const removeSessionProgressCallback: (cb: SessionProgressCallback
 export declare const MulmoStudioContextMethods: {
     resolveAssetPath(context: MulmoStudioContext, relativePath: string): string;
     getAudioDirPath(context: MulmoStudioContext): string;
+    getImageDirPath(context: MulmoStudioContext): string;
+    getImageProjectDirPath(context: MulmoStudioContext): string;
+    getOutDirPath(context: MulmoStudioContext): string;
     setSessionState(context: MulmoStudioContext, sessionType: SessionType, value: boolean): void;
     setBeatSessionState(context: MulmoStudioContext, sessionType: BeatSessionType, index: number, value: boolean): void;
 };

package/lib/methods/mulmo_studio_context.js

@@ -30,6 +30,16 @@ export const MulmoStudioContextMethods = {
     getAudioDirPath(context) {
         return context.fileDirs.audioDirPath;
     },
+    getImageDirPath(context) {
+        return context.fileDirs.imageDirPath;
+    },
+    getImageProjectDirPath(context) {
+        const imageDirPath = MulmoStudioContextMethods.getImageDirPath(context);
+        return `${imageDirPath}/${context.studio.filename}`;
+    },
+    getOutDirPath(context) {
+        return context.fileDirs.outDirPath;
+    },
     setSessionState(context, sessionType, value) {
         context.sessionState.inSession[sessionType] = value;
         notifyStateChange(context, sessionType);

package/lib/types/index.d.ts

@@ -1,3 +1,2 @@
 export * from "./type.js";
 export * from "./schema.js";
-export * from "./cli_types.js";

package/lib/types/index.js

@@ -1,3 +1,2 @@
 export * from "./type.js";
 export * from "./schema.js";
-export * from "./cli_types.js";

package/lib/types/schema.d.ts

@@ -879,6 +879,76 @@ export declare const mulmoAudioAssetSchema: z.ZodUnion<[z.ZodObject<{
     type: "midi";
     source: string;
 }>]>;
+export declare const mulmoImageParamsImagesSchema: z.ZodRecord<z.ZodString, z.ZodObject<{
+    type: z.ZodLiteral<"image">;
+    source: z.ZodDiscriminatedUnion<"kind", [z.ZodObject<{
+        kind: z.ZodLiteral<"url">;
+        url: z.ZodString;
+    }, "strict", z.ZodTypeAny, {
+        url: string;
+        kind: "url";
+    }, {
+        url: string;
+        kind: "url";
+    }>, z.ZodObject<{
+        kind: z.ZodLiteral<"base64">;
+        data: z.ZodString;
+    }, "strict", z.ZodTypeAny, {
+        kind: "base64";
+        data: string;
+    }, {
+        kind: "base64";
+        data: string;
+    }>, z.ZodObject<{
+        kind: z.ZodLiteral<"text">;
+        text: z.ZodString;
+    }, "strict", z.ZodTypeAny, {
+        text: string;
+        kind: "text";
+    }, {
+        text: string;
+        kind: "text";
+    }>, z.ZodObject<{
+        kind: z.ZodLiteral<"path">;
+        path: z.ZodString;
+    }, "strict", z.ZodTypeAny, {
+        path: string;
+        kind: "path";
+    }, {
+        path: string;
+        kind: "path";
+    }>]>;
+}, "strict", z.ZodTypeAny, {
+    type: "image";
+    source: {
+        url: string;
+        kind: "url";
+    } | {
+        kind: "base64";
+        data: string;
+    } | {
+        text: string;
+        kind: "text";
+    } | {
+        path: string;
+        kind: "path";
+    };
+}, {
+    type: "image";
+    source: {
+        url: string;
+        kind: "url";
+    } | {
+        kind: "base64";
+        data: string;
+    } | {
+        text: string;
+        kind: "text";
+    } | {
+        path: string;
+        kind: "path";
+    };
+}>>;
 export declare const mulmoImageParamsSchema: z.ZodObject<{
     model: z.ZodOptional<z.ZodString>;
     style: z.ZodOptional<z.ZodString>;
@@ -6470,43 +6540,6 @@ export declare const mulmoStudioSchema: z.ZodObject<{
         movieFile?: string | undefined;
         captionFile?: string | undefined;
     }>, "many">;
-    multiLingual: z.ZodArray<z.ZodObject<{
-        multiLingualTexts: z.ZodRecord<z.ZodString, z.ZodObject<{
-            text: z.ZodString;
-            lang: z.ZodString;
-            texts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
-            ttsTexts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
-            duration: z.ZodOptional<z.ZodNumber>;
-        }, "strict", z.ZodTypeAny, {
-            text: string;
-            lang: string;
-            texts?: string[] | undefined;
-            ttsTexts?: string[] | undefined;
-            duration?: number | undefined;
-        }, {
-            text: string;
-            lang: string;
-            texts?: string[] | undefined;
-            ttsTexts?: string[] | undefined;
-            duration?: number | undefined;
-        }>>;
-    }, "strip", z.ZodTypeAny, {
-        multiLingualTexts: Record<string, {
-            text: string;
-            lang: string;
-            texts?: string[] | undefined;
-            ttsTexts?: string[] | undefined;
-            duration?: number | undefined;
-        }>;
-    }, {
-        multiLingualTexts: Record<string, {
-            text: string;
-            lang: string;
-            texts?: string[] | undefined;
-            ttsTexts?: string[] | undefined;
-            duration?: number | undefined;
-        }>;
-    }>, "many">;
 }, "strict", z.ZodTypeAny, {
     beats: {
         duration?: number | undefined;
@@ -6516,15 +6549,6 @@ export declare const mulmoStudioSchema: z.ZodObject<{
         movieFile?: string | undefined;
         captionFile?: string | undefined;
     }[];
-    multiLingual: {
-        multiLingualTexts: Record<string, {
-            text: string;
-            lang: string;
-            texts?: string[] | undefined;
-            ttsTexts?: string[] | undefined;
-            duration?: number | undefined;
-        }>;
-    }[];
     script: {
         audioParams: {
             padding: number;
@@ -6782,15 +6806,6 @@ export declare const mulmoStudioSchema: z.ZodObject<{
         movieFile?: string | undefined;
         captionFile?: string | undefined;
     }[];
-    multiLingual: {
-        multiLingualTexts: Record<string, {
-            text: string;
-            lang: string;
-            texts?: string[] | undefined;
-            ttsTexts?: string[] | undefined;
-            duration?: number | undefined;
-        }>;
-    }[];
     script: {
         $mulmocast: {
             version: "1.0";

package/lib/types/schema.js

@@ -140,12 +140,13 @@ const mulmoMidiMediaSchema = z
     .strict();
 export const mulmoAudioAssetSchema = z.union([mulmoAudioMediaSchema, mulmoMidiMediaSchema]);
 const imageIdSchema = z.string();
+export const mulmoImageParamsImagesSchema = z.record(imageIdSchema, mulmoImageMediaSchema);
 export const mulmoImageParamsSchema = z
     .object({
     model: z.string().optional(), // default: provider specific
     style: z.string().optional(), // optional image style
     moderation: z.string().optional(), // optional image style
-    images: z.record(imageIdSchema, mulmoImageMediaSchema).optional(),
+    images: mulmoImageParamsImagesSchema.optional(),
 })
     .strict();
 export const textSlideParamsSchema = z
@@ -301,7 +302,6 @@ export const mulmoStudioSchema = z
     script: mulmoScriptSchema,
     filename: z.string(),
     beats: z.array(mulmoStudioBeatSchema).min(1),
-    multiLingual: mulmoStudioMultiLingualSchema,
 })
     .strict();
 export const mulmoScriptTemplateSchema = z

package/lib/types/type.d.ts

@@ -1,4 +1,4 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
 import { LLM } from "../utils/utils.js";
 import { z } from "zod";
@@ -9,6 +9,7 @@ export type MulmoSpeechParams = z.infer<typeof mulmoSpeechParamsSchema>;
 export type SpeechOptions = z.infer<typeof speechOptionsSchema>;
 export type SpeakerData = z.infer<typeof speakerDataSchema>;
 export type MulmoImageParams = z.infer<typeof mulmoImageParamsSchema>;
+export type MulmoImageParamsImages = z.infer<typeof mulmoImageParamsImagesSchema>;
 export type TextSlideParams = z.infer<typeof textSlideParamsSchema>;
 export type Text2ImageProvider = z.infer<typeof text2ImageProviderSchema>;
 export type Text2MovieProvider = z.infer<typeof text2MovieProviderSchema>;
@@ -51,6 +52,7 @@ export type MulmoStudioContext = {
     caption?: string;
     sessionState: MulmoSessionState;
     presentationStyle: MulmoPresentationStyle;
+    multiLingual: MulmoStudioMultiLingual;
 };
 export type ScriptingParams = {
     urls: string[];

package/lib/utils/file.d.ts

@@ -1,4 +1,4 @@
-import { MulmoScript, MulmoScriptTemplateFile } from "../types/index.js";
+import type { MulmoScript, MulmoScriptTemplateFile, MulmoStudioContext } from "../types/index.js";
 import { PDFMode } from "../types/index.js";
 import { ZodSchema } from "zod";
 export declare const updateNpmRoot: (_npmRoot: string) => void;
@@ -18,10 +18,15 @@ export declare const fetchMulmoScriptFile: (url: string) => Promise<{
     status: string | number;
 }>;
 export declare const getOutputStudioFilePath: (outDirPath: string, fileName: string) => string;
+export declare const getOutputMultilingualFilePath: (outDirPath: string, fileName: string) => string;
 export declare const resolveDirPath: (dirPath: string, studioFileName: string) => string;
 export declare const getAudioFilePath: (audioDirPath: string, dirName: string, fileName: string, lang?: string) => string;
 export declare const getAudioArtifactFilePath: (outDirPath: string, fileName: string) => string;
 export declare const getOutputVideoFilePath: (outDirPath: string, fileName: string, lang?: string, caption?: string) => string;
+export declare const imageSuffix = "p";
+export declare const getBeatPngImagePath: (context: MulmoStudioContext, index: number) => string;
+export declare const getBeatMoviePath: (context: MulmoStudioContext, index: number) => string;
+export declare const getReferenceImagePath: (context: MulmoStudioContext, key: string, extension: string) => string;
 export declare const getOutputPdfFilePath: (outDirPath: string, fileName: string, pdfMode: PDFMode, lang?: string) => string;
 export declare const getTemplateFilePath: (templateName: string) => string;
 export declare const mkdir: (dirPath: string) => void;

package/lib/utils/file.js

@@ -3,7 +3,7 @@ import path from "path";
 import { parse as yamlParse } from "yaml";
 import { fileURLToPath } from "url";
 import { GraphAILogger } from "graphai";
-import { MulmoScriptTemplateMethods } from "../methods/mulmo_script_template.js";
+import { MulmoScriptTemplateMethods, MulmoStudioContextMethods } from "../methods/index.js";
 import { mulmoScriptTemplateSchema } from "../types/schema.js";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -56,9 +56,13 @@ export const fetchMulmoScriptFile = async (url) => {
 export const getOutputStudioFilePath = (outDirPath, fileName) => {
     return path.resolve(outDirPath, fileName + "_studio.json");
 };
+export const getOutputMultilingualFilePath = (outDirPath, fileName) => {
+    return path.resolve(outDirPath, fileName + "_lang.json");
+};
 export const resolveDirPath = (dirPath, studioFileName) => {
     return path.resolve(dirPath, studioFileName);
 };
+// audio
 export const getAudioFilePath = (audioDirPath, dirName, fileName, lang) => {
     if (lang) {
         return path.resolve(audioDirPath, dirName, `${fileName}_${lang}.mp3`);
@@ -73,6 +77,21 @@ export const getOutputVideoFilePath = (outDirPath, fileName, lang, caption) => {
     const suffix2 = caption ? `__${caption}` : "";
     return path.resolve(outDirPath, `${fileName}${suffix}${suffix2}.mp4`);
 };
+// image
+export const imageSuffix = "p";
+export const getBeatPngImagePath = (context, index) => {
+    const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
+    return `${imageProjectDirPath}/${index}${imageSuffix}.png`;
+};
+export const getBeatMoviePath = (context, index) => {
+    const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
+    return `${imageProjectDirPath}/${index}.mov`;
+};
+export const getReferenceImagePath = (context, key, extension) => {
+    const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
+    return `${imageProjectDirPath}/${key}.${extension}`;
+};
+// pdf
 export const getOutputPdfFilePath = (outDirPath, fileName, pdfMode, lang) => {
     if (lang) {
         return path.resolve(outDirPath, `${fileName}_${pdfMode}_${lang}.pdf`);

package/lib/utils/preprocess.d.ts

@@ -8,15 +8,6 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         movieFile?: string | undefined;
         captionFile?: string | undefined;
     }[];
-    multiLingual: {
-        multiLingualTexts: Record<string, {
-            text: string;
-            lang: string;
-            texts?: string[] | undefined;
-            ttsTexts?: string[] | undefined;
-            duration?: number | undefined;
-        }>;
-    }[];
     script: {
         audioParams: {
             padding: number;

package/lib/utils/preprocess.js

@@ -1,7 +1,8 @@
 import { GraphAILogger } from "graphai";
-import { mulmoScriptSchema, mulmoBeatSchema, mulmoStudioSchema } from "../types/index.js";
+import { mulmoScriptSchema, mulmoStudioSchema } from "../types/index.js";
 const rebuildStudio = (currentStudio, mulmoScript, fileName) => {
-    const parsed = mulmoStudioSchema.safeParse(currentStudio);
+    const isTest = process.env.NODE_ENV === "test";
+    const parsed = isTest && currentStudio ? { data: mulmoStudioSchema.parse(currentStudio), success: true, error: null } : mulmoStudioSchema.safeParse(currentStudio);
     if (parsed.success) {
         return parsed.data;
     }
@@ -13,7 +14,6 @@ const rebuildStudio = (currentStudio, mulmoScript, fileName) => {
         script: mulmoScript,
         filename: fileName,
         beats: [...Array(mulmoScript.beats.length)].map(() => ({})),
-        multiLingual: [...Array(mulmoScript.beats.length)].map(() => ({ multiLingualTexts: {} })),
     });
 };
 const mulmoCredit = (speaker) => {
@@ -46,12 +46,5 @@ export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName)
     }
     studio.script = mulmoScriptSchema.parse(mulmoScript); // update the script
     studio.beats = studio.script.beats.map((_, index) => studio.beats[index] ?? {});
-    mulmoScript.beats.forEach((beat, index) => {
-        // Filling the default values
-        studio.script.beats[index] = mulmoBeatSchema.parse(beat);
-        if (!studio.multiLingual[index]) {
-            studio.multiLingual[index] = { multiLingualTexts: {} };
-        }
-    });
     return studio;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.0.16",
+  "version": "0.0.17",
   "description": "",
   "type": "module",
   "main": "lib/index.js",
@@ -27,7 +27,7 @@
     "preprocess": "npx tsx ./src/cli/bin.ts preprocess",
     "pdf": "npx tsx ./src/cli/bin.ts pdf",
     "test": "rm -f scratchpad/test*.* && npx tsx ./src/audio.ts scripts/test/test.json && npx tsx ./src/images.ts scripts/test/test.json && npx tsx ./src/movie.ts scripts/test/test.json",
-    "ci_test": "tsx --test ./test/*/test_*.ts",
+    "ci_test": "NODE_ENV=test tsx --test ./test/*/test_*.ts",
     "lint": "eslint src test",
     "build": "tsc",
     "build_test": "tsc && git checkout -- lib/*",
@@ -50,7 +50,7 @@
   "dependencies": {
     "@google-cloud/text-to-speech": "^6.1.0",
     "@graphai/anthropic_agent": "^2.0.2",
-    "@graphai/browserless_agent": "^2.0.0",
+    "@graphai/browserless_agent": "^2.0.1",
     "@graphai/gemini_agent": "^2.0.0",
     "@graphai/groq_agent": "^2.0.0",
     "@graphai/input_agents": "^1.0.1",
@@ -68,10 +68,10 @@
     "inquirer": "^12.6.1",
     "marked": "^15.0.12",
     "ora": "^8.2.0",
-    "puppeteer": "^24.10.0",
+    "puppeteer": "^24.10.2",
     "yaml": "^2.8.0",
     "yargs": "^17.7.2",
-    "zod": "^3.25.51",
+    "zod": "^3.25.67",
     "zod-to-json-schema": "^3.24.5"
   },
   "devDependencies": {
@@ -82,14 +82,14 @@
     "@types/yargs": "^17.0.33",
     "eslint": "^9.29.0",
     "eslint-config-prettier": "^10.1.5",
-    "eslint-plugin-prettier": "^5.4.1",
+    "eslint-plugin-prettier": "^5.5.0",
     "prettier": "^3.3.3",
     "ts-node": "^10.9.2",
     "tsx": "^4.20.3",
     "typescript": "^5.7.3",
-    "typescript-eslint": "^8.34.0"
+    "typescript-eslint": "^8.34.1"
   },
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=18.0.0"
   }
 }