mulmocast 1.1.4 → 1.1.6

package/lib/actions/audio.d.ts

@@ -2,6 +2,5 @@ import "dotenv/config";
 import type { CallbackFunction } from "graphai";
 import { MulmoStudioContext, MulmoBeat } from "../types/index.js";
 export declare const getBeatAudioPath: (text: string, context: MulmoStudioContext, beat: MulmoBeat, lang?: string) => string | undefined;
-export declare const audioFilePath: (context: MulmoStudioContext) => string;
 export declare const generateBeatAudio: (index: number, context: MulmoStudioContext, settings?: Record<string, string>, callbacks?: CallbackFunction[]) => Promise<void>;
 export declare const audio: (context: MulmoStudioContext, settings?: Record<string, string>, callbacks?: CallbackFunction[]) => Promise<MulmoStudioContext>;

package/lib/actions/audio.js

@@ -9,8 +9,8 @@ import ttsGoogleAgent from "../agents/tts_google_agent.js";
 import ttsElevenlabsAgent from "../agents/tts_elevenlabs_agent.js";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { MulmoPresentationStyleMethods } from "../methods/index.js";
-import { text2SpeechProviderSchema, } from "../types/index.js";
-import { fileCacheAgentFilter } from "../utils/filters.js";
+import { text2SpeechProviderSchema } from "../types/index.js";
+import { fileCacheAgentFilter, nijovoiceTextAgentFilter } from "../utils/filters.js";
 import { getAudioArtifactFilePath, getAudioFilePath, getOutputStudioFilePath, resolveDirPath, defaultBGMPath, mkdir, writingMessage } from "../utils/file.js";
 import { text2hash, localizedText, settings2GraphAIConfig } from "../utils/utils.js";
 import { provider2TTSAgent } from "../utils/provider2agent.js";
@@ -30,15 +30,15 @@ const getAudioPath = (context, beat, audioFile) => {
     }
     return audioFile;
 };
-const getAudioParam = (presentationStyle, beat) => {
-    const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
+const getAudioParam = (context, beat) => {
+    const speaker = MulmoPresentationStyleMethods.getSpeaker(context, beat);
     const speechOptions = { ...speaker.speechOptions, ...beat.speechOptions };
     const provider = text2SpeechProviderSchema.parse(speaker.provider);
     return { voiceId: speaker.voiceId, provider, speechOptions, model: speaker.model };
 };
 export const getBeatAudioPath = (text, context, beat, lang) => {
     const audioDirPath = MulmoStudioContextMethods.getAudioDirPath(context);
-    const { voiceId, provider, speechOptions, model } = getAudioParam(context.presentationStyle, beat);
+    const { voiceId, provider, speechOptions, model } = getAudioParam(context, beat);
     const hash_string = [text, voiceId, speechOptions?.instruction ?? "", speechOptions?.speed ?? 1.0, provider, model ?? ""].join(":");
     const audioFileName = `${context.studio.filename}_${text2hash(hash_string)}`;
     const audioFile = getAudioFilePath(audioDirPath, context.studio.filename, audioFileName, lang);
@@ -46,9 +46,9 @@ export const getBeatAudioPath = (text, context, beat, lang) => {
 };
 const preprocessor = (namedInputs) => {
     const { beat, studioBeat, multiLingual, context } = namedInputs;
-    const { lang, presentationStyle } = context;
+    const { lang } = context;
     const text = localizedText(beat, multiLingual, lang);
-    const { voiceId, provider, speechOptions, model } = getAudioParam(presentationStyle, beat);
+    const { voiceId, provider, speechOptions, model } = getAudioParam(context, beat);
     const audioPath = getBeatAudioPath(text, context, beat, lang);
     studioBeat.audioFile = audioPath; // TODO: Passing by reference is difficult to maintain, so pass it using graphai inputs
     const needsTTS = !beat.audio && audioPath !== undefined;
@@ -58,6 +58,8 @@ const preprocessor = (namedInputs) => {
         voiceId,
         speechOptions,
         model,
+        provider,
+        lang,
         audioPath,
         studioBeat,
         needsTTS,
@@ -84,6 +86,8 @@ const graph_tts = {
             agent: ":preprocessor.ttsAgent",
             inputs: {
                 text: ":preprocessor.text",
+                provider: ":preprocessor.provider",
+                lang: ":preprocessor.lang",
                 cache: {
                     force: [":context.force"],
                     file: ":preprocessor.audioPath",
@@ -173,12 +177,12 @@ const agentFilters = [
         agent: fileCacheAgentFilter,
         nodeIds: ["tts"],
     },
+    {
+        name: "nijovoiceTextAgentFilter",
+        agent: nijovoiceTextAgentFilter,
+        nodeIds: ["tts"],
+    },
 ];
-export const audioFilePath = (context) => {
-    const fileName = MulmoStudioContextMethods.getFileName(context);
-    const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
-    return getAudioArtifactFilePath(outDirPath, fileName);
-};
 const getConcurrency = (context) => {
     // Check if any speaker uses nijivoice or elevenlabs (providers that require concurrency = 1)
     const hasLimitedConcurrencyProvider = Object.values(context.presentationStyle.speechParams.speakers).some((speaker) => {
@@ -231,7 +235,7 @@ export const audio = async (context, settings, callbacks) => {
         const fileName = MulmoStudioContextMethods.getFileName(context);
         const audioDirPath = MulmoStudioContextMethods.getAudioDirPath(context);
         const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
-        const audioArtifactFilePath = audioFilePath(context);
+        const audioArtifactFilePath = getAudioArtifactFilePath(context);
         const audioSegmentDirPath = resolveDirPath(audioDirPath, fileName);
         const audioCombinedFilePath = getAudioFilePath(audioDirPath, fileName, fileName, context.lang);
         const outputStudioFilePath = getOutputStudioFilePath(outDirPath, fileName);
@@ -253,6 +257,7 @@ export const audio = async (context, settings, callbacks) => {
         const result = await graph.run();
         writingMessage(audioCombinedFilePath);
         MulmoStudioContextMethods.setSessionState(context, "audio", false);
+        writingMessage(audioArtifactFilePath);
         return result.combineFiles;
     }
     catch (__error) {

package/lib/actions/image_agents.d.ts

@@ -23,10 +23,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
     };
     lipSyncFile?: string;
     lipSyncModel?: string;
-    lipSyncAgentInfo?: {
-        agentName: string;
-        defaultModel: string;
-    };
+    lipSyncAgentName?: string;
     audioFile?: string;
     beatDuration?: number;
     htmlPrompt?: undefined;
@@ -61,10 +58,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
     };
     lipSyncFile?: string;
     lipSyncModel?: string;
-    lipSyncAgentInfo?: {
-        agentName: string;
-        defaultModel: string;
-    };
+    lipSyncAgentName?: string;
     audioFile?: string;
     beatDuration?: number;
     htmlPrompt?: undefined;
@@ -102,10 +96,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
     };
     lipSyncFile?: string;
     lipSyncModel?: string;
-    lipSyncAgentInfo?: {
-        agentName: string;
-        defaultModel: string;
-    };
+    lipSyncAgentName?: string;
     audioFile?: string;
     beatDuration?: number;
     htmlPrompt?: undefined;

package/lib/actions/image_agents.js

@@ -25,16 +25,20 @@ export const imagePreprocessAgent = async (namedInputs) => {
         movieFile: beat.moviePrompt ? moviePaths.movieFile : undefined,
         beatDuration: beat.duration ?? studioBeat?.duration,
     };
-    if (beat.soundEffectPrompt) {
-        returnValue.soundEffectAgentInfo = MulmoPresentationStyleMethods.getSoundEffectAgentInfo(context.presentationStyle, beat);
-        returnValue.soundEffectModel =
-            beat.soundEffectParams?.model ?? context.presentationStyle.soundEffectParams?.model ?? returnValue.soundEffectAgentInfo.defaultModel;
-        returnValue.soundEffectFile = moviePaths.soundEffectFile;
-        returnValue.soundEffectPrompt = beat.soundEffectPrompt;
+    const isMovie = Boolean(beat.moviePrompt || beat?.image?.type === "movie");
+    if (isMovie) {
+        if (beat.soundEffectPrompt) {
+            returnValue.soundEffectAgentInfo = MulmoPresentationStyleMethods.getSoundEffectAgentInfo(context.presentationStyle, beat);
+            returnValue.soundEffectModel =
+                beat.soundEffectParams?.model ?? context.presentationStyle.soundEffectParams?.model ?? returnValue.soundEffectAgentInfo.defaultModel;
+            returnValue.soundEffectFile = moviePaths.soundEffectFile;
+            returnValue.soundEffectPrompt = beat.soundEffectPrompt;
+        }
     }
     if (beat.enableLipSync) {
-        returnValue.lipSyncAgentInfo = MulmoPresentationStyleMethods.getLipSyncAgentInfo(context.presentationStyle, beat);
-        returnValue.lipSyncModel = beat.lipSyncParams?.model ?? context.presentationStyle.lipSyncParams?.model ?? returnValue.lipSyncAgentInfo.defaultModel;
+        const lipSyncAgentInfo = MulmoPresentationStyleMethods.getLipSyncAgentInfo(context.presentationStyle, beat);
+        returnValue.lipSyncAgentName = lipSyncAgentInfo.agentName;
+        returnValue.lipSyncModel = beat.lipSyncParams?.model ?? context.presentationStyle.lipSyncParams?.model ?? lipSyncAgentInfo.defaultModel;
         returnValue.lipSyncFile = moviePaths.lipSyncFile;
         // Audio file will be set from the beat's audio file when available
         returnValue.audioFile = studioBeat?.audioFile;

package/lib/actions/images.js

@@ -135,6 +135,7 @@ const beat_graph_data = {
                     model: ":preprocessor.imageParams.model",
                     moderation: ":preprocessor.imageParams.moderation",
                     canvasSize: ":context.presentationStyle.canvasSize",
+                    quality: ":preprocessor.imageParams.quality",
                 },
             },
             defaultValue: {},
@@ -217,10 +218,11 @@ const beat_graph_data = {
         },
         lipSyncGenerator: {
             if: ":beat.enableLipSync",
-            agent: ":preprocessor.lipSyncAgentInfo.agentName",
+            agent: ":preprocessor.lipSyncAgentName",
             inputs: {
                 onComplete: [":soundEffectGenerator"], // to wait for soundEffectGenerator to finish
                 movieFile: ":preprocessor.movieFile",
+                imageFile: ":preprocessor.referenceImageForMovie",
                 audioFile: ":preprocessor.audioFile",
                 lipSyncFile: ":preprocessor.lipSyncFile",
                 params: {

package/lib/actions/movie.js

@@ -246,9 +246,7 @@ export const movieFilePath = (context) => {
 export const movie = async (context) => {
     MulmoStudioContextMethods.setSessionState(context, "video", true);
     try {
-        const fileName = MulmoStudioContextMethods.getFileName(context);
-        const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
-        const audioArtifactFilePath = getAudioArtifactFilePath(outDirPath, fileName);
+        const audioArtifactFilePath = getAudioArtifactFilePath(context);
         const outputVideoPath = movieFilePath(context);
         if (await createVideo(audioArtifactFilePath, outputVideoPath, context)) {
             writingMessage(outputVideoPath);

package/lib/actions/translate.js

@@ -1,9 +1,9 @@
 import "dotenv/config";
-import { GraphAI, assert } from "graphai";
+import { GraphAI, assert, isNull } from "graphai";
 import * as agents from "@graphai/vanilla";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-import { recursiveSplitJa, replacementsJa, replacePairsJa } from "../utils/string.js";
+import { recursiveSplitJa } from "../utils/string.js";
 import { settings2GraphAIConfig } from "../utils/utils.js";
 import { getOutputMultilingualFilePath, mkdir, writingMessage } from "../utils/file.js";
 import { translateSystemPrompt, translatePrompts } from "../utils/prompt.js";
@@ -13,17 +13,9 @@ const translateGraph = {
     version: 0.5,
     nodes: {
         context: {},
-        defaultLang: {},
         outDirPath: {},
         outputMultilingualFilePath: {},
-        lang: {
-            agent: "stringUpdateTextAgent",
-            inputs: {
-                newText: ":context.studio.script.lang",
-                oldText: ":defaultLang",
-            },
-        },
-        targetLangs: {}, // TODO
+        targetLangs: {},
         mergeStudioResult: {
             isResult: true,
             agent: "mergeObjectAgent",
@@ -37,7 +29,6 @@ const translateGraph = {
                 targetLangs: ":targetLangs",
                 context: ":context",
                 rows: ":context.studio.script.beats",
-                lang: ":lang",
             },
             params: {
                 rowKey: "beat",
@@ -62,7 +53,7 @@ const translateGraph = {
                             beat: ":beat",
                             multiLingual: ":multiLingual",
                             rows: ":targetLangs",
-                            lang: ":lang.text",
+                            lang: ":context.studio.script.lang",
                             context: ":context",
                             beatIndex: ":__mapIndex",
                         },
@@ -120,17 +111,11 @@ const translateGraph = {
                                 },
                                 ttsTexts: {
                                     agent: (namedInputs) => {
-                                        const { localizedText, targetLang } = namedInputs;
+                                        const { localizedText } = namedInputs;
                                         // cache
                                         if (localizedText.ttsTexts) {
                                             return localizedText;
                                         }
-                                        if (targetLang === "ja") {
-                                            return {
-                                                ...localizedText,
-                                                ttsTexts: localizedText?.texts?.map((text) => replacePairsJa(text, replacementsJa)),
-                                            };
-                                        }
                                         return {
                                             ...localizedText,
                                             ttsTexts: localizedText.texts,
@@ -180,18 +165,14 @@ const localizedTextCacheAgentFilter = async (context, next) => {
     if (!beat.text) {
         return { text: "" };
     }
-    // The original text is unchanged and the target language text is present
-    if (multiLingual.multiLingualTexts &&
-        multiLingual.multiLingualTexts[lang] &&
-        multiLingual.multiLingualTexts[lang].text === beat.text &&
-        multiLingual.multiLingualTexts[targetLang] &&
-        multiLingual.multiLingualTexts[targetLang].text) {
-        return { text: multiLingual.multiLingualTexts[targetLang].text };
-    }
     // same language
     if (targetLang === lang) {
         return { text: beat.text };
     }
+    // The original text is unchanged and the target language text is present
+    if (multiLingual.multiLingualTexts?.[lang]?.text === beat.text && multiLingual.multiLingualTexts[targetLang]?.text) {
+        return { text: multiLingual.multiLingualTexts[targetLang].text };
+    }
     try {
         MulmoStudioContextMethods.setBeatSessionState(mulmoContext, "multiLingual", beatIndex, true);
         return await next(context);
@@ -207,8 +188,6 @@ const agentFilters = [
         nodeIds: ["localizedTexts"],
     },
 ];
-const defaultLang = "en";
-const targetLangs = ["ja", "en"];
 export const translate = async (context, args) => {
     const { settings, callbacks } = args ?? {};
     try {
@@ -217,11 +196,14 @@ export const translate = async (context, args) => {
         const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
         const outputMultilingualFilePath = getOutputMultilingualFilePath(outDirPath, fileName);
         mkdir(outDirPath);
+        const langs = (context.multiLingual ?? []).map((x) => Object.keys(x.multiLingualTexts)).flat(); // existing langs in multiLingual
+        const targetLangs = [
+            ...new Set([context.studio.script.lang, langs, context.lang, context.studio.script.captionParams?.lang].flat().filter((x) => !isNull(x))),
+        ];
         const config = settings2GraphAIConfig(settings, process.env);
         assert(!!config?.openAIAgent?.apiKey, "The OPENAI_API_KEY environment variable is missing or empty");
         const graph = new GraphAI(translateGraph, { ...vanillaAgents, fileWriteAgent, openAIAgent }, { agentFilters, config });
         graph.injectValue("context", context);
-        graph.injectValue("defaultLang", defaultLang);
         graph.injectValue("targetLangs", targetLangs);
         graph.injectValue("outDirPath", outDirPath);
         graph.injectValue("outputMultilingualFilePath", outputMultilingualFilePath);

package/lib/agents/image_openai_agent.js

@@ -6,7 +6,7 @@ import { provider2ImageAgent } from "../utils/provider2agent.js";
 // https://platform.openai.com/docs/guides/image-generation
 export const imageOpenaiAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, referenceImages } = namedInputs;
-    const { moderation, canvasSize } = params;
+    const { moderation, canvasSize, quality } = params;
     const { apiKey, baseURL } = { ...config };
     const model = params.model ?? provider2ImageAgent["openai"].defaultModel;
     const openai = new OpenAI({ apiKey, baseURL });
@@ -42,6 +42,9 @@ export const imageOpenaiAgent = async ({ namedInputs, params, config, }) => {
     };
     if (model === "gpt-image-1") {
         imageOptions.moderation = moderation || "auto";
+        if (quality) {
+            imageOptions.quality = quality;
+        }
     }
     const response = await (async () => {
         try {

package/lib/agents/lipsync_replicate_agent.js

@@ -3,7 +3,7 @@ import { GraphAILogger } from "graphai";
 import Replicate from "replicate";
 import { provider2LipSyncAgent } from "../utils/provider2agent.js";
 export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) => {
-    const { movieFile, audioFile } = namedInputs;
+    const { movieFile, audioFile, imageFile } = namedInputs;
     const apiKey = config?.apiKey;
     const model = params.model ?? provider2LipSyncAgent.replicate.defaultModel;
     if (!apiKey) {
@@ -12,10 +12,12 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
     const replicate = new Replicate({
         auth: apiKey,
     });
-    const videoBuffer = readFileSync(movieFile);
+    const videoBuffer = movieFile ? readFileSync(movieFile) : undefined;
     const audioBuffer = readFileSync(audioFile);
-    const videoUri = `data:video/quicktime;base64,${videoBuffer.toString("base64")}`;
+    const imageBuffer = imageFile ? readFileSync(imageFile) : undefined;
+    const videoUri = videoBuffer ? `data:video/quicktime;base64,${videoBuffer.toString("base64")}` : undefined;
     const audioUri = `data:audio/wav;base64,${audioBuffer.toString("base64")}`;
+    const imageUri = imageBuffer ? `data:image/png;base64,${imageBuffer.toString("base64")}` : undefined;
     const input = {
         video: undefined,
         video_input: undefined,
@@ -23,6 +25,7 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
         audio: undefined,
         audio_input: undefined,
         audio_file: undefined,
+        image: undefined,
     };
     const modelParams = provider2LipSyncAgent.replicate.modelParams[model];
     if (!modelParams) {
@@ -30,12 +33,16 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
     }
     const videoParam = modelParams.video;
     const audioParam = modelParams.audio;
+    const imageParam = modelParams.image;
     if (videoParam === "video" || videoParam === "video_input" || videoParam === "video_url") {
         input[videoParam] = videoUri;
     }
     if (audioParam === "audio" || audioParam === "audio_input" || audioParam === "audio_file") {
         input[audioParam] = audioUri;
     }
+    if (imageParam === "image") {
+        input[imageParam] = imageUri;
+    }
     const model_identifier = provider2LipSyncAgent.replicate.modelParams[model]?.identifier ?? model;
     try {
         const output = await replicate.run(model_identifier, {

package/lib/cli/commands/audio/handler.js

@@ -5,6 +5,6 @@ export const handler = async (argv) => {
     if (!context) {
         process.exit(1);
     }
-    await runTranslateIfNeeded(context, argv);
+    await runTranslateIfNeeded(context);
     await audio(context);
 };

package/lib/cli/commands/image/handler.js

@@ -5,6 +5,6 @@ export const handler = async (argv) => {
     if (!context) {
         process.exit(1);
     }
-    await runTranslateIfNeeded(context, argv);
+    await runTranslateIfNeeded(context);
     await images(context);
 };

package/lib/cli/commands/movie/handler.js

@@ -5,6 +5,6 @@ export const handler = async (argv) => {
     if (!context) {
         process.exit(1);
     }
-    await runTranslateIfNeeded(context, argv);
+    await runTranslateIfNeeded(context, true);
     await audio(context).then(images).then(captions).then(movie);
 };

package/lib/cli/commands/pdf/handler.js

@@ -5,7 +5,7 @@ export const handler = async (argv) => {
     if (!context) {
         process.exit(1);
     }
-    await runTranslateIfNeeded(context, argv);
+    await runTranslateIfNeeded(context);
     await images(context);
     await pdf(context, argv.pdf_mode, argv.pdf_size);
 };

package/lib/cli/helpers.d.ts

@@ -1,9 +1,6 @@
 import type { CliArgs } from "../types/cli_types.js";
 import { FileObject, InitOptions, MulmoStudioContext } from "../types/index.js";
-export declare const runTranslateIfNeeded: (context: MulmoStudioContext, argv: {
-    l?: string;
-    c?: string;
-}) => Promise<void>;
+export declare const runTranslateIfNeeded: (context: MulmoStudioContext, includeCaption?: boolean) => Promise<void>;
 export declare const setGraphAILogger: (verbose: boolean | undefined, logValues?: Record<string, unknown>) => void;
 export declare const getFileObject: (args: {
     basedir?: string;

package/lib/cli/helpers.js

@@ -5,10 +5,11 @@ import clipboardy from "clipboardy";
 import { getBaseDirPath, getFullPath, getOutputStudioFilePath, resolveDirPath, mkdir, getOutputMultilingualFilePath, generateTimestampedFileName, } from "../utils/file.js";
 import { isHttp } from "../utils/utils.js";
 import { outDirName, imageDirName, audioDirName } from "../utils/const.js";
+import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
 import { translate } from "../actions/translate.js";
 import { initializeContextFromFiles } from "../utils/context.js";
-export const runTranslateIfNeeded = async (context, argv) => {
-    if (argv.l || context.studio.script.captionParams?.lang) {
+export const runTranslateIfNeeded = async (context, includeCaption = false) => {
+    if (MulmoStudioContextMethods.needTranslate(context, includeCaption)) {
         GraphAILogger.log("run translate");
         await translate(context);
     }

package/lib/mcp/server.js

@@ -104,7 +104,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             throw new Error("Failed to initialize context from MulmoScript");
         }
         // Run translation if needed
-        await runTranslateIfNeeded(context, argv);
+        await runTranslateIfNeeded(context);
         // Execute the requested command
         switch (cmd) {
             case "movie":

package/lib/methods/mulmo_presentation_style.d.ts

@@ -1,12 +1,11 @@
 import "dotenv/config";
-import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider } from "../types/index.js";
+import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider, MulmoStudioContext } from "../types/index.js";
 export declare const MulmoPresentationStyleMethods: {
     getCanvasSize(presentationStyle: MulmoPresentationStyle): MulmoCanvasDimension;
     getAllSpeechProviders(presentationStyle: MulmoPresentationStyle): Set<Text2SpeechProvider>;
     getTextSlideStyle(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
     getDefaultSpeaker(presentationStyle: MulmoPresentationStyle): string;
-    getSpeaker(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SpeakerData;
-    getTTSModel(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string | undefined;
+    getSpeaker(context: MulmoStudioContext, beat: MulmoBeat): SpeakerData;
     getText2ImageProvider(provider: Text2ImageProvider | undefined): Text2ImageProvider;
     getImageAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): Text2ImageAgentInfo;
     getMovieAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): {
@@ -37,9 +36,10 @@ export declare const MulmoPresentationStyleMethods: {
         defaultModel: import("../utils/provider2agent.js").ReplicateModel;
         models: import("../utils/provider2agent.js").ReplicateModel[];
         modelParams: Record<import("../utils/provider2agent.js").ReplicateModel, {
-            identifier?: `${string}/${string}:${string}`;
-            video: string;
+            identifier?: `${string}/${string}:${string}` | `${string}/${string}`;
+            video?: string;
             audio: string;
+            image?: string;
         }>;
     };
     getConcurrency(presentationStyle: MulmoPresentationStyle): 4 | 16;

package/lib/methods/mulmo_presentation_style.js

@@ -46,18 +46,24 @@ export const MulmoPresentationStyleMethods = {
         }
         return keys[0];
     },
-    getSpeaker(presentationStyle, beat) {
-        userAssert(!!presentationStyle?.speechParams?.speakers, "presentationStyle.speechParams.speakers is not set!!");
-        const speakerId = beat?.speaker ?? MulmoPresentationStyleMethods.getDefaultSpeaker(presentationStyle);
-        userAssert(!!speakerId, "beat.speaker and default speaker is not set");
-        const speaker = presentationStyle.speechParams.speakers[speakerId];
+    getSpeaker(context, beat) {
+        userAssert(!!context.presentationStyle?.speechParams?.speakers, "presentationStyle.speechParams.speakers is not set!!");
+        const speakerId = beat?.speaker ?? MulmoPresentationStyleMethods.getDefaultSpeaker(context.presentationStyle);
+        const speaker = context.presentationStyle.speechParams.speakers[speakerId];
         userAssert(!!speaker, `speaker is not set: speaker "${speakerId}"`);
+        // Check if the speaker has a language-specific version
+        const lang = context.lang ?? context.studio.script.lang;
+        if (speaker.lang && lang && speaker.lang[lang]) {
+            return speaker.lang[lang];
+        }
         return speaker;
     },
-    getTTSModel(presentationStyle, beat) {
-        const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
-        return speaker.model;
+    /* NOTE: This method is not used.
+    getTTSModel(context: MulmoStudioContext, beat: MulmoBeat): string | undefined {
+      const speaker = MulmoPresentationStyleMethods.getSpeaker(context, beat);
+      return speaker.model;
     },
+    */
     getText2ImageProvider(provider) {
         return text2ImageProviderSchema.parse(provider);
     },

package/lib/methods/mulmo_script.js

@@ -18,6 +18,9 @@ const validators = [{ from: "1.0", to: "1.1", validator: validate_1_0 }];
 export const MulmoScriptMethods = {
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     validate(script) {
+        const version = script.$mulmocast.version;
+        // lang was optional in 1.0 and 1.1
+        const defaultLang = version === "1.0" || version === "1.1" ? { lang: "en" } : {};
         const validatedScript = validators.reduce((acc, validator) => {
             if (acc.$mulmocast.version === validator.from) {
                 const validated = validator.validator(acc);
@@ -25,7 +28,7 @@ export const MulmoScriptMethods = {
                 return validated;
             }
             return acc;
-        }, script);
+        }, { ...defaultLang, ...script });
         return mulmoScriptSchema.parse(validatedScript);
     },
 };

package/lib/methods/mulmo_studio_context.d.ts

@@ -11,4 +11,5 @@ export declare const MulmoStudioContextMethods: {
     getCaption(context: MulmoStudioContext): string | undefined;
     setSessionState(context: MulmoStudioContext, sessionType: SessionType, value: boolean): void;
     setBeatSessionState(context: MulmoStudioContext, sessionType: BeatSessionType, index: number, value: boolean): void;
+    needTranslate(context: MulmoStudioContext, includeCaption?: boolean): boolean | "" | undefined;
 };

package/lib/methods/mulmo_studio_context.js

@@ -63,4 +63,12 @@ export const MulmoStudioContextMethods = {
         }
         notifyBeatStateChange(context, sessionType, index);
     },
+    needTranslate(context, includeCaption = false) {
+        // context.studio.script.lang = defaultLang, context.lang = targetLanguage.
+        if (includeCaption) {
+            return (context.studio.script.lang !== context.lang ||
+                (context.studio.script.captionParams?.lang && context.studio.script.lang !== context.studio.script.captionParams?.lang));
+        }
+        return context.studio.script.lang !== context.lang;
+    },
 };

package/lib/types/agent.d.ts

@@ -1,11 +1,13 @@
 export type OpenAIImageSize = "1792x1024" | "1024x1792" | "1024x1024" | "1536x1024" | "1024x1536";
 export type OpenAIImageModeration = "low" | "auto";
+export type OpenAIImageQuality = "low" | "medium" | "high" | "auto";
 export type OpenAIImageOptions = {
     model: string;
     prompt: string;
     n: number;
     size: OpenAIImageSize;
     moderation?: OpenAIImageModeration;
+    quality?: OpenAIImageQuality;
 };
 export type AgentBufferResult = {
     buffer: Buffer;
@@ -35,6 +37,7 @@ export type ImageAgentParams = {
 };
 export type OpenAIImageAgentParams = ImageAgentParams & {
     moderation: OpenAIImageModeration | null | undefined;
+    quality?: OpenAIImageQuality;
 };
 export type OpenAIImageAgentConfig = {
     baseURL?: string;
@@ -74,6 +77,7 @@ export type LipSyncAgentInputs = {
     lipSyncFile: string;
     movieFile: string;
     audioFile: string;
+    imageFile: string;
 };
 export type GoogleMovieAgentConfig = GoogleImageAgentConfig;
 export type ReplicateMovieAgentConfig = AgentConfig;