mulmocast 1.1.6 → 1.1.7

package/lib/actions/translate.d.ts

@@ -1,7 +1,56 @@
 import "dotenv/config";
 import type { CallbackFunction } from "graphai";
-import { MulmoStudioContext } from "../types/index.js";
+import { LANG, LocalizedText, MulmoStudioContext } from "../types/index.js";
+export declare const translateTextGraph: {
+    version: number;
+    nodes: {
+        localizedText: {
+            inputs: {
+                targetLang: string;
+                beat: string;
+                multiLingual: string;
+                lang: string;
+                beatIndex: string;
+                mulmoContext: string;
+                system: string;
+                prompt: string[];
+            };
+            passThrough: {
+                lang: string;
+            };
+            output: {
+                text: string;
+            };
+            agent: string;
+        };
+        splitText: {
+            agent: (namedInputs: {
+                localizedText: LocalizedText;
+                targetLang: LANG;
+            }) => string[];
+            inputs: {
+                targetLang: string;
+                localizedText: string;
+            };
+        };
+        textTranslateResult: {
+            isResult: boolean;
+            agent: string;
+            inputs: {
+                lang: string;
+                text: string;
+                texts: string;
+                ttsTexts: string;
+                cacheKey: string;
+            };
+        };
+    };
+};
+export declare const translateBeat: (index: number, context: MulmoStudioContext, targetLangs: string[], args?: {
+    settings?: Record<string, string>;
+    callbacks?: CallbackFunction[];
+}) => Promise<void>;
 export declare const translate: (context: MulmoStudioContext, args?: {
     callbacks?: CallbackFunction[];
     settings?: Record<string, string>;
-}) => Promise<void>;
+}) => Promise<MulmoStudioContext>;

package/lib/actions/translate.js

@@ -1,14 +1,152 @@
 import "dotenv/config";
-import { GraphAI, assert, isNull } from "graphai";
+import { createHash } from "crypto";
+import fs from "fs";
+import { GraphAI, assert, isNull, GraphAILogger } from "graphai";
 import * as agents from "@graphai/vanilla";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { recursiveSplitJa } from "../utils/string.js";
 import { settings2GraphAIConfig } from "../utils/utils.js";
+import { getMultiLingual } from "../utils/context.js";
+import { currentMulmoScriptVersion } from "../utils/const.js";
 import { getOutputMultilingualFilePath, mkdir, writingMessage } from "../utils/file.js";
 import { translateSystemPrompt, translatePrompts } from "../utils/prompt.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
 const vanillaAgents = agents.default ?? agents;
+const hashSHA256 = (text) => {
+    return createHash("sha256").update(text, "utf8").digest("hex");
+};
+// 1. translateGraph / map each beats.
+// 2. beatGraph / map each target lang.
+// 3. translateTextGraph / translate text.
+export const translateTextGraph = {
+    version: 0.5,
+    nodes: {
+        localizedText: {
+            inputs: {
+                targetLang: ":targetLang", // for cache
+                beat: ":beat", // for cache
+                multiLingual: ":multiLingual", // for cache
+                lang: ":lang", // for cache
+                beatIndex: ":beatIndex", // for cache (state)
+                mulmoContext: ":context", // for cache (state)
+                system: translateSystemPrompt,
+                prompt: translatePrompts,
+            },
+            passThrough: {
+                lang: ":targetLang",
+            },
+            output: {
+                text: ".text",
+            },
+            // return { lang, text } <- localizedText
+            agent: "openAIAgent",
+        },
+        splitText: {
+            agent: (namedInputs) => {
+                const { localizedText, targetLang } = namedInputs;
+                // Cache
+                if (localizedText.texts) {
+                    return localizedText.texts;
+                }
+                if (targetLang === "ja") {
+                    return recursiveSplitJa(localizedText.text);
+                }
+                // not split
+                return [localizedText.text];
+            },
+            inputs: {
+                targetLang: ":targetLang",
+                localizedText: ":localizedText",
+            },
+        },
+        textTranslateResult: {
+            isResult: true,
+            agent: "copyAgent",
+            inputs: {
+                lang: ":targetLang",
+                text: ":localizedText.text",
+                texts: ":splitText",
+                ttsTexts: ":splitText",
+                cacheKey: ":multiLingual.cacheKey",
+            },
+        },
+    },
+};
+const beatGraph = {
+    version: 0.5,
+    nodes: {
+        targetLangs: {},
+        context: {},
+        beat: {},
+        __mapIndex: {},
+        // for cache
+        multiLingual: {
+            agent: (namedInputs) => {
+                const { multiLinguals, beatIndex, text } = namedInputs;
+                const cacheKey = hashSHA256(text ?? "");
+                const multiLingual = multiLinguals?.[beatIndex];
+                if (!multiLingual) {
+                    return { cacheKey, multiLingualTexts: {} };
+                }
+                return {
+                    multiLingualTexts: Object.keys(multiLingual.multiLingualTexts).reduce((tmp, lang) => {
+                        if (multiLingual.multiLingualTexts[lang].cacheKey === cacheKey) {
+                            tmp[lang] = multiLingual.multiLingualTexts[lang];
+                        }
+                        return tmp;
+                    }, {}),
+                    cacheKey,
+                };
+            },
+            inputs: {
+                text: ":beat.text",
+                beatIndex: ":__mapIndex",
+                multiLinguals: ":context.multiLingual",
+            },
+        },
+        preprocessMultiLingual: {
+            agent: "mapAgent",
+            inputs: {
+                beat: ":beat",
+                multiLingual: ":multiLingual",
+                rows: ":targetLangs",
+                lang: ":context.studio.script.lang",
+                context: ":context",
+                beatIndex: ":__mapIndex",
+            },
+            params: {
+                compositeResult: true,
+                rowKey: "targetLang",
+            },
+            graph: translateTextGraph,
+        },
+        mergeLocalizedText: {
+            // console: { after: true},
+            agent: "arrayToObjectAgent",
+            inputs: {
+                items: ":preprocessMultiLingual.textTranslateResult",
+            },
+            params: {
+                key: "lang",
+            },
+        },
+        multiLingualTexts: {
+            agent: "mergeObjectAgent",
+            inputs: {
+                items: [":multiLingual.multiLingualTexts", ":mergeLocalizedText"],
+            },
+        },
+        mergeMultiLingualData: {
+            isResult: true,
+            // console: { after: true},
+            agent: "mergeObjectAgent",
+            inputs: {
+                items: [":multiLingual", { multiLingualTexts: ":multiLingualTexts" }],
+            },
+        },
+    },
+};
 const translateGraph = {
     version: 0.5,
     nodes: {
@@ -18,9 +156,10 @@ const translateGraph = {
         targetLangs: {},
         mergeStudioResult: {
             isResult: true,
-            agent: "mergeObjectAgent",
+            agent: "copyAgent",
             inputs: {
-                items: [{ multiLingual: ":beatsMap.mergeMultiLingualData" }],
+                version: "1.1",
+                multiLingual: ":beatsMap.mergeMultiLingualData",
             },
         },
         beatsMap: {
@@ -34,127 +173,13 @@ const translateGraph = {
                 rowKey: "beat",
                 compositeResult: true,
             },
-            graph: {
-                version: 0.5,
-                nodes: {
-                    // for cache
-                    multiLingual: {
-                        agent: (namedInputs) => {
-                            return (namedInputs.rows && namedInputs.rows[namedInputs.index]) || {};
-                        },
-                        inputs: {
-                            index: ":__mapIndex",
-                            rows: ":context.multiLingual",
-                        },
-                    },
-                    preprocessMultiLingual: {
-                        agent: "mapAgent",
-                        inputs: {
-                            beat: ":beat",
-                            multiLingual: ":multiLingual",
-                            rows: ":targetLangs",
-                            lang: ":context.studio.script.lang",
-                            context: ":context",
-                            beatIndex: ":__mapIndex",
-                        },
-                        params: {
-                            compositeResult: true,
-                            rowKey: "targetLang",
-                        },
-                        graph: {
-                            version: 0.5,
-                            nodes: {
-                                localizedTexts: {
-                                    inputs: {
-                                        targetLang: ":targetLang", // for cache
-                                        beat: ":beat", // for cache
-                                        multiLingual: ":multiLingual", // for cache
-                                        lang: ":lang", // for cache
-                                        beatIndex: ":beatIndex", // for cache
-                                        mulmoContext: ":context", // for cache
-                                        system: translateSystemPrompt,
-                                        prompt: translatePrompts,
-                                    },
-                                    passThrough: {
-                                        lang: ":targetLang",
-                                    },
-                                    output: {
-                                        text: ".text",
-                                    },
-                                    // return { lang, text } <- localizedText
-                                    agent: "openAIAgent",
-                                },
-                                splitText: {
-                                    agent: (namedInputs) => {
-                                        const { localizedText, targetLang } = namedInputs;
-                                        // Cache
-                                        if (localizedText.texts) {
-                                            return localizedText;
-                                        }
-                                        if (targetLang === "ja") {
-                                            return {
-                                                ...localizedText,
-                                                texts: recursiveSplitJa(localizedText.text),
-                                            };
-                                        }
-                                        // not split
-                                        return {
-                                            ...localizedText,
-                                            texts: [localizedText.text],
-                                        };
-                                        // return { lang, text, texts }
-                                    },
-                                    inputs: {
-                                        targetLang: ":targetLang",
-                                        localizedText: ":localizedTexts",
-                                    },
-                                },
-                                ttsTexts: {
-                                    agent: (namedInputs) => {
-                                        const { localizedText } = namedInputs;
-                                        // cache
-                                        if (localizedText.ttsTexts) {
-                                            return localizedText;
-                                        }
-                                        return {
-                                            ...localizedText,
-                                            ttsTexts: localizedText.texts,
-                                        };
-                                    },
-                                    inputs: {
-                                        targetLang: ":targetLang",
-                                        localizedText: ":splitText",
-                                    },
-                                    isResult: true,
-                                },
-                            },
-                        },
-                    },
-                    mergeLocalizedText: {
-                        agent: "arrayToObjectAgent",
-                        inputs: {
-                            items: ":preprocessMultiLingual.ttsTexts",
-                        },
-                        params: {
-                            key: "lang",
-                        },
-                    },
-                    mergeMultiLingualData: {
-                        isResult: true,
-                        agent: "mergeObjectAgent",
-                        inputs: {
-                            items: [":multiLingual", { multiLingualTexts: ":mergeLocalizedText" }],
-                        },
-                    },
-                },
-            },
+            graph: beatGraph,
         },
         writeOutput: {
-            // console: { before: true },
             agent: "fileWriteAgent",
             inputs: {
                 file: ":outputMultilingualFilePath",
-                text: ":mergeStudioResult.multiLingual.toJSON()",
+                text: ":mergeStudioResult.toJSON()",
             },
         },
     },
@@ -170,7 +195,7 @@ const localizedTextCacheAgentFilter = async (context, next) => {
         return { text: beat.text };
     }
     // The original text is unchanged and the target language text is present
-    if (multiLingual.multiLingualTexts?.[lang]?.text === beat.text && multiLingual.multiLingualTexts[targetLang]?.text) {
+    if (multiLingual.cacheKey === multiLingual.multiLingualTexts[targetLang]?.cacheKey) {
         return { text: multiLingual.multiLingualTexts[targetLang].text };
     }
     try {
@@ -185,9 +210,49 @@ const agentFilters = [
     {
         name: "localizedTextCacheAgentFilter",
         agent: localizedTextCacheAgentFilter,
-        nodeIds: ["localizedTexts"],
+        nodeIds: ["localizedText"],
     },
 ];
+export const translateBeat = async (index, context, targetLangs, args) => {
+    const { settings, callbacks } = args ?? {};
+    // Validate inputs
+    if (index < 0 || index >= context.studio.script.beats.length) {
+        throw new Error(`Invalid beat index: ${index}. Must be between 0 and ${context.studio.script.beats.length - 1}`);
+    }
+    if (!targetLangs || targetLangs.length === 0) {
+        throw new Error("targetLangs must be a non-empty array");
+    }
+    try {
+        const fileName = MulmoStudioContextMethods.getFileName(context);
+        const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
+        const outputMultilingualFilePath = getOutputMultilingualFilePath(outDirPath, fileName);
+        mkdir(outDirPath);
+        const config = settings2GraphAIConfig(settings, process.env);
+        assert(!!config?.openAIAgent?.apiKey, "The OPENAI_API_KEY environment variable is missing or empty");
+        const graph = new GraphAI(beatGraph, { ...vanillaAgents, fileWriteAgent, openAIAgent }, { agentFilters, config });
+        graph.injectValue("context", context);
+        graph.injectValue("targetLangs", targetLangs);
+        graph.injectValue("beat", context.studio.script.beats[index]);
+        graph.injectValue("__mapIndex", index);
+        if (callbacks) {
+            callbacks.forEach((callback) => {
+                graph.registerCallback(callback);
+            });
+        }
+        const results = await graph.run();
+        const multiLingual = getMultiLingual(outputMultilingualFilePath, context.studio.beats.length);
+        multiLingual[index] = results.mergeMultiLingualData;
+        const data = {
+            version: currentMulmoScriptVersion,
+            multiLingual,
+        };
+        fs.writeFileSync(outputMultilingualFilePath, JSON.stringify(data, null, 2), "utf8");
+        writingMessage(outputMultilingualFilePath);
+    }
+    catch (error) {
+        GraphAILogger.log(error);
+    }
+};
 export const translate = async (context, args) => {
     const { settings, callbacks } = args ?? {};
     try {
@@ -196,10 +261,7 @@ export const translate = async (context, args) => {
         const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
         const outputMultilingualFilePath = getOutputMultilingualFilePath(outDirPath, fileName);
         mkdir(outDirPath);
-        const langs = (context.multiLingual ?? []).map((x) => Object.keys(x.multiLingualTexts)).flat(); // existing langs in multiLingual
-        const targetLangs = [
-            ...new Set([context.studio.script.lang, langs, context.lang, context.studio.script.captionParams?.lang].flat().filter((x) => !isNull(x))),
-        ];
+        const targetLangs = [...new Set([context.lang, context.studio.script.captionParams?.lang].filter((x) => !isNull(x)))];
         const config = settings2GraphAIConfig(settings, process.env);
         assert(!!config?.openAIAgent?.apiKey, "The OPENAI_API_KEY environment variable is missing or empty");
         const graph = new GraphAI(translateGraph, { ...vanillaAgents, fileWriteAgent, openAIAgent }, { agentFilters, config });
@@ -221,4 +283,5 @@ export const translate = async (context, args) => {
     finally {
         MulmoStudioContextMethods.setSessionState(context, "multiLingual", false);
     }
+    return context;
 };

package/lib/agents/combine_audio_files_agent.js

@@ -94,7 +94,7 @@ const voiceOverProcess = (context, mediaDurations, movieDuration, beatDurations,
         if (voiceStartAt) {
             const remainingDuration = movieDuration - voiceStartAt;
             const duration = remaining - remainingDuration;
-            userAssert(duration >= 0, `Invalid startAt: At index(${idx}), avaiable duration(${duration}) < 0`);
+            userAssert(duration >= 0, `Invalid startAt: At index(${idx}), available duration(${duration}) < 0`);
             beatDurations.push(duration);
             subBeatDurations.silenceDuration = duration - subBeatDurations.audioDuration;
             userAssert(subBeatDurations.silenceDuration >= 0, `Duration Overwrap: At index(${idx}), silenceDuration(${subBeatDurations.silenceDuration}) < 0`);

package/lib/agents/tts_nijivoice_agent.js

@@ -27,7 +27,7 @@ export const ttsNijivoiceAgent = async ({ params, namedInputs, config, }) => {
     try {
         const voiceRes = await fetch(url, options);
         const voiceJson = await voiceRes.json();
-        if (voiceJson && voiceJson.generatedVoice && voiceJson.generatedVoice.audioFileDownloadUrl) {
+        if (voiceJson?.generatedVoice?.audioFileDownloadUrl) {
             const audioRes = await fetch(voiceJson.generatedVoice.audioFileDownloadUrl);
             const buffer = Buffer.from(await audioRes.arrayBuffer());
             return { buffer };

package/lib/index.common.d.ts

@@ -1,2 +1,3 @@
 export * from "./types/index.js";
 export * from "./utils/provider2agent.js";
+export * from "./utils/const.js";

package/lib/index.common.js

@@ -1,3 +1,4 @@
 // Entry point for universal code
 export * from "./types/index.js";
 export * from "./utils/provider2agent.js";
+export * from "./utils/const.js";

package/lib/methods/mulmo_script.d.ts

@@ -1,4 +1,7 @@
-import { MulmoScript } from "../types/index.js";
+import { type MulmoScript, type MulmoStudioMultiLingual } from "../types/index.js";
 export declare const MulmoScriptMethods: {
     validate(script: any): MulmoScript;
 };
+export declare const MulmoStudioMultiLingualMethod: {
+    validate(jsonData: any, studioBeatsLength: number): MulmoStudioMultiLingual;
+};

package/lib/methods/mulmo_script.js

@@ -1,4 +1,4 @@
-import { mulmoScriptSchema } from "../types/index.js";
+import { mulmoScriptSchema, mulmoStudioMultiLingualFileSchema } from "../types/index.js";
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const validate_1_0 = (script) => {
     if (script.speechParams?.provider) {
@@ -32,3 +32,16 @@ export const MulmoScriptMethods = {
         return mulmoScriptSchema.parse(validatedScript);
     },
 };
+export const MulmoStudioMultiLingualMethod = {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    validate(jsonData, studioBeatsLength) {
+        // TODO version check
+        const result = mulmoStudioMultiLingualFileSchema.safeParse(jsonData);
+        const dataSet = result.success ? result.data.multiLingual : [];
+        while (dataSet.length < studioBeatsLength) {
+            dataSet.push({ multiLingualTexts: {} });
+        }
+        dataSet.length = studioBeatsLength;
+        return dataSet;
+    },
+};