mulmocast 0.1.5 → 0.1.6

package/lib/actions/images.js

@@ -293,17 +293,11 @@ export const graphOption = async (context, settings) => {
     const config = settings2GraphAIConfig(settings, process.env);
     // We need to get google's auth token only if the google is the text2image provider.
     if (provider === "google" || context.presentationStyle.movieParams?.provider === "google") {
-        userAssert(!!process.env.GOOGLE_PROJECT_ID, "GOOGLE_PROJECT_ID is not set");
+        userAssert(!!config.movieGoogleAgent || !!config.imageGoogleAgent, "GOOGLE_PROJECT_ID is not set");
         GraphAILogger.log("google was specified as text2image engine");
         const token = await googleAuth();
-        config["imageGoogleAgent"] = {
-            projectId: process.env.GOOGLE_PROJECT_ID,
-            token,
-        };
-        config["movieGoogleAgent"] = {
-            projectId: process.env.GOOGLE_PROJECT_ID,
-            token,
-        };
+        config["imageGoogleAgent"].token = token;
+        config["movieGoogleAgent"].token = token;
     }
     options.config = config;
     return options;

package/lib/actions/translate.d.ts

@@ -1,4 +1,7 @@
 import "dotenv/config";
 import type { CallbackFunction } from "graphai";
 import { MulmoStudioContext } from "../types/index.js";
-export declare const translate: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;
+export declare const translate: (context: MulmoStudioContext, args?: {
+    callbacks?: CallbackFunction[];
+    settings?: Record<string, string>;
+}) => Promise<void>;

package/lib/actions/translate.js

@@ -4,6 +4,7 @@ import * as agents from "@graphai/vanilla";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { recursiveSplitJa, replacementsJa, replacePairsJa } from "../utils/string.js";
+import { settings2GraphAIConfig } from "../utils/utils.js";
 import { getOutputMultilingualFilePath, mkdir, writingMessage } from "../utils/file.js";
 import { translateSystemPrompt, translatePrompts } from "../utils/prompt.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
@@ -208,15 +209,17 @@ const agentFilters = [
 ];
 const defaultLang = "en";
 const targetLangs = ["ja", "en"];
-export const translate = async (context, callbacks) => {
+export const translate = async (context, args) => {
+    const { settings, callbacks } = args ?? {};
     try {
         MulmoStudioContextMethods.setSessionState(context, "multiLingual", true);
         const fileName = MulmoStudioContextMethods.getFileName(context);
         const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
         const outputMultilingualFilePath = getOutputMultilingualFilePath(outDirPath, fileName);
         mkdir(outDirPath);
-        assert(!!process.env.OPENAI_API_KEY, "The OPENAI_API_KEY environment variable is missing or empty");
-        const graph = new GraphAI(translateGraph, { ...vanillaAgents, fileWriteAgent, openAIAgent }, { agentFilters });
+        const config = settings2GraphAIConfig(settings, process.env);
+        assert(!!config?.openAIAgent?.apiKey, "The OPENAI_API_KEY environment variable is missing or empty");
+        const graph = new GraphAI(translateGraph, { ...vanillaAgents, fileWriteAgent, openAIAgent }, { agentFilters, config });
         graph.injectValue("context", context);
         graph.injectValue("defaultLang", defaultLang);
         graph.injectValue("targetLangs", targetLangs);

package/lib/agents/combine_audio_files_agent.js

@@ -1,6 +1,6 @@
 import { assert, GraphAILogger } from "graphai";
 import { silent60secPath } from "../utils/file.js";
-import { FfmpegContextInit, FfmpegContextGenerateOutput, FfmpegContextInputFormattedAudio, ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
+import { FfmpegContextInit, FfmpegContextGenerateOutput, FfmpegContextInputFormattedAudio, ffmpegGetMediaDuration, } from "../utils/ffmpeg_utils.js";
 import { userAssert } from "../utils/utils.js";
 const getMovieDuration = async (beat) => {
     if (beat.image?.type === "movie" && (beat.image.source.kind === "url" || beat.image.source.kind === "path")) {
@@ -62,6 +62,93 @@ const getGroupBeatDurations = (context, group, audioDuration) => {
     });
     return durations;
 };
+const getInputIds = (context, mediaDurations, ffmpegContext, silentIds) => {
+    const inputIds = [];
+    context.studio.beats.forEach((studioBeat, index) => {
+        const { silenceDuration } = mediaDurations[index];
+        const paddingId = `[padding_${index}]`;
+        if (studioBeat.audioFile) {
+            const audioId = FfmpegContextInputFormattedAudio(ffmpegContext, studioBeat.audioFile);
+            inputIds.push(audioId);
+        }
+        if (silenceDuration > 0) {
+            const silentId = silentIds.pop();
+            ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${silenceDuration}${paddingId}`);
+            inputIds.push(paddingId);
+        }
+    });
+    return inputIds;
+};
+const voiceOverProcess = (context, mediaDurations, movieDuration, beatDurations, groupLength) => {
+    return (remaining, idx, iGroup) => {
+        const subBeatDurations = mediaDurations[idx];
+        userAssert(subBeatDurations.audioDuration <= remaining, `Duration Overflow: At index(${idx}) audioDuration(${subBeatDurations.audioDuration}) > remaining(${remaining})`);
+        if (iGroup === groupLength - 1) {
+            beatDurations.push(remaining);
+            subBeatDurations.silenceDuration = remaining - subBeatDurations.audioDuration;
+            return 0;
+        }
+        const nextBeat = context.studio.script.beats[idx + 1];
+        assert(nextBeat.image?.type === "voice_over", "nextBeat.image.type !== voice_over");
+        const voiceStartAt = nextBeat.image?.startAt;
+        if (voiceStartAt) {
+            const remainingDuration = movieDuration - voiceStartAt;
+            const duration = remaining - remainingDuration;
+            userAssert(duration >= 0, `Invalid startAt: At index(${idx}), avaiable duration(${duration}) < 0`);
+            beatDurations.push(duration);
+            subBeatDurations.silenceDuration = duration - subBeatDurations.audioDuration;
+            userAssert(subBeatDurations.silenceDuration >= 0, `Duration Overwrap: At index(${idx}), silenceDuration(${subBeatDurations.silenceDuration}) < 0`);
+            return remainingDuration;
+        }
+        beatDurations.push(subBeatDurations.audioDuration);
+        return remaining - subBeatDurations.audioDuration;
+    };
+};
+const getVoiceOverGroup = (context, index) => {
+    const group = [index];
+    for (let i = index + 1; i < context.studio.beats.length && context.studio.script.beats[i].image?.type === "voice_over"; i++) {
+        group.push(i);
+    }
+    return group;
+};
+const getSpillOverGroup = (context, mediaDurations, index) => {
+    const group = [index];
+    for (let i = index + 1; i < context.studio.beats.length && !mediaDurations[i].hasMedia; i++) {
+        group.push(i);
+    }
+    return group;
+};
+const spilledOverAudio = (context, group, audioDuration, beatDurations, mediaDurations) => {
+    const groupBeatsDurations = getGroupBeatDurations(context, group, audioDuration);
+    // Yes, the current beat has spilled over audio.
+    const beatsTotalDuration = groupBeatsDurations.reduce((a, b) => a + b, 0);
+    if (beatsTotalDuration > audioDuration + 0.01) {
+        // 0.01 is a tolerance to avoid floating point precision issues
+        group.reduce((remaining, idx, iGroup) => {
+            if (remaining >= groupBeatsDurations[iGroup]) {
+                return remaining - groupBeatsDurations[iGroup];
+            }
+            mediaDurations[idx].silenceDuration = groupBeatsDurations[iGroup] - remaining;
+            return 0;
+        }, audioDuration);
+    }
+    else if (audioDuration > beatsTotalDuration) {
+        // Last beat gets the rest of the audio.
+        groupBeatsDurations[groupBeatsDurations.length - 1] += audioDuration - beatsTotalDuration;
+    }
+    beatDurations.push(...groupBeatsDurations);
+};
+const noSpilledOverAudio = (context, beat, index, movieDuration, audioDuration, beatDurations, mediaDurations) => {
+    // padding is the amount of audio padding specified in the script.
+    const padding = getPadding(context, beat, index);
+    // totalPadding is the amount of audio padding to be added to the audio file.
+    const totalPadding = Math.round(getTotalPadding(padding, movieDuration, audioDuration, beat.duration) * 100) / 100;
+    const beatDuration = audioDuration + totalPadding;
+    beatDurations.push(beatDuration);
+    if (totalPadding > 0) {
+        mediaDurations[index].silenceDuration = totalPadding;
+    }
+};
 const combineAudioFilesAgent = async ({ namedInputs, }) => {
     const { context, combinedFileName } = namedInputs;
     const ffmpegContext = FfmpegContextInit();
@@ -77,91 +164,37 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
         const { audioDuration, movieDuration } = mediaDurations[index];
         // Check if we are processing a voice-over beat.
         if (movieDuration > 0) {
-            const group = [index];
-            for (let i = index + 1; i < context.studio.beats.length && context.studio.script.beats[i].image?.type === "voice_over"; i++) {
-                group.push(i);
-            }
+            const group = getVoiceOverGroup(context, index);
             if (group.length > 1) {
-                group.reduce((remaining, idx, iGroup) => {
-                    const subBeatDurations = mediaDurations[idx];
-                    userAssert(subBeatDurations.audioDuration <= remaining, `Duration Overflow: At index(${idx}) audioDuration(${subBeatDurations.audioDuration}) > remaining(${remaining})`);
-                    if (iGroup === group.length - 1) {
-                        beatDurations.push(remaining);
-                        subBeatDurations.silenceDuration = remaining - subBeatDurations.audioDuration;
-                        return 0;
-                    }
-                    const nextBeat = context.studio.script.beats[idx + 1];
-                    assert(nextBeat.image?.type === "voice_over", "nextBeat.image.type !== voice_over");
-                    const voiceStartAt = nextBeat.image?.startAt;
-                    if (voiceStartAt) {
-                        const remainingDuration = movieDuration - voiceStartAt;
-                        const duration = remaining - remainingDuration;
-                        userAssert(duration >= 0, `Invalid startAt: At index(${idx}), avaiable duration(${duration}) < 0`);
-                        beatDurations.push(duration);
-                        subBeatDurations.silenceDuration = duration - subBeatDurations.audioDuration;
-                        userAssert(subBeatDurations.silenceDuration >= 0, `Duration Overwrap: At index(${idx}), silenceDuration(${subBeatDurations.silenceDuration}) < 0`);
-                        return remainingDuration;
-                    }
-                    beatDurations.push(subBeatDurations.audioDuration);
-                    return remaining - subBeatDurations.audioDuration;
-                }, movieDuration);
+                GraphAILogger.log(`Voice over group: ${group.length}`);
+                group.reduce(voiceOverProcess(context, mediaDurations, movieDuration, beatDurations, group.length), movieDuration);
                 return;
             }
         }
         // Check if the current beat has media and the next beat does not have media.
         if (audioDuration > 0) {
             // Check if the current beat has spilled over audio.
-            const group = [index];
-            for (let i = index + 1; i < context.studio.beats.length && !mediaDurations[i].hasMedia; i++) {
-                group.push(i);
-            }
+            const group = getSpillOverGroup(context, mediaDurations, index);
             if (group.length > 1) {
-                const groupBeatsDurations = getGroupBeatDurations(context, group, audioDuration);
-                // Yes, the current beat has spilled over audio.
-                const beatsTotalDuration = groupBeatsDurations.reduce((a, b) => a + b, 0);
-                if (beatsTotalDuration > audioDuration + 0.01) {
-                    // 0.01 is a tolerance to avoid floating point precision issues
-                    group.reduce((remaining, idx, iGroup) => {
-                        if (remaining >= groupBeatsDurations[iGroup]) {
-                            return remaining - groupBeatsDurations[iGroup];
-                        }
-                        mediaDurations[idx].silenceDuration = groupBeatsDurations[iGroup] - remaining;
-                        return 0;
-                    }, audioDuration);
-                }
-                else {
-                    // Last beat gets the rest of the audio.
-                    if (audioDuration > beatsTotalDuration) {
-                        groupBeatsDurations[groupBeatsDurations.length - 1] += audioDuration - beatsTotalDuration;
-                    }
-                }
-                beatDurations.push(...groupBeatsDurations);
-            }
-            else {
-                // No spilled over audio.
-                assert(beatDurations.length === index, "beatDurations.length !== index");
-                // padding is the amount of audio padding specified in the script.
-                const padding = getPadding(context, beat, index);
-                // totalPadding is the amount of audio padding to be added to the audio file.
-                const totalPadding = Math.round(getTotalPadding(padding, movieDuration, audioDuration, beat.duration) * 100) / 100;
-                const beatDuration = audioDuration + totalPadding;
-                beatDurations.push(beatDuration);
-                if (totalPadding > 0) {
-                    mediaDurations[index].silenceDuration = totalPadding;
-                }
+                GraphAILogger.log(`Spill over group: ${group.length}`);
+                spilledOverAudio(context, group, audioDuration, beatDurations, mediaDurations);
+                return;
             }
+            // No spilled over audio.
+            assert(beatDurations.length === index, "beatDurations.length !== index");
+            noSpilledOverAudio(context, beat, index, movieDuration, audioDuration, beatDurations, mediaDurations);
+            return;
         }
-        else if (movieDuration > 0) {
+        if (movieDuration > 0) {
             // This beat has only a movie, not audio.
             beatDurations.push(movieDuration);
             mediaDurations[index].silenceDuration = movieDuration;
+            return;
         }
-        else {
-            // The current beat has no audio, nor no spilled over audio
-            const beatDuration = beat.duration ?? (movieDuration > 0 ? movieDuration : 1.0);
-            beatDurations.push(beatDuration);
-            mediaDurations[index].silenceDuration = beatDuration;
-        }
+        // The current beat has no audio, nor no spilled over audio
+        const beatDuration = beat.duration ?? (movieDuration > 0 ? movieDuration : 1.0);
+        beatDurations.push(beatDuration);
+        mediaDurations[index].silenceDuration = beatDuration;
     });
     assert(beatDurations.length === context.studio.beats.length, "beatDurations.length !== studio.beats.length");
     // We cannot reuse longSilentId. We need to explicitly split it for each beat.
@@ -170,20 +203,7 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
         const longSilentId = FfmpegContextInputFormattedAudio(ffmpegContext, silent60secPath(), undefined, ["-stream_loop", "-1"]);
         ffmpegContext.filterComplex.push(`${longSilentId}asplit=${silentIds.length}${silentIds.join("")}`);
     }
-    const inputIds = [];
-    context.studio.beats.forEach((studioBeat, index) => {
-        const { silenceDuration } = mediaDurations[index];
-        const paddingId = `[padding_${index}]`;
-        if (studioBeat.audioFile) {
-            const audioId = FfmpegContextInputFormattedAudio(ffmpegContext, studioBeat.audioFile);
-            inputIds.push(audioId);
-        }
-        if (silenceDuration > 0) {
-            const silentId = silentIds.pop();
-            ffmpegContext.filterComplex.push(`${silentId}atrim=start=0:end=${silenceDuration}${paddingId}`);
-            inputIds.push(paddingId);
-        }
-    });
+    const inputIds = getInputIds(context, mediaDurations, ffmpegContext, silentIds);
     assert(silentIds.length === 0, "silentIds.length !== 0");
     GraphAILogger.log("filterComplex:", ffmpegContext.filterComplex.join("\n"));
     // Finally, combine all audio files.

package/lib/agents/movie_replicate_agent.js

@@ -62,7 +62,7 @@ export const movieReplicateAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, imagePath } = namedInputs;
     const aspectRatio = getAspectRatio(params.canvasSize);
     const duration = params.duration ?? 5;
-    const apiKey = config?.apiKey ?? process.env.REPLICATE_API_TOKEN;
+    const apiKey = config?.apiKey;
     if (!apiKey) {
         throw new Error("REPLICATE_API_TOKEN environment variable is required");
     }

package/lib/agents/tts_elevenlabs_agent.js

@@ -3,7 +3,7 @@ import { provider2TTSAgent } from "../utils/provider2agent.js";
 export const ttsElevenlabsAgent = async ({ namedInputs, params, config, }) => {
     const { text } = namedInputs;
     const { voice, model, stability, similarityBoost, suppressError } = params;
-    const apiKey = config?.apiKey ?? process.env.ELEVENLABS_API_KEY;
+    const apiKey = config?.apiKey;
     if (!apiKey) {
         throw new Error("ELEVENLABS_API_KEY environment variable is required");
     }

package/lib/agents/tts_nijivoice_agent.js

@@ -1,5 +1,4 @@
 import { GraphAILogger, assert } from "graphai";
-const nijovoiceApiKey = process.env.NIJIVOICE_API_KEY ?? "";
 const errorMessage = [
     "TTS NijiVoice: No API key. ",
     "You have the following options:",
@@ -10,12 +9,12 @@ export const ttsNijivoiceAgent = async ({ params, namedInputs, config, }) => {
     const { suppressError, voice, speed, speed_global } = params;
     const { apiKey } = config ?? {};
     const { text } = namedInputs;
-    assert(!!(apiKey ?? nijovoiceApiKey), errorMessage);
+    assert(!!apiKey, errorMessage);
     const url = `https://api.nijivoice.com/api/platform/v1/voice-actors/${voice}/generate-voice`;
     const options = {
         method: "POST",
         headers: {
-            "x-api-key": apiKey ?? nijovoiceApiKey,
+            "x-api-key": apiKey,
             accept: "application/json",
             "content-type": "application/json",
         },

package/lib/index.browser.d.ts

@@ -1,4 +1,2 @@
-export * from "./types/type.js";
-export * from "./types/schema.js";
-export * from "./utils/provider2agent.js";
+export * from "./index.common.js";
 export * from "./agents/validate_schema_agent.js";

package/lib/index.browser.js

@@ -1,5 +1,3 @@
-// Entry point that exposes only APIs available for use in the browser
-export * from "./types/type.js";
-export * from "./types/schema.js";
-export * from "./utils/provider2agent.js";
+// Entry point for browser code
+export * from "./index.common.js";
 export * from "./agents/validate_schema_agent.js";

package/lib/index.common.d.ts

@@ -0,0 +1,2 @@
+export * from "./types/index.js";
+export * from "./utils/provider2agent.js";

package/lib/index.common.js

@@ -0,0 +1,3 @@
+// Entry point for universal code
+export * from "./types/index.js";
+export * from "./utils/provider2agent.js";

package/lib/index.node.d.ts

@@ -0,0 +1,7 @@
+export * from "./index.common.js";
+export * from "./actions/index.js";
+export * from "./cli/helpers.js";
+export * from "./utils/file.js";
+export * from "./utils/ffmpeg_utils.js";
+export * from "./methods/index.js";
+export * from "./agents/index.js";

package/lib/index.node.js

@@ -0,0 +1,8 @@
+// Entry point for node.js code
+export * from "./index.common.js";
+export * from "./actions/index.js";
+export * from "./cli/helpers.js";
+export * from "./utils/file.js";
+export * from "./utils/ffmpeg_utils.js";
+export * from "./methods/index.js";
+export * from "./agents/index.js";

package/lib/utils/image_plugins/image.d.ts

@@ -1,3 +1,3 @@
 export declare const imageType = "image";
-export declare const process: (params: import("../../index.browser.js").ImageProcessorParams) => string | undefined;
-export declare const path: (params: import("../../index.browser.js").ImageProcessorParams) => string | undefined;
+export declare const process: (params: import("../../index.common.js").ImageProcessorParams) => string | undefined;
+export declare const path: (params: import("../../index.common.js").ImageProcessorParams) => string | undefined;

package/lib/utils/image_plugins/movie.d.ts

@@ -1,3 +1,3 @@
 export declare const imageType = "movie";
-export declare const process: (params: import("../../index.browser.js").ImageProcessorParams) => string | undefined;
-export declare const path: (params: import("../../index.browser.js").ImageProcessorParams) => string | undefined;
+export declare const process: (params: import("../../index.common.js").ImageProcessorParams) => string | undefined;
+export declare const path: (params: import("../../index.common.js").ImageProcessorParams) => string | undefined;

package/lib/utils/utils.js

@@ -56,12 +56,18 @@ export const settings2GraphAIConfig = (settings, env) => {
             apiKey: getKey("IMAGE", "OPENAI_API_KEY"),
             baseURL: getKey("IMAGE", "OPENAI_BASE_URL"),
         },
+        imageGoogleAgent: {
+            projectId: getKey("IMAGE", "GOOGLE_PROJECT_ID"),
+        },
         anthropicAgent: {
             apiKey: getKey("LLM", "ANTHROPIC_API_TOKEN"),
         },
         movieReplicateAgent: {
             apiKey: getKey("MOVIE", "REPLICATE_API_TOKEN"),
         },
+        movieGoogleAgent: {
+            projectId: getKey("MOVIE", "GOOGLE_PROJECT_ID"),
+        },
         ttsNijivoiceAgent: {
             apiKey: getKey("TTS", "NIJIVOICE_API_KEY"),
         },

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "mulmocast",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "",
   "type": "module",
-  "main": "lib/index.js",
+  "main": "lib/index.node.js",
   "exports": {
     ".": {
-      "types": "./lib/index.d.ts",
-      "default": "./lib/index.js"
+      "types": "./lib/index.node.d.ts",
+      "default": "./lib/index.node.js"
     },
     "./browser": {
       "types": "./lib/index.browser.d.ts",
@@ -75,7 +75,7 @@
     "clipboardy": "^4.0.0",
     "dotenv": "^17.2.0",
     "fluent-ffmpeg": "^2.1.3",
-    "google-auth-library": "^9.15.1",
+    "google-auth-library": "^10.1.0",
     "graphai": "^2.0.12",
     "inquirer": "^12.7.0",
     "marked": "^16.0.0",