mulmocast 0.1.4 → 0.1.5

package/lib/actions/audio.js

@@ -34,12 +34,13 @@ const getAudioParam = (presentationStyle, beat) => {
     // Use speaker-specific provider if available, otherwise fall back to script-level provider
     const provider = MulmoPresentationStyleMethods.getTTSProvider(presentationStyle, beat);
     const speechOptions = MulmoPresentationStyleMethods.getSpeechOptions(presentationStyle, beat);
-    return { voiceId, provider, speechOptions };
+    const model = MulmoPresentationStyleMethods.getTTSModel(presentationStyle, beat);
+    return { voiceId, provider, speechOptions, model };
 };
 export const getBeatAudioPath = (text, context, beat, lang) => {
     const audioDirPath = MulmoStudioContextMethods.getAudioDirPath(context);
-    const { voiceId, provider, speechOptions } = getAudioParam(context.presentationStyle, beat);
-    const hash_string = [text, voiceId, speechOptions?.instruction ?? "", speechOptions?.speed ?? 1.0, provider].join(":");
+    const { voiceId, provider, speechOptions, model } = getAudioParam(context.presentationStyle, beat);
+    const hash_string = [text, voiceId, speechOptions?.instruction ?? "", speechOptions?.speed ?? 1.0, provider, model ?? ""].join(":");
     const audioFileName = `${context.studio.filename}_${text2hash(hash_string)}`;
     const audioFile = getAudioFilePath(audioDirPath, context.studio.filename, audioFileName, lang);
     return getAudioPath(context, beat, audioFile);
@@ -48,15 +49,16 @@ const preprocessor = (namedInputs) => {
     const { beat, studioBeat, multiLingual, context } = namedInputs;
     const { lang, presentationStyle } = context;
     const text = localizedText(beat, multiLingual, lang);
-    const { voiceId, provider, speechOptions } = getAudioParam(presentationStyle, beat);
+    const { voiceId, provider, speechOptions, model } = getAudioParam(presentationStyle, beat);
     const audioPath = getBeatAudioPath(text, context, beat, lang);
-    studioBeat.audioFile = audioPath; // TODO
+    studioBeat.audioFile = audioPath; // TODO: Passing by reference is difficult to maintain, so pass it using graphai inputs
     const needsTTS = !beat.audio && audioPath !== undefined;
     return {
         ttsAgent: provider2TTSAgent[provider].agentName,
         text,
         voiceId,
         speechOptions,
+        model,
         audioPath,
         studioBeat,
         needsTTS,
@@ -94,6 +96,7 @@ const graph_tts = {
                     voice: ":preprocessor.voiceId",
                     speed: ":preprocessor.speechOptions.speed",
                     instructions: ":preprocessor.speechOptions.instruction",
+                    model: ":preprocessor.model",
                 },
             },
         },

package/lib/actions/image_agents.d.ts

@@ -14,8 +14,8 @@ export declare const imagePreprocessAgent: (namedInputs: {
     referenceImageForMovie: string | undefined;
     imageParams: {
         provider: string;
-        style?: string | undefined;
         model?: string | undefined;
+        style?: string | undefined;
         moderation?: string | undefined;
         images?: Record<string, {
             type: "image";
@@ -58,8 +58,8 @@ export declare const imagePreprocessAgent: (namedInputs: {
     };
     imageParams: {
         provider: string;
-        style?: string | undefined;
         model?: string | undefined;
+        style?: string | undefined;
         moderation?: string | undefined;
         images?: Record<string, {
             type: "image";
@@ -105,8 +105,8 @@ export declare const imagePreprocessAgent: (namedInputs: {
     };
     imageParams: {
         provider: string;
-        style?: string | undefined;
         model?: string | undefined;
+        style?: string | undefined;
         moderation?: string | undefined;
         images?: Record<string, {
             type: "image";

package/lib/actions/image_references.js

@@ -1,5 +1,5 @@
 import fs from "fs";
-import { GraphAI } from "graphai";
+import { GraphAI, GraphAILogger } from "graphai";
 import { getReferenceImagePath } from "../utils/file.js";
 import { getExtention } from "../utils/utils.js";
 import { graphOption } from "./images.js";
@@ -13,6 +13,7 @@ export const generateReferenceImage = async (inputs) => {
     // generate image
     const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle);
     const prompt = `${image.prompt}\n${imageAgentInfo.imageParams.style || ""}`;
+    GraphAILogger.info(`Generating reference image for ${key}: ${prompt}`);
     const image_graph_data = {
         version: 0.5,
         nodes: {

package/lib/actions/images.js

@@ -11,7 +11,7 @@ import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../met
 import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
 import { userAssert, settings2GraphAIConfig } from "../utils/utils.js";
-import { extractImageFromMovie } from "../utils/ffmpeg_utils.js";
+import { extractImageFromMovie, ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
 import { getImageRefs } from "./image_references.js";
 import { imagePreprocessAgent, imagePluginAgent, htmlImageGeneratorAgent } from "./image_agents.js";
 const vanillaAgents = vanilla.default ?? vanilla;
@@ -166,16 +166,30 @@ const beat_graph_data = {
             },
             defaultValue: {},
         },
+        audioChecker: {
+            if: ":preprocessor.movieFile",
+            agent: async (namedInputs) => {
+                const { hasAudio } = await ffmpegGetMediaDuration(namedInputs.movieFile);
+                return { hasMovieAudio: hasAudio };
+            },
+            inputs: {
+                onComplete: [":movieGenerator"], // to wait for movieGenerator to finish
+                movieFile: ":preprocessor.movieFile",
+            },
+            defaultValue: {},
+        },
         output: {
             agent: "copyAgent",
             inputs: {
-                onComplete: [":imageFromMovie", ":htmlImageGenerator"], // to wait for imageFromMovie to finish
+                onComplete: [":imageFromMovie", ":htmlImageGenerator", ":audioChecker"], // to wait for imageFromMovie to finish
                 imageFile: ":preprocessor.imagePath",
                 movieFile: ":preprocessor.movieFile",
+                hasMovieAudio: ":audioChecker.hasMovieAudio",
             },
             output: {
                 imageFile: ".imageFile",
                 movieFile: ".movieFile",
+                hasMovieAudio: ".hasMovieAudio",
             },
             isResult: true,
         },

package/lib/actions/movie.js

@@ -204,8 +204,9 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, context) => {
             videoIdsForBeats.push(videoId);
         }
         // NOTE: We don't support audio if the speed is not 1.0.
-        if (beat.image?.type == "movie" && beat.image.mixAudio > 0.0 && speed === 1.0) {
-            const { audioId, audioPart } = getAudioPart(inputIndex, duration, timestamp, beat.image.mixAudio);
+        const movieVolume = beat.audioParams?.movieVolume ?? 1.0;
+        if (studioBeat.hasMovieAudio && movieVolume > 0.0 && speed === 1.0) {
+            const { audioId, audioPart } = getAudioPart(inputIndex, duration, timestamp, movieVolume);
             audioIdsFromMovieBeats.push(audioId);
             ffmpegContext.filterComplex.push(audioPart);
         }

package/lib/agents/add_bgm_agent.js

@@ -10,7 +10,7 @@ const addBGMAgent = async ({ namedInputs, params, }) => {
     if (!musicFile.match(/^http/) && !fs.existsSync(musicFile)) {
         throw new Error(`AddBGMAgent musicFile not exist: ${musicFile}`);
     }
-    const speechDuration = await ffmpegGetMediaDuration(voiceFile);
+    const { duration: speechDuration } = await ffmpegGetMediaDuration(voiceFile);
     const introPadding = context.presentationStyle.audioParams.introPadding;
     const outroPadding = context.presentationStyle.audioParams.outroPadding;
     const totalDuration = speechDuration + introPadding + outroPadding;

package/lib/agents/combine_audio_files_agent.js

@@ -2,13 +2,14 @@ import { assert, GraphAILogger } from "graphai";
 import { silent60secPath } from "../utils/file.js";
 import { FfmpegContextInit, FfmpegContextGenerateOutput, FfmpegContextInputFormattedAudio, ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
 import { userAssert } from "../utils/utils.js";
-const getMovieDulation = async (beat) => {
+const getMovieDuration = async (beat) => {
     if (beat.image?.type === "movie" && (beat.image.source.kind === "url" || beat.image.source.kind === "path")) {
         const pathOrUrl = beat.image.source.kind === "url" ? beat.image.source.url : beat.image.source.path;
         const speed = beat.movieParams?.speed ?? 1.0;
-        return (await ffmpegGetMediaDuration(pathOrUrl)) / speed;
+        const { duration, hasAudio } = await ffmpegGetMediaDuration(pathOrUrl);
+        return { duration: duration / speed, hasAudio };
     }
-    return 0;
+    return { duration: 0, hasAudio: false };
 };
 const getPadding = (context, beat, index) => {
     if (beat.audioParams?.padding !== undefined) {
@@ -29,16 +30,17 @@ const getTotalPadding = (padding, movieDuration, audioDuration, duration) => {
     }
     return padding;
 };
-const getMediaDurations = (context) => {
+const getMediaDurationsOfAllBeats = (context) => {
     return Promise.all(context.studio.beats.map(async (studioBeat, index) => {
         const beat = context.studio.script.beats[index];
-        const movieDuration = await getMovieDulation(beat);
-        const audioDuration = studioBeat.audioFile ? await ffmpegGetMediaDuration(studioBeat.audioFile) : 0;
+        const { duration: movieDuration, hasAudio: hasMovieAudio } = await getMovieDuration(beat);
+        const audioDuration = studioBeat.audioFile ? (await ffmpegGetMediaDuration(studioBeat.audioFile)).duration : 0;
         return {
             movieDuration,
             audioDuration,
             hasMedia: movieDuration + audioDuration > 0,
             silenceDuration: 0,
+            hasMovieAudio,
         };
     }));
 };
@@ -64,7 +66,7 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
     const { context, combinedFileName } = namedInputs;
     const ffmpegContext = FfmpegContextInit();
     // First, get the audio durations of all beats, taking advantage of multi-threading capability of ffmpeg.
-    const mediaDurations = await getMediaDurations(context);
+    const mediaDurations = await getMediaDurationsOfAllBeats(context);
     const beatDurations = [];
     context.studio.script.beats.forEach((beat, index) => {
         if (beatDurations.length > index) {
@@ -196,6 +198,7 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
                 audioDuration: mediaDurations[index].audioDuration,
                 movieDuration: mediaDurations[index].movieDuration,
                 silenceDuration: mediaDurations[index].silenceDuration,
+                hasMovieAudio: mediaDurations[index].hasMovieAudio,
             })),
         },
     };

package/lib/agents/tts_elevenlabs_agent.d.ts

@@ -1,4 +1,5 @@
 import type { AgentFunction, AgentFunctionInfo } from "graphai";
-export declare const ttsElevenlabsAgent: AgentFunction;
+import type { ElevenlabsTTSAgentParams, AgentBufferResult, AgentTextInputs, AgentErrorResult, AgentConfig } from "../types/agent.js";
+export declare const ttsElevenlabsAgent: AgentFunction<ElevenlabsTTSAgentParams, AgentBufferResult | AgentErrorResult, AgentTextInputs, AgentConfig>;
 declare const ttsElevenlabsAgentInfo: AgentFunctionInfo;
 export default ttsElevenlabsAgentInfo;

package/lib/agents/tts_elevenlabs_agent.js

@@ -1,5 +1,6 @@
 import { GraphAILogger } from "graphai";
-export const ttsElevenlabsAgent = async ({ namedInputs, params, config }) => {
+import { provider2TTSAgent } from "../utils/provider2agent.js";
+export const ttsElevenlabsAgent = async ({ namedInputs, params, config, }) => {
     const { text } = namedInputs;
     const { voice, model, stability, similarityBoost, suppressError } = params;
     const apiKey = config?.apiKey ?? process.env.ELEVENLABS_API_KEY;
@@ -7,12 +8,12 @@ export const ttsElevenlabsAgent = async ({ namedInputs, params, config }) => {
         throw new Error("ELEVENLABS_API_KEY environment variable is required");
     }
     if (!voice) {
-        throw new Error("Voice ID is required");
+        throw new Error("ELEVENLABS Voice ID is required");
     }
     try {
         const requestBody = {
             text,
-            model_id: model ?? "eleven_monolingual_v1",
+            model_id: model ?? provider2TTSAgent.elevenlabs.defaultModel,
             voice_settings: {
                 stability: stability ?? 0.5,
                 similarity_boost: similarityBoost ?? 0.75,

package/lib/agents/tts_google_agent.d.ts

@@ -1,12 +1,5 @@
 import type { AgentFunction, AgentFunctionInfo } from "graphai";
-export declare const ttsGoogleAgent: AgentFunction<{
-    voice: string;
-    speed: number;
-    suppressError: boolean;
-}, {
-    buffer?: Buffer | null;
-}, {
-    text: string;
-}>;
+import type { GoogleTTSAgentParams, AgentBufferResult, AgentTextInputs, AgentErrorResult } from "../types/agent.js";
+export declare const ttsGoogleAgent: AgentFunction<GoogleTTSAgentParams, AgentBufferResult | AgentErrorResult, AgentTextInputs>;
 declare const ttsGoogleAgentInfo: AgentFunctionInfo;
 export default ttsGoogleAgentInfo;

package/lib/agents/tts_nijivoice_agent.d.ts

@@ -1,4 +1,5 @@
 import type { AgentFunction, AgentFunctionInfo } from "graphai";
-export declare const ttsNijivoiceAgent: AgentFunction;
+import type { NijivoiceTTSAgentParams, AgentBufferResult, AgentTextInputs, AgentErrorResult, AgentConfig } from "../types/agent.js";
+export declare const ttsNijivoiceAgent: AgentFunction<NijivoiceTTSAgentParams, AgentBufferResult | AgentErrorResult, AgentTextInputs, AgentConfig>;
 declare const ttsNijivoiceAgentInfo: AgentFunctionInfo;
 export default ttsNijivoiceAgentInfo;

package/lib/agents/tts_nijivoice_agent.js

@@ -6,11 +6,11 @@ const errorMessage = [
     "1. Obtain an API key from Niji Voice (https://platform.nijivoice.com/) and set it as the NIJIVOICE_API_KEY environment variable.",
     '2. Use OpenAI\'s TTS instead of Niji Voice by changing speechParams.provider from "nijivoice" to "openai".',
 ].join("\n");
-export const ttsNijivoiceAgent = async ({ params, namedInputs, config }) => {
+export const ttsNijivoiceAgent = async ({ params, namedInputs, config, }) => {
     const { suppressError, voice, speed, speed_global } = params;
     const { apiKey } = config ?? {};
     const { text } = namedInputs;
-    assert(apiKey ?? nijovoiceApiKey, errorMessage);
+    assert(!!(apiKey ?? nijovoiceApiKey), errorMessage);
     const url = `https://api.nijivoice.com/api/platform/v1/voice-actors/${voice}/generate-voice`;
     const options = {
         method: "POST",
@@ -31,7 +31,7 @@ export const ttsNijivoiceAgent = async ({ params, namedInputs, config }) => {
         if (voiceJson && voiceJson.generatedVoice && voiceJson.generatedVoice.audioFileDownloadUrl) {
             const audioRes = await fetch(voiceJson.generatedVoice.audioFileDownloadUrl);
             const buffer = Buffer.from(await audioRes.arrayBuffer());
-            return { buffer, generatedVoice: voiceJson.generatedVoice };
+            return { buffer };
         }
         if (suppressError) {
             return {

package/lib/agents/tts_openai_agent.d.ts

@@ -1,16 +1,5 @@
 import type { AgentFunction, AgentFunctionInfo } from "graphai";
-export declare const ttsOpenaiAgent: AgentFunction<{
-    model: string;
-    voice: string;
-    instructions: string;
-    suppressError: boolean;
-}, {
-    buffer?: Buffer;
-}, {
-    text: string;
-}, {
-    baseURL?: string;
-    apiKey?: string;
-}>;
+import type { OpenAITTSAgentParams, AgentBufferResult, AgentTextInputs, AgentErrorResult, OpenAIImageAgentConfig } from "../types/agent.js";
+export declare const ttsOpenaiAgent: AgentFunction<OpenAITTSAgentParams, AgentBufferResult | AgentErrorResult, AgentTextInputs, OpenAIImageAgentConfig>;
 declare const ttsOpenaiAgentInfo: AgentFunctionInfo;
 export default ttsOpenaiAgentInfo;

package/lib/agents/tts_openai_agent.js

@@ -1,14 +1,15 @@
 import { GraphAILogger } from "graphai";
 import OpenAI from "openai";
-export const ttsOpenaiAgent = async ({ namedInputs, params, config }) => {
+import { provider2TTSAgent } from "../utils/provider2agent.js";
+export const ttsOpenaiAgent = async ({ namedInputs, params, config, }) => {
     const { text } = namedInputs;
     const { model, voice, suppressError, instructions } = params;
     const { apiKey, baseURL } = config ?? {};
     const openai = new OpenAI({ apiKey, baseURL });
     try {
         const tts_options = {
-            model: model ?? "gpt-4o-mini-tts", // "tts-1",
-            voice: voice ?? "shimmer",
+            model: model ?? provider2TTSAgent.openai.defaultModel,
+            voice: voice ?? provider2TTSAgent.openai.defaultVoice,
             input: text,
         };
         if (instructions) {

package/lib/index.browser.d.ts

@@ -1,3 +1,4 @@
 export * from "./types/type.js";
 export * from "./types/schema.js";
+export * from "./utils/provider2agent.js";
 export * from "./agents/validate_schema_agent.js";

package/lib/index.browser.js

@@ -1,4 +1,5 @@
 // Entry point that exposes only APIs available for use in the browser
 export * from "./types/type.js";
 export * from "./types/schema.js";
+export * from "./utils/provider2agent.js";
 export * from "./agents/validate_schema_agent.js";

package/lib/index.js

@@ -1,3 +1,4 @@
+// NOTE: If you want to support usage in the browser codebase, also add to src/index.browser.ts
 export * from "./actions/index.js";
 export * from "./cli/helpers.js";
 export * from "./utils/file.js";

package/lib/methods/mulmo_presentation_style.d.ts

@@ -8,6 +8,7 @@ export declare const MulmoPresentationStyleMethods: {
     getSpeechOptions(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SpeechOptions | undefined;
     getSpeaker(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SpeakerData;
     getTTSProvider(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): Text2SpeechProvider;
+    getTTSModel(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string | undefined;
     getVoiceId(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
     getText2ImageProvider(provider: Text2ImageProvider | undefined): Text2ImageProvider;
     getImageAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): Text2ImageAgentInfo;

package/lib/methods/mulmo_presentation_style.js

@@ -53,6 +53,10 @@ export const MulmoPresentationStyleMethods = {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
         return speaker.provider ?? presentationStyle.speechParams.provider;
     },
+    getTTSModel(presentationStyle, beat) {
+        const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
+        return speaker.model ?? presentationStyle.speechParams.model;
+    },
     getVoiceId(presentationStyle, beat) {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
         return speaker.voiceId;

package/lib/types/agent.d.ts

@@ -13,6 +13,15 @@ export type AgentBufferResult = {
 export type AgentPromptInputs = {
     prompt: string;
 };
+export type AgentTextInputs = {
+    text: string;
+};
+export type AgentErrorResult = {
+    error: unknown;
+};
+export type AgentConfig = {
+    apiKey?: string;
+};
 export type ImageAgentInputs = AgentPromptInputs;
 export type OpenAIImageAgentInputs = AgentPromptInputs & {
     referenceImages: string[] | null | undefined;
@@ -50,6 +59,24 @@ export type ReplicateMovieAgentParams = {
     duration?: number;
 };
 export type GoogleMovieAgentConfig = GoogleImageAgentConfig;
-export type ReplicateMovieAgentConfig = {
-    apiKey?: string;
+export type ReplicateMovieAgentConfig = AgentConfig;
+export type TTSAgentParams = {
+    suppressError: boolean;
+    voice: string;
+};
+export type OpenAITTSAgentParams = TTSAgentParams & {
+    instructions: string;
+    model: string;
+};
+export type NijivoiceTTSAgentParams = TTSAgentParams & {
+    speed: number;
+    speed_global: number;
+};
+export type GoogleTTSAgentParams = TTSAgentParams & {
+    speed: number;
+};
+export type ElevenlabsTTSAgentParams = TTSAgentParams & {
+    model: string;
+    stability: number;
+    similarityBoost: number;
 };

package/lib/types/agent.js

@@ -1,3 +1,2 @@
 // for image agent
 export {};
-// end of image agent