mulmocast 1.2.42 → 1.2.44

package/lib/actions/audio.d.ts

@@ -1,6 +1,7 @@
 import type { GraphData } from "graphai";
 import { MulmoStudioContext, MulmoBeat, PublicAPIArgs } from "../types/index.js";
 export declare const getBeatAudioPath: (text: string, context: MulmoStudioContext, beat: MulmoBeat, lang?: string) => string | undefined;
+export declare const getBeatAudioPathOrUrl: (text: string, context: MulmoStudioContext, beat: MulmoBeat, lang?: string) => string | undefined;
 export declare const listLocalizedAudioPaths: (context: MulmoStudioContext) => (string | undefined)[];
 export declare const audio_graph_data: GraphData;
 export declare const generateBeatAudio: (index: number, context: MulmoStudioContext, args?: PublicAPIArgs & {

package/lib/actions/audio.js

@@ -9,38 +9,43 @@ import { getAudioArtifactFilePath, getAudioFilePath, getOutputStudioFilePath, re
 import { localizedText, settings2GraphAIConfig } from "../utils/utils.js";
 import { text2hash } from "../utils/utils_node.js";
 import { provider2TTSAgent } from "../utils/provider2agent.js";
+import { invalidAudioSourceError } from "../utils/error_cause.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
 import { MulmoMediaSourceMethods } from "../methods/mulmo_media_source.js";
 dotenv.config({ quiet: true });
 const vanillaAgents = agents.default ?? agents;
-const getAudioPath = (context, beat, audioFile) => {
+const getAudioPathOrUrl = (context, beat, maybeAudioFile) => {
     if (beat.audio?.type === "audio") {
-        const path = MulmoMediaSourceMethods.resolve(beat.audio.source, context);
-        if (path) {
-            return path;
+        const pathOrUrl = MulmoMediaSourceMethods.resolve(beat.audio.source, context);
+        if (pathOrUrl) {
+            return pathOrUrl;
         }
-        throw new Error("Invalid audio source");
+        throw new Error("Invalid audio source", { cause: invalidAudioSourceError(context.studio.script.beats.indexOf(beat)) });
     }
     if (beat.text === undefined || beat.text === "" || context.studio.script.audioParams.suppressSpeech) {
         return undefined; // It indicates that the audio is not needed.
     }
-    return audioFile;
+    return maybeAudioFile;
 };
+// for back forward compatible
 export const getBeatAudioPath = (text, context, beat, lang) => {
+    return getBeatAudioPathOrUrl(text, context, beat, lang);
+};
+export const getBeatAudioPathOrUrl = (text, context, beat, lang) => {
     const audioDirPath = MulmoStudioContextMethods.getAudioDirPath(context);
     const { voiceId, provider, speechOptions, model } = MulmoStudioContextMethods.getAudioParam(context, beat, lang);
     const hash_string = [text, voiceId, speechOptions?.instruction ?? "", speechOptions?.speed ?? 1.0, provider, model ?? ""].join(":");
-    GraphAILogger.log(`getBeatAudioPath [${hash_string}]`);
+    GraphAILogger.log(`getBeatAudioPathOrUrl [${hash_string}]`);
     const audioFileName = `${context.studio.filename}_${text2hash(hash_string)}`;
-    const audioFile = getAudioFilePath(audioDirPath, context.studio.filename, audioFileName, lang);
-    return getAudioPath(context, beat, audioFile);
+    const maybeAudioFile = getAudioFilePath(audioDirPath, context.studio.filename, audioFileName, lang);
+    return getAudioPathOrUrl(context, beat, maybeAudioFile);
 };
 export const listLocalizedAudioPaths = (context) => {
     const lang = context.lang ?? context.studio.script.lang;
     return context.studio.script.beats.map((beat, index) => {
         const multiLingual = context.multiLingual[index];
         const text = localizedText(beat, multiLingual, lang);
-        return getBeatAudioPath(text, context, beat, lang);
+        return getBeatAudioPathOrUrl(text, context, beat, lang);
     });
 };
 const preprocessorAgent = (namedInputs) => {
@@ -48,7 +53,7 @@ const preprocessorAgent = (namedInputs) => {
     // const { lang } = context;
     const text = localizedText(beat, multiLingual, lang);
     const { voiceId, provider, speechOptions, model } = MulmoStudioContextMethods.getAudioParam(context, beat, lang);
-    const audioPath = getBeatAudioPath(text, context, beat, lang);
+    const audioPath = getBeatAudioPathOrUrl(text, context, beat, lang);
     studioBeat.audioFile = audioPath; // TODO: Passing by reference is difficult to maintain, so pass it using graphai inputs
     const needsTTS = !beat.audio && audioPath !== undefined;
     return {

package/lib/actions/image_references.js

@@ -1,9 +1,7 @@
-import fs from "fs";
 import { GraphAI, GraphAILogger } from "graphai";
-import { getReferenceImagePath, resolveAssetPath } from "../utils/file.js";
-import { getExtention } from "../utils/utils.js";
+import { getReferenceImagePath } from "../utils/file.js";
 import { graphOption } from "./images.js";
-import { MulmoPresentationStyleMethods } from "../methods/index.js";
+import { MulmoPresentationStyleMethods, MulmoMediaSourceMethods } from "../methods/index.js";
 import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent, imageReplicateAgent } from "../agents/index.js";
 // public api
 // Application may call this function directly to generate reference image.
@@ -43,18 +41,6 @@ export const generateReferenceImage = async (inputs) => {
     await graph.run();
     return imagePath;
 };
-const downLoadImage = async (context, key, url) => {
-    const response = await fetch(url);
-    if (!response.ok) {
-        throw new Error(`Failed to download image: ${url}`);
-    }
-    const buffer = Buffer.from(await response.arrayBuffer());
-    // Detect file extension from Content-Type header or URL
-    const extension = getExtention(response.headers.get("content-type"), url);
-    const imagePath = getReferenceImagePath(context, key, extension);
-    await fs.promises.writeFile(imagePath, buffer);
-    return imagePath;
-};
 export const getImageRefs = async (context) => {
     const images = context.presentationStyle.imageParams?.images;
     if (!images) {
@@ -69,12 +55,7 @@ export const getImageRefs = async (context) => {
             imageRefs[key] = await generateReferenceImage({ context, key, index, image, force: false });
         }
         else if (image.type === "image") {
-            if (image.source.kind === "path") {
-                imageRefs[key] = resolveAssetPath(context, image.source.path);
-            }
-            else if (image.source.kind === "url") {
-                imageRefs[key] = await downLoadImage(context, key, image.source.url);
-            }
+            imageRefs[key] = await MulmoMediaSourceMethods.imageReference(image.source, context, key);
         }
     }));
     return imageRefs;

package/lib/actions/images.js

@@ -10,6 +10,7 @@ import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../met
 import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
 import { settings2GraphAIConfig } from "../utils/utils.js";
+import { audioCheckerError } from "../utils/error_cause.js";
 import { extractImageFromMovie, ffmpegGetMediaDuration, trimMusic } from "../utils/ffmpeg_utils.js";
 import { getImageRefs } from "./image_references.js";
 import { imagePreprocessAgent, imagePluginAgent, htmlImageGeneratorAgent } from "./image_agents.js";
@@ -201,13 +202,8 @@ export const beat_graph_data = {
                 }
                 catch (error) {
                     GraphAILogger.error(error);
-                    throw new Error("audioChecker: ffmpegGetMediaDuration error.", {
-                        cause: {
-                            type: "FileNotExist",
-                            action: "images",
-                            agentName: "audioChecker",
-                            beat_index: namedInputs.index,
-                        },
+                    throw new Error(`audioChecker: ffmpegGetMediaDuration error:  index=${namedInputs.index} file=${sourceFile}`, {
+                        cause: audioCheckerError(namedInputs.index, sourceFile),
                     });
                 }
             },

package/lib/actions/movie.js

@@ -2,6 +2,7 @@ import { GraphAILogger, assert } from "graphai";
 import { mulmoTransitionSchema, mulmoFillOptionSchema } from "../types/index.js";
 import { MulmoPresentationStyleMethods } from "../methods/index.js";
 import { getAudioArtifactFilePath, getOutputVideoFilePath, writingMessage, isFile } from "../utils/file.js";
+import { createVideoFileError, createVideoSourceError } from "../utils/error_cause.js";
 import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextPushFormattedAudio, FfmpegContextGenerateOutput, } from "../utils/ffmpeg_utils.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
 // const isMac = process.platform === "darwin";
@@ -163,8 +164,8 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, context) => {
             return timestamp; // Skip voice-over beats.
         }
         const sourceFile = studioBeat.lipSyncFile ?? studioBeat.soundEffectFile ?? studioBeat.movieFile ?? studioBeat.htmlImageFile ?? studioBeat.imageFile;
-        assert(!!sourceFile, `studioBeat.imageFile or studioBeat.movieFile is not set: index=${index}`);
-        assert(isFile(sourceFile), `studioBeat.imageFile or studioBeat.movieFile is not exist or not file: index=${index}`);
+        assert(!!sourceFile, `studioBeat.imageFile or studioBeat.movieFile is not set: index=${index}`, false, createVideoSourceError(index));
+        assert(isFile(sourceFile), `studioBeat.imageFile or studioBeat.movieFile is not exist or not file: index=${index} file=${sourceFile}`, false, createVideoFileError(index, sourceFile));
         assert(!!studioBeat.duration, `studioBeat.duration is not set: index=${index}`);
         const extraPadding = (() => {
             // We need to consider only intro and outro padding because the other paddings were already added to the beat.duration

package/lib/actions/translate.js

@@ -241,7 +241,7 @@ export const translateBeat = async (index, context, targetLangs, args) => {
     try {
         const { outputMultilingualFilePath } = getOutputMultilingualFilePathAndMkdir(context);
         const config = settings2GraphAIConfig(settings, process.env);
-        assert(!!config?.openAIAgent?.apiKey, "The OPENAI_API_KEY environment variable is missing or empty");
+        assert(!!config?.openAIAgent?.apiKey, "The OPENAI_API_KEY environment variable is missing or empty"); // TODO: cause
         const graph = new GraphAI(beatGraph, { ...vanillaAgents, fileWriteAgent, openAIAgent }, { agentFilters, config });
         graph.injectValue("context", context);
         graph.injectValue("targetLangs", targetLangs);
@@ -276,7 +276,7 @@ export const translate = async (context, args) => {
             ? args?.targetLangs
             : [...new Set([context.lang, context.studio.script.captionParams?.lang].filter((x) => !isNull(x)))];
         const config = settings2GraphAIConfig(settings, process.env);
-        assert(!!config?.openAIAgent?.apiKey, "The OPENAI_API_KEY environment variable is missing or empty");
+        assert(!!config?.openAIAgent?.apiKey, "The OPENAI_API_KEY environment variable is missing or empty"); // TODO: cause
         const graph = new GraphAI(translate_graph_data, { ...vanillaAgents, fileWriteAgent, openAIAgent }, { agentFilters, config });
         graph.injectValue("context", context);
         graph.injectValue("targetLangs", targetLangs);

package/lib/agents/add_bgm_agent.js

@@ -1,14 +1,14 @@
-import fs from "fs";
 import { GraphAILogger } from "graphai";
 import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextGenerateOutput, ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
+import { isFile } from "../utils/file.js";
 const addBGMAgent = async ({ namedInputs, params, }) => {
     const { voiceFile, outputFile, context } = namedInputs;
     const { musicFile } = params;
-    if (!fs.existsSync(voiceFile)) {
+    if (!isFile(voiceFile)) {
         throw new Error(`AddBGMAgent voiceFile not exist: ${voiceFile}`);
     }
-    if (!musicFile.match(/^http/) && !fs.existsSync(musicFile)) {
+    if (!musicFile.match(/^http/) && !isFile(musicFile)) {
         throw new Error(`AddBGMAgent musicFile not exist: ${musicFile}`);
     }
     const { duration: speechDuration } = await ffmpegGetMediaDuration(voiceFile);

package/lib/agents/combine_audio_files_agent.js

@@ -1,8 +1,9 @@
 import { assert, GraphAILogger } from "graphai";
-import { silent60secPath } from "../utils/file.js";
+import { silent60secPath, isFile } from "../utils/file.js";
 import { FfmpegContextInit, FfmpegContextGenerateOutput, FfmpegContextInputFormattedAudio, ffmpegGetMediaDuration, } from "../utils/ffmpeg_utils.js";
 import { MulmoMediaSourceMethods } from "../methods/mulmo_media_source.js";
 import { userAssert } from "../utils/utils.js";
+import { getAudioInputIdsError } from "../utils/error_cause.js";
 const getMovieDuration = async (context, beat) => {
     if (beat.image?.type === "movie") {
         const pathOrUrl = MulmoMediaSourceMethods.resolve(beat.image.source, context);
@@ -71,6 +72,9 @@ const getInputIds = (context, mediaDurations, ffmpegContext, silentIds) => {
         const { silenceDuration } = mediaDurations[index];
         const paddingId = `[padding_${index}]`;
         if (studioBeat.audioFile) {
+            if (!/^https?:\/\//.test(studioBeat.audioFile)) {
+                assert(isFile(studioBeat.audioFile), `studioBeat.audioFile is not exist or not file: index=${index} file=${studioBeat.audioFile}`, false, getAudioInputIdsError(index, studioBeat.audioFile));
+            }
             const audioId = FfmpegContextInputFormattedAudio(ffmpegContext, studioBeat.audioFile);
             inputIds.push(audioId);
         }

package/lib/methods/mulmo_beat.js

@@ -9,7 +9,7 @@ export const MulmoBeatMethods = {
     getPlugin(beat) {
         const plugin = findImagePlugin(beat?.image?.type);
         if (!plugin) {
-            throw new Error(`invalid beat image type: ${beat.image}`);
+            throw new Error(`invalid beat image type: ${beat.image}`); // TODO: cause
         }
         return plugin;
     },

package/lib/methods/mulmo_media_source.d.ts

@@ -1,5 +1,9 @@
-import { MulmoMediaSource, MulmoStudioContext } from "../types/index.js";
+import type { MulmoMediaSource, MulmoMediaMermaidSource, MulmoStudioContext, ImageType } from "../types/index.js";
+export declare const getExtention: (contentType: string | null, url: string) => string;
 export declare const MulmoMediaSourceMethods: {
-    getText(mediaSource: MulmoMediaSource, context: MulmoStudioContext): Promise<string | null>;
+    getText(mediaSource: MulmoMediaMermaidSource, context: MulmoStudioContext): Promise<string | null>;
     resolve(mediaSource: MulmoMediaSource | undefined, context: MulmoStudioContext): string | null;
+    imageReference(mediaSource: MulmoMediaSource, context: MulmoStudioContext, key: string): Promise<string>;
+    imagePluginSource(mediaSource: MulmoMediaSource, context: MulmoStudioContext, expectImagePath: string, imageType: ImageType): Promise<string>;
+    imagePluginSourcePath(mediaSource: MulmoMediaSource, context: MulmoStudioContext, expectImagePath: string, imageType: ImageType): string | undefined;
 };

package/lib/methods/mulmo_media_source.js

@@ -1,16 +1,52 @@
 import fs from "fs";
-import { getFullPath, resolveAssetPath } from "../utils/file.js";
+import { GraphAILogger, assert } from "graphai";
+import { getFullPath, getReferenceImagePath, resolveAssetPath } from "../utils/file.js";
+import { downLoadReferenceImageError, getTextError, imageReferenceUnknownMediaError, downloadImagePluginError, imagePluginUnknownMediaError, } from "../utils/error_cause.js";
+// for image reference
+export const getExtention = (contentType, url) => {
+    if (contentType?.includes("jpeg") || contentType?.includes("jpg")) {
+        return "jpg";
+    }
+    else if (contentType?.includes("png")) {
+        return "png";
+    }
+    // Fall back to URL extension
+    const urlExtension = url.split(".").pop()?.toLowerCase();
+    if (urlExtension && ["jpg", "jpeg", "png"].includes(urlExtension)) {
+        return urlExtension === "jpeg" ? "jpg" : urlExtension;
+    }
+    return "png"; // default
+};
+const downLoadReferenceImage = async (context, key, url) => {
+    const response = await fetch(url);
+    assert(response.ok, `Failed to download reference image: ${url}`, false, downLoadReferenceImageError(key, url));
+    const buffer = Buffer.from(await response.arrayBuffer());
+    // Detect file extension from Content-Type header or URL
+    const extension = getExtention(response.headers.get("content-type"), url);
+    const imagePath = getReferenceImagePath(context, key, extension);
+    await fs.promises.writeFile(imagePath, buffer);
+    return imagePath;
+};
+// for image
+function pluginSourceFixExtention(path, imageType) {
+    if (imageType === "movie") {
+        if (!path.endsWith(".png")) {
+            GraphAILogger.warn(`Expected .png extension for movie type, got: ${path}`);
+        }
+        return path.replace(/\.png$/, ".mov");
+    }
+    return path;
+}
+// end of util
 export const MulmoMediaSourceMethods = {
     async getText(mediaSource, context) {
         if (mediaSource.kind === "text") {
             return mediaSource.text;
         }
         if (mediaSource.kind === "url") {
-            const res = await fetch(mediaSource.url);
-            if (!res.ok) {
-                throw new Error(`Failed to fetch media source: ${mediaSource.url}`);
-            }
-            return await res.text();
+            const response = await fetch(mediaSource.url);
+            assert(response.ok, `Failed to download mermaid code text: ${mediaSource.url}`, false, getTextError(mediaSource.url)); // TODO: index
+            return await response.text();
         }
         if (mediaSource.kind === "path") {
             const path = getFullPath(context.fileDirs.mulmoFileDirPath, mediaSource.path);
@@ -29,4 +65,43 @@ export const MulmoMediaSourceMethods = {
         }
         return null;
     },
+    // if url then download image and save it to file. both case return local image path. For image reference
+    async imageReference(mediaSource, context, key) {
+        if (mediaSource.kind === "path") {
+            return resolveAssetPath(context, mediaSource.path);
+        }
+        else if (mediaSource.kind === "url") {
+            return await downLoadReferenceImage(context, key, mediaSource.url);
+        }
+        // TODO base64
+        throw new Error(`imageReference media unknown error`, { cause: imageReferenceUnknownMediaError(key) });
+    },
+    async imagePluginSource(mediaSource, context, expectImagePath, imageType) {
+        if (mediaSource.kind === "url") {
+            const response = await fetch(mediaSource.url);
+            assert(response.ok, `Failed to download image plugin: ${imageType} ${mediaSource.url}`, false, downloadImagePluginError(mediaSource.url, imageType)); // TODO: key, id, index
+            const buffer = Buffer.from(await response.arrayBuffer());
+            // Detect file extension from Content-Type header or URL
+            const imagePath = pluginSourceFixExtention(expectImagePath, imageType);
+            await fs.promises.writeFile(imagePath, buffer);
+            return imagePath;
+        }
+        const path = MulmoMediaSourceMethods.resolve(mediaSource, context);
+        if (path) {
+            return path;
+        }
+        // base64??
+        GraphAILogger.error(`Image Plugin unknown ${imageType} source type:`, mediaSource);
+        throw new Error(`ERROR: unknown ${imageType} source type`, { cause: imagePluginUnknownMediaError(imageType) }); // TODO index
+    },
+    imagePluginSourcePath(mediaSource, context, expectImagePath, imageType) {
+        if (mediaSource?.kind === "url") {
+            return pluginSourceFixExtention(expectImagePath, imageType);
+        }
+        const path = MulmoMediaSourceMethods.resolve(mediaSource, context);
+        if (path) {
+            return path;
+        }
+        return undefined;
+    },
 };