mulmocast 0.0.14 → 0.0.16

package/lib/actions/images.js

@@ -1,27 +1,27 @@
 import dotenv from "dotenv";
 import fs from "fs";
 import { GraphAI, GraphAILogger } from "graphai";
+import { TaskManager } from "graphai/lib/task_manager.js";
 import * as agents from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
-import imageGoogleAgent from "../agents/image_google_agent.js";
-import imageOpenaiAgent from "../agents/image_openai_agent.js";
-import movieGoogleAgent from "../agents/movie_google_agent.js";
-import { MulmoScriptMethods, MulmoStudioContextMethods } from "../methods/index.js";
+import { imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, mediaMockAgent } from "../agents/index.js";
+import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
 import { imagePlugins } from "../utils/image_plugins/index.js";
 import { imagePrompt } from "../utils/prompt.js";
 const vanillaAgents = agents.default ?? agents;
 dotenv.config();
 // const openai = new OpenAI();
 import { GoogleAuth } from "google-auth-library";
-const htmlStyle = (script, beat) => {
+import { extractImageFromMovie } from "../utils/ffmpeg_utils.js";
+const htmlStyle = (context, beat) => {
     return {
-        canvasSize: MulmoScriptMethods.getCanvasSize(script),
-        textSlideStyle: MulmoScriptMethods.getTextSlideStyle(script, beat),
+        canvasSize: MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle),
+        textSlideStyle: MulmoPresentationStyleMethods.getTextSlideStyle(context.presentationStyle, beat),
     };
 };
-const imagePreprocessAgent = async (namedInputs) => {
+export const imagePreprocessAgent = async (namedInputs) => {
     const { context, beat, index, suffix, imageDirPath, imageAgentInfo, imageRefs } = namedInputs;
     const imageParams = { ...imageAgentInfo.imageParams, ...beat.imageParams };
     const imagePath = `${imageDirPath}/${context.studio.filename}/${index}${suffix}.png`;
@@ -34,10 +34,10 @@ const imagePreprocessAgent = async (namedInputs) => {
         if (plugin) {
             try {
                 MulmoStudioContextMethods.setBeatSessionState(context, "image", index, true);
-                const processorParams = { beat, context, imagePath, ...htmlStyle(context.studio.script, beat) };
+                const processorParams = { beat, context, imagePath, ...htmlStyle(context, beat) };
                 const path = await plugin.process(processorParams);
                 // undefined prompt indicates that image generation is not needed
-                return { imagePath: path, ...returnValue };
+                return { imagePath: path, referenceImage: path, ...returnValue };
             }
             finally {
                 MulmoStudioContextMethods.setBeatSessionState(context, "image", index, false);
@@ -51,10 +51,102 @@ const imagePreprocessAgent = async (namedInputs) => {
         return sources.filter((source) => source !== undefined);
     })();
     if (beat.moviePrompt && !beat.imagePrompt) {
-        return { ...returnValue, images }; // no image prompt, only movie prompt
+        return { ...returnValue, imagePath, images, imageFromMovie: true }; // no image prompt, only movie prompt
     }
     const prompt = imagePrompt(beat, imageParams.style);
-    return { imagePath, prompt, ...returnValue, images };
+    return { imagePath, referenceImage: imagePath, prompt, ...returnValue, images };
+};
+const beat_graph_data = {
+    version: 0.5,
+    concurrency: 4,
+    nodes: {
+        context: {},
+        imageDirPath: {},
+        imageAgentInfo: {},
+        movieAgentInfo: {},
+        imageRefs: {},
+        beat: {},
+        __mapIndex: {},
+        preprocessor: {
+            agent: imagePreprocessAgent,
+            inputs: {
+                context: ":context",
+                beat: ":beat",
+                index: ":__mapIndex",
+                suffix: "p",
+                imageDirPath: ":imageDirPath",
+                imageAgentInfo: ":imageAgentInfo",
+                imageRefs: ":imageRefs",
+            },
+        },
+        imageGenerator: {
+            if: ":preprocessor.prompt",
+            agent: ":imageAgentInfo.agent",
+            retry: 3,
+            inputs: {
+                prompt: ":preprocessor.prompt",
+                images: ":preprocessor.images",
+                file: ":preprocessor.imagePath", // only for fileCacheAgentFilter
+                text: ":preprocessor.prompt", // only for fileCacheAgentFilter
+                force: ":context.force", // only for fileCacheAgentFilter
+                mulmoContext: ":context", // for fileCacheAgentFilter
+                index: ":__mapIndex", // for fileCacheAgentFilter
+                sessionType: "image", // for fileCacheAgentFilter
+                params: {
+                    model: ":preprocessor.imageParams.model",
+                    moderation: ":preprocessor.imageParams.moderation",
+                    canvasSize: ":context.presentationStyle.canvasSize",
+                },
+            },
+            defaultValue: {},
+        },
+        movieGenerator: {
+            if: ":preprocessor.movieFile",
+            agent: ":movieAgentInfo.agent",
+            inputs: {
+                onComplete: ":imageGenerator", // to wait for imageGenerator to finish
+                prompt: ":beat.moviePrompt",
+                imagePath: ":preprocessor.referenceImage",
+                file: ":preprocessor.movieFile",
+                studio: ":context.studio", // for cache
+                mulmoContext: ":context", // for fileCacheAgentFilter
+                index: ":__mapIndex", // for cache
+                sessionType: "movie", // for cache
+                params: {
+                    model: ":context.presentationStyle.movieParams.model",
+                    duration: ":beat.duration",
+                    canvasSize: ":context.presentationStyle.canvasSize",
+                },
+            },
+            defaultValue: {},
+        },
+        imageFromMovie: {
+            if: ":preprocessor.imageFromMovie",
+            agent: async (namedInputs) => {
+                await extractImageFromMovie(namedInputs.movieFile, namedInputs.imageFile);
+                return { generatedImage: true };
+            },
+            inputs: {
+                onComplete: ":movieGenerator", // to wait for movieGenerator to finish
+                imageFile: ":preprocessor.imagePath",
+                movieFile: ":preprocessor.movieFile",
+            },
+            defaultValue: { generatedImage: false },
+        },
+        output: {
+            agent: "copyAgent",
+            inputs: {
+                onComplete: ":imageFromMovie", // to wait for imageFromMovie to finish
+                imageFile: ":preprocessor.imagePath",
+                movieFile: ":preprocessor.movieFile",
+            },
+            output: {
+                imageFile: ".imageFile",
+                movieFile: ".movieFile",
+            },
+            isResult: true,
+        },
+    },
 };
 const graph_data = {
     version: 0.5,
@@ -63,6 +155,7 @@ const graph_data = {
         context: {},
         imageDirPath: {},
         imageAgentInfo: {},
+        movieAgentInfo: {},
         outputStudioFilePath: {},
         imageRefs: {},
         map: {
@@ -71,6 +164,7 @@ const graph_data = {
                 rows: ":context.studio.script.beats",
                 context: ":context",
                 imageAgentInfo: ":imageAgentInfo",
+                movieAgentInfo: ":movieAgentInfo",
                 imageDirPath: ":imageDirPath",
                 imageRefs: ":imageRefs",
             },
@@ -79,80 +173,10 @@ const graph_data = {
                 rowKey: "beat",
                 compositeResult: true,
             },
-            graph: {
-                nodes: {
-                    preprocessor: {
-                        agent: imagePreprocessAgent,
-                        inputs: {
-                            context: ":context",
-                            beat: ":beat",
-                            index: ":__mapIndex",
-                            suffix: "p",
-                            imageDirPath: ":imageDirPath",
-                            imageAgentInfo: ":imageAgentInfo",
-                            imageRefs: ":imageRefs",
-                        },
-                    },
-                    imageGenerator: {
-                        if: ":preprocessor.prompt",
-                        agent: ":imageAgentInfo.agent",
-                        retry: 3,
-                        inputs: {
-                            prompt: ":preprocessor.prompt",
-                            images: ":preprocessor.images",
-                            file: ":preprocessor.imagePath", // only for fileCacheAgentFilter
-                            text: ":preprocessor.prompt", // only for fileCacheAgentFilter
-                            force: ":context.force", // only for fileCacheAgentFilter
-                            mulmoContext: ":context", // for fileCacheAgentFilter
-                            index: ":__mapIndex", // for fileCacheAgentFilter
-                            sessionType: "image", // for fileCacheAgentFilter
-                            params: {
-                                model: ":preprocessor.imageParams.model",
-                                moderation: ":preprocessor.imageParams.moderation",
-                                canvasSize: ":context.studio.script.canvasSize",
-                            },
-                        },
-                        defaultValue: {},
-                    },
-                    movieGenerator: {
-                        if: ":preprocessor.movieFile",
-                        agent: "movieGoogleAgent",
-                        inputs: {
-                            onComplete: ":imageGenerator", // to wait for imageGenerator to finish
-                            prompt: ":beat.moviePrompt",
-                            imagePath: ":preprocessor.imagePath",
-                            file: ":preprocessor.movieFile",
-                            studio: ":context.studio", // for cache
-                            index: ":__mapIndex", // for cache
-                            sessionType: "movie", // for cache
-                            params: {
-                                model: ":context.studio.script.movieParams.model",
-                                duration: ":beat.duration",
-                                canvasSize: ":context.studio.script.canvasSize",
-                            },
-                        },
-                        defaultValue: {},
-                    },
-                    onComplete: {
-                        agent: "copyAgent",
-                        inputs: {
-                            onComplete: ":movieGenerator", // to wait for movieGenerator to finish
-                            imageFile: ":preprocessor.imagePath",
-                            movieFile: ":preprocessor.movieFile",
-                        },
-                    },
-                    output: {
-                        agent: "copyAgent",
-                        inputs: {
-                            imageFile: ":onComplete.imageFile",
-                            movieFile: ":onComplete.movieFile",
-                        },
-                        isResult: true,
-                    },
-                },
-            },
+            graph: beat_graph_data,
         },
         mergeResult: {
+            isResult: true,
             agent: (namedInputs) => {
                 const { array, context } = namedInputs;
                 const { studio } = context;
@@ -207,10 +231,7 @@ const googleAuth = async () => {
         throw error;
     }
 };
-const generateImages = async (context, callbacks) => {
-    const { studio, fileDirs } = context;
-    const { outDirPath, imageDirPath } = fileDirs;
-    mkdir(`${imageDirPath}/${studio.filename}`);
+const graphOption = async (context) => {
     const agentFilters = [
         {
             name: "fileCacheAgentFilter",
@@ -218,12 +239,14 @@ const generateImages = async (context, callbacks) => {
             nodeIds: ["imageGenerator", "movieGenerator"],
         },
     ];
+    const taskManager = new TaskManager(getConcurrency(context));
     const options = {
         agentFilters,
+        taskManager,
     };
-    const imageAgentInfo = MulmoScriptMethods.getImageAgentInfo(studio.script);
+    const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle);
     // We need to get google's auth token only if the google is the text2image provider.
-    if (imageAgentInfo.provider === "google" || studio.script.movieParams?.provider === "google") {
+    if (imageAgentInfo.provider === "google" || context.presentationStyle.movieParams?.provider === "google") {
         GraphAILogger.log("google was specified as text2image engine");
         const token = await googleAuth();
         options.config = {
@@ -237,14 +260,15 @@ const generateImages = async (context, callbacks) => {
             },
         };
     }
-    if (imageAgentInfo.provider === "openai") {
-        // NOTE: Here are the rate limits of OpenAI's text2image API (1token = 32x32 patch).
-        // dall-e-3: 7,500 RPM、15 images per minute (4 images for max resolution)
-        // gpt-image-1：3,000,000 TPM、150 images per minute
-        graph_data.concurrency = imageAgentInfo.imageParams.model === "dall-e-3" ? 4 : 16;
-    }
+    return options;
+};
+const prepareGenerateImages = async (context) => {
+    const { studio, fileDirs } = context;
+    const { outDirPath, imageDirPath } = fileDirs;
+    mkdir(`${imageDirPath}/${studio.filename}`);
+    const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle, context.dryRun);
     const imageRefs = {};
-    const images = studio.script.imageParams?.images;
+    const images = context.presentationStyle.imageParams?.images;
     if (images) {
         await Promise.all(Object.keys(images).map(async (key) => {
             const image = images[key];
@@ -285,11 +309,29 @@ const generateImages = async (context, callbacks) => {
     const injections = {
         context,
         imageAgentInfo,
+        movieAgentInfo: {
+            agent: context.dryRun ? "mediaMockAgent" : "movieGoogleAgent",
+        },
         outputStudioFilePath: getOutputStudioFilePath(outDirPath, studio.filename),
         imageDirPath,
         imageRefs,
     };
-    const graph = new GraphAI(graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, fileWriteAgent }, options);
+    return injections;
+};
+const getConcurrency = (context) => {
+    const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle);
+    if (imageAgentInfo.provider === "openai") {
+        // NOTE: Here are the rate limits of OpenAI's text2image API (1token = 32x32 patch).
+        // dall-e-3: 7,500 RPM、15 images per minute (4 images for max resolution)
+        // gpt-image-1：3,000,000 TPM、150 images per minute
+        return imageAgentInfo.imageParams.model === "dall-e-3" ? 4 : 16;
+    }
+    return 4;
+};
+const generateImages = async (context, callbacks) => {
+    const options = await graphOption(context);
+    const injections = await prepareGenerateImages(context);
+    const graph = new GraphAI(graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent }, options);
     Object.keys(injections).forEach((key) => {
         graph.injectValue(key, injections[key]);
     });
@@ -298,7 +340,8 @@ const generateImages = async (context, callbacks) => {
             graph.registerCallback(callback);
         });
     }
-    await graph.run();
+    const res = await graph.run();
+    return res.mergeResult;
 };
 export const images = async (context, callbacks) => {
     try {
@@ -309,3 +352,21 @@ export const images = async (context, callbacks) => {
         MulmoStudioContextMethods.setSessionState(context, "image", false);
     }
 };
+export const generateBeatImage = async (index, context, callbacks) => {
+    const options = await graphOption(context);
+    const injections = await prepareGenerateImages(context);
+    const graph = new GraphAI(beat_graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent }, options);
+    Object.keys(injections).forEach((key) => {
+        if ("outputStudioFilePath" !== key) {
+            graph.injectValue(key, injections[key]);
+        }
+    });
+    graph.injectValue("__mapIndex", index);
+    graph.injectValue("beat", context.studio.script.beats[index]);
+    if (callbacks) {
+        callbacks.forEach((callback) => {
+            graph.registerCallback(callback);
+        });
+    }
+    await graph.run();
+};

package/lib/actions/movie.js

@@ -1,6 +1,6 @@
 import { GraphAILogger, assert } from "graphai";
 import { mulmoTransitionSchema } from "../types/index.js";
-import { MulmoScriptMethods } from "../methods/index.js";
+import { MulmoPresentationStyleMethods } from "../methods/index.js";
 import { getAudioArtifactFilePath, getOutputVideoFilePath, writingMessage } from "../utils/file.js";
 import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextPushFormattedAudio, FfmpegContextGenerateOutput } from "../utils/ffmpeg_utils.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
@@ -59,22 +59,22 @@ const getOutputOption = (audioId, videoId) => {
         "-b:a 128k", // Audio bitrate
     ];
 };
-const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, caption) => {
+const createVideo = async (audioArtifactFilePath, outputVideoPath, context, caption) => {
     const start = performance.now();
     const ffmpegContext = FfmpegContextInit();
-    const missingIndex = studio.beats.findIndex((beat) => !beat.imageFile && !beat.movieFile);
+    const missingIndex = context.studio.beats.findIndex((beat) => !beat.imageFile && !beat.movieFile);
     if (missingIndex !== -1) {
         GraphAILogger.info(`ERROR: beat.imageFile or beat.movieFile is not set on beat ${missingIndex}.`);
         return false;
     }
-    const canvasInfo = MulmoScriptMethods.getCanvasSize(studio.script);
+    const canvasInfo = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
     // Add each image input
     const filterComplexVideoIds = [];
     const filterComplexAudioIds = [];
     const transitionVideoIds = [];
     const beatTimestamps = [];
-    studio.beats.reduce((timestamp, studioBeat, index) => {
-        const beat = studio.script.beats[index];
+    context.studio.beats.reduce((timestamp, studioBeat, index) => {
+        const beat = context.studio.script.beats[index];
         const sourceFile = studioBeat.movieFile ?? studioBeat.imageFile;
         if (!sourceFile) {
             throw new Error(`studioBeat.imageFile or studioBeat.movieFile is not set: index=${index}`);
@@ -83,14 +83,14 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, capti
             throw new Error(`studioBeat.duration is not set: index=${index}`);
         }
         const inputIndex = FfmpegContextAddInput(ffmpegContext, sourceFile);
-        const mediaType = studioBeat.movieFile ? "movie" : MulmoScriptMethods.getImageType(studio.script, beat);
+        const mediaType = studioBeat.movieFile ? "movie" : MulmoPresentationStyleMethods.getImageType(context.presentationStyle, beat);
         const extraPadding = (() => {
             // We need to consider only intro and outro padding because the other paddings were already added to the beat.duration
             if (index === 0) {
-                return studio.script.audioParams.introPadding;
+                return context.presentationStyle.audioParams.introPadding;
             }
-            else if (index === studio.beats.length - 1) {
-                return studio.script.audioParams.outroPadding;
+            else if (index === context.studio.beats.length - 1) {
+                return context.presentationStyle.audioParams.outroPadding;
             }
             return 0;
         })();
@@ -106,11 +106,18 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, capti
         else {
             filterComplexVideoIds.push(videoId);
         }
-        if (studio.script.movieParams?.transition && index < studio.beats.length - 1) {
+        if (context.presentationStyle.movieParams?.transition && index < context.studio.beats.length - 1) {
             const sourceId = filterComplexVideoIds.pop();
             ffmpegContext.filterComplex.push(`[${sourceId}]split=2[${sourceId}_0][${sourceId}_1]`);
             filterComplexVideoIds.push(`${sourceId}_0`);
-            transitionVideoIds.push(`${sourceId}_1`);
+            if (mediaType === "movie") {
+                // For movie beats, extract the last frame for transition
+                ffmpegContext.filterComplex.push(`[${sourceId}_1]reverse,select='eq(n,0)',reverse,tpad=stop_mode=clone:stop_duration=${duration},fps=30,setpts=PTS-STARTPTS[${sourceId}_2]`);
+                transitionVideoIds.push(`${sourceId}_2`);
+            }
+            else {
+                transitionVideoIds.push(`${sourceId}_1`);
+            }
         }
         if (beat.image?.type == "movie" && beat.image.mixAudio > 0.0) {
             const { audioId, audioPart } = getAudioPart(inputIndex, duration, timestamp, beat.image.mixAudio);
@@ -120,20 +127,19 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, capti
         beatTimestamps.push(timestamp);
         return timestamp + duration;
     }, 0);
-    assert(filterComplexVideoIds.length === studio.beats.length, "videoIds.length !== studio.beats.length");
-    assert(beatTimestamps.length === studio.beats.length, "beatTimestamps.length !== studio.beats.length");
+    assert(filterComplexVideoIds.length === context.studio.beats.length, "videoIds.length !== studio.beats.length");
+    assert(beatTimestamps.length === context.studio.beats.length, "beatTimestamps.length !== studio.beats.length");
     // console.log("*** images", images.audioIds);
     // Concatenate the trimmed images
     const concatVideoId = "concat_video";
-    ffmpegContext.filterComplex.push(`${filterComplexVideoIds.map((id) => `[${id}]`).join("")}concat=n=${studio.beats.length}:v=1:a=0[${concatVideoId}]`);
+    ffmpegContext.filterComplex.push(`${filterComplexVideoIds.map((id) => `[${id}]`).join("")}concat=n=${context.studio.beats.length}:v=1:a=0[${concatVideoId}]`);
     // Add tranditions if needed
     const mixedVideoId = (() => {
-        if (studio.script.movieParams?.transition && transitionVideoIds.length > 1) {
-            const transition = mulmoTransitionSchema.parse(studio.script.movieParams.transition);
+        if (context.presentationStyle.movieParams?.transition && transitionVideoIds.length > 0) {
+            const transition = mulmoTransitionSchema.parse(context.presentationStyle.movieParams.transition);
             return transitionVideoIds.reduce((acc, transitionVideoId, index) => {
                 const transitionStartTime = beatTimestamps[index + 1] - 0.05; // 0.05 is to avoid flickering
                 const processedVideoId = `${transitionVideoId}_f`;
-                // TODO: This mechanism does not work for video beats yet. It works only with image beats.
                 // If we can to add other transition types than fade, we need to add them here.
                 ffmpegContext.filterComplex.push(`[${transitionVideoId}]format=yuva420p,fade=t=out:d=${transition.duration}:alpha=1,setpts=PTS-STARTPTS+${transitionStartTime}/TB[${processedVideoId}]`);
                 const outputId = `${transitionVideoId}_o`;
@@ -156,11 +162,12 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, capti
         }
         return artifactAudioId;
     })();
+    // GraphAILogger.debug("filterComplex", ffmpegContext.filterComplex);
     await FfmpegContextGenerateOutput(ffmpegContext, outputVideoPath, getOutputOption(ffmpegContextAudioId, mixedVideoId));
     const end = performance.now();
     GraphAILogger.info(`Video created successfully! ${Math.round(end - start) / 1000} sec`);
-    GraphAILogger.info(studio.script.title);
-    GraphAILogger.info((studio.script.references ?? []).map((reference) => `${reference.title} (${reference.url})`).join("\n"));
+    GraphAILogger.info(context.studio.script.title);
+    GraphAILogger.info((context.studio.script.references ?? []).map((reference) => `${reference.title} (${reference.url})`).join("\n"));
     return true;
 };
 export const movieFilePath = (context) => {
@@ -174,7 +181,7 @@ export const movie = async (context) => {
         const { outDirPath } = fileDirs;
         const audioArtifactFilePath = getAudioArtifactFilePath(outDirPath, studio.filename);
         const outputVideoPath = movieFilePath(context);
-        if (await createVideo(audioArtifactFilePath, outputVideoPath, studio, caption)) {
+        if (await createVideo(audioArtifactFilePath, outputVideoPath, context, caption)) {
             writingMessage(outputVideoPath);
         }
     }

package/lib/actions/pdf.d.ts

@@ -1,2 +1,3 @@
 import { MulmoStudioContext, PDFMode, PDFSize } from "../types/index.js";
+export declare const pdfFilePath: (context: MulmoStudioContext, pdfMode: PDFMode) => string;
 export declare const pdf: (context: MulmoStudioContext, pdfMode: PDFMode, pdfSize: PDFSize) => Promise<void>;