mulmocast 0.0.5 → 0.0.7

package/lib/actions/captions.js

@@ -0,0 +1,75 @@
+import { GraphAI, GraphAILogger } from "graphai";
+import * as agents from "@graphai/vanilla";
+import { getHTMLFile } from "../utils/file.js";
+import { renderHTMLToImage, interpolate } from "../utils/markdown.js";
+import { MulmoStudioMethods } from "../methods/mulmo_studio.js";
+const { default: __, ...vanillaAgents } = agents;
+const graph_data = {
+    version: 0.5,
+    nodes: {
+        context: {},
+        map: {
+            agent: "mapAgent",
+            inputs: { rows: ":context.studio.script.beats", context: ":context" },
+            isResult: true,
+            params: {
+                rowKey: "beat",
+                compositeResult: true,
+            },
+            graph: {
+                nodes: {
+                    generateCaption: {
+                        agent: async (namedInputs) => {
+                            const { beat, context, index } = namedInputs;
+                            try {
+                                MulmoStudioMethods.setBeatSessionState(context.studio, "caption", index, true);
+                                const { fileDirs } = namedInputs.context;
+                                const { caption } = context;
+                                const { imageDirPath } = fileDirs;
+                                const { canvasSize } = context.studio.script;
+                                const imagePath = `${imageDirPath}/${context.studio.filename}/${index}_caption.png`;
+                                const template = getHTMLFile("caption");
+                                const text = (() => {
+                                    const multiLingual = context.studio.multiLingual;
+                                    if (caption && multiLingual) {
+                                        return multiLingual[index].multiLingualTexts[caption].text;
+                                    }
+                                    GraphAILogger.warn(`No multiLingual caption found for beat ${index}, lang: ${caption}`);
+                                    return beat.text;
+                                })();
+                                const htmlData = interpolate(template, {
+                                    caption: text,
+                                    width: `${canvasSize.width}`,
+                                    height: `${canvasSize.height}`,
+                                });
+                                await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, false, true);
+                                context.studio.beats[index].captionFile = imagePath;
+                                return imagePath;
+                            }
+                            finally {
+                                MulmoStudioMethods.setBeatSessionState(context.studio, "caption", index, false);
+                            }
+                        },
+                        inputs: {
+                            beat: ":beat",
+                            context: ":context",
+                            index: ":__mapIndex",
+                        },
+                        isResult: true,
+                    },
+                },
+            },
+        },
+    },
+};
+export const captions = async (context) => {
+    try {
+        MulmoStudioMethods.setSessionState(context.studio, "caption", true);
+        const graph = new GraphAI(graph_data, { ...vanillaAgents });
+        graph.injectValue("context", context);
+        await graph.run();
+    }
+    finally {
+        MulmoStudioMethods.setSessionState(context.studio, "caption", false);
+    }
+};

package/lib/actions/images.js

@@ -8,10 +8,12 @@ import imageGoogleAgent from "../agents/image_google_agent.js";
 import imageOpenaiAgent from "../agents/image_openai_agent.js";
 import { MulmoScriptMethods } from "../methods/index.js";
 import { imagePlugins } from "../utils/image_plugins/index.js";
+import { imagePrompt } from "../utils/prompt.js";
 const { default: __, ...vanillaAgents } = agents;
 dotenv.config();
 // const openai = new OpenAI();
 import { GoogleAuth } from "google-auth-library";
+import { MulmoStudioMethods } from "../methods/mulmo_studio.js";
 const htmlStyle = (script, beat) => {
     return {
         canvasSize: MulmoScriptMethods.getCanvasSize(script),
@@ -29,18 +31,24 @@ const imagePreprocessAgent = async (namedInputs) => {
     if (beat.image) {
         const plugin = imagePlugins.find((plugin) => plugin.imageType === beat?.image?.type);
         if (plugin) {
-            const processorParams = { beat, context, imagePath, ...htmlStyle(context.studio.script, beat) };
-            const path = await plugin.process(processorParams);
-            // undefined prompt indicates that image generation is not needed
-            return { path, ...returnValue };
+            try {
+                MulmoStudioMethods.setBeatSessionState(context.studio, "image", index, true);
+                const processorParams = { beat, context, imagePath, ...htmlStyle(context.studio.script, beat) };
+                const path = await plugin.process(processorParams);
+                // undefined prompt indicates that image generation is not needed
+                return { path, ...returnValue };
+            }
+            finally {
+                MulmoStudioMethods.setBeatSessionState(context.studio, "image", index, false);
+            }
         }
     }
-    const prompt = (beat.imagePrompt || beat.text) + "\n" + (imageParams.style || "");
+    const prompt = imagePrompt(beat, imageParams.style);
     return { path: imagePath, prompt, ...returnValue };
 };
 const graph_data = {
     version: 0.5,
-    concurrency: 2,
+    concurrency: 4,
     nodes: {
         context: {},
         imageDirPath: {},
@@ -70,17 +78,21 @@ const graph_data = {
                     imageGenerator: {
                         if: ":preprocessor.prompt",
                         agent: ":imageAgentInfo.agent",
-                        params: {
-                            model: ":preprocessor.imageParams.model",
-                            size: ":preprocessor.imageParams.size",
-                            moderation: ":preprocessor.imageParams.moderation",
-                            aspectRatio: ":preprocessor.aspectRatio",
-                        },
+                        retry: 3,
                         inputs: {
                             prompt: ":preprocessor.prompt",
                             file: ":preprocessor.path", // only for fileCacheAgentFilter
                             text: ":preprocessor.prompt", // only for fileCacheAgentFilter
                             force: ":context.force",
+                            studio: ":context.studio", // for cache
+                            index: ":__mapIndex", // for cache
+                            sessionType: "image", // for cache
+                            params: {
+                                model: ":preprocessor.imageParams.model",
+                                size: ":preprocessor.imageParams.size",
+                                moderation: ":preprocessor.imageParams.moderation",
+                                aspectRatio: ":preprocessor.aspectRatio",
+                            },
                         },
                         defaultValue: {},
                     },
@@ -132,7 +144,7 @@ const googleAuth = async () => {
     const accessToken = await client.getAccessToken();
     return accessToken.token;
 };
-export const images = async (context) => {
+const generateImages = async (context) => {
     const { studio, fileDirs } = context;
     const { outDirPath, imageDirPath } = fileDirs;
     mkdir(`${imageDirPath}/${studio.filename}`);
@@ -171,3 +183,12 @@ export const images = async (context) => {
     });
     await graph.run();
 };
+export const images = async (context) => {
+    try {
+        MulmoStudioMethods.setSessionState(context.studio, "image", true);
+        await generateImages(context);
+    }
+    finally {
+        MulmoStudioMethods.setSessionState(context.studio, "image", false);
+    }
+};

package/lib/actions/index.d.ts

@@ -3,3 +3,4 @@ export * from "./images.js";
 export * from "./movie.js";
 export * from "./pdf.js";
 export * from "./translate.js";
+export * from "./captions.js";

package/lib/actions/index.js

@@ -3,3 +3,4 @@ export * from "./images.js";
 export * from "./movie.js";
 export * from "./pdf.js";
 export * from "./translate.js";
+export * from "./captions.js";

package/lib/actions/movie.js

@@ -1,26 +1,29 @@
-import ffmpeg from "fluent-ffmpeg";
 import { GraphAILogger } from "graphai";
 import { MulmoScriptMethods } from "../methods/index.js";
 import { getAudioArtifactFilePath, getOutputVideoFilePath, writingMessage } from "../utils/file.js";
-const isMac = process.platform === "darwin";
-const videoCodec = isMac ? "h264_videotoolbox" : "libx264";
+import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextPushFormattedAudio, FfmpegContextGenerateOutput } from "../utils/ffmpeg_utils.js";
+import { MulmoStudioMethods } from "../methods/mulmo_studio.js";
+// const isMac = process.platform === "darwin";
+const videoCodec = "libx264"; // "h264_videotoolbox" (macOS only) is too noisy
 export const getVideoPart = (inputIndex, mediaType, duration, canvasInfo) => {
     const videoId = `v${inputIndex}`;
+    const videoFilters = [];
+    // Handle different media types
+    if (mediaType === "image") {
+        videoFilters.push("loop=loop=-1:size=1:start=0");
+    }
+    else if (mediaType === "movie") {
+        // For videos, extend with last frame if shorter than required duration
+        // tpad will extend the video by cloning the last frame, then trim will ensure exact duration
+        videoFilters.push(`tpad=stop_mode=clone:stop_duration=${duration * 2}`); // Use 2x duration to ensure coverage
+    }
+    // Common filters for all media types
+    videoFilters.push(`trim=duration=${duration}`, "fps=30", "setpts=PTS-STARTPTS", `scale=w=${canvasInfo.width}:h=${canvasInfo.height}:force_original_aspect_ratio=decrease`, 
+    // In case of the aspect ratio mismatch, we fill the extra space with black color.
+    `pad=${canvasInfo.width}:${canvasInfo.height}:(ow-iw)/2:(oh-ih)/2:color=black`, "setsar=1", "format=yuv420p");
     return {
         videoId,
-        videoPart: `[${inputIndex}:v]` +
-            [
-                mediaType === "image" ? "loop=loop=-1:size=1:start=0" : "",
-                `trim=duration=${duration}`,
-                "fps=30",
-                "setpts=PTS-STARTPTS",
-                `scale=${canvasInfo.width}:${canvasInfo.height}`,
-                "setsar=1",
-                "format=yuv420p",
-            ]
-                .filter((a) => a)
-                .join(",") +
-            `[${videoId}]`,
+        videoPart: `[${inputIndex}:v]` + videoFilters.filter((a) => a).join(",") + `[${videoId}]`,
     };
 };
 export const getAudioPart = (inputIndex, duration, delay) => {
@@ -29,112 +32,110 @@ export const getAudioPart = (inputIndex, duration, delay) => {
         audioId,
         audioPart: `[${inputIndex}:a]` +
             `atrim=duration=${duration},` + // Trim to beat duration
-            `adelay=${delay}|${delay},` +
+            `adelay=${delay * 1000}|${delay * 1000},` +
             `aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo` +
             `[${audioId}]`,
     };
 };
 const getOutputOption = (audioId) => {
     return [
-        "-preset veryfast", // Faster encoding
+        "-preset medium", // Changed from veryfast to medium for better compression
         "-map [v]", // Map the video stream
         `-map ${audioId}`, // Map the audio stream
         `-c:v ${videoCodec}`, // Set video codec
+        ...(videoCodec === "libx264" ? ["-crf", "26"] : []), // Add CRF for libx264
         "-threads 8",
         "-filter_threads 8",
-        "-b:v 5M", // bitrate (only for videotoolbox)
+        "-b:v 2M", // Reduced from 5M to 2M
         "-bufsize",
-        "10M", // Add buffer size for better quality
+        "4M", // Reduced buffer size
         "-maxrate",
-        "7M", // Maximum bitrate
+        "3M", // Reduced from 7M to 3M
         "-r 30", // Set frame rate
         "-pix_fmt yuv420p", // Set pixel format for better compatibility
+        "-c:a aac", // Audio codec
+        "-b:a 128k", // Audio bitrate
     ];
 };
-const createVideo = (audioArtifactFilePath, outputVideoPath, studio) => {
-    return new Promise((resolve, reject) => {
-        const start = performance.now();
-        const ffmpegContext = {
-            command: ffmpeg(),
-            inputCount: 0,
-        };
-        function addInput(input) {
-            ffmpegContext.command = ffmpegContext.command.input(input);
-            ffmpegContext.inputCount++;
-            return ffmpegContext.inputCount - 1; // returned the index of the input
+const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, caption) => {
+    const start = performance.now();
+    const ffmpegContext = FfmpegContextInit();
+    if (studio.beats.some((beat) => !beat.imageFile)) {
+        GraphAILogger.info("beat.imageFile is not set. Please run `yarn run images ${file}` ");
+        return;
+    }
+    const canvasInfo = MulmoScriptMethods.getCanvasSize(studio.script);
+    // Add each image input
+    const filterComplexVideoIds = [];
+    const filterComplexAudioIds = [];
+    studio.beats.reduce((timestamp, beat, index) => {
+        if (!beat.imageFile || !beat.duration) {
+            throw new Error(`beat.imageFile or beat.duration is not set: index=${index}`);
         }
-        if (studio.beats.some((beat) => !beat.imageFile)) {
-            GraphAILogger.info("beat.imageFile is not set. Please run `yarn run images ${file}` ");
-            return;
-        }
-        const canvasInfo = MulmoScriptMethods.getCanvasSize(studio.script);
-        const padding = MulmoScriptMethods.getPadding(studio.script) / 1000;
-        // Add each image input
-        const filterComplexParts = [];
-        const filterComplexVideoIds = [];
-        const filterComplexAudioIds = [];
-        studio.beats.reduce((timestamp, beat, index) => {
-            if (!beat.imageFile || !beat.duration) {
-                throw new Error(`beat.imageFile is not set: index=${index}`);
-            }
-            const inputIndex = addInput(beat.imageFile);
-            const mediaType = MulmoScriptMethods.getImageType(studio.script, studio.script.beats[index]);
-            const headOrTail = index === 0 || index === studio.beats.length - 1;
-            const duration = beat.duration + (headOrTail ? padding : 0);
-            const { videoId, videoPart } = getVideoPart(inputIndex, mediaType, duration, canvasInfo);
-            filterComplexVideoIds.push(videoId);
-            filterComplexParts.push(videoPart);
-            if (mediaType === "movie") {
-                const { audioId, audioPart } = getAudioPart(inputIndex, duration, timestamp * 1000);
-                filterComplexAudioIds.push(audioId);
-                filterComplexParts.push(audioPart);
+        const inputIndex = FfmpegContextAddInput(ffmpegContext, beat.imageFile);
+        const mediaType = MulmoScriptMethods.getImageType(studio.script, studio.script.beats[index]);
+        const extraPadding = (() => {
+            // We need to consider only intro and outro padding because the other paddings were already added to the beat.duration
+            if (index === 0) {
+                return studio.script.audioParams.introPadding;
             }
-            return timestamp + duration;
-        }, 0);
-        // console.log("*** images", images.audioIds);
-        // Concatenate the trimmed images
-        filterComplexParts.push(`${filterComplexVideoIds.map((id) => `[${id}]`).join("")}concat=n=${studio.beats.length}:v=1:a=0[v]`);
-        const audioIndex = addInput(audioArtifactFilePath); // Add audio input
-        const artifactAudioId = `${audioIndex}:a`;
-        const ffmpegContextAudioId = (() => {
-            if (filterComplexAudioIds.length > 0) {
-                const mainAudioId = "mainaudio";
-                const compositeAudioId = "composite";
-                const audioIds = filterComplexAudioIds.map((id) => `[${id}]`).join("");
-                filterComplexParts.push(`[${artifactAudioId}]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo[${mainAudioId}]`);
-                filterComplexParts.push(`[${mainAudioId}]${audioIds}amix=inputs=${filterComplexAudioIds.length + 1}:duration=first:dropout_transition=2[${compositeAudioId}]`);
-                return `[${compositeAudioId}]`; // notice that we need to use [mainaudio] instead of mainaudio
+            else if (index === studio.beats.length - 1) {
+                return studio.script.audioParams.outroPadding;
             }
-            return artifactAudioId;
+            return 0;
         })();
-        // Apply the filter complex for concatenation and map audio input
-        ffmpegContext.command
-            .complexFilter(filterComplexParts)
-            .outputOptions(getOutputOption(ffmpegContextAudioId))
-            .on("start", (__cmdLine) => {
-            GraphAILogger.log("Started FFmpeg ..."); // with command:', cmdLine);
-        })
-            .on("error", (err, stdout, stderr) => {
-            GraphAILogger.error("Error occurred:", err);
-            GraphAILogger.error("FFmpeg stdout:", stdout);
-            GraphAILogger.error("FFmpeg stderr:", stderr);
-            GraphAILogger.info("Video creation failed. An unexpected error occurred.");
-            reject();
-        })
-            .on("end", () => {
-            const end = performance.now();
-            GraphAILogger.info(`Video created successfully! ${Math.round(end - start) / 1000} sec`);
-            resolve(0);
-        })
-            .output(outputVideoPath)
-            .run();
-    });
+        const duration = beat.duration + extraPadding;
+        const { videoId, videoPart } = getVideoPart(inputIndex, mediaType, duration, canvasInfo);
+        ffmpegContext.filterComplex.push(videoPart);
+        if (caption && beat.captionFile) {
+            const captionInputIndex = FfmpegContextAddInput(ffmpegContext, beat.captionFile);
+            const compositeVideoId = `c${index}`;
+            ffmpegContext.filterComplex.push(`[${videoId}][${captionInputIndex}:v]overlay=format=auto[${compositeVideoId}]`);
+            filterComplexVideoIds.push(compositeVideoId);
+        }
+        else {
+            filterComplexVideoIds.push(videoId);
+        }
+        if (mediaType === "movie") {
+            const { audioId, audioPart } = getAudioPart(inputIndex, duration, timestamp);
+            filterComplexAudioIds.push(audioId);
+            ffmpegContext.filterComplex.push(audioPart);
+        }
+        return timestamp + duration;
+    }, 0);
+    // console.log("*** images", images.audioIds);
+    // Concatenate the trimmed images
+    ffmpegContext.filterComplex.push(`${filterComplexVideoIds.map((id) => `[${id}]`).join("")}concat=n=${studio.beats.length}:v=1:a=0[v]`);
+    const audioIndex = FfmpegContextAddInput(ffmpegContext, audioArtifactFilePath); // Add audio input
+    const artifactAudioId = `${audioIndex}:a`;
+    const ffmpegContextAudioId = (() => {
+        if (filterComplexAudioIds.length > 0) {
+            const mainAudioId = "mainaudio";
+            const compositeAudioId = "composite";
+            const audioIds = filterComplexAudioIds.map((id) => `[${id}]`).join("");
+            FfmpegContextPushFormattedAudio(ffmpegContext, `[${artifactAudioId}]`, `[${mainAudioId}]`);
+            ffmpegContext.filterComplex.push(`[${mainAudioId}]${audioIds}amix=inputs=${filterComplexAudioIds.length + 1}:duration=first:dropout_transition=2[${compositeAudioId}]`);
+            return `[${compositeAudioId}]`; // notice that we need to use [mainaudio] instead of mainaudio
+        }
+        return artifactAudioId;
+    })();
+    await FfmpegContextGenerateOutput(ffmpegContext, outputVideoPath, getOutputOption(ffmpegContextAudioId));
+    const end = performance.now();
+    GraphAILogger.info(`Video created successfully! ${Math.round(end - start) / 1000} sec`);
+    GraphAILogger.info(studio.script.title);
+    GraphAILogger.info((studio.script.references ?? []).map((reference) => `${reference.title} (${reference.url})`).join("\n"));
 };
 export const movie = async (context) => {
-    const { studio, fileDirs } = context;
-    const { outDirPath } = fileDirs;
-    const audioArtifactFilePath = getAudioArtifactFilePath(outDirPath, studio.filename);
-    const outputVideoPath = getOutputVideoFilePath(outDirPath, studio.filename);
-    await createVideo(audioArtifactFilePath, outputVideoPath, studio);
-    writingMessage(outputVideoPath);
+    MulmoStudioMethods.setSessionState(context.studio, "video", true);
+    try {
+        const { studio, fileDirs, caption } = context;
+        const { outDirPath } = fileDirs;
+        const audioArtifactFilePath = getAudioArtifactFilePath(outDirPath, studio.filename);
+        const outputVideoPath = getOutputVideoFilePath(outDirPath, studio.filename, context.lang, caption);
+        await createVideo(audioArtifactFilePath, outputVideoPath, studio, caption);
+        writingMessage(outputVideoPath);
+    }
+    finally {
+        MulmoStudioMethods.setSessionState(context.studio, "video", false);
+    }
 };

package/lib/actions/pdf.js

@@ -2,10 +2,11 @@ import fs from "fs";
 import path from "path";
 import { rgb, PDFDocument } from "pdf-lib";
 import fontkit from "@pdf-lib/fontkit";
-import { chunkArray, isHttp } from "../utils/utils.js";
+import { chunkArray, isHttp, localizedText } from "../utils/utils.js";
 import { getOutputPdfFilePath, writingMessage } from "../utils/file.js";
 import { MulmoScriptMethods } from "../methods/index.js";
 import { fontSize, textMargin, drawSize, wrapText } from "../utils/pdf.js";
+import { MulmoStudioMethods } from "../methods/mulmo_studio.js";
 const imagesPerPage = 4;
 const offset = 10;
 const handoutImageRatio = 0.5;
@@ -19,7 +20,14 @@ const readImage = async (imagePath, pdfDoc) => {
         return fs.readFileSync(imagePath);
     })();
     const ext = path.extname(imagePath).toLowerCase();
-    return ext === ".jpg" || ext === ".jpeg" ? await pdfDoc.embedJpg(imageBytes) : await pdfDoc.embedPng(imageBytes);
+    if (ext === ".jpg" || ext === ".jpeg") {
+        return await pdfDoc.embedJpg(imageBytes);
+    }
+    if (ext === ".png") {
+        return await pdfDoc.embedPng(imageBytes);
+    }
+    // workaround. TODO: movie, image should convert to png/jpeg image
+    return await pdfDoc.embedPng(fs.readFileSync("assets/images/mulmocast_credit.png"));
 };
 const pdfSlide = async (pageWidth, pageHeight, imagePaths, pdfDoc) => {
     const cellRatio = pageHeight / pageWidth;
@@ -182,16 +190,19 @@ const outputSize = (pdfSize, isLandscapeImage, isRotate) => {
     }
     return { width: 612, height: 792 };
 };
-export const pdf = async (context, pdfMode, pdfSize) => {
-    const { studio, fileDirs } = context;
+const generatePdf = async (context, pdfMode, pdfSize) => {
+    const { studio, fileDirs, lang } = context;
+    const { multiLingual } = studio;
     const { outDirPath } = fileDirs;
     const { width: imageWidth, height: imageHeight } = MulmoScriptMethods.getCanvasSize(studio.script);
     const isLandscapeImage = imageWidth > imageHeight;
     const isRotate = pdfMode === "handout";
     const { width: pageWidth, height: pageHeight } = outputSize(pdfSize, isLandscapeImage, isRotate);
     const imagePaths = studio.beats.map((beat) => beat.imageFile);
-    const texts = studio.script.beats.map((beat) => beat.text);
-    const outputPdfPath = getOutputPdfFilePath(outDirPath, studio.filename, pdfMode);
+    const texts = studio.script.beats.map((beat, index) => {
+        return localizedText(beat, multiLingual?.[index], lang);
+    });
+    const outputPdfPath = getOutputPdfFilePath(outDirPath, studio.filename, pdfMode, lang);
     const pdfDoc = await PDFDocument.create();
     pdfDoc.registerFontkit(fontkit);
     const fontBytes = fs.readFileSync("assets/font/NotoSansJP-Regular.ttf");
@@ -209,3 +220,12 @@ export const pdf = async (context, pdfMode, pdfSize) => {
     fs.writeFileSync(outputPdfPath, pdfBytes);
     writingMessage(outputPdfPath);
 };
+export const pdf = async (context, pdfMode, pdfSize) => {
+    try {
+        MulmoStudioMethods.setSessionState(context.studio, "pdf", true);
+        await generatePdf(context, pdfMode, pdfSize);
+    }
+    finally {
+        MulmoStudioMethods.setSessionState(context.studio, "pdf", false);
+    }
+};