mulmocast 1.2.66 → 1.2.68

package/lib/actions/bundle.js

@@ -2,38 +2,66 @@ import path from "path";
 import fs from "fs";
 import { GraphAILogger } from "graphai";
 import { listLocalizedAudioPaths } from "./audio.js";
-import { imagePreprocessAgent } from "./image_agents.js";
 import { mkdir } from "../utils/file.js";
 import { ZipBuilder } from "../utils/zip.js";
 import { bundleTargetLang } from "../utils/const.js";
-const beatImage = (context) => {
-    return async (beat, index) => {
-        try {
-            const res = await imagePreprocessAgent({ context, beat, index, imageRefs: {} });
-            if ("htmlPrompt" in res) {
-                return { htmlImageSource: res.htmlImageFile, imageSource: res.imagePath };
-            }
-            const { imagePath, movieFile, lipSyncFile } = res;
-            return { imageSource: imagePath, videoSource: movieFile, videoWithAudioSource: lipSyncFile };
-        }
-        catch (e) {
-            GraphAILogger.log(e);
-            return {};
+import { createSilentAudio } from "../utils/ffmpeg_utils.js";
+const downloadFile = async (url, destPath) => {
+    const response = await fetch(url);
+    if (!response.ok) {
+        throw new Error(`Failed to download file from ${url}: ${response.statusText}`);
+    }
+    const buffer = await response.arrayBuffer();
+    fs.writeFileSync(destPath, Buffer.from(buffer));
+};
+const processBgm = async (bgm, dir, zipper) => {
+    if (!bgm) {
+        return undefined;
+    }
+    if (bgm.kind === "path") {
+        // Local file path
+        const sourcePath = path.resolve(bgm.path);
+        if (!fs.existsSync(sourcePath)) {
+            GraphAILogger.log(`BGM file not found: ${sourcePath}`);
+            return undefined;
         }
-    };
+        const fileName = path.basename(bgm.path);
+        const destPath = path.resolve(dir, fileName);
+        fs.copyFileSync(sourcePath, destPath);
+        zipper.addFile(sourcePath, fileName);
+        return fileName;
+    }
+    else if (bgm.kind === "url") {
+        // URL download
+        const fileName = path.basename(new URL(bgm.url).pathname) || "bgm.mp3";
+        const destPath = path.resolve(dir, fileName);
+        await downloadFile(bgm.url, destPath);
+        zipper.addFile(destPath);
+        return fileName;
+    }
+    // base64 or other formats are not supported
+    return undefined;
 };
-// TODO reference
 const viewJsonFileName = "mulmo_view.json";
 const zipFileName = "mulmo.zip";
+const imageSourceMappings = [
+    ["imageFile", "imageSource"],
+    ["movieFile", "videoSource"],
+    ["soundEffectFile", "soundEffectSource"],
+    ["lipSyncFile", "videoWithAudioSource"],
+    ["htmlImageFile", "htmlImageSource"],
+];
 export const mulmoViewerBundle = async (context) => {
     const isZip = true;
     const dir = path.resolve(context.fileDirs.fileName);
     mkdir(dir);
     const zipper = new ZipBuilder(path.resolve(dir, zipFileName));
+    // text
     const resultJson = [];
     context.studio.script.beats.forEach((beat) => {
         resultJson.push({ text: beat.text, duration: beat.duration, audioSources: {}, multiLinguals: {} });
     });
+    // audio
     for (const lang of bundleTargetLang) {
         const audios = listLocalizedAudioPaths({ ...context, lang });
         audios.forEach((audio, index) => {
@@ -49,14 +77,13 @@ export const mulmoViewerBundle = async (context) => {
             }
         });
     }
-    const images = await Promise.all(context.studio.script.beats.map(beatImage(context)));
-    images.forEach((image, index) => {
+    // image, movie
+    context.studio.beats.forEach((image, index) => {
         const data = resultJson[index];
-        const keys = ["htmlImageSource", "imageSource", "videoSource", "videoWithAudioSource"];
-        keys.forEach((key) => {
+        imageSourceMappings.forEach(([key, source]) => {
             const value = image[key];
-            if (value) {
-                data[key] = path.basename(value);
+            if (typeof value === "string") {
+                data[source] = path.basename(value);
                 if (fs.existsSync(value)) {
                     fs.copyFileSync(value, path.resolve(dir, path.basename(value)));
                     zipper.addFile(value);
@@ -64,6 +91,23 @@ export const mulmoViewerBundle = async (context) => {
             }
         });
     });
+    // silent
+    await Promise.all(context.studio.script.beats.map(async (__, index) => {
+        const data = resultJson[index];
+        if (data.audioSources &&
+            Object.keys(data.audioSources).length === 0 &&
+            data.videoSource === undefined &&
+            data.videoWithAudioSource === undefined &&
+            data.duration) {
+            const file = `silent_${index}.mp3`;
+            const audioFile = path.resolve(dir, file);
+            await createSilentAudio(audioFile, data.duration);
+            zipper.addFile(audioFile);
+            data.audioSources.ja = file;
+            data.audioSources.en = file;
+        }
+    }));
+    // multiLinguals
     context.multiLingual.forEach((beat, index) => {
         bundleTargetLang.forEach((lang) => {
             if (resultJson[index] && resultJson[index].multiLinguals) {
@@ -71,7 +115,10 @@ export const mulmoViewerBundle = async (context) => {
             }
         });
     });
-    fs.writeFileSync(path.resolve(dir, viewJsonFileName), JSON.stringify({ beats: resultJson, bgmSource: context.studio?.script.audioParams?.bgm }, null, 2));
+    // BGM
+    const bgmFileName = await processBgm(context.studio?.script.audioParams?.bgm, dir, zipper);
+    const bundleData = { beats: resultJson, bgmSource: bgmFileName };
+    fs.writeFileSync(path.resolve(dir, viewJsonFileName), JSON.stringify(bundleData, null, 2));
     zipper.addFile(path.resolve(dir, viewJsonFileName));
     if (isZip) {
         await zipper.finalize();

package/lib/agents/image_genai_agent.d.ts

@@ -1,5 +1,6 @@
 import type { AgentFunction, AgentFunctionInfo } from "graphai";
 import type { AgentBufferResult, ImageAgentInputs, ImageAgentParams, GenAIImageAgentConfig } from "../types/agent.js";
+export declare const ratio2BlankPath: (aspectRatio: string) => string;
 export declare const imageGenAIAgent: AgentFunction<ImageAgentParams, AgentBufferResult, ImageAgentInputs, GenAIImageAgentConfig>;
 declare const imageGenAIAgentInfo: AgentFunctionInfo;
 export default imageGenAIAgentInfo;

package/lib/agents/image_genai_agent.js

@@ -11,9 +11,53 @@ const getAspectRatio = (canvasSize) => {
     else if (canvasSize.width < canvasSize.height) {
         return "9:16";
     }
-    else {
-        return "1:1";
+    return "1:1";
+};
+export const ratio2BlankPath = (aspectRatio) => {
+    if (aspectRatio === "9:16") {
+        return blankVerticalImagePath();
+    }
+    else if (aspectRatio === "1:1") {
+        return blankSquareImagePath();
+    }
+    return blankImagePath();
+};
+const getGeminiContents = (prompt, aspectRatio, referenceImages) => {
+    const contents = [{ text: prompt }];
+    const images = [...(referenceImages ?? [])];
+    // NOTE: There is no way to explicitly specify the aspect ratio for Gemini. This is just a hint.
+    images.push(ratio2BlankPath(aspectRatio));
+    images.forEach((imagePath) => {
+        const imageData = fs.readFileSync(imagePath);
+        const base64Image = imageData.toString("base64");
+        contents.push({ inlineData: { mimeType: "image/png", data: base64Image } });
+    });
+    return contents;
+};
+const geminiFlashResult = (response) => {
+    if (!response.candidates?.[0]?.content?.parts) {
+        throw new Error("ERROR: generateContent returned no candidates", {
+            cause: agentInvalidResponseError("imageGenAIAgent", imageAction, imageFileTarget),
+        });
+    }
+    for (const part of response.candidates[0].content.parts) {
+        if (part.text) {
+            GraphAILogger.info("Gemini image generation response:", part.text);
+        }
+        else if (part.inlineData) {
+            const imageData = part.inlineData.data;
+            if (!imageData) {
+                throw new Error("ERROR: generateContent returned no image data", {
+                    cause: agentInvalidResponseError("imageGenAIAgent", imageAction, imageFileTarget),
+                });
+            }
+            const buffer = Buffer.from(imageData, "base64");
+            return { buffer };
+        }
     }
+    throw new Error("ERROR: generateContent returned no image data", {
+        cause: agentInvalidResponseError("imageGenAIAgent", imageAction, imageFileTarget),
+    });
 };
 export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, referenceImages } = namedInputs;
@@ -28,47 +72,9 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
     try {
         const ai = new GoogleGenAI({ apiKey });
         if (model === "gemini-2.5-flash-image-preview") {
-            const contents = [{ text: prompt }];
-            const images = [...(referenceImages ?? [])];
-            // NOTE: There is no way to explicitly specify the aspect ratio for Gemini. This is just a hint.
-            if (aspectRatio === "9:16") {
-                images.push(blankVerticalImagePath());
-            }
-            else if (aspectRatio === "1:1") {
-                images.push(blankSquareImagePath());
-            }
-            else {
-                images.push(blankImagePath());
-            }
-            images.forEach((imagePath) => {
-                const imageData = fs.readFileSync(imagePath);
-                const base64Image = imageData.toString("base64");
-                contents.push({ inlineData: { mimeType: "image/png", data: base64Image } });
-            });
+            const contents = getGeminiContents(prompt, aspectRatio, referenceImages);
             const response = await ai.models.generateContent({ model, contents });
-            if (!response.candidates?.[0]?.content?.parts) {
-                throw new Error("ERROR: generateContent returned no candidates", {
-                    cause: agentInvalidResponseError("imageGenAIAgent", imageAction, imageFileTarget),
-                });
-            }
-            for (const part of response.candidates[0].content.parts) {
-                if (part.text) {
-                    GraphAILogger.info("Gemini image generation response:", part.text);
-                }
-                else if (part.inlineData) {
-                    const imageData = part.inlineData.data;
-                    if (!imageData) {
-                        throw new Error("ERROR: generateContent returned no image data", {
-                            cause: agentInvalidResponseError("imageGenAIAgent", imageAction, imageFileTarget),
-                        });
-                    }
-                    const buffer = Buffer.from(imageData, "base64");
-                    return { buffer };
-                }
-            }
-            throw new Error("ERROR: generateContent returned no image data", {
-                cause: agentInvalidResponseError("imageGenAIAgent", imageAction, imageFileTarget),
-            });
+            return geminiFlashResult(response);
         }
         else {
             const response = await ai.models.generateImages({

package/lib/agents/puppeteer_crawler_agent.js

@@ -11,6 +11,7 @@ const waitStable = async (page, ms = 1200, step = 200) => {
     let last = -1;
     let stable = 0;
     while (stable < ms) {
+        // eslint-disable-next-line no-undef
         const len = await page.evaluate(() => document.body?.innerText?.length || 0);
         stable = len === last ? stable + step : 0;
         last = len;
@@ -38,6 +39,7 @@ const fetchArticle = async (url) => {
         let finalText = text;
         if (finalText.length < 100) {
             const raw = await page.evaluate(() => {
+                // eslint-disable-next-line no-undef
                 const el = document.querySelector("article, main, [role=main], .article, .post") || document.body;
                 return el?.textContent || "";
             });

package/lib/types/type.d.ts

@@ -134,3 +134,18 @@ export type PublicAPIArgs = {
     callbacks?: CallbackFunction[];
 };
 export type ImageType = "image" | "movie";
+export type MulmoViewerBeat = {
+    text?: string;
+    duration?: number;
+    multiLinguals?: Record<string, string>;
+    audioSources?: Record<string, string>;
+    imageSource?: string;
+    videoSource?: string;
+    videoWithAudioSource?: string;
+    htmlImageSource?: string;
+    soundEffectSource?: string;
+};
+export type MulmoViewerData = {
+    beats: MulmoViewerBeat[];
+    bgmSource?: string;
+};

package/lib/utils/ffmpeg_utils.d.ts

@@ -17,3 +17,4 @@ export declare const ffmpegGetMediaDuration: (filePath: string) => Promise<{
 }>;
 export declare const extractImageFromMovie: (movieFile: string, imagePath: string) => Promise<object>;
 export declare const trimMusic: (inputFile: string, startTime: number, duration: number) => Promise<Buffer>;
+export declare const createSilentAudio: (filePath: string, durationSec: number) => Promise<void>;

package/lib/utils/ffmpeg_utils.js

@@ -132,3 +132,15 @@ export const trimMusic = (inputFile, startTime, duration) => {
         });
     });
 };
+export const createSilentAudio = (filePath, durationSec) => {
+    const filter = `anullsrc=r=44100:cl=stereo,atrim=duration=${durationSec},aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo[a]`;
+    return new Promise((resolve, reject) => {
+        ffmpeg()
+            .complexFilter([filter])
+            .outputOptions(["-map", "[a]"])
+            .output(filePath)
+            .on("end", () => resolve())
+            .on("error", (err) => reject(err))
+            .run();
+    });
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "1.2.66",
+  "version": "1.2.68",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -92,7 +92,7 @@
     "dotenv": "^17.2.3",
     "fluent-ffmpeg": "^2.1.3",
     "graphai": "^2.0.16",
-    "jsdom": "^27.0.0",
+    "jsdom": "^27.0.1",
     "marked": "^16.4.1",
     "mulmocast-vision": "^1.0.4",
     "ora": "^9.0.0",
@@ -116,7 +116,7 @@
     "prettier": "^3.6.2",
     "tsx": "^4.20.6",
     "typescript": "^5.9.3",
-    "typescript-eslint": "^8.46.1"
+    "typescript-eslint": "^8.46.2"
   },
   "engines": {
     "node": ">=20.0.0"