mulmocast 1.2.64 → 1.2.66

package/lib/actions/bundle.js

@@ -5,15 +5,16 @@ import { listLocalizedAudioPaths } from "./audio.js";
 import { imagePreprocessAgent } from "./image_agents.js";
 import { mkdir } from "../utils/file.js";
 import { ZipBuilder } from "../utils/zip.js";
+import { bundleTargetLang } from "../utils/const.js";
 const beatImage = (context) => {
     return async (beat, index) => {
         try {
             const res = await imagePreprocessAgent({ context, beat, index, imageRefs: {} });
             if ("htmlPrompt" in res) {
-                return { htmlImageFile: res.htmlImageFile, imagePath: res.imagePath };
+                return { htmlImageSource: res.htmlImageFile, imageSource: res.imagePath };
             }
             const { imagePath, movieFile, lipSyncFile } = res;
-            return { imagePath, movieFile, lipSyncFile };
+            return { imageSource: imagePath, videoSource: movieFile, videoWithAudioSource: lipSyncFile };
         }
         catch (e) {
             GraphAILogger.log(e);
@@ -21,32 +22,37 @@ const beatImage = (context) => {
         }
     };
 };
+// TODO reference
 const viewJsonFileName = "mulmo_view.json";
 const zipFileName = "mulmo.zip";
 export const mulmoViewerBundle = async (context) => {
     const isZip = true;
-    const audios = listLocalizedAudioPaths(context);
-    const images = await Promise.all(context.studio.script.beats.map(beatImage(context)));
     const dir = path.resolve(context.fileDirs.fileName);
     mkdir(dir);
     const zipper = new ZipBuilder(path.resolve(dir, zipFileName));
     const resultJson = [];
-    audios.forEach((audio) => {
-        if (audio) {
-            const fileName = path.basename(audio ?? "");
-            resultJson.push({ audio: fileName });
-            if (fs.existsSync(audio)) {
-                fs.copyFileSync(audio, path.resolve(dir, fileName));
-                zipper.addFile(audio, fileName);
-            }
-        }
-        else {
-            resultJson.push({});
-        }
+    context.studio.script.beats.forEach((beat) => {
+        resultJson.push({ text: beat.text, duration: beat.duration, audioSources: {}, multiLinguals: {} });
     });
+    for (const lang of bundleTargetLang) {
+        const audios = listLocalizedAudioPaths({ ...context, lang });
+        audios.forEach((audio, index) => {
+            if (audio) {
+                const fileName = path.basename(audio ?? "");
+                if (resultJson[index] && resultJson[index].audioSources) {
+                    resultJson[index].audioSources[lang] = fileName;
+                }
+                if (fs.existsSync(audio)) {
+                    fs.copyFileSync(audio, path.resolve(dir, fileName));
+                    zipper.addFile(audio, fileName);
+                }
+            }
+        });
+    }
+    const images = await Promise.all(context.studio.script.beats.map(beatImage(context)));
     images.forEach((image, index) => {
         const data = resultJson[index];
-        const keys = ["htmlImageFile", "imagePath", "movieFile", "lipSyncFile"];
+        const keys = ["htmlImageSource", "imageSource", "videoSource", "videoWithAudioSource"];
         keys.forEach((key) => {
             const value = image[key];
             if (value) {
@@ -58,7 +64,14 @@ export const mulmoViewerBundle = async (context) => {
             }
         });
     });
-    fs.writeFileSync(path.resolve(dir, viewJsonFileName), JSON.stringify(resultJson, null, 2));
+    context.multiLingual.forEach((beat, index) => {
+        bundleTargetLang.forEach((lang) => {
+            if (resultJson[index] && resultJson[index].multiLinguals) {
+                resultJson[index].multiLinguals[lang] = beat.multiLingualTexts[lang].text;
+            }
+        });
+    });
+    fs.writeFileSync(path.resolve(dir, viewJsonFileName), JSON.stringify({ beats: resultJson, bgmSource: context.studio?.script.audioParams?.bgm }, null, 2));
     zipper.addFile(path.resolve(dir, viewJsonFileName));
     if (isZip) {
         await zipper.finalize();

package/lib/actions/image_agents.js

@@ -1,3 +1,4 @@
+import { GraphAILogger } from "graphai";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods, MulmoBeatMethods, MulmoMediaSourceMethods } from "../methods/index.js";
 import { getBeatPngImagePath, getBeatMoviePaths, getAudioFilePath } from "../utils/file.js";
 import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
@@ -28,14 +29,17 @@ export const imagePreprocessAgent = async (namedInputs) => {
         beatDuration: beat.duration ?? studioBeat?.duration,
     };
     const isMovie = Boolean(beat.moviePrompt || beat?.image?.type === "movie");
-    if (isMovie) {
-        if (beat.soundEffectPrompt) {
+    if (beat.soundEffectPrompt) {
+        if (isMovie) {
             returnValue.soundEffectAgentInfo = MulmoPresentationStyleMethods.getSoundEffectAgentInfo(context.presentationStyle, beat);
             returnValue.soundEffectModel =
                 beat.soundEffectParams?.model ?? context.presentationStyle.soundEffectParams?.model ?? returnValue.soundEffectAgentInfo.defaultModel;
             returnValue.soundEffectFile = moviePaths.soundEffectFile;
             returnValue.soundEffectPrompt = beat.soundEffectPrompt;
         }
+        else {
+            GraphAILogger.warn(`soundEffectPrompt is set, but there is no video. beat: ${index}`);
+        }
     }
     if (beat.enableLipSync) {
         const lipSyncAgentInfo = MulmoPresentationStyleMethods.getLipSyncAgentInfo(context.presentationStyle, beat);
@@ -70,8 +74,10 @@ export const imagePreprocessAgent = async (namedInputs) => {
         // ImagePluginPreprocessAgentResponse
         return {
             ...returnValue,
-            imagePath: isTypeMovie ? undefined : pluginPath,
+            // imagePath: isTypeMovie ? undefined : pluginPath,
+            imagePath: isTypeMovie ? imagePath : pluginPath,
             movieFile: isTypeMovie ? pluginPath : undefined,
+            imageFromMovie: isTypeMovie,
             referenceImageForMovie: pluginPath,
             markdown,
             html,

package/lib/actions/images.js

@@ -354,11 +354,33 @@ export const images_graph_data = {
                 studio.beats.forEach((studioBeat, index) => {
                     const beat = studio.script.beats[index];
                     if (beat.image?.type === "beat") {
-                        if (beat.image.id && beatIndexMap[beat.image.id] !== undefined) {
-                            studioBeat.imageFile = studio.beats[beatIndexMap[beat.image.id]].imageFile;
+                        // reference Beat by plugin
+                        const referenceBeat = (() => {
+                            if (beat.image.id) {
+                                if (beatIndexMap[beat.image.id] !== undefined) {
+                                    return studio.beats[beatIndexMap[beat.image.id]];
+                                }
+                                else {
+                                    GraphAILogger.info(`reference beat not exist: id=${beat.image.id}`);
+                                }
+                            }
+                            else if (index > 0) {
+                                return studio.beats[index - 1];
+                            }
+                        })();
+                        if (referenceBeat === undefined) {
+                            // error?
+                            GraphAILogger.info(`reference beat not exist: index=${index}`);
                         }
-                        else if (index > 0) {
-                            studioBeat.imageFile = studio.beats[index - 1].imageFile;
+                        else {
+                            studioBeat.imageFile = referenceBeat.imageFile;
+                            studioBeat.movieFile = referenceBeat.movieFile;
+                            studioBeat.soundEffectFile = referenceBeat.soundEffectFile;
+                            studioBeat.lipSyncFile = referenceBeat.lipSyncFile;
+                            studioBeat.hasMovieAudio = referenceBeat.hasMovieAudio;
+                            studioBeat.htmlImageFile = referenceBeat.htmlImageFile;
+                            studioBeat.markdown = referenceBeat.markdown;
+                            studioBeat.html = referenceBeat.html;
                         }
                     }
                 });

package/lib/actions/pdf.js

@@ -4,14 +4,14 @@ import puppeteer from "puppeteer";
 import { GraphAILogger, sleep } from "graphai";
 import { MulmoPresentationStyleMethods } from "../methods/index.js";
 import { localizedText, isHttp } from "../utils/utils.js";
-import { getOutputPdfFilePath, writingMessage, getHTMLFile } from "../utils/file.js";
+import { getOutputPdfFilePath, writingMessage, getHTMLFile, mulmoCreditPath } from "../utils/file.js";
 import { interpolate } from "../utils/markdown.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
 const isCI = process.env.CI === "true";
 const getPdfSize = (pdfSize) => {
     return pdfSize === "a4" ? "A4" : "Letter";
 };
-const loadImage = async (imagePath) => {
+const loadImage = async (imagePath, index) => {
     try {
         const imageData = isHttp(imagePath) ? Buffer.from(await (await fetch(imagePath)).arrayBuffer()) : fs.readFileSync(imagePath);
         const ext = path.extname(imagePath).toLowerCase().replace(".", "");
@@ -19,8 +19,8 @@ const loadImage = async (imagePath) => {
         return `data:image/${mimeType};base64,${imageData.toString("base64")}`;
     }
     catch (error) {
-        GraphAILogger.info("loadImage failed", error);
-        const placeholderData = fs.readFileSync("assets/images/mulmocast_credit.png");
+        GraphAILogger.info(`loadImage failed: file: ${imagePath} index: ${index}`, error);
+        const placeholderData = fs.readFileSync(mulmoCreditPath());
         return `data:image/png;base64,${placeholderData.toString("base64")}`;
     }
 };
@@ -100,9 +100,9 @@ const generatePDFHTML = async (context, pdfMode, pdfSize) => {
     const { studio, multiLingual, lang = "en" } = context;
     const { width: imageWidth, height: imageHeight } = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
     const isLandscapeImage = imageWidth > imageHeight;
-    const imagePaths = studio.beats.map((beat) => beat.imageFile);
+    const imageFiles = studio.beats.map((beat) => beat.htmlImageFile ?? beat.imageFile);
     const texts = studio.script.beats.map((beat, index) => localizedText(beat, multiLingual?.[index], lang));
-    const imageDataUrls = await Promise.all(imagePaths.map(loadImage));
+    const imageDataUrls = await Promise.all(imageFiles.map(loadImage));
     const defaultPageSize = `${getPdfSize(pdfSize)} ${isLandscapeImage ? "landscape" : "portrait"}`;
     const pageSize = pdfMode === "handout" ? `${getPdfSize(pdfSize)} portrait` : defaultPageSize;
     const pagesHTML = generatePagesHTML(pdfMode, imageDataUrls, texts);

package/lib/cli/commands/bundle/handler.js

@@ -1,10 +1,14 @@
-import { mulmoViewerBundle, audio, images } from "../../../actions/index.js";
-import { initializeContext, runTranslateIfNeeded } from "../../helpers.js";
+import { mulmoViewerBundle, audio, images, translate } from "../../../actions/index.js";
+import { initializeContext } from "../../helpers.js";
+import { bundleTargetLang } from "../../../utils/const.js";
 export const handler = async (argv) => {
     const context = await initializeContext(argv);
     if (!context) {
         process.exit(1);
     }
-    await runTranslateIfNeeded(context, true);
+    await translate(context, { targetLangs: bundleTargetLang });
+    for (const lang of bundleTargetLang.filter((_lang) => _lang !== context.lang)) {
+        await audio({ ...context, lang });
+    }
     await audio(context).then(images).then(mulmoViewerBundle);
 };

package/lib/utils/const.d.ts

@@ -10,3 +10,4 @@ export declare const storyToScriptGenerateMode: {
     stepWise: string;
     oneStep: string;
 };
+export declare const bundleTargetLang: string[];

package/lib/utils/const.js

@@ -10,3 +10,4 @@ export const storyToScriptGenerateMode = {
     stepWise: "step_wise",
     oneStep: "one_step",
 };
+export const bundleTargetLang = ["ja", "en"];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "1.2.64",
+  "version": "1.2.66",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -84,7 +84,7 @@
     "@graphai/vanilla_node_agents": "^2.0.4",
     "@inquirer/input": "^4.2.5",
     "@inquirer/select": "^4.4.0",
-    "@modelcontextprotocol/sdk": "^1.20.0",
+    "@modelcontextprotocol/sdk": "^1.20.1",
     "@mozilla/readability": "^0.6.0",
     "@tavily/core": "^0.5.11",
     "archiver": "^7.0.1",
@@ -93,10 +93,10 @@
     "fluent-ffmpeg": "^2.1.3",
     "graphai": "^2.0.16",
     "jsdom": "^27.0.0",
-    "marked": "^16.4.0",
+    "marked": "^16.4.1",
     "mulmocast-vision": "^1.0.4",
     "ora": "^9.0.0",
-    "puppeteer": "^24.24.1",
+    "puppeteer": "^24.25.0",
     "replicate": "^1.3.0",
     "yaml": "^2.8.1",
     "yargs": "^18.0.0",
@@ -109,7 +109,7 @@
     "@types/fluent-ffmpeg": "^2.1.26",
     "@types/jsdom": "^27.0.0",
     "@types/yargs": "^17.0.33",
-    "eslint": "^9.37.0",
+    "eslint": "^9.38.0",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-prettier": "^5.5.4",
     "eslint-plugin-sonarjs": "^3.0.5",

package/scripts/test/test_media.json

@@ -49,7 +49,7 @@
     },
     {
       "speaker": "Presenter",
-      "text": "This is a reference beat.",
+      "text": "This is a image reference beat.",
       "duration": 0.5,
       "image": {
         "type": "beat",
@@ -81,6 +81,7 @@
       }
     },
     {
+      "id": "textSlide",
       "speaker": "Presenter",
       "text": "",
       "duration": 2,
@@ -94,6 +95,7 @@
     },
     {
       "speaker": "Presenter",
+      "id": "pingpongmov",
       "text": "This is a local movie with audio.",
       "image": {
         "type": "movie",
@@ -253,6 +255,24 @@
           "</footer>"
         ]
       }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "This is a text slide reference beat.",
+      "duration": 0.5,
+      "image": {
+        "type": "beat",
+        "id": "textSlide"
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "This is a movie reference beat.",
+      "duration": 0.5,
+      "image": {
+        "type": "beat",
+        "id": "pingpongmov"
+      }
     }
   ]
 }