mulmocast 0.0.2 → 0.0.4

package/README.md

@@ -1,6 +1,14 @@
 # mulmocast
 
-A CLI tool for generating podcast and video content from script files. Automates the process of creating audio, images, and video from structured script files.
+A CLI tool for generating podcast and video content from script files (MulmoScript). Automates the process of creating audio, images, and video from structured MulmoScript files.
+
+## What is MulmoScript?
+
+**MulmoScript** is a simple JSON/YAML format for describing multi-modal content.  
+You can define speakers, text, images, and layout — all in one script.
+
+A Hello World script is available in [./docs/scripts](./docs/scripts).   
+See [MulmoScript Format](#mulmoscript-format) for details on the structure.
 
 ## Installation
 
@@ -22,11 +30,11 @@ brew install ffmpeg
 Create a `.env` file in your project directory with the following API keys:
 
 ### Required
-```
+```bash
 OPENAI_API_KEY=your_openai_api_key
 ```
 ### Optional
-```
+```bash
 DEFAULT_OPENAI_IMAGE_MODEL=gpt-image-1 # for the advanced image generation model
 GOOGLE_PROJECT_ID=your_google_project_id # for Google's image generation model
 NIJIVOICE_API_KEY=your_nijivoice_api_key # for Nijivoice's TTS model
@@ -42,13 +50,13 @@ BROWSERLESS_API_TOKEN=your_browserless_api_token # to access web in mulmo-tool
 
 ## Quick Start
 
-```
+```bash
 # Generate script with interactive mode
-mulmo-tool scripting -i -t children_book -o ./ -f story
+mulmo-tool scripting -i -t children_book -o ./ -s story
 ```
 After running this command, you'll create a story script through an interactive conversation with the AI.
 
-```
+```bash
 # Generate both audio and images, then combine into video
 mulmo movie {generated_script_file}
 ```
@@ -57,7 +65,7 @@ Replace `{generated_script_file}` with the output file from the previous command
 ## Generate MulmoScript
 
 ```bash
-# Generate script from web content
+# Generate script from web content (requires Browserless API KEY)
 mulmo-tool scripting -u https://example.com
 
 # Generate script with interactive mode
@@ -113,8 +121,13 @@ https://github.com/receptron/mulmocast-cli/tree/main/scripts
 
 CLI Usage
 
+### `mulmo -h` Output
+Use this command to generate content (audio, images, movie) from an existing MulmoScript.
+
+```bash
+mulmo -h
 ```
-$ mulmo -h
+```bash
 mulmo <action> <file>
 
 Run mulmocast
@@ -136,9 +149,14 @@ Options:
 
 ```
 
+### `mulmo-tool -h` Output
+Use this command to generate a new MulmoScript from a URL or interactively.
+
+```bash
+mulmo-tool -h
 ```
-$ mulmo-tool -h
 
+```bash
 mulmo-tool <action>
 
 Run mulmocast tool

package/assets/html/chart.html

@@ -5,20 +5,11 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>Simple Chart.js Bar Chart</title>
     <style>
-        body {
-            font-family: Arial, sans-serif;
-            padding: 40px; padding-top: 60px;
-        }
+        ${style}
         .chart-container {
             width: ${width}px;
             margin: 0 auto;
         }
-        h1 {
-            text-align: center;
-            font-size: 50px;
-            color: #333;
-            margin-bottom: 80px;
-        }
     </style>
     <!-- Include Chart.js from CDN -->
     <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>

package/assets/html/mermaid.html

@@ -5,24 +5,12 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>Mermaid Diagram</title>
     <style>
-        body {
-            box-sizing: border-box;
-            font-family: Arial, sans-serif;
-            padding: 40px;
-            padding-top: 60px;
-            height: 100vh;
-        }
+        ${style}
         .container {
             height: 100%;
             display: flex;
             flex-direction: column;
         }
-        h1 {
-            text-align: center;
-            font-size: 50px;
-            color: #333;
-            margin-bottom: 30px;
-        }
         .mermaid {
             width: 100%;
             height: 100%;

package/assets/templates/business.json

@@ -1,20 +1,13 @@
 {
   "title": "Business presentation",
   "description": "Template for business presentation.",
-  "systemPrompt": "Generate a script for a business presentation of the given topic. Use textSlides or markdown to show slides or use imagePrompt to generate an appropriate image for that page. Use the JSON below as a template. chartData is the data for Chart.js",
+  "systemPrompt": "Generate a script for a business presentation of the given topic. Use textSlides, markdown, mermaid, or chart to show slides. Extract image links in the article to reuse them in the presentation. Use the JSON below as a template. chartData is the data for Chart.js",
   "script": {
     "$mulmocast": {
       "version": "1.0",
       "credit": "closing"
     },
     "title": "Sample Title",
-    "canvasSize": {
-      "width": 1536,
-      "height": 1024
-    },
-    "imageParams": {
-      "style": "Style appropriate for business environment."
-    },
     "speechParams": {
       "speakers": {
         "Presenter": {
@@ -26,19 +19,6 @@
         }
       }
     },
-    "textSlideParams": {
-      "cssStyles": [
-        "body { padding: 40px; padding-top: 60px; font-family: Arial, sans-serif;color:#333; font-size: 36px }",
-        "h1 { font-size: 50px; text-align: center }",
-        "ul { margin-left: 36px } ",
-        "pre { background: #eeeecc; font-size: 24px; padding:10px }",
-        "p { margin-left: 36px }",
-        "table { font-size: 36px; margin: auto; border: 1px solid gray; border-collapse: collapse }",
-        "th { border-bottom: 1px solid gray }",
-        "td, th { padding: 8px }",
-        "tr:nth-child(even) { background-color: #eee }"
-      ]
-    },
     "lang": "en",
     "beats": [
       {
@@ -90,11 +70,6 @@
           ]
         }
       },
-      {
-        "speaker": "Presenter",
-        "text": "Our story begins with tree-dwelling primates—small mammals with forward-facing eyes and grasping hands. These ancestors lived in forests and evolved traits useful for climbing and visual depth perception.",
-        "imagePrompt": "tree-dwelling primates with forward-facing eyes in a jungle."
-      },
       {
         "speaker": "Presenter",
         "text": "This page shows the sales and profits of this company from January 2024 to June 2024.",
@@ -135,7 +110,21 @@
         "image": {
           "type": "mermaid",
           "title": "Business Process Flow",
-          "code": "graph LR\n    A[Market Research] --> B[Product Planning]\n    B --> C[Development]\n    C --> D[Testing]\n    D --> E[Manufacturing]\n    E --> F[Marketing]\n    F --> G[Sales]\n    G --> H[Customer Support]\n    H --> A"
+          "code": {
+            "kind": "text",
+            "text": "graph LR\n    A[Market Research] --> B[Product Planning]\n    B --> C[Development]\n    C --> D[Testing]\n    D --> E[Manufacturing]\n    E --> F[Marketing]\n    F --> G[Sales]\n    G --> H[Customer Support]\n    H --> A"
+          }
+        }
+      },
+      {
+        "speaker": "Presenter",
+        "text": "This is the image of a pingpong ball.",
+        "image": {
+          "type": "image",
+          "source": {
+            "kind": "url",
+            "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/test/pingpong.png"
+          }
         }
       }
     ]

package/assets/templates/coding.json

@@ -1,20 +1,13 @@
 {
   "title": "Coding presentation",
   "description": "Template for software and coding presentation.",
-  "systemPrompt": "Generate a script for a technical presentation of the given topic. Use markdown with a code block to show some code on a slide. Use the JSON below as a template.",
+  "systemPrompt": "Generate a script for a technical presentation of the given topic. Use markdown with a code block to show some code on a slide. Avoid long coding examples, which may not fit in a single slide. Use the JSON below as a template.",
   "script": {
     "$mulmocast": {
       "version": "1.0",
       "credit": "closing"
     },
     "title": "Sample Title",
-    "canvasSize": {
-      "width": 1536,
-      "height": 1024
-    },
-    "imageParams": {
-      "style": "Style appropriate for software presentation."
-    },
     "speechParams": {
       "speakers": {
         "Presenter": {
@@ -27,19 +20,6 @@
       }
     },
     "lang": "en",
-    "textSlideParams": {
-      "cssStyles": [
-        "body { margin: 40px; margin-top: 60px; color:#333; font-size: 48px }",
-        "h1 { font-size: 60px; text-align: center }",
-        "ul { margin-left: 40px } ",
-        "pre { background: #eeeecc; font-size: 24px; padding:10px }",
-        "p { margin-left: 40px }",
-        "table { font-size: 40px; margin: auto; border: 1px solid gray; border-collapse: collapse }",
-        "th { border-bottom: 1px solid gray }",
-        "td, th { padding: 8px }",
-        "tr:nth-child(even) { background-color: #eee }"
-      ]
-    },
     "beats": [
       {
         "speaker": "Presenter",
@@ -97,6 +77,63 @@
             "| Codecademy Hoodie |  False   | 42.99 |"
           ]
         }
+      },
+      {
+        "speaker": "Presenter",
+        "text": "Next, let's look at a diagram of our business process flow. This illustrates the key steps from product development to sales.",
+        "image": {
+          "type": "mermaid",
+          "title": "Business Process Flow",
+          "code": {
+            "kind": "text",
+            "text": "graph LR\n    A[Market Research] --> B[Product Planning]\n    B --> C[Development]\n    C --> D[Testing]\n    D --> E[Manufacturing]\n    E --> F[Marketing]\n    F --> G[Sales]\n    G --> H[Customer Support]\n    H --> A"
+          }
+        }
+      },
+      {
+        "speaker": "Presenter",
+        "text": "This page shows the sales and profits of this company from January 2024 to June 2024.",
+        "image": {
+          "type": "chart",
+          "title": "Sales and Profits (from Jan to June)",
+          "chartData": {
+            "type": "bar",
+            "data": {
+              "labels": ["January", "February", "March", "April", "May", "June"],
+              "datasets": [
+                {
+                  "label": "Revenue ($1000s)",
+                  "data": [120, 135, 180, 155, 170, 190],
+                  "backgroundColor": "rgba(54, 162, 235, 0.5)",
+                  "borderColor": "rgba(54, 162, 235, 1)",
+                  "borderWidth": 1
+                },
+                {
+                  "label": "Profit ($1000s)",
+                  "data": [45, 52, 68, 53, 61, 73],
+                  "backgroundColor": "rgba(75, 192, 192, 0.5)",
+                  "borderColor": "rgba(75, 192, 192, 1)",
+                  "borderWidth": 1
+                }
+              ]
+            },
+            "options": {
+              "responsive": true,
+              "animation": false
+            }
+          }
+        }
+      },
+      {
+        "speaker": "Presenter",
+        "text": "This is the image of a pingpong ball.",
+        "image": {
+          "type": "image",
+          "source": {
+            "kind": "url",
+            "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/test/pingpong.png"
+          }
+        }
       }
     ]
   }

package/lib/actions/audio.d.ts

@@ -1,3 +1,3 @@
 import "dotenv/config";
 import { MulmoStudioContext } from "../types/index.js";
-export declare const audio: (context: MulmoStudioContext, concurrency: number) => Promise<void>;
+export declare const audio: (context: MulmoStudioContext) => Promise<void>;

package/lib/actions/audio.js

@@ -8,43 +8,59 @@ import ttsOpenaiAgent from "../agents/tts_openai_agent.js";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { MulmoScriptMethods } from "../methods/index.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
-import { getAudioArtifactFilePath, getAudioSegmentDirPath, getAudioCombinedFilePath, getOutputStudioFilePath, defaultBGMPath, mkdir, writingMessage, } from "../utils/file.js";
+import { getAudioArtifactFilePath, getAudioSegmentDirPath, getAudioCombinedFilePath, getOutputStudioFilePath, defaultBGMPath, mkdir, writingMessage, getAudioSegmentFilePath, resolveMediaSource, } from "../utils/file.js";
+import { text2hash } from "../utils/utils.js";
 const { default: __, ...vanillaAgents } = agents;
 // const rion_takanashi_voice = "b9277ce3-ba1c-4f6f-9a65-c05ca102ded0"; // たかなし りおん
 // const ben_carter_voice = "bc06c63f-fef6-43b6-92f7-67f919bd5dae"; // ベン・カーター
+const provider_to_agent = {
+    nijivoice: "ttsNijivoiceAgent",
+    openai: "ttsOpenaiAgent",
+};
+const getAudioPath = (context, beat, audioFile, audioDirPath) => {
+    if (beat.audio?.type === "audio") {
+        const path = resolveMediaSource(beat.audio.source, context);
+        if (path) {
+            return path;
+        }
+        throw new Error("Invalid audio source");
+    }
+    return getAudioSegmentFilePath(audioDirPath, context.studio.filename, audioFile);
+};
+const preprocessor = (namedInputs) => {
+    const { beat, index, context, audioDirPath } = namedInputs;
+    const studioBeat = context.studio.beats[index];
+    const voiceId = context.studio.script.speechParams.speakers[beat.speaker].voiceId;
+    const speechOptions = MulmoScriptMethods.getSpeechOptions(context.studio.script, beat);
+    const hash_string = `${beat.text}${voiceId}${speechOptions?.instruction ?? ""}${speechOptions?.speed ?? 1.0}`;
+    const audioFile = `${context.studio.filename}_${index}_${text2hash(hash_string)}`;
+    const audioPath = getAudioPath(context, beat, audioFile, audioDirPath);
+    studioBeat.audioFile = audioPath;
+    return {
+        ttsAgent: provider_to_agent[context.studio.script.speechParams.provider],
+        studioBeat,
+        voiceId,
+        speechOptions,
+        audioPath,
+    };
+};
 const graph_tts = {
     nodes: {
         preprocessor: {
-            agent: (namedInputs) => {
-                const { beat, script, speakers } = namedInputs;
-                return {
-                    voiceId: speakers[beat.speaker].voiceId,
-                    speechOptions: MulmoScriptMethods.getSpeechOptions(script, beat),
-                };
-            },
+            agent: preprocessor,
             inputs: {
                 beat: ":beat",
-                script: ":script",
-                speakers: ":script.speechParams.speakers",
-            },
-        },
-        ttsAgent: {
-            agent: (namedInputs) => {
-                if (namedInputs.provider === "nijivoice") {
-                    return "ttsNijivoiceAgent";
-                }
-                return "ttsOpenaiAgent";
-            },
-            inputs: {
-                provider: ":script.speechParams.provider",
+                index: ":__mapIndex",
+                context: ":context",
+                audioDirPath: ":audioDirPath",
             },
         },
         tts: {
             unless: ":beat.audio",
-            agent: ":ttsAgent",
+            agent: ":preprocessor.ttsAgent",
             inputs: {
                 text: ":beat.text",
-                file: "${:audioSegmentDirPath}/${:beat.audioFile}.mp3", // TODO
+                file: ":preprocessor.audioPath",
                 force: ":context.force",
             },
             params: {
@@ -68,8 +84,8 @@ const graph_data = {
         map: {
             agent: "mapAgent",
             inputs: {
-                rows: ":context.studio.beats",
-                script: ":context.studio.script",
+                rows: ":context.studio.script.beats",
+                studio: ":context.studio",
                 audioDirPath: ":audioDirPath",
                 audioSegmentDirPath: ":audioSegmentDirPath",
                 context: ":context",
@@ -128,7 +144,7 @@ const agentFilters = [
         nodeIds: ["tts"],
     },
 ];
-export const audio = async (context, concurrency) => {
+export const audio = async (context) => {
     const { studio, fileDirs } = context;
     const { outDirPath, audioDirPath } = fileDirs;
     const audioArtifactFilePath = getAudioArtifactFilePath(outDirPath, studio.filename);
@@ -137,7 +153,7 @@ export const audio = async (context, concurrency) => {
     const outputStudioFilePath = getOutputStudioFilePath(outDirPath, studio.filename);
     mkdir(outDirPath);
     mkdir(audioSegmentDirPath);
-    graph_data.concurrency = concurrency;
+    graph_data.concurrency = MulmoScriptMethods.getSpeechProvider(studio.script) === "nijivoice" ? 1 : 8;
     const graph = new GraphAI(graph_data, {
         ...vanillaAgents,
         fileWriteAgent,

package/lib/actions/images.js

@@ -7,42 +7,36 @@ import { fileCacheAgentFilter } from "../utils/filters.js";
 import imageGoogleAgent from "../agents/image_google_agent.js";
 import imageOpenaiAgent from "../agents/image_openai_agent.js";
 import { MulmoScriptMethods } from "../methods/index.js";
-import { processChart, processMarkdown, processTextSlide, processImage, processMermaid } from "../utils/image_preprocess.js";
+import { imagePlugins } from "../utils/image_plugins/index.js";
 const { default: __, ...vanillaAgents } = agents;
 dotenv.config();
 // const openai = new OpenAI();
 import { GoogleAuth } from "google-auth-library";
+const htmlStyle = (script, beat) => {
+    return {
+        canvasSize: MulmoScriptMethods.getCanvasSize(script),
+        textSlideStyle: MulmoScriptMethods.getTextSlideStyle(script, beat),
+    };
+};
 const imagePreprocessAgent = async (namedInputs) => {
     const { context, beat, index, suffix, imageDirPath, imageAgentInfo } = namedInputs;
     const imageParams = { ...imageAgentInfo.imageParams, ...beat.imageParams };
-    const prompt = (beat.imagePrompt || beat.text) + "\n" + (imageParams.style || "");
     const imagePath = `${imageDirPath}/${context.studio.filename}/${index}${suffix}.png`;
-    const aspectRatio = MulmoScriptMethods.getAspectRatio(context.studio.script);
-    const textSlideStyle = MulmoScriptMethods.getTextSlideStyle(context.studio.script, beat);
+    const returnValue = {
+        aspectRatio: MulmoScriptMethods.getAspectRatio(context.studio.script),
+        imageParams,
+    };
     if (beat.image) {
-        const canvasSize = MulmoScriptMethods.getCanvasSize(context.studio.script);
-        const processorParams = { beat, context, imagePath, textSlideStyle, canvasSize };
-        if (beat.image.type === "textSlide") {
-            await processTextSlide(processorParams);
-        }
-        else if (beat.image.type === "markdown") {
-            await processMarkdown(processorParams);
-        }
-        else if (beat.image.type === "image") {
-            const path = processImage(processorParams);
-            if (path) {
-                // undefined prompt indicates that image generation is not needed
-                return { path, prompt: undefined, imageParams, aspectRatio };
-            }
-        }
-        else if (beat.image.type === "chart") {
-            await processChart(processorParams);
-        }
-        else if (beat.image.type === "mermaid") {
-            await processMermaid(processorParams);
+        const plugin = imagePlugins.find((plugin) => plugin.imageType === beat?.image?.type);
+        if (plugin) {
+            const processorParams = { beat, context, imagePath, ...htmlStyle(context.studio.script, beat) };
+            const path = await plugin.process(processorParams);
+            // undefined prompt indicates that image generation is not needed
+            return { path, ...returnValue };
         }
     }
-    return { path: imagePath, prompt, imageParams, aspectRatio };
+    const prompt = (beat.imagePrompt || beat.text) + "\n" + (imageParams.style || "");
+    return { path: imagePath, prompt, ...returnValue };
 };
 const graph_data = {
     version: 0.5,
@@ -54,7 +48,7 @@ const graph_data = {
         outputStudioFilePath: {},
         map: {
             agent: "mapAgent",
-            inputs: { rows: ":context.studio.beats", context: ":context", imageAgentInfo: ":imageAgentInfo", imageDirPath: ":imageDirPath" },
+            inputs: { rows: ":context.studio.script.beats", context: ":context", imageAgentInfo: ":imageAgentInfo", imageDirPath: ":imageDirPath" },
             isResult: true,
             params: {
                 rowKey: "beat",

package/lib/actions/index.d.ts

@@ -0,0 +1,5 @@
+export * from "./audio.js";
+export * from "./images.js";
+export * from "./movie.js";
+export * from "./pdf.js";
+export * from "./translate.js";

package/lib/actions/index.js

@@ -0,0 +1,5 @@
+export * from "./audio.js";
+export * from "./images.js";
+export * from "./movie.js";
+export * from "./pdf.js";
+export * from "./translate.js";

package/lib/actions/movie.d.ts

@@ -1,2 +1,10 @@
-import { MulmoStudioContext } from "../types/index.js";
+import { MulmoStudioContext, MulmoCanvasDimension, BeatMediaType } from "../types/index.js";
+export declare const getVideoPart: (inputIndex: number, mediaType: BeatMediaType, duration: number, canvasInfo: MulmoCanvasDimension) => {
+    videoId: string;
+    videoPart: string;
+};
+export declare const getAudioPart: (inputIndex: number, duration: number, delay: number) => {
+    audioId: string;
+    audioPart: string;
+};
 export declare const movie: (context: MulmoStudioContext) => Promise<void>;