mulmocast 2.0.3 → 2.0.5

package/lib/actions/audio.js

@@ -2,7 +2,7 @@ import dotenv from "dotenv";
 import { GraphAI, TaskManager, GraphAILogger } from "graphai";
 import * as agents from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-import { ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, ttsElevenlabsAgent, addBGMAgent, combineAudioFilesAgent, mediaMockAgent } from "../agents/index.js";
+import { ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, ttsGeminiAgent, ttsElevenlabsAgent, addBGMAgent, combineAudioFilesAgent, mediaMockAgent, } from "../agents/index.js";
 import { text2SpeechProviderSchema } from "../types/index.js";
 import { fileCacheAgentFilter, nijovoiceTextAgentFilter } from "../utils/filters.js";
 import { getAudioArtifactFilePath, getAudioFilePath, getOutputStudioFilePath, resolveDirPath, defaultBGMPath, mkdir, writingMessage } from "../utils/file.js";
@@ -221,6 +221,7 @@ const audioAgents = {
     ttsOpenaiAgent,
     ttsNijivoiceAgent,
     ttsGoogleAgent,
+    ttsGeminiAgent,
     ttsElevenlabsAgent,
     mediaMockAgent,
     addBGMAgent,

package/lib/agents/image_genai_agent.js

@@ -22,11 +22,13 @@ export const ratio2BlankPath = (aspectRatio) => {
     }
     return blankImagePath();
 };
-const getGeminiContents = (prompt, aspectRatio, referenceImages) => {
+const getGeminiContents = (prompt, referenceImages, aspectRatio) => {
     const contents = [{ text: prompt }];
     const images = [...(referenceImages ?? [])];
     // NOTE: There is no way to explicitly specify the aspect ratio for Gemini. This is just a hint.
-    images.push(ratio2BlankPath(aspectRatio));
+    if (aspectRatio) {
+        images.push(ratio2BlankPath(aspectRatio));
+    }
     images.forEach((imagePath) => {
         const imageData = fs.readFileSync(imagePath);
         const base64Image = imageData.toString("base64");
@@ -71,11 +73,25 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
     }
     try {
         const ai = new GoogleGenAI({ apiKey });
-        if (model === "gemini-2.5-flash-image" || model === "gemini-3-pro-image-preview") {
-            const contents = getGeminiContents(prompt, aspectRatio, referenceImages);
+        if (model === "gemini-2.5-flash-image") {
+            const contents = getGeminiContents(prompt, referenceImages, aspectRatio);
             const response = await ai.models.generateContent({ model, contents });
             return geminiFlashResult(response);
         }
+        else if (model === "gemini-3-pro-image-preview") {
+            const contents = getGeminiContents(prompt, referenceImages);
+            const response = await ai.models.generateContent({
+                model,
+                contents,
+                config: {
+                    imageConfig: {
+                        // '1:1', '2:3', '3:2', '3:4', '4:3', '4:5', '5:4', '9:16', '16:9', or '21:9'.
+                        aspectRatio,
+                    },
+                },
+            });
+            return geminiFlashResult(response);
+        }
         else {
             const response = await ai.models.generateImages({
                 model,

package/lib/agents/index.d.ts

@@ -11,6 +11,7 @@ import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
 import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
 import ttsOpenaiAgent from "./tts_openai_agent.js";
 import ttsGoogleAgent from "./tts_google_agent.js";
+import ttsGeminiAgent from "./tts_gemini_agent.js";
 import validateSchemaAgent from "./validate_schema_agent.js";
 import soundEffectReplicateAgent from "./sound_effect_replicate_agent.js";
 import lipSyncReplicateAgent from "./lipsync_replicate_agent.js";
@@ -19,4 +20,4 @@ import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, ttsGeminiAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };

package/lib/agents/index.js

@@ -11,6 +11,7 @@ import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
 import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
 import ttsOpenaiAgent from "./tts_openai_agent.js";
 import ttsGoogleAgent from "./tts_google_agent.js";
+import ttsGeminiAgent from "./tts_gemini_agent.js";
 import validateSchemaAgent from "./validate_schema_agent.js";
 import soundEffectReplicateAgent from "./sound_effect_replicate_agent.js";
 import lipSyncReplicateAgent from "./lipsync_replicate_agent.js";
@@ -20,4 +21,4 @@ import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 // import * as vanilla from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, ttsGeminiAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };

package/lib/agents/tts_gemini_agent.d.ts

@@ -0,0 +1,5 @@
+import type { AgentFunction, AgentFunctionInfo } from "graphai";
+import type { GoogleTTSAgentParams, AgentBufferResult, AgentTextInputs, AgentErrorResult } from "../types/agent.js";
+export declare const ttsGeminiAgent: AgentFunction<GoogleTTSAgentParams, AgentBufferResult | AgentErrorResult, AgentTextInputs>;
+declare const ttsGeminiAgentInfo: AgentFunctionInfo;
+export default ttsGeminiAgentInfo;

package/lib/agents/tts_gemini_agent.js

@@ -0,0 +1,64 @@
+import { GraphAILogger } from "graphai";
+import { GoogleGenAI } from "@google/genai";
+import { provider2TTSAgent } from "../utils/provider2agent.js";
+import { apiKeyMissingError, agentGenerationError, audioAction, audioFileTarget } from "../utils/error_cause.js";
+import { pcmToMp3 } from "../utils/ffmpeg_utils.js";
+export const ttsGeminiAgent = async ({ namedInputs, params, config, }) => {
+    const { text } = namedInputs;
+    const { voice, suppressError } = params;
+    const apiKey = config?.apiKey;
+    if (!apiKey) {
+        throw new Error("Google GenAI API key is required (GEMINI_API_KEY)", {
+            cause: apiKeyMissingError("ttsGeminiAgent", audioAction, "GEMINI_API_KEY"),
+        });
+    }
+    try {
+        const ai = new GoogleGenAI({ apiKey });
+        const response = await ai.models.generateContent({
+            model: "gemini-2.5-flash-preview-tts",
+            contents: [{ parts: [{ text }] }],
+            config: {
+                responseModalities: ["AUDIO"],
+                speechConfig: {
+                    voiceConfig: {
+                        prebuiltVoiceConfig: { voiceName: voice ?? provider2TTSAgent.gemini.defaultVoice },
+                    },
+                },
+            },
+        });
+        const inlineData = response.candidates?.[0]?.content?.parts?.[0]?.inlineData;
+        const pcmBase64 = inlineData?.data;
+        const mimeType = inlineData?.mimeType;
+        if (!pcmBase64)
+            throw new Error("No audio data returned");
+        // Extract sample rate from mimeType (e.g., "audio/L16;codec=pcm;rate=24000")
+        const rateMatch = mimeType?.match(/rate=(\d+)/);
+        const sampleRate = rateMatch ? parseInt(rateMatch[1]) : 24000;
+        const rawPcm = Buffer.from(pcmBase64, "base64");
+        return { buffer: await pcmToMp3(rawPcm, sampleRate) };
+    }
+    catch (e) {
+        if (suppressError) {
+            return {
+                error: e,
+            };
+        }
+        GraphAILogger.info(e);
+        throw new Error("TTS Gemini Error", {
+            cause: agentGenerationError("ttsGeminiAgent", audioAction, audioFileTarget),
+        });
+    }
+};
+const ttsGeminiAgentInfo = {
+    name: "ttsGeminiAgent",
+    agent: ttsGeminiAgent,
+    mock: ttsGeminiAgent,
+    samples: [],
+    description: "Google Gemini TTS agent",
+    category: ["tts"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    license: "MIT",
+    environmentVariables: ["GEMINI_API_KEY"],
+};
+export default ttsGeminiAgentInfo;

package/lib/cli/commands/tool/scripting/builder.d.ts

@@ -16,7 +16,7 @@ export declare const builder: (yargs: Argv) => Argv<{
 } & {
     s: string;
 } & {
-    llm: "mock" | "openai" | "anthropic" | "gemini" | "groq" | undefined;
+    llm: "mock" | "openai" | "gemini" | "anthropic" | "groq" | undefined;
 } & {
     llm_model: string | undefined;
 }>;

package/lib/cli/commands/tool/story_to_script/builder.d.ts

@@ -10,7 +10,7 @@ export declare const builder: (yargs: Argv) => Argv<{
 } & {
     beats_per_scene: number;
 } & {
-    llm: "mock" | "openai" | "anthropic" | "gemini" | "groq" | undefined;
+    llm: "mock" | "openai" | "gemini" | "anthropic" | "groq" | undefined;
 } & {
     llm_model: string | undefined;
 } & {

package/lib/utils/ffmpeg_utils.d.ts

@@ -18,3 +18,4 @@ export declare const ffmpegGetMediaDuration: (filePath: string) => Promise<{
 export declare const extractImageFromMovie: (movieFile: string, imagePath: string) => Promise<object>;
 export declare const trimMusic: (inputFile: string, startTime: number, duration: number) => Promise<Buffer>;
 export declare const createSilentAudio: (filePath: string, durationSec: number) => Promise<void>;
+export declare const pcmToMp3: (rawPcm: Buffer, sampleRate?: number) => Promise<Buffer>;

package/lib/utils/ffmpeg_utils.js

@@ -2,6 +2,7 @@ import ffmpeg from "fluent-ffmpeg";
 import { GraphAILogger } from "graphai";
 import { isFile } from "./file.js";
 import fs from "fs";
+import { Readable, PassThrough } from "node:stream";
 export const setFfmpegPath = (ffmpegPath) => {
     ffmpeg.setFfmpegPath(ffmpegPath);
 };
@@ -144,3 +145,25 @@ export const createSilentAudio = (filePath, durationSec) => {
             .run();
     });
 };
+export const pcmToMp3 = (rawPcm, sampleRate = 24000) => {
+    return new Promise((resolve, reject) => {
+        const inputStream = new Readable({
+            read() {
+                this.push(rawPcm);
+                this.push(null);
+            },
+        });
+        const outputChunks = [];
+        const outputStream = new PassThrough();
+        outputStream.on("data", (chunk) => outputChunks.push(chunk));
+        outputStream.on("end", () => resolve(Buffer.concat(outputChunks)));
+        outputStream.on("error", reject);
+        ffmpeg(inputStream)
+            .inputFormat("s16le")
+            .inputOptions([`-ar ${sampleRate}`, "-ac 1"])
+            .audioCodec("libmp3lame")
+            .format("mp3")
+            .on("error", reject)
+            .pipe(outputStream);
+    });
+};

package/lib/utils/provider2agent.d.ts

@@ -16,6 +16,12 @@ export declare const provider2TTSAgent: {
         hasLimitedConcurrency: boolean;
         keyName: string;
     };
+    gemini: {
+        agentName: string;
+        hasLimitedConcurrency: boolean;
+        defaultVoice: string;
+        keyName: string;
+    };
     elevenlabs: {
         agentName: string;
         hasLimitedConcurrency: boolean;

package/lib/utils/provider2agent.js

@@ -17,6 +17,12 @@ export const provider2TTSAgent = {
         hasLimitedConcurrency: false,
         keyName: "GEMINI_API_KEY",
     },
+    gemini: {
+        agentName: "ttsGeminiAgent",
+        hasLimitedConcurrency: false,
+        defaultVoice: "Kore",
+        keyName: "GEMINI_API_KEY",
+    },
     elevenlabs: {
         agentName: "ttsElevenlabsAgent",
         hasLimitedConcurrency: true,

package/lib/utils/utils.js

@@ -73,6 +73,9 @@ export const settings2GraphAIConfig = (settings, env) => {
         ttsNijivoiceAgent: {
             apiKey: getKey("TTS", "NIJIVOICE_API_KEY"),
         },
+        ttsGeminiAgent: {
+            apiKey: getKey("TTS", "GEMINI_API_KEY"),
+        },
         ttsElevenlabsAgent: {
             apiKey: getKey("TTS", "ELEVENLABS_API_KEY"),
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.0.3",
+  "version": "2.0.5",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -102,7 +102,7 @@
     "replicate": "^1.4.0",
     "yaml": "^2.8.1",
     "yargs": "^18.0.0",
-    "zod": "^4.1.12"
+    "zod": "^4.1.13"
   },
   "devDependencies": {
     "@receptron/test_utils": "^2.0.3",
@@ -117,7 +117,7 @@
     "prettier": "^3.6.2",
     "tsx": "^4.20.6",
     "typescript": "^5.9.3",
-    "typescript-eslint": "^8.47.0"
+    "typescript-eslint": "^8.48.0"
   },
   "engines": {
     "node": ">=20.0.0"

package/scripts/test/test_audio_gemini.json

@@ -0,0 +1,67 @@
+{
+  "$mulmocast": {
+    "version": "1.1"
+  },
+  "lang": "en",
+  "title": "Audio Instructions Test",
+  "speechParams": {
+    "speakers": {
+      "Presenter": {
+        "provider": "gemini",
+        "voiceId": "Kore"
+      },
+      "Presenter2": {
+        "provider": "gemini",
+        "voiceId": "Puck"
+      }
+    }
+  },
+  "beats": [
+    {
+      "speaker": "Presenter",
+      "text": "Hello, I'm a presenter. I have no instructions.",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Presenter"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter2",
+      "text": "Hello, I'm a presenter 2. My instructions are 'Speak in a cheerful and positive tone'.",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Presenter 2"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "Hello, I'm a presenter. I have a British English instruction.",
+      "speechOptions": {
+        "instruction": "Speak in British English."
+      },
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Presenter with British English instruction"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "Hello, I'm a presenter. I have a whisper instruction.",
+      "speechOptions": {
+        "instruction": "Whisper softly, like a pillow talk."
+      },
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Presenter with whisper instruction"
+        }
+      }
+    }
+  ]
+}

package/scripts/test/test_audio_gemini.json~

@@ -0,0 +1,67 @@
+{
+  "$mulmocast": {
+    "version": "1.1"
+  },
+  "lang": "en",
+  "title": "Audio Instructions Test",
+  "speechParams": {
+    "speakers": {
+      "Presenter": {
+        "provider": "google",
+        "voiceId": "ja-JP-Standard-A"
+      },
+      "Presenter2": {
+        "provider": "google",
+        "voiceId": "ja-JP-Standard-B"
+      }
+    }
+  },
+  "beats": [
+    {
+      "speaker": "Presenter",
+      "text": "Hello, I'm a presenter. I have no instructions.",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Presenter"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter2",
+      "text": "Hello, I'm a presenter 2. My instructions are 'Speak in a cheerful and positive tone'.",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Presenter 2"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "Hello, I'm a presenter. I have a British English instruction.",
+      "speechOptions": {
+        "instruction": "Speak in British English."
+      },
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Presenter with British English instruction"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "Hello, I'm a presenter. I have a whisper instruction.",
+      "speechOptions": {
+        "instruction": "Whisper softly, like a pillow talk."
+      },
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Presenter with whisper instruction"
+        }
+      }
+    }
+  ]
+}

package/scripts/test/test_genai2.json

@@ -0,0 +1,25 @@
+{
+  "$mulmocast": { "version": "1.1" },
+  "imageParams": {
+    "provider": "google",
+    "style": "<style>Photo realistic.</style>"
+  },
+  "canvasSize": {
+    "width": 720,
+    "height": 1280
+  },
+  "movieParams": {
+    "provider": "google"
+  },
+  "lang": "en",
+  "beats": [
+    {
+      "id": "gemini_3_pro_image_preview",
+      "text": "image generated by gemini-3-pro-image-preview",
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "imageParams": {
+        "model": "gemini-3-pro-image-preview"
+      }
+    }
+  ]
+}

package/scripts/test/test_genai2.json~

@@ -0,0 +1,84 @@
+{
+  "$mulmocast": { "version": "1.1" },
+  "imageParams": {
+    "provider": "google",
+    "style": "<style>Photo realistic.</style>"
+  },
+  "movieParams": {
+    "provider": "google"
+  },
+  "lang": "en",
+  "beats": [
+    {
+      "id": "gemini_3_pro_image_preview",
+      "text": "image generated by gemini-3-pro-image-preview",
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "imageParams": {
+        "model": "gemini-3-pro-image-preview"
+      }
+    },
+    {
+      "id": "gemini_2_5_flash_image",
+      "text": "image generated by gemini-2.5-flash-image",
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "imageParams": {
+        "model": "gemini-2.5-flash-image"
+      }
+    },
+    {
+      "id": "imagen_4",
+      "text": "image generated by imagen-4",
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses"
+    },
+    {
+      "id": "imagen_4_ultra",
+      "text": "image generated by imagen-4",
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "imageParams": {
+        "model": "imagen-4.0-ultra-generate-preview-06-06"
+      }
+    },
+    {
+      "id": "genai_veo2",
+      "text": "movie generated by veo2",
+      "duration": 5,
+      "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses"
+    },
+    {
+      "id": "genai_veo2_image",
+      "text": "movie generated by veo2 with image",
+      "duration": 5,
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "moviePrompt": "a woman takes a selfie with her phone"
+    },
+    {
+      "id": "genai_veo3",
+      "text": "movie generated by veo3",
+      "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "movieParams": {
+        "model": "veo-3.0-generate-001"
+      }
+    },
+    {
+      "id": "genai_veo3_1",
+      "text": "movie generated by veo3_1",
+      "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "movieParams": {
+        "model": "veo-3.1-generate-preview"
+      }
+    },
+    {
+      "id": "genai_veo3_image",
+      "text": "movie generated by veo3",
+      "duration": 5,
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "imageParams": {
+        "model": "gemini-2.5-flash-image"
+      },
+      "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "movieParams": {
+        "model": "veo-3.0-generate-001"
+      }
+    }
+  ]
+}