mulmocast 2.6.8 → 2.6.10

package/lib/agents/image_genai_agent.js

@@ -60,14 +60,16 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
     const model = params.model ?? provider2ImageAgent["google"].defaultModel;
     const apiKey = config?.apiKey;
     const ai = params.vertexai_project
-        ? new GoogleGenAI({
-            vertexai: true,
-            project: params.vertexai_project,
-            location: params.vertexai_location ?? "us-central1",
-        })
+        ? (() => {
+            const location = params.vertexai_location ?? "us-central1";
+            if (model === "gemini-3-pro-image-preview" && location !== "global") {
+                GraphAILogger.warn(`imageGenAIAgent: model "${model}" on Vertex AI is only available in location "global", but got "${location}". Set imageParams.vertexai_location to "global".`);
+            }
+            return new GoogleGenAI({ vertexai: true, project: params.vertexai_project, location });
+        })()
         : (() => {
             if (!apiKey) {
-                throw new Error("Google GenAI API key is required (GEMINI_API_KEY)", {
+                throw new Error("Google GenAI authentication is required. Either set GEMINI_API_KEY (Gemini API) or specify imageParams.vertexai_project (Vertex AI). See docs/vertexai_en.md or docs/vertexai_ja.md.", {
                     cause: apiKeyMissingError("imageGenAIAgent", imageAction, "GEMINI_API_KEY"),
                 });
             }

package/lib/agents/image_replicate_agent.js

@@ -4,6 +4,22 @@ import Replicate from "replicate";
 import { getAspectRatio } from "./movie_replicate_agent.js";
 import { apiKeyMissingError, agentIncorrectAPIKeyError, agentGenerationError, agentInvalidResponseError, imageAction, imageFileTarget, hasCause, } from "../utils/error_cause.js";
 import { provider2ImageAgent } from "../types/provider2agent.js";
+// Replicate image models return one of: FileOutput (object with url() method),
+// Array<FileOutput>, string URL, or { url: string }. Normalize to URL string/URL.
+const extractImageUrl = (output) => {
+    if (typeof output === "string")
+        return output;
+    if (Array.isArray(output))
+        return output.length > 0 ? extractImageUrl(output[0]) : undefined;
+    if (output && typeof output === "object" && "url" in output) {
+        const url = output.url;
+        if (typeof url === "function")
+            return url();
+        if (typeof url === "string")
+            return url;
+    }
+    return undefined;
+};
 export const imageReplicateAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, referenceImages } = namedInputs;
     const { canvasSize } = params;
@@ -29,22 +45,21 @@ export const imageReplicateAgent = async ({ namedInputs, params, config, }) => {
     }
     try {
         const output = await replicate.run(model, { input });
-        // Download the generated video
-        if (output && Array.isArray(output) && output.length > 0 && typeof output[0] === "object" && "url" in output[0]) {
-            const imageUrl = output[0].url();
-            const imageResponse = await fetch(imageUrl);
-            if (!imageResponse.ok) {
-                throw new Error(`Error downloading image: ${imageResponse.status} - ${imageResponse.statusText}`, {
-                    cause: agentGenerationError("imageReplicateAgent", imageAction, imageFileTarget),
-                });
-            }
-            const arrayBuffer = await imageResponse.arrayBuffer();
-            const buffer = Buffer.from(arrayBuffer);
-            return { buffer };
+        const imageUrl = extractImageUrl(output);
+        if (!imageUrl) {
+            throw new Error("ERROR: generateImage returned undefined", {
+                cause: agentInvalidResponseError("imageReplicateAgent", imageAction, imageFileTarget),
+            });
         }
-        throw new Error("ERROR: generateImage returned undefined", {
-            cause: agentInvalidResponseError("imageReplicateAgent", imageAction, imageFileTarget),
-        });
+        const imageResponse = await fetch(imageUrl);
+        if (!imageResponse.ok) {
+            throw new Error(`Error downloading image: ${imageResponse.status} - ${imageResponse.statusText}`, {
+                cause: agentGenerationError("imageReplicateAgent", imageAction, imageFileTarget),
+            });
+        }
+        const arrayBuffer = await imageResponse.arrayBuffer();
+        const buffer = Buffer.from(arrayBuffer);
+        return { buffer };
     }
     catch (error) {
         GraphAILogger.info("Replicate generation error:", error);

package/lib/agents/movie_genai_agent.js

@@ -175,7 +175,7 @@ export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
             })
             : (() => {
                 if (!apiKey) {
-                    throw new Error("Google GenAI API key is required (GEMINI_API_KEY)", {
+                    throw new Error("Google GenAI authentication is required. Either set GEMINI_API_KEY (Gemini API) or specify movieParams.vertexai_project (Vertex AI). See docs/vertexai_en.md or docs/vertexai_ja.md.", {
                         cause: apiKeyMissingError("movieGenAIAgent", imageAction, "GEMINI_API_KEY"),
                     });
                 }

package/lib/agents/movie_replicate_agent.js

@@ -136,6 +136,11 @@ export const movieReplicateAgent = async ({ namedInputs, params, config, }) => {
             cause: agentGenerationError("movieReplicateAgent", imageAction, unsupportedModelTarget),
         });
     }
+    if (provider2MovieAgent.replicate.modelParams[model].start_image_required && !imagePath) {
+        throw new Error(`Model ${model} requires a start image (image-to-video only)`, {
+            cause: agentGenerationError("movieReplicateAgent", imageAction, unsupportedModelTarget),
+        });
+    }
     const duration = getModelDuration("replicate", model, params.duration);
     if (duration === undefined || !provider2MovieAgent.replicate.modelParams[model].durations.includes(duration)) {
         throw new Error(`Duration ${duration} is not supported for model ${model}. Supported durations: ${provider2MovieAgent.replicate.modelParams[model].durations.join(", ")}`, {

package/lib/agents/tts_google_agent.js

@@ -2,6 +2,10 @@ import { GraphAILogger } from "graphai";
 import * as textToSpeech from "@google-cloud/text-to-speech";
 import { agentGenerationError, audioAction, audioFileTarget } from "../utils/error_cause.js";
 const client = new textToSpeech.TextToSpeechClient();
+// Hard cap so a hung Google TTS RPC can't pin a beat indefinitely.
+// Most synthesizeSpeech calls return in seconds; 60s leaves headroom
+// for long inputs and slow regions while still failing loud.
+const SYNTHESIZE_TIMEOUT_MS = 60_000;
 const getPrompt = (text, instructions) => {
     if (instructions) {
         return `### DIRECTOR'S NOTES\n${instructions}\n\n#### TRANSCRIPT\n${text}`;
@@ -37,7 +41,7 @@ export const ttsGoogleAgent = async ({ namedInputs, params }) => {
     };
     try {
         // Call the Text-to-Speech API
-        const [response] = await client.synthesizeSpeech(request);
+        const [response] = await client.synthesizeSpeech(request, { timeout: SYNTHESIZE_TIMEOUT_MS });
         return { buffer: response.audioContent };
     }
     catch (e) {
@@ -47,11 +51,23 @@ export const ttsGoogleAgent = async ({ namedInputs, params }) => {
             };
         }
         GraphAILogger.info(e);
-        throw new Error("TTS Google Error", {
+        // gRPC errors from @google-cloud/text-to-speech are ServiceError
+        // (extends Error with a `details` string). Surface that human-readable
+        // text so callers don't see only "TTS Google Error".
+        throw new Error(`TTS Google Error: ${grpcErrorDetail(e)}`, {
             cause: agentGenerationError("ttsGoogleAgent", audioAction, audioFileTarget),
         });
     }
 };
+const grpcErrorDetail = (e) => {
+    if (e instanceof Error) {
+        const details = e.details;
+        if (typeof details === "string" && details)
+            return details;
+        return e.message;
+    }
+    return String(e);
+};
 const ttsGoogleAgentInfo = {
     name: "ttsGoogleAgent",
     agent: ttsGoogleAgent,

package/lib/types/provider2agent.d.ts

@@ -84,6 +84,7 @@ type MovieAudioSpec = {
 type ReplicateMovieModelParams = {
     durations: number[];
     start_image: string | undefined;
+    start_image_required?: boolean;
     last_image?: string;
     reference_images_param?: string;
     audio: MovieAudioSpec;

package/lib/types/provider2agent.js

@@ -44,7 +44,7 @@ export const provider2TTSAgent = {
         models: ["mock-model"],
     },
 };
-export const gptImages = ["gpt-image-1.5", "gpt-image-1", "gpt-image-1-mini"];
+export const gptImages = ["gpt-image-2", "gpt-image-1.5", "gpt-image-1", "gpt-image-1-mini"];
 export const provider2ImageAgent = {
     openai: {
         agentName: "imageOpenaiAgent",
@@ -69,7 +69,23 @@ export const provider2ImageAgent = {
     replicate: {
         agentName: "imageReplicateAgent",
         defaultModel: "bytedance/seedream-4",
-        models: ["bytedance/seedream-4", "qwen/qwen-image"],
+        models: [
+            "bytedance/seedream-4",
+            "qwen/qwen-image",
+            "black-forest-labs/flux-2-pro",
+            "black-forest-labs/flux-2-dev",
+            "black-forest-labs/flux-1.1-pro",
+            "black-forest-labs/flux-1.1-pro-ultra",
+            "black-forest-labs/flux-pro",
+            "black-forest-labs/flux-dev",
+            "black-forest-labs/flux-schnell",
+            "ideogram-ai/ideogram-v3-turbo",
+            "ideogram-ai/ideogram-v3-balanced",
+            "ideogram-ai/ideogram-v3-quality",
+            "recraft-ai/recraft-v3",
+            "stability-ai/stable-diffusion-3.5-large",
+            "luma/photon",
+        ],
         keyName: "REPLICATE_API_TOKEN",
     },
     mock: {
@@ -112,6 +128,10 @@ export const provider2MovieAgent = {
             "runwayml/gen-4.5",
             "kwaivgi/kling-v3-omni-video",
             "kwaivgi/kling-v3-video",
+            "alibaba/happyhorse-1.0",
+            "minimax/hailuo-2.3",
+            "minimax/hailuo-2.3-fast",
+            "pixverse/pixverse-v5",
         ],
         modelParams: {
             "bytedance/seedance-1-lite": {
@@ -273,6 +293,36 @@ export const provider2MovieAgent = {
                 audio: { mode: AUDIO_MODE_OPTIONAL, param: "generate_audio" },
                 price_per_sec: 0.3,
             },
+            // TODO: price_per_sec for the models below is a coarse approximation.
+            // Actual Replicate pricing varies by resolution / duration / quality and
+            // cannot be expressed as a single per-second number. Verify each model at
+            // https://replicate.com/<owner>/<model> when this field starts being consumed.
+            "alibaba/happyhorse-1.0": {
+                durations: [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
+                start_image: "image",
+                audio: { mode: AUDIO_MODE_NEVER },
+                price_per_sec: 0.05,
+            },
+            "minimax/hailuo-2.3": {
+                durations: [6, 10],
+                start_image: "first_frame_image",
+                audio: { mode: AUDIO_MODE_NEVER },
+                price_per_sec: 0.1,
+            },
+            "minimax/hailuo-2.3-fast": {
+                durations: [6, 10],
+                start_image: "first_frame_image",
+                start_image_required: true,
+                audio: { mode: AUDIO_MODE_NEVER },
+                price_per_sec: 0.06,
+            },
+            "pixverse/pixverse-v5": {
+                durations: [5, 8],
+                start_image: "image",
+                last_image: "last_frame_image",
+                audio: { mode: AUDIO_MODE_NEVER },
+                price_per_sec: 0.12,
+            },
         },
     },
     google: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.6.8",
+  "version": "2.6.10",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -109,11 +109,11 @@
     "dotenv": "^17.4.2",
     "fluent-ffmpeg": "^2.1.3",
     "graphai": "^2.0.16",
-    "jsdom": "^29.0.2",
+    "jsdom": "^29.1.0",
     "marked": "^18.0.2",
     "mulmocast-vision": "^1.0.9",
-    "ora": "^9.3.0",
-    "puppeteer": "^24.41.0",
+    "ora": "^9.4.0",
+    "puppeteer": "^24.42.0",
     "replicate": "^1.4.0",
     "yaml": "^2.8.3",
     "yargs": "^18.0.0",
@@ -136,7 +136,7 @@
     "prettier": "^3.8.3",
     "tsx": "^4.21.0",
     "typescript": "6.0.3",
-    "typescript-eslint": "^8.58.1"
+    "typescript-eslint": "^8.59.1"
   },
   "engines": {
     "node": ">=22.0.0"

package/scripts/test/README.md

@@ -106,7 +106,7 @@ Language setting tests
 Provider-specific feature tests
 
 - [**test_hello_google.json**](./test_hello_google.json) - Google TTS専用テスト / Google TTS specific test
-- [**gpt.json**](./gpt.json) - GPTモデルテスト / GPT model test
+- [**test_gpt_image.json**](./test_gpt_image.json) - GPT image model test
 - [**mulmo_story.json**](./mulmo_story.json) - ストーリー形式テスト / Story format test
 - [**nano_banana.json**](./nano_banana.json) - カスタムサンプル / Custom sample
 

package/scripts/test/test_flux2.json

@@ -0,0 +1,29 @@
+{
+  "$mulmocast": {
+    "version": "1.1"
+  },
+  "canvasSize": {
+    "width": 1024,
+    "height": 1024
+  },
+  "imageParams": {
+    "provider": "replicate",
+    "model": "black-forest-labs/flux-2-pro"
+  },
+  "speechParams": {
+    "speakers": {
+      "Presenter": {
+        "voiceId": "shimmer",
+        "provider": "openai"
+      }
+    }
+  },
+  "lang": "en",
+  "beats": [
+    {
+      "speaker": "Presenter",
+      "text": "Testing flux-2-pro on Replicate.",
+      "imagePrompt": "A cinematic photo of a red panda holding a tiny espresso cup, soft window light, shallow depth of field"
+    }
+  ]
+}

package/scripts/test/{gpt.json → test_gpt_image.json}

@@ -21,6 +21,14 @@
     }
   },
   "beats": [
+    {
+      "speaker": "Host",
+      "text": "How are you?",
+      "imagePrompt": "A witch in Harajuku",
+      "imageParams": {
+        "model": "gpt-image-2"
+      }
+    },
     {
       "speaker": "Host",
       "text": "How are you?",

package/scripts/test/test_replicate_images.json

@@ -0,0 +1,79 @@
+{
+  "$mulmocast": { "version": "1.1" },
+  "canvasSize": { "width": 1024, "height": 1024 },
+  "imageParams": {
+    "provider": "replicate"
+  },
+  "speechParams": {
+    "speakers": {
+      "Presenter": {
+        "voiceId": "shimmer",
+        "provider": "openai"
+      }
+    }
+  },
+  "audioParams": { "bgmVolume": 0 },
+  "lang": "en",
+  "title": "Replicate Image Models Showcase",
+  "beats": [
+    {
+      "speaker": "Presenter",
+      "text": "PROMPT: A red panda holding a tiny espresso cup, soft window light, cinematic.",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "PROMPT: A red panda holding a tiny espresso cup, soft window light, cinematic."
+        }
+      }
+    },
+    {
+      "id": "flux-2-pro",
+      "speaker": "Presenter",
+      "text": "Flux 2 Pro from Black Forest Labs — high quality text-to-image.",
+      "imagePrompt": "A red panda holding a tiny espresso cup, soft window light, cinematic",
+      "imageParams": { "model": "black-forest-labs/flux-2-pro" }
+    },
+    {
+      "id": "flux-1.1-pro-ultra",
+      "speaker": "Presenter",
+      "text": "Flux 1.1 Pro Ultra — ultra-high resolution variant.",
+      "imagePrompt": "A red panda holding a tiny espresso cup, soft window light, cinematic",
+      "imageParams": { "model": "black-forest-labs/flux-1.1-pro-ultra" }
+    },
+    {
+      "id": "flux-schnell",
+      "speaker": "Presenter",
+      "text": "Flux Schnell — fastest open Flux model.",
+      "imagePrompt": "A red panda holding a tiny espresso cup, soft window light, cinematic",
+      "imageParams": { "model": "black-forest-labs/flux-schnell" }
+    },
+    {
+      "id": "ideogram-v3-turbo",
+      "speaker": "Presenter",
+      "text": "Ideogram v3 Turbo — strong at typography and posters.",
+      "imagePrompt": "A red panda holding a tiny espresso cup, soft window light, cinematic",
+      "imageParams": { "model": "ideogram-ai/ideogram-v3-turbo" }
+    },
+    {
+      "id": "recraft-v3",
+      "speaker": "Presenter",
+      "text": "Recraft v3 — strong at clean illustration and text in images.",
+      "imagePrompt": "A red panda holding a tiny espresso cup, soft window light, cinematic",
+      "imageParams": { "model": "recraft-ai/recraft-v3" }
+    },
+    {
+      "id": "sd-3.5-large",
+      "speaker": "Presenter",
+      "text": "Stable Diffusion 3.5 Large — open weights baseline.",
+      "imagePrompt": "A red panda holding a tiny espresso cup, soft window light, cinematic",
+      "imageParams": { "model": "stability-ai/stable-diffusion-3.5-large" }
+    },
+    {
+      "id": "luma-photon",
+      "speaker": "Presenter",
+      "text": "Luma Photon — strong photorealism and style transfer.",
+      "imagePrompt": "A red panda holding a tiny espresso cup, soft window light, cinematic",
+      "imageParams": { "model": "luma/photon" }
+    }
+  ]
+}

package/scripts/test/test_replicate_new_movies.json

@@ -0,0 +1,50 @@
+{
+  "$mulmocast": { "version": "1.1" },
+  "movieParams": {
+    "provider": "replicate"
+  },
+  "audioParams": { "bgmVolume": 0 },
+  "captionParams": { "lang": "en" },
+  "lang": "en",
+  "title": "Replicate New Movie Models Showcase",
+  "beats": [
+    {
+      "text": "PROMPT: a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "PROMPT: a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses"
+        }
+      }
+    },
+    {
+      "id": "alibaba-happyhorse",
+      "text": "alibaba/happyhorse-1.0",
+      "duration": 5,
+      "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "movieParams": { "model": "alibaba/happyhorse-1.0" }
+    },
+    {
+      "id": "hailuo-23",
+      "text": "minimax/hailuo-2.3",
+      "duration": 6,
+      "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "movieParams": { "model": "minimax/hailuo-2.3" }
+    },
+    {
+      "id": "pixverse-v5",
+      "text": "pixverse/pixverse-v5",
+      "duration": 5,
+      "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "movieParams": { "model": "pixverse/pixverse-v5" }
+    },
+    {
+      "id": "hailuo-23-fast",
+      "text": "minimax/hailuo-2.3-fast (i2v: starts from a generated still image)",
+      "duration": 6,
+      "imagePrompt": "a woman wearing dark sunglasses standing on a busy Tokyo street at night, neon reflections, cinematic still",
+      "moviePrompt": "the woman starts walking forward, neon reflections shimmer on the wet pavement",
+      "movieParams": { "model": "minimax/hailuo-2.3-fast" }
+    }
+  ]
+}