mulmocast 2.0.4 → 2.0.6

package/lib/agents/image_genai_agent.js

@@ -22,11 +22,13 @@ export const ratio2BlankPath = (aspectRatio) => {
     }
     return blankImagePath();
 };
-const getGeminiContents = (prompt, aspectRatio, referenceImages) => {
+const getGeminiContents = (prompt, referenceImages, aspectRatio) => {
     const contents = [{ text: prompt }];
     const images = [...(referenceImages ?? [])];
     // NOTE: There is no way to explicitly specify the aspect ratio for Gemini. This is just a hint.
-    images.push(ratio2BlankPath(aspectRatio));
+    if (aspectRatio) {
+        images.push(ratio2BlankPath(aspectRatio));
+    }
     images.forEach((imagePath) => {
         const imageData = fs.readFileSync(imagePath);
         const base64Image = imageData.toString("base64");
@@ -71,11 +73,25 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
     }
     try {
         const ai = new GoogleGenAI({ apiKey });
-        if (model === "gemini-2.5-flash-image" || model === "gemini-3-pro-image-preview") {
-            const contents = getGeminiContents(prompt, aspectRatio, referenceImages);
+        if (model === "gemini-2.5-flash-image") {
+            const contents = getGeminiContents(prompt, referenceImages, aspectRatio);
             const response = await ai.models.generateContent({ model, contents });
             return geminiFlashResult(response);
         }
+        else if (model === "gemini-3-pro-image-preview") {
+            const contents = getGeminiContents(prompt, referenceImages);
+            const response = await ai.models.generateContent({
+                model,
+                contents,
+                config: {
+                    imageConfig: {
+                        // '1:1', '2:3', '3:2', '3:4', '4:3', '4:5', '5:4', '9:16', '16:9', or '21:9'.
+                        aspectRatio,
+                    },
+                },
+            });
+            return geminiFlashResult(response);
+        }
         else {
             const response = await ai.models.generateImages({
                 model,

package/lib/agents/movie_genai_agent.js

@@ -1,7 +1,7 @@
 import { readFileSync } from "fs";
 import { GraphAILogger, sleep } from "graphai";
-import { apiKeyMissingError, agentGenerationError, agentInvalidResponseError, imageAction, movieFileTarget, videoDurationTarget, hasCause, } from "../utils/error_cause.js";
 import { GoogleGenAI, PersonGeneration } from "@google/genai";
+import { apiKeyMissingError, agentGenerationError, agentInvalidResponseError, imageAction, movieFileTarget, videoDurationTarget, hasCause, } from "../utils/error_cause.js";
 import { getModelDuration, provider2MovieAgent } from "../utils/provider2agent.js";
 export const getAspectRatio = (canvasSize) => {
     if (canvasSize.width > canvasSize.height) {
@@ -14,6 +14,105 @@ export const getAspectRatio = (canvasSize) => {
         return "1:1";
     }
 };
+const pollUntilDone = async (ai, operation) => {
+    const response = { operation };
+    while (!response.operation.done) {
+        await sleep(5000);
+        response.operation = await ai.operations.getVideosOperation(response);
+    }
+    return response;
+};
+const getVideoFromResponse = (response, iteration) => {
+    const iterationInfo = iteration !== undefined ? ` in iteration ${iteration}` : "";
+    if (!response.operation.response?.generatedVideos) {
+        throw new Error(`No video${iterationInfo}: ${JSON.stringify(response.operation, null, 2)}`, {
+            cause: agentInvalidResponseError("movieGenAIAgent", imageAction, movieFileTarget),
+        });
+    }
+    const video = response.operation.response.generatedVideos[0].video;
+    if (!video) {
+        throw new Error(`No video${iterationInfo}`, {
+            cause: agentInvalidResponseError("movieGenAIAgent", imageAction, movieFileTarget),
+        });
+    }
+    return video;
+};
+const loadImageAsBase64 = (imagePath) => {
+    const buffer = readFileSync(imagePath);
+    return {
+        imageBytes: buffer.toString("base64"),
+        mimeType: "image/png",
+    };
+};
+const downloadVideo = async (ai, video, movieFile) => {
+    await ai.files.download({
+        file: video,
+        downloadPath: movieFile,
+    });
+    await sleep(5000); // HACK: Without this, the file is not ready yet.
+    return { saved: movieFile };
+};
+const createVeo31Payload = (model, prompt, aspectRatio, source) => ({
+    model,
+    prompt,
+    config: {
+        aspectRatio,
+        resolution: "720p",
+        numberOfVideos: 1,
+    },
+    ...source,
+});
+const generateExtendedVideo = async (ai, model, prompt, aspectRatio, imagePath, requestedDuration, movieFile) => {
+    const initialDuration = 8;
+    const maxExtensionDuration = 8;
+    const extensionsNeeded = Math.ceil((requestedDuration - initialDuration) / maxExtensionDuration);
+    GraphAILogger.info(`Veo 3.1 video extension: ${extensionsNeeded} extensions needed for ${requestedDuration}s target`);
+    const generateIteration = async (iteration, accumulatedDuration, previousVideo) => {
+        const isInitial = iteration === 0;
+        const remainingDuration = requestedDuration - accumulatedDuration;
+        const extensionDuration = isInitial ? initialDuration : (getModelDuration("google", model, remainingDuration) ?? maxExtensionDuration);
+        const getSource = () => {
+            if (isInitial)
+                return imagePath ? { image: loadImageAsBase64(imagePath) } : undefined;
+            return previousVideo?.uri ? { video: { uri: previousVideo.uri } } : undefined;
+        };
+        const payload = createVeo31Payload(model, prompt, aspectRatio, getSource());
+        GraphAILogger.info(isInitial ? "Generating initial 8s video..." : `Extending video: iteration ${iteration}/${extensionsNeeded} (+${extensionDuration}s)...`);
+        const operation = await ai.models.generateVideos(payload);
+        const response = await pollUntilDone(ai, operation);
+        const video = getVideoFromResponse(response, iteration);
+        const totalDuration = accumulatedDuration + extensionDuration;
+        GraphAILogger.info(`Video ${isInitial ? "generated" : "extended"}: ~${totalDuration}s total`);
+        return { video, duration: totalDuration };
+    };
+    const result = await Array.from({ length: extensionsNeeded + 1 }).reduce(async (prev, _, index) => {
+        const { video, duration } = await prev;
+        return generateIteration(index, duration, video);
+    }, Promise.resolve({ video: undefined, duration: 0 }));
+    if (!result.video) {
+        throw new Error("Failed to generate extended video", {
+            cause: agentInvalidResponseError("movieGenAIAgent", imageAction, movieFileTarget),
+        });
+    }
+    return downloadVideo(ai, result.video, movieFile);
+};
+const generateStandardVideo = async (ai, model, prompt, aspectRatio, imagePath, duration, movieFile) => {
+    const isVeo3 = model === "veo-3.0-generate-001" || model === "veo-3.1-generate-preview";
+    const payload = {
+        model,
+        prompt,
+        config: {
+            durationSeconds: isVeo3 ? undefined : duration,
+            aspectRatio,
+            personGeneration: imagePath ? undefined : PersonGeneration.ALLOW_ALL,
+        },
+        image: imagePath ? loadImageAsBase64(imagePath) : undefined,
+    };
+    const operation = await ai.models.generateVideos(payload);
+    const response = await pollUntilDone(ai, operation);
+    const video = getVideoFromResponse(response);
+    return downloadVideo(ai, video, movieFile);
+};
 export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, imagePath, movieFile } = namedInputs;
     const aspectRatio = getAspectRatio(params.canvasSize);
@@ -33,54 +132,12 @@ export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
             });
         }
         const ai = new GoogleGenAI({ apiKey });
-        const payload = {
-            model,
-            prompt,
-            config: {
-                durationSeconds: duration,
-                aspectRatio,
-                personGeneration: undefined,
-            },
-            image: undefined,
-        };
-        if (model === "veo-3.0-generate-001" || model === "veo-3.1-generate-preview") {
-            payload.config.durationSeconds = undefined;
+        // Veo 3.1: Video extension mode for videos longer than 8s
+        if (model === "veo-3.1-generate-preview" && requestedDuration > 8 && params.canvasSize) {
+            return generateExtendedVideo(ai, model, prompt, aspectRatio, imagePath, requestedDuration, movieFile);
         }
-        if (imagePath) {
-            const buffer = readFileSync(imagePath);
-            const imageBytes = buffer.toString("base64");
-            payload.image = {
-                imageBytes,
-                mimeType: "image/png",
-            };
-        }
-        else {
-            payload.config.personGeneration = PersonGeneration.ALLOW_ALL;
-        }
-        const operation = await ai.models.generateVideos(payload);
-        const response = { operation };
-        // Poll the operation status until the video is ready.
-        while (!response.operation.done) {
-            await sleep(5000);
-            response.operation = await ai.operations.getVideosOperation(response);
-        }
-        if (!response.operation.response?.generatedVideos) {
-            throw new Error(`No video: ${JSON.stringify(response.operation, null, 2)}`, {
-                cause: agentInvalidResponseError("movieGenAIAgent", imageAction, movieFileTarget),
-            });
-        }
-        const video = response.operation.response.generatedVideos[0].video;
-        if (!video) {
-            throw new Error(`No video: ${JSON.stringify(response.operation, null, 2)}`, {
-                cause: agentInvalidResponseError("movieGenAIAgent", imageAction, movieFileTarget),
-            });
-        }
-        await ai.files.download({
-            file: video,
-            downloadPath: movieFile,
-        });
-        await sleep(5000); // HACK: Without this, the file is not ready yet.
-        return { saved: movieFile };
+        // Standard mode
+        return generateStandardVideo(ai, model, prompt, aspectRatio, imagePath, duration, movieFile);
     }
     catch (error) {
         GraphAILogger.info("Failed to generate movie:", error.message);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.0.4",
+  "version": "2.0.6",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",

package/scripts/test/test_genai2.json

@@ -0,0 +1,25 @@
+{
+  "$mulmocast": { "version": "1.1" },
+  "imageParams": {
+    "provider": "google",
+    "style": "<style>Photo realistic.</style>"
+  },
+  "canvasSize": {
+    "width": 720,
+    "height": 1280
+  },
+  "movieParams": {
+    "provider": "google"
+  },
+  "lang": "en",
+  "beats": [
+    {
+      "id": "gemini_3_pro_image_preview",
+      "text": "image generated by gemini-3-pro-image-preview",
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "imageParams": {
+        "model": "gemini-3-pro-image-preview"
+      }
+    }
+  ]
+}

package/scripts/test/test_genai2.json~

@@ -0,0 +1,84 @@
+{
+  "$mulmocast": { "version": "1.1" },
+  "imageParams": {
+    "provider": "google",
+    "style": "<style>Photo realistic.</style>"
+  },
+  "movieParams": {
+    "provider": "google"
+  },
+  "lang": "en",
+  "beats": [
+    {
+      "id": "gemini_3_pro_image_preview",
+      "text": "image generated by gemini-3-pro-image-preview",
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "imageParams": {
+        "model": "gemini-3-pro-image-preview"
+      }
+    },
+    {
+      "id": "gemini_2_5_flash_image",
+      "text": "image generated by gemini-2.5-flash-image",
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "imageParams": {
+        "model": "gemini-2.5-flash-image"
+      }
+    },
+    {
+      "id": "imagen_4",
+      "text": "image generated by imagen-4",
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses"
+    },
+    {
+      "id": "imagen_4_ultra",
+      "text": "image generated by imagen-4",
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "imageParams": {
+        "model": "imagen-4.0-ultra-generate-preview-06-06"
+      }
+    },
+    {
+      "id": "genai_veo2",
+      "text": "movie generated by veo2",
+      "duration": 5,
+      "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses"
+    },
+    {
+      "id": "genai_veo2_image",
+      "text": "movie generated by veo2 with image",
+      "duration": 5,
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "moviePrompt": "a woman takes a selfie with her phone"
+    },
+    {
+      "id": "genai_veo3",
+      "text": "movie generated by veo3",
+      "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "movieParams": {
+        "model": "veo-3.0-generate-001"
+      }
+    },
+    {
+      "id": "genai_veo3_1",
+      "text": "movie generated by veo3_1",
+      "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "movieParams": {
+        "model": "veo-3.1-generate-preview"
+      }
+    },
+    {
+      "id": "genai_veo3_image",
+      "text": "movie generated by veo3",
+      "duration": 5,
+      "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "imageParams": {
+        "model": "gemini-2.5-flash-image"
+      },
+      "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
+      "movieParams": {
+        "model": "veo-3.0-generate-001"
+      }
+    }
+  ]
+}

package/scripts/test/test_genai_movie.json

@@ -0,0 +1,26 @@
+{
+  "$mulmocast": { "version": "1.1" },
+  "imageParams": {
+    "provider": "google",
+    "style": "<style>Photo realistic.</style>"
+  },
+  "canvasSize": {
+    "width": 720,
+    "height": 1280
+  },
+  "lang": "en",
+  "movieParams": {
+    "provider": "google",
+    "model": "veo-3.1-generate-preview"
+  },
+  "beats": [
+    {
+      "moviePrompt": "A butterfly flying in slow motion",
+      "duration": 20
+    },
+    {
+      "moviePrompt": "Running Tiger",
+      "duration": 25
+    }
+  ]
+}

package/scripts/test/test_genai_movie.json~

@@ -0,0 +1,22 @@
+{
+  "$mulmocast": { "version": "1.1" },
+  "imageParams": {
+    "provider": "google",
+    "style": "<style>Photo realistic.</style>"
+  },
+  "canvasSize": {
+    "width": 720,
+    "height": 1280
+  },
+  "lang": "en",
+  "movieParams": {
+    "provider": "google",
+    "model": "veo-3.1-generate-preview"
+  },
+  "beats": [
+    {
+      "moviePrompt": "A butterfly flying in slow motion",
+      "duration": 20
+    }
+  ]
+}