mulmocast 2.1.23 → 2.1.25

package/README.md

@@ -128,9 +128,6 @@ REPLICATE_API_TOKEN=your_replicate_api_key
 
 #### (Optional) For TTS models
 ```bash
-# For Nijivoice TTS
-NIJIVOICE_API_KEY=your_nijivoice_api_key
-
 # For ElevenLabs TTS
 ELEVENLABS_API_KEY=your_elevenlabs_api_key
 ```
@@ -140,6 +137,76 @@ ELEVENLABS_API_KEY=your_elevenlabs_api_key
 BROWSERLESS_API_TOKEN=your_browserless_api_token # to access web in mulmo tool
 ```
 
+### Google Vertex AI
+
+For enterprise/production environments or to access models like Imagen 4, use Vertex AI with Application Default Credentials (ADC):
+
+```bash
+# Install gcloud CLI and authenticate
+gcloud auth application-default login
+```
+
+Configure in MulmoScript:
+```json
+{
+  "imageParams": {
+    "provider": "google",
+    "model": "imagen-4.0-generate-001",
+    "vertexai_project": "your-project-id",
+    "vertexai_location": "us-central1"
+  }
+}
+```
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `vertexai_project` | Google Cloud Project ID | None (enables Vertex AI mode when set) |
+| `vertexai_location` | Region | `us-central1` |
+
+For detailed setup instructions, see [Vertex AI Setup Guide](./docs/vertexai_en.md).
+
+### Azure OpenAI
+
+To use Azure OpenAI instead of OpenAI API:
+
+```bash
+# For image generation
+IMAGE_OPENAI_API_KEY=<your-azure-openai-api-key>
+IMAGE_OPENAI_BASE_URL=https://<resource-name>.openai.azure.com/
+
+# For TTS (Text-to-Speech)
+TTS_OPENAI_API_KEY=<your-azure-openai-api-key>
+TTS_OPENAI_BASE_URL=https://<resource-name>.openai.azure.com/
+
+# For LLM (translate, scripting)
+LLM_OPENAI_API_KEY=<your-azure-openai-api-key>
+LLM_OPENAI_BASE_URL=https://<resource-name>.openai.azure.com/
+LLM_OPENAI_API_VERSION=2025-04-01-preview  # optional
+```
+
+MulmoScript configuration (same as OpenAI):
+```json
+{
+  "imageParams": {
+    "provider": "openai",
+    "model": "gpt-image-1.5"
+  },
+  "speechParams": {
+    "speakers": {
+      "Presenter": {
+        "provider": "openai",
+        "voiceId": "alloy",
+        "model": "tts"
+      }
+    }
+  }
+}
+```
+
+**Important**: Azure deployment names must match model names exactly (e.g., deployment name `gpt-image-1.5` for model `gpt-image-1.5`).
+
+For detailed setup and region availability, see [Azure OpenAI Usage Guide](./docs/azure_openai_usage.md).
+
 ## Workflow
 
 1. Create a MulmoScript JSON file with `mulmo tool scripting`
@@ -217,8 +284,7 @@ mulmo tool scripting --input-file story.txt
 mulmo tool scripting -i
 ```
 
-Note: 
-- When using the `⁠sensei_and_taro` template, a Nijivoice API key is required
+Note:
 - When -i is specified, --input-file value will be ignored
 - When --input-file is specified, -u value will be ignored
 

package/lib/actions/captions.js

@@ -58,21 +58,27 @@ const calculateCumulativeRatios = (ratios) => {
 };
 // Generate caption files for a single beat
 const generateBeatCaptions = async (beat, context, index) => {
-    const captionParams = mulmoCaptionParamsSchema.parse({ ...context.studio.script.captionParams, ...beat.captionParams });
+    const globalCaptionParamsRaw = context.studio.script.captionParams ?? {};
+    const beatCaptionParamsRaw = beat.captionParams ?? {};
+    const mergedCaptionParams = mulmoCaptionParamsSchema.parse({
+        ...globalCaptionParamsRaw,
+        ...beatCaptionParamsRaw,
+        styles: Object.hasOwn(beatCaptionParamsRaw, "styles") ? beatCaptionParamsRaw.styles : globalCaptionParamsRaw.styles,
+    });
     const canvasSize = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
     const template = getHTMLFile("caption");
-    if (captionParams.lang && !context.multiLingual?.[index]?.multiLingualTexts?.[captionParams.lang]) {
-        GraphAILogger.warn(`No multiLingual caption found for beat ${index}, lang: ${captionParams.lang}`);
+    if (mergedCaptionParams.lang && !context.multiLingual?.[index]?.multiLingualTexts?.[mergedCaptionParams.lang]) {
+        GraphAILogger.warn(`No multiLingual caption found for beat ${index}, lang: ${mergedCaptionParams.lang}`);
     }
-    const text = localizedText(beat, context.multiLingual?.[index], captionParams.lang, context.studio.script.lang);
+    const text = localizedText(beat, context.multiLingual?.[index], mergedCaptionParams.lang, context.studio.script.lang);
     // Get beat timing info
     const studioBeat = context.studio.beats[index];
     const beatStartAt = studioBeat.startAt ?? 0;
     const beatDuration = studioBeat.duration ?? 0;
     const introPadding = MulmoStudioContextMethods.getIntroPadding(context);
     // Determine split texts based on captionSplit setting
-    const captionSplit = captionParams.captionSplit ?? "none";
-    const splitTexts = captionSplit === "estimate" ? getSplitTexts(text, beat.texts, captionParams.textSplit) : [text];
+    const captionSplit = mergedCaptionParams.captionSplit ?? "none";
+    const splitTexts = captionSplit === "estimate" ? getSplitTexts(text, beat.texts, mergedCaptionParams.textSplit) : [text];
     // Calculate timing
     const cumulativeRatios = calculateCumulativeRatios(calculateTimingRatios(splitTexts));
     // Generate caption images with absolute timing
@@ -82,7 +88,7 @@ const generateBeatCaptions = async (beat, context, index) => {
             caption: processLineBreaks(segmentText),
             width: `${canvasSize.width}`,
             height: `${canvasSize.height}`,
-            styles: captionParams.styles.join(";\n"),
+            styles: (mergedCaptionParams.styles ?? []).join(";\n"),
         });
         await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, false, true);
         return {

package/lib/agents/image_genai_agent.js

@@ -59,12 +59,20 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, referenceImages } = namedInputs;
     const model = params.model ?? provider2ImageAgent["google"].defaultModel;
     const apiKey = config?.apiKey;
-    if (!apiKey) {
-        throw new Error("Google GenAI API key is required (GEMINI_API_KEY)", {
-            cause: apiKeyMissingError("imageGenAIAgent", imageAction, "GEMINI_API_KEY"),
-        });
-    }
-    const ai = new GoogleGenAI({ apiKey });
+    const ai = params.vertexai_project
+        ? new GoogleGenAI({
+            vertexai: true,
+            project: params.vertexai_project,
+            location: params.vertexai_location ?? "us-central1",
+        })
+        : (() => {
+            if (!apiKey) {
+                throw new Error("Google GenAI API key is required (GEMINI_API_KEY)", {
+                    cause: apiKeyMissingError("imageGenAIAgent", imageAction, "GEMINI_API_KEY"),
+                });
+            }
+            return new GoogleGenAI({ apiKey });
+        })();
     if (model === "gemini-2.5-flash-image" || model === "gemini-3-pro-image-preview") {
         const contentParams = (() => {
             const contents = getGeminiContents(prompt, referenceImages);

package/lib/agents/movie_genai_agent.js

@@ -1,4 +1,4 @@
-import { readFileSync } from "fs";
+import { readFileSync, writeFileSync } from "fs";
 import { GraphAILogger, sleep } from "graphai";
 import { GoogleGenAI, PersonGeneration } from "@google/genai";
 import { apiKeyMissingError, agentGenerationError, agentInvalidResponseError, imageAction, movieFileTarget, videoDurationTarget, hasCause, } from "../utils/error_cause.js";
@@ -35,12 +35,19 @@ const loadImageAsBase64 = (imagePath) => {
         mimeType: "image/png",
     };
 };
-const downloadVideo = async (ai, video, movieFile) => {
-    await ai.files.download({
-        file: video,
-        downloadPath: movieFile,
-    });
-    await sleep(5000); // HACK: Without this, the file is not ready yet.
+const downloadVideo = async (ai, video, movieFile, isVertexAI) => {
+    if (isVertexAI) {
+        // Vertex AI returns videoBytes directly
+        writeFileSync(movieFile, Buffer.from(video.videoBytes, "base64"));
+    }
+    else {
+        // Gemini API requires download via uri
+        await ai.files.download({
+            file: video,
+            downloadPath: movieFile,
+        });
+        await sleep(5000); // HACK: Without this, the file is not ready yet.
+    }
     return { saved: movieFile };
 };
 const createVeo31Payload = (model, prompt, aspectRatio, source) => ({
@@ -53,7 +60,7 @@ const createVeo31Payload = (model, prompt, aspectRatio, source) => ({
     },
     ...source,
 });
-const generateExtendedVideo = async (ai, model, prompt, aspectRatio, imagePath, requestedDuration, movieFile) => {
+const generateExtendedVideo = async (ai, model, prompt, aspectRatio, imagePath, requestedDuration, movieFile, isVertexAI) => {
     const initialDuration = 8;
     const maxExtensionDuration = 8;
     const extensionsNeeded = Math.ceil((requestedDuration - initialDuration) / maxExtensionDuration);
@@ -85,9 +92,9 @@ const generateExtendedVideo = async (ai, model, prompt, aspectRatio, imagePath,
             cause: agentInvalidResponseError("movieGenAIAgent", imageAction, movieFileTarget),
         });
     }
-    return downloadVideo(ai, result.video, movieFile);
+    return downloadVideo(ai, result.video, movieFile, isVertexAI);
 };
-const generateStandardVideo = async (ai, model, prompt, aspectRatio, imagePath, duration, movieFile) => {
+const generateStandardVideo = async (ai, model, prompt, aspectRatio, imagePath, duration, movieFile, isVertexAI) => {
     const isVeo3 = model === "veo-3.0-generate-001" || model === "veo-3.1-generate-preview";
     const payload = {
         model,
@@ -102,18 +109,13 @@ const generateStandardVideo = async (ai, model, prompt, aspectRatio, imagePath,
     const operation = await ai.models.generateVideos(payload);
     const response = await pollUntilDone(ai, operation);
     const video = getVideoFromResponse(response);
-    return downloadVideo(ai, video, movieFile);
+    return downloadVideo(ai, video, movieFile, isVertexAI);
 };
 export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, imagePath, movieFile } = namedInputs;
     const aspectRatio = getAspectRatio(params.canvasSize, ASPECT_RATIOS);
     const model = params.model ?? provider2MovieAgent.google.defaultModel;
     const apiKey = config?.apiKey;
-    if (!apiKey) {
-        throw new Error("Google GenAI API key is required (GEMINI_API_KEY)", {
-            cause: apiKeyMissingError("movieGenAIAgent", imageAction, "GEMINI_API_KEY"),
-        });
-    }
     try {
         const requestedDuration = params.duration ?? 8;
         const duration = getModelDuration("google", model, requestedDuration);
@@ -122,13 +124,27 @@ export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
                 cause: agentGenerationError("movieGenAIAgent", imageAction, videoDurationTarget),
             });
         }
-        const ai = new GoogleGenAI({ apiKey });
+        const isVertexAI = !!params.vertexai_project;
+        const ai = isVertexAI
+            ? new GoogleGenAI({
+                vertexai: true,
+                project: params.vertexai_project,
+                location: params.vertexai_location ?? "us-central1",
+            })
+            : (() => {
+                if (!apiKey) {
+                    throw new Error("Google GenAI API key is required (GEMINI_API_KEY)", {
+                        cause: apiKeyMissingError("movieGenAIAgent", imageAction, "GEMINI_API_KEY"),
+                    });
+                }
+                return new GoogleGenAI({ apiKey });
+            })();
         // Veo 3.1: Video extension mode for videos longer than 8s
         if (model === "veo-3.1-generate-preview" && requestedDuration > 8 && params.canvasSize) {
-            return generateExtendedVideo(ai, model, prompt, aspectRatio, imagePath, requestedDuration, movieFile);
+            return generateExtendedVideo(ai, model, prompt, aspectRatio, imagePath, requestedDuration, movieFile, isVertexAI);
         }
         // Standard mode
-        return generateStandardVideo(ai, model, prompt, aspectRatio, imagePath, duration, movieFile);
+        return generateStandardVideo(ai, model, prompt, aspectRatio, imagePath, duration, movieFile, isVertexAI);
     }
     catch (error) {
         GraphAILogger.info("Failed to generate movie:", error.message);

package/lib/cli/commands/tool/complete/builder.d.ts

@@ -0,0 +1,8 @@
+import { Argv } from "yargs";
+export declare const builder: (yargs: Argv) => Argv<{
+    o: string | undefined;
+} & {
+    t: string | undefined;
+} & {
+    file: string;
+}>;

package/lib/cli/commands/tool/complete/builder.js

@@ -0,0 +1,23 @@
+import { getAvailablePromptTemplates } from "../../../../utils/file.js";
+const availableTemplateNames = getAvailablePromptTemplates().map((template) => template.filename);
+export const builder = (yargs) => {
+    return yargs
+        .option("o", {
+        alias: "output",
+        description: "Output file path (default: <file>_completed.json)",
+        demandOption: false,
+        type: "string",
+    })
+        .option("t", {
+        alias: "template",
+        description: "Template/style name to apply",
+        demandOption: false,
+        choices: availableTemplateNames,
+        type: "string",
+    })
+        .positional("file", {
+        description: "Input beats file path (JSON)",
+        type: "string",
+        demandOption: true,
+    });
+};

package/lib/cli/commands/tool/complete/handler.d.ts

@@ -0,0 +1,8 @@
+type CompleteHandlerArgs = {
+    file: string;
+    o?: string;
+    t?: string;
+    v?: boolean;
+};
+export declare const handler: (argv: CompleteHandlerArgs) => Promise<void>;
+export {};

package/lib/cli/commands/tool/complete/handler.js

@@ -0,0 +1,40 @@
+import { readFileSync, writeFileSync } from "fs";
+import path from "path";
+import { GraphAILogger } from "graphai";
+import { completeScript, templateExists } from "../../../../tools/complete_script.js";
+export const handler = async (argv) => {
+    const { file, o: outputPath, t: templateName, v: verbose } = argv;
+    if (!file) {
+        GraphAILogger.error("Error: Input file is required");
+        process.exit(1);
+    }
+    const inputPath = path.resolve(file);
+    const inputData = (() => {
+        try {
+            const content = readFileSync(inputPath, "utf-8");
+            return JSON.parse(content);
+        }
+        catch (error) {
+            GraphAILogger.error(`Error reading file: ${inputPath}`);
+            GraphAILogger.error(error.message);
+            process.exit(1);
+        }
+    })();
+    if (templateName && !templateExists(templateName)) {
+        GraphAILogger.warn(`Warning: Template '${templateName}' not found`);
+    }
+    const result = completeScript(inputData, templateName);
+    if (!result.success) {
+        GraphAILogger.error("Validation errors:");
+        result.error.issues.forEach((issue) => {
+            GraphAILogger.error(`  - ${issue.path.join(".")}: ${issue.message}`);
+        });
+        process.exit(1);
+    }
+    if (verbose && templateName) {
+        GraphAILogger.info(`Applied template: ${templateName}`);
+    }
+    const outputFilePath = outputPath ? path.resolve(outputPath) : inputPath.replace(/\.json$/, "_completed.json");
+    writeFileSync(outputFilePath, JSON.stringify(result.data, null, 2));
+    GraphAILogger.info(`Completed script written to: ${outputFilePath}`);
+};

package/lib/cli/commands/tool/complete/index.d.ts

@@ -0,0 +1,4 @@
+export declare const command = "complete <file>";
+export declare const desc = "Complete MulmoScript with schema defaults and optional style";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/commands/tool/complete/index.js

@@ -0,0 +1,4 @@
+export const command = "complete <file>";
+export const desc = "Complete MulmoScript with schema defaults and optional style";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/commands/tool/index.js

@@ -3,7 +3,8 @@ import * as promptCmd from "./prompt/index.js";
 import * as schemaCmd from "./schema/index.js";
 import * as storyToScriptCmd from "./story_to_script/index.js";
 import * as whisperCmd from "./whisper/index.js";
+import * as completeCmd from "./complete/index.js";
 export const command = "tool <command>";
 export const desc = "Generate Mulmo script and other tools";
-export const builder = (y) => y.command(scriptingCmd).command(promptCmd).command(schemaCmd).command(storyToScriptCmd).command(whisperCmd).demandCommand().strict();
+export const builder = (y) => y.command(scriptingCmd).command(promptCmd).command(schemaCmd).command(storyToScriptCmd).command(whisperCmd).command(completeCmd).demandCommand().strict();
 export const handler = (__argv) => { };

package/lib/data/styles.d.ts

@@ -0,0 +1,255 @@
+export declare const styles: ({
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            images: {
+                girl: {
+                    source: {
+                        kind: string;
+                        url: string;
+                    };
+                    type: string;
+                };
+                ani?: undefined;
+                presenter?: undefined;
+                optimus?: undefined;
+            };
+            style: string;
+            model?: undefined;
+            provider?: undefined;
+        };
+        audioParams?: undefined;
+        movieParams?: undefined;
+        speechParams?: undefined;
+    };
+} | {
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        audioParams: {
+            bgm: {
+                kind: string;
+                url: string;
+            };
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            images: {
+                ani: {
+                    source: {
+                        kind: string;
+                        url: string;
+                    };
+                    type: string;
+                };
+                girl?: undefined;
+                presenter?: undefined;
+                optimus?: undefined;
+            };
+            model: string;
+            provider: string;
+            style: string;
+        };
+        movieParams: {
+            model: string;
+            provider: string;
+        };
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    provider: string;
+                    speechOptions: {
+                        instruction: string;
+                    };
+                    voiceId: string;
+                };
+            };
+            provider?: undefined;
+        };
+    };
+} | {
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            style: string;
+            images?: undefined;
+            model?: undefined;
+            provider?: undefined;
+        };
+        audioParams?: undefined;
+        movieParams?: undefined;
+        speechParams?: undefined;
+    };
+} | {
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            images: {
+                presenter: {
+                    source: {
+                        kind: string;
+                        url: string;
+                    };
+                    type: string;
+                };
+                girl?: undefined;
+                ani?: undefined;
+                optimus?: undefined;
+            };
+            style: string;
+            model?: undefined;
+            provider?: undefined;
+        };
+        audioParams?: undefined;
+        movieParams?: undefined;
+        speechParams?: undefined;
+    };
+} | {
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            images: {
+                presenter: {
+                    source: {
+                        kind: string;
+                        url: string;
+                    };
+                    type: string;
+                };
+                girl?: undefined;
+                ani?: undefined;
+                optimus?: undefined;
+            };
+            style: string;
+            model?: undefined;
+            provider?: undefined;
+        };
+        speechParams: {
+            provider: string;
+            speakers: {
+                Presenter: {
+                    speechOptions: {
+                        instruction: string;
+                    };
+                    voiceId: string;
+                    provider?: undefined;
+                };
+            };
+        };
+        audioParams?: undefined;
+        movieParams?: undefined;
+    };
+} | {
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            images: {
+                optimus: {
+                    source: {
+                        kind: string;
+                        url: string;
+                    };
+                    type: string;
+                };
+                presenter: {
+                    source: {
+                        kind: string;
+                        url: string;
+                    };
+                    type: string;
+                };
+                girl?: undefined;
+                ani?: undefined;
+            };
+            style: string;
+            model?: undefined;
+            provider?: undefined;
+        };
+        audioParams?: undefined;
+        movieParams?: undefined;
+        speechParams?: undefined;
+    };
+} | {
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        audioParams: {
+            bgm: {
+                kind: string;
+                url: string;
+            };
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            model: string;
+            provider: string;
+            images?: undefined;
+            style?: undefined;
+        };
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    provider: string;
+                    speechOptions: {
+                        instruction: string;
+                    };
+                    voiceId: string;
+                };
+            };
+            provider?: undefined;
+        };
+        movieParams?: undefined;
+    };
+})[];