mulmocast 2.1.24 → 2.1.25

package/README.md

@@ -128,9 +128,6 @@ REPLICATE_API_TOKEN=your_replicate_api_key
 
 #### (Optional) For TTS models
 ```bash
-# For Nijivoice TTS
-NIJIVOICE_API_KEY=your_nijivoice_api_key
-
 # For ElevenLabs TTS
 ELEVENLABS_API_KEY=your_elevenlabs_api_key
 ```
@@ -140,6 +137,76 @@ ELEVENLABS_API_KEY=your_elevenlabs_api_key
 BROWSERLESS_API_TOKEN=your_browserless_api_token # to access web in mulmo tool
 ```
 
+### Google Vertex AI
+
+For enterprise/production environments or to access models like Imagen 4, use Vertex AI with Application Default Credentials (ADC):
+
+```bash
+# Install gcloud CLI and authenticate
+gcloud auth application-default login
+```
+
+Configure in MulmoScript:
+```json
+{
+  "imageParams": {
+    "provider": "google",
+    "model": "imagen-4.0-generate-001",
+    "vertexai_project": "your-project-id",
+    "vertexai_location": "us-central1"
+  }
+}
+```
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `vertexai_project` | Google Cloud Project ID | None (enables Vertex AI mode when set) |
+| `vertexai_location` | Region | `us-central1` |
+
+For detailed setup instructions, see [Vertex AI Setup Guide](./docs/vertexai_en.md).
+
+### Azure OpenAI
+
+To use Azure OpenAI instead of OpenAI API:
+
+```bash
+# For image generation
+IMAGE_OPENAI_API_KEY=<your-azure-openai-api-key>
+IMAGE_OPENAI_BASE_URL=https://<resource-name>.openai.azure.com/
+
+# For TTS (Text-to-Speech)
+TTS_OPENAI_API_KEY=<your-azure-openai-api-key>
+TTS_OPENAI_BASE_URL=https://<resource-name>.openai.azure.com/
+
+# For LLM (translate, scripting)
+LLM_OPENAI_API_KEY=<your-azure-openai-api-key>
+LLM_OPENAI_BASE_URL=https://<resource-name>.openai.azure.com/
+LLM_OPENAI_API_VERSION=2025-04-01-preview  # optional
+```
+
+MulmoScript configuration (same as OpenAI):
+```json
+{
+  "imageParams": {
+    "provider": "openai",
+    "model": "gpt-image-1.5"
+  },
+  "speechParams": {
+    "speakers": {
+      "Presenter": {
+        "provider": "openai",
+        "voiceId": "alloy",
+        "model": "tts"
+      }
+    }
+  }
+}
+```
+
+**Important**: Azure deployment names must match model names exactly (e.g., deployment name `gpt-image-1.5` for model `gpt-image-1.5`).
+
+For detailed setup and region availability, see [Azure OpenAI Usage Guide](./docs/azure_openai_usage.md).
+
 ## Workflow
 
 1. Create a MulmoScript JSON file with `mulmo tool scripting`
@@ -217,8 +284,7 @@ mulmo tool scripting --input-file story.txt
 mulmo tool scripting -i
 ```
 
-Note: 
-- When using the `⁠sensei_and_taro` template, a Nijivoice API key is required
+Note:
 - When -i is specified, --input-file value will be ignored
 - When --input-file is specified, -u value will be ignored
 

package/lib/actions/captions.js

@@ -58,21 +58,27 @@ const calculateCumulativeRatios = (ratios) => {
 };
 // Generate caption files for a single beat
 const generateBeatCaptions = async (beat, context, index) => {
-    const captionParams = mulmoCaptionParamsSchema.parse({ ...context.studio.script.captionParams, ...beat.captionParams });
+    const globalCaptionParamsRaw = context.studio.script.captionParams ?? {};
+    const beatCaptionParamsRaw = beat.captionParams ?? {};
+    const mergedCaptionParams = mulmoCaptionParamsSchema.parse({
+        ...globalCaptionParamsRaw,
+        ...beatCaptionParamsRaw,
+        styles: Object.hasOwn(beatCaptionParamsRaw, "styles") ? beatCaptionParamsRaw.styles : globalCaptionParamsRaw.styles,
+    });
     const canvasSize = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
     const template = getHTMLFile("caption");
-    if (captionParams.lang && !context.multiLingual?.[index]?.multiLingualTexts?.[captionParams.lang]) {
-        GraphAILogger.warn(`No multiLingual caption found for beat ${index}, lang: ${captionParams.lang}`);
+    if (mergedCaptionParams.lang && !context.multiLingual?.[index]?.multiLingualTexts?.[mergedCaptionParams.lang]) {
+        GraphAILogger.warn(`No multiLingual caption found for beat ${index}, lang: ${mergedCaptionParams.lang}`);
     }
-    const text = localizedText(beat, context.multiLingual?.[index], captionParams.lang, context.studio.script.lang);
+    const text = localizedText(beat, context.multiLingual?.[index], mergedCaptionParams.lang, context.studio.script.lang);
     // Get beat timing info
     const studioBeat = context.studio.beats[index];
     const beatStartAt = studioBeat.startAt ?? 0;
     const beatDuration = studioBeat.duration ?? 0;
     const introPadding = MulmoStudioContextMethods.getIntroPadding(context);
     // Determine split texts based on captionSplit setting
-    const captionSplit = captionParams.captionSplit ?? "none";
-    const splitTexts = captionSplit === "estimate" ? getSplitTexts(text, beat.texts, captionParams.textSplit) : [text];
+    const captionSplit = mergedCaptionParams.captionSplit ?? "none";
+    const splitTexts = captionSplit === "estimate" ? getSplitTexts(text, beat.texts, mergedCaptionParams.textSplit) : [text];
     // Calculate timing
     const cumulativeRatios = calculateCumulativeRatios(calculateTimingRatios(splitTexts));
     // Generate caption images with absolute timing
@@ -82,7 +88,7 @@ const generateBeatCaptions = async (beat, context, index) => {
             caption: processLineBreaks(segmentText),
             width: `${canvasSize.width}`,
             height: `${canvasSize.height}`,
-            styles: captionParams.styles.join(";\n"),
+            styles: (mergedCaptionParams.styles ?? []).join(";\n"),
         });
         await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, false, true);
         return {

package/lib/cli/commands/tool/complete/builder.d.ts

@@ -0,0 +1,8 @@
+import { Argv } from "yargs";
+export declare const builder: (yargs: Argv) => Argv<{
+    o: string | undefined;
+} & {
+    t: string | undefined;
+} & {
+    file: string;
+}>;

package/lib/cli/commands/tool/complete/builder.js

@@ -0,0 +1,23 @@
+import { getAvailablePromptTemplates } from "../../../../utils/file.js";
+const availableTemplateNames = getAvailablePromptTemplates().map((template) => template.filename);
+export const builder = (yargs) => {
+    return yargs
+        .option("o", {
+        alias: "output",
+        description: "Output file path (default: <file>_completed.json)",
+        demandOption: false,
+        type: "string",
+    })
+        .option("t", {
+        alias: "template",
+        description: "Template/style name to apply",
+        demandOption: false,
+        choices: availableTemplateNames,
+        type: "string",
+    })
+        .positional("file", {
+        description: "Input beats file path (JSON)",
+        type: "string",
+        demandOption: true,
+    });
+};

package/lib/cli/commands/tool/complete/handler.d.ts

@@ -0,0 +1,8 @@
+type CompleteHandlerArgs = {
+    file: string;
+    o?: string;
+    t?: string;
+    v?: boolean;
+};
+export declare const handler: (argv: CompleteHandlerArgs) => Promise<void>;
+export {};

package/lib/cli/commands/tool/complete/handler.js

@@ -0,0 +1,40 @@
+import { readFileSync, writeFileSync } from "fs";
+import path from "path";
+import { GraphAILogger } from "graphai";
+import { completeScript, templateExists } from "../../../../tools/complete_script.js";
+export const handler = async (argv) => {
+    const { file, o: outputPath, t: templateName, v: verbose } = argv;
+    if (!file) {
+        GraphAILogger.error("Error: Input file is required");
+        process.exit(1);
+    }
+    const inputPath = path.resolve(file);
+    const inputData = (() => {
+        try {
+            const content = readFileSync(inputPath, "utf-8");
+            return JSON.parse(content);
+        }
+        catch (error) {
+            GraphAILogger.error(`Error reading file: ${inputPath}`);
+            GraphAILogger.error(error.message);
+            process.exit(1);
+        }
+    })();
+    if (templateName && !templateExists(templateName)) {
+        GraphAILogger.warn(`Warning: Template '${templateName}' not found`);
+    }
+    const result = completeScript(inputData, templateName);
+    if (!result.success) {
+        GraphAILogger.error("Validation errors:");
+        result.error.issues.forEach((issue) => {
+            GraphAILogger.error(`  - ${issue.path.join(".")}: ${issue.message}`);
+        });
+        process.exit(1);
+    }
+    if (verbose && templateName) {
+        GraphAILogger.info(`Applied template: ${templateName}`);
+    }
+    const outputFilePath = outputPath ? path.resolve(outputPath) : inputPath.replace(/\.json$/, "_completed.json");
+    writeFileSync(outputFilePath, JSON.stringify(result.data, null, 2));
+    GraphAILogger.info(`Completed script written to: ${outputFilePath}`);
+};

package/lib/cli/commands/tool/complete/index.d.ts

@@ -0,0 +1,4 @@
+export declare const command = "complete <file>";
+export declare const desc = "Complete MulmoScript with schema defaults and optional style";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/commands/tool/complete/index.js

@@ -0,0 +1,4 @@
+export const command = "complete <file>";
+export const desc = "Complete MulmoScript with schema defaults and optional style";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/commands/tool/index.js

@@ -3,7 +3,8 @@ import * as promptCmd from "./prompt/index.js";
 import * as schemaCmd from "./schema/index.js";
 import * as storyToScriptCmd from "./story_to_script/index.js";
 import * as whisperCmd from "./whisper/index.js";
+import * as completeCmd from "./complete/index.js";
 export const command = "tool <command>";
 export const desc = "Generate Mulmo script and other tools";
-export const builder = (y) => y.command(scriptingCmd).command(promptCmd).command(schemaCmd).command(storyToScriptCmd).command(whisperCmd).demandCommand().strict();
+export const builder = (y) => y.command(scriptingCmd).command(promptCmd).command(schemaCmd).command(storyToScriptCmd).command(whisperCmd).command(completeCmd).demandCommand().strict();
 export const handler = (__argv) => { };

package/lib/data/styles.d.ts

@@ -0,0 +1,255 @@
+export declare const styles: ({
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            images: {
+                girl: {
+                    source: {
+                        kind: string;
+                        url: string;
+                    };
+                    type: string;
+                };
+                ani?: undefined;
+                presenter?: undefined;
+                optimus?: undefined;
+            };
+            style: string;
+            model?: undefined;
+            provider?: undefined;
+        };
+        audioParams?: undefined;
+        movieParams?: undefined;
+        speechParams?: undefined;
+    };
+} | {
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        audioParams: {
+            bgm: {
+                kind: string;
+                url: string;
+            };
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            images: {
+                ani: {
+                    source: {
+                        kind: string;
+                        url: string;
+                    };
+                    type: string;
+                };
+                girl?: undefined;
+                presenter?: undefined;
+                optimus?: undefined;
+            };
+            model: string;
+            provider: string;
+            style: string;
+        };
+        movieParams: {
+            model: string;
+            provider: string;
+        };
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    provider: string;
+                    speechOptions: {
+                        instruction: string;
+                    };
+                    voiceId: string;
+                };
+            };
+            provider?: undefined;
+        };
+    };
+} | {
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            style: string;
+            images?: undefined;
+            model?: undefined;
+            provider?: undefined;
+        };
+        audioParams?: undefined;
+        movieParams?: undefined;
+        speechParams?: undefined;
+    };
+} | {
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            images: {
+                presenter: {
+                    source: {
+                        kind: string;
+                        url: string;
+                    };
+                    type: string;
+                };
+                girl?: undefined;
+                ani?: undefined;
+                optimus?: undefined;
+            };
+            style: string;
+            model?: undefined;
+            provider?: undefined;
+        };
+        audioParams?: undefined;
+        movieParams?: undefined;
+        speechParams?: undefined;
+    };
+} | {
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            images: {
+                presenter: {
+                    source: {
+                        kind: string;
+                        url: string;
+                    };
+                    type: string;
+                };
+                girl?: undefined;
+                ani?: undefined;
+                optimus?: undefined;
+            };
+            style: string;
+            model?: undefined;
+            provider?: undefined;
+        };
+        speechParams: {
+            provider: string;
+            speakers: {
+                Presenter: {
+                    speechOptions: {
+                        instruction: string;
+                    };
+                    voiceId: string;
+                    provider?: undefined;
+                };
+            };
+        };
+        audioParams?: undefined;
+        movieParams?: undefined;
+    };
+} | {
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            images: {
+                optimus: {
+                    source: {
+                        kind: string;
+                        url: string;
+                    };
+                    type: string;
+                };
+                presenter: {
+                    source: {
+                        kind: string;
+                        url: string;
+                    };
+                    type: string;
+                };
+                girl?: undefined;
+                ani?: undefined;
+            };
+            style: string;
+            model?: undefined;
+            provider?: undefined;
+        };
+        audioParams?: undefined;
+        movieParams?: undefined;
+        speechParams?: undefined;
+    };
+} | {
+    filename: string;
+    presentationStyle: {
+        $mulmocast: {
+            credit: string;
+            version: string;
+        };
+        audioParams: {
+            bgm: {
+                kind: string;
+                url: string;
+            };
+        };
+        canvasSize: {
+            height: number;
+            width: number;
+        };
+        imageParams: {
+            model: string;
+            provider: string;
+            images?: undefined;
+            style?: undefined;
+        };
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    provider: string;
+                    speechOptions: {
+                        instruction: string;
+                    };
+                    voiceId: string;
+                };
+            };
+            provider?: undefined;
+        };
+        movieParams?: undefined;
+    };
+})[];