mulmocast 0.0.23 → 0.0.25

package/README.md

@@ -103,6 +103,14 @@ GOOGLE_PROJECT_ID=your_google_project_id
 
 See also [pre-requisites for Google's image generation model](./docs/pre-requisites-google.md)
 
+#### (Optional) For AI providers
+```bash
+# For Anthropic Claude (htmlPrompt feature)
+ANTHROPIC_API_TOKEN=your_anthropic_api_token
+```
+
+For htmlPrompt configuration, see [docs/image.md](./docs/image.md#2-htmlprompt).
+
 #### (Optional) For Movie models
 ```bash
 REPLICATE_API_TOKEN=your_replicate_api_key

package/assets/html/caption.html

@@ -19,6 +19,7 @@
     }
     .caption {
       /* Text positioned at the bottom */
+      width: 80%;
       position: absolute;
       bottom: 0px;
       /* Enable text wrapping */
@@ -34,6 +35,7 @@
       padding-right: 10%;
       padding-top: 4px;
       background: rgba(0, 0, 0, 0.4);
+      ${styles}
     }
   </style>
 </head>

package/lib/actions/audio.d.ts

@@ -3,5 +3,5 @@ import type { CallbackFunction } from "graphai";
 import { MulmoStudioContext, MulmoBeat } from "../types/index.js";
 export declare const getBeatAudioPath: (text: string, context: MulmoStudioContext, beat: MulmoBeat, lang?: string) => string | undefined;
 export declare const audioFilePath: (context: MulmoStudioContext) => string;
-export declare const generateBeatAudio: (index: number, context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;
-export declare const audio: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<MulmoStudioContext>;
+export declare const generateBeatAudio: (index: number, context: MulmoStudioContext, settings?: Record<string, string>, callbacks?: CallbackFunction[]) => Promise<void>;
+export declare const audio: (context: MulmoStudioContext, settings?: Record<string, string>, callbacks?: CallbackFunction[]) => Promise<MulmoStudioContext>;

package/lib/actions/audio.js

@@ -1,6 +1,5 @@
 import "dotenv/config";
-import { GraphAI } from "graphai";
-import { TaskManager } from "graphai/lib/task_manager.js";
+import { GraphAI, TaskManager } from "graphai";
 import * as agents from "@graphai/vanilla";
 import ttsNijivoiceAgent from "../agents/tts_nijivoice_agent.js";
 import addBGMAgent from "../agents/add_bgm_agent.js";
@@ -12,7 +11,7 @@ import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { MulmoPresentationStyleMethods } from "../methods/index.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
 import { getAudioArtifactFilePath, getAudioFilePath, getOutputStudioFilePath, resolveDirPath, defaultBGMPath, mkdir, writingMessage } from "../utils/file.js";
-import { text2hash, localizedText } from "../utils/utils.js";
+import { text2hash, localizedText, settings2GraphAIConfig } from "../utils/utils.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
 import { MulmoMediaSourceMethods } from "../methods/mulmo_media_source.js";
 const vanillaAgents = agents.default ?? agents;
@@ -200,7 +199,7 @@ const audioAgents = {
     addBGMAgent,
     combineAudioFilesAgent,
 };
-export const generateBeatAudio = async (index, context, callbacks) => {
+export const generateBeatAudio = async (index, context, settings, callbacks) => {
     try {
         MulmoStudioContextMethods.setSessionState(context, "audio", true);
         const fileName = MulmoStudioContextMethods.getFileName(context);
@@ -209,8 +208,9 @@ export const generateBeatAudio = async (index, context, callbacks) => {
         const audioSegmentDirPath = resolveDirPath(audioDirPath, fileName);
         mkdir(outDirPath);
         mkdir(audioSegmentDirPath);
+        const config = settings2GraphAIConfig(settings);
         const taskManager = new TaskManager(getConcurrency(context));
-        const graph = new GraphAI(graph_tts, audioAgents, { agentFilters, taskManager });
+        const graph = new GraphAI(graph_tts, audioAgents, { agentFilters, taskManager, config });
         graph.injectValue("__mapIndex", index);
         graph.injectValue("beat", context.studio.script.beats[index]);
         graph.injectValue("studioBeat", context.studio.beats[index]);
@@ -227,7 +227,7 @@ export const generateBeatAudio = async (index, context, callbacks) => {
         MulmoStudioContextMethods.setSessionState(context, "audio", false);
     }
 };
-export const audio = async (context, callbacks) => {
+export const audio = async (context, settings, callbacks) => {
     try {
         MulmoStudioContextMethods.setSessionState(context, "audio", true);
         const fileName = MulmoStudioContextMethods.getFileName(context);
@@ -239,8 +239,9 @@ export const audio = async (context, callbacks) => {
         const outputStudioFilePath = getOutputStudioFilePath(outDirPath, fileName);
         mkdir(outDirPath);
         mkdir(audioSegmentDirPath);
+        const config = settings2GraphAIConfig(settings);
         const taskManager = new TaskManager(getConcurrency(context));
-        const graph = new GraphAI(graph_data, audioAgents, { agentFilters, taskManager });
+        const graph = new GraphAI(graph_data, audioAgents, { agentFilters, taskManager, config });
         graph.injectValue("context", context);
         graph.injectValue("audioArtifactFilePath", audioArtifactFilePath);
         graph.injectValue("audioCombinedFilePath", audioCombinedFilePath);

package/lib/actions/captions.js

@@ -1,3 +1,4 @@
+import { mulmoCaptionParamsSchema } from "../types/index.js";
 import { GraphAI, GraphAILogger } from "graphai";
 import * as agents from "@graphai/vanilla";
 import { getHTMLFile, getCaptionImagePath } from "../utils/file.js";
@@ -23,22 +24,23 @@ const graph_data = {
                             const { beat, context, index } = namedInputs;
                             try {
                                 MulmoStudioContextMethods.setBeatSessionState(context, "caption", index, true);
-                                const caption = MulmoStudioContextMethods.getCaption(context);
+                                const captionParams = mulmoCaptionParamsSchema.parse({ ...context.studio.script.captionParams, ...beat.captionParams });
                                 const canvasSize = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
                                 const imagePath = getCaptionImagePath(context, index);
                                 const template = getHTMLFile("caption");
                                 const text = (() => {
                                     const multiLingual = context.multiLingual;
-                                    if (caption && multiLingual) {
-                                        return multiLingual[index].multiLingualTexts[caption].text;
+                                    if (captionParams.lang && multiLingual) {
+                                        return multiLingual[index].multiLingualTexts[captionParams.lang].text;
                                     }
-                                    GraphAILogger.warn(`No multiLingual caption found for beat ${index}, lang: ${caption}`);
+                                    GraphAILogger.warn(`No multiLingual caption found for beat ${index}, lang: ${captionParams.lang}`);
                                     return beat.text;
                                 })();
                                 const htmlData = interpolate(template, {
                                     caption: text,
                                     width: `${canvasSize.width}`,
                                     height: `${canvasSize.height}`,
+                                    styles: captionParams.styles.join(";\n"),
                                 });
                                 await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, false, true);
                                 context.studio.beats[index].captionFile = imagePath;

package/lib/actions/images.d.ts

@@ -32,11 +32,11 @@ export declare const imagePreprocessAgent: (namedInputs: {
     imagePath: string | undefined;
     referenceImage: string | undefined;
     htmlPrompt?: undefined;
-    htmlSystemPrompt?: undefined;
+    htmlImageSystemPrompt?: undefined;
 } | {
     imagePath: string;
     htmlPrompt: string;
-    htmlSystemPrompt: string[];
+    htmlImageSystemPrompt: string[];
 } | {
     imagePath: string;
     images: string[];
@@ -64,7 +64,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
     };
     movieFile: string | undefined;
     htmlPrompt?: undefined;
-    htmlSystemPrompt?: undefined;
+    htmlImageSystemPrompt?: undefined;
 } | {
     images: string[];
     imageParams: {
@@ -93,7 +93,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
     referenceImage: string;
     prompt: string;
     htmlPrompt?: undefined;
-    htmlSystemPrompt?: undefined;
+    htmlImageSystemPrompt?: undefined;
 }>;
 export declare const imagePluginAgent: (namedInputs: {
     context: MulmoStudioContext;
@@ -101,5 +101,5 @@ export declare const imagePluginAgent: (namedInputs: {
     index: number;
 }) => Promise<void>;
 export declare const getImageRefs: (context: MulmoStudioContext) => Promise<Record<string, string>>;
-export declare const images: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<MulmoStudioContext>;
-export declare const generateBeatImage: (index: number, context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;
+export declare const images: (context: MulmoStudioContext, settings?: Record<string, string>, callbacks?: CallbackFunction[]) => Promise<MulmoStudioContext>;
+export declare const generateBeatImage: (index: number, context: MulmoStudioContext, settings?: Record<string, string>, callbacks?: CallbackFunction[]) => Promise<void>;

package/lib/actions/images.js

@@ -1,7 +1,6 @@
 import dotenv from "dotenv";
 import fs from "fs";
-import { GraphAI, GraphAILogger } from "graphai";
-import { TaskManager } from "graphai/lib/task_manager.js";
+import { GraphAI, GraphAILogger, TaskManager } from "graphai";
 import * as agents from "@graphai/vanilla";
 import { openAIAgent } from "@graphai/openai_agent";
 import { anthropicAgent } from "@graphai/anthropic_agent";
@@ -11,7 +10,8 @@ import { fileCacheAgentFilter } from "../utils/filters.js";
 import { imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent } from "../agents/index.js";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
 import { findImagePlugin } from "../utils/image_plugins/index.js";
-import { imagePrompt } from "../utils/prompt.js";
+import { userAssert, settings2GraphAIConfig } from "../utils/utils.js";
+import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
 import { defaultOpenAIImageModel } from "../utils/const.js";
 import { renderHTMLToImage } from "../utils/markdown.js";
 const vanillaAgents = agents.default ?? agents;
@@ -43,15 +43,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
     }
     if (beat.htmlPrompt) {
         const htmlPrompt = beat.htmlPrompt.prompt + (beat.htmlPrompt.data ? "\n\n data\n" + JSON.stringify(beat.htmlPrompt.data, null, 2) : "");
-        const htmlSystemPrompt = [
-            "Based on the provided information, create a single slide HTML page using Tailwind CSS.",
-            `The view port size is ${context.presentationStyle.canvasSize.width}x${context.presentationStyle.canvasSize.height}. Make sure the HTML fits within the view port.`,
-            "If charts are needed, use Chart.js to present them in a clean and visually appealing way.",
-            "Include a balanced mix of comments, graphs, and illustrations to enhance visual impact.",
-            "Output only the HTML code. Do not include any comments, explanations, or additional information outside the HTML.",
-            "If data is provided, use it effectively to populate the slide.",
-        ];
-        return { imagePath, htmlPrompt, htmlSystemPrompt };
+        return { imagePath, htmlPrompt, htmlImageSystemPrompt: htmlImageSystemPrompt(context.presentationStyle.canvasSize) };
     }
     // images for "edit_image"
     const images = (() => {
@@ -85,6 +77,9 @@ export const imagePluginAgent = async (namedInputs) => {
 };
 const htmlImageGeneratorAgent = async (namedInputs) => {
     const { html, file, canvasSize } = namedInputs;
+    // Save HTML file
+    const htmlFile = file.replace(/\.[^/.]+$/, ".html");
+    await fs.promises.writeFile(htmlFile, html, "utf8");
     await renderHTMLToImage(html, file, canvasSize.width, canvasSize.height);
 };
 const beat_graph_data = {
@@ -123,12 +118,13 @@ const beat_graph_data = {
             if: ":preprocessor.htmlPrompt",
             defaultValue: {},
             agent: ":htmlImageAgentInfo.agent",
-            params: {
-                mode: ":htmlImageAgentInfo.model",
-            },
             inputs: {
                 prompt: ":preprocessor.htmlPrompt",
-                system: ":preprocessor.htmlSystemPrompt",
+                system: ":preprocessor.htmlImageSystemPrompt",
+                params: {
+                    model: ":htmlImageAgentInfo.model",
+                    max_tokens: ":htmlImageAgentInfo.max_tokens",
+                },
             },
         },
         htmlImageGenerator: {
@@ -297,7 +293,7 @@ const googleAuth = async () => {
         throw error;
     }
 };
-const graphOption = async (context) => {
+const graphOption = async (context, settings) => {
     const agentFilters = [
         {
             name: "fileCacheAgentFilter",
@@ -311,21 +307,22 @@ const graphOption = async (context) => {
         taskManager,
     };
     const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle);
+    const config = settings2GraphAIConfig(settings);
     // We need to get google's auth token only if the google is the text2image provider.
     if (imageAgentInfo.provider === "google" || context.presentationStyle.movieParams?.provider === "google") {
+        userAssert(!!process.env.GOOGLE_PROJECT_ID, "GOOGLE_PROJECT_ID is not set");
         GraphAILogger.log("google was specified as text2image engine");
         const token = await googleAuth();
-        options.config = {
-            imageGoogleAgent: {
-                projectId: process.env.GOOGLE_PROJECT_ID,
-                token,
-            },
-            movieGoogleAgent: {
-                projectId: process.env.GOOGLE_PROJECT_ID,
-                token,
-            },
+        config["imageGoogleAgent"] = {
+            projectId: process.env.GOOGLE_PROJECT_ID,
+            token,
+        };
+        config["movieGoogleAgent"] = {
+            projectId: process.env.GOOGLE_PROJECT_ID,
+            token,
         };
     }
+    options.config = config;
     return options;
 };
 // TODO: unit test
@@ -417,8 +414,8 @@ const getConcurrency = (context) => {
     }
     return 4;
 };
-const generateImages = async (context, callbacks) => {
-    const options = await graphOption(context);
+const generateImages = async (context, settings, callbacks) => {
+    const options = await graphOption(context, settings);
     const injections = await prepareGenerateImages(context);
     const graph = new GraphAI(graph_data, {
         ...vanillaAgents,
@@ -442,10 +439,10 @@ const generateImages = async (context, callbacks) => {
     const res = await graph.run();
     return res.mergeResult;
 };
-export const images = async (context, callbacks) => {
+export const images = async (context, settings, callbacks) => {
     try {
         MulmoStudioContextMethods.setSessionState(context, "image", true);
-        const newContext = await generateImages(context, callbacks);
+        const newContext = await generateImages(context, settings, callbacks);
         MulmoStudioContextMethods.setSessionState(context, "image", false);
         return newContext;
     }
@@ -454,8 +451,8 @@ export const images = async (context, callbacks) => {
         throw error;
     }
 };
-export const generateBeatImage = async (index, context, callbacks) => {
-    const options = await graphOption(context);
+export const generateBeatImage = async (index, context, settings, callbacks) => {
+    const options = await graphOption(context, settings);
     const injections = await prepareGenerateImages(context);
     const graph = new GraphAI(beat_graph_data, {
         ...vanillaAgents,

package/lib/actions/movie.js

@@ -69,7 +69,8 @@ const getOutputOption = (audioId, videoId) => {
         "-b:a 128k", // Audio bitrate
     ];
 };
-const createVideo = async (audioArtifactFilePath, outputVideoPath, context, caption) => {
+const createVideo = async (audioArtifactFilePath, outputVideoPath, context) => {
+    const caption = MulmoStudioContextMethods.getCaption(context);
     const start = performance.now();
     const ffmpegContext = FfmpegContextInit();
     const missingIndex = context.studio.beats.findIndex((beat) => !beat.imageFile && !beat.movieFile);
@@ -208,12 +209,11 @@ export const movieFilePath = (context) => {
 export const movie = async (context) => {
     MulmoStudioContextMethods.setSessionState(context, "video", true);
     try {
-        const caption = MulmoStudioContextMethods.getCaption(context);
         const fileName = MulmoStudioContextMethods.getFileName(context);
         const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
         const audioArtifactFilePath = getAudioArtifactFilePath(outDirPath, fileName);
         const outputVideoPath = movieFilePath(context);
-        if (await createVideo(audioArtifactFilePath, outputVideoPath, context, caption)) {
+        if (await createVideo(audioArtifactFilePath, outputVideoPath, context)) {
             writingMessage(outputVideoPath);
         }
     }

package/lib/agents/image_openai_agent.d.ts

@@ -13,6 +13,8 @@ export declare const imageOpenaiAgent: AgentFunction<{
 }, {
     prompt: string;
     images: string[] | null | undefined;
+}, {
+    apiKey?: string;
 }>;
 declare const imageOpenaiAgentInfo: AgentFunctionInfo;
 export default imageOpenaiAgentInfo;

package/lib/agents/image_openai_agent.js

@@ -4,9 +4,10 @@ import { GraphAILogger } from "graphai";
 import OpenAI, { toFile } from "openai";
 import { defaultOpenAIImageModel } from "../utils/const.js";
 // https://platform.openai.com/docs/guides/image-generation
-export const imageOpenaiAgent = async ({ namedInputs, params }) => {
+export const imageOpenaiAgent = async ({ namedInputs, params, config }) => {
     const { prompt, images } = namedInputs;
-    const { apiKey, moderation, canvasSize } = params;
+    const { moderation, canvasSize } = params;
+    const { apiKey } = { ...config };
     const model = params.model ?? defaultOpenAIImageModel;
     const openai = new OpenAI({ apiKey });
     const size = (() => {

package/lib/agents/tts_elevenlabs_agent.js

@@ -1,8 +1,8 @@
 import { GraphAILogger } from "graphai";
-export const ttsElevenlabsAgent = async ({ namedInputs, params }) => {
+export const ttsElevenlabsAgent = async ({ namedInputs, params, config }) => {
     const { text } = namedInputs;
     const { voice, model, stability, similarityBoost, suppressError } = params;
-    const apiKey = process.env.ELEVENLABS_API_KEY;
+    const apiKey = config?.apiKey ?? process.env.ELEVENLABS_API_KEY;
     if (!apiKey) {
         throw new Error("ELEVENLABS_API_KEY environment variable is required");
     }

package/lib/agents/tts_nijivoice_agent.js

@@ -6,8 +6,9 @@ const errorMessage = [
     "1. Obtain an API key from Niji Voice (https://platform.nijivoice.com/) and set it as the NIJIVOICE_API_KEY environment variable.",
     '2. Use OpenAI\'s TTS instead of Niji Voice by changing speechParams.provider from "nijivoice" to "openai".',
 ].join("\n");
-export const ttsNijivoiceAgent = async ({ params, namedInputs }) => {
-    const { apiKey, suppressError, voice, speed, speed_global } = params;
+export const ttsNijivoiceAgent = async ({ params, namedInputs, config }) => {
+    const { suppressError, voice, speed, speed_global } = params;
+    const { apiKey } = config ?? {};
     const { text } = namedInputs;
     assert(apiKey ?? nijovoiceApiKey, errorMessage);
     const url = `https://api.nijivoice.com/api/platform/v1/voice-actors/${voice}/generate-voice`;

package/lib/agents/tts_openai_agent.js

@@ -1,8 +1,9 @@
 import { GraphAILogger } from "graphai";
 import OpenAI from "openai";
-export const ttsOpenaiAgent = async ({ namedInputs, params }) => {
+export const ttsOpenaiAgent = async ({ namedInputs, params, config }) => {
     const { text } = namedInputs;
-    const { apiKey, model, voice, suppressError, instructions } = params;
+    const { model, voice, suppressError, instructions } = params;
+    const { apiKey } = config ?? {};
     const openai = new OpenAI({ apiKey });
     try {
         const tts_options = {

package/lib/cli/commands/tool/scripting/builder.d.ts

@@ -16,7 +16,7 @@ export declare const builder: (yargs: Argv) => Argv<{
 } & {
     s: string;
 } & {
-    llm: "anthropic" | "openAI" | "gemini" | "groq" | undefined;
+    llm: "openai" | "anthropic" | "gemini" | "groq" | undefined;
 } & {
     llm_model: string | undefined;
 }>;

package/lib/cli/commands/tool/story_to_script/builder.d.ts

@@ -10,7 +10,7 @@ export declare const builder: (yargs: Argv) => Argv<{
 } & {
     beats_per_scene: number;
 } & {
-    llm: "anthropic" | "openAI" | "gemini" | "groq" | undefined;
+    llm: "openai" | "anthropic" | "gemini" | "groq" | undefined;
 } & {
     llm_model: string | undefined;
 } & {

package/lib/methods/mulmo_presentation_style.js

@@ -1,5 +1,5 @@
 import "dotenv/config";
-import { userAssert } from "../utils/utils.js";
+import { userAssert, llmConfig } from "../utils/utils.js";
 import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema } from "../types/schema.js";
 import { defaultOpenAIImageModel } from "../utils/const.js";
 const defaultTextSlideStyles = [
@@ -72,16 +72,13 @@ export const MulmoPresentationStyleMethods = {
     },
     getHtmlImageAgentInfo(presentationStyle) {
         const provider = text2HtmlImageProviderSchema.parse(presentationStyle.htmlImageParams?.provider);
-        const agent = provider === "anthropic" ? "anthropicAgent" : "openAIAgent";
-        const model = presentationStyle.htmlImageParams?.model
-            ? presentationStyle.htmlImageParams?.model
-            : provider === "anthropic"
-                ? "claude-3-7-sonnet-20250219"
-                : "gpt-4o-mini";
+        const defaultConfig = llmConfig[provider];
+        const model = presentationStyle.htmlImageParams?.model ? presentationStyle.htmlImageParams?.model : defaultConfig.defaultModel;
         return {
             provider,
-            agent,
+            agent: defaultConfig.agent,
             model,
+            max_tokens: defaultConfig.max_tokens,
         };
     },
     getImageType(_, beat) {