mulmocast 0.0.22 → 0.0.23

package/README.md

@@ -103,6 +103,11 @@ GOOGLE_PROJECT_ID=your_google_project_id
 
 See also [pre-requisites for Google's image generation model](./docs/pre-requisites-google.md)
 
+#### (Optional) For Movie models
+```bash
+REPLICATE_API_TOKEN=your_replicate_api_key
+```
+
 #### (Optional) For TTS models
 ```bash
 # For Nijivoice TTS

package/lib/actions/captions.js

@@ -61,7 +61,7 @@ const graph_data = {
     },
 };
 export const captions = async (context, callbacks) => {
-    if (context.caption) {
+    if (MulmoStudioContextMethods.getCaption(context)) {
         try {
             MulmoStudioContextMethods.setSessionState(context, "caption", true);
             const graph = new GraphAI(graph_data, { ...vanillaAgents });

package/lib/actions/images.d.ts

@@ -8,8 +8,8 @@ export declare const imagePreprocessAgent: (namedInputs: {
     imageRefs: Record<string, string>;
 }) => Promise<{
     imageParams: {
-        model?: string | undefined;
         style?: string | undefined;
+        model?: string | undefined;
         moderation?: string | undefined;
         images?: Record<string, {
             type: "image";
@@ -32,16 +32,18 @@ export declare const imagePreprocessAgent: (namedInputs: {
     imagePath: string | undefined;
     referenceImage: string | undefined;
     htmlPrompt?: undefined;
+    htmlSystemPrompt?: undefined;
 } | {
     imagePath: string;
     htmlPrompt: string;
+    htmlSystemPrompt: string[];
 } | {
     imagePath: string;
     images: string[];
     imageFromMovie: boolean;
     imageParams: {
-        model?: string | undefined;
         style?: string | undefined;
+        model?: string | undefined;
         moderation?: string | undefined;
         images?: Record<string, {
             type: "image";
@@ -62,11 +64,12 @@ export declare const imagePreprocessAgent: (namedInputs: {
     };
     movieFile: string | undefined;
     htmlPrompt?: undefined;
+    htmlSystemPrompt?: undefined;
 } | {
     images: string[];
     imageParams: {
-        model?: string | undefined;
         style?: string | undefined;
+        model?: string | undefined;
         moderation?: string | undefined;
         images?: Record<string, {
             type: "image";
@@ -90,6 +93,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
     referenceImage: string;
     prompt: string;
     htmlPrompt?: undefined;
+    htmlSystemPrompt?: undefined;
 }>;
 export declare const imagePluginAgent: (namedInputs: {
     context: MulmoStudioContext;

package/lib/actions/images.js

@@ -4,10 +4,11 @@ import { GraphAI, GraphAILogger } from "graphai";
 import { TaskManager } from "graphai/lib/task_manager.js";
 import * as agents from "@graphai/vanilla";
 import { openAIAgent } from "@graphai/openai_agent";
+import { anthropicAgent } from "@graphai/anthropic_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { getOutputStudioFilePath, getBeatPngImagePath, getBeatMoviePath, getReferenceImagePath, mkdir } from "../utils/file.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
-import { imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, mediaMockAgent } from "../agents/index.js";
+import { imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent } from "../agents/index.js";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
 import { findImagePlugin } from "../utils/image_plugins/index.js";
 import { imagePrompt } from "../utils/prompt.js";
@@ -15,7 +16,6 @@ import { defaultOpenAIImageModel } from "../utils/const.js";
 import { renderHTMLToImage } from "../utils/markdown.js";
 const vanillaAgents = agents.default ?? agents;
 dotenv.config();
-// const openai = new OpenAI();
 import { GoogleAuth } from "google-auth-library";
 import { extractImageFromMovie } from "../utils/ffmpeg_utils.js";
 const htmlStyle = (context, beat) => {
@@ -43,7 +43,15 @@ export const imagePreprocessAgent = async (namedInputs) => {
     }
     if (beat.htmlPrompt) {
         const htmlPrompt = beat.htmlPrompt.prompt + (beat.htmlPrompt.data ? "\n\n data\n" + JSON.stringify(beat.htmlPrompt.data, null, 2) : "");
-        return { imagePath, htmlPrompt };
+        const htmlSystemPrompt = [
+            "Based on the provided information, create a single slide HTML page using Tailwind CSS.",
+            `The view port size is ${context.presentationStyle.canvasSize.width}x${context.presentationStyle.canvasSize.height}. Make sure the HTML fits within the view port.`,
+            "If charts are needed, use Chart.js to present them in a clean and visually appealing way.",
+            "Include a balanced mix of comments, graphs, and illustrations to enhance visual impact.",
+            "Output only the HTML code. Do not include any comments, explanations, or additional information outside the HTML.",
+            "If data is provided, use it effectively to populate the slide.",
+        ];
+        return { imagePath, htmlPrompt, htmlSystemPrompt };
     }
     // images for "edit_image"
     const images = (() => {
@@ -85,6 +93,7 @@ const beat_graph_data = {
     nodes: {
         context: {},
         imageAgentInfo: {},
+        htmlImageAgentInfo: {},
         movieAgentInfo: {},
         imageRefs: {},
         beat: {},
@@ -113,25 +122,21 @@ const beat_graph_data = {
         htmlImageAgent: {
             if: ":preprocessor.htmlPrompt",
             defaultValue: {},
-            agent: "openAIAgent",
+            agent: ":htmlImageAgentInfo.agent",
+            params: {
+                mode: ":htmlImageAgentInfo.model",
+            },
             inputs: {
                 prompt: ":preprocessor.htmlPrompt",
-                system: [
-                    "Based on the provided information, create a single slide HTML page using Tailwind CSS.",
-                    "If charts are needed, use Chart.js to present them in a clean and visually appealing way.",
-                    "Include a balanced mix of comments, graphs, and illustrations to enhance visual impact.",
-                    "Output only the HTML code. Do not include any comments, explanations, or additional information outside the HTML.",
-                    "If data is provided, use it effectively to populate the slide.",
-                ],
+                system: ":preprocessor.htmlSystemPrompt",
             },
         },
         htmlImageGenerator: {
             if: ":preprocessor.htmlPrompt",
             defaultValue: {},
             agent: htmlImageGeneratorAgent,
-            // console: { before: true, after: true },
             inputs: {
-                html: ":htmlImageAgent.text.codeBlock()",
+                html: ":htmlImageAgent.text.codeBlockOrRaw()",
                 canvasSize: ":context.presentationStyle.canvasSize",
                 file: ":preprocessor.imagePath", // only for fileCacheAgentFilter
                 mulmoContext: ":context", // for fileCacheAgentFilter
@@ -213,6 +218,7 @@ const graph_data = {
     nodes: {
         context: {},
         imageAgentInfo: {},
+        htmlImageAgentInfo: {},
         movieAgentInfo: {},
         outputStudioFilePath: {},
         imageRefs: {},
@@ -222,6 +228,7 @@ const graph_data = {
                 rows: ":context.studio.script.beats",
                 context: ":context",
                 imageAgentInfo: ":imageAgentInfo",
+                htmlImageAgentInfo: ":htmlImageAgentInfo",
                 movieAgentInfo: ":movieAgentInfo",
                 imageRefs: ":imageRefs",
             },
@@ -268,7 +275,6 @@ const graph_data = {
             },
         },
         writeOutput: {
-            // console: { before: true },
             agent: "fileWriteAgent",
             inputs: {
                 file: ":outputStudioFilePath",
@@ -370,13 +376,28 @@ const prepareGenerateImages = async (context) => {
     const outDirPath = MulmoStudioContextMethods.getOutDirPath(context);
     mkdir(imageProjectDirPath);
     const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle, context.dryRun);
+    const htmlImageAgentInfo = MulmoPresentationStyleMethods.getHtmlImageAgentInfo(context.presentationStyle);
     const imageRefs = await getImageRefs(context);
+    // Determine movie agent based on provider
+    const getMovieAgent = () => {
+        if (context.dryRun)
+            return "mediaMockAgent";
+        const provider = context.presentationStyle.movieParams?.provider ?? "google";
+        switch (provider) {
+            case "replicate":
+                return "movieReplicateAgent";
+            case "google":
+            default:
+                return "movieGoogleAgent";
+        }
+    };
     GraphAILogger.info(`text2image: provider=${imageAgentInfo.provider} model=${imageAgentInfo.imageParams.model}`);
     const injections = {
         context,
         imageAgentInfo,
+        htmlImageAgentInfo,
         movieAgentInfo: {
-            agent: context.dryRun ? "mediaMockAgent" : "movieGoogleAgent",
+            agent: getMovieAgent(),
         },
         outputStudioFilePath: getOutputStudioFilePath(outDirPath, fileName),
         imageRefs,
@@ -384,6 +405,9 @@ const prepareGenerateImages = async (context) => {
     return injections;
 };
 const getConcurrency = (context) => {
+    if (context.presentationStyle.movieParams?.provider === "replicate") {
+        return 4;
+    }
     const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle);
     if (imageAgentInfo.provider === "openai") {
         // NOTE: Here are the rate limits of OpenAI's text2image API (1token = 32x32 patch).
@@ -396,7 +420,17 @@ const getConcurrency = (context) => {
 const generateImages = async (context, callbacks) => {
     const options = await graphOption(context);
     const injections = await prepareGenerateImages(context);
-    const graph = new GraphAI(graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent, openAIAgent }, options);
+    const graph = new GraphAI(graph_data, {
+        ...vanillaAgents,
+        imageGoogleAgent,
+        movieGoogleAgent,
+        movieReplicateAgent,
+        imageOpenaiAgent,
+        mediaMockAgent,
+        fileWriteAgent,
+        openAIAgent,
+        anthropicAgent,
+    }, options);
     Object.keys(injections).forEach((key) => {
         graph.injectValue(key, injections[key]);
     });
@@ -423,7 +457,17 @@ export const images = async (context, callbacks) => {
 export const generateBeatImage = async (index, context, callbacks) => {
     const options = await graphOption(context);
     const injections = await prepareGenerateImages(context);
-    const graph = new GraphAI(beat_graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent, openAIAgent }, options);
+    const graph = new GraphAI(beat_graph_data, {
+        ...vanillaAgents,
+        imageGoogleAgent,
+        movieGoogleAgent,
+        movieReplicateAgent,
+        imageOpenaiAgent,
+        mediaMockAgent,
+        fileWriteAgent,
+        openAIAgent,
+        anthropicAgent,
+    }, options);
     Object.keys(injections).forEach((key) => {
         if ("outputStudioFilePath" !== key) {
             graph.injectValue(key, injections[key]);

package/lib/actions/movie.d.ts

@@ -1,5 +1,5 @@
-import { MulmoStudioContext, MulmoCanvasDimension, BeatMediaType } from "../types/index.js";
-export declare const getVideoPart: (inputIndex: number, mediaType: BeatMediaType, duration: number, canvasInfo: MulmoCanvasDimension) => {
+import { MulmoStudioContext, MulmoCanvasDimension, BeatMediaType, MulmoFillOption } from "../types/index.js";
+export declare const getVideoPart: (inputIndex: number, mediaType: BeatMediaType, duration: number, canvasInfo: MulmoCanvasDimension, fillOption: MulmoFillOption) => {
     videoId: string;
     videoPart: string;
 };

package/lib/actions/movie.js

@@ -1,12 +1,12 @@
 import { GraphAILogger, assert } from "graphai";
-import { mulmoTransitionSchema } from "../types/index.js";
+import { mulmoTransitionSchema, mulmoFillOptionSchema } from "../types/index.js";
 import { MulmoPresentationStyleMethods } from "../methods/index.js";
 import { getAudioArtifactFilePath, getOutputVideoFilePath, writingMessage } from "../utils/file.js";
 import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextPushFormattedAudio, FfmpegContextGenerateOutput } from "../utils/ffmpeg_utils.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
 // const isMac = process.platform === "darwin";
 const videoCodec = "libx264"; // "h264_videotoolbox" (macOS only) is too noisy
-export const getVideoPart = (inputIndex, mediaType, duration, canvasInfo) => {
+export const getVideoPart = (inputIndex, mediaType, duration, canvasInfo, fillOption) => {
     const videoId = `v${inputIndex}`;
     const videoFilters = [];
     // Handle different media types
@@ -19,9 +19,19 @@ export const getVideoPart = (inputIndex, mediaType, duration, canvasInfo) => {
         videoFilters.push(`tpad=stop_mode=clone:stop_duration=${duration * 2}`); // Use 2x duration to ensure coverage
     }
     // Common filters for all media types
-    videoFilters.push(`trim=duration=${duration}`, "fps=30", "setpts=PTS-STARTPTS", `scale=w=${canvasInfo.width}:h=${canvasInfo.height}:force_original_aspect_ratio=decrease`, 
-    // In case of the aspect ratio mismatch, we fill the extra space with black color.
-    `pad=${canvasInfo.width}:${canvasInfo.height}:(ow-iw)/2:(oh-ih)/2:color=black`, "setsar=1", "format=yuv420p");
+    videoFilters.push(`trim=duration=${duration}`, "fps=30", "setpts=PTS-STARTPTS");
+    // Apply scaling based on fill option
+    if (fillOption.style === "aspectFill") {
+        // For aspect fill: scale to fill the canvas completely, cropping if necessary
+        videoFilters.push(`scale=w=${canvasInfo.width}:h=${canvasInfo.height}:force_original_aspect_ratio=increase`, `crop=${canvasInfo.width}:${canvasInfo.height}`);
+    }
+    else {
+        // For aspect fit: scale to fit within canvas, padding if necessary
+        videoFilters.push(`scale=w=${canvasInfo.width}:h=${canvasInfo.height}:force_original_aspect_ratio=decrease`, 
+        // In case of the aspect ratio mismatch, we fill the extra space with black color.
+        `pad=${canvasInfo.width}:${canvasInfo.height}:(ow-iw)/2:(oh-ih)/2:color=black`);
+    }
+    videoFilters.push("setsar=1", "format=yuv420p");
     return {
         videoId,
         videoPart: `[${inputIndex}:v]` + videoFilters.filter((a) => a).join(",") + `[${videoId}]`,
@@ -95,7 +105,12 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, context, capt
             return 0;
         })();
         const duration = studioBeat.duration + extraPadding;
-        const { videoId, videoPart } = getVideoPart(inputIndex, mediaType, duration, canvasInfo);
+        // Get fillOption from merged imageParams (global + beat-specific)
+        const globalFillOption = context.presentationStyle.movieParams?.fillOption;
+        const beatFillOption = beat.movieParams?.fillOption;
+        const defaultFillOption = mulmoFillOptionSchema.parse({}); // let the schema infer the default value
+        const fillOption = { ...defaultFillOption, ...globalFillOption, ...beatFillOption };
+        const { videoId, videoPart } = getVideoPart(inputIndex, mediaType, duration, canvasInfo, fillOption);
         ffmpegContext.filterComplex.push(videoPart);
         if (caption && studioBeat.captionFile) {
             const captionInputIndex = FfmpegContextAddInput(ffmpegContext, studioBeat.captionFile);

package/lib/agents/combine_audio_files_agent.js

@@ -1,4 +1,4 @@
-import { assert } from "graphai";
+import { assert, GraphAILogger } from "graphai";
 import { silent60secPath } from "../utils/file.js";
 import { FfmpegContextInit, FfmpegContextGenerateOutput, FfmpegContextInputFormattedAudio, ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
 const getMovieDulation = async (beat) => {
@@ -77,7 +77,8 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
                 const groupBeatsDurations = getGroupBeatDurations(context, group, audioDuration);
                 // Yes, the current beat has spilled over audio.
                 const beatsTotalDuration = groupBeatsDurations.reduce((a, b) => a + b, 0);
-                if (beatsTotalDuration > audioDuration) {
+                if (beatsTotalDuration > audioDuration + 0.01) {
+                    // 0.01 is a tolerance to avoid floating point precision issues
                     group.reduce((remaining, idx, iGroup) => {
                         if (remaining >= groupBeatsDurations[iGroup]) {
                             return remaining - groupBeatsDurations[iGroup];
@@ -88,7 +89,9 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
                 }
                 else {
                     // Last beat gets the rest of the audio.
-                    groupBeatsDurations[groupBeatsDurations.length - 1] += audioDuration - beatsTotalDuration;
+                    if (audioDuration > beatsTotalDuration) {
+                        groupBeatsDurations[groupBeatsDurations.length - 1] += audioDuration - beatsTotalDuration;
+                    }
                 }
                 beatDurations.push(...groupBeatsDurations);
             }
@@ -98,7 +101,7 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
                 // padding is the amount of audio padding specified in the script.
                 const padding = getPadding(context, beat, index);
                 // totalPadding is the amount of audio padding to be added to the audio file.
-                const totalPadding = getTotalPadding(padding, movieDuration, audioDuration, beat.duration);
+                const totalPadding = Math.round(getTotalPadding(padding, movieDuration, audioDuration, beat.duration) * 100) / 100;
                 const beatDuration = audioDuration + totalPadding;
                 beatDurations.push(beatDuration);
                 if (totalPadding > 0) {
@@ -124,7 +127,7 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
     // We cannot reuse longSilentId. We need to explicitly split it for each beat.
     const silentIds = mediaDurations.filter((md) => md.silenceDuration > 0).map((_, index) => `[ls_${index}]`);
     if (silentIds.length > 0) {
-        const longSilentId = FfmpegContextInputFormattedAudio(ffmpegContext, silent60secPath());
+        const longSilentId = FfmpegContextInputFormattedAudio(ffmpegContext, silent60secPath(), undefined, ["-stream_loop", "-1"]);
         ffmpegContext.filterComplex.push(`${longSilentId}asplit=${silentIds.length}${silentIds.join("")}`);
     }
     const inputIds = [];
@@ -142,6 +145,7 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
         }
     });
     assert(silentIds.length === 0, "silentIds.length !== 0");
+    GraphAILogger.log("filterComplex:", ffmpegContext.filterComplex.join("\n"));
     // Finally, combine all audio files.
     ffmpegContext.filterComplex.push(`${inputIds.join("")}concat=n=${inputIds.length}:v=0:a=1[aout]`);
     await FfmpegContextGenerateOutput(ffmpegContext, combinedFileName, ["-map", "[aout]"]);

package/lib/agents/index.d.ts

@@ -4,6 +4,7 @@ import imageGoogleAgent from "./image_google_agent.js";
 import imageOpenaiAgent from "./image_openai_agent.js";
 import tavilySearchAgent from "./tavily_agent.js";
 import movieGoogleAgent from "./movie_google_agent.js";
+import movieReplicateAgent from "./movie_replicate_agent.js";
 import mediaMockAgent from "./media_mock_agent.js";
 import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
 import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
@@ -13,4 +14,4 @@ import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };

package/lib/agents/index.js

@@ -4,6 +4,7 @@ import imageGoogleAgent from "./image_google_agent.js";
 import imageOpenaiAgent from "./image_openai_agent.js";
 import tavilySearchAgent from "./tavily_agent.js";
 import movieGoogleAgent from "./movie_google_agent.js";
+import movieReplicateAgent from "./movie_replicate_agent.js";
 import mediaMockAgent from "./media_mock_agent.js";
 import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
 import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
@@ -14,4 +15,4 @@ import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 // import * as vanilla from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };

package/lib/agents/movie_replicate_agent.d.ts

@@ -0,0 +1,23 @@
+import type { AgentFunction, AgentFunctionInfo } from "graphai";
+export declare const getAspectRatio: (canvasSize: {
+    width: number;
+    height: number;
+}) => string;
+export type MovieReplicateConfig = {
+    apiKey?: string;
+};
+export declare const movieReplicateAgent: AgentFunction<{
+    model: `${string}/${string}` | undefined;
+    canvasSize: {
+        width: number;
+        height: number;
+    };
+    duration?: number;
+}, {
+    buffer: Buffer;
+}, {
+    prompt: string;
+    imagePath?: string;
+}, MovieReplicateConfig>;
+declare const movieReplicateAgentInfo: AgentFunctionInfo;
+export default movieReplicateAgentInfo;

package/lib/agents/movie_replicate_agent.js

@@ -0,0 +1,93 @@
+import { readFileSync } from "fs";
+import { GraphAILogger } from "graphai";
+import Replicate from "replicate";
+async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, duration) {
+    const replicate = new Replicate({
+        auth: apiKey,
+    });
+    const input = {
+        prompt: prompt,
+        duration: duration,
+        image: undefined,
+        start_image: undefined,
+        aspect_ratio: aspectRatio, // only for bytedance/seedance-1-lite
+        // resolution: "720p", // only for bytedance/seedance-1-lite
+        // fps: 24, // only for bytedance/seedance-1-lite
+        // camera_fixed: false, // only for bytedance/seedance-1-lite
+        // mode: "standard" // only for kwaivgi/kling-v2.1
+        // negative_prompt: "" // only for kwaivgi/kling-v2.1
+    };
+    // Add image if provided (for image-to-video generation)
+    if (imagePath) {
+        const buffer = readFileSync(imagePath);
+        const base64Image = `data:image/png;base64,${buffer.toString("base64")}`;
+        if (model === "kwaivgi/kling-v2.1") {
+            input.start_image = base64Image;
+        }
+        else {
+            input.image = base64Image;
+        }
+    }
+    try {
+        const output = await replicate.run(model ?? "bytedance/seedance-1-lite", { input });
+        // Download the generated video
+        if (output && typeof output === "object" && "url" in output) {
+            const videoUrl = output.url();
+            const videoResponse = await fetch(videoUrl);
+            if (!videoResponse.ok) {
+                throw new Error(`Error downloading video: ${videoResponse.status} - ${videoResponse.statusText}`);
+            }
+            const arrayBuffer = await videoResponse.arrayBuffer();
+            return Buffer.from(arrayBuffer);
+        }
+        return undefined;
+    }
+    catch (error) {
+        GraphAILogger.info("Replicate generation error:", error);
+        throw error;
+    }
+}
+export const getAspectRatio = (canvasSize) => {
+    if (canvasSize.width > canvasSize.height) {
+        return "16:9";
+    }
+    else if (canvasSize.width < canvasSize.height) {
+        return "9:16";
+    }
+    else {
+        return "1:1";
+    }
+};
+export const movieReplicateAgent = async ({ namedInputs, params, config }) => {
+    const { prompt, imagePath } = namedInputs;
+    const aspectRatio = getAspectRatio(params.canvasSize);
+    const duration = params.duration ?? 5;
+    const apiKey = config?.apiKey ?? process.env.REPLICATE_API_TOKEN;
+    if (!apiKey) {
+        throw new Error("REPLICATE_API_TOKEN environment variable is required");
+    }
+    try {
+        const buffer = await generateMovie(params.model, apiKey, prompt, imagePath, aspectRatio, duration);
+        if (buffer) {
+            return { buffer };
+        }
+        throw new Error("ERROR: generateMovie returned undefined");
+    }
+    catch (error) {
+        GraphAILogger.info("Failed to generate movie:", error.message);
+        throw error;
+    }
+};
+const movieReplicateAgentInfo = {
+    name: "movieReplicateAgent",
+    agent: movieReplicateAgent,
+    mock: movieReplicateAgent,
+    samples: [],
+    description: "Replicate Movie agent using seedance-1-lite",
+    category: ["movie"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    license: "MIT",
+    environmentVariables: ["REPLICATE_API_TOKEN"],
+};
+export default movieReplicateAgentInfo;

package/lib/cli/commands/tool/scripting/builder.d.ts

@@ -5,6 +5,8 @@ export declare const builder: (yargs: Argv) => Argv<{
     b: string | undefined;
 } & {
     u: string[] | never[];
+} & {
+    "input-file": string | undefined;
 } & {
     i: boolean | undefined;
 } & {
@@ -14,7 +16,7 @@ export declare const builder: (yargs: Argv) => Argv<{
 } & {
     s: string;
 } & {
-    llm: "openAI" | "anthropic" | "gemini" | "groq" | undefined;
+    llm: "anthropic" | "openAI" | "gemini" | "groq" | undefined;
 } & {
     llm_model: string | undefined;
 }>;

package/lib/cli/commands/tool/scripting/builder.js

@@ -22,6 +22,11 @@ export const builder = (yargs) => {
         default: [],
         type: "array",
         string: true,
+    })
+        .option("input-file", {
+        description: "input file name",
+        demandOption: false,
+        type: "string",
     })
         .option("i", {
         alias: "interactive",

package/lib/cli/commands/tool/scripting/handler.d.ts

@@ -6,6 +6,7 @@ export declare const handler: (argv: ToolCliArgs<{
     u?: string[];
     i?: boolean;
     t?: string;
+    "input-file"?: string;
     c?: string;
     s?: string;
     llm?: LLM;

package/lib/cli/commands/tool/scripting/handler.js

@@ -1,18 +1,23 @@
 import { getBaseDirPath, getFullPath } from "../../../../utils/file.js";
 import { outDirName, cacheDirName } from "../../../../utils/const.js";
 import { getUrlsIfNeeded, selectTemplate } from "../../../../utils/inquirer.js";
-import { createMulmoScriptFromUrl } from "../../../../tools/create_mulmo_script_from_url.js";
+import { createMulmoScriptFromUrl, createMulmoScriptFromFile } from "../../../../tools/create_mulmo_script_from_url.js";
 import { createMulmoScriptInteractively } from "../../../../tools/create_mulmo_script_interactively.js";
 import { setGraphAILogger } from "../../../../cli/helpers.js";
 export const handler = async (argv) => {
-    const { o: outdir, b: basedir, v: verbose, i: interactive, c: cache, s: filename, llm, llm_model } = argv;
+    const { o: outdir, b: basedir, "input-file": inputFile, v: verbose, i: interactive, c: cache, s: filename, llm, llm_model } = argv;
     let { t: template } = argv;
     const urls = argv.u || [];
     const baseDirPath = getBaseDirPath(basedir);
     const outDirPath = getFullPath(baseDirPath, outdir ?? outDirName);
     const cacheDirPath = getFullPath(outDirPath, cache ?? cacheDirName);
     if (!template) {
-        template = await selectTemplate();
+        if (interactive) {
+            template = await selectTemplate();
+        }
+        else {
+            template = "business";
+        }
     }
     setGraphAILogger(verbose, {
         baseDirPath,
@@ -22,13 +27,17 @@ export const handler = async (argv) => {
         urls,
         interactive,
         filename,
+        inputFile,
         llm,
         llm_model,
     });
-    const context = { outDirPath, templateName: template, urls, filename: filename, cacheDirPath, llm_model, llm };
+    const context = { outDirPath, templateName: template, urls, filename: filename, cacheDirPath, llm_model, llm, verbose };
     if (interactive) {
         await createMulmoScriptInteractively(context);
     }
+    if (inputFile) {
+        await createMulmoScriptFromFile(inputFile, context);
+    }
     else {
         context.urls = await getUrlsIfNeeded(urls);
         await createMulmoScriptFromUrl(context);

package/lib/cli/commands/tool/story_to_script/builder.d.ts

@@ -10,7 +10,7 @@ export declare const builder: (yargs: Argv) => Argv<{
 } & {
     beats_per_scene: number;
 } & {
-    llm: "openAI" | "anthropic" | "gemini" | "groq" | undefined;
+    llm: "anthropic" | "openAI" | "gemini" | "groq" | undefined;
 } & {
     llm_model: string | undefined;
 } & {

package/lib/cli/helpers.js

@@ -7,7 +7,7 @@ import { isHttp } from "../utils/utils.js";
 import { createOrUpdateStudioData } from "../utils/preprocess.js";
 import { outDirName, imageDirName, audioDirName } from "../utils/const.js";
 import { translate } from "../actions/translate.js";
-import { mulmoPresentationStyleSchema, mulmoStudioMultiLingualSchema } from "../types/schema.js";
+import { mulmoCaptionParamsSchema, mulmoPresentationStyleSchema, mulmoStudioMultiLingualSchema } from "../types/schema.js";
 export const setGraphAILogger = (verbose, logValues) => {
     if (verbose) {
         if (logValues) {
@@ -126,13 +126,18 @@ export const initializeContext = async (argv) => {
         // validate mulmoStudioSchema. skip if __test_invalid__ is true
         const studio = createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName);
         const multiLingual = getMultiLingual(outputMultilingualFilePath, studio.beats.length);
+        if (argv.c) {
+            studio.script.captionParams = mulmoCaptionParamsSchema.parse({
+                ...(studio.script.captionParams ?? {}),
+                lang: argv.c,
+            });
+        }
         return {
             studio,
             fileDirs: files,
             force: Boolean(argv.f),
             dryRun: Boolean(argv.dryRun),
             lang: argv.l,
-            caption: argv.c,
             sessionState: {
                 inSession: {
                     audio: false,
@@ -160,7 +165,7 @@ export const initializeContext = async (argv) => {
     }
 };
 export const runTranslateIfNeeded = async (context, argv) => {
-    if (argv.l || argv.c) {
+    if (argv.l || context.studio.script.captionParams?.lang) {
         GraphAILogger.log("run translate");
         await translate(context);
     }