mulmocast 1.1.0 → 1.1.2

package/assets/templates/akira_comic.json

@@ -24,5 +24,5 @@
       }
     }
   },
-  "scriptName": "image_prompts_template.json"
+  "scriptName": "image_prompts_template"
 }

package/assets/templates/ani.json

@@ -1,6 +1,6 @@
 {
-  "title": "Presentation with Ani in Japanese",
-  "description": "Template for presentation with Ani in Japanese.",
+  "title": "Presentation with Ani",
+  "description": "Template for presentation with Ani.",
   "systemPrompt": "Generate a script for a presentation of the given topic. 言葉づかいは少しツンデレにして。Another AI will generate comic for each beat based on the image prompt of that beat. You don't need to specify the style of the image, just describe the scene. Mention the reference in one of beats, if it exists. Use the JSON below as a template. Create appropriate amount of beats, and make sure the beats are coherent and flow well.",
   "presentationStyle": {
     "$mulmocast": {
@@ -44,5 +44,5 @@
       }
     }
   },
-  "scriptName": "image_prompts_template.json"
+  "scriptName": "image_prompts_template"
 }

package/assets/templates/ani_ja.json

@@ -1,6 +1,6 @@
 {
-  "title": "Presentation with Ani",
-  "description": "Template for presentation with Ani.",
+  "title": "Presentation with Ani in Japanese",
+  "description": "Template for presentation with Ani in Japanese.",
   "systemPrompt": "Generate a Japanese script for a presentation of the given topic. 言葉づかいは少しツンデレにして。Another AI will generate comic for each beat based on the image prompt of that beat. You don't need to specify the style of the image, just describe the scene. Mention the reference in one of beats, if it exists. Use the JSON below as a template. Create appropriate amount of beats, and make sure the beats are coherent and flow well.",
   "presentationStyle": {
     "$mulmocast": {
@@ -40,5 +40,5 @@
       }
     }
   },
-  "scriptName": "image_prompts_template.json"
+  "scriptName": "image_prompts_template"
 }

package/assets/templates/business.json

@@ -2,5 +2,5 @@
   "title": "Business presentation",
   "description": "Template for business presentation.",
   "systemPrompt": "Generate a script for a business presentation of the given topic. Use textSlides, markdown, mermaid, or chart to show slides. Extract image links in the article (from <img> tag) to reuse them in the presentation. Mention the reference in one of beats, if it exists. Use the JSON below as a template. chartData is the data for Chart.js",
-  "scriptName": "business.json"
+  "scriptName": "business"
 }

package/assets/templates/characters.json

@@ -12,5 +12,5 @@
     }
   },
   "systemPrompt": "Generate a script for a the given story with multiple characters. Generate image prompts for each character, and make references to them in the beats. Use the JSON below as a template.",
-  "scriptName": "image_refs.json"
+  "scriptName": "image_refs"
 }

package/assets/templates/children_book.json

@@ -15,5 +15,5 @@
       "style": "A hand-drawn style illustration with a warm, nostalgic atmosphere. The background is rich with natural scenery—lush forests, cloudy skies, and traditional Japanese architecture. Characters have expressive eyes, soft facial features, and are portrayed with gentle lighting and subtle shading. The color palette is muted yet vivid, using earthy tones and watercolor-like textures. The overall scene feels magical and peaceful, with a sense of quiet wonder and emotional depth, reminiscent of classic 1980s and 1990s Japanese animation."
     }
   },
-  "scriptName": "children_book.json"
+  "scriptName": "children_book"
 }

package/assets/templates/coding.json

@@ -2,5 +2,5 @@
   "title": "Coding presentation",
   "description": "Template for software and coding presentation.",
   "systemPrompt": "Generate a script for a technical presentation of the given topic. Use markdown with a code block to show some code on a slide. Avoid long coding examples, which may not fit in a single slide. Mention the reference in one of beats, if it exists. Use the JSON below as a template.",
-  "scriptName": "coding.json"
+  "scriptName": "coding"
 }

package/assets/templates/comic_strips.json

@@ -15,5 +15,5 @@
       "style": "<style>A multi panel comic strips. 1990s American workplace humor. Clean, minimalist line art with muted colors. One character is a nerdy office worker with glasses</style>"
     }
   },
-  "scriptName": "text_only_template.json"
+  "scriptName": "text_only_template"
 }

package/assets/templates/drslump_comic.json

@@ -24,5 +24,5 @@
       }
     }
   },
-  "scriptName": "image_prompts_template.json"
+  "scriptName": "image_prompts_template"
 }

package/assets/templates/ghibli_comic.json

@@ -24,5 +24,5 @@
       }
     }
   },
-  "scriptName": "image_prompts_template.json"
+  "scriptName": "image_prompts_template"
 }

package/assets/templates/ghibli_image_only.json

@@ -24,5 +24,5 @@
       }
     }
   },
-  "scriptName": "image_prompt_only_template.json"
+  "scriptName": "image_prompt_only_template"
 }

package/assets/templates/ghibli_shorts.json

@@ -29,5 +29,5 @@
       }
     }
   },
-  "scriptName": "image_prompts_template.json"
+  "scriptName": "image_prompts_template"
 }

package/assets/templates/ghost_comic.json

@@ -31,5 +31,5 @@
       }
     }
   },
-  "scriptName": "image_prompts_template.json"
+  "scriptName": "image_prompts_template"
 }

package/assets/templates/html.json

@@ -2,5 +2,5 @@
   "title": "Business presentation in HTML",
   "description": "Template for business presentation in HTML.",
   "systemPrompt": "Generate a script for a business presentation of the given topic. Another LLM will generate actual slides from the prompt and data for each beat. Adding optional data would help it to generate more compelling slide. Mention the reference in one of beats, if it exists. The valid type of reference is 'article', 'paper', 'image', 'video', 'audio'. Use the JSON below as a template.",
-  "scriptName": "html.json"
+  "scriptName": "html"
 }

package/assets/templates/onepiece_comic.json

@@ -24,5 +24,5 @@
       }
     }
   },
-  "scriptName": "image_prompts_template.json"
+  "scriptName": "image_prompts_template"
 }

package/assets/templates/portrait_movie.json

@@ -24,5 +24,5 @@
       }
     }
   },
-  "scriptName": "movie_prompts_template.json"
+  "scriptName": "movie_prompts_template"
 }

package/assets/templates/realistic_movie.json

@@ -24,5 +24,5 @@
       }
     }
   },
-  "scriptName": "movie_prompts_template.json"
+  "scriptName": "movie_prompts_template"
 }

package/assets/templates/sensei_and_taro.json

@@ -22,5 +22,5 @@
       }
     }
   },
-  "scriptName": "sensei_and_taro.json"
+  "scriptName": "sensei_and_taro"
 }

package/assets/templates/shorts.json

@@ -14,5 +14,5 @@
       "style": "<style>Photo realistic, cinematic.</style>"
     }
   },
-  "scriptName": "movie_prompts_template.json"
+  "scriptName": "movie_prompts_template"
 }

package/assets/templates/text_and_image.json

@@ -2,5 +2,5 @@
   "title": "Text and Image",
   "description": "Template for Text and Image Script.",
   "systemPrompt": "Generate a script for a presentation of the given topic. Another AI will generate comic strips for each beat based on the imagePrompt of that beat. Mention the reference in one of beats, if it exists. Use the JSON below as a template.",
-  "scriptName": "image_prompts_template.json"
+  "scriptName": "image_prompts_template"
 }

package/assets/templates/text_only.json

@@ -2,5 +2,5 @@
   "title": "Text Only",
   "description": "Template for Text Only Script.",
   "systemPrompt": "Generate a script for a presentation of the given topic. Another AI will generate comic strips for each beat based on the text description of that beat. Mention the reference in one of beats, if it exists. Use the JSON below as a template.",
-  "scriptName": "text_only_template.json"
+  "scriptName": "text_only_template"
 }

package/assets/templates/trailer.json

@@ -21,5 +21,5 @@
       "bgm": { "kind": "url", "url": "https://raw.githubusercontent.com/receptron/mulmocast-media/refs/heads/main/bgms/trailer_dramatic.mp3" }
     }
   },
-  "scriptName": "movie_prompts_no_text_template.json"
+  "scriptName": "movie_prompts_no_text_template"
 }

package/lib/actions/image_agents.d.ts

@@ -21,6 +21,14 @@ export declare const imagePreprocessAgent: (namedInputs: {
         agentName: string;
         defaultModel: string;
     };
+    lipSyncFile?: string;
+    lipSyncModel?: string;
+    lipSyncAgentInfo?: {
+        agentName: string;
+        defaultModel: string;
+    };
+    audioFile?: string;
+    beatDuration?: number;
     htmlPrompt?: undefined;
     htmlPath?: undefined;
     htmlImageSystemPrompt?: undefined;
@@ -51,6 +59,14 @@ export declare const imagePreprocessAgent: (namedInputs: {
         agentName: string;
         defaultModel: string;
     };
+    lipSyncFile?: string;
+    lipSyncModel?: string;
+    lipSyncAgentInfo?: {
+        agentName: string;
+        defaultModel: string;
+    };
+    audioFile?: string;
+    beatDuration?: number;
     htmlPrompt?: undefined;
     htmlPath?: undefined;
     htmlImageSystemPrompt?: undefined;
@@ -84,6 +100,14 @@ export declare const imagePreprocessAgent: (namedInputs: {
         agentName: string;
         defaultModel: string;
     };
+    lipSyncFile?: string;
+    lipSyncModel?: string;
+    lipSyncAgentInfo?: {
+        agentName: string;
+        defaultModel: string;
+    };
+    audioFile?: string;
+    beatDuration?: number;
     htmlPrompt?: undefined;
     htmlPath?: undefined;
     htmlImageSystemPrompt?: undefined;

package/lib/actions/image_agents.js

@@ -11,6 +11,7 @@ const htmlStyle = (context, beat) => {
 };
 export const imagePreprocessAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs } = namedInputs;
+    const studioBeat = context.studio.beats[index];
     const imagePath = getBeatPngImagePath(context, index);
     if (beat.htmlPrompt) {
         const htmlPrompt = MulmoBeatMethods.getHtmlPrompt(beat);
@@ -22,6 +23,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
     const returnValue = {
         imageParams: imageAgentInfo.imageParams,
         movieFile: beat.moviePrompt ? moviePaths.movieFile : undefined,
+        beatDuration: beat.duration ?? studioBeat?.duration,
     };
     if (beat.soundEffectPrompt) {
         returnValue.soundEffectAgentInfo = MulmoPresentationStyleMethods.getSoundEffectAgentInfo(context.presentationStyle, beat);
@@ -30,6 +32,13 @@ export const imagePreprocessAgent = async (namedInputs) => {
         returnValue.soundEffectFile = moviePaths.soundEffectFile;
         returnValue.soundEffectPrompt = beat.soundEffectPrompt;
     }
+    if (beat.enableLipSync) {
+        returnValue.lipSyncAgentInfo = MulmoPresentationStyleMethods.getLipSyncAgentInfo(context.presentationStyle, beat);
+        returnValue.lipSyncModel = beat.lipSyncParams?.model ?? context.presentationStyle.lipSyncParams?.model ?? returnValue.lipSyncAgentInfo.defaultModel;
+        returnValue.lipSyncFile = moviePaths.lipSyncFile;
+        // Audio file will be set from the beat's audio file when available
+        returnValue.audioFile = studioBeat?.audioFile;
+    }
     if (beat.image) {
         const plugin = MulmoBeatMethods.getPlugin(beat);
         const pluginPath = plugin.path({ beat, context, imagePath, ...htmlStyle(context, beat) });

package/lib/actions/images.js

@@ -6,7 +6,7 @@ import * as vanilla from "@graphai/vanilla";
 import { openAIAgent } from "@graphai/openai_agent";
 import { anthropicAgent } from "@graphai/anthropic_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-import { imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, soundEffectReplicateAgent } from "../agents/index.js";
+import { imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, } from "../agents/index.js";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
 import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
@@ -26,11 +26,15 @@ const movieAgents = {
 const soundEffectAgents = {
     soundEffectReplicateAgent,
 };
+const lipSyncAgents = {
+    lipSyncReplicateAgent,
+};
 const defaultAgents = {
     ...vanillaAgents,
     ...imageAgents,
     ...movieAgents,
     ...soundEffectAgents,
+    ...lipSyncAgents,
     mediaMockAgent,
     fileWriteAgent,
     openAIAgent,
@@ -151,7 +155,7 @@ const beat_graph_data = {
                 },
                 params: {
                     model: ":preprocessor.movieAgentInfo.movieParams.model",
-                    duration: ":beat.duration",
+                    duration: ":preprocessor.beatDuration",
                     canvasSize: ":context.presentationStyle.canvasSize",
                 },
             },
@@ -171,6 +175,7 @@ const beat_graph_data = {
         },
         audioChecker: {
             agent: async (namedInputs) => {
+                // NOTE: We intentinonally don't check lipSyncFile here.
                 if (namedInputs.soundEffectFile) {
                     return { hasMovieAudio: true };
                 }
@@ -182,7 +187,7 @@ const beat_graph_data = {
                 return { hasMovieAudio: hasAudio };
             },
             inputs: {
-                onComplete: [":movieGenerator", ":htmlImageGenerator", ":soundEffectGenerator"], // to wait for movieGenerator and htmlImageGenerator to finish
+                onComplete: [":movieGenerator", ":htmlImageGenerator", ":soundEffectGenerator"],
                 movieFile: ":preprocessor.movieFile",
                 imageFile: ":preprocessor.imagePath",
                 soundEffectFile: ":preprocessor.soundEffectFile",
@@ -198,7 +203,7 @@ const beat_graph_data = {
                 soundEffectFile: ":preprocessor.soundEffectFile",
                 params: {
                     model: ":preprocessor.soundEffectModel",
-                    duration: ":beat.duration",
+                    duration: ":preprocessor.beatDuration",
                 },
                 cache: {
                     force: [":context.force"],
@@ -210,19 +215,43 @@ const beat_graph_data = {
             },
             defaultValue: {},
         },
+        lipSyncGenerator: {
+            if: ":beat.enableLipSync",
+            agent: ":preprocessor.lipSyncAgentInfo.agentName",
+            inputs: {
+                onComplete: [":soundEffectGenerator"], // to wait for soundEffectGenerator to finish
+                movieFile: ":preprocessor.movieFile",
+                audioFile: ":preprocessor.audioFile",
+                lipSyncFile: ":preprocessor.lipSyncFile",
+                params: {
+                    model: ":preprocessor.lipSyncModel",
+                    duration: ":preprocessor.beatDuration",
+                },
+                cache: {
+                    force: [":context.force"],
+                    file: ":preprocessor.lipSyncFile",
+                    index: ":__mapIndex",
+                    sessionType: "lipSync",
+                    mulmoContext: ":context",
+                },
+            },
+            defaultValue: {},
+        },
         output: {
             agent: "copyAgent",
             inputs: {
-                onComplete: [":imageFromMovie", ":htmlImageGenerator", ":audioChecker", ":soundEffectGenerator"], // to wait for imageFromMovie to finish
+                onComplete: [":imageFromMovie", ":htmlImageGenerator", ":audioChecker", ":soundEffectGenerator", ":lipSyncGenerator"], // to wait for imageFromMovie, soundEffectGenerator, and lipSyncGenerator to finish
                 imageFile: ":preprocessor.imagePath",
                 movieFile: ":preprocessor.movieFile",
                 soundEffectFile: ":preprocessor.soundEffectFile",
+                lipSyncFile: ":preprocessor.lipSyncFile",
                 hasMovieAudio: ":audioChecker.hasMovieAudio",
             },
             output: {
                 imageFile: ".imageFile",
                 movieFile: ".movieFile",
                 soundEffectFile: ".soundEffectFile",
+                lipSyncFile: ".lipSyncFile",
                 hasMovieAudio: ".hasMovieAudio",
             },
             isResult: true,
@@ -316,7 +345,7 @@ export const graphOption = async (context, settings) => {
             {
                 name: "fileCacheAgentFilter",
                 agent: fileCacheAgentFilter,
-                nodeIds: ["imageGenerator", "movieGenerator", "htmlImageAgent", "soundEffectGenerator"],
+                nodeIds: ["imageGenerator", "movieGenerator", "htmlImageAgent", "soundEffectGenerator", "lipSyncGenerator"],
             },
         ],
         taskManager: new TaskManager(MulmoPresentationStyleMethods.getConcurrency(context.presentationStyle)),

package/lib/actions/movie.js

@@ -162,7 +162,7 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, context) => {
             beatTimestamps.push(timestamp);
             return timestamp; // Skip voice-over beats.
         }
-        const sourceFile = studioBeat.soundEffectFile ?? studioBeat.movieFile ?? studioBeat.imageFile;
+        const sourceFile = studioBeat.lipSyncFile ?? studioBeat.soundEffectFile ?? studioBeat.movieFile ?? studioBeat.imageFile;
         assert(!!sourceFile, `studioBeat.imageFile or studioBeat.movieFile is not set: index=${index}`);
         assert(!!studioBeat.duration, `studioBeat.duration is not set: index=${index}`);
         const extraPadding = (() => {
@@ -183,7 +183,7 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, context) => {
         const defaultFillOption = mulmoFillOptionSchema.parse({}); // let the schema infer the default value
         const fillOption = { ...defaultFillOption, ...globalFillOption, ...beatFillOption };
         const inputIndex = FfmpegContextAddInput(ffmpegContext, sourceFile);
-        const mediaType = studioBeat.movieFile ? "movie" : MulmoPresentationStyleMethods.getImageType(context.presentationStyle, beat);
+        const mediaType = studioBeat.lipSyncFile || studioBeat.movieFile ? "movie" : MulmoPresentationStyleMethods.getImageType(context.presentationStyle, beat);
         const speed = beat.movieParams?.speed ?? 1.0;
         const { videoId, videoPart } = getVideoPart(inputIndex, mediaType, duration, canvasInfo, fillOption, speed);
         ffmpegContext.filterComplex.push(videoPart);
@@ -206,6 +206,7 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, context) => {
         // NOTE: We don't support audio if the speed is not 1.0.
         const movieVolume = beat.audioParams?.movieVolume ?? 1.0;
         if (studioBeat.hasMovieAudio && movieVolume > 0.0 && speed === 1.0) {
+            // TODO: Handle a special case where it has lipSyncFile AND hasMovieAudio is on (the source file has an audio, such as sound effect).
             const { audioId, audioPart } = getAudioPart(inputIndex, duration, timestamp, movieVolume);
             audioIdsFromMovieBeats.push(audioId);
             ffmpegContext.filterComplex.push(audioPart);

package/lib/agents/index.d.ts

@@ -11,8 +11,9 @@ import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
 import ttsOpenaiAgent from "./tts_openai_agent.js";
 import validateSchemaAgent from "./validate_schema_agent.js";
 import soundEffectReplicateAgent from "./sound_effect_replicate_agent.js";
+import lipSyncReplicateAgent from "./lipsync_replicate_agent.js";
 import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, soundEffectReplicateAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, };

package/lib/agents/index.js

@@ -11,9 +11,10 @@ import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
 import ttsOpenaiAgent from "./tts_openai_agent.js";
 import validateSchemaAgent from "./validate_schema_agent.js";
 import soundEffectReplicateAgent from "./sound_effect_replicate_agent.js";
+import lipSyncReplicateAgent from "./lipsync_replicate_agent.js";
 import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 // import * as vanilla from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, soundEffectReplicateAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, };

package/lib/agents/lipsync_replicate_agent.d.ts

@@ -0,0 +1,5 @@
+import type { AgentFunction, AgentFunctionInfo } from "graphai";
+import type { AgentBufferResult, LipSyncAgentInputs, ReplicateLipSyncAgentParams, ReplicateLipSyncAgentConfig } from "../types/agent.js";
+export declare const lipSyncReplicateAgent: AgentFunction<ReplicateLipSyncAgentParams, AgentBufferResult, LipSyncAgentInputs, ReplicateLipSyncAgentConfig>;
+declare const lipSyncReplicateAgentInfo: AgentFunctionInfo;
+export default lipSyncReplicateAgentInfo;

package/lib/agents/lipsync_replicate_agent.js

@@ -0,0 +1,72 @@
+import { readFileSync } from "fs";
+import { GraphAILogger } from "graphai";
+import Replicate from "replicate";
+import { provider2LipSyncAgent } from "../utils/provider2agent.js";
+export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) => {
+    const { movieFile, audioFile } = namedInputs;
+    const apiKey = config?.apiKey;
+    const model = params.model ?? provider2LipSyncAgent.replicate.defaultModel;
+    if (!apiKey) {
+        throw new Error("REPLICATE_API_TOKEN environment variable is required");
+    }
+    const replicate = new Replicate({
+        auth: apiKey,
+    });
+    const videoBuffer = readFileSync(movieFile);
+    const audioBuffer = readFileSync(audioFile);
+    const videoUri = `data:video/quicktime;base64,${videoBuffer.toString("base64")}`;
+    const audioUri = `data:audio/wav;base64,${audioBuffer.toString("base64")}`;
+    const input = {
+        video: undefined,
+        video_input: undefined,
+        video_url: undefined,
+        audio: undefined,
+        audio_input: undefined,
+        audio_file: undefined,
+    };
+    const modelParams = provider2LipSyncAgent.replicate.modelParams[model];
+    if (!modelParams) {
+        throw new Error(`Model ${model} is not supported`);
+    }
+    const videoParam = modelParams.video;
+    const audioParam = modelParams.audio;
+    if (videoParam === "video" || videoParam === "video_input" || videoParam === "video_url") {
+        input[videoParam] = videoUri;
+    }
+    if (audioParam === "audio" || audioParam === "audio_input" || audioParam === "audio_file") {
+        input[audioParam] = audioUri;
+    }
+    const model_identifier = provider2LipSyncAgent.replicate.modelParams[model]?.identifier ?? model;
+    try {
+        const output = await replicate.run(model_identifier, {
+            input,
+        });
+        if (output && typeof output === "object" && "url" in output) {
+            const videoUrl = output.url();
+            const videoResponse = await fetch(videoUrl);
+            if (!videoResponse.ok) {
+                throw new Error(`Error downloading video: ${videoResponse.status} - ${videoResponse.statusText}`);
+            }
+            const arrayBuffer = await videoResponse.arrayBuffer();
+            return { buffer: Buffer.from(arrayBuffer) };
+        }
+        return undefined;
+    }
+    catch (error) {
+        GraphAILogger.info("Failed to generate lip sync:", error.message);
+        throw error;
+    }
+};
+const lipSyncReplicateAgentInfo = {
+    name: "lipSyncReplicateAgent",
+    agent: lipSyncReplicateAgent,
+    mock: lipSyncReplicateAgent,
+    samples: [],
+    description: "Replicate Lip Sync agent (video + audio to video)",
+    category: ["movie"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    license: "MIT",
+    environmentVariables: ["REPLICATE_API_TOKEN"],
+};
+export default lipSyncReplicateAgentInfo;

package/lib/agents/movie_replicate_agent.js

@@ -71,7 +71,16 @@ export const movieReplicateAgent = async ({ namedInputs, params, config, }) => {
     if (!provider2MovieAgent.replicate.modelParams[model]) {
         throw new Error(`Model ${model} is not supported`);
     }
-    const duration = params.duration ?? provider2MovieAgent.replicate.modelParams[model].durations[0] ?? 5;
+    const duration = (() => {
+        const durations = provider2MovieAgent.replicate.modelParams[model].durations;
+        if (params.duration) {
+            const largerDurations = durations.filter((d) => d >= params.duration);
+            return largerDurations.length > 0 ? largerDurations[0] : durations[durations.length - 1];
+        }
+        else {
+            return durations[0];
+        }
+    })();
     if (!provider2MovieAgent.replicate.modelParams[model].durations.includes(duration)) {
         throw new Error(`Duration ${duration} is not supported for model ${model}. Supported durations: ${provider2MovieAgent.replicate.modelParams[model].durations.join(", ")}`);
     }

package/lib/cli/commands/tool/prompt/builder.js

@@ -1,5 +1,5 @@
-import { getAvailableTemplates } from "../../../../utils/file.js";
-const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
+import { getAvailablePromptTemplates } from "../../../../utils/file.js";
+const availableTemplateNames = getAvailablePromptTemplates().map((template) => template.filename);
 export const builder = (yargs) => {
     return yargs.option("t", {
         alias: "template",

package/lib/cli/commands/tool/scripting/builder.js

@@ -1,6 +1,6 @@
 import { llm } from "../../../../utils/provider2agent.js";
-import { getAvailableTemplates } from "../../../../utils/file.js";
-const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
+import { getAvailablePromptTemplates } from "../../../../utils/file.js";
+const availableTemplateNames = getAvailablePromptTemplates().map((template) => template.filename);
 export const builder = (yargs) => {
     return yargs
         .option("o", {

package/lib/cli/commands/tool/story_to_script/builder.js

@@ -1,7 +1,7 @@
-import { getAvailableTemplates } from "../../../../utils/file.js";
+import { getAvailablePromptTemplates } from "../../../../utils/file.js";
 import { llm } from "../../../../utils/provider2agent.js";
 import { storyToScriptGenerateMode } from "../../../../utils/const.js";
-const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
+const availableTemplateNames = getAvailablePromptTemplates().map((template) => template.filename);
 export const builder = (yargs) => {
     return yargs
         .option("o", {

package/lib/data/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./promptTemplates.js";
+export * from "./scriptTemplates.js";

package/lib/data/index.js

@@ -0,0 +1,2 @@
+export * from "./promptTemplates.js";
+export * from "./scriptTemplates.js";