mulmocast 1.1.1 → 1.1.3

package/lib/actions/image_agents.d.ts

@@ -28,6 +28,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
         defaultModel: string;
     };
     audioFile?: string;
+    beatDuration?: number;
     htmlPrompt?: undefined;
     htmlPath?: undefined;
     htmlImageSystemPrompt?: undefined;
@@ -65,6 +66,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
         defaultModel: string;
     };
     audioFile?: string;
+    beatDuration?: number;
     htmlPrompt?: undefined;
     htmlPath?: undefined;
     htmlImageSystemPrompt?: undefined;
@@ -105,6 +107,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
         defaultModel: string;
     };
     audioFile?: string;
+    beatDuration?: number;
     htmlPrompt?: undefined;
     htmlPath?: undefined;
     htmlImageSystemPrompt?: undefined;

package/lib/actions/image_agents.js

@@ -11,6 +11,7 @@ const htmlStyle = (context, beat) => {
 };
 export const imagePreprocessAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs } = namedInputs;
+    const studioBeat = context.studio.beats[index];
     const imagePath = getBeatPngImagePath(context, index);
     if (beat.htmlPrompt) {
         const htmlPrompt = MulmoBeatMethods.getHtmlPrompt(beat);
@@ -22,6 +23,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
     const returnValue = {
         imageParams: imageAgentInfo.imageParams,
         movieFile: beat.moviePrompt ? moviePaths.movieFile : undefined,
+        beatDuration: beat.duration ?? studioBeat?.duration,
     };
     if (beat.soundEffectPrompt) {
         returnValue.soundEffectAgentInfo = MulmoPresentationStyleMethods.getSoundEffectAgentInfo(context.presentationStyle, beat);
@@ -35,7 +37,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
         returnValue.lipSyncModel = beat.lipSyncParams?.model ?? context.presentationStyle.lipSyncParams?.model ?? returnValue.lipSyncAgentInfo.defaultModel;
         returnValue.lipSyncFile = moviePaths.lipSyncFile;
         // Audio file will be set from the beat's audio file when available
-        returnValue.audioFile = context.studio.beats[index]?.audioFile;
+        returnValue.audioFile = studioBeat?.audioFile;
     }
     if (beat.image) {
         const plugin = MulmoBeatMethods.getPlugin(beat);

package/lib/actions/images.js

@@ -155,7 +155,7 @@ const beat_graph_data = {
                 },
                 params: {
                     model: ":preprocessor.movieAgentInfo.movieParams.model",
-                    duration: ":beat.duration",
+                    duration: ":preprocessor.beatDuration",
                     canvasSize: ":context.presentationStyle.canvasSize",
                 },
             },
@@ -175,8 +175,8 @@ const beat_graph_data = {
         },
         audioChecker: {
             agent: async (namedInputs) => {
+                // NOTE: We intentinonally don't check lipSyncFile here.
                 if (namedInputs.soundEffectFile) {
-                    // NOTE: We intentinonally don't check lipSyncFile here.
                     return { hasMovieAudio: true };
                 }
                 const sourceFile = namedInputs.movieFile || namedInputs.imageFile;
@@ -187,7 +187,7 @@ const beat_graph_data = {
                 return { hasMovieAudio: hasAudio };
             },
             inputs: {
-                onComplete: [":movieGenerator", ":htmlImageGenerator", ":soundEffectGenerator"], // to wait for movieGenerator, htmlImageGenerator, soundEffectGenerator, and lipSyncGenerator to finish
+                onComplete: [":movieGenerator", ":htmlImageGenerator", ":soundEffectGenerator"],
                 movieFile: ":preprocessor.movieFile",
                 imageFile: ":preprocessor.imagePath",
                 soundEffectFile: ":preprocessor.soundEffectFile",
@@ -203,7 +203,7 @@ const beat_graph_data = {
                 soundEffectFile: ":preprocessor.soundEffectFile",
                 params: {
                     model: ":preprocessor.soundEffectModel",
-                    duration: ":beat.duration",
+                    duration: ":preprocessor.beatDuration",
                 },
                 cache: {
                     force: [":context.force"],
@@ -225,7 +225,7 @@ const beat_graph_data = {
                 lipSyncFile: ":preprocessor.lipSyncFile",
                 params: {
                     model: ":preprocessor.lipSyncModel",
-                    duration: ":beat.duration",
+                    duration: ":preprocessor.beatDuration",
                 },
                 cache: {
                     force: [":context.force"],

package/lib/agents/lipsync_replicate_agent.js

@@ -17,12 +17,27 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
     const videoUri = `data:video/quicktime;base64,${videoBuffer.toString("base64")}`;
     const audioUri = `data:audio/wav;base64,${audioBuffer.toString("base64")}`;
     const input = {
-        video: videoUri,
-        audio: audioUri,
-        duration: params.duration,
+        video: undefined,
+        video_input: undefined,
+        video_url: undefined,
+        audio: undefined,
+        audio_input: undefined,
+        audio_file: undefined,
     };
+    const modelParams = provider2LipSyncAgent.replicate.modelParams[model];
+    if (!modelParams) {
+        throw new Error(`Model ${model} is not supported`);
+    }
+    const videoParam = modelParams.video;
+    const audioParam = modelParams.audio;
+    if (videoParam === "video" || videoParam === "video_input" || videoParam === "video_url") {
+        input[videoParam] = videoUri;
+    }
+    if (audioParam === "audio" || audioParam === "audio_input" || audioParam === "audio_file") {
+        input[audioParam] = audioUri;
+    }
+    const model_identifier = provider2LipSyncAgent.replicate.modelParams[model]?.identifier ?? model;
     try {
-        const model_identifier = provider2LipSyncAgent.replicate.modelParams[model]?.identifier ?? model;
         const output = await replicate.run(model_identifier, {
             input,
         });

package/lib/agents/movie_replicate_agent.js

@@ -71,7 +71,16 @@ export const movieReplicateAgent = async ({ namedInputs, params, config, }) => {
     if (!provider2MovieAgent.replicate.modelParams[model]) {
         throw new Error(`Model ${model} is not supported`);
     }
-    const duration = params.duration ?? provider2MovieAgent.replicate.modelParams[model].durations[0] ?? 5;
+    const duration = (() => {
+        const durations = provider2MovieAgent.replicate.modelParams[model].durations;
+        if (params.duration) {
+            const largerDurations = durations.filter((d) => d >= params.duration);
+            return largerDurations.length > 0 ? largerDurations[0] : durations[durations.length - 1];
+        }
+        else {
+            return durations[0];
+        }
+    })();
     if (!provider2MovieAgent.replicate.modelParams[model].durations.includes(duration)) {
         throw new Error(`Duration ${duration} is not supported for model ${model}. Supported durations: ${provider2MovieAgent.replicate.modelParams[model].durations.join(", ")}`);
     }

package/lib/cli/commands/tool/prompt/builder.js

@@ -1,5 +1,5 @@
-import { getAvailableTemplates } from "../../../../utils/file.js";
-const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
+import { getAvailablePromptTemplates } from "../../../../utils/file.js";
+const availableTemplateNames = getAvailablePromptTemplates().map((template) => template.filename);
 export const builder = (yargs) => {
     return yargs.option("t", {
         alias: "template",

package/lib/cli/commands/tool/scripting/builder.js

@@ -1,6 +1,6 @@
 import { llm } from "../../../../utils/provider2agent.js";
-import { getAvailableTemplates } from "../../../../utils/file.js";
-const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
+import { getAvailablePromptTemplates } from "../../../../utils/file.js";
+const availableTemplateNames = getAvailablePromptTemplates().map((template) => template.filename);
 export const builder = (yargs) => {
     return yargs
         .option("o", {

package/lib/cli/commands/tool/story_to_script/builder.js

@@ -1,7 +1,7 @@
-import { getAvailableTemplates } from "../../../../utils/file.js";
+import { getAvailablePromptTemplates } from "../../../../utils/file.js";
 import { llm } from "../../../../utils/provider2agent.js";
 import { storyToScriptGenerateMode } from "../../../../utils/const.js";
-const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
+const availableTemplateNames = getAvailablePromptTemplates().map((template) => template.filename);
 export const builder = (yargs) => {
     return yargs
         .option("o", {