mulmocast 2.7.2 → 2.8.0

package/lib/agents/combine_audio_files_agent.js

@@ -2,7 +2,7 @@ import { assert, GraphAILogger } from "graphai";
 import { silent60secPath, isFile } from "../utils/file.js";
 import { FfmpegContextInit, FfmpegContextGenerateOutput, FfmpegContextInputFormattedAudio, ffmpegGetMediaDuration, } from "../utils/ffmpeg_utils.js";
 import { MulmoMediaSourceMethods } from "../methods/mulmo_media_source.js";
-import { MulmoBeatMethods } from "../methods/index.js";
+import { MulmoBeatMethods, MulmoPresentationStyleMethods } from "../methods/index.js";
 import { userAssert } from "../utils/utils.js";
 import { getAudioInputIdsError } from "../utils/error_cause.js";
 const getMovieDuration = async (context, beat) => {
@@ -126,7 +126,22 @@ const getVoiceOverGroup = (context, index) => {
 };
 const getSpillOverGroup = (context, mediaDurations, index) => {
     const group = [index];
-    for (let i = index + 1; i < context.studio.beats.length && !mediaDurations[i].hasMedia; i++) {
+    for (let i = index + 1; i < context.studio.beats.length; i++) {
+        const media = mediaDurations[i];
+        // An empty-text beat with only a moviePrompt (no generated audio requested) continues
+        // the preceding narration (spill-over). Generated movies don't exist yet at this point,
+        // so movieDuration === 0; imported movies and animated html_tailwind beats have
+        // movieDuration > 0 and keep their own timeline, ending the group as before.
+        const beat = context.studio.script.beats[i];
+        const isSilentContinuation = !beat.text?.trim() &&
+            media.audioDuration === 0 &&
+            media.movieDuration === 0 &&
+            Boolean(beat.moviePrompt) &&
+            !beat.soundEffectPrompt &&
+            !MulmoPresentationStyleMethods.generatedMovieHasAudio(context.presentationStyle, beat);
+        if (media.hasMedia && !isSilentContinuation) {
+            break;
+        }
         group.push(i);
     }
     return group;

package/lib/agents/movie_replicate_agent.js

@@ -81,11 +81,14 @@ async function generateMovie(model, apiKey, prompt, imagePath, lastFrameImagePat
     }
     // Add generate_audio if the model supports it
     const audio = provider2MovieAgent.replicate.modelParams[model].audio;
+    if (audio.mode === AUDIO_MODE_OPTIONAL) {
+        // Always send the flag: several models (e.g. seedance-2.0's generate_audio, p-video's
+        // save_audio) default it to true, which silently embeds generated audio.
+        // Silent unless explicitly requested.
+        input[audio.param] = generateAudio ?? false;
+    }
     if (generateAudio !== undefined) {
-        if (audio.mode === AUDIO_MODE_OPTIONAL) {
-            input[audio.param] = generateAudio;
-        }
-        else if (audio.mode === AUDIO_MODE_NEVER && generateAudio === true) {
+        if (audio.mode === AUDIO_MODE_NEVER && generateAudio === true) {
             throw new Error(`Model ${model} does not support audio generation`, {
                 cause: agentGenerationError("movieReplicateAgent", imageAction, unsupportedModelTarget),
             });

package/lib/methods/mulmo_presentation_style.d.ts

@@ -204,6 +204,7 @@ export declare const MulmoPresentationStyleMethods: {
         };
         keyName: string;
     };
+    generatedMovieHasAudio(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): boolean;
     getSoundEffectAgentInfo(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): {
         agentName: string;
         defaultModel: import("../types/provider2agent.js").ReplicateModel;

package/lib/methods/mulmo_presentation_style.js

@@ -7,7 +7,7 @@ import { isNull } from "graphai";
 import { userAssert } from "../utils/utils.js";
 import { slideThemes } from "../data/slideThemes.js";
 import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, mulmoTransitionSchema, } from "../types/schema.js";
-import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, } from "../types/provider2agent.js";
+import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, getModelAudio, AUDIO_MODE_ALWAYS, } from "../types/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; background-color:#fff; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
@@ -131,6 +131,17 @@ export const MulmoPresentationStyleMethods = {
             keyName: agentInfo.keyName,
         };
     },
+    // True when the movie generated for this beat will carry an audio track:
+    // the model always embeds audio, or audio generation is explicitly requested.
+    generatedMovieHasAudio(presentationStyle, beat) {
+        const movieParams = { ...presentationStyle.movieParams, ...beat.movieParams };
+        if (movieParams?.generateAudio === true) {
+            return true;
+        }
+        const provider = text2MovieProviderSchema.parse(movieParams?.provider ?? defaultProviders.text2movie);
+        const model = movieParams?.model ?? provider2MovieAgent[provider].defaultModel;
+        return getModelAudio(provider, model)?.mode === AUDIO_MODE_ALWAYS;
+    },
     getSoundEffectAgentInfo(presentationStyle, beat) {
         const soundEffectProvider = (beat.soundEffectParams?.provider ??
             presentationStyle.soundEffectParams?.provider ??

package/lib/types/provider2agent.d.ts

@@ -88,7 +88,7 @@ export type ReplicateModel = `${string}/${string}`;
 export declare const AUDIO_MODE_NEVER: "never";
 export declare const AUDIO_MODE_ALWAYS: "always";
 export declare const AUDIO_MODE_OPTIONAL: "optional";
-type MovieAudioSpec = {
+export type MovieAudioSpec = {
     mode: typeof AUDIO_MODE_NEVER;
 } | {
     mode: typeof AUDIO_MODE_ALWAYS;
@@ -213,6 +213,7 @@ export declare const defaultProviders: {
 export declare const llm: (keyof typeof provider2LLMAgent)[];
 export type LLM = keyof typeof provider2LLMAgent;
 export declare const htmlLLMProvider: string[];
+export declare const getModelAudio: (provider: keyof typeof provider2MovieAgent, model: string) => MovieAudioSpec | undefined;
 export declare const getModelDuration: (provider: keyof typeof provider2MovieAgent, model: string, movieDuration?: number) => number | undefined;
 export type ModelPricing = {
     unit: "tokens" | "chars" | "seconds" | "images";

package/lib/types/provider2agent.js

@@ -144,10 +144,12 @@ export const provider2MovieAgent = {
             "minimax/hailuo-2.3",
             "minimax/hailuo-2.3-fast",
             "pixverse/pixverse-v5",
+            "prunaai/p-video",
+            "xai/grok-imagine-video-1.5",
         ],
         modelParams: {
             "bytedance/seedance-1-lite": {
-                durations: [5, 10],
+                durations: [4, 5, 6, 7, 8, 9, 10, 11, 12],
                 start_image: "image",
                 last_image: "last_frame_image",
                 audio: { mode: AUDIO_MODE_NEVER },
@@ -259,10 +261,13 @@ export const provider2MovieAgent = {
                 price_per_sec: 0.12,
             },
             "wan-video/wan-2.2-i2v-fast": {
+                // No duration input: length is num_frames (default 81) / frames_per_second (default 16) ≈ 5s.
                 durations: [5],
                 start_image: "image",
+                start_image_required: true,
+                last_image: "last_image",
                 audio: { mode: AUDIO_MODE_NEVER },
-                price_per_sec: 0.012,
+                price_per_sec: 0.01, // $0.05 per 81-frame 480p video; up to $0.145 at 720p
             },
             "wan-video/wan-2.2-t2v-fast": {
                 durations: [5],
@@ -295,7 +300,7 @@ export const provider2MovieAgent = {
                 last_image: "end_image",
                 reference_images_param: "reference_images",
                 audio: { mode: AUDIO_MODE_OPTIONAL, param: "generate_audio" },
-                price_per_sec: 0.3,
+                price_per_sec: 0.28, // 'pro' (1080p, default); 'standard' $0.168, '4k' $0.42
             },
             "kwaivgi/kling-v3-video": {
                 durations: [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
@@ -335,6 +340,20 @@ export const provider2MovieAgent = {
                 audio: { mode: AUDIO_MODE_NEVER },
                 price_per_sec: 0.12,
             },
+            "prunaai/p-video": {
+                durations: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20],
+                start_image: "image",
+                last_image: "last_frame_image",
+                audio: { mode: AUDIO_MODE_OPTIONAL, param: "save_audio" },
+                price_per_sec: 0.02, // 720p, draft mode off ($0.04 at 1080p)
+            },
+            "xai/grok-imagine-video-1.5": {
+                durations: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
+                start_image: "image",
+                start_image_required: true,
+                audio: { mode: AUDIO_MODE_ALWAYS },
+                price_per_sec: 0.08,
+            },
         },
     },
     google: {
@@ -508,6 +527,10 @@ export const defaultProviders = {
 };
 export const llm = Object.keys(provider2LLMAgent);
 export const htmlLLMProvider = ["openai", "anthropic", "mock"];
+export const getModelAudio = (provider, model) => {
+    const modelParams = provider2MovieAgent[provider]?.modelParams;
+    return modelParams?.[model]?.audio;
+};
 export const getModelDuration = (provider, model, movieDuration) => {
     const modelParams = provider2MovieAgent[provider]?.modelParams;
     const { durations } = modelParams[model];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.7.2",
+  "version": "2.8.0",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",