mulmocast 2.7.1 → 2.8.0

package/lib/agents/combine_audio_files_agent.js

@@ -2,7 +2,7 @@ import { assert, GraphAILogger } from "graphai";
 import { silent60secPath, isFile } from "../utils/file.js";
 import { FfmpegContextInit, FfmpegContextGenerateOutput, FfmpegContextInputFormattedAudio, ffmpegGetMediaDuration, } from "../utils/ffmpeg_utils.js";
 import { MulmoMediaSourceMethods } from "../methods/mulmo_media_source.js";
-import { MulmoBeatMethods } from "../methods/index.js";
+import { MulmoBeatMethods, MulmoPresentationStyleMethods } from "../methods/index.js";
 import { userAssert } from "../utils/utils.js";
 import { getAudioInputIdsError } from "../utils/error_cause.js";
 const getMovieDuration = async (context, beat) => {
@@ -126,7 +126,22 @@ const getVoiceOverGroup = (context, index) => {
 };
 const getSpillOverGroup = (context, mediaDurations, index) => {
     const group = [index];
-    for (let i = index + 1; i < context.studio.beats.length && !mediaDurations[i].hasMedia; i++) {
+    for (let i = index + 1; i < context.studio.beats.length; i++) {
+        const media = mediaDurations[i];
+        // An empty-text beat with only a moviePrompt (no generated audio requested) continues
+        // the preceding narration (spill-over). Generated movies don't exist yet at this point,
+        // so movieDuration === 0; imported movies and animated html_tailwind beats have
+        // movieDuration > 0 and keep their own timeline, ending the group as before.
+        const beat = context.studio.script.beats[i];
+        const isSilentContinuation = !beat.text?.trim() &&
+            media.audioDuration === 0 &&
+            media.movieDuration === 0 &&
+            Boolean(beat.moviePrompt) &&
+            !beat.soundEffectPrompt &&
+            !MulmoPresentationStyleMethods.generatedMovieHasAudio(context.presentationStyle, beat);
+        if (media.hasMedia && !isSilentContinuation) {
+            break;
+        }
         group.push(i);
     }
     return group;

package/lib/agents/movie_replicate_agent.js

@@ -81,11 +81,14 @@ async function generateMovie(model, apiKey, prompt, imagePath, lastFrameImagePat
     }
     // Add generate_audio if the model supports it
     const audio = provider2MovieAgent.replicate.modelParams[model].audio;
+    if (audio.mode === AUDIO_MODE_OPTIONAL) {
+        // Always send the flag: several models (e.g. seedance-2.0's generate_audio, p-video's
+        // save_audio) default it to true, which silently embeds generated audio.
+        // Silent unless explicitly requested.
+        input[audio.param] = generateAudio ?? false;
+    }
     if (generateAudio !== undefined) {
-        if (audio.mode === AUDIO_MODE_OPTIONAL) {
-            input[audio.param] = generateAudio;
-        }
-        else if (audio.mode === AUDIO_MODE_NEVER && generateAudio === true) {
+        if (audio.mode === AUDIO_MODE_NEVER && generateAudio === true) {
             throw new Error(`Model ${model} does not support audio generation`, {
                 cause: agentGenerationError("movieReplicateAgent", imageAction, unsupportedModelTarget),
             });

package/lib/methods/index.d.ts

@@ -1,5 +1,5 @@
 export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
-export * from "./mulmo_beat.js";
+export * from "./mulmo_beat_node.js";
 export * from "./mulmo_script.js";

package/lib/methods/index.js

@@ -1,5 +1,7 @@
 export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
-export * from "./mulmo_beat.js";
+// Node-only MulmoBeatMethods (includes getPlugin). The browser-safe subset lives in
+// ./mulmo_beat.js and is re-exported by ../index.common.js for mulmocast/browser.
+export * from "./mulmo_beat_node.js";
 export * from "./mulmo_script.js";

package/lib/methods/mulmo_beat.d.ts

@@ -9,13 +9,6 @@ export declare const MulmoBeatMethods: {
     isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
     isMovieMode: (animation: unknown) => boolean;
     getHtmlPrompt(beat: MulmoBeat): string | undefined;
-    getPlugin(beat: MulmoBeat): {
-        imageType: string;
-        process: (params: import("../types/type.js").ImageProcessorParams) => Promise<string | undefined> | void;
-        path: (params: import("../types/type.js").ImageProcessorParams) => string | undefined;
-        markdown?: (params: import("../types/type.js").ImageProcessorParams) => string | undefined;
-        html?: (params: import("../types/type.js").ImageProcessorParams) => Promise<string | undefined>;
-    };
     getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
 };
 export {};

package/lib/methods/mulmo_beat.js

@@ -1,4 +1,3 @@
-import { findImagePlugin } from "../utils/image_plugins/index.js";
 /** Type guard: checks if animation value is an object config like { fps: 30 } */
 const isAnimationObject = (animation) => {
     return typeof animation === "object" && animation !== null && !Array.isArray(animation);
@@ -29,13 +28,6 @@ export const MulmoBeatMethods = {
         }
         return beat?.htmlPrompt?.prompt;
     },
-    getPlugin(beat) {
-        const plugin = findImagePlugin(beat?.image?.type);
-        if (!plugin) {
-            throw new Error(`invalid beat image type: ${beat.image}`); // TODO: cause
-        }
-        return plugin;
-    },
     getImageReferenceForImageGenerator(beat, imageRefs) {
         const imageNames = beat.imageNames ?? Object.keys(imageRefs); // use all images if imageNames is not specified
         const sources = imageNames.map((name) => imageRefs[name]);

package/lib/methods/mulmo_beat_node.d.ts

@@ -0,0 +1,22 @@
+import { MulmoBeat } from "../types/index.js";
+export declare const MulmoBeatMethods: {
+    getPlugin(beat: MulmoBeat): {
+        imageType: string;
+        process: (params: import("../types/type.js").ImageProcessorParams) => Promise<string | undefined> | void;
+        path: (params: import("../types/type.js").ImageProcessorParams) => string | undefined;
+        markdown?: (params: import("../types/type.js").ImageProcessorParams) => string | undefined;
+        html?: (params: import("../types/type.js").ImageProcessorParams) => Promise<string | undefined>;
+    };
+    isAnimationEnabled: (animation: unknown) => animation is true | {
+        fps?: number;
+        movie?: boolean;
+    };
+    isAnimationObject: (animation: unknown) => animation is {
+        fps?: number;
+        movie?: boolean;
+    };
+    isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
+    isMovieMode: (animation: unknown) => boolean;
+    getHtmlPrompt(beat: MulmoBeat): string | undefined;
+    getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
+};

package/lib/methods/mulmo_beat_node.js

@@ -0,0 +1,10 @@
+import { MulmoBeatMethods as MulmoBeatMethodsBrowser } from "./mulmo_beat.js";
+import { getBeatPlugin } from "../utils/image_plugins/index.js";
+// Node-only extension of MulmoBeatMethods. Kept out of methods/mulmo_beat.ts so
+// browser bundles (mulmocast/browser) do not pull in image_plugins → mulmocast-vision → puppeteer.
+export const MulmoBeatMethods = {
+    ...MulmoBeatMethodsBrowser,
+    getPlugin(beat) {
+        return getBeatPlugin(beat);
+    },
+};

package/lib/methods/mulmo_presentation_style.d.ts

@@ -204,6 +204,7 @@ export declare const MulmoPresentationStyleMethods: {
         };
         keyName: string;
     };
+    generatedMovieHasAudio(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): boolean;
     getSoundEffectAgentInfo(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): {
         agentName: string;
         defaultModel: import("../types/provider2agent.js").ReplicateModel;

package/lib/methods/mulmo_presentation_style.js

@@ -7,7 +7,7 @@ import { isNull } from "graphai";
 import { userAssert } from "../utils/utils.js";
 import { slideThemes } from "../data/slideThemes.js";
 import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, mulmoTransitionSchema, } from "../types/schema.js";
-import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, } from "../types/provider2agent.js";
+import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, getModelAudio, AUDIO_MODE_ALWAYS, } from "../types/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; background-color:#fff; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
@@ -131,6 +131,17 @@ export const MulmoPresentationStyleMethods = {
             keyName: agentInfo.keyName,
         };
     },
+    // True when the movie generated for this beat will carry an audio track:
+    // the model always embeds audio, or audio generation is explicitly requested.
+    generatedMovieHasAudio(presentationStyle, beat) {
+        const movieParams = { ...presentationStyle.movieParams, ...beat.movieParams };
+        if (movieParams?.generateAudio === true) {
+            return true;
+        }
+        const provider = text2MovieProviderSchema.parse(movieParams?.provider ?? defaultProviders.text2movie);
+        const model = movieParams?.model ?? provider2MovieAgent[provider].defaultModel;
+        return getModelAudio(provider, model)?.mode === AUDIO_MODE_ALWAYS;
+    },
     getSoundEffectAgentInfo(presentationStyle, beat) {
         const soundEffectProvider = (beat.soundEffectParams?.provider ??
             presentationStyle.soundEffectParams?.provider ??

package/lib/types/provider2agent.d.ts

@@ -88,7 +88,7 @@ export type ReplicateModel = `${string}/${string}`;
 export declare const AUDIO_MODE_NEVER: "never";
 export declare const AUDIO_MODE_ALWAYS: "always";
 export declare const AUDIO_MODE_OPTIONAL: "optional";
-type MovieAudioSpec = {
+export type MovieAudioSpec = {
     mode: typeof AUDIO_MODE_NEVER;
 } | {
     mode: typeof AUDIO_MODE_ALWAYS;
@@ -213,6 +213,7 @@ export declare const defaultProviders: {
 export declare const llm: (keyof typeof provider2LLMAgent)[];
 export type LLM = keyof typeof provider2LLMAgent;
 export declare const htmlLLMProvider: string[];
+export declare const getModelAudio: (provider: keyof typeof provider2MovieAgent, model: string) => MovieAudioSpec | undefined;
 export declare const getModelDuration: (provider: keyof typeof provider2MovieAgent, model: string, movieDuration?: number) => number | undefined;
 export type ModelPricing = {
     unit: "tokens" | "chars" | "seconds" | "images";

package/lib/types/provider2agent.js

@@ -144,10 +144,12 @@ export const provider2MovieAgent = {
             "minimax/hailuo-2.3",
             "minimax/hailuo-2.3-fast",
             "pixverse/pixverse-v5",
+            "prunaai/p-video",
+            "xai/grok-imagine-video-1.5",
         ],
         modelParams: {
             "bytedance/seedance-1-lite": {
-                durations: [5, 10],
+                durations: [4, 5, 6, 7, 8, 9, 10, 11, 12],
                 start_image: "image",
                 last_image: "last_frame_image",
                 audio: { mode: AUDIO_MODE_NEVER },
@@ -259,10 +261,13 @@ export const provider2MovieAgent = {
                 price_per_sec: 0.12,
             },
             "wan-video/wan-2.2-i2v-fast": {
+                // No duration input: length is num_frames (default 81) / frames_per_second (default 16) ≈ 5s.
                 durations: [5],
                 start_image: "image",
+                start_image_required: true,
+                last_image: "last_image",
                 audio: { mode: AUDIO_MODE_NEVER },
-                price_per_sec: 0.012,
+                price_per_sec: 0.01, // $0.05 per 81-frame 480p video; up to $0.145 at 720p
             },
             "wan-video/wan-2.2-t2v-fast": {
                 durations: [5],
@@ -295,7 +300,7 @@ export const provider2MovieAgent = {
                 last_image: "end_image",
                 reference_images_param: "reference_images",
                 audio: { mode: AUDIO_MODE_OPTIONAL, param: "generate_audio" },
-                price_per_sec: 0.3,
+                price_per_sec: 0.28, // 'pro' (1080p, default); 'standard' $0.168, '4k' $0.42
             },
             "kwaivgi/kling-v3-video": {
                 durations: [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
@@ -335,6 +340,20 @@ export const provider2MovieAgent = {
                 audio: { mode: AUDIO_MODE_NEVER },
                 price_per_sec: 0.12,
             },
+            "prunaai/p-video": {
+                durations: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20],
+                start_image: "image",
+                last_image: "last_frame_image",
+                audio: { mode: AUDIO_MODE_OPTIONAL, param: "save_audio" },
+                price_per_sec: 0.02, // 720p, draft mode off ($0.04 at 1080p)
+            },
+            "xai/grok-imagine-video-1.5": {
+                durations: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
+                start_image: "image",
+                start_image_required: true,
+                audio: { mode: AUDIO_MODE_ALWAYS },
+                price_per_sec: 0.08,
+            },
         },
     },
     google: {
@@ -508,6 +527,10 @@ export const defaultProviders = {
 };
 export const llm = Object.keys(provider2LLMAgent);
 export const htmlLLMProvider = ["openai", "anthropic", "mock"];
+export const getModelAudio = (provider, model) => {
+    const modelParams = provider2MovieAgent[provider]?.modelParams;
+    return modelParams?.[model]?.audio;
+};
 export const getModelDuration = (provider, model, movieDuration) => {
     const modelParams = provider2MovieAgent[provider]?.modelParams;
     const { durations } = modelParams[model];

package/lib/types/schema.js

@@ -423,8 +423,16 @@ export const audioParamsSchema = z
     .object({
     padding: z.number().optional().default(0.3).describe("Padding between beats"), // seconds
     introPadding: z.number().optional().default(1.0).describe("Padding at the beginning of the audio"), // seconds
-    closingPadding: z.number().optional().default(0.8).describe("Padding before the last beat"), // seconds
-    outroPadding: z.number().optional().default(1.0).describe("Padding at the end of the audio"), // seconds
+    closingPadding: z
+        .number()
+        .optional()
+        .default(0.8)
+        .describe("Padding before the last beat (typically the closing credit). For padding after the last beat, use outroPadding"), // seconds
+    outroPadding: z
+        .number()
+        .optional()
+        .default(1.0)
+        .describe("Padding after the last beat, at the very end of the audio where the BGM fades out (distinct from closingPadding, which comes before the last beat)"), // seconds
     bgm: mediaSourceSchema.optional(),
     bgmVolume: z.number().optional().default(0.2).describe("Volume of the background music"),
     audioVolume: z.number().optional().default(1.0).describe("Volume of the audio"),

package/lib/utils/image_plugins/index.d.ts

@@ -1,4 +1,4 @@
-import { ImageProcessorParams } from "../../types/index.js";
+import { ImageProcessorParams, MulmoBeat } from "../../types/index.js";
 export declare const findImagePlugin: (imageType?: string) => {
     imageType: string;
     process: (params: ImageProcessorParams) => Promise<string | undefined> | void;
@@ -6,3 +6,10 @@ export declare const findImagePlugin: (imageType?: string) => {
     markdown?: (params: ImageProcessorParams) => string | undefined;
     html?: (params: ImageProcessorParams) => Promise<string | undefined>;
 } | undefined;
+export declare const getBeatPlugin: (beat: MulmoBeat) => {
+    imageType: string;
+    process: (params: ImageProcessorParams) => Promise<string | undefined> | void;
+    path: (params: ImageProcessorParams) => string | undefined;
+    markdown?: (params: ImageProcessorParams) => string | undefined;
+    html?: (params: ImageProcessorParams) => Promise<string | undefined>;
+};

package/lib/utils/image_plugins/index.js

@@ -25,3 +25,10 @@ const imagePlugins = [
 export const findImagePlugin = (imageType) => {
     return imagePlugins.find((plugin) => plugin.imageType === imageType);
 };
+export const getBeatPlugin = (beat) => {
+    const plugin = findImagePlugin(beat?.image?.type);
+    if (!plugin) {
+        throw new Error(`invalid beat image type: ${beat.image}`);
+    }
+    return plugin;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.7.1",
+  "version": "2.8.0",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",