mulmocast 2.12.1 → 2.14.0

package/lib/actions/movie.js

@@ -105,6 +105,30 @@ const getOutputOption = (audioId, videoId) => {
         "-b:a 128k", // Audio bitrate
     ];
 };
+// Talking avatars (actions/avatar.ts): each track is a see-through video as long as the whole movie,
+// overlaid after transitions so they do not move it. "-c:v libvpx-vp9" is needed to decode the alpha.
+// A track is shown in stretches (placements); each stretch is trimmed out of the track, kept at its
+// time, scaled to its size and placed where it says, so no filter runs outside its stretch.
+export const addAvatars = (ffmpegContext, videoId, context) => {
+    return (context.studio.avatarTracks ?? []).reduce((accId, track, trackIndex) => {
+        if (track.placements.length === 0)
+            return accId;
+        const inputIndex = FfmpegContextAddInput(ffmpegContext, track.file, ["-c:v", "libvpx-vp9"]);
+        const sources = track.placements.length === 1 ? [`${inputIndex}:v`] : track.placements.map((_, index) => `avatar${trackIndex}_src${index}`);
+        const labels = sources.map((id) => "[" + id + "]").join("");
+        if (track.placements.length > 1)
+            ffmpegContext.filterComplex.push(`[${inputIndex}:v]split=${sources.length}${labels}`);
+        return track.placements.reduce((prevId, placement, index) => {
+            const partId = `avatar${trackIndex}_${index}s`;
+            const resized = placement.width !== track.width || placement.height !== track.height;
+            const scale = resized ? `,scale=${placement.width}:${placement.height}` : "";
+            ffmpegContext.filterComplex.push(`[${sources[index]}]trim=start=${placement.start}:end=${placement.end},setpts=PTS-STARTPTS+${placement.start}/TB${scale}[${partId}]`);
+            const outId = `avatar${trackIndex}_${index}`;
+            ffmpegContext.filterComplex.push(`[${prevId}][${partId}]overlay=x=${placement.x}:y=${placement.y}:format=auto:eof_action=pass:enable='gte(t,${placement.start})*lt(t,${placement.end})'[${outId}]`);
+            return outId;
+        }, accId);
+    }, videoId);
+};
 const addCaptions = (ffmpegContext, concatVideoId, context, caption) => {
     const beatsWithCaptions = context.studio.beats.filter(({ captionFiles }) => captionFiles && captionFiles.length > 0);
     if (caption && beatsWithCaptions.length > 0) {
@@ -470,8 +494,13 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
     // Concatenate the trimmed images
     const concatVideoId = "concat_video";
     ffmpegContext.filterComplex.push(getConcatVideoFilter(concatVideoId, videoIdsForBeats));
-    const captionedVideoId = addCaptions(ffmpegContext, concatVideoId, context, caption);
-    const mixedVideoId = addTransitionEffects(ffmpegContext, captionedVideoId, context, transitionVideoIds, beatTimestamps, videoIdsForBeats);
+    // With talking avatars, captions go on last so that an avatar never covers them. Without, the order
+    // is unchanged: captions under the transitions.
+    const hasAvatars = (context.studio.avatarTracks ?? []).some((track) => track.placements.length > 0);
+    const captionedVideoId = hasAvatars ? concatVideoId : addCaptions(ffmpegContext, concatVideoId, context, caption);
+    const transitionedVideoId = addTransitionEffects(ffmpegContext, captionedVideoId, context, transitionVideoIds, beatTimestamps, videoIdsForBeats);
+    const avatarVideoId = addAvatars(ffmpegContext, transitionedVideoId, context);
+    const mixedVideoId = hasAvatars ? addCaptions(ffmpegContext, avatarVideoId, context, caption) : avatarVideoId;
     if (isTest) {
         return ffmpegContext.filterComplex;
     }
@@ -491,8 +520,17 @@ export const movieFilePath = (context) => {
     const caption = MulmoStudioContextMethods.getCaption(context);
     return getOutputVideoFilePath(outDirPath, fileName, context.lang, caption);
 };
+// A speaker has an avatar but no track was rendered: the `avatar` action did not run before `movie`.
+const warnIfAvatarsMissing = (context) => {
+    if (context.studio.avatarTracks)
+        return;
+    const speaksWithAvatar = context.studio.script.beats.some((beat) => MulmoPresentationStyleMethods.getSpeakerAvatar(context.presentationStyle, beat, context.lang));
+    if (speaksWithAvatar)
+        GraphAILogger.warn("movie: a speaker has an avatar, but no avatar track was rendered. Run the avatar action before movie.");
+};
 export const movie = async (context) => {
     MulmoStudioContextMethods.setSessionState(context, "video", true);
+    warnIfAvatarsMissing(context);
     try {
         const audioArtifactFilePath = getAudioArtifactFilePath(context);
         const outputVideoPath = movieFilePath(context);

package/lib/agents/combine_audio_files_agent.js

@@ -14,8 +14,8 @@ const getMovieDuration = async (context, beat) => {
             return { duration: duration / speed, hasAudio };
         }
     }
-    // Animated html_tailwind beats with explicit duration act as movie-like for voice_over grouping
-    if (MulmoBeatMethods.isAnimatedHtmlTailwind(beat) && beat.duration !== undefined) {
+    // Plugin-video beats (animated html_tailwind, remotion) with explicit duration act as movie-like for voice_over grouping
+    if (MulmoBeatMethods.isPluginVideo(beat) && beat.duration !== undefined) {
         return { duration: beat.duration, hasAudio: false };
     }
     return { duration: 0, hasAudio: false };

package/lib/cli/commands/movie/handler.js

@@ -1,4 +1,4 @@
-import { audio, images, movie, captions } from "../../../actions/index.js";
+import { audio, images, movie, captions, avatar } from "../../../actions/index.js";
 import { dumpUsageIfRequested, initializeContext, runTranslateIfNeeded, printUsageEstimate } from "../../helpers.js";
 export const handler = async (argv) => {
     const context = await initializeContext(argv);
@@ -10,6 +10,6 @@ export const handler = async (argv) => {
         return;
     }
     await runTranslateIfNeeded(context, true);
-    await audio(context).then(images).then(captions).then(movie);
+    await audio(context).then(images).then(captions).then(avatar).then(movie);
     dumpUsageIfRequested(context);
 };

package/lib/index.remotion.d.ts

@@ -0,0 +1,3 @@
+export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
+export { REMOTION_PACKAGES, REMOTION_RENDER_PACKAGES, REMOTION_SCENE_PACKAGES, remotionInstallCommand } from "./utils/remotion/packages.js";
+export { ensureRemotionPackages, missingInstalledRemotionPackages } from "./utils/remotion/render.js";

package/lib/index.remotion.js

@@ -0,0 +1,4 @@
+// Entry point for hosts whose own agent writes the remotion beat's component and passes it as `code`.
+export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
+export { REMOTION_PACKAGES, REMOTION_RENDER_PACKAGES, REMOTION_SCENE_PACKAGES, remotionInstallCommand } from "./utils/remotion/packages.js";
+export { ensureRemotionPackages, missingInstalledRemotionPackages } from "./utils/remotion/render.js";

package/lib/mcp/server.js

@@ -7,7 +7,7 @@ import fs from "fs";
 import path from "path";
 import { fileURLToPath } from "url";
 import { GraphAILogger } from "graphai";
-import { audio, images, movie, captions, pdf } from "../actions/index.js";
+import { audio, images, movie, captions, avatar, pdf } from "../actions/index.js";
 import { initializeContext, runTranslateIfNeeded } from "../cli/helpers.js";
 import { outDirName } from "../types/const.js";
 import { resolveDirPath, mkdir, generateTimestampedFileName } from "../utils/file.js";
@@ -111,8 +111,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         // Execute the requested command
         switch (cmd) {
             case "movie":
-                // Generate movie (audio + images + captions + movie)
-                await audio(context).then(images).then(captions).then(movie);
+                // Generate movie (audio + images + captions + avatar + movie)
+                await audio(context).then(images).then(captions).then(avatar).then(movie);
                 return {
                     content: [
                         {

package/lib/methods/mulmo_beat.d.ts

@@ -7,6 +7,7 @@ export declare const MulmoBeatMethods: {
     isAnimationEnabled: (animation: unknown) => animation is true | AnimationConfig;
     isAnimationObject: (animation: unknown) => animation is AnimationConfig;
     isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
+    isPluginVideo: (beat: MulmoBeat) => boolean;
     isMovieMode: (animation: unknown) => boolean;
     isVoiceOver: (beat?: MulmoBeat) => boolean;
     getPrevRenderedBeatIndex(beats: MulmoBeat[], index: number): number;

package/lib/methods/mulmo_beat.js

@@ -18,12 +18,15 @@ const isAnimatedHtmlTailwind = (beat) => {
     const animation = beat.image.animation;
     return isAnimationEnabled(animation);
 };
+// Beats whose image plugin writes the beat's .mp4 itself, at the length the audio gives it.
+const isPluginVideo = (beat) => isAnimatedHtmlTailwind(beat) || beat.image?.type === ImageMediaType.Remotion;
 // voice_over beats share the preceding beat's shot, and contribute no video segment of their own.
 const isVoiceOver = (beat) => beat?.image?.type === ImageMediaType.VoiceOver;
 export const MulmoBeatMethods = {
     isAnimationEnabled,
     isAnimationObject,
     isAnimatedHtmlTailwind,
+    isPluginVideo,
     isMovieMode,
     isVoiceOver,
     // Index of the closest preceding beat which is actually rendered as a video segment (-1 if none).

package/lib/methods/mulmo_beat_node.d.ts

@@ -16,6 +16,7 @@ export declare const MulmoBeatMethods: {
         movie?: boolean;
     };
     isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
+    isPluginVideo: (beat: MulmoBeat) => boolean;
     isMovieMode: (animation: unknown) => boolean;
     isVoiceOver: (beat?: MulmoBeat) => boolean;
     getPrevRenderedBeatIndex(beats: MulmoBeat[], index: number): number;

package/lib/methods/mulmo_presentation_style.d.ts

@@ -4,7 +4,7 @@
  * Works in both Node.js and modern browsers.
  */
 import type { SlideTheme } from "@mulmocast/deck";
-import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider, MulmoStudioContext, MulmoTransition, MulmoFillOption } from "../types/index.js";
+import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, MulmoSpeakerAvatar, Text2ImageProvider, MulmoStudioContext, MulmoTransition, MulmoFillOption } from "../types/index.js";
 export declare const MulmoPresentationStyleMethods: {
     getCanvasSize(presentationStyle: MulmoPresentationStyle): MulmoCanvasDimension;
     getAllSpeechProviders(presentationStyle: MulmoPresentationStyle): Set<Text2SpeechProvider>;
@@ -28,6 +28,10 @@ export declare const MulmoPresentationStyleMethods: {
     getResolvedSlideTheme(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SlideTheme;
     getDefaultSpeaker(presentationStyle: MulmoPresentationStyle): string;
     getSpeakerData(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat, lang: string | undefined): SpeakerData;
+    getSpeakerAvatar(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat, lang: string | undefined): {
+        speakerId: string;
+        avatar: MulmoSpeakerAvatar;
+    } | null;
     getSpeaker(context: MulmoStudioContext, beat: MulmoBeat, targetLang: string | undefined): SpeakerData;
     getMovieTransition(context: MulmoStudioContext, beat: MulmoBeat): MulmoTransition | null;
     getNeedFirstFrame(context: MulmoStudioContext): boolean[];

package/lib/methods/mulmo_presentation_style.js

@@ -92,6 +92,14 @@ export const MulmoPresentationStyleMethods = {
         }
         return speaker;
     },
+    // The speaker of a beat and the avatar it speaks with, if any. A language-specific speaker
+    // replaces the whole speaker, so its avatar falls back to the base speaker's.
+    getSpeakerAvatar(presentationStyle, beat, lang) {
+        const speakerId = beat?.speaker ?? MulmoPresentationStyleMethods.getDefaultSpeaker(presentationStyle);
+        const speaker = presentationStyle.speechParams?.speakers?.[speakerId];
+        const avatar = (lang ? speaker?.lang?.[lang]?.avatar : undefined) ?? speaker?.avatar;
+        return avatar ? { speakerId, avatar } : null;
+    },
     getSpeaker(context, beat, targetLang) {
         // Normally, lang is determined by the context, but lang may be specified when using the API.
         const lang = targetLang ?? context.lang ?? context?.studio?.script?.lang;