mulmocast 2.12.0 → 2.13.0

package/lib/actions/avatar.d.ts

@@ -0,0 +1,37 @@
+import type { MulmoAvatarPosition, MulmoStudioContext } from "../types/index.js";
+export type AvatarSegmentPlan = {
+    beatIndex: number;
+    text: string;
+    audio: string;
+    /** absolute seconds on the final timeline */
+    start: number;
+    emotion?: "neutral" | "happy" | "sad" | "angry" | "surprised" | "relaxed";
+    motions?: {
+        motion: string;
+        at?: string;
+    }[];
+};
+/** Where the avatar is during one stretch of the video (absolute seconds, end exclusive). */
+export type AvatarPlacementPlan = {
+    start: number;
+    end: number;
+    position: Required<MulmoAvatarPosition>;
+};
+export type AvatarTrackPlan = {
+    speakerId: string;
+    /** absolute path of the avatar package */
+    source: string;
+    segments: AvatarSegmentPlan[];
+    /** where the avatar is shown, beat by beat; it is not shown outside these */
+    placements: AvatarPlacementPlan[];
+    /** length of the whole video, seconds */
+    duration: number;
+};
+/**
+ * Which beats each avatar speaks, and where it stands, beat by beat. Pure: reads the studio,
+ * writes nothing. Position: the beat's avatarParams, then the speaker's avatar, then the script's
+ * avatarParams, then the defaults; during other speakers' beats the avatar stays where it was.
+ */
+export declare const planAvatarTracks: (context: MulmoStudioContext) => AvatarTrackPlan[];
+/** The `avatar` action: renders the avatar tracks into studio.avatarTracks (none when no speaker has an avatar). */
+export declare const avatar: (context: MulmoStudioContext) => Promise<MulmoStudioContext>;

package/lib/actions/avatar.js

@@ -0,0 +1,176 @@
+// Talking avatars (AvatarScript): for each speaker that has an avatar, one continuous, see-through
+// avatar track for the whole video, speaking that speaker's beats. movie.ts overlays the tracks.
+// See docs/plans/plan_avatar.md.
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { GraphAILogger } from "graphai";
+import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
+import { getFullPath, getOutputStudioFilePath } from "../utils/file.js";
+import { localizedText } from "../utils/utils.js";
+const DEFAULT_POSITION = { x: "84%", y: "100%", scale: "62%" };
+// Show the whole avatar image. A rig may crop the top of its image (a flat cut edge, hidden when the
+// avatar fills the frame); over a slide the cut would run through the head.
+const PAD_TOP = 0;
+const mergePosition = (...positions) => positions.reduce((acc, p) => ({ ...acc, ...Object.fromEntries(Object.entries(p ?? {}).filter(([, v]) => v !== undefined)) }), {
+    ...DEFAULT_POSITION,
+});
+const percent = (value, of) => (of * parseFloat(value)) / 100;
+const samePosition = (a, b) => a.x === b.x && a.y === b.y && a.scale === b.scale;
+/** Adds [start, end) at `position`, joining it to the previous stretch when nothing changes. */
+const place = (track, start, end, position) => {
+    const last = track.placements.at(-1);
+    if (last && last.end === start && samePosition(last.position, position))
+        last.end = end;
+    else
+        track.placements.push({ start, end, position });
+};
+/**
+ * Which beats each avatar speaks, and where it stands, beat by beat. Pure: reads the studio,
+ * writes nothing. Position: the beat's avatarParams, then the speaker's avatar, then the script's
+ * avatarParams, then the defaults; during other speakers' beats the avatar stays where it was.
+ */
+export const planAvatarTracks = (context) => {
+    const { script, beats: studioBeats } = context.studio;
+    const introPadding = MulmoStudioContextMethods.getIntroPadding(context);
+    const duration = studioBeats.reduce((total, _, index) => total + MulmoStudioContextMethods.getBeatDuration(context, index), 0);
+    const scriptPosition = context.presentationStyle.avatarParams?.position;
+    const tracks = new Map();
+    script.beats.forEach((beat) => {
+        const found = MulmoPresentationStyleMethods.getSpeakerAvatar(context.presentationStyle, beat, context.lang);
+        if (!found || tracks.has(found.speakerId))
+            return;
+        tracks.set(found.speakerId, {
+            speakerId: found.speakerId,
+            avatar: found.avatar,
+            source: getFullPath(context.fileDirs.mulmoFileDirPath, found.avatar.source),
+            segments: [],
+            placements: [],
+            duration,
+            current: mergePosition(scriptPosition, found.avatar.position),
+        });
+    });
+    // each beat's stretch of the video: the first one starts at 0 (its intro padding), the rest at their narration
+    const starts = studioBeats.map((studioBeat, index) => (index === 0 ? 0 : (studioBeat.startAt ?? 0) + introPadding));
+    script.beats.forEach((beat, index) => {
+        const studioBeat = studioBeats[index];
+        const windowEnd = starts[index + 1] ?? duration;
+        const speaker = MulmoPresentationStyleMethods.getSpeakerAvatar(context.presentationStyle, beat, context.lang)?.speakerId;
+        tracks.forEach((track) => {
+            const own = track.speakerId === speaker;
+            if (own)
+                track.current = mergePosition(scriptPosition, track.avatar.position, beat.avatarParams?.position);
+            if (!(own && beat.avatarParams?.hidden))
+                place(track, starts[index], windowEnd, track.current);
+            const text = localizedText(beat, context.multiLingual?.[index], context.lang, script.lang);
+            if (!own || beat.avatarParams?.hidden || !text || !studioBeat?.audioFile)
+                return;
+            track.segments.push({
+                beatIndex: index,
+                text,
+                audio: studioBeat.audioFile,
+                start: (studioBeat.startAt ?? 0) + introPadding,
+                emotion: beat.avatarParams?.emotion,
+                motions: beat.avatarParams?.motions,
+            });
+        });
+    });
+    return [...tracks.values()].map(({ avatar: __avatar, current: __current, ...track }) => track);
+};
+const INSTALL_HINT = "Avatars need the avatarscript and onnxruntime-node packages: npm install avatarscript onnxruntime-node";
+// a variable, so TypeScript does not need onnxruntime-node installed (it is optional, ~290 MB)
+const ONNX_RUNTIME = "onnxruntime-node";
+const loadAvatarScript = () => import("avatarscript").catch((error) => {
+    throw new Error(INSTALL_HINT, { cause: error });
+});
+// AvatarScript times the speech by forced alignment, which runs on onnxruntime-node. Only rendering
+// needs it.
+const ensureOnnxRuntime = () => import(ONNX_RUNTIME).catch((error) => {
+    throw new Error(INSTALL_HINT, { cause: error });
+});
+// Size and modification time: what the cache key sees of a file, so that replacing it (new speech at
+// the same path, an edited avatar) renders the track again.
+const fileStamp = (file) => {
+    const stat = fs.statSync(file, { throwIfNoEntry: false });
+    return stat ? `${stat.size}:${stat.mtimeMs}` : null;
+};
+const folderStamp = (dir) => {
+    const stat = fs.statSync(dir, { throwIfNoEntry: false });
+    if (!stat?.isDirectory())
+        return fileStamp(dir);
+    return fs
+        .readdirSync(dir, { recursive: true, withFileTypes: true })
+        .filter((entry) => entry.isFile())
+        .map((entry) => path.join(entry.parentPath, entry.name))
+        .sort()
+        .map((file) => [path.relative(dir, file), fileStamp(file)]);
+};
+const even = (n) => Math.max(2, Math.round(n / 2) * 2); // video codecs need even sizes
+/** Renders one track (or reuses it), returning where it goes on the canvas. */
+const renderAvatarTrack = async (plan, context, getAvatarScript) => {
+    const canvas = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
+    // rendered once, at the largest size it is shown; smaller placements scale it down
+    const height = even(Math.max(...plan.placements.map((p) => percent(p.position.scale, canvas.height))));
+    const segments = plan.segments.map((segment) => ({ ...segment, audioStamp: fileStamp(segment.audio) }));
+    const identity = { version: 4, segments, source: plan.source, sourceStamp: folderStamp(plan.source), duration: plan.duration, height, padTop: PAD_TOP };
+    const hash = createHash("sha256").update(JSON.stringify(identity)).digest("hex").slice(0, 12);
+    const dir = MulmoStudioContextMethods.getImageProjectDirPath(context);
+    const name = `avatar_${plan.speakerId.replace(/[^\w-]/g, "_")}_${hash}`;
+    const file = path.resolve(dir, `${name}.webm`);
+    // the avatar's width / height, kept next to the track so that reusing it needs no avatarscript
+    const infoFile = path.resolve(dir, `${name}.json`);
+    const aspect = await (async () => {
+        if (!context.force && fs.existsSync(file) && fs.existsSync(infoFile)) {
+            GraphAILogger.info(`avatar: reusing ${file}`);
+            return JSON.parse(fs.readFileSync(infoFile, "utf-8")).aspect;
+        }
+        GraphAILogger.info(`avatar: rendering ${plan.speakerId} (${plan.segments.length} beats, ${plan.duration.toFixed(1)} s)`);
+        const avatarscript = await getAvatarScript();
+        await ensureOnnxRuntime();
+        const avatar = await avatarscript.loadAvatar(plan.source);
+        const avatarAspect = avatarscript.avatarAspect(avatar.rig, PAD_TOP);
+        fs.mkdirSync(dir, { recursive: true });
+        const { score, audio } = await avatarscript.compileTimeline(plan.segments, {
+            lang: context.lang,
+            duration: plan.duration,
+            view: { background: "transparent" },
+        });
+        const voice = path.resolve(dir, `${name}.wav`);
+        fs.writeFileSync(voice, avatarscript.toWav(audio));
+        await avatarscript.render({ avatar, score, audioPath: voice, audio: false, out: file, width: even(height * avatarAspect), height, padTop: PAD_TOP });
+        fs.writeFileSync(infoFile, JSON.stringify({ aspect: avatarAspect }));
+        return avatarAspect;
+    })();
+    const width = even(height * aspect);
+    const placements = plan.placements.map(({ start, end, position }) => {
+        const h = even(percent(position.scale, canvas.height));
+        const w = even(h * aspect);
+        return { start, end, x: Math.round(percent(position.x, canvas.width) - w / 2), y: Math.round(percent(position.y, canvas.height) - h), width: w, height: h };
+    });
+    return { speaker: plan.speakerId, file, width, height, placements };
+};
+/** The `avatar` action: renders the avatar tracks into studio.avatarTracks (none when no speaker has an avatar). */
+export const avatar = async (context) => {
+    const allPlans = planAvatarTracks(context);
+    const plans = allPlans.filter((plan) => plan.segments.length > 0);
+    if (plans.length === 0) {
+        // no stale tracks from an earlier run; [] when there are avatars but nothing for them to say
+        if (allPlans.length > 0)
+            context.studio.avatarTracks = [];
+        else
+            delete context.studio.avatarTracks;
+        return context;
+    }
+    // loaded only when a track has to be rendered: a cached track needs neither avatarscript nor onnxruntime-node
+    const getAvatarScript = (() => {
+        const loaded = {};
+        return () => (loaded.module ??= loadAvatarScript());
+    })();
+    const tracks = [];
+    for (const plan of plans)
+        tracks.push(await renderAvatarTrack(plan, context, getAvatarScript));
+    context.studio.avatarTracks = tracks;
+    const outputStudioFilePath = getOutputStudioFilePath(MulmoStudioContextMethods.getOutDirPath(context), MulmoStudioContextMethods.getFileName(context));
+    fs.writeFileSync(outputStudioFilePath, JSON.stringify(context.studio, null, 2));
+    return context;
+};

package/lib/actions/image_agents.js

@@ -205,12 +205,12 @@ export const imagePreprocessAgent = async (namedInputs) => {
         const markdown = plugin.markdown ? plugin.markdown({ beat, context }) : undefined;
         const html = plugin.html ? await plugin.html({ beat, context }) : undefined;
         const isTypeMovie = beat.image.type === ImageMediaType.Movie;
-        const isAnimatedHtml = MulmoBeatMethods.isAnimatedHtmlTailwind(beat);
-        // animation and moviePrompt cannot be used together
-        if (isAnimatedHtml && beat.moviePrompt) {
-            throw new Error("html_tailwind animation and moviePrompt cannot be used together on the same beat. Use either animation or moviePrompt, not both.");
+        const isPluginVideo = MulmoBeatMethods.isPluginVideo(beat);
+        // the plugin's own video and moviePrompt cannot be used together
+        if (isPluginVideo && beat.moviePrompt) {
+            throw new Error(`${beat.image.type} video and moviePrompt cannot be used together on the same beat. Use one of them, not both.`);
         }
-        if (isAnimatedHtml) {
+        if (isPluginVideo) {
             const animatedVideoPath = getBeatAnimatedVideoPath(context, index);
             // ImagePluginPreprocessAgentResponse
             // imageFromMovie is false: the plugin generates both the .mp4 video AND
@@ -218,7 +218,8 @@ export const imagePreprocessAgent = async (namedInputs) => {
             return {
                 ...returnValue,
                 imagePath, // static final-frame PNG (generated by the plugin)
-                movieFile: animatedVideoPath, // .mp4 path for the pipeline
+                // Without a known duration the plugin writes only the PNG, so there is no .mp4 to point at.
+                movieFile: returnValue.beatDuration !== undefined ? animatedVideoPath : undefined,
                 referenceImageForMovie: pluginPath,
                 markdown,
                 html,
@@ -275,9 +276,8 @@ export const imagePluginAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs, movieRefs } = namedInputs;
     const { imagePath } = getBeatPngImagePath(context, index);
     const plugin = MulmoBeatMethods.getPlugin(beat);
-    // For animated html_tailwind, use the .mp4 path so the plugin writes video there
-    const isAnimatedHtml = MulmoBeatMethods.isAnimatedHtmlTailwind(beat);
-    const effectiveImagePath = isAnimatedHtml ? getBeatAnimatedVideoPath(context, index) : imagePath;
+    // Plugins that write their own video (animated html_tailwind, remotion) get the .mp4 path
+    const effectiveImagePath = MulmoBeatMethods.isPluginVideo(beat) ? getBeatAnimatedVideoPath(context, index) : imagePath;
     try {
         MulmoStudioContextMethods.setBeatSessionState(context, "image", index, beat.id, true);
         const studioBeat = context.studio.beats[index];

package/lib/actions/index.d.ts

@@ -1,6 +1,7 @@
 export * from "./audio.js";
 export * from "./images.js";
 export * from "./captions.js";
+export * from "./avatar.js";
 export * from "./image_references.js";
 export * from "./image_agents.js";
 export * from "./movie.js";

package/lib/actions/index.js

@@ -1,6 +1,7 @@
 export * from "./audio.js";
 export * from "./images.js";
 export * from "./captions.js";
+export * from "./avatar.js";
 export * from "./image_references.js";
 export * from "./image_agents.js";
 export * from "./movie.js";

package/lib/actions/movie.d.ts

@@ -9,6 +9,7 @@ export declare const getAudioPart: (inputIndex: number, duration: number, delay:
     audioId: string;
     audioPart: string;
 };
+export declare const addAvatars: (ffmpegContext: FfmpegContext, videoId: string, context: MulmoStudioContext) => string;
 export declare const getOutOverlayCoords: (transitionType: string, d: number, t: number) => string;
 export declare const getInOverlayCoords: (transitionType: string, d: number, t: number) => string;
 export declare const isExplicitMixMode: (context: MulmoStudioContext) => boolean;

package/lib/actions/movie.js

@@ -105,6 +105,30 @@ const getOutputOption = (audioId, videoId) => {
         "-b:a 128k", // Audio bitrate
     ];
 };
+// Talking avatars (actions/avatar.ts): each track is a see-through video as long as the whole movie,
+// overlaid after transitions so they do not move it. "-c:v libvpx-vp9" is needed to decode the alpha.
+// A track is shown in stretches (placements); each stretch is trimmed out of the track, kept at its
+// time, scaled to its size and placed where it says, so no filter runs outside its stretch.
+export const addAvatars = (ffmpegContext, videoId, context) => {
+    return (context.studio.avatarTracks ?? []).reduce((accId, track, trackIndex) => {
+        if (track.placements.length === 0)
+            return accId;
+        const inputIndex = FfmpegContextAddInput(ffmpegContext, track.file, ["-c:v", "libvpx-vp9"]);
+        const sources = track.placements.length === 1 ? [`${inputIndex}:v`] : track.placements.map((_, index) => `avatar${trackIndex}_src${index}`);
+        const labels = sources.map((id) => "[" + id + "]").join("");
+        if (track.placements.length > 1)
+            ffmpegContext.filterComplex.push(`[${inputIndex}:v]split=${sources.length}${labels}`);
+        return track.placements.reduce((prevId, placement, index) => {
+            const partId = `avatar${trackIndex}_${index}s`;
+            const resized = placement.width !== track.width || placement.height !== track.height;
+            const scale = resized ? `,scale=${placement.width}:${placement.height}` : "";
+            ffmpegContext.filterComplex.push(`[${sources[index]}]trim=start=${placement.start}:end=${placement.end},setpts=PTS-STARTPTS+${placement.start}/TB${scale}[${partId}]`);
+            const outId = `avatar${trackIndex}_${index}`;
+            ffmpegContext.filterComplex.push(`[${prevId}][${partId}]overlay=x=${placement.x}:y=${placement.y}:format=auto:eof_action=pass:enable='gte(t,${placement.start})*lt(t,${placement.end})'[${outId}]`);
+            return outId;
+        }, accId);
+    }, videoId);
+};
 const addCaptions = (ffmpegContext, concatVideoId, context, caption) => {
     const beatsWithCaptions = context.studio.beats.filter(({ captionFiles }) => captionFiles && captionFiles.length > 0);
     if (caption && beatsWithCaptions.length > 0) {
@@ -470,8 +494,13 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
     // Concatenate the trimmed images
     const concatVideoId = "concat_video";
     ffmpegContext.filterComplex.push(getConcatVideoFilter(concatVideoId, videoIdsForBeats));
-    const captionedVideoId = addCaptions(ffmpegContext, concatVideoId, context, caption);
-    const mixedVideoId = addTransitionEffects(ffmpegContext, captionedVideoId, context, transitionVideoIds, beatTimestamps, videoIdsForBeats);
+    // With talking avatars, captions go on last so that an avatar never covers them. Without, the order
+    // is unchanged: captions under the transitions.
+    const hasAvatars = (context.studio.avatarTracks ?? []).some((track) => track.placements.length > 0);
+    const captionedVideoId = hasAvatars ? concatVideoId : addCaptions(ffmpegContext, concatVideoId, context, caption);
+    const transitionedVideoId = addTransitionEffects(ffmpegContext, captionedVideoId, context, transitionVideoIds, beatTimestamps, videoIdsForBeats);
+    const avatarVideoId = addAvatars(ffmpegContext, transitionedVideoId, context);
+    const mixedVideoId = hasAvatars ? addCaptions(ffmpegContext, avatarVideoId, context, caption) : avatarVideoId;
     if (isTest) {
         return ffmpegContext.filterComplex;
     }
@@ -491,8 +520,17 @@ export const movieFilePath = (context) => {
     const caption = MulmoStudioContextMethods.getCaption(context);
     return getOutputVideoFilePath(outDirPath, fileName, context.lang, caption);
 };
+// A speaker has an avatar but no track was rendered: the `avatar` action did not run before `movie`.
+const warnIfAvatarsMissing = (context) => {
+    if (context.studio.avatarTracks)
+        return;
+    const speaksWithAvatar = context.studio.script.beats.some((beat) => MulmoPresentationStyleMethods.getSpeakerAvatar(context.presentationStyle, beat, context.lang));
+    if (speaksWithAvatar)
+        GraphAILogger.warn("movie: a speaker has an avatar, but no avatar track was rendered. Run the avatar action before movie.");
+};
 export const movie = async (context) => {
     MulmoStudioContextMethods.setSessionState(context, "video", true);
+    warnIfAvatarsMissing(context);
     try {
         const audioArtifactFilePath = getAudioArtifactFilePath(context);
         const outputVideoPath = movieFilePath(context);

package/lib/agents/combine_audio_files_agent.js

@@ -14,8 +14,8 @@ const getMovieDuration = async (context, beat) => {
             return { duration: duration / speed, hasAudio };
         }
     }
-    // Animated html_tailwind beats with explicit duration act as movie-like for voice_over grouping
-    if (MulmoBeatMethods.isAnimatedHtmlTailwind(beat) && beat.duration !== undefined) {
+    // Plugin-video beats (animated html_tailwind, remotion) with explicit duration act as movie-like for voice_over grouping
+    if (MulmoBeatMethods.isPluginVideo(beat) && beat.duration !== undefined) {
         return { duration: beat.duration, hasAudio: false };
     }
     return { duration: 0, hasAudio: false };

package/lib/cli/commands/movie/handler.js

@@ -1,4 +1,4 @@
-import { audio, images, movie, captions } from "../../../actions/index.js";
+import { audio, images, movie, captions, avatar } from "../../../actions/index.js";
 import { dumpUsageIfRequested, initializeContext, runTranslateIfNeeded, printUsageEstimate } from "../../helpers.js";
 export const handler = async (argv) => {
     const context = await initializeContext(argv);
@@ -10,6 +10,6 @@ export const handler = async (argv) => {
         return;
     }
     await runTranslateIfNeeded(context, true);
-    await audio(context).then(images).then(captions).then(movie);
+    await audio(context).then(images).then(captions).then(avatar).then(movie);
     dumpUsageIfRequested(context);
 };

package/lib/mcp/server.js

@@ -7,7 +7,7 @@ import fs from "fs";
 import path from "path";
 import { fileURLToPath } from "url";
 import { GraphAILogger } from "graphai";
-import { audio, images, movie, captions, pdf } from "../actions/index.js";
+import { audio, images, movie, captions, avatar, pdf } from "../actions/index.js";
 import { initializeContext, runTranslateIfNeeded } from "../cli/helpers.js";
 import { outDirName } from "../types/const.js";
 import { resolveDirPath, mkdir, generateTimestampedFileName } from "../utils/file.js";
@@ -111,8 +111,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         // Execute the requested command
         switch (cmd) {
             case "movie":
-                // Generate movie (audio + images + captions + movie)
-                await audio(context).then(images).then(captions).then(movie);
+                // Generate movie (audio + images + captions + avatar + movie)
+                await audio(context).then(images).then(captions).then(avatar).then(movie);
                 return {
                     content: [
                         {

package/lib/methods/mulmo_beat.d.ts

@@ -7,6 +7,7 @@ export declare const MulmoBeatMethods: {
     isAnimationEnabled: (animation: unknown) => animation is true | AnimationConfig;
     isAnimationObject: (animation: unknown) => animation is AnimationConfig;
     isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
+    isPluginVideo: (beat: MulmoBeat) => boolean;
     isMovieMode: (animation: unknown) => boolean;
     isVoiceOver: (beat?: MulmoBeat) => boolean;
     getPrevRenderedBeatIndex(beats: MulmoBeat[], index: number): number;

package/lib/methods/mulmo_beat.js

@@ -18,12 +18,15 @@ const isAnimatedHtmlTailwind = (beat) => {
     const animation = beat.image.animation;
     return isAnimationEnabled(animation);
 };
+// Beats whose image plugin writes the beat's .mp4 itself, at the length the audio gives it.
+const isPluginVideo = (beat) => isAnimatedHtmlTailwind(beat) || beat.image?.type === ImageMediaType.Remotion;
 // voice_over beats share the preceding beat's shot, and contribute no video segment of their own.
 const isVoiceOver = (beat) => beat?.image?.type === ImageMediaType.VoiceOver;
 export const MulmoBeatMethods = {
     isAnimationEnabled,
     isAnimationObject,
     isAnimatedHtmlTailwind,
+    isPluginVideo,
     isMovieMode,
     isVoiceOver,
     // Index of the closest preceding beat which is actually rendered as a video segment (-1 if none).

package/lib/methods/mulmo_beat_node.d.ts

@@ -16,6 +16,7 @@ export declare const MulmoBeatMethods: {
         movie?: boolean;
     };
     isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
+    isPluginVideo: (beat: MulmoBeat) => boolean;
     isMovieMode: (animation: unknown) => boolean;
     isVoiceOver: (beat?: MulmoBeat) => boolean;
     getPrevRenderedBeatIndex(beats: MulmoBeat[], index: number): number;

package/lib/methods/mulmo_presentation_style.d.ts

@@ -4,7 +4,7 @@
  * Works in both Node.js and modern browsers.
  */
 import type { SlideTheme } from "@mulmocast/deck";
-import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider, MulmoStudioContext, MulmoTransition, MulmoFillOption } from "../types/index.js";
+import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, MulmoSpeakerAvatar, Text2ImageProvider, MulmoStudioContext, MulmoTransition, MulmoFillOption } from "../types/index.js";
 export declare const MulmoPresentationStyleMethods: {
     getCanvasSize(presentationStyle: MulmoPresentationStyle): MulmoCanvasDimension;
     getAllSpeechProviders(presentationStyle: MulmoPresentationStyle): Set<Text2SpeechProvider>;
@@ -28,6 +28,10 @@ export declare const MulmoPresentationStyleMethods: {
     getResolvedSlideTheme(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SlideTheme;
     getDefaultSpeaker(presentationStyle: MulmoPresentationStyle): string;
     getSpeakerData(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat, lang: string | undefined): SpeakerData;
+    getSpeakerAvatar(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat, lang: string | undefined): {
+        speakerId: string;
+        avatar: MulmoSpeakerAvatar;
+    } | null;
     getSpeaker(context: MulmoStudioContext, beat: MulmoBeat, targetLang: string | undefined): SpeakerData;
     getMovieTransition(context: MulmoStudioContext, beat: MulmoBeat): MulmoTransition | null;
     getNeedFirstFrame(context: MulmoStudioContext): boolean[];

package/lib/methods/mulmo_presentation_style.js

@@ -92,6 +92,14 @@ export const MulmoPresentationStyleMethods = {
         }
         return speaker;
     },
+    // The speaker of a beat and the avatar it speaks with, if any. A language-specific speaker
+    // replaces the whole speaker, so its avatar falls back to the base speaker's.
+    getSpeakerAvatar(presentationStyle, beat, lang) {
+        const speakerId = beat?.speaker ?? MulmoPresentationStyleMethods.getDefaultSpeaker(presentationStyle);
+        const speaker = presentationStyle.speechParams?.speakers?.[speakerId];
+        const avatar = (lang ? speaker?.lang?.[lang]?.avatar : undefined) ?? speaker?.avatar;
+        return avatar ? { speakerId, avatar } : null;
+    },
     getSpeaker(context, beat, targetLang) {
         // Normally, lang is determined by the context, but lang may be specified when using the API.
         const lang = targetLang ?? context.lang ?? context?.studio?.script?.lang;