mulmocast 2.12.1 → 2.14.0

package/assets/schemas/mulmo_script.json

@@ -119,6 +119,41 @@
               "apiVersion": {
                 "type": "string"
               },
+              "avatar": {
+                "description": "Avatar shown over the beat visuals, speaking this speaker's lines",
+                "type": "object",
+                "properties": {
+                  "source": {
+                    "type": "string",
+                    "description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project), relative to the script"
+                  },
+                  "position": {
+                    "type": "object",
+                    "properties": {
+                      "x": {
+                        "description": "Horizontal centre of the avatar, % of the canvas width. Default 84%",
+                        "type": "string",
+                        "pattern": "^-?\\d+(\\.\\d+)?%$"
+                      },
+                      "y": {
+                        "description": "Bottom edge of the avatar, % of the canvas height. Default 100%",
+                        "type": "string",
+                        "pattern": "^-?\\d+(\\.\\d+)?%$"
+                      },
+                      "scale": {
+                        "description": "Height of the avatar, % of the canvas height. Default 62%",
+                        "type": "string",
+                        "pattern": "^-?\\d+(\\.\\d+)?%$"
+                      }
+                    },
+                    "additionalProperties": false
+                  }
+                },
+                "required": [
+                  "source"
+                ],
+                "additionalProperties": false
+              },
               "lang": {
                 "type": "object",
                 "propertyNames": {
@@ -184,6 +219,41 @@
                     },
                     "apiVersion": {
                       "type": "string"
+                    },
+                    "avatar": {
+                      "description": "Avatar shown over the beat visuals, speaking this speaker's lines",
+                      "type": "object",
+                      "properties": {
+                        "source": {
+                          "type": "string",
+                          "description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project), relative to the script"
+                        },
+                        "position": {
+                          "type": "object",
+                          "properties": {
+                            "x": {
+                              "description": "Horizontal centre of the avatar, % of the canvas width. Default 84%",
+                              "type": "string",
+                              "pattern": "^-?\\d+(\\.\\d+)?%$"
+                            },
+                            "y": {
+                              "description": "Bottom edge of the avatar, % of the canvas height. Default 100%",
+                              "type": "string",
+                              "pattern": "^-?\\d+(\\.\\d+)?%$"
+                            },
+                            "scale": {
+                              "description": "Height of the avatar, % of the canvas height. Default 62%",
+                              "type": "string",
+                              "pattern": "^-?\\d+(\\.\\d+)?%$"
+                            }
+                          },
+                          "additionalProperties": false
+                        }
+                      },
+                      "required": [
+                        "source"
+                      ],
+                      "additionalProperties": false
                     }
                   },
                   "required": [
@@ -1860,6 +1930,69 @@
       },
       "additionalProperties": false
     },
+    "avatarParams": {
+      "type": "object",
+      "properties": {
+        "position": {
+          "type": "object",
+          "properties": {
+            "x": {
+              "description": "Horizontal centre of the avatar, % of the canvas width. Default 84%",
+              "type": "string",
+              "pattern": "^-?\\d+(\\.\\d+)?%$"
+            },
+            "y": {
+              "description": "Bottom edge of the avatar, % of the canvas height. Default 100%",
+              "type": "string",
+              "pattern": "^-?\\d+(\\.\\d+)?%$"
+            },
+            "scale": {
+              "description": "Height of the avatar, % of the canvas height. Default 62%",
+              "type": "string",
+              "pattern": "^-?\\d+(\\.\\d+)?%$"
+            }
+          },
+          "additionalProperties": false
+        },
+        "emotion": {
+          "description": "The avatar's expression during this beat",
+          "type": "string",
+          "enum": [
+            "neutral",
+            "happy",
+            "sad",
+            "angry",
+            "surprised",
+            "relaxed"
+          ]
+        },
+        "motions": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "motion": {
+                "type": "string",
+                "description": "Motion name, e.g. nod, tilt, think, giggle, surprise"
+              },
+              "at": {
+                "description": "Words of the beat's text where the motion starts. Default: the start of the beat",
+                "type": "string"
+              }
+            },
+            "required": [
+              "motion"
+            ],
+            "additionalProperties": false
+          }
+        },
+        "hidden": {
+          "description": "Hide the speaker's avatar during this beat",
+          "type": "boolean"
+        }
+      },
+      "additionalProperties": false
+    },
     "htmlImageParams": {
       "type": "object",
       "properties": {
@@ -2207,6 +2340,16 @@
       ],
       "additionalProperties": false
     },
+    "remotionParams": {
+      "type": "object",
+      "properties": {
+        "brief": {
+          "description": "Whole-video art direction shared by every remotion beat (palette, type, mood), so all scenes look like one film.",
+          "type": "string"
+        }
+      },
+      "additionalProperties": false
+    },
     "captionParams": {
       "type": "object",
       "properties": {
@@ -13372,6 +13515,97 @@
                   "slide"
                 ],
                 "additionalProperties": false
+              },
+              {
+                "anyOf": [
+                  {
+                    "type": "object",
+                    "properties": {
+                      "type": {
+                        "type": "string",
+                        "const": "remotion"
+                      },
+                      "fps": {
+                        "description": "Frame rate of the rendered video. Default: 30.",
+                        "type": "number",
+                        "minimum": 1,
+                        "maximum": 60
+                      },
+                      "prompt": {
+                        "type": "string",
+                        "minLength": 1,
+                        "description": "What the scene shows. Claude Code (claude -p) writes a Remotion component from it."
+                      }
+                    },
+                    "required": [
+                      "type",
+                      "prompt"
+                    ],
+                    "additionalProperties": false
+                  },
+                  {
+                    "type": "object",
+                    "properties": {
+                      "type": {
+                        "type": "string",
+                        "const": "remotion"
+                      },
+                      "fps": {
+                        "description": "Frame rate of the rendered video. Default: 30.",
+                        "type": "number",
+                        "minimum": 1,
+                        "maximum": 60
+                      },
+                      "code": {
+                        "oneOf": [
+                          {
+                            "type": "object",
+                            "properties": {
+                              "kind": {
+                                "type": "string",
+                                "const": "text"
+                              },
+                              "text": {
+                                "type": "string",
+                                "minLength": 1
+                              }
+                            },
+                            "required": [
+                              "kind",
+                              "text"
+                            ],
+                            "additionalProperties": false
+                          },
+                          {
+                            "type": "object",
+                            "properties": {
+                              "kind": {
+                                "type": "string",
+                                "const": "path"
+                              },
+                              "path": {
+                                "type": "string",
+                                "minLength": 1,
+                                "description": "Relative to the script"
+                              }
+                            },
+                            "required": [
+                              "kind",
+                              "path"
+                            ],
+                            "additionalProperties": false
+                          }
+                        ],
+                        "description": "A finished Remotion component (TSX that default-exports the scene). It is rendered as is, without claude -p."
+                      }
+                    },
+                    "required": [
+                      "type",
+                      "code"
+                    ],
+                    "additionalProperties": false
+                  }
+                ]
               }
             ]
           },
@@ -14686,6 +14920,69 @@
             },
             "additionalProperties": false
           },
+          "avatarParams": {
+            "type": "object",
+            "properties": {
+              "position": {
+                "type": "object",
+                "properties": {
+                  "x": {
+                    "description": "Horizontal centre of the avatar, % of the canvas width. Default 84%",
+                    "type": "string",
+                    "pattern": "^-?\\d+(\\.\\d+)?%$"
+                  },
+                  "y": {
+                    "description": "Bottom edge of the avatar, % of the canvas height. Default 100%",
+                    "type": "string",
+                    "pattern": "^-?\\d+(\\.\\d+)?%$"
+                  },
+                  "scale": {
+                    "description": "Height of the avatar, % of the canvas height. Default 62%",
+                    "type": "string",
+                    "pattern": "^-?\\d+(\\.\\d+)?%$"
+                  }
+                },
+                "additionalProperties": false
+              },
+              "emotion": {
+                "description": "The avatar's expression during this beat",
+                "type": "string",
+                "enum": [
+                  "neutral",
+                  "happy",
+                  "sad",
+                  "angry",
+                  "surprised",
+                  "relaxed"
+                ]
+              },
+              "motions": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "motion": {
+                      "type": "string",
+                      "description": "Motion name, e.g. nod, tilt, think, giggle, surprise"
+                    },
+                    "at": {
+                      "description": "Words of the beat's text where the motion starts. Default: the start of the beat",
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "motion"
+                  ],
+                  "additionalProperties": false
+                }
+              },
+              "hidden": {
+                "description": "Hide the speaker's avatar during this beat",
+                "type": "boolean"
+              }
+            },
+            "additionalProperties": false
+          },
           "htmlImageParams": {
             "type": "object",
             "properties": {

package/lib/actions/avatar.d.ts

@@ -0,0 +1,37 @@
+import type { MulmoAvatarPosition, MulmoStudioContext } from "../types/index.js";
+export type AvatarSegmentPlan = {
+    beatIndex: number;
+    text: string;
+    audio: string;
+    /** absolute seconds on the final timeline */
+    start: number;
+    emotion?: "neutral" | "happy" | "sad" | "angry" | "surprised" | "relaxed";
+    motions?: {
+        motion: string;
+        at?: string;
+    }[];
+};
+/** Where the avatar is during one stretch of the video (absolute seconds, end exclusive). */
+export type AvatarPlacementPlan = {
+    start: number;
+    end: number;
+    position: Required<MulmoAvatarPosition>;
+};
+export type AvatarTrackPlan = {
+    speakerId: string;
+    /** absolute path of the avatar package */
+    source: string;
+    segments: AvatarSegmentPlan[];
+    /** where the avatar is shown, beat by beat; it is not shown outside these */
+    placements: AvatarPlacementPlan[];
+    /** length of the whole video, seconds */
+    duration: number;
+};
+/**
+ * Which beats each avatar speaks, and where it stands, beat by beat. Pure: reads the studio,
+ * writes nothing. Position: the beat's avatarParams, then the speaker's avatar, then the script's
+ * avatarParams, then the defaults; during other speakers' beats the avatar stays where it was.
+ */
+export declare const planAvatarTracks: (context: MulmoStudioContext) => AvatarTrackPlan[];
+/** The `avatar` action: renders the avatar tracks into studio.avatarTracks (none when no speaker has an avatar). */
+export declare const avatar: (context: MulmoStudioContext) => Promise<MulmoStudioContext>;

package/lib/actions/avatar.js

@@ -0,0 +1,176 @@
+// Talking avatars (AvatarScript): for each speaker that has an avatar, one continuous, see-through
+// avatar track for the whole video, speaking that speaker's beats. movie.ts overlays the tracks.
+// See docs/plans/plan_avatar.md.
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { GraphAILogger } from "graphai";
+import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
+import { getFullPath, getOutputStudioFilePath } from "../utils/file.js";
+import { localizedText } from "../utils/utils.js";
+const DEFAULT_POSITION = { x: "84%", y: "100%", scale: "62%" };
+// Show the whole avatar image. A rig may crop the top of its image (a flat cut edge, hidden when the
+// avatar fills the frame); over a slide the cut would run through the head.
+const PAD_TOP = 0;
+const mergePosition = (...positions) => positions.reduce((acc, p) => ({ ...acc, ...Object.fromEntries(Object.entries(p ?? {}).filter(([, v]) => v !== undefined)) }), {
+    ...DEFAULT_POSITION,
+});
+const percent = (value, of) => (of * parseFloat(value)) / 100;
+const samePosition = (a, b) => a.x === b.x && a.y === b.y && a.scale === b.scale;
+/** Adds [start, end) at `position`, joining it to the previous stretch when nothing changes. */
+const place = (track, start, end, position) => {
+    const last = track.placements.at(-1);
+    if (last && last.end === start && samePosition(last.position, position))
+        last.end = end;
+    else
+        track.placements.push({ start, end, position });
+};
+/**
+ * Which beats each avatar speaks, and where it stands, beat by beat. Pure: reads the studio,
+ * writes nothing. Position: the beat's avatarParams, then the speaker's avatar, then the script's
+ * avatarParams, then the defaults; during other speakers' beats the avatar stays where it was.
+ */
+export const planAvatarTracks = (context) => {
+    const { script, beats: studioBeats } = context.studio;
+    const introPadding = MulmoStudioContextMethods.getIntroPadding(context);
+    const duration = studioBeats.reduce((total, _, index) => total + MulmoStudioContextMethods.getBeatDuration(context, index), 0);
+    const scriptPosition = context.presentationStyle.avatarParams?.position;
+    const tracks = new Map();
+    script.beats.forEach((beat) => {
+        const found = MulmoPresentationStyleMethods.getSpeakerAvatar(context.presentationStyle, beat, context.lang);
+        if (!found || tracks.has(found.speakerId))
+            return;
+        tracks.set(found.speakerId, {
+            speakerId: found.speakerId,
+            avatar: found.avatar,
+            source: getFullPath(context.fileDirs.mulmoFileDirPath, found.avatar.source),
+            segments: [],
+            placements: [],
+            duration,
+            current: mergePosition(scriptPosition, found.avatar.position),
+        });
+    });
+    // each beat's stretch of the video: the first one starts at 0 (its intro padding), the rest at their narration
+    const starts = studioBeats.map((studioBeat, index) => (index === 0 ? 0 : (studioBeat.startAt ?? 0) + introPadding));
+    script.beats.forEach((beat, index) => {
+        const studioBeat = studioBeats[index];
+        const windowEnd = starts[index + 1] ?? duration;
+        const speaker = MulmoPresentationStyleMethods.getSpeakerAvatar(context.presentationStyle, beat, context.lang)?.speakerId;
+        tracks.forEach((track) => {
+            const own = track.speakerId === speaker;
+            if (own)
+                track.current = mergePosition(scriptPosition, track.avatar.position, beat.avatarParams?.position);
+            if (!(own && beat.avatarParams?.hidden))
+                place(track, starts[index], windowEnd, track.current);
+            const text = localizedText(beat, context.multiLingual?.[index], context.lang, script.lang);
+            if (!own || beat.avatarParams?.hidden || !text || !studioBeat?.audioFile)
+                return;
+            track.segments.push({
+                beatIndex: index,
+                text,
+                audio: studioBeat.audioFile,
+                start: (studioBeat.startAt ?? 0) + introPadding,
+                emotion: beat.avatarParams?.emotion,
+                motions: beat.avatarParams?.motions,
+            });
+        });
+    });
+    return [...tracks.values()].map(({ avatar: __avatar, current: __current, ...track }) => track);
+};
+const INSTALL_HINT = "Avatars need the avatarscript and onnxruntime-node packages: npm install avatarscript onnxruntime-node";
+// a variable, so TypeScript does not need onnxruntime-node installed (it is optional, ~290 MB)
+const ONNX_RUNTIME = "onnxruntime-node";
+const loadAvatarScript = () => import("avatarscript").catch((error) => {
+    throw new Error(INSTALL_HINT, { cause: error });
+});
+// AvatarScript times the speech by forced alignment, which runs on onnxruntime-node. Only rendering
+// needs it.
+const ensureOnnxRuntime = () => import(ONNX_RUNTIME).catch((error) => {
+    throw new Error(INSTALL_HINT, { cause: error });
+});
+// Size and modification time: what the cache key sees of a file, so that replacing it (new speech at
+// the same path, an edited avatar) renders the track again.
+const fileStamp = (file) => {
+    const stat = fs.statSync(file, { throwIfNoEntry: false });
+    return stat ? `${stat.size}:${stat.mtimeMs}` : null;
+};
+const folderStamp = (dir) => {
+    const stat = fs.statSync(dir, { throwIfNoEntry: false });
+    if (!stat?.isDirectory())
+        return fileStamp(dir);
+    return fs
+        .readdirSync(dir, { recursive: true, withFileTypes: true })
+        .filter((entry) => entry.isFile())
+        .map((entry) => path.join(entry.parentPath, entry.name))
+        .sort()
+        .map((file) => [path.relative(dir, file), fileStamp(file)]);
+};
+const even = (n) => Math.max(2, Math.round(n / 2) * 2); // video codecs need even sizes
+/** Renders one track (or reuses it), returning where it goes on the canvas. */
+const renderAvatarTrack = async (plan, context, getAvatarScript) => {
+    const canvas = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
+    // rendered once, at the largest size it is shown; smaller placements scale it down
+    const height = even(Math.max(...plan.placements.map((p) => percent(p.position.scale, canvas.height))));
+    const segments = plan.segments.map((segment) => ({ ...segment, audioStamp: fileStamp(segment.audio) }));
+    const identity = { version: 4, segments, source: plan.source, sourceStamp: folderStamp(plan.source), duration: plan.duration, height, padTop: PAD_TOP };
+    const hash = createHash("sha256").update(JSON.stringify(identity)).digest("hex").slice(0, 12);
+    const dir = MulmoStudioContextMethods.getImageProjectDirPath(context);
+    const name = `avatar_${plan.speakerId.replace(/[^\w-]/g, "_")}_${hash}`;
+    const file = path.resolve(dir, `${name}.webm`);
+    // the avatar's width / height, kept next to the track so that reusing it needs no avatarscript
+    const infoFile = path.resolve(dir, `${name}.json`);
+    const aspect = await (async () => {
+        if (!context.force && fs.existsSync(file) && fs.existsSync(infoFile)) {
+            GraphAILogger.info(`avatar: reusing ${file}`);
+            return JSON.parse(fs.readFileSync(infoFile, "utf-8")).aspect;
+        }
+        GraphAILogger.info(`avatar: rendering ${plan.speakerId} (${plan.segments.length} beats, ${plan.duration.toFixed(1)} s)`);
+        const avatarscript = await getAvatarScript();
+        await ensureOnnxRuntime();
+        const avatar = await avatarscript.loadAvatar(plan.source);
+        const avatarAspect = avatarscript.avatarAspect(avatar.rig, PAD_TOP);
+        fs.mkdirSync(dir, { recursive: true });
+        const { score, audio } = await avatarscript.compileTimeline(plan.segments, {
+            lang: context.lang,
+            duration: plan.duration,
+            view: { background: "transparent" },
+        });
+        const voice = path.resolve(dir, `${name}.wav`);
+        fs.writeFileSync(voice, avatarscript.toWav(audio));
+        await avatarscript.render({ avatar, score, audioPath: voice, audio: false, out: file, width: even(height * avatarAspect), height, padTop: PAD_TOP });
+        fs.writeFileSync(infoFile, JSON.stringify({ aspect: avatarAspect }));
+        return avatarAspect;
+    })();
+    const width = even(height * aspect);
+    const placements = plan.placements.map(({ start, end, position }) => {
+        const h = even(percent(position.scale, canvas.height));
+        const w = even(h * aspect);
+        return { start, end, x: Math.round(percent(position.x, canvas.width) - w / 2), y: Math.round(percent(position.y, canvas.height) - h), width: w, height: h };
+    });
+    return { speaker: plan.speakerId, file, width, height, placements };
+};
+/** The `avatar` action: renders the avatar tracks into studio.avatarTracks (none when no speaker has an avatar). */
+export const avatar = async (context) => {
+    const allPlans = planAvatarTracks(context);
+    const plans = allPlans.filter((plan) => plan.segments.length > 0);
+    if (plans.length === 0) {
+        // no stale tracks from an earlier run; [] when there are avatars but nothing for them to say
+        if (allPlans.length > 0)
+            context.studio.avatarTracks = [];
+        else
+            delete context.studio.avatarTracks;
+        return context;
+    }
+    // loaded only when a track has to be rendered: a cached track needs neither avatarscript nor onnxruntime-node
+    const getAvatarScript = (() => {
+        const loaded = {};
+        return () => (loaded.module ??= loadAvatarScript());
+    })();
+    const tracks = [];
+    for (const plan of plans)
+        tracks.push(await renderAvatarTrack(plan, context, getAvatarScript));
+    context.studio.avatarTracks = tracks;
+    const outputStudioFilePath = getOutputStudioFilePath(MulmoStudioContextMethods.getOutDirPath(context), MulmoStudioContextMethods.getFileName(context));
+    fs.writeFileSync(outputStudioFilePath, JSON.stringify(context.studio, null, 2));
+    return context;
+};

package/lib/actions/image_agents.js

@@ -205,12 +205,12 @@ export const imagePreprocessAgent = async (namedInputs) => {
         const markdown = plugin.markdown ? plugin.markdown({ beat, context }) : undefined;
         const html = plugin.html ? await plugin.html({ beat, context }) : undefined;
         const isTypeMovie = beat.image.type === ImageMediaType.Movie;
-        const isAnimatedHtml = MulmoBeatMethods.isAnimatedHtmlTailwind(beat);
-        // animation and moviePrompt cannot be used together
-        if (isAnimatedHtml && beat.moviePrompt) {
-            throw new Error("html_tailwind animation and moviePrompt cannot be used together on the same beat. Use either animation or moviePrompt, not both.");
+        const isPluginVideo = MulmoBeatMethods.isPluginVideo(beat);
+        // the plugin's own video and moviePrompt cannot be used together
+        if (isPluginVideo && beat.moviePrompt) {
+            throw new Error(`${beat.image.type} video and moviePrompt cannot be used together on the same beat. Use one of them, not both.`);
         }
-        if (isAnimatedHtml) {
+        if (isPluginVideo) {
             const animatedVideoPath = getBeatAnimatedVideoPath(context, index);
             // ImagePluginPreprocessAgentResponse
             // imageFromMovie is false: the plugin generates both the .mp4 video AND
@@ -218,7 +218,8 @@ export const imagePreprocessAgent = async (namedInputs) => {
             return {
                 ...returnValue,
                 imagePath, // static final-frame PNG (generated by the plugin)
-                movieFile: animatedVideoPath, // .mp4 path for the pipeline
+                // Without a known duration the plugin writes only the PNG, so there is no .mp4 to point at.
+                movieFile: returnValue.beatDuration !== undefined ? animatedVideoPath : undefined,
                 referenceImageForMovie: pluginPath,
                 markdown,
                 html,
@@ -275,9 +276,8 @@ export const imagePluginAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs, movieRefs } = namedInputs;
     const { imagePath } = getBeatPngImagePath(context, index);
     const plugin = MulmoBeatMethods.getPlugin(beat);
-    // For animated html_tailwind, use the .mp4 path so the plugin writes video there
-    const isAnimatedHtml = MulmoBeatMethods.isAnimatedHtmlTailwind(beat);
-    const effectiveImagePath = isAnimatedHtml ? getBeatAnimatedVideoPath(context, index) : imagePath;
+    // Plugins that write their own video (animated html_tailwind, remotion) get the .mp4 path
+    const effectiveImagePath = MulmoBeatMethods.isPluginVideo(beat) ? getBeatAnimatedVideoPath(context, index) : imagePath;
     try {
         MulmoStudioContextMethods.setBeatSessionState(context, "image", index, beat.id, true);
         const studioBeat = context.studio.beats[index];

package/lib/actions/index.d.ts

@@ -1,6 +1,7 @@
 export * from "./audio.js";
 export * from "./images.js";
 export * from "./captions.js";
+export * from "./avatar.js";
 export * from "./image_references.js";
 export * from "./image_agents.js";
 export * from "./movie.js";

package/lib/actions/index.js

@@ -1,6 +1,7 @@
 export * from "./audio.js";
 export * from "./images.js";
 export * from "./captions.js";
+export * from "./avatar.js";
 export * from "./image_references.js";
 export * from "./image_agents.js";
 export * from "./movie.js";

package/lib/actions/movie.d.ts

@@ -9,6 +9,7 @@ export declare const getAudioPart: (inputIndex: number, duration: number, delay:
     audioId: string;
     audioPart: string;
 };
+export declare const addAvatars: (ffmpegContext: FfmpegContext, videoId: string, context: MulmoStudioContext) => string;
 export declare const getOutOverlayCoords: (transitionType: string, d: number, t: number) => string;
 export declare const getInOverlayCoords: (transitionType: string, d: number, t: number) => string;
 export declare const isExplicitMixMode: (context: MulmoStudioContext) => boolean;