mulmocast 2.12.0 → 2.13.0

package/lib/types/schema.js

@@ -34,6 +34,37 @@ export const speechOptionsSchema = z
 const speakerIdSchema = z.string();
 export const defaultSpeaker = "Presenter";
 export const text2SpeechProviderSchema = z.enum(Object.keys(provider2TTSAgent)).default(defaultProviders.tts);
+// Talking avatars (AvatarScript): a speaker can speak with an avatar shown over the beat visuals.
+const avatarPercentSchema = z.string().regex(/^-?\d+(\.\d+)?%$/, 'a percentage such as "84%"');
+export const avatarPositionSchema = z
+    .object({
+    x: avatarPercentSchema.optional().describe("Horizontal centre of the avatar, % of the canvas width. Default 84%"),
+    y: avatarPercentSchema.optional().describe("Bottom edge of the avatar, % of the canvas height. Default 100%"),
+    scale: avatarPercentSchema.optional().describe("Height of the avatar, % of the canvas height. Default 62%"),
+})
+    .strict();
+export const avatarEmotionSchema = z.enum(["neutral", "happy", "sad", "angry", "surprised", "relaxed"]);
+export const speakerAvatarSchema = z
+    .object({
+    source: z.string().describe("AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project), relative to the script"),
+    position: avatarPositionSchema.optional(),
+})
+    .strict();
+export const mulmoAvatarParamsSchema = z
+    .object({
+    position: avatarPositionSchema.optional(),
+    emotion: avatarEmotionSchema.optional().describe("The avatar's expression during this beat"),
+    motions: z
+        .array(z
+        .object({
+        motion: z.string().describe("Motion name, e.g. nod, tilt, think, giggle, surprise"),
+        at: z.string().optional().describe("Words of the beat's text where the motion starts. Default: the start of the beat"),
+    })
+        .strict())
+        .optional(),
+    hidden: z.boolean().optional().describe("Hide the speaker's avatar during this beat"),
+})
+    .strict();
 export const speakerDataSchema = z
     .object({
     displayName: z.record(langSchema, z.string()).optional(),
@@ -44,6 +75,7 @@ export const speakerDataSchema = z
     model: z.string().optional().describe("TTS model to use for this speaker"),
     baseURL: z.string().optional(), // Azure/custom endpoint URL
     apiVersion: z.string().optional(), // Azure API version (e.g., "2025-04-01-preview")
+    avatar: speakerAvatarSchema.optional().describe("Avatar shown over the beat visuals, speaking this speaker's lines"),
 })
     .strict();
 export const speakerSchema = speakerDataSchema.extend({
@@ -139,6 +171,7 @@ export const ImageMediaType = {
     MoviePrompt: "moviePrompt",
     Beat: "beat",
     VoiceOver: "voice_over",
+    Remotion: "remotion",
 };
 // Beat image types whose still derives from the movie or another beat — they cannot supply
 // their own still for the $beatImage frame sentinel.
@@ -326,6 +359,18 @@ export const mulmoVisionMediaSchema = z
     data: z.record(z.string(), z.any()),
 })
     .strict();
+export const mulmoRemotionMediaSchema = z
+    .object({
+    type: z.literal(ImageMediaType.Remotion),
+    prompt: z.string().min(1).describe("What the scene shows. Claude Code (claude -p) writes a Remotion component from it."),
+    fps: z.number().min(1).max(60).optional().describe("Frame rate of the rendered video. Default: 30."),
+})
+    .strict();
+export const mulmoRemotionParamsSchema = z
+    .object({
+    brief: z.string().optional().describe("Whole-video art direction shared by every remotion beat (palette, type, mood), so all scenes look like one film."),
+})
+    .strict();
 export const mulmoImageAssetSchema = z.union([
     mulmoMarkdownMediaSchema,
     mulmoWebMediaSchema,
@@ -341,6 +386,7 @@ export const mulmoImageAssetSchema = z.union([
     mulmoVoiceOverMediaSchema,
     mulmoVisionMediaSchema,
     mulmoSlideMediaSchema,
+    mulmoRemotionMediaSchema,
 ]);
 const mulmoAudioMediaSchema = z
     .object({
@@ -567,6 +613,7 @@ export const mulmoBeatSchema = z
         .optional(),
     soundEffectParams: mulmoSoundEffectParamsSchema.optional(),
     lipSyncParams: mulmoLipSyncParamsSchema.optional(),
+    avatarParams: mulmoAvatarParamsSchema.optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema.optional(),
     speechOptions: speechOptionsSchema.optional(),
     textSlideParams: textSlideParamsSchema.optional(),
@@ -626,6 +673,7 @@ export const mulmoPresentationStyleSchema = z.object({
         provider: defaultProviders.soundEffect,
     }),
     lipSyncParams: mulmoLipSyncParamsSchema.optional(),
+    avatarParams: mulmoAvatarParamsSchema.optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema
         .extend({
         provider: text2HtmlImageProviderSchema,
@@ -635,6 +683,8 @@ export const mulmoPresentationStyleSchema = z.object({
     textSlideParams: textSlideParamsSchema.optional(),
     // for slide plugin
     slideParams: mulmoSlideParamsSchema.optional(),
+    // for remotion plugin
+    remotionParams: mulmoRemotionParamsSchema.optional(),
     captionParams: mulmoCaptionParamsSchema.optional(),
     audioParams: audioParamsSchema.default({
         introPadding: 1.0,
@@ -730,11 +780,22 @@ export const mulmoSessionStateSchema = z.object({
         lipSync: z.record(z.string(), z.boolean()),
     }),
 });
+export const mulmoAvatarTrackSchema = z
+    .object({
+    speaker: z.string(),
+    file: z.string(), // see-through, avatar-sized video (WebM VP9 with alpha)
+    width: z.number(), // the size it was rendered at, pixels
+    height: z.number(),
+    // where it is shown: absolute [start, end) seconds, and its place and size on the canvas in pixels
+    placements: z.array(z.object({ start: z.number(), end: z.number(), x: z.number(), y: z.number(), width: z.number(), height: z.number() }).strict()),
+})
+    .strict();
 export const mulmoStudioSchema = z
     .object({
     script: mulmoScriptSchema,
     filename: z.string(),
     beats: z.array(mulmoStudioBeatSchema).min(1),
+    avatarTracks: z.array(mulmoAvatarTrackSchema).optional(),
 })
     .strict();
 export const mulmoPromptTemplateSchema = z

package/lib/types/type.d.ts

@@ -1,6 +1,6 @@
 import { type CallbackFunction } from "graphai";
 import type { UsageCollectorAPI } from "./usage.js";
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoImageReferenceSchema, mulmoMovieMediaSchema, mulmoMoviePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoRemotionMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoImageReferenceSchema, mulmoMovieMediaSchema, mulmoMoviePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema, mulmoAvatarParamsSchema, avatarPositionSchema, speakerAvatarSchema, mulmoAvatarTrackSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "./const.js";
 import type { LLM } from "./provider2agent.js";
 import { z } from "zod";
@@ -44,6 +44,10 @@ export type MultiLingualTexts = z.infer<typeof multiLingualTextsSchema>;
 export type MulmoMovieParams = z.infer<typeof mulmoMovieParamsSchema>;
 export type MulmoSoundEffectParams = z.infer<typeof mulmoSoundEffectParamsSchema>;
 export type MulmoLipSyncParams = z.infer<typeof mulmoLipSyncParamsSchema>;
+export type MulmoAvatarParams = z.infer<typeof mulmoAvatarParamsSchema>;
+export type MulmoAvatarPosition = z.infer<typeof avatarPositionSchema>;
+export type MulmoSpeakerAvatar = z.infer<typeof speakerAvatarSchema>;
+export type MulmoAvatarTrack = z.infer<typeof mulmoAvatarTrackSchema>;
 export type MulmoOpenAIImageModel = z.infer<typeof mulmoOpenAIImageModelSchema>;
 export type MulmoGoogleImageModel = z.infer<typeof mulmoGoogleImageModelSchema>;
 export type MulmoGoogleMovieModel = z.infer<typeof mulmoGoogleMovieModelSchema>;
@@ -61,6 +65,7 @@ export type MulmoImageMedia = z.infer<typeof mulmoImageMediaSchema>;
 export type MulmoMovieMedia = z.infer<typeof mulmoMovieMediaSchema>;
 export type MulmoChartMedia = z.infer<typeof mulmoChartMediaSchema>;
 export type MulmoMermaidMedia = z.infer<typeof mulmoMermaidMediaSchema>;
+export type MulmoRemotionMedia = z.infer<typeof mulmoRemotionMediaSchema>;
 export type MulmoSessionState = z.infer<typeof mulmoSessionStateSchema>;
 export type MulmoStudioContext = {
     fileDirs: FileObject;

package/lib/utils/context.d.ts

@@ -53,6 +53,14 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                 model?: string | undefined;
                 baseURL?: string | undefined;
                 apiVersion?: string | undefined;
+                avatar?: {
+                    source: string;
+                    position?: {
+                        x?: string | undefined;
+                        y?: string | undefined;
+                        scale?: string | undefined;
+                    } | undefined;
+                } | undefined;
                 lang?: Record<string, {
                     voiceId: string;
                     displayName?: Record<string, string> | undefined;
@@ -68,6 +76,14 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                     model?: string | undefined;
                     baseURL?: string | undefined;
                     apiVersion?: string | undefined;
+                    avatar?: {
+                        source: string;
+                        position?: {
+                            x?: string | undefined;
+                            y?: string | undefined;
+                            scale?: string | undefined;
+                        } | undefined;
+                    } | undefined;
                 }> | undefined;
             }>;
         };
@@ -532,6 +548,10 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                 type: "vision";
                 style: string;
                 data: Record<string, any>;
+            } | {
+                type: "remotion";
+                prompt: string;
+                fps?: number | undefined;
             } | {
                 type: "slide";
                 slide: {
@@ -2371,6 +2391,19 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                 provider?: string | undefined;
                 model?: string | undefined;
             } | undefined;
+            avatarParams?: {
+                position?: {
+                    x?: string | undefined;
+                    y?: string | undefined;
+                    scale?: string | undefined;
+                } | undefined;
+                emotion?: "neutral" | "happy" | "sad" | "angry" | "surprised" | "relaxed" | undefined;
+                motions?: {
+                    motion: string;
+                    at?: string | undefined;
+                }[] | undefined;
+                hidden?: boolean | undefined;
+            } | undefined;
             htmlImageParams?: {
                 model?: string | undefined;
             } | undefined;
@@ -2474,6 +2507,19 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
             provider?: string | undefined;
             model?: string | undefined;
         } | undefined;
+        avatarParams?: {
+            position?: {
+                x?: string | undefined;
+                y?: string | undefined;
+                scale?: string | undefined;
+            } | undefined;
+            emotion?: "neutral" | "happy" | "sad" | "angry" | "surprised" | "relaxed" | undefined;
+            motions?: {
+                motion: string;
+                at?: string | undefined;
+            }[] | undefined;
+            hidden?: boolean | undefined;
+        } | undefined;
         htmlImageParams?: {
             provider: string;
             model?: string | undefined;
@@ -2540,6 +2586,9 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                 } | undefined;
             } | undefined;
         } | undefined;
+        remotionParams?: {
+            brief?: string | undefined;
+        } | undefined;
         captionParams?: {
             lang?: string | undefined;
             styles?: string[] | undefined;
@@ -2588,6 +2637,20 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
         markdown?: string | undefined;
         html?: string | undefined;
     }[];
+    avatarTracks?: {
+        speaker: string;
+        file: string;
+        width: number;
+        height: number;
+        placements: {
+            start: number;
+            end: number;
+            x: number;
+            y: number;
+            width: number;
+            height: number;
+        }[];
+    }[] | undefined;
 };
 export declare const fetchScript: (isHttpPath: boolean, mulmoFilePath: string, fileOrUrl: string) => Promise<MulmoScript | null>;
 export declare const getMultiLingual: (multilingualFilePath: string, beats: MulmoStudioBeat[]) => MulmoStudioMultiLingual;
@@ -2619,6 +2682,14 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     model?: string | undefined;
                     baseURL?: string | undefined;
                     apiVersion?: string | undefined;
+                    avatar?: {
+                        source: string;
+                        position?: {
+                            x?: string | undefined;
+                            y?: string | undefined;
+                            scale?: string | undefined;
+                        } | undefined;
+                    } | undefined;
                     lang?: Record<string, {
                         voiceId: string;
                         displayName?: Record<string, string> | undefined;
@@ -2634,6 +2705,14 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         model?: string | undefined;
                         baseURL?: string | undefined;
                         apiVersion?: string | undefined;
+                        avatar?: {
+                            source: string;
+                            position?: {
+                                x?: string | undefined;
+                                y?: string | undefined;
+                                scale?: string | undefined;
+                            } | undefined;
+                        } | undefined;
                     }> | undefined;
                 }>;
             };
@@ -3098,6 +3177,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     type: "vision";
                     style: string;
                     data: Record<string, any>;
+                } | {
+                    type: "remotion";
+                    prompt: string;
+                    fps?: number | undefined;
                 } | {
                     type: "slide";
                     slide: {
@@ -4937,6 +5020,19 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     provider?: string | undefined;
                     model?: string | undefined;
                 } | undefined;
+                avatarParams?: {
+                    position?: {
+                        x?: string | undefined;
+                        y?: string | undefined;
+                        scale?: string | undefined;
+                    } | undefined;
+                    emotion?: "neutral" | "happy" | "sad" | "angry" | "surprised" | "relaxed" | undefined;
+                    motions?: {
+                        motion: string;
+                        at?: string | undefined;
+                    }[] | undefined;
+                    hidden?: boolean | undefined;
+                } | undefined;
                 htmlImageParams?: {
                     model?: string | undefined;
                 } | undefined;
@@ -5040,6 +5136,19 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 provider?: string | undefined;
                 model?: string | undefined;
             } | undefined;
+            avatarParams?: {
+                position?: {
+                    x?: string | undefined;
+                    y?: string | undefined;
+                    scale?: string | undefined;
+                } | undefined;
+                emotion?: "neutral" | "happy" | "sad" | "angry" | "surprised" | "relaxed" | undefined;
+                motions?: {
+                    motion: string;
+                    at?: string | undefined;
+                }[] | undefined;
+                hidden?: boolean | undefined;
+            } | undefined;
             htmlImageParams?: {
                 provider: string;
                 model?: string | undefined;
@@ -5106,6 +5215,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     } | undefined;
                 } | undefined;
             } | undefined;
+            remotionParams?: {
+                brief?: string | undefined;
+            } | undefined;
             captionParams?: {
                 lang?: string | undefined;
                 styles?: string[] | undefined;
@@ -5154,6 +5266,20 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             markdown?: string | undefined;
             html?: string | undefined;
         }[];
+        avatarTracks?: {
+            speaker: string;
+            file: string;
+            width: number;
+            height: number;
+            placements: {
+                start: number;
+                end: number;
+                x: number;
+                y: number;
+                width: number;
+                height: number;
+            }[];
+        }[] | undefined;
     };
     multiLingual: {
         multiLingualTexts: Record<string, {
@@ -5192,6 +5318,14 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 model?: string | undefined;
                 baseURL?: string | undefined;
                 apiVersion?: string | undefined;
+                avatar?: {
+                    source: string;
+                    position?: {
+                        x?: string | undefined;
+                        y?: string | undefined;
+                        scale?: string | undefined;
+                    } | undefined;
+                } | undefined;
                 lang?: Record<string, {
                     voiceId: string;
                     displayName?: Record<string, string> | undefined;
@@ -5207,6 +5341,14 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     model?: string | undefined;
                     baseURL?: string | undefined;
                     apiVersion?: string | undefined;
+                    avatar?: {
+                        source: string;
+                        position?: {
+                            x?: string | undefined;
+                            y?: string | undefined;
+                            scale?: string | undefined;
+                        } | undefined;
+                    } | undefined;
                 }> | undefined;
             }>;
         };
@@ -5498,6 +5640,19 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             provider?: string | undefined;
             model?: string | undefined;
         } | undefined;
+        avatarParams?: {
+            position?: {
+                x?: string | undefined;
+                y?: string | undefined;
+                scale?: string | undefined;
+            } | undefined;
+            emotion?: "neutral" | "happy" | "sad" | "angry" | "surprised" | "relaxed" | undefined;
+            motions?: {
+                motion: string;
+                at?: string | undefined;
+            }[] | undefined;
+            hidden?: boolean | undefined;
+        } | undefined;
         htmlImageParams?: {
             provider: string;
             model?: string | undefined;
@@ -5564,6 +5719,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | undefined;
             } | undefined;
         } | undefined;
+        remotionParams?: {
+            brief?: string | undefined;
+        } | undefined;
         captionParams?: {
             lang?: string | undefined;
             styles?: string[] | undefined;

package/lib/utils/image_plugins/index.js

@@ -9,6 +9,7 @@ import * as pluginBeat from "./beat.js";
 import * as pluginVoiceOver from "./voice_over.js";
 import * as pluginVision from "./vision.js";
 import * as pluginSlide from "./slide.js";
+import * as pluginRemotion from "./remotion.js";
 const imagePlugins = [
     pluginTextSlide,
     pluginMarkdown,
@@ -21,6 +22,7 @@ const imagePlugins = [
     pluginVoiceOver,
     pluginVision,
     pluginSlide,
+    pluginRemotion,
 ];
 export const findImagePlugin = (imageType) => {
     return imagePlugins.find((plugin) => plugin.imageType === imageType);

package/lib/utils/image_plugins/remotion.d.ts

@@ -0,0 +1,16 @@
+import { ImageProcessorParams } from "../../types/index.js";
+import { ComponentReviewer, ComponentWriter } from "../remotion/claude_runner.js";
+import { SceneRenderer } from "../remotion/render.js";
+export declare const imageType: "remotion";
+export type RemotionDeps = {
+    ensurePackages: () => void;
+    writeComponent: ComponentWriter;
+    reviewComponent: ComponentReviewer;
+    renderScene: SceneRenderer;
+};
+export declare const remotionWorkDir: (imagePath: string) => string;
+export declare const remotionStillPath: (imagePath: string) => string;
+export declare const reviewCandidatePath: (filePath: string) => string;
+export declare const createRemotionProcess: (deps: RemotionDeps) => (params: ImageProcessorParams) => Promise<string | undefined>;
+export declare const process: (params: ImageProcessorParams) => Promise<string | undefined>;
+export declare const path: (params: import("../../types/type.js").BeatPathParams) => string;

package/lib/utils/image_plugins/remotion.js

@@ -0,0 +1,164 @@
+import fs from "node:fs";
+import nodePath from "node:path";
+import { GraphAILogger } from "graphai";
+import { ImageMediaType } from "../../types/index.js";
+import { localizedText } from "../utils.js";
+import { REMOTION_COMPONENT_FILE, buildRepairPrompt, buildReviewPrompt, buildScenePrompt, remotionCacheKey, } from "../remotion/claude_prompt.js";
+import { reviewComponentWithClaude, writeComponentWithClaude } from "../remotion/claude_runner.js";
+import { ensureRemotionPackages, renderRemotionScene } from "../remotion/render.js";
+import { commitFiles } from "../remotion/commit_files.js";
+import { parrotingImagePath } from "./utils.js";
+export const imageType = ImageMediaType.Remotion;
+const DEFAULT_FPS = 30;
+const MAX_REPAIR_ATTEMPTS = 2;
+// Without audio the length is unknown; any length gives the same final frame, since the system
+// prompt makes every animation finish before the end.
+const STILL_ONLY_DURATION_SEC = 10;
+const REVIEW_FRACTIONS = [0.3, 0.6, 0.95];
+export const remotionWorkDir = (imagePath) => imagePath.replace(/(_animated)?\.(mp4|png)$/, "_remotion");
+export const remotionStillPath = (imagePath) => imagePath.replace(/_animated\.mp4$/, ".png");
+const loadOrWriteComponent = async (job, force, deps) => {
+    if (!force && fs.existsSync(job.codePath)) {
+        return { code: fs.readFileSync(job.codePath, "utf8"), isNew: false };
+    }
+    const code = await deps.writeComponent(buildScenePrompt(job.spec), job.workDir);
+    fs.writeFileSync(job.codePath, code);
+    return { code, isNew: true };
+};
+// Resolves to the code that rendered, which is a repaired version when the given one failed.
+const renderWithRepair = async (job, code, attemptsLeft, deps, reviewStills = []) => {
+    fs.writeFileSync(nodePath.join(job.workDir, REMOTION_COMPONENT_FILE), code);
+    try {
+        const extraStills = reviewStills.map(({ frame, file }) => ({ frame, path: file }));
+        await deps.renderScene({ workDir: job.workDir, props: job.props, videoPath: job.videoPath, stillPath: job.stillPath, extraStills });
+        return code;
+    }
+    catch (error) {
+        if (attemptsLeft <= 0) {
+            throw new Error(`remotion beat failed to render after repairs (code: ${job.codePath})`, { cause: error });
+        }
+        const message = error instanceof Error ? error.message : String(error);
+        GraphAILogger.info(`remotion: render failed, asking claude -p to repair (${attemptsLeft} left): ${message.slice(0, 200)}`);
+        const repaired = await deps.writeComponent(buildRepairPrompt(job.spec, code, message), job.workDir);
+        fs.writeFileSync(job.codePath, repaired);
+        return await renderWithRepair(job, repaired, attemptsLeft - 1, deps, reviewStills);
+    }
+};
+const reviewStillsFor = (job) => REVIEW_FRACTIONS.map((fraction, index) => ({
+    fraction,
+    frame: Math.min(job.props.durationInFrames - 1, Math.floor(job.props.durationInFrames * fraction)),
+    file: nodePath.join(job.workDir, `review_${index}.png`),
+}));
+const errorMessage = (error) => (error instanceof Error ? error.message : String(error));
+// null when the review itself failed; undefined when the reviewer approved the scene as it is.
+const askForReview = async (job, renderedCode, reviewStills, deps) => {
+    try {
+        return await deps.reviewComponent(buildReviewPrompt(job.spec, renderedCode, reviewStills), job.workDir);
+    }
+    catch (error) {
+        GraphAILogger.info(`remotion: visual review skipped: ${errorMessage(error)}`);
+        return null;
+    }
+};
+const nodeFileOps = {
+    exists: (file) => fs.existsSync(file),
+    rename: (from, to) => fs.renameSync(from, to),
+    remove: (file) => fs.rmSync(file, { force: true }),
+};
+export const reviewCandidatePath = (filePath) => filePath.replace(/(\.[^./]+)$/, ".review$1");
+// The reviewed version renders to side files that replace the first version's output only once it
+// succeeded, so a failure leaves the scene that already rendered — files and cached code — untouched.
+const renderReviewedVersion = async (job, improved, deps) => {
+    const candidate = {
+        ...job,
+        codePath: reviewCandidatePath(job.codePath),
+        videoPath: job.videoPath && reviewCandidatePath(job.videoPath),
+        stillPath: reviewCandidatePath(job.stillPath),
+    };
+    try {
+        const renderedImprovement = await renderWithRepair(candidate, improved, MAX_REPAIR_ATTEMPTS, deps);
+        fs.writeFileSync(candidate.codePath, renderedImprovement);
+        const moves = [
+            ...(job.videoPath && candidate.videoPath ? [{ from: candidate.videoPath, to: job.videoPath }] : []),
+            { from: candidate.stillPath, to: job.stillPath },
+            { from: candidate.codePath, to: job.codePath },
+        ];
+        commitFiles(moves, nodeFileOps);
+    }
+    catch (error) {
+        GraphAILogger.info(`remotion: the reviewed version was not applied, keeping the first one: ${errorMessage(error)}`);
+    }
+    finally {
+        [candidate.codePath, candidate.videoPath, candidate.stillPath].forEach((file) => file && fs.rmSync(file, { force: true }));
+    }
+};
+const reviewAndImprove = async (job, renderedCode, reviewStills, deps) => {
+    const improved = await askForReview(job, renderedCode, reviewStills, deps);
+    if (improved === null)
+        return;
+    if (improved === undefined) {
+        GraphAILogger.info(`remotion: visual review approved ${job.codePath}`);
+        return;
+    }
+    GraphAILogger.info(`remotion: visual review improved ${job.codePath}`);
+    await renderReviewedVersion(job, improved, deps);
+};
+const toFrameCount = (durationSec, fps) => {
+    const frames = Math.floor(durationSec * fps);
+    if (frames <= 0) {
+        throw new Error(`remotion: frame count is ${frames} (duration=${durationSec}, fps=${fps}). Increase duration or fps.`);
+    }
+    return frames;
+};
+// Lets a scene number itself and pace its place in the video; unknown when the beat is not in the script.
+const scenePosition = (beats, index) => (index < 0 ? undefined : { index, count: beats.length });
+// The words actually spoken over the scene: the translation when the video is rendered in another language.
+const spokenNarration = (context, beat, index) => (index < 0 ? beat.text : localizedText(beat, context.multiLingual?.[index], context.lang, context.studio.script.lang)) || undefined;
+const buildJob = (params, prompt, fps) => {
+    const { beat, context, imagePath, canvasSize } = params;
+    const duration = params.beatDuration ?? beat.duration;
+    const index = context.studio.script.beats.indexOf(beat);
+    const spec = {
+        prompt,
+        narration: spokenNarration(context, beat, index),
+        brief: context.presentationStyle.remotionParams?.brief || undefined,
+        position: scenePosition(context.studio.script.beats, index),
+        fps,
+        width: canvasSize.width,
+        height: canvasSize.height,
+    };
+    const workDir = remotionWorkDir(imagePath);
+    return {
+        spec,
+        workDir,
+        codePath: nodePath.join(workDir, `${remotionCacheKey(spec)}.tsx`),
+        props: { durationInFrames: toFrameCount(duration ?? STILL_ONLY_DURATION_SEC, fps), fps, width: spec.width, height: spec.height },
+        videoPath: duration !== undefined ? imagePath : undefined,
+        stillPath: remotionStillPath(imagePath),
+    };
+};
+export const createRemotionProcess = (deps) => async (params) => {
+    const { beat, context } = params;
+    if (!beat.image || beat.image.type !== imageType)
+        return;
+    const job = buildJob(params, beat.image.prompt, beat.image.fps ?? DEFAULT_FPS);
+    deps.ensurePackages();
+    fs.mkdirSync(job.workDir, { recursive: true });
+    const { code, isNew } = await loadOrWriteComponent(job, context.force, deps);
+    if (isNew) {
+        const reviewStills = reviewStillsFor(job);
+        const renderedCode = await renderWithRepair(job, code, MAX_REPAIR_ATTEMPTS, deps, reviewStills);
+        await reviewAndImprove(job, renderedCode, reviewStills, deps);
+    }
+    else {
+        await renderWithRepair(job, code, MAX_REPAIR_ATTEMPTS, deps);
+    }
+    return job.videoPath ?? job.stillPath;
+};
+export const process = createRemotionProcess({
+    ensurePackages: ensureRemotionPackages,
+    writeComponent: writeComponentWithClaude,
+    reviewComponent: reviewComponentWithClaude,
+    renderScene: renderRemotionScene,
+});
+export const path = parrotingImagePath;

package/lib/utils/remotion/claude_prompt.d.ts

@@ -0,0 +1,33 @@
+export declare const REMOTION_COMPONENT_FILE = "Generated.tsx";
+export declare const REMOTION_ENTRY_FILE = "index.tsx";
+export declare const REMOTION_COMPOSITION_ID = "Beat";
+export type RemotionSceneSpec = {
+    prompt: string;
+    narration?: string;
+    brief?: string;
+    position?: {
+        index: number;
+        count: number;
+    };
+    fps: number;
+    width: number;
+    height: number;
+};
+export declare const buildClaudeArgs: (userPrompt: string, systemPrompt?: string, tools?: string) => string[];
+export declare const buildScenePrompt: (spec: RemotionSceneSpec) => string;
+export declare const buildRepairPrompt: (spec: RemotionSceneSpec, code: string, error: string) => string;
+export type ReviewFrame = {
+    fraction: number;
+    file: string;
+};
+export declare const buildReviewPrompt: (spec: RemotionSceneSpec, code: string, frames: ReviewFrame[]) => string;
+export declare const parseReviewReply: (replyText: string) => string | undefined;
+export declare const remotionCacheKey: (spec: RemotionSceneSpec, systemPrompt?: string) => string;
+export declare const extractTsx: (replyText: string) => string | undefined;
+type ClaudeJsonReply = {
+    result: string;
+    is_error?: boolean;
+};
+export declare const parseClaudeReply: (stdout: string) => ClaudeJsonReply;
+export declare const buildEntrySource: () => string;
+export {};