mulmocast 2.12.1 → 2.14.0

package/lib/utils/image_plugins/remotion.js

@@ -0,0 +1,202 @@
+import fs from "node:fs";
+import nodePath from "node:path";
+import { GraphAILogger } from "graphai";
+import { ImageMediaType } from "../../types/index.js";
+import { localizedText } from "../utils.js";
+import { getFullPath } from "../file.js";
+import { REMOTION_COMPONENT_FILE, buildRepairPrompt, buildReviewPrompt, buildScenePrompt, remotionCacheKey, } from "../remotion/claude_prompt.js";
+import { reviewComponentWithClaude, writeComponentWithClaude } from "../remotion/claude_runner.js";
+import { ensureRemotionPackages, renderRemotionScene } from "../remotion/render.js";
+import { commitFiles } from "../remotion/commit_files.js";
+import { parrotingImagePath } from "./utils.js";
+export const imageType = ImageMediaType.Remotion;
+const DEFAULT_FPS = 30;
+const MAX_REPAIR_ATTEMPTS = 2;
+// Without audio the length is unknown; any length gives the same final frame, since the system
+// prompt makes every animation finish before the end.
+const STILL_ONLY_DURATION_SEC = 10;
+const REVIEW_FRACTIONS = [0.3, 0.6, 0.95];
+export const remotionWorkDir = (imagePath) => imagePath.replace(/(_animated)?\.(mp4|png)$/, "_remotion");
+export const remotionStillPath = (imagePath) => imagePath.replace(/_animated\.mp4$/, ".png");
+const loadOrWriteComponent = async (job, force, deps) => {
+    if (!force && fs.existsSync(job.codePath)) {
+        return { code: fs.readFileSync(job.codePath, "utf8"), isNew: false };
+    }
+    const code = await deps.writeComponent(buildScenePrompt(job.spec), job.workDir);
+    fs.writeFileSync(job.codePath, code);
+    return { code, isNew: true };
+};
+// Resolves to the code that rendered, which is a repaired version when the given one failed.
+const renderWithRepair = async (job, code, attemptsLeft, deps, reviewStills = []) => {
+    fs.writeFileSync(nodePath.join(job.workDir, REMOTION_COMPONENT_FILE), code);
+    try {
+        const extraStills = reviewStills.map(({ frame, file }) => ({ frame, path: file }));
+        await deps.renderScene({ workDir: job.workDir, props: job.props, videoPath: job.videoPath, stillPath: job.stillPath, extraStills });
+        return code;
+    }
+    catch (error) {
+        if (attemptsLeft <= 0) {
+            throw new Error(`remotion beat failed to render after repairs (code: ${job.codePath})`, { cause: error });
+        }
+        const message = error instanceof Error ? error.message : String(error);
+        GraphAILogger.info(`remotion: render failed, asking claude -p to repair (${attemptsLeft} left): ${message.slice(0, 200)}`);
+        const repaired = await deps.writeComponent(buildRepairPrompt(job.spec, code, message), job.workDir);
+        fs.writeFileSync(job.codePath, repaired);
+        return await renderWithRepair(job, repaired, attemptsLeft - 1, deps, reviewStills);
+    }
+};
+const reviewStillsFor = (job) => REVIEW_FRACTIONS.map((fraction, index) => ({
+    fraction,
+    frame: Math.min(job.props.durationInFrames - 1, Math.floor(job.props.durationInFrames * fraction)),
+    file: nodePath.join(job.workDir, `review_${index}.png`),
+}));
+const errorMessage = (error) => (error instanceof Error ? error.message : String(error));
+// null when the review itself failed; undefined when the reviewer approved the scene as it is.
+const askForReview = async (job, renderedCode, reviewStills, deps) => {
+    try {
+        return await deps.reviewComponent(buildReviewPrompt(job.spec, renderedCode, reviewStills), job.workDir);
+    }
+    catch (error) {
+        GraphAILogger.info(`remotion: visual review skipped: ${errorMessage(error)}`);
+        return null;
+    }
+};
+const nodeFileOps = {
+    exists: (file) => fs.existsSync(file),
+    rename: (from, to) => fs.renameSync(from, to),
+    remove: (file) => fs.rmSync(file, { force: true }),
+};
+export const reviewCandidatePath = (filePath) => filePath.replace(/(\.[^./]+)$/, ".review$1");
+// The reviewed version renders to side files that replace the first version's output only once it
+// succeeded, so a failure leaves the scene that already rendered — files and cached code — untouched.
+const renderReviewedVersion = async (job, improved, deps) => {
+    const candidate = {
+        ...job,
+        codePath: reviewCandidatePath(job.codePath),
+        videoPath: job.videoPath && reviewCandidatePath(job.videoPath),
+        stillPath: reviewCandidatePath(job.stillPath),
+    };
+    try {
+        const renderedImprovement = await renderWithRepair(candidate, improved, MAX_REPAIR_ATTEMPTS, deps);
+        fs.writeFileSync(candidate.codePath, renderedImprovement);
+        const moves = [
+            ...(job.videoPath && candidate.videoPath ? [{ from: candidate.videoPath, to: job.videoPath }] : []),
+            { from: candidate.stillPath, to: job.stillPath },
+            { from: candidate.codePath, to: job.codePath },
+        ];
+        commitFiles(moves, nodeFileOps);
+    }
+    catch (error) {
+        GraphAILogger.info(`remotion: the reviewed version was not applied, keeping the first one: ${errorMessage(error)}`);
+    }
+    finally {
+        [candidate.codePath, candidate.videoPath, candidate.stillPath].forEach((file) => file && fs.rmSync(file, { force: true }));
+    }
+};
+const reviewAndImprove = async (job, renderedCode, reviewStills, deps) => {
+    const improved = await askForReview(job, renderedCode, reviewStills, deps);
+    if (improved === null)
+        return;
+    if (improved === undefined) {
+        GraphAILogger.info(`remotion: visual review approved ${job.codePath}`);
+        return;
+    }
+    GraphAILogger.info(`remotion: visual review improved ${job.codePath}`);
+    await renderReviewedVersion(job, improved, deps);
+};
+const toFrameCount = (durationSec, fps) => {
+    const frames = Math.floor(durationSec * fps);
+    if (frames <= 0) {
+        throw new Error(`remotion: frame count is ${frames} (duration=${durationSec}, fps=${fps}). Increase duration or fps.`);
+    }
+    return frames;
+};
+// Lets a scene number itself and pace its place in the video; unknown when the beat is not in the script.
+const scenePosition = (beats, index) => (index < 0 ? undefined : { index, count: beats.length });
+// The words actually spoken over the scene: the translation when the video is rendered in another language.
+const spokenNarration = (context, beat, index) => (index < 0 ? beat.text : localizedText(beat, context.multiLingual?.[index], context.lang, context.studio.script.lang)) || undefined;
+const buildSceneOutput = (params, fps) => {
+    const { beat, imagePath, canvasSize } = params;
+    const duration = params.beatDuration ?? beat.duration;
+    return {
+        workDir: remotionWorkDir(imagePath),
+        props: { durationInFrames: toFrameCount(duration ?? STILL_ONLY_DURATION_SEC, fps), fps, width: canvasSize.width, height: canvasSize.height },
+        videoPath: duration !== undefined ? imagePath : undefined,
+        stillPath: remotionStillPath(imagePath),
+    };
+};
+const buildJob = (params, prompt, fps) => {
+    const { beat, context, canvasSize } = params;
+    const index = context.studio.script.beats.indexOf(beat);
+    const spec = {
+        prompt,
+        narration: spokenNarration(context, beat, index),
+        brief: context.presentationStyle.remotionParams?.brief || undefined,
+        position: scenePosition(context.studio.script.beats, index),
+        fps,
+        width: canvasSize.width,
+        height: canvasSize.height,
+    };
+    const output = buildSceneOutput(params, fps);
+    return { ...output, spec, codePath: nodePath.join(output.workDir, `${remotionCacheKey(spec)}.tsx`) };
+};
+// Named in errors, so whoever wrote the component (usually the host's agent) knows what to fix.
+export const remotionCodeLocation = (code, context, beatIndex) => {
+    if (code.kind === "path")
+        return getFullPath(context.fileDirs.mulmoFileDirPath, code.path);
+    return beatIndex < 0 ? "inline code" : `inline code of beat ${beatIndex + 1}`;
+};
+const readGivenComponent = (code, location) => {
+    if (code.kind === "text")
+        return code.text;
+    try {
+        return fs.readFileSync(location, "utf8");
+    }
+    catch (error) {
+        throw new Error(`remotion: cannot read the component file ${location}`, { cause: error });
+    }
+};
+// A given component is the caller's to fix, so a failed render stops here instead of asking claude -p to repair it.
+const renderGivenComponent = async (params, code, fps, deps) => {
+    const output = buildSceneOutput(params, fps);
+    const location = remotionCodeLocation(code, params.context, params.context.studio.script.beats.indexOf(params.beat));
+    deps.ensurePackages();
+    const source = readGivenComponent(code, location);
+    fs.mkdirSync(output.workDir, { recursive: true });
+    fs.writeFileSync(nodePath.join(output.workDir, REMOTION_COMPONENT_FILE), source);
+    try {
+        await deps.renderScene({ workDir: output.workDir, props: output.props, videoPath: output.videoPath, stillPath: output.stillPath });
+    }
+    catch (error) {
+        throw new Error(`remotion beat failed to render the given component (code: ${location}): ${errorMessage(error)}`, { cause: error });
+    }
+    return output.videoPath ?? output.stillPath;
+};
+export const createRemotionProcess = (deps) => async (params) => {
+    const { beat, context } = params;
+    if (!beat.image || beat.image.type !== imageType)
+        return;
+    const fps = beat.image.fps ?? DEFAULT_FPS;
+    if ("code" in beat.image)
+        return await renderGivenComponent(params, beat.image.code, fps, deps);
+    const job = buildJob(params, beat.image.prompt, fps);
+    deps.ensurePackages();
+    fs.mkdirSync(job.workDir, { recursive: true });
+    const { code, isNew } = await loadOrWriteComponent(job, context.force, deps);
+    if (isNew) {
+        const reviewStills = reviewStillsFor(job);
+        const renderedCode = await renderWithRepair(job, code, MAX_REPAIR_ATTEMPTS, deps, reviewStills);
+        await reviewAndImprove(job, renderedCode, reviewStills, deps);
+    }
+    else {
+        await renderWithRepair(job, code, MAX_REPAIR_ATTEMPTS, deps);
+    }
+    return job.videoPath ?? job.stillPath;
+};
+export const process = createRemotionProcess({
+    ensurePackages: ensureRemotionPackages,
+    writeComponent: writeComponentWithClaude,
+    reviewComponent: reviewComponentWithClaude,
+    renderScene: renderRemotionScene,
+});
+export const path = parrotingImagePath;

package/lib/utils/remotion/claude_prompt.d.ts

@@ -0,0 +1,33 @@
+export declare const REMOTION_COMPONENT_FILE = "Generated.tsx";
+export declare const REMOTION_ENTRY_FILE = "index.tsx";
+export declare const REMOTION_COMPOSITION_ID = "Beat";
+export type RemotionSceneSpec = {
+    prompt: string;
+    narration?: string;
+    brief?: string;
+    position?: {
+        index: number;
+        count: number;
+    };
+    fps: number;
+    width: number;
+    height: number;
+};
+export declare const buildClaudeArgs: (userPrompt: string, systemPrompt?: string, tools?: string) => string[];
+export declare const buildScenePrompt: (spec: RemotionSceneSpec) => string;
+export declare const buildRepairPrompt: (spec: RemotionSceneSpec, code: string, error: string) => string;
+export type ReviewFrame = {
+    fraction: number;
+    file: string;
+};
+export declare const buildReviewPrompt: (spec: RemotionSceneSpec, code: string, frames: ReviewFrame[]) => string;
+export declare const parseReviewReply: (replyText: string) => string | undefined;
+export declare const remotionCacheKey: (spec: RemotionSceneSpec, systemPrompt?: string) => string;
+export declare const extractTsx: (replyText: string) => string | undefined;
+type ClaudeJsonReply = {
+    result: string;
+    is_error?: boolean;
+};
+export declare const parseClaudeReply: (stdout: string) => ClaudeJsonReply;
+export declare const buildEntrySource: () => string;
+export {};

package/lib/utils/remotion/claude_prompt.js

@@ -0,0 +1,111 @@
+import { createHash } from "node:crypto";
+import { REMOTION_REVIEW_MARKER, REMOTION_SYSTEM_PROMPT } from "./system_prompt.js";
+export const REMOTION_COMPONENT_FILE = "Generated.tsx";
+export const REMOTION_ENTRY_FILE = "index.tsx";
+export const REMOTION_COMPOSITION_ID = "Beat";
+// No user/project settings: the reply depends only on these prompts, and hooks, plugins and MCP
+// servers neither run nor add their context to every call. Writing needs no tools; review reads images,
+// and --restricted keeps that Read inside the beat's work directory — a script's prompt cannot point it
+// at other files.
+export const buildClaudeArgs = (userPrompt, systemPrompt = REMOTION_SYSTEM_PROMPT, tools = "") => [
+    "-p",
+    userPrompt,
+    "--system-prompt",
+    systemPrompt,
+    "--tools",
+    tools,
+    ...(tools ? ["--allowedTools", tools] : []),
+    "--setting-sources",
+    "",
+    "--strict-mcp-config",
+    "--restricted",
+    "--output-format",
+    "json",
+    "--no-session-persistence",
+];
+export const buildScenePrompt = (spec) => [
+    ...(spec.brief ? [`Art direction for the whole video (every scene follows it exactly, so they look like one film): ${spec.brief}`, ""] : []),
+    ...(spec.position ? [`This is scene ${spec.position.index + 1} of ${spec.position.count}.`, ""] : []),
+    spec.prompt,
+    "",
+    ...(spec.narration ? [`Narration spoken over this scene: ${spec.narration}`, ""] : []),
+    `Canvas: ${spec.width}x${spec.height} pixels at ${spec.fps} fps.`,
+].join("\n");
+export const buildRepairPrompt = (spec, code, error) => [
+    buildScenePrompt(spec),
+    "",
+    "Your previous component failed to bundle or render. Fix it and reply with the whole corrected component.",
+    "",
+    "Error:",
+    error,
+    "",
+    "Previous component:",
+    "```tsx",
+    code,
+    "```",
+].join("\n");
+export const buildReviewPrompt = (spec, code, frames) => [
+    buildScenePrompt(spec),
+    "",
+    "Rendered frames of your component (read each image file):",
+    ...frames.map((frame) => `- at ${Math.round(frame.fraction * 100)}% of the scene: ${frame.file}`),
+    "",
+    "Your component:",
+    "```tsx",
+    code,
+    "```",
+].join("\n");
+// undefined means the reviewer approved the scene as it is.
+export const parseReviewReply = (replyText) => {
+    if (replyText.trim() === REMOTION_REVIEW_MARKER)
+        return undefined;
+    return extractTsx(replyText);
+};
+export const remotionCacheKey = (spec, systemPrompt = REMOTION_SYSTEM_PROMPT) => createHash("sha256")
+    .update(JSON.stringify([spec.prompt, spec.narration ?? "", spec.brief ?? "", spec.position ?? null, spec.fps, spec.width, spec.height, systemPrompt]), "utf8")
+    .digest("hex");
+const CODE_BLOCK_PATTERN = /```(?:tsx|typescript|ts|jsx)?[ \t]*\r?\n([\s\S]*?)```/;
+export const extractTsx = (replyText) => {
+    const match = CODE_BLOCK_PATTERN.exec(replyText);
+    const code = match ? match[1] : replyText;
+    return code.includes("export default") ? code.trim() + "\n" : undefined;
+};
+const isClaudeJsonReply = (value) => typeof value === "object" && value !== null && "result" in value && typeof value.result === "string";
+export const parseClaudeReply = (stdout) => {
+    const parsed = JSON.parse(stdout);
+    if (!isClaudeJsonReply(parsed)) {
+        throw new Error(`claude -p returned JSON without a "result" string: ${stdout.slice(0, 200)}`);
+    }
+    if (parsed.is_error) {
+        throw new Error(`claude -p reported an error: ${parsed.result}`);
+    }
+    return parsed;
+};
+export const buildEntrySource = () => [
+    'import React from "react";',
+    'import { Composition, registerRoot } from "remotion";',
+    `import Scene from "./${REMOTION_COMPONENT_FILE.replace(/\.tsx$/, "")}";`,
+    "",
+    "type SceneProps = { durationInFrames: number; fps: number; width: number; height: number };",
+    "",
+    "const Root: React.FC = () => (",
+    "  <Composition",
+    `    id="${REMOTION_COMPOSITION_ID}"`,
+    "    component={Scene}",
+    "    durationInFrames={1}",
+    "    fps={30}",
+    "    width={1280}",
+    "    height={720}",
+    "    defaultProps={{ durationInFrames: 1, fps: 30, width: 1280, height: 720 }}",
+    "    calculateMetadata={({ props }: { props: SceneProps }) => ({",
+    "      durationInFrames: props.durationInFrames,",
+    "      fps: props.fps,",
+    "      width: props.width,",
+    "      height: props.height,",
+    "    })}",
+    "  />",
+    ");",
+    "",
+    "registerRoot(Root);",
+    "",
+].join("\n");

package/lib/utils/remotion/claude_runner.d.ts

@@ -0,0 +1,4 @@
+export type ComponentWriter = (userPrompt: string, workDir: string) => Promise<string>;
+export type ComponentReviewer = (userPrompt: string, workDir: string) => Promise<string | undefined>;
+export declare const writeComponentWithClaude: ComponentWriter;
+export declare const reviewComponentWithClaude: ComponentReviewer;

package/lib/utils/remotion/claude_runner.js

@@ -0,0 +1,47 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { buildClaudeArgs, extractTsx, parseClaudeReply, parseReviewReply } from "./claude_prompt.js";
+import { REMOTION_REVIEW_SYSTEM_PROMPT, REMOTION_SYSTEM_PROMPT } from "./system_prompt.js";
+const execFileAsync = promisify(execFile);
+const CLAUDE_COMMAND = "claude";
+const CLAUDE_TIMEOUT_MS = 10 * 60 * 1000;
+const CLAUDE_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
+const REVIEW_TOOLS = "Read";
+const describeFailure = (error, workDir) => {
+    if (error instanceof Error && error.name === "AbortError") {
+        return new Error(`claude -p timed out after ${CLAUDE_TIMEOUT_MS}ms (workDir: ${workDir})`, { cause: error });
+    }
+    if (error instanceof Error && "code" in error && error.code === "ENOENT") {
+        return new Error(`"${CLAUDE_COMMAND}" was not found. The remotion beat needs Claude Code installed and logged in.`, { cause: error });
+    }
+    const message = error instanceof Error ? error.message : String(error);
+    return new Error(`Running "${CLAUDE_COMMAND} -p" failed (workDir: ${workDir}): ${message}`, { cause: error });
+};
+// Run from the beat's work directory so no project CLAUDE.md is picked up and the reviewer can read
+// the frames there; stdin is closed so `claude -p` does not wait for piped input.
+const runClaude = async (args, workDir) => {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), CLAUDE_TIMEOUT_MS);
+    try {
+        const child = execFileAsync(CLAUDE_COMMAND, args, { cwd: workDir, signal: controller.signal, maxBuffer: CLAUDE_MAX_OUTPUT_BYTES });
+        child.child.stdin?.end();
+        const { stdout } = await child;
+        return parseClaudeReply(stdout).result;
+    }
+    catch (error) {
+        throw describeFailure(error, workDir);
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+};
+export const writeComponentWithClaude = async (userPrompt, workDir) => {
+    const code = extractTsx(await runClaude(buildClaudeArgs(userPrompt, REMOTION_SYSTEM_PROMPT), workDir));
+    if (!code) {
+        throw new Error(`claude -p did not return a component with "export default" (workDir: ${workDir})`);
+    }
+    return code;
+};
+export const reviewComponentWithClaude = async (userPrompt, workDir) => {
+    return parseReviewReply(await runClaude(buildClaudeArgs(userPrompt, REMOTION_REVIEW_SYSTEM_PROMPT, REVIEW_TOOLS), workDir));
+};

package/lib/utils/remotion/commit_files.d.ts

@@ -0,0 +1,11 @@
+export type FileOps = {
+    exists: (file: string) => boolean;
+    rename: (from: string, to: string) => void;
+    remove: (file: string) => void;
+};
+export type FileMove = {
+    from: string;
+    to: string;
+};
+export declare const backupPathOf: (file: string) => string;
+export declare const commitFiles: (moves: FileMove[], ops: FileOps) => void;

package/lib/utils/remotion/commit_files.js

@@ -0,0 +1,37 @@
+export const backupPathOf = (file) => `${file}.pre-review`;
+// Moves every `from` onto its `to`, or none of them: on any failure the moves already made are undone
+// and the files they replaced are put back, then the error is rethrown.
+export const commitFiles = (moves, ops) => {
+    const undoSteps = [];
+    try {
+        moves.forEach(({ from, to }) => {
+            if (ops.exists(to)) {
+                ops.rename(to, backupPathOf(to));
+                undoSteps.push(() => ops.rename(backupPathOf(to), to));
+            }
+            ops.rename(from, to);
+            undoSteps.push(() => ops.rename(to, from));
+        });
+    }
+    catch (error) {
+        // Undo in reverse; one failing undo must not stop the others from restoring their files.
+        undoSteps.reverse().forEach((undo) => {
+            try {
+                undo();
+            }
+            catch {
+                // the original error below is what the caller needs to see
+            }
+        });
+        throw error;
+    }
+    // The moves are committed by now; a backup that cannot be removed is a leftover, not a failed commit.
+    moves.forEach(({ to }) => {
+        try {
+            ops.remove(backupPathOf(to));
+        }
+        catch {
+            // leave it; the next commit onto this path overwrites it
+        }
+    });
+};

package/lib/utils/remotion/packages.d.ts

@@ -0,0 +1,5 @@
+export declare const REMOTION_RENDER_PACKAGES: readonly ["remotion", "@remotion/bundler", "@remotion/renderer", "react", "react-dom"];
+export declare const REMOTION_SCENE_PACKAGES: readonly ["@remotion/three", "three", "@react-three/fiber", "@remotion/effects", "@remotion/paths", "@remotion/noise", "@remotion/shapes", "@remotion/transitions", "@remotion/motion-blur", "@remotion/layout-utils"];
+export declare const REMOTION_PACKAGES: readonly string[];
+export declare const remotionInstallCommand: (packages?: readonly string[]) => string;
+export declare const missingRemotionPackages: (isInstalled: (name: string) => boolean, packages?: readonly string[]) => string[];

package/lib/utils/remotion/packages.js

@@ -0,0 +1,20 @@
+// Every package the renderer needs or the generated component may import. They are optional peer
+// dependencies of mulmocast, so a host installs them itself; one list keeps the install hint, the
+// pre-flight check and the system prompt's import allowance from drifting apart.
+export const REMOTION_RENDER_PACKAGES = ["remotion", "@remotion/bundler", "@remotion/renderer", "react", "react-dom"];
+export const REMOTION_SCENE_PACKAGES = [
+    "@remotion/three",
+    "three",
+    "@react-three/fiber",
+    "@remotion/effects",
+    "@remotion/paths",
+    "@remotion/noise",
+    "@remotion/shapes",
+    "@remotion/transitions",
+    "@remotion/motion-blur",
+    "@remotion/layout-utils",
+];
+export const REMOTION_PACKAGES = [...REMOTION_RENDER_PACKAGES, ...REMOTION_SCENE_PACKAGES];
+export const remotionInstallCommand = (packages = REMOTION_PACKAGES) => `npm install ${packages.join(" ")}`;
+// `isInstalled` is injected so the rule can be tested without touching node_modules.
+export const missingRemotionPackages = (isInstalled, packages = REMOTION_PACKAGES) => packages.filter((name) => !isInstalled(name));

package/lib/utils/remotion/render.d.ts

@@ -0,0 +1,20 @@
+export type RemotionSceneProps = {
+    durationInFrames: number;
+    fps: number;
+    width: number;
+    height: number;
+};
+export type RemotionRenderRequest = {
+    workDir: string;
+    props: RemotionSceneProps;
+    videoPath?: string;
+    stillPath: string;
+    extraStills?: {
+        frame: number;
+        path: string;
+    }[];
+};
+export type SceneRenderer = (request: RemotionRenderRequest) => Promise<void>;
+export declare const missingInstalledRemotionPackages: () => string[];
+export declare const ensureRemotionPackages: () => void;
+export declare const renderRemotionScene: SceneRenderer;

package/lib/utils/remotion/render.js

@@ -0,0 +1,65 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createRequire } from "node:module";
+import { REMOTION_COMPOSITION_ID, REMOTION_ENTRY_FILE, buildEntrySource } from "./claude_prompt.js";
+import { missingRemotionPackages, remotionInstallCommand } from "./packages.js";
+// WebGL (three.js scenes) needs the ANGLE backend; the default one cannot create a context headless.
+const CHROMIUM_OPTIONS = { gl: "angle" };
+const loadRemotion = async () => {
+    try {
+        const [bundler, renderer] = await Promise.all([import("@remotion/bundler"), import("@remotion/renderer")]);
+        return { bundle: bundler.bundle, renderer };
+    }
+    catch (error) {
+        throw new Error(`The remotion beat needs these packages: ${remotionInstallCommand()}`, { cause: error });
+    }
+};
+// A package that hides its package.json (three does) is still installed; only "not found" means missing.
+const isPackageInstalled = (name) => {
+    try {
+        createRequire(import.meta.url).resolve(`${name}/package.json`);
+        return true;
+    }
+    catch (error) {
+        return !(error instanceof Error && "code" in error && error.code === "MODULE_NOT_FOUND");
+    }
+};
+// Resolved from mulmocast's own location, which is where the renderer looks for them.
+export const missingInstalledRemotionPackages = () => missingRemotionPackages(isPackageInstalled);
+// Checked before asking claude -p for a component: a generated import of a missing optional peer
+// would otherwise surface as an opaque bundling error after the generation was already paid for.
+export const ensureRemotionPackages = () => {
+    const missing = missingInstalledRemotionPackages();
+    if (missing.length > 0) {
+        throw new Error(`The remotion beat needs packages that are not installed (${missing.join(", ")}). Install them with: ${remotionInstallCommand()}`);
+    }
+};
+// The generated code sits in the output directory, which usually has no node_modules above it that
+// holds react/remotion — so resolve them from wherever remotion itself is installed.
+const remotionNodeModulesDir = () => {
+    const remotionPackageJson = createRequire(import.meta.url).resolve("remotion/package.json");
+    return path.dirname(path.dirname(remotionPackageJson));
+};
+export const renderRemotionScene = async ({ workDir, props, videoPath, stillPath, extraStills = [] }) => {
+    const { bundle, renderer } = await loadRemotion();
+    const entryPoint = path.join(workDir, REMOTION_ENTRY_FILE);
+    fs.writeFileSync(entryPoint, buildEntrySource());
+    const nodeModulesDir = remotionNodeModulesDir();
+    const serveUrl = await bundle({
+        entryPoint,
+        outDir: path.join(workDir, "bundle"),
+        webpackOverride: (config) => ({
+            ...config,
+            resolve: { ...config.resolve, modules: [...(config.resolve?.modules ?? ["node_modules"]), nodeModulesDir] },
+        }),
+    });
+    const composition = await renderer.selectComposition({ serveUrl, id: REMOTION_COMPOSITION_ID, inputProps: props, chromiumOptions: CHROMIUM_OPTIONS });
+    if (videoPath) {
+        await renderer.renderMedia({ composition, serveUrl, codec: "h264", outputLocation: videoPath, inputProps: props, chromiumOptions: CHROMIUM_OPTIONS });
+    }
+    const stills = [...extraStills, { frame: composition.durationInFrames - 1, path: stillPath }];
+    await stills.reduce(async (previous, still) => {
+        await previous;
+        await renderer.renderStill({ composition, serveUrl, output: still.path, frame: still.frame, inputProps: props, chromiumOptions: CHROMIUM_OPTIONS });
+    }, Promise.resolve());
+};

package/lib/utils/remotion/system_prompt.d.ts

@@ -0,0 +1,4 @@
+export declare const REMOTION_SYSTEM_PROMPT: string;
+export declare const REMOTION_COMPONENT_GUIDE: string;
+export declare const REMOTION_REVIEW_MARKER = "LGTM";
+export declare const REMOTION_REVIEW_SYSTEM_PROMPT: string;

package/lib/utils/remotion/system_prompt.js

@@ -0,0 +1,66 @@
+import { REMOTION_SCENE_PACKAGES } from "./packages.js";
+const ALLOWED_IMPORTS = ["react", "remotion", ...REMOTION_SCENE_PACKAGES].map((name) => `"${name}"`).join(", ");
+// The contract and the house style for the component `claude -p` writes for one remotion beat.
+// Every snippet in the toolkit section was rendered with this repository's renderer settings.
+const ROLE = `You are a senior motion designer and Remotion (v4.0.533) engineer. You write ONE React component in TypeScript (TSX) that renders one scene of a commercial-quality video.`;
+const REPLY_FORMAT = `- Reply with ONLY the TSX source in one \`\`\`tsx code block. No explanation before or after.`;
+const COMPONENT_RULES = `- One self-contained file: no relative imports, since it is rendered on its own.`;
+const CONTRACT_BODY = `- \`export default\` the scene component. It receives props \`{ durationInFrames: number; fps: number; width: number; height: number }\`; read them with \`useVideoConfig()\`.
+- Allowed imports, nothing else: ${ALLOWED_IMPORTS}. Subpaths of these packages are allowed too, e.g. "@remotion/transitions/fade" (also "/slide", "/wipe", "/iris", "/cross-zoom", "/film-burn", "/dissolve", "/clock-wipe") and "@remotion/effects/<name>".
+- No network, no external URLs, no image/video/audio files, no web fonts. Draw everything with DOM, SVG, canvas or three.js.`;
+const OUTPUT_CONTRACT = [ROLE, "", "# Output contract", REPLY_FORMAT, CONTRACT_BODY].join("\n");
+const TIMING_RULES = `# Timing (hard rules — breaking them causes flicker or broken renders)
+- Everything is a pure function of \`useCurrentFrame()\`. NEVER use CSS animations/transitions, \`setTimeout\`, \`requestAnimationFrame\`, \`Date\`, or three.js \`useFrame\`.
+- NEVER use \`Math.random()\`; use \`random("seed-" + i)\` from "remotion" or \`noise2D/noise3D\` from "@remotion/noise".
+- NEVER hardcode the scene length. Express every moment as a fraction of \`durationInFrames\` (e.g. \`const at = (f: number) => Math.round(durationInFrames * f)\`), so the scene fits any narration length.
+- All entrances finish by 75% of the duration; the rest holds the composed final state (with only subtle motion). Do not fade the scene out at the end — the next scene cuts in.
+- Always pass \`{ extrapolateLeft: "clamp", extrapolateRight: "clamp" }\` to \`interpolate\`.
+- Easing: \`Easing.bezier(0.16, 1, 0.3, 1)\` for entrances, \`Easing.bezier(0.7, 0, 0.84, 0)\` for exits, \`spring({ frame, fps, config: { damping: 200 } })\` for physical settle. Stagger related items by about 0.25 s (\`Math.round(fps * 0.25)\` frames).`;
+const DESIGN_SYSTEM = `# Design system (aim: a polished brand film or keynote, not a slide)
+- Canvas units: define \`const u = width / 1920;\` and size everything with it. Safe margins: 120u left/right, 96u top/bottom. Nothing important outside them, and nothing may touch or cross the canvas edge unintentionally.
+- Type scale at 1920 wide: hero 160–280u (numbers can go bigger), title 72–110u, body 40–52u, label/kicker 22–28u uppercase with letter-spacing 0.18em. Never below 22u.
+- Fonts: \`"Inter", "Hiragino Sans", "Noto Sans JP", "Helvetica Neue", Arial, sans-serif\` for text, \`"SF Mono", "Menlo", monospace\` for code. Weight 800–900 for heroes with letter-spacing -0.03em; 400–500 for body. Use \`fontVariantNumeric: "tabular-nums"\` for changing numbers.
+- Fit every text: before placing long text, compute its size with \`fitText({ text, withinWidth, fontFamily, fontWeight })\` from "@remotion/layout-utils" and cap it with \`Math.min(...)\`, or keep strings short. Text must never overflow, clip, or overlap other elements.
+- Layout: one hero per scene. Use a 12-column grid inside the safe margins, flex/grid layouts, thin 1–2u rules, aligned edges. Avoid centered stacks of identical boxes. Prefer asymmetric, editorial compositions with deliberate negative space.
+- Color: deep background (near-black, deep navy or rich dark tone), off-white text (#F5F7FA), muted secondary text (rgba(245,247,250,0.6)), and ONE saturated accent (plus at most one supporting tint). Use gradients only as soft lighting (radial, low contrast). No rainbow, no emoji.
+- Depth and polish: layered backgrounds (soft radial light, fine grid or grain), subtle parallax between layers, gentle camera drift (scale 1.00→1.04 over the scene), and light effects sparingly.`;
+const MOTION_LANGUAGE = `# Motion language
+- Kinetic typography: split headlines into words or characters and reveal each with a staggered mask (clipPath inset or overflow-hidden translateY 100%→0), not a plain fade. Pair every opacity change with translate, scale or blur.
+- Data as motion: counters that count up with tabular numerals, bars and rings that fill, lines that draw on (\`evolvePath\`), tables whose cells fill in order, comparisons that resolve into a ratio chip.
+- Choreography: one focal action at a time, in the order the narration says things. Secondary elements settle quietly. Keep a living background (noise drift, slow light sweep, particles from seeded random) so no frame is ever static.
+- Transitions inside the scene: wipes, clip-path reveals, scale-through, light sweeps, or \`TransitionSeries\` for multi-part scenes.
+- When narration is given, the screen shows its key facts (numbers, names, short phrases), timed in the order they are spoken — never the full sentence as a subtitle.`;
+const TOOLKIT = `# Toolkit (verified snippets — copy the patterns exactly)
+- Draw a line/path on:
+  \`import { evolvePath } from "@remotion/paths";\`
+  \`const { strokeDasharray, strokeDashoffset } = evolvePath(progress, d);\` → put both on the SVG <path> with fill="none".
+- Organic drift: \`import { noise2D } from "@remotion/noise";\` \`const dx = noise2D("x", frame / 90, 0) * 20 * u;\`
+- Seeded randomness: \`import { random } from "remotion";\` \`const x = random("star-" + i) * width;\`
+- 3D (three.js via react-three-fiber):
+  \`import { ThreeCanvas } from "@remotion/three";\`
+  \`<ThreeCanvas width={width} height={height} camera={{ position: [0, 0, 6], fov: 45 }}> <ambientLight intensity={0.5} /> <pointLight position={[3, 3, 4]} intensity={40} color="#22d3ee" /> <mesh rotation={[a, b, 0]}> <torusKnotGeometry args={[0.9, 0.28, 160, 24]} /> <meshStandardMaterial color="#e2e8f0" metalness={0.6} roughness={0.2} /> </mesh> </ThreeCanvas>\`
+  Drive every rotation/position/camera move from \`frame\`. Always add lights. Any <Sequence> inside ThreeCanvas needs \`layout="none"\`. Layer DOM/SVG text above the canvas with absolute positioning.
+- Post effects on a whole layer (glow, chromatic aberration, vignette, grain, scanlines, halftone, light leak, blur, zoom blur, shine, light trail…):
+  \`import { HtmlInCanvas } from "remotion";\` \`import { glow } from "@remotion/effects/glow";\` \`import { vignette } from "@remotion/effects/vignette";\`
+  \`<HtmlInCanvas width={width} height={height} effects={[glow({}), vignette({})]}> <AbsoluteFill>…DOM/SVG…</AbsoluteFill> </HtmlInCanvas>\`
+  Subpath → function: glow→glow, vignette→vignette, chromatic-aberration→chromaticAberration, scanlines→scanlines, halftone→halftone, noise→noise, blur→blur, zoom-blur→zoomBlur, shine→shine, light-trail→lightTrail, light-leak→lightLeak. Call each with \`({})\` for defaults. Use effects sparingly; they must serve the scene.
+- Light leak overlay: \`import { Solid } from "remotion"; import { lightLeak } from "@remotion/effects/light-leak";\` \`<Solid width={width} height={height} color="transparent" effects={[lightLeak({ progress, seed: 4 })]} style={{ position: "absolute", mixBlendMode: "screen" }} />\`
+- Shapes: \`import { Circle, Rect, Star, Polygon, Pie } from "@remotion/shapes";\` (SVG elements; animate their props from frame).`;
+const SELF_CHECK = `# Before you answer, check silently
+- Every import is in the allowed list and every used symbol is imported. The code type-checks.
+- At 30%, 60% and 100% of the duration the frame is fully composed, readable, balanced, and nothing overflows the safe margins or overlaps.
+- The scene matches the brief and the narration, and looks like a frame from a premium brand film.`;
+export const REMOTION_SYSTEM_PROMPT = [OUTPUT_CONTRACT, TIMING_RULES, DESIGN_SYSTEM, MOTION_LANGUAGE, TOOLKIT, SELF_CHECK].join("\n\n");
+// The same contract for a host's agent that writes the component file itself and passes it as \`code\`.
+// The system prompt stays byte-identical, since it is part of every cached component's key.
+const COMPONENT_CONTRACT = ["# Component contract", COMPONENT_RULES, CONTRACT_BODY].join("\n");
+export const REMOTION_COMPONENT_GUIDE = [COMPONENT_CONTRACT, TIMING_RULES, DESIGN_SYSTEM, MOTION_LANGUAGE, TOOLKIT, SELF_CHECK].join("\n\n");
+export const REMOTION_REVIEW_MARKER = "LGTM";
+export const REMOTION_REVIEW_SYSTEM_PROMPT = [
+    REMOTION_SYSTEM_PROMPT,
+    `# Your task now: visual review
+You are given rendered frames of a scene you wrote and its source. Read every frame image with the Read tool, then judge it as a creative director shipping a commercial.
+Look for: text overflowing, clipped by the canvas edge or by its container; elements overlapping or colliding; unreadable contrast or too-small text; empty, unbalanced or flat composition; broken or missing rendering (black areas, missing 3D, artifacts); anything that does not match the brief or the narration.
+- If the scene is ready to ship, reply with exactly ${REMOTION_REVIEW_MARKER} and nothing else.
+- Otherwise reply with the whole improved component in one \`\`\`tsx code block, fixing every problem you found while keeping what works.`,
+].join("\n\n");