mulmocast 2.14.0 → 2.16.0

package/lib/utils/remotion/render.d.ts

@@ -8,7 +8,7 @@ export type RemotionRenderRequest = {
     workDir: string;
     props: RemotionSceneProps;
     videoPath?: string;
-    stillPath: string;
+    stillPath?: string;
     extraStills?: {
         frame: number;
         path: string;

package/lib/utils/remotion/render.js

@@ -1,10 +1,15 @@
 import fs from "node:fs";
 import path from "node:path";
+import net from "node:net";
 import { createRequire } from "node:module";
+import { GraphAILogger } from "graphai";
 import { REMOTION_COMPOSITION_ID, REMOTION_ENTRY_FILE, buildEntrySource } from "./claude_prompt.js";
 import { missingRemotionPackages, remotionInstallCommand } from "./packages.js";
+import { isAllowedMediaDownload, withContentSecurityPolicy } from "./network_policy.js";
+import { guardSceneNetwork } from "./network_guard.js";
 // WebGL (three.js scenes) needs the ANGLE backend; the default one cannot create a context headless.
 const CHROMIUM_OPTIONS = { gl: "angle" };
+const MAX_LOGGED_URL_LENGTH = 200;
 const loadRemotion = async () => {
     try {
         const [bundler, renderer] = await Promise.all([import("@remotion/bundler"), import("@remotion/renderer")]);
@@ -40,8 +45,7 @@ const remotionNodeModulesDir = () => {
     const remotionPackageJson = createRequire(import.meta.url).resolve("remotion/package.json");
     return path.dirname(path.dirname(remotionPackageJson));
 };
-export const renderRemotionScene = async ({ workDir, props, videoPath, stillPath, extraStills = [] }) => {
-    const { bundle, renderer } = await loadRemotion();
+const bundleScene = async (bundle, workDir) => {
     const entryPoint = path.join(workDir, REMOTION_ENTRY_FILE);
     fs.writeFileSync(entryPoint, buildEntrySource());
     const nodeModulesDir = remotionNodeModulesDir();
@@ -53,13 +57,53 @@ export const renderRemotionScene = async ({ workDir, props, videoPath, stillPath
             resolve: { ...config.resolve, modules: [...(config.resolve?.modules ?? ["node_modules"]), nodeModulesDir] },
         }),
     });
-    const composition = await renderer.selectComposition({ serveUrl, id: REMOTION_COMPOSITION_ID, inputProps: props, chromiumOptions: CHROMIUM_OPTIONS });
+    const indexHtml = path.join(serveUrl, "index.html");
+    fs.writeFileSync(indexHtml, withContentSecurityPolicy(fs.readFileSync(indexHtml, "utf8")));
+    return serveUrl;
+};
+const findFreePort = () => new Promise((resolve, reject) => {
+    const server = net.createServer();
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", () => {
+        const address = server.address();
+        const port = typeof address === "object" && address ? address.port : 0;
+        server.close(() => (port > 0 ? resolve(port) : reject(new Error("remotion: could not pick a port for the bundle server"))));
+    });
+});
+const logBlockedRequest = (url) => GraphAILogger.info(`remotion: blocked a network request from the scene: ${url}`);
+// Throwing here stops remotion before it fetches, and fails the render.
+const rejectMediaDownload = (src) => {
+    if (!isAllowedMediaDownload(src))
+        throw new Error(`remotion: a scene may not load media files (${src.slice(0, MAX_LOGGED_URL_LENGTH)})`);
+    return undefined;
+};
+// The bundle server listens on the port given here, the only local origin the guard lets the scene reach.
+const renderOutputs = async (renderer, browser, serveUrl, port, outputs) => {
+    const { props, videoPath, stillPath, extraStills = [] } = outputs;
+    const shared = { serveUrl, inputProps: props, chromiumOptions: CHROMIUM_OPTIONS, puppeteerInstance: browser, port };
+    const downloadGuard = { onDownload: rejectMediaDownload };
+    const composition = await renderer.selectComposition({ ...shared, id: REMOTION_COMPOSITION_ID });
     if (videoPath) {
-        await renderer.renderMedia({ composition, serveUrl, codec: "h264", outputLocation: videoPath, inputProps: props, chromiumOptions: CHROMIUM_OPTIONS });
+        await renderer.renderMedia({ ...shared, ...downloadGuard, composition, codec: "h264", outputLocation: videoPath });
     }
-    const stills = [...extraStills, { frame: composition.durationInFrames - 1, path: stillPath }];
+    const stills = stillPath ? [...extraStills, { frame: composition.durationInFrames - 1, path: stillPath }] : extraStills;
     await stills.reduce(async (previous, still) => {
         await previous;
-        await renderer.renderStill({ composition, serveUrl, output: still.path, frame: still.frame, inputProps: props, chromiumOptions: CHROMIUM_OPTIONS });
+        await renderer.renderStill({ ...shared, ...downloadGuard, composition, output: still.path, frame: still.frame });
     }, Promise.resolve());
 };
+export const renderRemotionScene = async ({ workDir, ...outputs }) => {
+    const { bundle, renderer } = await loadRemotion();
+    const serveUrl = await bundleScene(bundle, workDir);
+    const port = await findFreePort();
+    const browser = await renderer.openBrowser("chrome", { chromiumOptions: CHROMIUM_OPTIONS });
+    try {
+        const guard = await guardSceneNetwork(browser, port, logBlockedRequest);
+        await renderOutputs(renderer, browser, serveUrl, port, outputs);
+        if (guard.failures.length > 0)
+            throw new Error(`remotion: the network guard failed: ${guard.failures.join("; ")}`);
+    }
+    finally {
+        await browser.close({ silent: true });
+    }
+};

package/lib/utils/remotion/render_frames.d.ts

@@ -0,0 +1,23 @@
+import { SceneRenderer } from "./render.js";
+export type RemotionFramesRequest = {
+    code: string;
+    durationSec: number;
+    outDir: string;
+    fps?: number;
+    width?: number;
+    height?: number;
+    fractions?: readonly number[];
+};
+export type RemotionFrame = {
+    fraction: number;
+    frame: number;
+    path: string;
+};
+export type RemotionFramesDeps = {
+    ensurePackages: () => void;
+    renderScene: SceneRenderer;
+    makeWorkDir: () => string;
+};
+export declare const framePlan: (durationInFrames: number, fractions: readonly number[], outDir: string) => RemotionFrame[];
+export declare const createRemotionFrameRenderer: (deps: RemotionFramesDeps) => (request: RemotionFramesRequest) => Promise<RemotionFrame[]>;
+export declare const renderRemotionFrames: (request: RemotionFramesRequest) => Promise<RemotionFrame[]>;

package/lib/utils/remotion/render_frames.js

@@ -0,0 +1,61 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { REMOTION_COMPONENT_FILE } from "./claude_prompt.js";
+import { REMOTION_REVIEW_FRACTIONS, frameAtFraction, toFrameCount } from "./frames.js";
+import { ensureRemotionPackages, renderRemotionScene } from "./render.js";
+const DEFAULT_FPS = 30;
+const DEFAULT_WIDTH = 1920;
+const DEFAULT_HEIGHT = 1080;
+const isValidFraction = (fraction) => Number.isFinite(fraction) && fraction >= 0 && fraction <= 1;
+const checkFractions = (fractions) => {
+    if (fractions.length === 0)
+        throw new Error("remotion: fractions is empty");
+    const invalid = fractions.filter((fraction) => !isValidFraction(fraction));
+    if (invalid.length > 0)
+        throw new Error(`remotion: fractions must be between 0 and 1 (got ${invalid.join(", ")})`);
+};
+// Named by percent so a reader can tell which moment each file is; two fractions on one percent keep their order.
+export const framePlan = (durationInFrames, fractions, outDir) => fractions.map((fraction, index) => {
+    const percent = String(Math.round(fraction * 100)).padStart(3, "0");
+    return { fraction, frame: frameAtFraction(durationInFrames, fraction), path: path.join(outDir, `frame_${index}_${percent}.png`) };
+});
+const isPositiveInteger = (value) => Number.isInteger(value) && value > 0;
+const checkCanvas = (width, height) => {
+    if (!isPositiveInteger(width) || !isPositiveInteger(height))
+        throw new Error(`remotion: width and height must be positive integers (got ${width}x${height})`);
+};
+const errorMessage = (error) => (error instanceof Error ? error.message : String(error));
+export const createRemotionFrameRenderer = (deps) => async (request) => {
+    const fps = request.fps ?? DEFAULT_FPS;
+    const props = {
+        durationInFrames: toFrameCount(request.durationSec, fps),
+        fps,
+        width: request.width ?? DEFAULT_WIDTH,
+        height: request.height ?? DEFAULT_HEIGHT,
+    };
+    const fractions = request.fractions ?? REMOTION_REVIEW_FRACTIONS;
+    checkFractions(fractions);
+    checkCanvas(props.width, props.height);
+    deps.ensurePackages();
+    const frames = framePlan(props.durationInFrames, fractions, request.outDir);
+    fs.mkdirSync(request.outDir, { recursive: true });
+    const workDir = deps.makeWorkDir();
+    try {
+        fs.writeFileSync(path.join(workDir, REMOTION_COMPONENT_FILE), request.code);
+        await deps.renderScene({ workDir, props, extraStills: frames.map(({ frame, path: file }) => ({ frame, path: file })) });
+        return frames;
+    }
+    catch (error) {
+        throw new Error(`remotion: failed to render the component's frames: ${errorMessage(error)}`, { cause: error });
+    }
+    finally {
+        fs.rmSync(workDir, { recursive: true, force: true });
+    }
+};
+// The bundle and entry files go to a temporary directory, so outDir ends up holding only the PNGs.
+export const renderRemotionFrames = createRemotionFrameRenderer({
+    ensurePackages: ensureRemotionPackages,
+    renderScene: renderRemotionScene,
+    makeWorkDir: () => fs.mkdtempSync(path.join(os.tmpdir(), "mulmocast-remotion-frames-")),
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.14.0",
+  "version": "2.16.0",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -20,6 +20,10 @@
       "types": "./lib/index.remotion.d.ts",
       "default": "./lib/index.remotion.js"
     },
+    "./remotion/guide": {
+      "types": "./lib/index.remotion_guide.d.ts",
+      "default": "./lib/index.remotion_guide.js"
+    },
     "./assets/*": "./assets/*",
     "./scripts/*": "./scripts/*",
     "./tools/complete_script": {
@@ -145,7 +149,7 @@
     "@types/jsdom": "^28.0.3",
     "@types/node": "^26.6.4",
     "@types/yargs": "^17.0.35",
-    "avatarscript": "^0.2.1",
+    "avatarscript": "^0.3.2",
     "cross-env": "^10.1.0",
     "esbuild": "^0.28.2",
     "eslint": "^10.12.0",
@@ -175,7 +179,7 @@
     "@remotion/shapes": "^4.0.533",
     "@remotion/three": "^4.0.533",
     "@remotion/transitions": "^4.0.533",
-    "avatarscript": "^0.2.1",
+    "avatarscript": "^0.3.2",
     "onnxruntime-node": "^1.30.0",
     "react": "^19.2.0",
     "react-dom": "^19.2.0",

package/scripts/templates/html.json

@@ -13,7 +13,7 @@
   "title": "[TITLE: Brief, engaging title for the topic]",
   "htmlImageParams": {
     "provider": "anthropic",
-    "model": "claude-sonnet-4-5-20250929"
+    "model": "claude-sonnet-4-6"
   },
   "lang": "en",
   "beats": [

package/scripts/templates/presentation.json

@@ -5,7 +5,7 @@
   },
   "htmlImageParams": {
     "provider": "anthropic",
-    "model": "claude-sonnet-4-5-20250929"
+    "model": "claude-sonnet-4-6"
   },
   "title": "Sample Title",
   "references": [

package/scripts/test/test_avatar.json

@@ -1,46 +1,120 @@
 {
-  "$mulmocast": { "version": "1.1" },
+  "$mulmocast": {
+    "version": "1.1"
+  },
   "lang": "ja",
   "title": "Talking avatar over slides",
-  "description": "Needs mesh-avatar-studio checked out next to this repository (for its sample avatar), and npm packages avatarscript and onnxruntime-node.",
-  "avatarParams": { "position": { "x": "82%", "y": "100%", "scale": "62%" } },
+  "description": "Loads the sample avatar ani from mulmocast-media by URL. Needs the npm packages avatarscript and onnxruntime-node.",
+  "avatarParams": {
+    "position": {
+      "x": "82%",
+      "y": "100%",
+      "scale": "62%"
+    }
+  },
   "speechParams": {
     "speakers": {
-      "Miko": {
-        "provider": "openai",
-        "voiceId": "coral",
-        "displayName": { "ja": "ミコ" },
-        "avatar": { "source": "../../../mesh-avatar-studio/samples/miko-qipao" }
+      "Ani": {
+        "provider": "gemini",
+        "voiceId": "Leda",
+        "speechOptions": {
+          "instruction": "Speak in a slightly high-pitched, curt tone with sudden flustered shifts—like a tsundere anime girl."
+        },
+        "displayName": {
+          "ja": "アニ"
+        },
+        "avatar": {
+          "source": "https://raw.githubusercontent.com/receptron/mulmocast-media/main/avatars/ani"
+        }
       }
     }
   },
   "beats": [
     {
-      "speaker": "Miko",
-      "text": "みなさん、こんにちは！今日は、スライドの上でしゃべるアバターを紹介します。",
-      "image": { "type": "markdown", "markdown": ["# Talking avatars", "", "- MulmoScript + AvatarScript", "- スライドの上でアバターが話す"] },
-      "avatarParams": { "emotion": "happy", "motions": [{ "motion": "nod", "at": "こんにちは" }] }
+      "speaker": "Ani",
+      "text": "べ、別にあなたのために説明するわけじゃないんだからね！今日は、スライドの上でしゃべるアバターを紹介してあげる。",
+      "image": {
+        "type": "markdown",
+        "markdown": ["# Talking avatars", "", "- MulmoScript + AvatarScript", "- スライドの上でアバターが話す"]
+      },
+      "avatarParams": {
+        "emotion": "happy",
+        "motions": [
+          {
+            "motion": "nod",
+            "at": "紹介"
+          }
+        ]
+      }
+    },
+    {
+      "speaker": "Ani",
+      "text": "このアバターはね、メッシュ・アバター・スタジオで、たった一枚のイラストから作ったの。それを、マルモキャストの上で動かしてるのよ。",
+      "image": {
+        "type": "markdown",
+        "markdown": ["# How it's made", "", "- mesh-avatar-studio: 1枚のイラスト → メッシュアバター", "- MulmoCast + AvatarScript: 台本 → 音声 → 口パク動画"]
+      },
+      "avatarParams": {
+        "emotion": "relaxed",
+        "motions": [
+          {
+            "motion": "nod",
+            "at": "マルモキャスト"
+          }
+        ]
+      }
     },
     {
-      "speaker": "Miko",
-      "text": "位置と大きさは、台本全体で決めておいて、ビートごとに変えられます。このスライドでは、大きくなっています。",
-      "image": { "type": "markdown", "markdown": ["# Position and size", "", "- 台本全体: `avatarParams.position`", "- ビートごとに上書き"] },
-      "avatarParams": { "emotion": "relaxed", "position": { "x": "76%", "scale": "82%" }, "motions": [{ "motion": "tilt", "at": "大きく" }] }
+      "speaker": "Ani",
+      "text": "位置と大きさは、台本全体で決めておいて、ビートごとに変えられるの。ほら、大きくなったでしょ。ちゃんと見なさいよね！",
+      "image": {
+        "type": "markdown",
+        "markdown": ["# Position and size", "", "- 台本全体: `avatarParams.position`", "- ビートごとに上書き"]
+      },
+      "avatarParams": {
+        "emotion": "relaxed",
+        "position": {
+          "x": "76%",
+          "scale": "82%"
+        },
+        "motions": [
+          {
+            "motion": "tilt",
+            "at": "大きく"
+          }
+        ]
+      }
     },
     {
-      "speaker": "Miko",
-      "text": "こちらでは、左側に小さく移動しました。",
+      "speaker": "Ani",
+      "text": "ちょ、ちょっと！今度は左に小さくされちゃったんだけど！",
       "image": {
         "type": "markdown",
         "markdown": ["# Moved", "", "```json", "\"avatarParams\": {", "  \"position\": { \"x\": \"18%\", \"scale\": \"48%\" }", "}", "```"]
       },
-      "avatarParams": { "emotion": "surprised", "position": { "x": "18%", "scale": "48%" }, "motions": [{ "motion": "surprise" }] }
+      "avatarParams": {
+        "emotion": "surprised",
+        "position": {
+          "x": "18%",
+          "scale": "48%"
+        },
+        "motions": [
+          {
+            "motion": "surprise"
+          }
+        ]
+      }
     },
     {
-      "speaker": "Miko",
-      "text": "最後は、元の位置に戻ります。それでは、始めましょう！",
-      "image": { "type": "markdown", "markdown": ["# Let's start"] },
-      "avatarParams": { "emotion": "happy" }
+      "speaker": "Ani",
+      "text": "最後は元の位置に戻るわ。べ、別に、こんなことが簡単にできるって、自慢してるわけじゃないんだからね！",
+      "image": {
+        "type": "markdown",
+        "markdown": ["# 簡単…でしょ？"]
+      },
+      "avatarParams": {
+        "emotion": "angry"
+      }
     }
   ]
 }

package/scripts/test/test_genai.json

@@ -59,7 +59,7 @@
       "text": "movie generated by veo3",
       "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
       "movieParams": {
-        "model": "veo-3.0-generate-001"
+        "model": "veo-3.1-lite-generate-preview"
       }
     },
     {
@@ -80,7 +80,7 @@
       },
       "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
       "movieParams": {
-        "model": "veo-3.0-generate-001"
+        "model": "veo-3.1-lite-generate-preview"
       }
     }
   ]

package/scripts/test/test_gpt_image.json

@@ -7,7 +7,7 @@
   "title": "Testing OpenAI's new Image Generation",
   "description": "Hello",
   "imageParams": {
-    "model": "gpt-image-1",
+    "model": "gpt-image-2.5-sunburst",
     "style": "<style>Japanese animation with soft watercolor backgrounds, characters with simple rounded faces, large expressive eyes, small nose and mouth, soft jawlines, minimalist facial features, pastel color palette, detailed natural environments, whimsical magical elements, hand-drawn aesthetic, gentle lighting, flowing movement in hair and clothing, nostalgic countryside scenery with fantasy elements."
   },
   "speechParams": {

package/scripts/test/test_replicate.json

@@ -96,12 +96,12 @@
       }
     },
     {
-      "id": "kling-v1.6-pro",
-      "text": "kwaivgi/kling-v1.6-pro with gpt4-image",
+      "id": "kling-v2.1-start-image",
+      "text": "kwaivgi/kling-v2.1 with gpt4-image",
       "imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
       "moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
       "movieParams": {
-        "model": "kwaivgi/kling-v1.6-pro"
+        "model": "kwaivgi/kling-v2.1"
       }
     },
     {

package/scripts/test/test_vertexai.json

@@ -24,7 +24,7 @@
   },
   "movieParams": {
     "provider": "google",
-    "model": "veo-2.0-generate-001",
+    "model": "veo-3.1-generate-001",
     "vertexai_project": "YOUR_PROJECT_ID",
     "vertexai_location": "us-central1"
   },
@@ -76,7 +76,7 @@
       "text": "TTS: Gemini 2.5 Pro TTS Kore. Movie: Veo 3 override.",
       "moviePrompt": "Ocean waves crashing on a beach at sunset",
       "movieParams": {
-        "model": "veo-3.0-generate-001",
+        "model": "veo-3.1-generate-001",
         "vertexai_project": "YOUR_PROJECT_ID",
         "vertexai_location": "us-central1"
       }
@@ -101,7 +101,7 @@
       },
       "moviePrompt": "The robot turns its head and waves at the camera",
       "movieParams": {
-        "model": "veo-3.0-generate-001",
+        "model": "veo-3.1-generate-001",
         "vertexai_project": "YOUR_PROJECT_ID",
         "vertexai_location": "us-central1"
       },