cozyclay 1.9.0 → 2.0.0

package/tools/gt-render/render.mjs

@@ -0,0 +1,514 @@
+#!/usr/bin/env node
+/**
+ * Ground-truth renderer (#428): cskel27 motion npz -> GVHMR-ready videos
+ * rendered by the real CozyClay Studio from ONE fixed, known camera, with the
+ * exact camera and joint ground truth written next to every video.
+ *
+ *   node tools/gt-render/render.mjs --out <dir> [options] <motion.npz ...>
+ *
+ * Per motion it starts (or reuses, --url) a Vite dev server of this checkout
+ * and a headless Chrome, loads the take through `?motion=`, and:
+ *   1. pre-pass: scrubs every timeline frame, CPU-skins every drawable vertex
+ *      of the rig and reduces it (plus the joints) to five support values;
+ *   2. solves the closest static camera at the requested azimuth/elevation and
+ *      integer GVHMR f-mm that keeps every vertex of every frame inside the
+ *      margin box (camera-math.mjs), identical for all variants;
+ *   3. renders every timeline frame at 832x480 through the Studio's export
+ *      capture (`window.__cozyclay.captureFraming`), encodes H.264 yuv420p
+ *      24 fps, and writes camera.json, joints.json and meta.json.
+ *
+ * Variants: `shaded` (part colours, shaded palette, what GVHMR's palette
+ * detector expects), `skin` (part colours off), `hue+N` / `hue-N` (the shaded
+ * frames through ffmpeg's `hue=h=N` filter; it rotates the chroma plane of
+ * EVERY pixel by the same N degrees, so all hues shift uniformly and neutral
+ * greys stay put. It does not reshuffle parts: the palette's relative layout is
+ * preserved).
+ *
+ * Output per motion:
+ *   <out>/<motion>/plate.png           the same framing with the character hidden
+ *   <out>/<motion>/mask/NNNNNN.png     exact per-frame silhouette (gray, 255 = character),
+ *                                      the character re-drawn unlit in one colour
+ *   <out>/<motion>/<variant>/{video.mp4,camera.json,joints.json,meta.json}
+ *
+ * Scoring renders (#431): `--camera <camera.json>` renders through that exact
+ * camera instead of solving one; `--no-video` captures only the silhouette
+ * masks and writes <out>/<motion>/{camera.json,joints.json,meta.json,mask/};
+ * `--transform` places the take in the scene first (take-transform.mjs, the
+ * Studio's sceneCalibration semantics); `--box` also writes contact.json, the
+ * per-frame signed distance of the closest skinned vertex to a scene box.
+ */
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { appendFileSync, copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { basename, dirname, join, resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { installSignalCleanup } from "../process-supervisor.mjs";
+import { openStudioClean, startChrome, startVite, terminateAll, waitFor } from "./browser.mjs";
+import { decodeMotionNpz } from "../../src/ardy/npz.js";
+import { writeNpz } from "../ardy/npz.mjs";
+import { boxContactValues, contactFromSigned, parseBox } from "../bench/metrics.mjs";
+import { buildCamera, cameraFromRecord, marginSlopes, mergeSupports, projectPoint, supportArgs, supportValues, translateCamera } from "./camera-math.mjs";
+import { installPageHelpers, MASK_RGB, RIG_JOINTS } from "./page.mjs";
+import { boxCorners, parseSceneBox, sceneBoxRecord } from "./scene-box.mjs";
+import { isIdentityTransform, parseTransform, takeToNpzMembers, transformTake } from "./take-transform.mjs";
+
+const ROOT = resolve(dirname(new URL(import.meta.url).pathname), "../..");
+const WIDTH = 832;
+const HEIGHT = 480;
+const FPS = 24;
+const CACHE_DIR = join(ROOT, "node_modules/.cache/gt-render");
+
+const USAGE = `usage: node tools/gt-render/render.mjs --out <dir> [options] <motion.npz ...>
+
+  --out <dir>          output root; each motion goes to <dir>/<npz basename>/<variant>/
+  --variants <list>    comma list of shaded, skin, hue+N, hue-N (default shaded)
+  --f-mm <int>         GVHMR --f-mm the camera is built for (default 35)
+  --azimuth <deg>      camera azimuth around the subject; 0 = on +Z looking -Z (default 0)
+  --elevation <deg>    camera elevation; positive looks down (default 5)
+  --margin <frac>      fraction of each image side kept clear of the body (default 0.08)
+  --crf <int>          libx264 CRF (default 12)
+  --keep-frames        keep the rendered PNG frames in <variant>/frames/
+  --url <origin>       reuse a running Vite dev server of THIS checkout (default: start one)
+  --port <n>           Vite port when starting one (default 5191)
+  --cdp-port <n>       headless Chrome DevTools port (default 9231)
+  --camera <file>      render through this camera.json (from an earlier render) instead of
+                       solving one; --f-mm/--azimuth/--elevation/--margin are then unused
+                       and a joint near or past the edge is reported, not fatal
+  --no-video           masks + joints only: <dir>/<motion>/{camera.json,joints.json,meta.json,mask/}
+  --transform <json>   place the take first, Studio sceneCalibration semantics:
+                       '{"yawDeg":0,"offsetX":0,"offsetY":0,"offsetZ":0,"scale":1}'
+                       (yaw about the frame-0 anchor, offsets in scene metres)
+  --scene-box <json>   visible Studio cube: {x,z,rot,sx,sy,sz}; base at floor, sizes in metres
+  --export-vertices   write vertices.f32 (frame/vertex/xyz, little-endian float32) + vertices.json
+  --box <json>         '{"min":[x,y,z],"max":[x,y,z]}' scene box; writes <dir>/<motion>/contact.json
+                       (closest skinned vertex per frame: distance / penetration)
+
+writes <dir>/<motion>/{plate.png,mask/NNNNNN.png} and
+<dir>/<motion>/<variant>/{video.mp4,camera.json,joints.json,meta.json}`;
+
+function parseOptions(argv) {
+	const { values, positionals } = parseArgs({
+		args: argv,
+		allowPositionals: true,
+		options: {
+			out: { type: "string" },
+			variants: { type: "string", default: "shaded" },
+			"f-mm": { type: "string", default: "35" },
+			azimuth: { type: "string", default: "0" },
+			elevation: { type: "string", default: "5" },
+			margin: { type: "string", default: "0.08" },
+			crf: { type: "string", default: "12" },
+			"keep-frames": { type: "boolean", default: false },
+			url: { type: "string" },
+			port: { type: "string", default: "5191" },
+			"cdp-port": { type: "string", default: "9231" },
+			camera: { type: "string" },
+			"no-video": { type: "boolean", default: false },
+			transform: { type: "string" },
+			box: { type: "string" },
+			"scene-box": { type: "string" },
+			"export-vertices": { type: "boolean", default: false },
+			help: { type: "boolean", short: "h", default: false },
+		},
+	});
+	if (values.help) {
+		console.log(USAGE);
+		process.exit(0);
+	}
+	const fail = (message) => {
+		console.error(`${message}\n\n${USAGE}`);
+		process.exit(2);
+	};
+	if (!values.out) fail("--out is required");
+	if (!positionals.length) fail("at least one motion npz is required");
+	const number = (name, integer = false) => {
+		const value = Number(values[name]);
+		if (!Number.isFinite(value) || (integer && !Number.isInteger(value))) fail(`--${name} must be ${integer ? "an integer" : "a number"}, got ${values[name]}`);
+		return value;
+	};
+	const variants = values.variants.split(",").map((entry) => entry.trim()).filter(Boolean).map((name) => {
+		if (name === "shaded" || name === "skin") return { name, kind: name };
+		const hue = /^hue([+-]\d+(?:\.\d+)?)$/.exec(name);
+		if (hue) return { name, kind: "hue", degrees: Number(hue[1]) };
+		return fail(`unknown variant ${name}`);
+	});
+	if (new Set(variants.map((variant) => variant.name)).size !== variants.length) fail("duplicate variant");
+	const motions = positionals.map((file) => resolve(file));
+	for (const file of motions) if (!existsSync(file)) fail(`no such motion: ${file}`);
+	const names = motions.map((file) => basename(file, ".npz"));
+	if (new Set(names).size !== names.length) fail(`two motions share an output name: ${names.join(", ")}`);
+	let camera = null;
+	if (values.camera) {
+		const file = resolve(values.camera);
+		if (!existsSync(file)) fail(`no such camera: ${file}`);
+		try {
+			camera = cameraFromRecord(JSON.parse(readFileSync(file, "utf8")));
+		} catch (error) {
+			fail(`--camera ${file}: ${error.message}`);
+		}
+		if (camera.width !== WIDTH || camera.height !== HEIGHT) fail(`--camera is ${camera.width}x${camera.height}; the renderer draws ${WIDTH}x${HEIGHT}`);
+		camera.source = file;
+	}
+	let transform = null;
+	if (values.transform) {
+		try {
+			transform = parseTransform(values.transform);
+		} catch (error) {
+			fail(`--transform: ${error.message}`);
+		}
+	}
+	let box = null;
+	if (values.box) {
+		try {
+			box = parseBox(values.box);
+		} catch (error) {
+			fail(`--box: ${error.message}`);
+		}
+	}
+	return {
+		camera,
+		sceneBox: values["scene-box"] ? parseSceneBox(values["scene-box"]) : null,
+		exportVertices: values["export-vertices"],
+		noVideo: values["no-video"],
+		transform,
+		box,
+		out: resolve(values.out),
+		variants,
+		fMm: number("f-mm", true),
+		azimuthDeg: number("azimuth"),
+		elevationDeg: number("elevation"),
+		margin: number("margin"),
+		crf: number("crf", true),
+		keepFrames: values["keep-frames"],
+		url: values.url?.replace(/\/$/, "") ?? null,
+		port: number("port", true),
+		cdpPort: number("cdp-port", true),
+		motions,
+	};
+}
+
+const round = (value, digits) => Math.round(value * 10 ** digits) / 10 ** digits;
+const triples = (flat, digits = 6) => Array.from({ length: flat.length / 3 }, (_, i) => [round(flat[i * 3], digits), round(flat[i * 3 + 1], digits), round(flat[i * 3 + 2], digits)]);
+
+function toolVersion() {
+	const pkg = JSON.parse(readFileSync(join(ROOT, "package.json"), "utf8"));
+	try {
+		const commit = execFileSync("git", ["rev-parse", "--short", "HEAD"], { cwd: ROOT, encoding: "utf8" }).trim();
+		const dirty = execFileSync("git", ["status", "--porcelain"], { cwd: ROOT, encoding: "utf8" }).trim() !== "";
+		return `${pkg.version}+${commit}${dirty ? "-dirty" : ""}`;
+	} catch {
+		return pkg.version;
+	}
+}
+
+function encode(framesDir, output, { crf, filter }) {
+	execFileSync("ffmpeg", [
+		"-hide_banner", "-loglevel", "error", "-y",
+		"-framerate", String(FPS), "-i", join(framesDir, "%06d.png"),
+		...(filter ? ["-vf", filter] : []),
+		"-c:v", "libx264", "-preset", "slow", "-crf", String(crf),
+		"-pix_fmt", "yuv420p", "-colorspace", "smpte170m", "-color_primaries", "smpte170m", "-color_trc", "smpte170m", "-color_range", "tv",
+		"-r", String(FPS), "-movflags", "+faststart", output,
+	], { stdio: ["ignore", "inherit", "inherit"] });
+}
+
+/** Magenta-on-scene mask captures -> gray PNGs, 255 where the pixel is at
+ * least half character (the MSAA edge blends linearly with the scene). */
+function writeMasks(rawDir, maskDir) {
+	rmSync(maskDir, { recursive: true, force: true });
+	mkdirSync(maskDir, { recursive: true });
+	const [r, g, b] = MASK_RGB;
+	if (!(r === 255 && g === 0 && b === 255)) throw new Error("writeMasks expects the magenta mask colour");
+	const test = "255*gt(r(X,Y)-g(X,Y),127)*gt(b(X,Y)-g(X,Y),127)";
+	execFileSync("ffmpeg", [
+		"-hide_banner", "-loglevel", "error", "-y",
+		"-framerate", String(FPS), "-i", join(rawDir, "%06d.png"),
+		"-vf", `format=gbrp,geq=r='${test}':g='${test}':b='${test}',format=gray`,
+		"-start_number", "0", join(maskDir, "%06d.png"),
+	], { stdio: ["ignore", "inherit", "inherit"] });
+}
+
+function cameraRecord(camera, shotCam) {
+	return {
+		width: camera.width,
+		height: camera.height,
+		fovDeg: camera.fovDeg,
+		fx: camera.fx,
+		fy: camera.fy,
+		cx: camera.cx,
+		cy: camera.cy,
+		K: [[camera.fx, 0, camera.cx], [0, camera.fy, camera.cy], [0, 0, 1]],
+		gvhmrFMm: camera.gvhmrFMm,
+		position: camera.position,
+		yaw: camera.yaw,
+		pitch: camera.pitch,
+		rotationOrder: "YXZ",
+		azimuthDeg: camera.azimuthDeg,
+		elevationDeg: camera.elevationDeg,
+		margin: camera.margin,
+		bindingAxis: camera.bindingAxis,
+		near: shotCam.near,
+		far: shotCam.far,
+		static: true,
+		worldToCamera: camera.worldToCameraCv,
+		worldToCameraGl: camera.worldToCameraGl,
+		cameraToWorldRotationThree: [camera.cameraToWorldRotation.slice(0, 3), camera.cameraToWorldRotation.slice(3, 6), camera.cameraToWorldRotation.slice(6, 9)],
+		convention: {
+			world: "CozyClay Studio / Three.js scene: right-handed, +Y up, metres. The character's front is +Z at azimuth 0.",
+			worldToCamera: "OpenCV: [X Y Z 1]^T = worldToCamera * [x y z 1]^T, camera x right, y DOWN, z forward (depth).",
+			projection: "u = fx * X / Z + cx, v = fy * Y / Z + cy. Square pixels, no distortion, no skew.",
+			image: "Pixels continuous from the top-left corner of the top-left pixel: u right, v DOWN; pixel (col,row) covers [col,col+1)x[row,row+1) and its centre is (col+0.5,row+0.5); cx = width/2, cy = height/2.",
+			worldToCameraGl: "Three/OpenGL view matrix (camera.matrixWorldInverse): x right, y up, looking down -z.",
+			threeCamera: "PerspectiveCamera(fov = fovDeg vertical, aspect = width/height), position, rotation.set(pitch, yaw, 0) with order YXZ (radians); rendered through the Studio export capture.",
+			gvhmr: "Run GVHMR with --f-mm gvhmrFMm: f_px = sqrt(W^2 + H^2) / sqrt(24^2 + 36^2) * f_mm equals fx = fy here.",
+		},
+	};
+}
+
+async function main() {
+	const options = parseOptions(process.argv.slice(2));
+	const children = [];
+	const cleanups = [];
+	const runCleanups = () => {
+		for (const cleanup of cleanups.splice(0)) cleanup();
+	};
+	const removeSignals = installSignalCleanup(() => children, () => {
+		runCleanups();
+		process.exit(130);
+	});
+	const version = toolVersion();
+	try {
+		const base = options.url ?? await startVite({ root: ROOT, port: options.port, children });
+		const cdp = await startChrome({ port: options.cdpPort, children, cleanups });
+		mkdirSync(CACHE_DIR, { recursive: true });
+		for (const file of options.motions) {
+			await renderMotion({ cdp, base, file, options, version });
+		}
+		cdp.close();
+	} finally {
+		removeSignals();
+		await terminateAll(children);
+		runCleanups();
+	}
+}
+
+/** The npz the Studio loads: the file itself, or the file with --transform's
+ * scale/offsetY/yaw baked in (take-transform.mjs). Either way it is put in
+ * the (gitignored) node_modules cache, where Vite can serve it to ?motion=. */
+async function stageTake(file, bytes, transform) {
+	if (isIdentityTransform(transform)) {
+		const sha256 = createHash("sha256").update(bytes).digest("hex");
+		const cached = join(CACHE_DIR, `${sha256.slice(0, 16)}.npz`);
+		if (!existsSync(cached)) copyFileSync(file, cached);
+		return { sha256, cacheName: `${sha256.slice(0, 16)}.npz`, sceneOffset: { x: 0, y: 0, z: 0 } };
+	}
+	const decoded = await decodeMotionNpz(new Uint8Array(bytes));
+	const { motion, sceneOffset } = transformTake(decoded, transform);
+	const staging = join(CACHE_DIR, `.staging-${process.pid}.npz`);
+	writeNpz(staging, takeToNpzMembers(motion));
+	const sha256 = createHash("sha256").update(readFileSync(staging)).digest("hex");
+	const cacheName = `${sha256.slice(0, 16)}.npz`;
+	copyFileSync(staging, join(CACHE_DIR, cacheName));
+	rmSync(staging, { force: true });
+	return { sha256, cacheName, sceneOffset };
+}
+
+async function renderMotion({ cdp, base, file, options, version }) {
+	const started = Date.now();
+	const bytes = readFileSync(file);
+	const sha256 = createHash("sha256").update(bytes).digest("hex");
+	const motionName = basename(file, ".npz");
+	const staged = await stageTake(file, bytes, options.transform);
+	// The page renders the take where playback anchors it; a scene offset is
+	// the camera moved the opposite way, and every reported point is shifted
+	// back into the scene by `offset`.
+	const offset = staged.sceneOffset;
+	const negOffset = { x: -offset.x, y: -offset.y, z: -offset.z };
+	const pageBox = options.box ? { min: options.box.min.map((v, i) => v - [offset.x, offset.y, offset.z][i]), max: options.box.max.map((v, i) => v - [offset.x, offset.y, offset.z][i]) } : null;
+	const motionUrl = `/node_modules/.cache/gt-render/${staged.cacheName}`;
+	const studioUrl = `${base}/app/?motion=${encodeURIComponent(motionUrl)}`;
+	const log = (message) => console.log(`[${motionName}] ${message}`);
+
+	await openStudioClean(cdp, studioUrl);
+	await waitFor("studio motion load", () => cdp.evaluate("!!(window.__cozyclay && window.__cozyclay.motion && window.__cozyclay.rigA && typeof window.__cozyclay.captureFraming === 'function')"), { timeoutMs: 180000, intervalMs: 250 });
+	await cdp.evaluate(`(${installPageHelpers.toString()})(${supportValues.toString()}, ${JSON.stringify(RIG_JOINTS)}, ${JSON.stringify(MASK_RGB)}, ${boxContactValues.toString()})`);
+	const state = await cdp.evaluate("window.__gtRender.state()");
+	if (state.motionUrl !== motionUrl) throw new Error(`Studio loaded ${state.motionUrl} instead of ${motionUrl}`);
+	if (state.fps !== FPS) throw new Error(`timeline runs at ${state.fps} fps, expected ${FPS}`);
+	const frames = state.frameCount;
+	log(`loaded ${frames} frames @ ${state.fps} fps, model ${state.characterModel}`);
+	const motionDir = join(options.out, motionName);
+	mkdirSync(motionDir, { recursive: true });
+	if (options.exportVertices) writeFileSync(join(motionDir, "vertices.f32"), "");
+	const sceneRecord = options.sceneBox ? { ...sceneBoxRecord(options.sceneBox), studio: await cdp.evaluate(`window.__gtRender.placeBox(${JSON.stringify(options.sceneBox)}, ${JSON.stringify(offset)})`) } : null;
+
+	// Pre-pass: the camera must see every vertex of every frame (solved
+	// camera), and --box needs every vertex anyway.
+	const geometry = { width: WIDTH, height: HEIGHT, fMm: options.fMm, azimuthDeg: options.azimuthDeg, elevationDeg: options.elevationDeg, margin: options.margin };
+	const givenMargin = options.camera ? (options.camera.margin ?? options.margin) : options.margin;
+	const { R, kx, ky } = options.camera
+		? { R: options.camera.cameraToWorldRotation, ...marginSlopes(options.camera, givenMargin) }
+		: supportArgs(geometry);
+	await cdp.evaluate(`window.__gtRender.setOrientation(${JSON.stringify(R)}, ${kx}, ${ky})`);
+	const sampleOptions = { vertices: !options.camera, box: pageBox, exportVertices: options.exportVertices };
+	const supports = [];
+	const prepassJoints = [];
+	const contacts = [];
+	let vertexCount = 0;
+	for (let frame = 0; frame < frames; frame += 1) {
+		const sample = await cdp.evaluate(`window.__gtRender.sample(${frame}, ${JSON.stringify(sampleOptions)})`);
+		if (sample.support) supports.push(sample.support);
+		if (sample.vertices) {
+			const points = Buffer.allocUnsafe(sample.vertices.length * 4);
+			sample.vertices.forEach((v, i) => points.writeFloatLE(v + [offset.x, offset.y, offset.z][i % 3], i * 4));
+			appendFileSync(join(motionDir, "vertices.f32"), points);
+		}
+		prepassJoints.push(sample.joints);
+		if (sample.contact) contacts.push(sample.contact);
+		vertexCount = sample.vertexCount;
+	}
+	if (options.exportVertices) writeFileSync(join(motionDir, "vertices.json"), JSON.stringify({ frames, fps: FPS, vertexCount, file: "vertices.f32", layout: "frame,vertex,xyz; float32 LE; world metres" }));
+	if (options.sceneBox && !options.camera) supports.push(supportValues(boxCorners(options.sceneBox).flatMap(p => p.map((v, i) => v - [offset.x, offset.y, offset.z][i])), R, kx, ky));
+	// pageCamera renders the anchored take; camera (scene) is what is reported.
+	const pageCamera = options.camera ? translateCamera(options.camera, negOffset) : buildCamera({ ...geometry, support: mergeSupports(supports) });
+	const camera = options.camera ?? translateCamera(pageCamera, offset);
+	const framing = { pos: pageCamera.position, yaw: pageCamera.yaw, pitch: pageCamera.pitch, fovDeg: pageCamera.fovDeg };
+	const output = { width: WIDTH, height: HEIGHT };
+	log(`camera at (${camera.position.x.toFixed(3)}, ${camera.position.y.toFixed(3)}, ${camera.position.z.toFixed(3)}), fov ${camera.fovDeg.toFixed(4)} deg, f ${camera.fy.toFixed(3)} px, ${options.camera ? `given (${options.camera.source})` : `${camera.bindingAxis}-bound`}, ${vertexCount} vertices/frame`);
+
+	// Scene joints (page + offset) projected with camera.json. A solved
+	// camera guarantees the margin box; a given one only reports it.
+	const sceneJoints = prepassJoints.map((joints) => triples(joints, 12).map(([x, y, z]) => [x + offset.x, y + offset.y, z + offset.z]));
+	const uv = sceneJoints.map((joints) => joints.map((point) => projectPoint(point, camera)));
+	let minMarginPx = Infinity;
+	for (const frameUv of uv) {
+		for (const [u, v, depth] of frameUv) minMarginPx = Math.min(minMarginPx, depth > 0 ? Math.min(u, v, WIDTH - u, HEIGHT - v) : -Infinity);
+	}
+	const marginPx = givenMargin * Math.min(WIDTH, HEIGHT);
+	if (!options.camera && minMarginPx < marginPx - 1e-6) throw new Error(`a joint projects ${minMarginPx.toFixed(2)} px from the edge, inside the ${marginPx} px margin`);
+	if (options.camera && minMarginPx < marginPx - 1e-6) log(`warning: a joint projects ${minMarginPx.toFixed(2)} px from the edge (margin ${marginPx} px); negative = outside the image or behind the camera`);
+
+	if (!options.noVideo) writeFileSync(join(motionDir, "plate.png"), Buffer.from(await cdp.evaluate(`window.__gtRender.plate(${JSON.stringify(framing)}, ${JSON.stringify(output)})`), "base64"));
+
+	const wanted = new Set(options.noVideo ? [] : options.variants.map((variant) => variant.name));
+	const needsShadedFrames = !options.noVideo && options.variants.some((variant) => variant.kind === "hue");
+	const browserPasses = options.noVideo ? [{ name: "mask", partColours: null }] : [
+		...(wanted.has("shaded") || needsShadedFrames ? [{ name: "shaded", partColours: true }] : []),
+		...(wanted.has("skin") ? [{ name: "skin", partColours: false }] : []),
+	];
+	const checks = { prepassVsRenderJointMaxM: 0, threeVsCameraJsonMaxPx: 0 };
+	const framesDirs = {};
+	const passState = {};
+	const maskRawDir = join(motionDir, ".mask-raw");
+	rmSync(maskRawDir, { recursive: true, force: true });
+	mkdirSync(maskRawDir, { recursive: true });
+	for (const [passIndex, pass] of browserPasses.entries()) {
+		const withMask = passIndex === 0;
+		const withColour = pass.partColours !== null;
+		if (withColour) passState[pass.name] = await cdp.evaluate(`window.__gtRender.setPartColours(${pass.partColours})`);
+		const framesDir = wanted.has(pass.name) ? join(motionDir, pass.name, "frames") : join(motionDir, `.${pass.name}-frames`);
+		if (withColour) {
+			rmSync(framesDir, { recursive: true, force: true });
+			mkdirSync(framesDir, { recursive: true });
+			framesDirs[pass.name] = framesDir;
+		}
+		for (let frame = 0; frame < frames; frame += 1) {
+			const captured = await cdp.evaluate(`window.__gtRender.capture(${frame}, ${JSON.stringify(framing)}, ${JSON.stringify(output)}, ${withMask}, ${withColour})`);
+			if (withColour) writeFileSync(join(framesDir, `${String(frame).padStart(6, "0")}.png`), Buffer.from(captured.png, "base64"));
+			if (withMask) writeFileSync(join(maskRawDir, `${String(frame).padStart(6, "0")}.png`), Buffer.from(captured.mask, "base64"));
+			for (let i = 0; i < captured.joints.length; i += 1) {
+				checks.prepassVsRenderJointMaxM = Math.max(checks.prepassVsRenderJointMaxM, Math.abs(captured.joints[i] - prepassJoints[frame][i]));
+			}
+			triples(captured.joints, 12).forEach((point, j) => {
+				const [u, v] = projectPoint(point, pageCamera);
+				checks.threeVsCameraJsonMaxPx = Math.max(checks.threeVsCameraJsonMaxPx, Math.abs(u - captured.threeUv[j * 2]), Math.abs(v - captured.threeUv[j * 2 + 1]));
+			});
+		}
+		log(`rendered ${pass.name}: ${frames} frames`);
+	}
+	writeMasks(maskRawDir, join(motionDir, "mask"));
+	rmSync(maskRawDir, { recursive: true, force: true });
+	// The pose must not depend on the pass: joints.json is the pre-pass pose.
+	if (checks.prepassVsRenderJointMaxM > 1e-6) throw new Error(`rendered pose differs from the pre-pass by ${checks.prepassVsRenderJointMaxM} m`);
+	if (checks.threeVsCameraJsonMaxPx > 1e-3) throw new Error(`camera.json projection differs from the Three capture camera by ${checks.threeVsCameraJsonMaxPx} px`);
+
+	const cameraJson = cameraRecord(camera, passState[browserPasses[0].name]?.shotCam ?? state.shotCam);
+	if (options.camera) cameraJson.source = options.camera.source;
+	const placement = {
+		transform: options.transform ?? null,
+		sceneOffset: offset,
+		anchor: state.motionAnchor,
+		note: "Studio placement: playback anchors the take's frame-0 root XZ on the subject (anchor) with the Character group's yaw. --transform: scale/offsetY/yaw baked into the loaded npz (yaw about the anchor), offsetX/offsetZ applied as the opposite camera move; joints, uv and contact are in the scene frame with the offset included.",
+		renderedNpzSha256: staged.sha256,
+	};
+	const contactJson = options.box ? {
+		box: options.box,
+		basis: "skinned-vertices",
+		vertexCount,
+		convention: "signedDistanceM: closest skinned vertex to the box, positive outside, negative inside (depth to the nearest face). minDistanceM = max(0, signed), maxPenetrationM = max(0, -signed). Scene metres, the placement's offset included.",
+		frames: contacts.map((contact) => ({ signedDistanceM: round(contact.minSignedDistance, 6), ...Object.fromEntries(Object.entries(contactFromSigned(contact.minSignedDistance)).map(([key, value]) => [key, round(value, 6)])), verticesInside: contact.insideCount })),
+	} : null;
+	if (contactJson) writeFileSync(join(motionDir, "contact.json"), `${JSON.stringify(contactJson)}\n`);
+	const jointsJson = {
+		convention: "world: metres in the camera.json world frame; uv: pixels in the camera.json image convention (origin top-left, v down), projected with camera.json; depth: camera-space Z in metres.",
+		fps: FPS,
+		frames,
+		joints: RIG_JOINTS.map(({ bone, cskel27 }) => ({ name: bone.replace(/^mixamorig/, ""), bone, cskel27 })),
+		note: "Rig bones of the rendered character (the pose actually drawn). Mixamo Spine/Spine1/Spine2 correspond to cskel27 Spine1/Spine2/Spine3 (src/ardy/playback.js); the Mixamo body is not congruent with cskel27 and playback scales the take's root travel to the rig's leg length, so these are NOT the npz's posed_joints: they differ by centimetres, growing with distance travelled (about 1% of it on walk-then-stop). They are the truth for what the video shows.",
+		world: sceneJoints.map((joints) => joints.map((point) => point.map((value) => round(value, 6)))),
+		uv: uv.map((frameUv) => frameUv.map(([u, v]) => [round(u, 4), round(v, 4)])),
+		depth: uv.map((frameUv) => frameUv.map(([, , z]) => round(z, 6))),
+	};
+	const shared = {
+		tool: "tools/gt-render/render.mjs",
+		toolVersion: version,
+		createdAt: new Date().toISOString(),
+		source: { path: file, sha256, bytes: bytes.length },
+		motionName,
+		frames,
+		fps: FPS,
+		placement,
+		camera: options.camera ? { source: options.camera.source, given: true } : { given: false },
+		video: { file: "video.mp4", width: WIDTH, height: HEIGHT, codec: "h264 (libx264)", pixFmt: "yuv420p", colorspace: "smpte170m, tv range", crf: options.crf },
+		characterModel: state.characterModel,
+		characterScale: state.characterScale,
+		studio: { url: studioUrl, captureHook: "window.__cozyclay.captureFraming" },
+		plate: "../plate.png (same camera, character hidden)",
+		mask: "../mask/%06d.png (8-bit gray, 255 = character: the frame re-rendered with the character unlit in one colour, >= 50% pixel coverage)",
+		checks: {
+			...checks,
+			minJointEdgeDistancePx: minMarginPx,
+			framingVerticesPerFrame: vertexCount,
+		},
+	};
+	if (sceneRecord) writeFileSync(join(motionDir, "scene.json"), `${JSON.stringify(sceneRecord, null, 2)}\n`);
+	if (options.noVideo) {
+		const { video, plate, ...rest } = shared;
+		writeFileSync(join(motionDir, "camera.json"), `${JSON.stringify(cameraJson, null, "\t")}\n`);
+		writeFileSync(join(motionDir, "joints.json"), `${JSON.stringify(jointsJson)}\n`);
+		writeFileSync(join(motionDir, "meta.json"), `${JSON.stringify({ ...rest, mask: "mask/%06d.png (8-bit gray, 255 = character: the frame re-rendered with the character unlit in one colour, >= 50% pixel coverage)", variant: null, noVideo: true, contact: contactJson ? "contact.json" : null }, null, "\t")}\n`);
+		log(`wrote joints + masks (no video)`);
+	}
+	for (const variant of options.noVideo ? [] : options.variants) {
+		const dir = join(motionDir, variant.name);
+		mkdirSync(dir, { recursive: true });
+		if (sceneRecord) writeFileSync(join(dir, "scene.json"), `${JSON.stringify(sceneRecord, null, 2)}\n`);
+		const filter = variant.kind === "hue" ? `hue=h=${variant.degrees}` : null;
+		encode(variant.kind === "hue" ? framesDirs.shaded : framesDirs[variant.name], join(dir, "video.mp4"), { crf: options.crf, filter });
+		writeFileSync(join(dir, "camera.json"), `${JSON.stringify(cameraJson, null, "\t")}\n`);
+		writeFileSync(join(dir, "joints.json"), `${JSON.stringify(jointsJson)}\n`);
+		const variantDetail = variant.kind === "hue"
+			? { partColours: "shaded", postFilter: filter, note: `ffmpeg hue filter on the shaded frames: rotates every pixel's chroma by ${variant.degrees} degrees, so all hues shift uniformly and greys are unchanged.` }
+			: { partColours: variant.kind === "shaded" ? "shaded" : "off", postFilter: null };
+		writeFileSync(join(dir, "meta.json"), `${JSON.stringify({ ...shared, variant: variant.name, variantDetail }, null, "\t")}\n`);
+		log(`wrote ${variant.name}`);
+	}
+	if (!options.keepFrames) for (const dir of Object.values(framesDirs)) rmSync(dir, { recursive: true, force: true });
+	else if (framesDirs.shaded && !wanted.has("shaded")) rmSync(framesDirs.shaded, { recursive: true, force: true });
+	log(`done in ${((Date.now() - started) / 1000).toFixed(1)} s`);
+}
+
+main().catch((error) => {
+	console.error(error?.stack || String(error));
+	process.exitCode = 1;
+});

package/tools/gt-render/scene-box.mjs

@@ -0,0 +1,31 @@
+import { projectPoint } from "./camera-math.mjs";
+import { elementByPath } from "../../src/studio-elements.js";
+
+/** Studio cube is 1 m on each axis, origin at the floor/footprint centre. */
+export function parseSceneBox(value) {
+	const b = typeof value === "string" ? JSON.parse(value) : value;
+	const out = { x: b?.x, z: b?.z, rot: b?.rot ?? 0, sx: b?.sx ?? b?.scaleX, sy: b?.sy ?? b?.scaleY, sz: b?.sz ?? b?.scaleZ };
+	if (!Object.values(out).every(Number.isFinite) || ![out.sx, out.sy, out.sz].every(v => v > 0)) throw new Error("scene-box needs finite x,z,rot and positive sx,sy,sz (scaleX/Y/Z also accepted)");
+	const size = elementByPath("object.scale"), position = elementByPath("object.position");
+	for (const axis of ["x", "y", "z"]) if (out[`s${axis}`] < size.min[axis] || out[`s${axis}`] > size.max[axis]) throw new Error(`scene-box s${axis} must be within Studio's ${size.min[axis]}..${size.max[axis]} m range`);
+	for (const axis of ["x", "z"]) if (out[axis] < position.min[axis] || out[axis] > position.max[axis]) throw new Error(`scene-box ${axis} exceeds Studio position bounds`);
+	out.rot = ((out.rot + 180) % 360 + 360) % 360 - 180;
+	return out;
+}
+export function boxCorners(box) {
+	const b = parseSceneBox(box), a = b.rot * Math.PI / 180, c = Math.cos(a), s = Math.sin(a);
+	return Array.from({ length: 8 }, (_, i) => {
+		const x = (i & 1 ? 1 : -1) * b.sx / 2, z = (i & 4 ? 1 : -1) * b.sz / 2;
+		return [b.x + c * x + s * z, i & 2 ? b.sy : 0, b.z - s * x + c * z];
+	});
+}
+export const BOX_EDGES = Array.from({ length: 8 }, (_, i) => [1, 2, 4].filter(bit => !(i & bit)).map(bit => [i, i | bit])).flat();
+export function sceneBoxRecord(box) {
+	const b = parseSceneBox(box);
+	return { kind: "cube", placement: b, centre: [b.x, b.sy / 2, b.z], halfExtents: [b.sx / 2, b.sy / 2, b.sz / 2], yawDeg: b.rot,
+		...(b.rot === 0 ? { min: [b.x - b.sx / 2, 0, b.z - b.sz / 2], max: [b.x + b.sx / 2, b.sy, b.z + b.sz / 2] } : {}),
+		corners: boxCorners(b), convention: "World metres, +Y up; yaw degrees about +Y. Cube base at y=0." };
+}
+export function projectBox(box, camera) {
+	return boxCorners(box).map(p => projectPoint(p, { ...camera, worldToCameraCv: camera.worldToCameraCv ?? camera.worldToCamera }));
+}

package/tools/gt-render/take-transform.mjs

@@ -0,0 +1,87 @@
+/**
+ * Scene transform of a cskel27 take for the renderer's --transform (#431),
+ * with the Studio's sceneCalibration semantics (src/App.jsx loadMotion):
+ *
+ *   scale, offsetY  applied to the take's arrays (applyMotionCalibration),
+ *                   exactly as the Studio's playback calibration does;
+ *   yawDeg          rotation about +Y through the take's frame-0 anchor. The
+ *                   Studio puts it on the Character group; here it is baked
+ *                   into the take (joints rotated by applyMotionCalibration,
+ *                   plus the root's local rotation, which that pass leaves
+ *                   alone), which is the same rigid rotation because playback
+ *                   subtracts the frame-0 root XZ before placing the rig;
+ *   offsetX/offsetZ scene metres added after the anchor. Playback anchors
+ *                   frame 0 on the subject, so an offset baked into the take
+ *                   would be cancelled; the renderer instead moves the camera
+ *                   by the opposite amount (identical character pixels) and
+ *                   adds the offset to every reported joint.
+ */
+import { applyMotionCalibration, normalizeMotionCalibration } from "../../src/ardy/motion-calibration.js";
+
+const KEYS = new Set(["scale", "yawDeg", "offsetX", "offsetY", "offsetZ"]);
+
+/** Parse and normalise a --transform JSON object; unknown keys are refused. */
+export function parseTransform(value) {
+	const raw = typeof value === "string" ? JSON.parse(value) : value;
+	if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new Error("transform must be a JSON object");
+	for (const [key, entry] of Object.entries(raw)) {
+		if (!KEYS.has(key)) throw new Error(`transform: unknown key ${key} (allowed: ${[...KEYS].join(", ")})`);
+		if (!Number.isFinite(Number(entry))) throw new Error(`transform: ${key} must be a finite number`);
+	}
+	const normalized = normalizeMotionCalibration(raw);
+	for (const key of Object.keys(raw)) {
+		const given = Number(raw[key]);
+		// normalizeMotionCalibration clamps silently; a scorer must not.
+		if (key !== "yawDeg" && Math.abs(normalized[key] - given) > 1e-12) throw new Error(`transform: ${key} ${given} is outside the Studio calibration range`);
+	}
+	return normalized;
+}
+
+export function isIdentityTransform(t) {
+	return !t || (t.scale === 1 && t.yawDeg === 0 && t.offsetX === 0 && t.offsetY === 0 && t.offsetZ === 0);
+}
+
+/**
+ * Bake scale, offsetY and yaw of `transform` into a decoded take
+ * ({ frames, rotMats, rootPos, posedJoints, ... }, src/ardy/npz.js). Returns
+ * the new take and the scene offset the caller still has to apply.
+ */
+export function transformTake(motion, transform) {
+	const { motion: moved } = applyMotionCalibration(motion, { ...transform, offsetX: 0, offsetZ: 0 });
+	const rotMats = Float32Array.from(motion.rotMats);
+	if (transform.yawDeg !== 0) {
+		const a = (transform.yawDeg * Math.PI) / 180;
+		const c = Math.cos(a);
+		const s = Math.sin(a);
+		// Ry with applyMotionCalibration's sign: x' = c x + s z, z' = -s x + c z.
+		const Ry = [c, 0, s, 0, 1, 0, -s, 0, c];
+		const stride = rotMats.length / motion.frames;
+		for (let f = 0; f < motion.frames; f += 1) {
+			const o = f * stride; // joint 0 (Hips) is the root: local == global
+			const m = Array.from(rotMats.subarray(o, o + 9));
+			for (let r = 0; r < 3; r += 1) {
+				for (let col = 0; col < 3; col += 1) {
+					rotMats[o + r * 3 + col] = Ry[r * 3] * m[col] + Ry[r * 3 + 1] * m[3 + col] + Ry[r * 3 + 2] * m[6 + col];
+				}
+			}
+		}
+	}
+	return {
+		motion: { ...moved, rotMats, rootPos: Float32Array.from(moved.rootPos), posedJoints: Float32Array.from(moved.posedJoints) },
+		sceneOffset: { x: transform.offsetX, y: 0, z: transform.offsetZ },
+	};
+}
+
+/** The npz members the Studio decoder reads, for tools/ardy/npz.mjs writeNpz. */
+export function takeToNpzMembers(motion) {
+	const joints = motion.posedJoints.length / (motion.frames * 3);
+	const members = {
+		local_rot_mats: { data: Float32Array.from(motion.rotMats), shape: [motion.frames, joints, 3, 3] },
+		root_positions: { data: Float32Array.from(motion.rootPos), shape: [motion.frames, 3] },
+		posed_joints: { data: Float32Array.from(motion.posedJoints), shape: [motion.frames, joints, 3] },
+		fps: { data: Int32Array.of(motion.fps), shape: [] },
+	};
+	if (motion.personScale !== undefined && motion.personScale !== 1) members.person_scale = { data: Float32Array.of(motion.personScale), shape: [] };
+	if (motion.boneScale) members.bone_scale = { data: Float32Array.from(motion.boneScale), shape: [motion.boneScale.length] };
+	return members;
+}