cozyclay 2.0.0 → 2.1.0

package/src/stage-canvas-rigs.jsx

@@ -0,0 +1,402 @@
+import { useEffect, useRef } from "react";
+import { useFrame, useThree } from "@react-three/fiber";
+import * as THREE from "three";
+import { aimAt } from "./controls.jsx";
+import { SUBJECT_HEIGHT_M } from "./shot.js";
+import { shotAtFrame } from "./cuts.js";
+import { sampleAt } from "./sample-at.js";
+import { RENDER_ACTIVITY_EVENT } from "./use-render-activity.js";
+import { PRESETS, editorFrameForSubject } from "./stage-startup-defaults.js";
+
+/** Selection marker for the picked cast member: a Unity-style XYZ tripod
+ * plus a ground ring at the feet. X/Z arrows also drag the character on the
+ * floor (Y is display-only — characters stand on the deck). */
+export function ShotRig({ preset, nonce, fovDeg, charA, charB, showB, probeX, probeZ, camRef, look, onMetrics, appliedPresetRef, lastPosRef }) {
+	useEffect(() => {
+		const cam = camRef.current;
+		if (!cam) return;
+		// A preset is applied once per (preset, nonce), not once per mount. The
+		// Canvas has no Suspense boundary of its own, so a cast member whose FBX
+		// is still downloading suspends the whole scene graph; when it resolves,
+		// React remounts the siblings — this rig included, on a fresh camera
+		// object — and a per-mount seed would snap the shot back to the preset,
+		// discarding a frame_shot or a manual move that landed in the meantime
+		// (#86 on slow CI runners). Both refs live in App so they survive the
+		// remount: the stamp says whether this preset was already applied, the
+		// position ref (with look.current) says where the camera last was.
+		const stamp = `${preset}:${nonce}`;
+		if (appliedPresetRef && appliedPresetRef.current === stamp && lastPosRef?.current) {
+			const last = lastPosRef.current;
+			cam.position.set(last.x, last.y, last.z);
+			cam.rotation.order = "YXZ";
+			cam.rotation.set(look.current.pitch, look.current.yaw, 0);
+			return;
+		}
+		if (appliedPresetRef) appliedPresetRef.current = stamp;
+		const p = PRESETS[preset];
+		const az = (p.azimuth * Math.PI) / 180;
+		const el = (p.elevation * Math.PI) / 180;
+		const aim =
+			p.aimMid && showB ? { x: (charA.x + charB.x) / 2, z: (charA.z + charB.z) / 2 } : { x: charA.x, z: charA.z };
+		const horizontal = p.distance * Math.cos(el);
+		cam.position.set(
+			aim.x + horizontal * Math.sin(az),
+			p.height ?? Math.max(p.targetY + p.distance * Math.sin(el), 0.15),
+			aim.z + horizontal * Math.cos(az),
+		);
+		const angles = aimAt(cam.position, { x: aim.x, y: p.targetY, z: aim.z });
+		look.current.yaw = angles.yaw;
+		look.current.pitch = angles.pitch;
+		// eslint-disable-next-line react-hooks/exhaustive-deps
+	}, [preset, nonce]);
+
+
+	useEffect(() => {
+		const cam = camRef.current;
+		if (!cam) return;
+		cam.fov = fovDeg;
+		cam.updateProjectionMatrix();
+	}, [fovDeg, camRef]);
+
+	// A blocking tool must not claim a framing it is not holding, so the subject's
+	// feet and eyes are projected into the frame every tick.
+	const probe = useRef(new THREE.Vector3());
+	const lastMetrics = useRef({ x: NaN, y: NaN, z: NaN, visible: null });
+	useFrame(() => {
+		const cam = camRef.current;
+		if (!cam) return;
+		let visible = false;
+		for (const y of [0.1, SUBJECT_HEIGHT_M * 0.94]) {
+			// During motion playback the probe follows the played root, so the
+			// "subject out of frame" caption stays honest.
+			probe.current.set(probeX ?? charA.x, y, probeZ ?? charA.z).project(cam);
+			if (probe.current.z < 1 && Math.abs(probe.current.x) <= 1 && Math.abs(probe.current.y) <= 1) visible = true;
+		}
+		const previous = lastMetrics.current;
+		if (
+			Math.abs(previous.x - cam.position.x) +
+				Math.abs(previous.y - cam.position.y) +
+				Math.abs(previous.z - cam.position.z) >
+				1e-4 ||
+			previous.visible !== visible
+		) {
+			lastMetrics.current = {
+				x: cam.position.x,
+				y: cam.position.y,
+				z: cam.position.z,
+				visible,
+			};
+			onMetrics(cam.position, visible);
+		}
+	});
+
+	return null;
+}
+
+/** Plays an authored A→B camera move by driving the shot camera exactly the
+    way a fly-drag does — position on the camera, orientation through the look
+    ref — so ShotRig's metrics, the slate, and the inset all stay honest for
+    free. A right-drag interrupts the dolly: the user always outranks it.
+
+    Two frame sources can drive the move. Standalone preview advances authored
+    frames at production cadence; follow mode reads the timeline playhead, so the camera and
+    the character motion are two views of the same time axis — playing or
+    scrubbing frame 72 at 24 fps puts the camera exactly 3 s into its move.
+
+    frameRef is the imperative playhead routed props and riders already read in
+    their own useFrame. The followFrame prop reaches this Canvas subtree one
+    commit later, so alone it would pose the camera a paint behind them. */
+export function MoveRig({ playing, following, followFrame, frameRef, fps, keys, shots, scene, camRef, look, isInterrupted, onDone }) {
+	const invalidate = useThree((state) => state.invalidate);
+	const preview = useRef({ frame: 0, finished: false, notified: false });
+	const handlers = useRef({ isInterrupted, onDone });
+	handlers.current = { isInterrupted, onDone };
+	// The follow branch only re-applies when its time changes; in demand mode
+	// each apply needs one more frame so ShotRig's metrics see the new pose.
+	const appliedFrame = useRef(null);
+	useEffect(() => {
+		// Arming or reshaping the move must schedule a frame by hand: this
+		// component owns no three objects, so in demand mode nothing else will.
+		appliedFrame.current = null;
+		if (following) invalidate();
+	}, [keys, shots, scene, fps, following, invalidate]);
+	useEffect(() => {
+		// A paused scrub changes the playhead without starting the render loop.
+		if (following && !playing) invalidate();
+	}, [followFrame, following, playing, invalidate]);
+	useEffect(() => {
+		if (!playing || keys.length < 1) return undefined;
+		const firstFrame = keys[0].frame;
+		const lastFrame = keys[keys.length - 1].frame;
+		const spanFrames = Math.max(lastFrame - firstFrame, 1);
+		const spanSeconds = Math.max(spanFrames / Math.max(fps, 1), 0.1);
+		const tickMs = (spanSeconds * 1000) / spanFrames;
+		let tick = 0;
+		preview.current = { frame: firstFrame, finished: false, notified: false };
+		invalidate();
+		const timer = window.setInterval(() => {
+			if (preview.current.finished) return;
+			if (handlers.current.isInterrupted?.()) {
+				preview.current.finished = true;
+				invalidate();
+				return;
+			}
+			tick += 1;
+			preview.current = {
+				frame: firstFrame + (lastFrame - firstFrame) * Math.min(tick / spanFrames, 1),
+				finished: tick >= spanFrames,
+				notified: false,
+			};
+			invalidate();
+		}, tickMs);
+		return () => window.clearInterval(timer);
+	}, [playing, keys, fps, invalidate]);
+	useFrame(() => {
+		if ((playing && keys.length < 1) || (!playing && following && !shots.some((shot) => shot.cameraKeys.length))) return;
+		const cam = camRef.current;
+		if (!cam) return;
+		const apply = (frame) => {
+			// Timeline/PlayView sampling follows the editorial strips. Standalone
+			// preview stays scoped to the selected strip's keys. Entering a Shot
+			// applies its authored camera immediately (a hard cut). Leaving into a
+			// gap returns null and deliberately applies nothing: the last physical
+			// camera pose stays put and FlyControls regain ownership, so we do not
+			// invent a second cut to an arbitrary "free camera" preset.
+			const timelineShot = following ? shotAtFrame(shots, frame) : null;
+			const sampledShot = timelineShot
+				? { ...timelineShot, camera: { mode: "keys" } }
+				: following ? null : { camera: { mode: "keys" }, cameraKeys: keys };
+			const f = sampleAt(scene, sampledShot, frame).camera;
+			if (!f) return null;
+			cam.position.set(f.pos.x, f.pos.y, f.pos.z);
+			look.current.yaw = f.yaw;
+			look.current.pitch = f.pitch;
+			if (Math.abs(cam.fov - f.fovDeg) > 1e-3) {
+				cam.fov = f.fovDeg;
+				cam.updateProjectionMatrix();
+			}
+			return f;
+		};
+		if (playing) {
+			const f = apply(preview.current.frame);
+			if (preview.current.finished && !preview.current.notified) {
+				preview.current.notified = true;
+				handlers.current.onDone(f ? f.fovDeg : cam.fov);
+			}
+			return;
+		}
+		if (!following) return;
+		if (isInterrupted?.()) return; // manual viewport framing owns the paused camera
+		const frame = frameRef?.current ?? followFrame;
+		if (appliedFrame.current === frame) return;
+		appliedFrame.current = frame;
+		apply(frame);
+		invalidate();
+	});
+	return null;
+}
+
+/**
+ * Applies the precomputed follow-camera track to the shot camera. The track
+ * is derived offline from the subject trajectory (camera-follow.js), so this
+ * rig only samples it at the playhead — scrub, play, PlayView and Record all
+ * replay the identical deterministic move. Null-rendering, so every apply
+ * must invalidate() by hand in demand mode, like MoveRig. Like MoveRig it
+ * prefers frameRef over the frame prop so the camera lands in the same paint
+ * as the routed props and riders it follows.
+ */
+export function FollowCamRig({ enabled, frame, frameRef, scene, shot, camRef, look, isInterrupted }) {
+	const invalidate = useThree((state) => state.invalidate);
+	const applied = useRef(null);
+	useEffect(() => {
+		applied.current = null;
+		if (enabled) invalidate();
+	}, [scene, shot, enabled, invalidate]);
+	useEffect(() => {
+		if (enabled) invalidate();
+	}, [frame, enabled, invalidate]);
+	useFrame(() => {
+		if (!enabled) return;
+		if (isInterrupted?.()) return; // the user is flying; yield until released
+		const cam = camRef.current;
+		if (!cam) return;
+		const playhead = frameRef?.current ?? frame;
+		const sample = sampleAt(scene, shot, playhead).camera;
+		if (!sample || applied.current === playhead) return;
+		applied.current = playhead;
+		cam.position.set(sample.pos.x, sample.pos.y, sample.pos.z);
+		look.current.yaw = sample.yaw;
+		look.current.pitch = sample.pitch;
+		invalidate();
+	});
+	return null;
+}
+
+/**
+ * Applies the shot camera's yaw/pitch from its look ref every frame. Before
+ * the camera split, FlyControls owned the shot camera and performed this
+ * apply as a side effect of flying; with the fly controls bound to the
+ * editor camera, the recording camera still needs its aim (ShotRig, follow,
+ * frame-selection) reflected in its actual rotation — the preview and the
+ * capture read the camera object, not the ref.
+ */
+export function ShotLookApplier({ camRef, look }) {
+	useFrame(() => {
+		const cam = camRef.current;
+		if (!cam) return;
+		if (cam.rotation.order !== "YXZ") cam.rotation.order = "YXZ";
+		if (cam.rotation.x !== look.current.pitch || cam.rotation.y !== look.current.yaw || cam.rotation.z !== 0) {
+			cam.rotation.set(look.current.pitch, look.current.yaw, 0);
+		}
+	});
+	return null;
+}
+
+/**
+ * Seeds the editor camera once, inside the Canvas root — App-level effects
+ * run before R3F mounts its children, so the ref is still null there. The
+ * seed frames both the subject and the shot camera: the first editor frame
+ * must read as "the set with the camera in it".
+ */
+/**
+ * A short editor-camera glide: 260 ms ease toward a target orientation (and
+ * optionally a position), instead of the teleport that made "look at the
+ * light" and F-framing feel like a cut. Any user press cancels it — the
+ * operator always outranks the tour guide.
+ */
+export function CameraGlide({ glide, camRef, lookRef, onDone }) {
+	const { gl } = useThree();
+	const invalidate = useThree((state) => state.invalidate);
+	const animRef = useRef(null);
+	useEffect(() => {
+		if (!glide) {
+			animRef.current = null;
+			return undefined;
+		}
+		const cam = camRef.current;
+		if (!cam) return undefined;
+		const fromPos = cam.position.clone();
+		const toPos = glide.position ? new THREE.Vector3(glide.position.x, glide.position.y, glide.position.z) : fromPos.clone();
+		const angles = aimAt(toPos, glide.target);
+		const from = { yaw: lookRef.current.yaw, pitch: lookRef.current.pitch };
+		let dYaw = angles.yaw - from.yaw;
+		while (dYaw > Math.PI) dYaw -= 2 * Math.PI;
+		while (dYaw < -Math.PI) dYaw += 2 * Math.PI;
+		// elapsed accumulates render-loop deltas: the glide needs no wall clock
+		animRef.current = { fromPos, toPos, from, dYaw, dPitch: angles.pitch - from.pitch, elapsed: 0 };
+		const cancel = () => {
+			animRef.current = null;
+			onDone?.();
+		};
+		gl.domElement.addEventListener("pointerdown", cancel, true);
+		invalidate();
+		return () => gl.domElement.removeEventListener("pointerdown", cancel, true);
+		// camRef/lookRef are stable refs; onDone identity is per-glide noise
+		// eslint-disable-next-line react-hooks/exhaustive-deps
+	}, [glide, gl, invalidate]);
+	useFrame((_, delta) => {
+		const anim = animRef.current;
+		const cam = camRef.current;
+		if (!anim || !cam) return;
+		anim.elapsed += Math.min(delta, 0.1) * 1000;
+		const t = Math.min(1, anim.elapsed / 260);
+		const eased = 1 - (1 - t) ** 3;
+		cam.position.lerpVectors(anim.fromPos, anim.toPos, eased);
+		const yaw = anim.from.yaw + anim.dYaw * eased;
+		const pitch = anim.from.pitch + anim.dPitch * eased;
+		lookRef.current = { yaw, pitch };
+		cam.rotation.order = "YXZ";
+		cam.rotation.set(pitch, yaw, 0);
+		if (t >= 1) {
+			animRef.current = null;
+			onDone?.();
+		} else {
+			invalidate();
+		}
+	});
+	return null;
+}
+
+export function EditorCamSeed({ camRef, lookRef, shotCamRef, subject }) {
+	const seeded = useRef(false);
+	const invalidate = useThree((state) => state.invalidate);
+	useEffect(() => {
+		if (seeded.current) return;
+		const cam = camRef.current;
+		if (!cam) return;
+		seeded.current = true;
+		// The selected character owns the first read. The shot camera remains a
+		// useful reference in the inset; mixing its position into this seed pushed
+		// the editor eye several metres back and made the mannequin look tiny.
+		const frame = editorFrameForSubject(subject);
+		const { target, position } = frame;
+		cam.position.set(position.x, position.y, position.z);
+		const angles = aimAt(cam.position, target);
+		lookRef.current = { yaw: angles.yaw, pitch: angles.pitch };
+		cam.rotation.order = "YXZ";
+		cam.rotation.set(angles.pitch, angles.yaw, 0);
+		invalidate();
+	});
+	return null;
+}
+
+export function RenderLoopController({ stageRef }) {
+	const frameloop = useThree((state) => state.frameloop);
+	const setFrameloop = useThree((state) => state.setFrameloop);
+	const invalidate = useThree((state) => state.invalidate);
+	useEffect(() => {
+		const apply = (next) => {
+			setFrameloop(next ? "always" : "demand");
+			if (next) invalidate();
+		};
+		const onActivity = (event) => apply(event.detail);
+		window.addEventListener(RENDER_ACTIVITY_EVENT, onActivity);
+		return () => window.removeEventListener(RENDER_ACTIVITY_EVENT, onActivity);
+	}, [invalidate, setFrameloop]);
+	useEffect(() => {
+		if (stageRef.current) stageRef.current.dataset.actualRenderLoop = frameloop;
+	}, [frameloop, stageRef]);
+	return null;
+}
+
+/** GPU resets — sleep/wake, a driver restart, VRAM pressure from an export —
+ * kill the WebGL context, and without listeners the stage goes black and
+ * stays black with no message. preventDefault on webglcontextlost is the
+ * documented opt-in for browser-driven restoration; on restore, kick the
+ * demand loop so the first frame actually paints. */
+export function ContextLossGuard({ onLostChange }) {
+	const gl = useThree((state) => state.gl);
+	const invalidate = useThree((state) => state.invalidate);
+	useEffect(() => {
+		const canvas = gl.domElement;
+		const onLost = (event) => {
+			event.preventDefault();
+			onLostChange(true);
+		};
+		const onRestored = () => {
+			onLostChange(false);
+			invalidate();
+		};
+		canvas.addEventListener("webglcontextlost", onLost);
+		canvas.addEventListener("webglcontextrestored", onRestored);
+		return () => {
+			canvas.removeEventListener("webglcontextlost", onLost);
+			canvas.removeEventListener("webglcontextrestored", onRestored);
+		};
+	}, [gl, invalidate, onLostChange]);
+	return null;
+}
+
+export function ViewportLayoutInvalidator({ insetX, insetY, insetWidth, insetHeight, hierarchyWidth, sidebarWidth, timelineHeight, planZoom }) {
+	const invalidate = useThree((state) => state.invalidate);
+	useEffect(() => {
+		// The pane frame is regular DOM while its picture is a scissored WebGL
+		// viewport. In demand mode a drag can commit its final DOM position just
+		// after the continuous loop stops, leaving the picture at the previous
+		// coordinates. Redraw now and once more after layout has settled.
+		invalidate();
+		const frame = requestAnimationFrame(invalidate);
+		return () => cancelAnimationFrame(frame);
+	}, [invalidate, insetX, insetY, insetWidth, insetHeight, hierarchyWidth, sidebarWidth, timelineHeight, planZoom]);
+	return null;
+}

package/src/stage-capture.jsx

@@ -0,0 +1,216 @@
+import { useEffect } from "react";
+import { useThree } from "@react-three/fiber";
+import * as THREE from "three";
+import { GIZMO_LAYER } from "./dualview.jsx";
+
+export const CAPTURE_W = 1920;
+export const CAPTURE_H = 1080;
+export const MCP_CAPTURE_W = 640;
+export const MCP_CAPTURE_H = 360;
+export const MCP_CAPTURE_SAMPLE_STEP = 32;
+// How far the deck runs in an EXPORTED frame. The viewport fades it out at
+// 18..54 m to keep the working view clean; a blocking frame needs the far edge
+// to survive, so the falloff starts later and ends further away, letting the
+// floor resolve into a horizon instead of dissolving into the background.
+//
+// Both values must stay inside the shot camera's far plane (100 m) — geometry
+// past it is clipped outright, so a fog range that ended beyond it would cut
+// the deck off mid-fade and take the horizon with it.
+export const CAPTURE_FOG_NEAR = 55;
+export const CAPTURE_FOG_FAR = 95;
+
+/** Offscreen aspect-aware read-back, always from the shot camera. */
+export function CaptureRig({ apiRef, camRef, width = CAPTURE_W, height = CAPTURE_H }) {
+	const { gl, scene } = useThree();
+	useEffect(() => {
+		// Each attempt owns its target independently of the live output size.
+		// Resizing the editor cannot dispose an in-flight export's resources.
+		function createExportCapture({ width, height }) {
+			const target = new THREE.WebGLRenderTarget(width, height, {
+				colorSpace: THREE.SRGBColorSpace,
+				samples: 4,
+			});
+			const buffer = new Uint8Array(width * height * 4);
+			return {
+				scene,
+				dispose: () => target.dispose(),
+				render() {
+					const source = camRef.current;
+					if (!source) return null;
+					const cam = source.clone();
+					// the transform gizmo is UI: it never reaches an exported frame
+					cam.layers.disable(GIZMO_LAYER);
+					// QA hook: the layer mask the export camera actually renders
+					// with — the browser suite asserts GIZMO_LAYER (the gizmo AND
+					// the selection cage) is never in it.
+					window.__captureCameraMask = cam.layers.mask;
+					// A view offset (a crop of a wider frustum) already fixes the aspect.
+					if (!cam.view?.enabled) cam.aspect = width / height;
+					cam.updateProjectionMatrix();
+					const previous = gl.getRenderTarget();
+					// The viewport's fog dissolves the deck into the background by ~54 m
+					// so the working view has no horizon to distract from blocking. An
+					// exported frame wants the opposite: the horizon IS the vanishing
+					// point, and without it the floor has no far edge to read depth
+					// against. Push the falloff back for this draw only, then restore
+					// it so the viewport is untouched.
+					const fog = scene.fog;
+					const fogNear = fog?.near;
+					const fogFar = fog?.far;
+					if (fog) {
+						fog.near = CAPTURE_FOG_NEAR;
+						fog.far = CAPTURE_FOG_FAR;
+					}
+					try {
+						gl.setRenderTarget(target);
+						gl.render(scene, cam);
+						gl.readRenderTargetPixels(target, 0, 0, width, height, buffer);
+					} finally {
+						gl.setRenderTarget(previous);
+						if (fog) {
+							fog.near = fogNear;
+							fog.far = fogFar;
+						}
+					}
+					return buffer;
+				},
+			};
+		}
+		const api = { ...createExportCapture({ width, height }), createExportCapture };
+		apiRef.current = api;
+		if (width === MCP_CAPTURE_W && height === MCP_CAPTURE_H) {
+			window.__cozyclayMcpCaptureReady = true;
+			window.dispatchEvent(new Event("cozyclay:mcp-capture-ready"));
+		}
+		// QA hook: run one real export render and report what the capture
+		// camera saw — the layer mask plus an amber scan of the output
+		// frame (the selection cage's warm tone). Lets the suite prove the
+		// deliverable frame stays free of editor furniture without driving
+		// the whole generation form.
+		window.__captureFrame = () => {
+			const buffer = apiRef.current?.render() ?? null;
+			if (!buffer) return null;
+			let amber = 0;
+			for (let i = 0; i < buffer.length; i += 4) {
+				const r = buffer[i];
+				const g = buffer[i + 1];
+				const b = buffer[i + 2];
+				if (r >= 180 && g >= 165 && g - b >= 35) amber++;
+			}
+			return { layersMask: window.__captureCameraMask ?? 0, amber, pixels: buffer.length / 4 };
+		};
+		return () => {
+			if (apiRef.current === api) apiRef.current = null;
+			if (width === MCP_CAPTURE_W && height === MCP_CAPTURE_H) window.__cozyclayMcpCaptureReady = false;
+			api.dispose();
+		};
+	}, [gl, scene, camRef, apiRef, width, height]);
+	return null;
+}
+
+export async function captureMcpFrame({ capture, camera, characters, activeCharacterId, objects, rigs, readAuthoredState, partColours = null }) {
+	if (!capture || !camera) throw new Error("No renderable shot camera is available for capture_frame.");
+	const authoredStateBefore = JSON.stringify(readAuthoredState());
+	const buffer = capture.render();
+	if (!buffer) throw new Error("No renderable shot camera is available for capture_frame.");
+	let nonBlackPixels = 0;
+	for (let offset = 0; offset < buffer.length; offset += 4) {
+		if (buffer[offset] > 5 || buffer[offset + 1] > 5 || buffer[offset + 2] > 5) nonBlackPixels += 1;
+	}
+	if (nonBlackPixels === 0) throw new Error("capture_frame rejected a black rendered frame.");
+	camera.updateMatrixWorld();
+	const raycaster = new THREE.Raycaster();
+	const roots = [];
+	capture.scene.traverse((node) => {
+		const id = node.userData?.sceneObjectId;
+		if (typeof id === "string" && objects.some((object) => object.id === id)) roots.push({ id, root: node });
+	});
+	const assertions = characters.map((character) => {
+		const rig = rigs[character.id];
+		if (!rig || character.hidden) return { id: character.id, renderable: false, behindCameraPlane: null, fartherAlongCameraForward: null, distanceToFloor: null, occludedBy: null, visiblePixelCount: 0 };
+		rig.updateWorldMatrix(true, true);
+		const bounds = new THREE.Box3().setFromObject(rig);
+		const center = bounds.getCenter(new THREE.Vector3());
+		const cameraSpace = camera.worldToLocal(center.clone());
+		const behindCameraPlane = cameraSpace.z >= 0;
+		const distanceToFloor = Math.max(0, bounds.min.y);
+		if (behindCameraPlane || bounds.isEmpty()) {
+			return { id: character.id, renderable: true, behindCameraPlane, fartherAlongCameraForward: -cameraSpace.z, distanceToFloor, occludedBy: null, visiblePixelCount: 0 };
+		}
+		const corners = [
+			new THREE.Vector3(bounds.min.x, bounds.min.y, bounds.min.z), new THREE.Vector3(bounds.min.x, bounds.min.y, bounds.max.z),
+			new THREE.Vector3(bounds.min.x, bounds.max.y, bounds.min.z), new THREE.Vector3(bounds.min.x, bounds.max.y, bounds.max.z),
+			new THREE.Vector3(bounds.max.x, bounds.min.y, bounds.min.z), new THREE.Vector3(bounds.max.x, bounds.min.y, bounds.max.z),
+			new THREE.Vector3(bounds.max.x, bounds.max.y, bounds.min.z), new THREE.Vector3(bounds.max.x, bounds.max.y, bounds.max.z),
+		].map((corner) => corner.project(camera));
+		const minX = Math.max(0, Math.floor(((Math.min(...corners.map((corner) => corner.x)) + 1) * 0.5) * MCP_CAPTURE_W));
+		const maxX = Math.min(MCP_CAPTURE_W - 1, Math.ceil(((Math.max(...corners.map((corner) => corner.x)) + 1) * 0.5) * MCP_CAPTURE_W));
+		const minY = Math.max(0, Math.floor(((1 - Math.max(...corners.map((corner) => corner.y))) * 0.5) * MCP_CAPTURE_H));
+		const maxY = Math.min(MCP_CAPTURE_H - 1, Math.ceil(((1 - Math.min(...corners.map((corner) => corner.y))) * 0.5) * MCP_CAPTURE_H));
+		const occluders = new Map();
+		let visiblePixelCount = 0;
+		for (let y = minY; y <= maxY; y += MCP_CAPTURE_SAMPLE_STEP) {
+			for (let x = minX; x <= maxX; x += MCP_CAPTURE_SAMPLE_STEP) {
+				raycaster.setFromCamera({ x: (x / MCP_CAPTURE_W) * 2 - 1, y: 1 - (y / MCP_CAPTURE_H) * 2 }, camera);
+				const characterHit = raycaster.intersectObject(rig, true)[0];
+				if (!characterHit) continue;
+				const objectHit = raycaster.intersectObjects(roots.map((entry) => entry.root), true)[0];
+				const pixels = MCP_CAPTURE_SAMPLE_STEP * MCP_CAPTURE_SAMPLE_STEP;
+				if (!objectHit || objectHit.distance >= characterHit.distance - 1e-4) {
+					visiblePixelCount += pixels;
+					continue;
+				}
+				const occluder = roots.find((entry) => {
+					let node = objectHit.object;
+					while (node) {
+						if (node === entry.root) return true;
+						node = node.parent;
+					}
+					return false;
+				});
+				if (occluder) occluders.set(occluder.id, (occluders.get(occluder.id) ?? 0) + pixels);
+			}
+		}
+		const occludedBy = [...occluders.entries()].sort((left, right) => right[1] - left[1])[0]?.[0] ?? null;
+		return { id: character.id, renderable: true, behindCameraPlane, fartherAlongCameraForward: -cameraSpace.z, distanceToFloor, occludedBy, visiblePixelCount };
+	});
+	const active = assertions.find((entry) => entry.id === activeCharacterId) ?? assertions[0];
+	const authoredStateAfter = JSON.stringify(readAuthoredState());
+	const canvas = document.createElement("canvas");
+	canvas.width = MCP_CAPTURE_W;
+	canvas.height = MCP_CAPTURE_H;
+	const context = canvas.getContext("2d");
+	if (!context) throw new Error("capture_frame could not encode the rendered image.");
+	const image = context.createImageData(MCP_CAPTURE_W, MCP_CAPTURE_H);
+	for (let row = 0; row < MCP_CAPTURE_H; row += 1) {
+		image.data.set(buffer.subarray((MCP_CAPTURE_H - 1 - row) * MCP_CAPTURE_W * 4, (MCP_CAPTURE_H - row) * MCP_CAPTURE_W * 4), row * MCP_CAPTURE_W * 4);
+	}
+	context.putImageData(image, 0, 0);
+	const blob = await new Promise((resolve) => canvas.toBlob(resolve, "image/png"));
+	if (!blob) throw new Error("capture_frame could not compress the rendered image.");
+	const bytes = new Uint8Array(await blob.arrayBuffer());
+	let binary = "";
+	for (const byte of bytes) binary += String.fromCharCode(byte);
+	return {
+		width: MCP_CAPTURE_W,
+		height: MCP_CAPTURE_H,
+		mimeType: "image/png",
+		encoding: "base64",
+		byteSize: bytes.length,
+		data: btoa(binary),
+		partColours,
+		authoredStateBefore,
+		authoredStateAfter,
+		assertions: {
+			renderable: true,
+			blackFrame: false,
+			nonBlackPixels,
+			behindCameraPlane: active?.behindCameraPlane ?? null,
+			fartherAlongCameraForward: active?.fartherAlongCameraForward ?? null,
+			distanceToFloor: active?.distanceToFloor ?? null,
+			occludedBy: active?.occludedBy ?? null,
+			visiblePixelCount: active?.visiblePixelCount ?? 0,
+			characters: assertions,
+		},
+	};
+}