cozyclay 1.2.0 → 1.3.0

package/src/pose-extract/detector.js

@@ -0,0 +1,75 @@
+// detector.js — the App-side loader for MediaPipe's PoseLandmarker. Model
+// creation stays out of browser.js on purpose (see that header): this module
+// is where the engine version, the download sources and the failure names
+// live, so "extraction is broken" always decomposes into which piece failed.
+//
+// Both the wasm engine and the .task weights are runtime downloads. The wasm
+// must match the npm bundle compiled into this build byte-for-byte, so its
+// URL is pinned to the same version; the weights are ~5 MB of Google binaries
+// that belong in neither this repo nor its npm package. Offline is therefore
+// a NAMED failure, never a hang or a silent fallback.
+
+export const TASKS_VISION_VERSION = "0.10.17"; // must equal the @mediapipe/tasks-vision entry in package.json — the JS bundle and the wasm ABI move together
+export const POSE_WASM_BASE = `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${TASKS_VISION_VERSION}/wasm`;
+export const POSE_MODEL_URL =
+	"https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task";
+
+/**
+ * Create a `{ detect, close }` pair for collectLandmarkTrack. Every failure
+ * mode has its own name:
+ *   pose-runtime-unavailable   — the JS bundle did not load (broken build)
+ *   pose-model-download-failed — the .task weights were unreachable
+ *   pose-engine-download-failed— the wasm engine was unreachable
+ *   pose-engine-init-failed    — the engine loaded but refused to start
+ * The GPU delegate is tried first and CPU is the retry: WebGPU blocklists
+ * are common and a slower extraction beats a refused one.
+ */
+export async function createPoseDetector({
+	wasmBase = POSE_WASM_BASE,
+	modelUrl = POSE_MODEL_URL,
+	fetchImpl,
+	loadRuntime,
+	numPoses = 2, // the footage guidance allows two performers in frame; selectMostConfidentPerson picks one
+} = {}) {
+	const doFetch = fetchImpl ?? globalThis.fetch;
+	let runtime;
+	try {
+		runtime = await (loadRuntime ?? (() => import("@mediapipe/tasks-vision")))();
+	} catch {
+		throw new Error("pose-runtime-unavailable");
+	}
+	let modelAssetBuffer;
+	try {
+		const response = await doFetch(modelUrl);
+		if (!response.ok) throw new Error(`http-${response.status}`);
+		modelAssetBuffer = new Uint8Array(await response.arrayBuffer());
+	} catch {
+		throw new Error("pose-model-download-failed");
+	}
+	let fileset;
+	try {
+		fileset = await runtime.FilesetResolver.forVisionTasks(wasmBase);
+	} catch {
+		throw new Error("pose-engine-download-failed");
+	}
+	const optionsFor = (delegate) => ({
+		baseOptions: { modelAssetBuffer, delegate },
+		runningMode: "VIDEO",
+		numPoses,
+		outputSegmentationMasks: false,
+	});
+	let landmarker;
+	try {
+		landmarker = await runtime.PoseLandmarker.createFromOptions(fileset, optionsFor("GPU"));
+	} catch {
+		try {
+			landmarker = await runtime.PoseLandmarker.createFromOptions(fileset, optionsFor("CPU"));
+		} catch {
+			throw new Error("pose-engine-init-failed");
+		}
+	}
+	return {
+		detect: (image, timestampMs) => landmarker.detectForVideo(image, Math.round(timestampMs)),
+		close: () => landmarker.close?.(),
+	};
+}

package/src/pose-extract/index.js

@@ -12,6 +12,9 @@ import { selectPoseKeyframes } from "./keyframes.js";
 import { filterLandmarkTrack, normalizeLandmarkTrack } from "./landmarks.js";
 
 export { collectLandmarkTrack, selectMostConfidentPerson } from "./browser.js";
+export { createPoseDetector, POSE_MODEL_URL, POSE_WASM_BASE, TASKS_VISION_VERSION } from "./detector.js";
+export { sampleTimes, videoFrames } from "./video-frames.js";
+export { bakeExtractedTake } from "./take.js";
 export { fitLandmarksToPose } from "./fit.js";
 export { poseChangeScores, selectPoseKeyframes } from "./keyframes.js";
 export { filterLandmarkTrack, normalizeLandmarkSample, normalizeLandmarkTrack, POSE_LANDMARK } from "./landmarks.js";

package/src/pose-extract/take.js

@@ -0,0 +1,87 @@
+// take.js — bake a raw landmark track into the exact motion shape the npz
+// decoder returns ({ frames, fps, rotMats, rootPos, posedJoints }), so the
+// existing timeline playback consumes an extracted take with zero new
+// playback code. This is deliberately NOT an ARDY surface: the take is a
+// measurement of the footage, and the generator is never in the loop.
+
+import { CSKEL27_JOINTS } from "../ardy/cskel27.js";
+import { poseToCskel27 } from "../ardy/to-cskel27.js";
+import { fitLandmarksToPose } from "./fit.js";
+import { filterLandmarkTrack, normalizeLandmarkTrack } from "./landmarks.js";
+
+/**
+ * `samples` are raw `{ timeS, landmarks }` from collectLandmarkTrack on the
+ * uniform `fps` grid, holes allowed — detection loses the person for a frame
+ * without warning. Holes hold the previous baked frame and the leading hole
+ * backfills from the first fitted one: a previs take must never cut to
+ * T-pose because one frame missed. `fitted`/`held` in the result say exactly
+ * how much of the take was measured versus held.
+ */
+export function bakeExtractedTake({ samples, rest, fps, durationS, ...options }) {
+	if (!Number.isFinite(fps) || fps <= 0) {
+		throw new Error("bakeExtractedTake: fps must be a positive finite number");
+	}
+	if (!Number.isFinite(durationS) || durationS <= 0) {
+		throw new Error("bakeExtractedTake: durationS must be a positive finite number");
+	}
+	const normalized = normalizeLandmarkTrack(samples, options);
+	const filtered = filterLandmarkTrack(normalized, options.filter);
+	const joints = CSKEL27_JOINTS.length;
+	// Same counting rule as frameCountFor: the frame at t=0 plus every whole
+	// frame inside the duration, so take length and timeline length agree.
+	const frames = Math.max(1, Math.floor(durationS * fps + 1e-6) + 1);
+	const rotMats = new Float32Array(frames * joints * 9);
+	const rootPos = new Float32Array(frames * 3);
+	const posedJoints = new Float32Array(frames * joints * 3);
+
+	const fitByFrame = new Map();
+	let fitted = 0;
+	for (const sample of filtered) {
+		if (!sample.valid) continue;
+		const frame = Math.round(sample.timeS * fps);
+		if (frame < 0 || frame >= frames) continue;
+		const { pose } = fitLandmarksToPose({ sample, rest, createdMs: options.createdMs ?? 0 });
+		fitByFrame.set(frame, poseToCskel27({ pose, rest }));
+		fitted += 1;
+	}
+	if (fitted === 0) throw new Error("no-usable-pose");
+
+	const writeFrame = (frame, converted) => {
+		const rotBase = frame * joints * 9;
+		const jointBase = frame * joints * 3;
+		for (let j = 0; j < joints; j += 1) {
+			const m = converted.local_rot_mats[j];
+			const o = rotBase + j * 9;
+			rotMats[o] = m[0][0]; rotMats[o + 1] = m[0][1]; rotMats[o + 2] = m[0][2];
+			rotMats[o + 3] = m[1][0]; rotMats[o + 4] = m[1][1]; rotMats[o + 5] = m[1][2];
+			rotMats[o + 6] = m[2][0]; rotMats[o + 7] = m[2][1]; rotMats[o + 8] = m[2][2];
+			const p = converted.posed_joints[j];
+			posedJoints[jointBase + j * 3] = p[0];
+			posedJoints[jointBase + j * 3 + 1] = p[1];
+			posedJoints[jointBase + j * 3 + 2] = p[2];
+		}
+		rootPos[frame * 3] = converted.posed_joints[0][0];
+		rootPos[frame * 3 + 1] = converted.posed_joints[0][1];
+		rootPos[frame * 3 + 2] = converted.posed_joints[0][2];
+	};
+
+	let last = null;
+	let held = 0;
+	const leading = [];
+	for (let frame = 0; frame < frames; frame += 1) {
+		const converted = fitByFrame.get(frame) ?? null;
+		if (converted) last = converted;
+		if (!last) {
+			leading.push(frame);
+			continue;
+		}
+		if (!converted) held += 1;
+		writeFrame(frame, last);
+	}
+	if (leading.length > 0) {
+		const first = fitByFrame.get(Math.min(...fitByFrame.keys()));
+		for (const frame of leading) writeFrame(frame, first);
+		held += leading.length;
+	}
+	return { frames, fps, rotMats, rootPos, posedJoints, fitted, held };
+}

package/src/pose-extract/video-frames.js

@@ -0,0 +1,91 @@
+// video-frames.js — seek-stepped frame supply for landmark extraction. The
+// video element is injected (same pattern as probeFootage) so the node suite
+// drives exactly the stepping the app runs. Frames come out in ascending time
+// order, which is also the contract detectForVideo enforces on timestamps.
+
+/**
+ * The times extraction samples: t=0 plus every whole step inside the
+ * duration — the same counting rule as frameCountFor, so a clip's sample
+ * count and its timeline frame count agree at equal rates. The final sample
+ * is backed off the stream end by 1 ms: a seek to the exact duration lands
+ * past the last presentable frame on some decoders.
+ */
+export function sampleTimes(durationS, sampleFps) {
+	if (!(durationS > 0) || !(sampleFps > 0)) return [];
+	const count = Math.floor(durationS * sampleFps + 1e-6) + 1;
+	const tail = Math.max(0, durationS - 1e-3);
+	const times = [];
+	for (let i = 0; i < count; i += 1) times.push(Math.min(i / sampleFps, tail));
+	if (times.length > 1 && times[times.length - 1] <= times[times.length - 2]) times.pop();
+	return times;
+}
+
+function waitForData(video, timeoutMs) {
+	if ((video.readyState ?? 0) >= 2) return Promise.resolve();
+	return new Promise((resolve, reject) => {
+		let settled = false;
+		const timer = setTimeout(() => finish(new Error("footage-load-timeout")), timeoutMs);
+		const onLoaded = () => finish(null);
+		const onError = () => finish(new Error("decode-failed"));
+		function finish(error) {
+			if (settled) return;
+			settled = true;
+			clearTimeout(timer);
+			video.removeEventListener?.("loadeddata", onLoaded);
+			video.removeEventListener?.("error", onError);
+			if (error) reject(error);
+			else resolve();
+		}
+		video.addEventListener("loadeddata", onLoaded);
+		video.addEventListener("error", onError);
+		video.load?.();
+	});
+}
+
+function seekTo(video, timeS, timeoutMs) {
+	// The first sample is usually t=0 with the decoder already sitting there;
+	// assigning currentTime=0 again fires no seeked event, so that case must
+	// resolve without waiting for one.
+	if (Math.abs((video.currentTime ?? 0) - timeS) < 1e-4 && (video.readyState ?? 0) >= 2) {
+		return Promise.resolve();
+	}
+	return new Promise((resolve, reject) => {
+		let settled = false;
+		const timer = setTimeout(() => finish(new Error("seek-timeout")), timeoutMs);
+		const onSeeked = () => finish(null);
+		const onError = () => finish(new Error("decode-failed"));
+		function finish(error) {
+			if (settled) return;
+			settled = true;
+			clearTimeout(timer);
+			video.removeEventListener?.("seeked", onSeeked);
+			video.removeEventListener?.("error", onError);
+			if (error) reject(error);
+			else resolve();
+		}
+		video.addEventListener("seeked", onSeeked);
+		video.addEventListener("error", onError);
+		video.currentTime = timeS;
+	});
+}
+
+/**
+ * Async-iterate `{ image, timeS }` frames from already-ingested footage.
+ * `image` is the video element itself — exactly what detectForVideo accepts —
+ * so no canvas copy sits between the decoder and the detector.
+ */
+export async function* videoFrames(objectUrl, { createVideo, sampleFps = 20, timeoutMs = 8000 } = {}) {
+	if (typeof createVideo !== "function") throw new Error("videoFrames: createVideo is required");
+	const video = createVideo();
+	video.src = objectUrl;
+	video.muted = true;
+	video.playsInline = true;
+	video.preload = "auto";
+	await waitForData(video, timeoutMs);
+	const times = sampleTimes(Number(video.duration), sampleFps);
+	if (times.length === 0) throw new Error("duration-unreadable");
+	for (const timeS of times) {
+		await seekTo(video, timeS, timeoutMs);
+		yield { image: video, timeS };
+	}
+}

package/src/pose-thumbs.js

@@ -0,0 +1,152 @@
+import * as THREE from "three";
+import { FBXLoader } from "three/examples/jsm/loaders/FBXLoader.js";
+import { SkeletonUtils } from "three/examples/jsm/Addons.js";
+import { applyPose, primeBindPose, POSE_BONES } from "./poses.js";
+
+/**
+ * Pose Studio thumbnails, rendered on demand. The studio's pose entries are
+ * pure bone data (no image assets ship), so each tile's preview is generated
+ * here: clone the character rig, apply the pose, render one 256px frame with
+ * the same clay look as the viewport, and hand back a data URL. Results are
+ * cached per model+pose for the session; the FBX itself is fetched once per
+ * model (the browser HTTP cache dedupes the studio's own copy).
+ */
+
+const THUMB_SIZE = 256;
+const BODY_CLAY = "#f2eee6";
+
+const modelCache = new Map(); // model id -> Promise<Group>
+const thumbCache = new Map(); // `${model}:${poseId}` -> Promise<dataURL>
+
+// Thumbnail renders are SERIALIZED and yield to the browser between tiles:
+// a burst of 16 skinned 165k-vert clones on the main thread freezes the
+// studio open for half a second, so each tile waits for an idle slice.
+let renderQueue = Promise.resolve();
+const idle = () => new Promise((resolve) => {
+	if (typeof requestIdleCallback === "function") requestIdleCallback(() => resolve(), { timeout: 120 });
+	else setTimeout(resolve, 16);
+});
+function enqueueRender(work) {
+	const run = renderQueue.then(async () => {
+		await idle();
+		return work();
+	});
+	renderQueue = run.catch(() => {});
+	return run;
+}
+
+function loadModel(model) {
+	if (!modelCache.has(model)) {
+		modelCache.set(model, new Promise((resolve, reject) => {
+			new FBXLoader().load(`/models/${model}.fbx`, resolve, undefined, reject);
+		}));
+	}
+	return modelCache.get(model);
+}
+
+let renderer = null;
+function thumbRenderer() {
+	if (renderer) return renderer;
+	const canvas = document.createElement("canvas");
+	canvas.width = THUMB_SIZE;
+	canvas.height = THUMB_SIZE;
+	renderer = new THREE.WebGLRenderer({ canvas, antialias: true, preserveDrawingBuffer: true });
+	renderer.setSize(THUMB_SIZE, THUMB_SIZE, false);
+	return renderer;
+}
+
+// One rig per model, RE-POSED per tile: cloning + material traversal per
+// pose was the bulk of the studio-open stall — one clone serves them all.
+const thumbRigCache = new Map(); // model id -> Promise<Group (prepared clone)>
+function thumbRig(model) {
+	if (!thumbRigCache.has(model)) {
+		thumbRigCache.set(model, loadModel(model).then((fbx) => {
+			const rig = SkeletonUtils.clone(fbx);
+			rig.scale.setScalar(0.01); // Mixamo exports centimetres
+			const jointTint = new THREE.Color(BODY_CLAY).multiplyScalar(0.45);
+			rig.traverse((child) => {
+				if (child.isMesh) {
+					child.material = new THREE.MeshStandardMaterial({
+						color: /_Joints$/i.test(child.name) ? jointTint : BODY_CLAY,
+						roughness: 0.82,
+						metalness: 0,
+					});
+					child.frustumCulled = false;
+				}
+			});
+			primeBindPose(rig);
+			return rig;
+		}));
+	}
+	return thumbRigCache.get(model);
+}
+
+function thumbScene() {
+	const scene = new THREE.Scene();
+	scene.background = new THREE.Color("#101014");
+	const hemi = new THREE.HemisphereLight(0xffffff, 0x3a3a40, 1.25);
+	const key = new THREE.DirectionalLight(0xffffff, 1.6);
+	key.position.set(2.2, 4, 3);
+	// A barely-lighter floor so floor poses (seated, kneel) read grounded and
+	// air poses (jump) read airborne instead of everyone floating in a void.
+	const floor = new THREE.Mesh(
+		new THREE.CircleGeometry(6, 48),
+		new THREE.MeshStandardMaterial({ color: "#17171d", roughness: 1 })
+	);
+	floor.rotation.x = -Math.PI / 2;
+	scene.add(hemi, key, floor);
+	return scene;
+}
+
+/** Frame the POSED rig, not a standing mannequin: hands-up, jump and floor
+ * poses all leave the fixed framing of a standing figure, so the camera is
+ * recomputed from the pose's own bone extents (plus a small margin). */
+function frameCamera(rig) {
+	rig.updateMatrixWorld(true);
+	const box = new THREE.Box3();
+	const point = new THREE.Vector3();
+	rig.traverse((object) => {
+		if (!object.isBone) return;
+		object.getWorldPosition(point);
+		box.expandByPoint(point);
+	});
+	if (box.isEmpty()) box.setFromObject(rig);
+	const center = box.getCenter(new THREE.Vector3());
+	const size = box.getSize(new THREE.Vector3());
+	// Hands and feet sit slightly past their bones; heads a touch more.
+	const padded = Math.max(size.x, size.y, size.z) * 0.5 + 0.22;
+	const camera = new THREE.PerspectiveCamera(35, 1, 0.01, 50);
+	const distance = padded / Math.tan(THREE.MathUtils.degToRad(camera.fov / 2));
+	const direction = new THREE.Vector3(0.3, 0.42, 1).normalize();
+	camera.position.copy(center).addScaledVector(direction, distance);
+	camera.lookAt(center.x, center.y - size.y * 0.04, center.z);
+	return camera;
+}
+
+/** Preload a rig model during browser idle so the first studio open does
+ * not pay the FBX parse in one frame. Fire-and-forget by design. */
+export function warmThumbnailModels(models) {
+	for (const model of models) enqueueRender(() => thumbRig(model));
+}
+
+/** Render one pose of one rig model to a PNG data URL. */
+export function poseThumbnail(model, pose) {
+	const key = `${model}:${pose.id}`;
+	if (!thumbCache.has(key)) {
+		thumbCache.set(key, enqueueRender(() => thumbRig(model).then((rig) => {
+			// Re-pose the shared rig: applyPose writes deltas over the stamped
+			// bind pose, so a rest stamp per tile resets the previous pose.
+			applyPose(rig, Object.fromEntries(POSE_BONES.map((bone) => [bone.id, [0, 0, 0]])));
+			applyPose(rig, pose.bones);
+			const scene = thumbScene();
+			scene.add(rig);
+			const camera = frameCamera(rig);
+			const gl = thumbRenderer();
+			gl.render(scene, camera);
+			const url = gl.domElement.toDataURL("image/png");
+			scene.remove(rig);
+			return url;
+		})));
+	}
+	return thumbCache.get(key);
+}