cozyclay 2.1.0 → 3.0.0

package/src/pose-extract/landmarks.js

@@ -1,162 +0,0 @@
-/**
- * Pure MediaPipe Pose landmark preparation.
- *
- * Input samples use MediaPipe's 33-landmark shape and camera coordinates
- * (X right, Y down, Z away from the camera). Output is hip-centred, Y-up,
- * forward-positive and torso-normalised. Root translation is intentionally
- * discarded: CozyClay authors it with waypoints and ARDY synthesises it.
- */
-
-export const POSE_LANDMARK = Object.freeze({
-	NOSE: 0,
-	LEFT_EYE: 2,
-	RIGHT_EYE: 5,
-	LEFT_EAR: 7,
-	RIGHT_EAR: 8,
-	LEFT_SHOULDER: 11,
-	RIGHT_SHOULDER: 12,
-	LEFT_ELBOW: 13,
-	RIGHT_ELBOW: 14,
-	LEFT_WRIST: 15,
-	RIGHT_WRIST: 16,
-	LEFT_PINKY: 17,
-	RIGHT_PINKY: 18,
-	LEFT_INDEX: 19,
-	RIGHT_INDEX: 20,
-	LEFT_HIP: 23,
-	RIGHT_HIP: 24,
-	LEFT_KNEE: 25,
-	RIGHT_KNEE: 26,
-	LEFT_ANKLE: 27,
-	RIGHT_ANKLE: 28,
-	LEFT_HEEL: 29,
-	RIGHT_HEEL: 30,
-	LEFT_FOOT_INDEX: 31,
-	RIGHT_FOOT_INDEX: 32,
-});
-
-const REQUIRED_CORE = [
-	POSE_LANDMARK.LEFT_SHOULDER,
-	POSE_LANDMARK.RIGHT_SHOULDER,
-	POSE_LANDMARK.LEFT_HIP,
-	POSE_LANDMARK.RIGHT_HIP,
-];
-
-function finite(value) {
-	return typeof value === "number" && Number.isFinite(value);
-}
-
-function midpoint(a, b) {
-	return [(a[0] + b[0]) / 2, (a[1] + b[1]) / 2, (a[2] + b[2]) / 2];
-}
-
-function distance(a, b) {
-	return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2]);
-}
-
-function readLandmark(entry, index, minVisibility, mirrored) {
-	if (!entry || !finite(entry.x) || !finite(entry.y) || !finite(entry.z)) return null;
-	const visibility = finite(entry.visibility) ? entry.visibility : 1;
-	if (visibility < minVisibility) return null;
-	return {
-		position: [(mirrored ? -1 : 1) * entry.x, -entry.y, -entry.z],
-		visibility,
-		index,
-	};
-}
-
-/** Normalize one `{ timeS, landmarks }` sample without using DOM or three.js. */
-export function normalizeLandmarkSample(sample, options = {}) {
-	const minVisibility = options.minVisibility ?? 0.35;
-	const mirrored = options.mirrored === true;
-	if (!sample || !finite(sample.timeS) || sample.timeS < 0) {
-		throw new Error("normalizeLandmarkSample: sample.timeS must be a non-negative finite number");
-	}
-	if (!Array.isArray(sample.landmarks) || sample.landmarks.length < 33) {
-		throw new Error("normalizeLandmarkSample: landmarks must contain the 33 MediaPipe Pose landmarks");
-	}
-	const landmarks = sample.landmarks.map((entry, index) =>
-		readLandmark(entry, index, minVisibility, mirrored)
-	);
-	if (REQUIRED_CORE.some((index) => landmarks[index] === null)) {
-		return { timeS: sample.timeS, landmarks, valid: false, scale: null };
-	}
-
-	const leftHip = landmarks[POSE_LANDMARK.LEFT_HIP].position;
-	const rightHip = landmarks[POSE_LANDMARK.RIGHT_HIP].position;
-	const leftShoulder = landmarks[POSE_LANDMARK.LEFT_SHOULDER].position;
-	const rightShoulder = landmarks[POSE_LANDMARK.RIGHT_SHOULDER].position;
-	const hipCentre = midpoint(leftHip, rightHip);
-	const shoulderCentre = midpoint(leftShoulder, rightShoulder);
-	const torso = distance(hipCentre, shoulderCentre);
-	const shoulderWidth = distance(leftShoulder, rightShoulder);
-	const hipWidth = distance(leftHip, rightHip);
-	const scale = torso > 1e-6 ? torso : Math.max(shoulderWidth, hipWidth);
-	if (!(scale > 1e-6)) return { timeS: sample.timeS, landmarks, valid: false, scale: null };
-
-	for (const landmark of landmarks) {
-		if (!landmark) continue;
-		landmark.position = landmark.position.map((value, axis) =>
-			(value - hipCentre[axis]) / scale
-		);
-	}
-	return { timeS: sample.timeS, landmarks, valid: true, scale };
-}
-
-export function normalizeLandmarkTrack(samples, options = {}) {
-	if (!Array.isArray(samples) || samples.length === 0) {
-		throw new Error("normalizeLandmarkTrack: samples must be a non-empty array");
-	}
-	let previous = -Infinity;
-	return samples.map((sample) => {
-		if (sample?.timeS <= previous) {
-			throw new Error("normalizeLandmarkTrack: sample times must be strictly ascending");
-		}
-		previous = sample.timeS;
-		return normalizeLandmarkSample(sample, options);
-	});
-}
-
-function smoothingAlpha(cutoff, dt) {
-	const tau = 1 / (2 * Math.PI * cutoff);
-	return 1 / (1 + tau / dt);
-}
-
-/** One-Euro filter over a normalised track. Missing/low-confidence points stay released. */
-export function filterLandmarkTrack(samples, options = {}) {
-	const minCutoff = options.minCutoff ?? 1;
-	const beta = options.beta ?? 0.08;
-	const derivativeCutoff = options.derivativeCutoff ?? 1;
-	const state = new Array(33).fill(null);
-	return samples.map((sample) => {
-		const landmarks = sample.landmarks.map((landmark, index) => {
-			if (!landmark) return null;
-			const previous = state[index];
-			if (!previous) {
-				state[index] = {
-				timeS: sample.timeS,
-				raw: landmark.position.slice(),
-				filtered: landmark.position.slice(),
-				derivative: [0, 0, 0],
-			};
-				return { ...landmark, position: landmark.position.slice() };
-			}
-			const dt = Math.max(sample.timeS - previous.timeS, 1e-6);
-			const derivativeAlpha = smoothingAlpha(derivativeCutoff, dt);
-			const rawDerivative = landmark.position.map((value, axis) =>
-				(value - previous.raw[axis]) / dt
-			);
-			const derivative = rawDerivative.map((value, axis) =>
-				previous.derivative[axis] + derivativeAlpha * (value - previous.derivative[axis])
-			);
-			const speed = Math.hypot(...derivative);
-			const alpha = smoothingAlpha(minCutoff + beta * speed, dt);
-			const filtered = landmark.position.map((value, axis) =>
-				previous.filtered[axis] + alpha * (value - previous.filtered[axis])
-			);
-			state[index] = { timeS: sample.timeS, raw: landmark.position.slice(), filtered, derivative };
-			return { ...landmark, position: filtered };
-		});
-		return { ...sample, landmarks };
-	});
-}

package/src/pose-extract/mirror.js

@@ -1,153 +0,0 @@
-// mirror.js — test-time augmentation for the single-photograph path.
-//
-// One detection of one still is the least evidence this pipeline ever works
-// from: there is no neighbouring frame to average against and no filter to run,
-// so every landmark error lands whole in the saved pose. Two things are cheap
-// to fix here. Detection noise is roughly independent between two views of the
-// same photograph, so averaging two views halves it. And the model carries a
-// lateral bias — it was trained on natural photographs, which are not
-// left/right symmetric — so a mirrored second pass biases the other way and the
-// average cancels it.
-//
-// The second view is the photograph flipped horizontally. Its detection comes
-// back in flipped coordinates with left and right limbs traded, and
-// mirrorLandmarks puts it back. Everything downstream sees one ordinary
-// 33-landmark array and cannot tell it was measured twice.
-
-import { selectMostConfidentPerson } from "./browser.js";
-
-// MediaPipe Pose topology, left index ↔ right index. Every landmark the model
-// emits in a pair is here; NOSE (0) is the only unpaired one, so it mirrors
-// onto itself. Indices follow the published landmark order:
-//   1,2,3 left eye inner/centre/outer   4,5,6 right eye inner/centre/outer
-//   7 left ear        8 right ear
-//   9 mouth left     10 mouth right
-//  11 shoulder  13 elbow  15 wrist  17 pinky  19 index  21 thumb   (left)
-//  12 shoulder  14 elbow  16 wrist  18 pinky  20 index  22 thumb   (right)
-//  23 hip  25 knee  27 ankle  29 heel  31 foot index                (left)
-//  24 hip  26 knee  28 ankle  30 heel  32 foot index                (right)
-export const LEFT_RIGHT_PAIRS = Object.freeze([
-	[1, 4], [2, 5], [3, 6], // eyes
-	[7, 8], // ears
-	[9, 10], // mouth corners
-	[11, 12], [13, 14], [15, 16], // shoulder, elbow, wrist
-	[17, 18], [19, 20], [21, 22], // pinky, index, thumb
-	[23, 24], [25, 26], [27, 28], // hip, knee, ankle
-	[29, 30], [31, 32], // heel, foot index
-].map((pair) => Object.freeze(pair)));
-
-const POSE_LANDMARK_COUNT = 33;
-
-// index -> the index whose measurement belongs to it after a mirror.
-const MIRROR_INDEX = Object.freeze((() => {
-	const table = new Array(POSE_LANDMARK_COUNT).fill(0).map((_, index) => index);
-	for (const [left, right] of LEFT_RIGHT_PAIRS) {
-		table[left] = right;
-		table[right] = left;
-	}
-	return table;
-})());
-
-function visibilityOf(point) {
-	return Number.isFinite(point?.visibility) ? point.visibility : 1;
-}
-
-/**
- * Un-mirror a detection that ran on a horizontally flipped image: negate X and
- * trade every left/right pair. World landmarks are hip-centred metric
- * coordinates about the body's own origin, so negating X *is* the spatial
- * mirror — no image width enters. Visibility (and any other field) rides along
- * with the landmark it was measured on, because it describes that body part.
- *
- * The transform is its own inverse: mirroring twice returns the input.
- */
-export function mirrorLandmarks(landmarks) {
-	if (!Array.isArray(landmarks) || landmarks.length < POSE_LANDMARK_COUNT) {
-		throw new Error("mirrorLandmarks: landmarks must contain the 33 MediaPipe Pose landmarks");
-	}
-	return landmarks.map((_, index) => {
-		const source = landmarks[MIRROR_INDEX[index]];
-		if (!source) return source ?? null;
-		return { ...source, x: -source.x };
-	});
-}
-
-/**
- * Per-landmark visibility-weighted mean of two detections of the same body.
- * Weighting by visibility lets the view that could actually see a limb carry
- * it; the averaged visibility is the plain mean of the two, not the max,
- * because a part one view could not see is genuinely less certain.
- * A missing landmark on one side yields the other side untouched.
- */
-export function averageLandmarkSets(a, b) {
-	if (!Array.isArray(a)) return Array.isArray(b) ? b : null;
-	if (!Array.isArray(b)) return a;
-	const length = Math.max(a.length, b.length);
-	const averaged = new Array(length);
-	for (let index = 0; index < length; index += 1) {
-		const pa = a[index];
-		const pb = b[index];
-		if (!pa || !pb) {
-			averaged[index] = pa ?? pb ?? null;
-			continue;
-		}
-		const wa = visibilityOf(pa);
-		const wb = visibilityOf(pb);
-		// Two invisible landmarks still have a position worth keeping; fall back
-		// to an unweighted mean rather than dividing by zero.
-		const total = wa + wb > 1e-9 ? wa + wb : 0;
-		const mix = (key) => (total > 0
-			? (pa[key] * wa + pb[key] * wb) / total
-			: (pa[key] + pb[key]) / 2);
-		averaged[index] = {
-			...pa,
-			x: mix("x"),
-			y: mix("y"),
-			z: mix("z"),
-			visibility: (wa + wb) / 2,
-		};
-	}
-	return averaged;
-}
-
-function sizeOf(image) {
-	const width = image?.naturalWidth || image?.videoWidth || image?.width || 0;
-	const height = image?.naturalHeight || image?.videoHeight || image?.height || 0;
-	return { width, height };
-}
-
-/**
- * Detect the photograph and its horizontal mirror, and return the average of
- * the two as one ordinary landmark array (or null if neither view found a
- * person). `detect` is the IMAGE-mode detector's `detect(image)`; `createCanvas`
- * is injectable so this is testable without a DOM.
- *
- * Named failures, in the style of detector.js:
- *   pose-mirror-image-unsized  — the element has no decoded size to flip
- *   pose-mirror-canvas-failed  — no 2d context to draw the flip on
- */
-export async function detectMirrorAveraged(image, detect, { createCanvas } = {}) {
-	if (typeof detect !== "function") throw new Error("detectMirrorAveraged: detect must be a function");
-	const { width, height } = sizeOf(image);
-	if (!(width > 0) || !(height > 0)) throw new Error("pose-mirror-image-unsized");
-	const makeCanvas = createCanvas ?? (() => document.createElement("canvas"));
-	const canvas = makeCanvas();
-	canvas.width = width;
-	canvas.height = height;
-	const context = canvas.getContext?.("2d");
-	if (!context) throw new Error("pose-mirror-canvas-failed");
-	// translate to the right edge, then scale X by -1: the draw walks back
-	// across the canvas and lands the image reversed.
-	context.translate(width, 0);
-	context.scale(-1, 1);
-	context.drawImage(image, 0, 0, width, height);
-
-	// Both passes are the same still at t=0; IMAGE mode ignores the timestamp.
-	const direct = selectMostConfidentPerson(await detect(image, 0));
-	const flipped = selectMostConfidentPerson(await detect(canvas, 0));
-	if (!direct && !flipped) return null;
-	if (!flipped) return direct;
-	const unmirrored = mirrorLandmarks(flipped);
-	if (!direct) return unmirrored;
-	return averageLandmarkSets(direct, unmirrored);
-}

package/src/pose-extract/take.js

@@ -1,131 +0,0 @@
-// take.js — bake a raw landmark track into the exact motion shape the npz
-// decoder returns ({ frames, fps, rotMats, rootPos, posedJoints }), so the
-// existing timeline playback consumes an extracted take with zero new
-// playback code. This is deliberately NOT an ARDY surface: the take is a
-// measurement of the footage, and the generator is never in the loop.
-
-import { CSKEL27_JOINTS } from "../ardy/cskel27.js";
-import { poseToCskel27 } from "../ardy/to-cskel27.js";
-import { fitLandmarksToPose } from "./fit.js";
-import { filterLandmarkTrack, normalizeLandmarkTrack } from "./landmarks.js";
-
-/**
- * `samples` are raw `{ timeS, landmarks }` from collectLandmarkTrack on the
- * uniform `fps` grid, holes allowed — detection loses the person for a frame
- * without warning. Holes hold the previous baked frame and the leading hole
- * backfills from the first fitted one: a previs take must never cut to
- * T-pose because one frame missed. `fitted`/`held` in the result say exactly
- * how much of the take was measured versus held.
- *
- * `releasedBones`/`confidence` carry the fit diagnostics out to the UI, which
- * is the only place that can tell the user why part of a pose did not move.
- * A bone is reported released only when EVERY fitted frame released it: a bone
- * the footage found even once was measured, and warning about it would be a
- * lie. `confidence` is the mean over fitted frames so one bad frame in a long
- * take cannot alone push the take under a warning threshold.
- */
-export function bakeExtractedTake({ samples, rest, fps, durationS, ...options }) {
-	if (!Number.isFinite(fps) || fps <= 0) {
-		throw new Error("bakeExtractedTake: fps must be a positive finite number");
-	}
-	if (!Number.isFinite(durationS) || durationS <= 0) {
-		throw new Error("bakeExtractedTake: durationS must be a positive finite number");
-	}
-	const normalized = normalizeLandmarkTrack(samples, options);
-	const filtered = filterLandmarkTrack(normalized, options.filter);
-	const joints = CSKEL27_JOINTS.length;
-	// Same counting rule as frameCountFor: the frame at t=0 plus every whole
-	// frame inside the duration, so take length and timeline length agree.
-	const frames = Math.max(1, Math.floor(durationS * fps + 1e-6) + 1);
-	const rotMats = new Float32Array(frames * joints * 9);
-	const rootPos = new Float32Array(frames * 3);
-	const posedJoints = new Float32Array(frames * joints * 3);
-
-	const fitByFrame = new Map();
-	let fitted = 0;
-	// Held in fit order, then narrowed frame by frame, so the reported names keep
-	// the skeleton's own ordering instead of Set insertion order.
-	let releasedBones = null;
-	let confidenceSum = 0;
-	for (const sample of filtered) {
-		if (!sample.valid) continue;
-		const frame = Math.round(sample.timeS * fps);
-		if (frame < 0 || frame >= frames) continue;
-		const fit = fitLandmarksToPose({ sample, rest, createdMs: options.createdMs ?? 0 });
-		fitByFrame.set(frame, poseToCskel27({ pose: fit.pose, rest }));
-		const released = new Set(fit.releasedBones);
-		releasedBones = releasedBones === null
-			? [...fit.releasedBones]
-			: releasedBones.filter((name) => released.has(name));
-		confidenceSum += fit.confidence;
-		fitted += 1;
-	}
-	if (fitted === 0) throw new Error("no-usable-pose");
-	const confidence = confidenceSum / fitted;
-
-	const writeFrame = (frame, converted) => {
-		const rotBase = frame * joints * 9;
-		const jointBase = frame * joints * 3;
-		for (let j = 0; j < joints; j += 1) {
-			const m = converted.local_rot_mats[j];
-			const o = rotBase + j * 9;
-			rotMats[o] = m[0][0]; rotMats[o + 1] = m[0][1]; rotMats[o + 2] = m[0][2];
-			rotMats[o + 3] = m[1][0]; rotMats[o + 4] = m[1][1]; rotMats[o + 5] = m[1][2];
-			rotMats[o + 6] = m[2][0]; rotMats[o + 7] = m[2][1]; rotMats[o + 8] = m[2][2];
-			const p = converted.posed_joints[j];
-			posedJoints[jointBase + j * 3] = p[0];
-			posedJoints[jointBase + j * 3 + 1] = p[1];
-			posedJoints[jointBase + j * 3 + 2] = p[2];
-		}
-		rootPos[frame * 3] = converted.posed_joints[0][0];
-		rootPos[frame * 3 + 1] = converted.posed_joints[0][1];
-		rootPos[frame * 3 + 2] = converted.posed_joints[0][2];
-	};
-
-	let last = null;
-	let held = 0;
-	const leading = [];
-	for (let frame = 0; frame < frames; frame += 1) {
-		const converted = fitByFrame.get(frame) ?? null;
-		if (converted) last = converted;
-		if (!last) {
-			leading.push(frame);
-			continue;
-		}
-		if (!converted) held += 1;
-		writeFrame(frame, last);
-	}
-	if (leading.length > 0) {
-		const first = fitByFrame.get(Math.min(...fitByFrame.keys()));
-		for (const frame of leading) writeFrame(frame, first);
-		held += leading.length;
-	}
-	return { frames, fps, rotMats, rootPos, posedJoints, fitted, held, releasedBones, confidence };
-}
-
-// A still has no duration to sample, so the take is one frame wide. Any
-// duration under one second at 1 fps produces exactly that single frame under
-// bakeExtractedTake's own counting rule; naming it here keeps the constant
-// from reading as an arbitrary number at the call site.
-const STILL_FPS = 1;
-const STILL_DURATION_S = 0.5;
-
-/**
- * Bake the one landmark sample a photograph yields into the same motion shape
- * playback consumes, so a still is posed through the take path that footage
- * already proves. The sample is pinned to t=0: a still has no timeline, and a
- * stray timestamp would otherwise bake a held frame instead of a measured one.
- */
-export function bakePoseFrame({ samples, rest, ...options }) {
-	const list = Array.isArray(samples) ? samples : [];
-	// A still that detected nobody is the same outcome as a clip that never
-	// fitted a frame, so it fails by that name rather than as a shape error.
-	if (list.length === 0) throw new Error("no-usable-pose");
-	return bakeExtractedTake({
-		samples: list.slice(0, 1).map((sample) => ({ ...sample, timeS: 0 })),
-		rest,
-		fps: STILL_FPS,
-		durationS: STILL_DURATION_S,
-		...options,
-	});
-}

package/src/pose-extract/video-frames.js

@@ -1,91 +0,0 @@
-// video-frames.js — seek-stepped frame supply for landmark extraction. The
-// video element is injected (same pattern as probeFootage) so the node suite
-// drives exactly the stepping the app runs. Frames come out in ascending time
-// order, which is also the contract detectForVideo enforces on timestamps.
-
-/**
- * The times extraction samples: t=0 plus every whole step inside the
- * duration — the same counting rule as frameCountFor, so a clip's sample
- * count and its timeline frame count agree at equal rates. The final sample
- * is backed off the stream end by 1 ms: a seek to the exact duration lands
- * past the last presentable frame on some decoders.
- */
-export function sampleTimes(durationS, sampleFps) {
-	if (!(durationS > 0) || !(sampleFps > 0)) return [];
-	const count = Math.floor(durationS * sampleFps + 1e-6) + 1;
-	const tail = Math.max(0, durationS - 1e-3);
-	const times = [];
-	for (let i = 0; i < count; i += 1) times.push(Math.min(i / sampleFps, tail));
-	if (times.length > 1 && times[times.length - 1] <= times[times.length - 2]) times.pop();
-	return times;
-}
-
-function waitForData(video, timeoutMs) {
-	if ((video.readyState ?? 0) >= 2) return Promise.resolve();
-	return new Promise((resolve, reject) => {
-		let settled = false;
-		const timer = setTimeout(() => finish(new Error("footage-load-timeout")), timeoutMs);
-		const onLoaded = () => finish(null);
-		const onError = () => finish(new Error("decode-failed"));
-		function finish(error) {
-			if (settled) return;
-			settled = true;
-			clearTimeout(timer);
-			video.removeEventListener?.("loadeddata", onLoaded);
-			video.removeEventListener?.("error", onError);
-			if (error) reject(error);
-			else resolve();
-		}
-		video.addEventListener("loadeddata", onLoaded);
-		video.addEventListener("error", onError);
-		video.load?.();
-	});
-}
-
-function seekTo(video, timeS, timeoutMs) {
-	// The first sample is usually t=0 with the decoder already sitting there;
-	// assigning currentTime=0 again fires no seeked event, so that case must
-	// resolve without waiting for one.
-	if (Math.abs((video.currentTime ?? 0) - timeS) < 1e-4 && (video.readyState ?? 0) >= 2) {
-		return Promise.resolve();
-	}
-	return new Promise((resolve, reject) => {
-		let settled = false;
-		const timer = setTimeout(() => finish(new Error("seek-timeout")), timeoutMs);
-		const onSeeked = () => finish(null);
-		const onError = () => finish(new Error("decode-failed"));
-		function finish(error) {
-			if (settled) return;
-			settled = true;
-			clearTimeout(timer);
-			video.removeEventListener?.("seeked", onSeeked);
-			video.removeEventListener?.("error", onError);
-			if (error) reject(error);
-			else resolve();
-		}
-		video.addEventListener("seeked", onSeeked);
-		video.addEventListener("error", onError);
-		video.currentTime = timeS;
-	});
-}
-
-/**
- * Async-iterate `{ image, timeS }` frames from already-ingested footage.
- * `image` is the video element itself — exactly what detectForVideo accepts —
- * so no canvas copy sits between the decoder and the detector.
- */
-export async function* videoFrames(objectUrl, { createVideo, sampleFps = 20, timeoutMs = 8000 } = {}) {
-	if (typeof createVideo !== "function") throw new Error("videoFrames: createVideo is required");
-	const video = createVideo();
-	video.src = objectUrl;
-	video.muted = true;
-	video.playsInline = true;
-	video.preload = "auto";
-	await waitForData(video, timeoutMs);
-	const times = sampleTimes(Number(video.duration), sampleFps);
-	if (times.length === 0) throw new Error("duration-unreadable");
-	for (const timeS of times) {
-		await seekTo(video, timeS, timeoutMs);
-		yield { image: video, timeS };
-	}
-}