cozyclay 2.1.0 → 3.0.0

package/tools/kimodo/read-npz.mjs

@@ -1,189 +0,0 @@
-/**
- * read-npz.mjs — minimal numpy .npz reader for Kimodo generation output.
- *
- * tools/ardy/npz.mjs writes npz files and src/ardy/npz.js decodes the cclay
- * MOTION npz, but that decoder is bound to the 27-joint cclay contract (member
- * set, joint count, browser DecompressionStream). Kimodo writes a somaskel77
- * file with a different member set, so reading it needs a plain reader that
- * returns members verbatim and leaves interpretation to the caller.
- *
- * `np.savez` writes STORED entries and `np.savez_compressed` writes DEFLATE;
- * both are accepted so the reader does not depend on which one Kimodo uses.
- */
-
-import { readFileSync } from "node:fs";
-import { inflateRawSync } from "node:zlib";
-
-const ZIP_EOCD_SIG = 0x06054b50;
-const ZIP_LOCAL_SIG = 0x04034b50;
-const NPY_MAGIC = "\x93NUMPY";
-
-/** Element readers by numpy dtype string. Little-endian only, which is what
- * numpy writes on every platform CozyClay runs generation on. */
-const DTYPES = {
-	"<f4": { bytes: 4, read: (buf, off) => buf.readFloatLE(off) },
-	"<f8": { bytes: 8, read: (buf, off) => buf.readDoubleLE(off) },
-	"<i4": { bytes: 4, read: (buf, off) => buf.readInt32LE(off) },
-	"<i8": { bytes: 8, read: (buf, off) => Number(buf.readBigInt64LE(off)) },
-	"<u4": { bytes: 4, read: (buf, off) => buf.readUInt32LE(off) },
-	"|b1": { bytes: 1, read: (buf, off) => buf.readUInt8(off) },
-	"|i1": { bytes: 1, read: (buf, off) => buf.readInt8(off) },
-};
-
-function findEocd(buf) {
-	// The comment field is variable-length, so the EOCD is found by scanning
-	// back from the end for its signature.
-	for (let offset = buf.length - 22; offset >= 0; offset -= 1) {
-		if (buf.readUInt32LE(offset) === ZIP_EOCD_SIG) return offset;
-	}
-	throw new Error("readNpz: not a zip archive (no end-of-central-directory record)");
-}
-
-function unzip(buf) {
-	const eocd = findEocd(buf);
-	const entryCount = buf.readUInt16LE(eocd + 10);
-	let cursor = buf.readUInt32LE(eocd + 16);
-	const files = new Map();
-	for (let index = 0; index < entryCount; index += 1) {
-		const method = buf.readUInt16LE(cursor + 10);
-		const compressedSize = buf.readUInt32LE(cursor + 20);
-		const nameLength = buf.readUInt16LE(cursor + 28);
-		const extraLength = buf.readUInt16LE(cursor + 30);
-		const commentLength = buf.readUInt16LE(cursor + 32);
-		const localOffset = buf.readUInt32LE(cursor + 42);
-		const name = buf.toString("utf8", cursor + 46, cursor + 46 + nameLength);
-
-		if (buf.readUInt32LE(localOffset) !== ZIP_LOCAL_SIG) {
-			throw new Error(`readNpz: bad local header for ${name}`);
-		}
-		// The local header repeats the name/extra lengths and they may differ
-		// from the central directory's, so the payload offset is taken here.
-		const localNameLength = buf.readUInt16LE(localOffset + 26);
-		const localExtraLength = buf.readUInt16LE(localOffset + 28);
-		const start = localOffset + 30 + localNameLength + localExtraLength;
-		const payload = buf.subarray(start, start + compressedSize);
-
-		if (method === 0) files.set(name, payload);
-		else if (method === 8) files.set(name, inflateRawSync(payload));
-		else throw new Error(`readNpz: unsupported zip compression method ${method} for ${name}`);
-
-		cursor += 46 + nameLength + extraLength + commentLength;
-	}
-	return files;
-}
-
-function parseNpy(buf, label) {
-	if (buf.toString("latin1", 0, 6) !== NPY_MAGIC) {
-		throw new Error(`readNpz: ${label} is not a .npy payload`);
-	}
-	const major = buf.readUInt8(6);
-	const headerLength = major === 1 ? buf.readUInt16LE(8) : buf.readUInt32LE(8);
-	const headerStart = major === 1 ? 10 : 12;
-	const header = buf.toString("latin1", headerStart, headerStart + headerLength);
-
-	const descr = /'descr'\s*:\s*'([^']+)'/.exec(header);
-	const fortran = /'fortran_order'\s*:\s*(True|False)/.exec(header);
-	const shapeText = /'shape'\s*:\s*\(([^)]*)\)/.exec(header);
-	if (!descr || !fortran || !shapeText) {
-		throw new Error(`readNpz: ${label} has an unreadable .npy header`);
-	}
-	if (fortran[1] === "True") {
-		throw new Error(`readNpz: ${label} is Fortran-ordered; only C order is supported`);
-	}
-	const shape = shapeText[1]
-		.split(",")
-		.map((part) => part.trim())
-		.filter((part) => part.length > 0)
-		.map((part) => {
-			const value = Number(part);
-			if (!Number.isInteger(value) || value < 0) {
-				throw new Error(`readNpz: ${label} has a bad shape entry ${part}`);
-			}
-			return value;
-		});
-
-	const dtype = DTYPES[descr[1]];
-	// A member this reader cannot type is reported, not thrown: real motion npz
-	// files carry non-numeric members alongside the arrays (ARDY writes the
-	// prompt as a `text` member with a unicode dtype), and refusing the whole
-	// archive over one of them would make a readable file unreadable. Callers
-	// that NEED a member still fail, in readKimodoMotion, by name.
-	if (!dtype) return { dtype: descr[1], shape, data: null, unsupported: true };
-
-	const count = shape.reduce((total, dim) => total * dim, 1);
-	const body = buf.subarray(headerStart + headerLength);
-	if (body.length < count * dtype.bytes) {
-		throw new Error(
-			`readNpz: ${label} is truncated (need ${count * dtype.bytes} bytes, have ${body.length})`
-		);
-	}
-	const data = new Float32Array(count);
-	for (let index = 0; index < count; index += 1) {
-		data[index] = dtype.read(body, index * dtype.bytes);
-	}
-	return { dtype: descr[1], shape, data };
-}
-
-/**
- * Read an .npz into `{ [name]: { dtype, shape, data: Float32Array } }`.
- * Values are widened to Float32Array so every member reads the same way;
- * int64 members (numpy's default for a python int, e.g. `fps`) are exact for
- * every magnitude this pipeline uses.
- */
-export function readNpz(path) {
-	const files = unzip(readFileSync(path));
-	const members = {};
-	for (const [name, payload] of files) {
-		if (!name.endsWith(".npy")) continue;
-		members[name.slice(0, -4)] = parseNpy(payload, name);
-	}
-	if (Object.keys(members).length === 0) {
-		throw new Error(`readNpz: ${path} contains no .npy members`);
-	}
-	return members;
-}
-
-/**
- * Pull the arrays soma77ToCskel27Motion needs out of a Kimodo motion npz.
- * Kimodo writes `posed_joints [T,J,3]`, `global_rot_mats [T,J,3,3]` and `fps`;
- * the joint count is read from the data rather than assumed so a G1 or SMPL-X
- * file fails with a clear message instead of silently misreading.
- */
-export function readKimodoMotion(path, { expectedJoints = 77 } = {}) {
-	const members = readNpz(path);
-	for (const required of ["posed_joints", "global_rot_mats"]) {
-		if (!members[required]) {
-			throw new Error(`readKimodoMotion: ${path} is missing ${required}`);
-		}
-		if (members[required].unsupported) {
-			throw new Error(
-				`readKimodoMotion: ${path} stores ${required} as unsupported dtype ${members[required].dtype}`
-			);
-		}
-	}
-	const posed = members.posed_joints;
-	const rots = members.global_rot_mats;
-	if (posed.shape.length !== 3 || posed.shape[2] !== 3) {
-		throw new Error(`readKimodoMotion: posed_joints must be [T,J,3], got [${posed.shape}]`);
-	}
-	if (rots.shape.length !== 4 || rots.shape[2] !== 3 || rots.shape[3] !== 3) {
-		throw new Error(`readKimodoMotion: global_rot_mats must be [T,J,3,3], got [${rots.shape}]`);
-	}
-	const [frames, joints] = posed.shape;
-	if (rots.shape[0] !== frames || rots.shape[1] !== joints) {
-		throw new Error(
-			`readKimodoMotion: posed_joints [${posed.shape}] and global_rot_mats [${rots.shape}] disagree`
-		);
-	}
-	if (joints !== expectedJoints) {
-		throw new Error(
-			`readKimodoMotion: expected a ${expectedJoints}-joint skeleton, got ${joints}. ` +
-				"Generate with a Kimodo-SOMA model — the G1 (34) and SMPL-X (22) skeletons are not supported."
-		);
-	}
-	const fps = members.fps ? Math.round(members.fps.data[0]) : null;
-	if (fps !== null && (!Number.isInteger(fps) || fps < 1)) {
-		throw new Error(`readKimodoMotion: fps member is not a positive integer (${fps})`);
-	}
-	return { frames, joints, fps, posedJoints: posed.data, globalRotMats: rots.data };
-}

package/tools/kimodo/run-edit-on-box.mjs

@@ -1,155 +0,0 @@
-#!/usr/bin/env node
-/**
- * run-edit-on-box.mjs — the Kimodo twin of tools/ardy/run-edit-on-box.sh.
- *
- * Regenerates one span of an existing take and splices it back, writing a cclay
- * motion npz the bridge and app consume exactly like any other generated take.
- *
- * The final stdout line matches the shape bridge.mjs greps for:
- *   run-kimodo-edit: done - <path> (<bytes>)
- */
-
-import { statSync } from "node:fs";
-import { motionArraysToNpzMembers, replaceMotionSegment, writeNpz } from "../ardy/npz.mjs";
-import { retimeMotion } from "../../src/ardy/retime.js";
-import { generateOnBox } from "./generate.mjs";
-import { planEditConstraints } from "./edit.mjs";
-import { readNpz } from "./read-npz.mjs";
-
-function usage(message) {
-	if (message) console.error(`run-kimodo-edit: ${message}`);
-	console.error(
-		"usage: run-edit-on-box.mjs --source <npz> --manifest <json> --prompt <text> " +
-			"[--context-before N] [--context-after N] [--seed N] [--target-fps N] --output <path>"
-	);
-	process.exit(2);
-}
-
-function parseArgs(argv) {
-	const out = { contextBefore: 0, contextAfter: 0, targetFps: 24 };
-	for (let i = 0; i < argv.length; i += 1) {
-		const flag = argv[i];
-		const next = () => argv[(i += 1)];
-		if (flag === "--source") out.source = next();
-		else if (flag === "--manifest") out.manifest = next();
-		else if (flag === "--prompt") out.prompt = next();
-		else if (flag === "--context-before") out.contextBefore = Number(next());
-		else if (flag === "--context-after") out.contextAfter = Number(next());
-		else if (flag === "--seed") out.seed = Number(next());
-		else if (flag === "--target-fps") out.targetFps = Number(next());
-		else if (flag === "--output") out.output = next();
-		else if (flag === "--cpu") { /* accepted for parity with the ARDY wrapper */ }
-		else usage(`unknown argument ${flag}`);
-	}
-	for (const key of ["source", "manifest", "prompt", "output"]) {
-		if (!out[key]) usage(`--${key} is required`);
-	}
-	return out;
-}
-
-const args = parseArgs(process.argv.slice(2));
-
-/** A written cclay motion npz back as the flat arrays replaceMotionSegment wants. */
-function loadMotion(path) {
-	const members = readNpz(path);
-	const frames = members.local_rot_mats.shape[0];
-	return {
-		frames,
-		fps: members.fps ? Math.round(members.fps.data[0]) : 20,
-		rotMats: members.local_rot_mats.data,
-		rootPos: members.root_positions.data,
-		posedJoints: members.posed_joints.data,
-	};
-}
-
-try {
-	const genFps = Number(process.env.CCLAY_KIMODO_GEN_FPS || 30);
-	const plan = planEditConstraints({
-		sourcePath: args.source,
-		manifestPath: args.manifest,
-		contextBefore: args.contextBefore,
-		contextAfter: args.contextAfter,
-		genFps,
-		appFps: args.targetFps,
-	});
-
-	const source = loadMotion(args.source);
-	const durationS = source.frames / source.fps;
-
-	// Which pinning the author's own keys got (contract C5). planEditConstraints
-	// owns the decision — every edited track maps to a limb chain, or nothing does
-	// — and it is announced BEFORE the GPU is booked, because it is the difference
-	// between "this edit moved an arm" and "this edit froze the body", and a take
-	// that came out over-pinned should be diagnosable from the run log alone.
-	console.log(
-		`run-kimodo-edit: constraints=${plan.constraintMode} for ${plan.poses.length} pinned frame(s)` +
-			`${plan.editedTracks.length > 0 ? ` (edited tracks: ${plan.editedTracks.join(", ")})` : ""}` +
-			`${plan.constraintMode === "effector" ? "; context anchors stay fullbody" : ""}`
-	);
-
-	// One prompt over the whole clip, constrained to the source on both sides of
-	// the edit. generateOnBox takes poses already in app frame space and rescales
-	// them itself, so the plan's poses — anchors and author keys, each tagged with
-	// its `kind` — are handed over rather than the plan's already-scaled
-	// constraints: the generator resolves the generation clock from the duration it
-	// is about to request, and scaling the same frames twice against two clocks is
-	// how the pins would drift off the take.
-	const { motion, raw } = await generateOnBox({
-		segments: [{ prompt: args.prompt, duration: durationS }],
-		poses: plan.poses,
-		effectorTracks: plan.constraintMode === "effector" ? plan.editedTracks : null,
-		appFps: args.targetFps,
-		seed: args.seed,
-		onLine: (line) => console.log(line),
-	});
-
-	const sourceFps = Math.round(raw.frames / durationS);
-	const retimed = retimeMotion({ ...motion, fps: sourceFps }, source.fps);
-
-	// Keep ONLY the edited span; everything else stays the author's take.
-	const span = plan.endFrame - plan.startFrame;
-	if (retimed.frames < plan.endFrame) {
-		throw new Error(`regenerated take has ${retimed.frames} frames, need at least ${plan.endFrame}`);
-	}
-	const replacement = {
-		frames: span,
-		fps: retimed.fps,
-		rotMats: retimed.rotMats.slice(plan.startFrame * 27 * 9, plan.endFrame * 27 * 9),
-		rootPos: retimed.rootPos.slice(plan.startFrame * 3, plan.endFrame * 3),
-		posedJoints: retimed.posedJoints.slice(plan.startFrame * 27 * 3, plan.endFrame * 27 * 3),
-	};
-	const edited = replaceMotionSegment(source, replacement, plan.startFrame);
-
-	writeNpz(args.output, motionArraysToNpzMembers(edited));
-	console.log(
-		JSON.stringify({
-			frames: edited.frames,
-			fps: edited.fps,
-			// Field names match ARDY's cclay_motion_edit.py report: the bridge's
-			// tryParseReport recognizes a motion-edit report only by edit_range +
-			// history_range + future_range and silently drops anything else — which
-			// left the App without commit verification and the new take uninstalled.
-			edit_range: [plan.startFrame, plan.endFrame],
-			history_range: [Math.max(0, plan.startFrame - (args.contextBefore || 0)), plan.startFrame],
-			future_range: [plan.endFrame, Math.min(edited.frames, plan.endFrame + (args.contextAfter || 0))],
-			constraints: plan.constraints.length,
-			pinned_frames: plan.constraints[0]?.frame_indices?.length ?? 0,
-			// "effector" = the author's keys pinned only their limb chains (+ hips
-			// and the root, which Kimodo's EE sets pin unconditionally); "fullbody" =
-			// round 1's whole-body pin. Context anchors are fullbody either way.
-			constraint_mode: plan.constraintMode,
-			// The bridge/App commit contract is backend-neutral. Kimodo's
-			// constrained splice has completed successfully at this point, so
-			// report the authored source-frame keys that were carried through.
-			committed_keys: plan.poses
-				.filter((entry) => entry.frame >= plan.startFrame && entry.frame < plan.endFrame)
-				.map((entry) => entry.frame),
-			commit_verified: true,
-		})
-	);
-	const { size } = statSync(args.output);
-	console.log(`run-kimodo-edit: done - ${args.output} (${size} bytes)`);
-} catch (error) {
-	console.error(`run-kimodo-edit: ${error.message}`);
-	process.exit(1);
-}

package/tools/kimodo/run-sequence-on-box.mjs

@@ -1,178 +0,0 @@
-#!/usr/bin/env node
-/**
- * run-sequence-on-box.mjs — the Kimodo twin of tools/ardy/run-sequence-on-box.sh.
- *
- * Generates a multi-segment take with Kimodo on the configured box and writes a
- * cclay motion npz that is byte-compatible with the ARDY path, so the bridge and
- * the app consume it without knowing which backend produced it.
- *
- * FRAME RATE. Kimodo-SOMA generates at 30 fps and the app timeline runs at
- * 24 fps (src/App.jsx). A raw 30 fps take would therefore be read as 1.25x its
- * real duration, so the take is retimed directly to the target rate before it
- * is written. retimeMotion re-grows
- * positions with FK, so bone lengths survive the resample exactly.
- *
- * The final stdout line matches the shape bridge.mjs greps for:
- *   run-kimodo-sequence: done - <path> (<bytes>)
- */
-
-import { statSync } from "node:fs";
-import { readNpz } from "./read-npz.mjs";
-import { writeNpz, motionArraysToNpzMembers } from "../ardy/npz.mjs";
-import { retimeMotion } from "../../src/ardy/retime.js";
-import { generateOnBox, continuityMetrics } from "./generate.mjs";
-
-const TARGET_FPS_DEFAULT = 24;
-
-function usage(message) {
-	if (message) console.error(`run-kimodo-sequence: ${message}`);
-	console.error(
-		'usage: run-sequence-on-box.mjs --segment "<prompt>" <seconds> [--segment ...] ' +
-			"[--root-2d <frame> <x> <z> <heading|none> ...] [--pose <npz> <dst-frame> ...] " +
-			"[--seed N] [--target-fps N] --output <path>"
-	);
-	process.exit(2);
-}
-
-/** A CozyClay pose npz -> the {local_rot_mats, posed_joints} the builder wants. */
-function loadPoseNpz(path) {
-	const members = readNpz(path);
-	for (const key of ["local_rot_mats", "posed_joints"]) {
-		if (!members[key]) throw new Error(`pose npz ${path} is missing ${key}`);
-	}
-	// Written as [1,27,3,3] and [1,27,3] by poseArraysToNpzMembers; the leading
-	// keyframe axis is dropped here so the builder always sees one pose.
-	const rot = members.local_rot_mats;
-	const pos = members.posed_joints;
-	const joints = rot.shape.at(-3);
-	if (joints !== 27) throw new Error(`pose npz ${path} has ${joints} joints, expected 27`);
-	const local_rot_mats = [];
-	for (let j = 0; j < 27; j += 1) {
-		const base = j * 9;
-		local_rot_mats.push([
-			[rot.data[base], rot.data[base + 1], rot.data[base + 2]],
-			[rot.data[base + 3], rot.data[base + 4], rot.data[base + 5]],
-			[rot.data[base + 6], rot.data[base + 7], rot.data[base + 8]],
-		]);
-	}
-	const posed_joints = [];
-	for (let j = 0; j < 27; j += 1) {
-		posed_joints.push([pos.data[j * 3], pos.data[j * 3 + 1], pos.data[j * 3 + 2]]);
-	}
-	return { local_rot_mats, posed_joints };
-}
-
-function parseArgs(argv) {
-	const segments = [];
-	const waypoints = [];
-	const poses = [];
-	let seed;
-	let output;
-	let targetFps = TARGET_FPS_DEFAULT;
-	for (let index = 0; index < argv.length; index += 1) {
-		const flag = argv[index];
-		if (flag === "--root-2d") {
-			// Same 4-token shape as tools/ardy/run-sequence-on-box.sh so a caller
-			// can swap backends without rewriting the path arguments.
-			const frame = Number(argv[index + 1]);
-			const x = Number(argv[index + 2]);
-			const z = Number(argv[index + 3]);
-			const rawHeading = argv[index + 4];
-			if (!Number.isInteger(frame) || !Number.isFinite(x) || !Number.isFinite(z) || rawHeading === undefined) {
-				usage("--root-2d needs FRAME X Z HEADING|none");
-			}
-			const heading = rawHeading === "none" ? null : Number(rawHeading);
-			if (heading !== null && !Number.isFinite(heading)) usage("--root-2d heading must be a number or 'none'");
-			waypoints.push({ frame, x, z, heading });
-			index += 4;
-		} else if (flag === "--pose") {
-			const npzPath = argv[index + 1];
-			const dstFrame = Number(argv[index + 2]);
-			if (!npzPath || !Number.isInteger(dstFrame) || dstFrame < 0) usage("--pose needs NPZ DST-FRAME");
-			poses.push({ frame: dstFrame, npz: npzPath });
-			index += 2;
-		} else if (flag === "--segment") {
-			const prompt = argv[index + 1];
-			const seconds = Number(argv[index + 2]);
-			if (prompt === undefined || !Number.isFinite(seconds)) usage("--segment needs PROMPT SECONDS");
-			segments.push({ prompt, duration: seconds });
-			index += 2;
-		} else if (flag === "--seed") {
-			seed = Number(argv[index += 1]);
-			if (!Number.isInteger(seed)) usage("--seed must be an integer");
-		} else if (flag === "--target-fps") {
-			targetFps = Number(argv[index += 1]);
-			if (!Number.isInteger(targetFps) || targetFps < 1) usage("--target-fps must be a positive integer");
-		} else if (flag === "--output") {
-			output = argv[index += 1];
-		} else if (flag === "--cpu") {
-			// accepted for parity with the ARDY wrapper; Kimodo picks its own
-			// device and already runs the text encoder on the CPU.
-		} else {
-			usage(`unknown argument ${flag}`);
-		}
-	}
-	if (segments.length === 0) usage("at least one --segment is required");
-	if (!output) usage("--output is required");
-	return { segments, waypoints, poses, seed, output, targetFps };
-}
-
-const { segments, waypoints, poses, seed, output, targetFps } = parseArgs(process.argv.slice(2));
-
-try {
-	const requestedS = segments.reduce((total, segment) => total + segment.duration, 0);
-	const { motion, raw, constraints } = await generateOnBox({
-		segments,
-		waypoints,
-		poses: poses.map((entry) => ({ frame: entry.frame, pose: loadPoseNpz(entry.npz) })),
-		// The authored path is in the SAME frame space this script retimes to,
-		// which is the app's clock — so target-fps is also the waypoint clock.
-		appFps: targetFps,
-		seed,
-		onLine: (line) => console.log(line),
-	});
-
-	// Kimodo's npz carries no fps member, so the real rate is recovered from the
-	// frame count against the duration that was actually asked for.
-	const sourceFps = Math.round(raw.frames / requestedS);
-	if (!Number.isFinite(sourceFps) || sourceFps < 1) {
-		throw new Error(`could not infer source fps from ${raw.frames} frames over ${requestedS}s`);
-	}
-	const atSourceRate = { ...motion, fps: sourceFps };
-	const retimed = retimeMotion(atSourceRate, targetFps);
-
-	const members = motionArraysToNpzMembers({
-		frames: retimed.frames,
-		fps: retimed.fps,
-		rotMats: retimed.rotMats,
-		rootPos: retimed.rootPos,
-		posedJoints: retimed.posedJoints,
-	});
-	writeNpz(output, members);
-
-	const boundaries = [];
-	let cursor = 0;
-	for (const segment of segments.slice(0, -1)) {
-		cursor += segment.duration;
-		boundaries.push(Math.round(cursor * retimed.fps));
-	}
-	console.log(
-		JSON.stringify({
-			frames: retimed.frames,
-			fps: retimed.fps,
-			source_fps: sourceFps,
-			model: process.env.CCLAY_KIMODO_MODEL || "Kimodo-SOMA-RP-v1.1",
-			segments: segments.map((segment) => ({ prompt: segment.prompt, requested_s: segment.duration })),
-			boundaries,
-			waypoints: waypoints.length,
-			poses: poses.length,
-			constraints: constraints.length,
-			continuity: continuityMetrics(retimed),
-		})
-	);
-	const { size } = statSync(output);
-	console.log(`run-kimodo-sequence: done - ${output} (${size} bytes)`);
-} catch (error) {
-	console.error(`run-kimodo-sequence: ${error.message}`);
-	process.exit(1);
-}