cozyclay 1.9.0 → 2.0.0

package/tools/bench/obs/mannequin-betas.json

@@ -0,0 +1,124 @@
+{
+  "schema": "obs.mannequin-betas.v1",
+  "method": {
+    "model": "smpl-x supermotion neutral",
+    "optimizer": "projected Adam warm start + bounded SLSQP minimax",
+    "adam_steps": 5000,
+    "adam_lr": 0.005,
+    "minimax_steps": 1500,
+    "minimax_starts": 3,
+    "minimax_success": true,
+    "minimax_status": 0,
+    "beta_limit": 3.0,
+    "smooth_max_weight": 0.0,
+    "smooth_max_temperature": 0.002,
+    "l2_prior": 1e-05,
+    "device": "cpu",
+    "source": "rendered rig rest pose (public/models/y-bot-tpose.fbx), segments: thigh_left, thigh_right, shin_left, shin_right, foot_left, foot_right, pelvis_width, spine_length, upper_arm_left, upper_arm_right, forearm_left, forearm_right, shoulder_width"
+  },
+  "betas": [
+    0.9879521131515503,
+    -0.43811583518981934,
+    0.23403924703598022,
+    1.1289187669754028,
+    0.4586992859840393,
+    -0.2744547724723816,
+    -2.9971723556518555,
+    -3.0,
+    2.999577045440674,
+    2.7470200061798096
+  ],
+  "target_geometry": {
+    "ybot_leg_height_m": 0.9650828829792554,
+    "canonical_cskel27_leg_height_m": 0.9544128,
+    "playback_scale": 1.0111797358326036,
+    "rest_contract": "public/ardy/cskel27-rest.json"
+  },
+  "residuals": {
+    "thigh_left": {
+      "target_m": 0.4059943638074366,
+      "fitted_m": 0.39750587940216064,
+      "residual_m": -0.008488484405275942,
+      "abs_residual_m": 0.008488484405275942
+    },
+    "thigh_right": {
+      "target_m": 0.4059943859177966,
+      "fitted_m": 0.4050813913345337,
+      "residual_m": -0.0009129945832628827,
+      "abs_residual_m": 0.0009129945832628827
+    },
+    "shin_left": {
+      "target_m": 0.42099025912196575,
+      "fitted_m": 0.4308183193206787,
+      "residual_m": 0.009828060198712962,
+      "abs_residual_m": 0.009828060198712962
+    },
+    "shin_right": {
+      "target_m": 0.42099023874729363,
+      "fitted_m": 0.43444767594337463,
+      "residual_m": 0.013457437196081001,
+      "abs_residual_m": 0.013457437196081001
+    },
+    "foot_left": {
+      "target_m": 0.15721559146548023,
+      "fitted_m": 0.14614197611808777,
+      "residual_m": -0.011073615347392457,
+      "abs_residual_m": 0.011073615347392457
+    },
+    "foot_right": {
+      "target_m": 0.15721558809599626,
+      "fitted_m": 0.14439848065376282,
+      "residual_m": -0.012817107442233439,
+      "abs_residual_m": 0.012817107442233439
+    },
+    "pelvis_width": {
+      "target_m": 0.18248906135565482,
+      "fitted_m": 0.15615954995155334,
+      "residual_m": -0.026329511404101474,
+      "abs_residual_m": 0.026329511404101474
+    },
+    "spine_length": {
+      "target_m": 0.3518987587650434,
+      "fitted_m": 0.3289218842983246,
+      "residual_m": -0.02297687446671881,
+      "abs_residual_m": 0.02297687446671881
+    },
+    "upper_arm_left": {
+      "target_m": 0.20827565969956363,
+      "fitted_m": 0.23210719227790833,
+      "residual_m": 0.023831532578344694,
+      "abs_residual_m": 0.023831532578344694
+    },
+    "upper_arm_right": {
+      "target_m": 0.2082745012754011,
+      "fitted_m": 0.2391563355922699,
+      "residual_m": 0.030881834316868806,
+      "abs_residual_m": 0.030881834316868806
+    },
+    "forearm_left": {
+      "target_m": 0.27614463806176104,
+      "fitted_m": 0.2709249258041382,
+      "residual_m": -0.005219712257622855,
+      "abs_residual_m": 0.005219712257622855
+    },
+    "forearm_right": {
+      "target_m": 0.2761446380616249,
+      "fitted_m": 0.26336047053337097,
+      "residual_m": -0.012784167528253954,
+      "abs_residual_m": 0.012784167528253954
+    },
+    "shoulder_width": {
+      "target_m": 0.3752159881593781,
+      "fitted_m": 0.34457817673683167,
+      "residual_m": -0.030637811422546457,
+      "abs_residual_m": 0.030637811422546457
+    }
+  },
+  "summary": {
+    "segment_count": 13,
+    "max_abs_beta": 3.0,
+    "max_abs_residual_m": 0.030881834316868806,
+    "rms_residual_m": 0.018625148220087758,
+    "max_bone_length_residual_m": 0.030881834316868806
+  }
+}

package/tools/bench/obs/remote.mjs

@@ -0,0 +1,83 @@
+import { spawn } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { appendFileSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { assertGpuIdle } from "../fit/remote.mjs";
+
+const sshOptions = ["-o", "BatchMode=yes", "-o", "ConnectTimeout=15"];
+const port = process.env.CCLAY_EXTRACT_SSH_PORT;
+const sshFlags = port ? [...sshOptions, "-p", port] : sshOptions;
+const scpFlags = port ? [...sshOptions, "-P", port] : sshOptions;
+const here = fileURLToPath(new URL("../", import.meta.url));
+let injectedExec = null;
+
+const quote = value => `'${String(value).replaceAll("'", "'\\''")}'`;
+
+function realExec(program, args, log) {
+	return new Promise((resolve, reject) => {
+		const child = spawn(program, args, { stdio: ["ignore", "pipe", "pipe"] });
+		let output = "";
+		const onData = chunk => { output += chunk; if (log) appendFileSync(log, chunk); };
+		child.stdout.on("data", onData); child.stderr.on("data", onData);
+		child.once("error", reject);
+		child.once("close", code => code === 0 ? resolve(output) : reject(new Error(`${program} exited ${code}: ${output.slice(-3000)}`)));
+	});
+}
+
+/** Test hook: replace ssh/scp execution with (program, args, log) => Promise. */
+export function setObsExec(exec) { injectedExec = exec; }
+
+function cameraK(camera) {
+	const value = typeof camera === "string" ? JSON.parse(readFileSync(camera, "utf8")) : camera;
+	const K = value?.K ?? value?.intrinsics?.K ?? value?.intrinsics;
+	if (!Array.isArray(K) || K.length !== 3 || K.some(row => !Array.isArray(row) || row.length !== 3)) {
+		throw new Error("camera must contain a 3x3 K array");
+	}
+	return K;
+}
+
+async function run(program, args, log) {
+	return (injectedExec ?? realExec)(program, args, log);
+}
+
+export async function extractObs({ host, video, camera, output, detector = "palette", keypoints = "vitpose", betas, log, force = false }) {
+	if (!host) throw new Error("missing host");
+	if (!video || !camera || !output) throw new Error("video, camera, and output are required");
+	if (!["palette", "yolo"].includes(detector)) throw new Error(`invalid detector: ${detector}`);
+	if (!["vitpose", "hybrid"].includes(keypoints)) throw new Error(`invalid keypoints: ${keypoints}`);
+	const exec = (program, args) => run(program, args, log);
+	if (!injectedExec) await assertGpuIdle(host);
+	else await exec("ssh", [...sshFlags, host, "python3 -c 'import sys; print(\"GPU idle for GVHMR\")'"]);
+
+	const remote = `/tmp/cclay-obs-${Date.now()}-${randomBytes(5).toString("hex")}`;
+	const localK = `${output}.K-${process.pid}.json`;
+	const K = cameraK(camera);
+	writeFileSync(localK, JSON.stringify(K));
+	let failure;
+	try {
+		await exec("ssh", [...sshFlags, host, `umask 077 && mkdir ${quote(remote)}`]);
+		await exec("scp", [...scpFlags, `${here}/cclay_bench_extract_obs.py`, `${host}:${remote}/cclay_bench_extract_obs.py`]);
+		await exec("scp", [...scpFlags, video, `${host}:${remote}/video.mp4`]);
+		await exec("scp", [...scpFlags, localK, `${host}:${remote}/K.json`]);
+		const args = [
+			`${remote}/cclay_bench_extract_obs.py`,
+			"cclay_gvhmr_extract.py",
+			`${remote}/video.mp4`, `${remote}/obs.npz`, "--K-json", `${remote}/K.json`,
+			"--detector", detector, "--keypoints", keypoints, "--out-root", `${remote}/cache`, "--check-pp",
+		];
+		if (betas) {
+			await exec("scp", [...scpFlags, betas, `${host}:${remote}/betas.json`]);
+			args.push("--betas-json", `${remote}/betas.json`);
+		}
+		await exec("ssh", [...sshFlags, host, `set -e; cd ~/cclay-ingest/GVHMR; .venv/bin/python ${args.map(quote).join(" ")}`]);
+		await exec("scp", [...scpFlags, `${host}:${remote}/obs.npz`, output]);
+	} catch (error) {
+		failure = error;
+	} finally {
+		try { unlinkSync(localK); } catch {}
+		try { await exec("ssh", [...sshFlags, host, `rm -rf ${quote(remote)}`]); }
+		catch (error) { failure = failure ? new AggregateError([failure, error], `remote cleanup failed: ${remote}`) : error; }
+	}
+	if (failure) throw failure;
+	return { output, remote };
+}

package/tools/bench/obs/rest_joints.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+"""SMPL rest joints for one betas vector (box helper for tools/bench/obs-bench.mjs).
+
+Run from ~/cclay-ingest/GVHMR with its venv:
+    .venv/bin/python rest_joints.py '<json list of 10 betas>'
+
+Reproduces cclay_gvhmr_extract.py main()'s ``smpl_rest_joints`` exactly:
+SMPL-X ``make_smplx("supermotion")`` zero pose for ``betas`` -> smplx2smpl
+sparse vertices -> neutral SMPL J-regressor, pelvis moved to the origin.
+Also reports the rest mesh's lowest vertex height (pelvis-relative), which
+gives the body's sole clearance under each foot joint. CPU only: the GPU is
+reserved for serial GVHMR extraction, so CUDA is hidden before torch loads.
+"""
+import json
+import os
+import sys
+
+os.environ["CUDA_VISIBLE_DEVICES"] = ""
+
+import numpy as np
+import torch
+
+from hmr4d.utils.smplx_utils import make_smplx
+
+
+def main():
+    assert not torch.cuda.is_available(), "rest_joints.py must not see the GPU"
+    betas = np.asarray(json.loads(sys.argv[1]), dtype=np.float32)
+    if betas.shape != (10,) or not np.isfinite(betas).all():
+        raise ValueError("expected 10 finite betas")
+    smplx = make_smplx("supermotion")
+    sparse = torch.load("hmr4d/utils/body_model/smplx2smpl_sparse.pt", map_location="cpu")
+    regressor = torch.load("hmr4d/utils/body_model/smpl_neutral_J_regressor.pt", map_location="cpu")
+    with torch.no_grad():
+        rest = smplx(betas=torch.from_numpy(betas)[None], body_pose=torch.zeros(1, 63),
+                     global_orient=torch.zeros(1, 3), transl=torch.zeros(1, 3))
+        verts = torch.matmul(sparse, rest.vertices[0])
+        joints = torch.einsum("jv,vi->ji", regressor, verts)
+        pelvis = joints[0].clone()
+        joints = joints - pelvis
+        min_y = float((verts[:, 1] - pelvis[1]).min())
+    print(json.dumps({
+        "schema": "obs.rest-joints.v1",
+        "betas": [float(v) for v in betas],
+        "restJoints": joints.numpy().astype(np.float32).tolist(),
+        "restMinVertexY": min_y,
+        "model": "make_smplx('supermotion') zero pose -> smplx2smpl_sparse -> smpl_neutral_J_regressor; pelvis at origin",
+    }))
+
+
+if __name__ == "__main__":
+    main()

package/tools/bench/obs/ybot-targets.mjs

@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+// Body-size targets for fit_mannequin_betas.py, measured on the character that
+// is actually rendered: the y-bot FBX rig in its rest (T) pose, at the Studio's
+// 0.01 armature scale. Only the character model is read (no motion, no video).
+// Usage: node tools/bench/obs/ybot-targets.mjs [model] > targets.json
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import * as THREE from "three";
+import { FBXLoader } from "three/examples/jsm/loaders/FBXLoader.js";
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), "../../..");
+
+// Segment name -> Mixamo bone pair, matching fit_mannequin_betas.py SEGMENTS
+// (SMPL joint pairs there). Upper arm: the SMPL spine3 -> shoulder span maps to
+// Spine2 -> Arm on the rig, the same visible shoulder reach.
+export const RIG_SEGMENTS = {
+	thigh_left: ["LeftUpLeg", "LeftLeg"], thigh_right: ["RightUpLeg", "RightLeg"],
+	shin_left: ["LeftLeg", "LeftFoot"], shin_right: ["RightLeg", "RightFoot"],
+	foot_left: ["LeftFoot", "LeftToeBase"], foot_right: ["RightFoot", "RightToeBase"],
+	pelvis_width: ["LeftUpLeg", "RightUpLeg"], spine_length: ["Hips", "Spine2"],
+	upper_arm_left: ["Spine2", "LeftArm"], upper_arm_right: ["Spine2", "RightArm"],
+	forearm_left: ["LeftForeArm", "LeftHand"], forearm_right: ["RightForeArm", "RightHand"],
+	shoulder_width: ["LeftArm", "RightArm"],
+};
+
+export function ybotTargets(model = "y-bot-tpose") {
+	const path = ["public", "dist"].map((d) => join(ROOT, d, "models", `${model}.fbx`)).find(existsSync);
+	if (!path) throw new Error(`character model ${model}.fbx not found under public/ or dist/models`);
+	const bytes = readFileSync(path);
+	const rig = new FBXLoader().parse(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength), "");
+	rig.scale.setScalar(0.01);
+	rig.updateMatrixWorld(true);
+	const bones = new Map();
+	rig.traverse((o) => { if (o.isBone) bones.set(o.name.replace(/^mixamorig:?/, ""), o); });
+	const at = (name) => {
+		const bone = bones.get(name);
+		if (!bone) throw new Error(`${model}: no bone ${name}`);
+		return new THREE.Vector3().setFromMatrixPosition(bone.matrixWorld);
+	};
+	const targets = Object.fromEntries(Object.entries(RIG_SEGMENTS).map(([name, [a, b]]) => [name, at(a).distanceTo(at(b))]));
+	// Standing height: the skinned mesh's rest bounding box (sole to crown).
+	const box = new THREE.Box3();
+	rig.traverse((o) => { if (o.isMesh) { o.geometry.computeBoundingBox(); box.union(o.geometry.boundingBox.clone().applyMatrix4(o.matrixWorld)); } });
+	targets.total_height = box.max.y - box.min.y;
+	return { schema: "obs.ybot-targets.v1", model: path.slice(ROOT.length + 1), scale: 0.01, pose: "FBX rest (T) pose", targets };
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) console.log(JSON.stringify(ybotTargets(process.argv[2]), null, 1));