cozyclay 2.1.0 → 3.0.0

package/tools/bench/obs/mannequin-betas.json

@@ -1,124 +0,0 @@
-{
-  "schema": "obs.mannequin-betas.v1",
-  "method": {
-    "model": "smpl-x supermotion neutral",
-    "optimizer": "projected Adam warm start + bounded SLSQP minimax",
-    "adam_steps": 5000,
-    "adam_lr": 0.005,
-    "minimax_steps": 1500,
-    "minimax_starts": 3,
-    "minimax_success": true,
-    "minimax_status": 0,
-    "beta_limit": 3.0,
-    "smooth_max_weight": 0.0,
-    "smooth_max_temperature": 0.002,
-    "l2_prior": 1e-05,
-    "device": "cpu",
-    "source": "rendered rig rest pose (public/models/y-bot-tpose.fbx), segments: thigh_left, thigh_right, shin_left, shin_right, foot_left, foot_right, pelvis_width, spine_length, upper_arm_left, upper_arm_right, forearm_left, forearm_right, shoulder_width"
-  },
-  "betas": [
-    0.9879521131515503,
-    -0.43811583518981934,
-    0.23403924703598022,
-    1.1289187669754028,
-    0.4586992859840393,
-    -0.2744547724723816,
-    -2.9971723556518555,
-    -3.0,
-    2.999577045440674,
-    2.7470200061798096
-  ],
-  "target_geometry": {
-    "ybot_leg_height_m": 0.9650828829792554,
-    "canonical_cskel27_leg_height_m": 0.9544128,
-    "playback_scale": 1.0111797358326036,
-    "rest_contract": "public/ardy/cskel27-rest.json"
-  },
-  "residuals": {
-    "thigh_left": {
-      "target_m": 0.4059943638074366,
-      "fitted_m": 0.39750587940216064,
-      "residual_m": -0.008488484405275942,
-      "abs_residual_m": 0.008488484405275942
-    },
-    "thigh_right": {
-      "target_m": 0.4059943859177966,
-      "fitted_m": 0.4050813913345337,
-      "residual_m": -0.0009129945832628827,
-      "abs_residual_m": 0.0009129945832628827
-    },
-    "shin_left": {
-      "target_m": 0.42099025912196575,
-      "fitted_m": 0.4308183193206787,
-      "residual_m": 0.009828060198712962,
-      "abs_residual_m": 0.009828060198712962
-    },
-    "shin_right": {
-      "target_m": 0.42099023874729363,
-      "fitted_m": 0.43444767594337463,
-      "residual_m": 0.013457437196081001,
-      "abs_residual_m": 0.013457437196081001
-    },
-    "foot_left": {
-      "target_m": 0.15721559146548023,
-      "fitted_m": 0.14614197611808777,
-      "residual_m": -0.011073615347392457,
-      "abs_residual_m": 0.011073615347392457
-    },
-    "foot_right": {
-      "target_m": 0.15721558809599626,
-      "fitted_m": 0.14439848065376282,
-      "residual_m": -0.012817107442233439,
-      "abs_residual_m": 0.012817107442233439
-    },
-    "pelvis_width": {
-      "target_m": 0.18248906135565482,
-      "fitted_m": 0.15615954995155334,
-      "residual_m": -0.026329511404101474,
-      "abs_residual_m": 0.026329511404101474
-    },
-    "spine_length": {
-      "target_m": 0.3518987587650434,
-      "fitted_m": 0.3289218842983246,
-      "residual_m": -0.02297687446671881,
-      "abs_residual_m": 0.02297687446671881
-    },
-    "upper_arm_left": {
-      "target_m": 0.20827565969956363,
-      "fitted_m": 0.23210719227790833,
-      "residual_m": 0.023831532578344694,
-      "abs_residual_m": 0.023831532578344694
-    },
-    "upper_arm_right": {
-      "target_m": 0.2082745012754011,
-      "fitted_m": 0.2391563355922699,
-      "residual_m": 0.030881834316868806,
-      "abs_residual_m": 0.030881834316868806
-    },
-    "forearm_left": {
-      "target_m": 0.27614463806176104,
-      "fitted_m": 0.2709249258041382,
-      "residual_m": -0.005219712257622855,
-      "abs_residual_m": 0.005219712257622855
-    },
-    "forearm_right": {
-      "target_m": 0.2761446380616249,
-      "fitted_m": 0.26336047053337097,
-      "residual_m": -0.012784167528253954,
-      "abs_residual_m": 0.012784167528253954
-    },
-    "shoulder_width": {
-      "target_m": 0.3752159881593781,
-      "fitted_m": 0.34457817673683167,
-      "residual_m": -0.030637811422546457,
-      "abs_residual_m": 0.030637811422546457
-    }
-  },
-  "summary": {
-    "segment_count": 13,
-    "max_abs_beta": 3.0,
-    "max_abs_residual_m": 0.030881834316868806,
-    "rms_residual_m": 0.018625148220087758,
-    "max_bone_length_residual_m": 0.030881834316868806
-  }
-}

package/tools/bench/obs/remote.mjs

@@ -1,83 +0,0 @@
-import { spawn } from "node:child_process";
-import { randomBytes } from "node:crypto";
-import { appendFileSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
-import { fileURLToPath } from "node:url";
-import { assertGpuIdle } from "../fit/remote.mjs";
-
-const sshOptions = ["-o", "BatchMode=yes", "-o", "ConnectTimeout=15"];
-const port = process.env.CCLAY_EXTRACT_SSH_PORT;
-const sshFlags = port ? [...sshOptions, "-p", port] : sshOptions;
-const scpFlags = port ? [...sshOptions, "-P", port] : sshOptions;
-const here = fileURLToPath(new URL("../", import.meta.url));
-let injectedExec = null;
-
-const quote = value => `'${String(value).replaceAll("'", "'\\''")}'`;
-
-function realExec(program, args, log) {
-	return new Promise((resolve, reject) => {
-		const child = spawn(program, args, { stdio: ["ignore", "pipe", "pipe"] });
-		let output = "";
-		const onData = chunk => { output += chunk; if (log) appendFileSync(log, chunk); };
-		child.stdout.on("data", onData); child.stderr.on("data", onData);
-		child.once("error", reject);
-		child.once("close", code => code === 0 ? resolve(output) : reject(new Error(`${program} exited ${code}: ${output.slice(-3000)}`)));
-	});
-}
-
-/** Test hook: replace ssh/scp execution with (program, args, log) => Promise. */
-export function setObsExec(exec) { injectedExec = exec; }
-
-function cameraK(camera) {
-	const value = typeof camera === "string" ? JSON.parse(readFileSync(camera, "utf8")) : camera;
-	const K = value?.K ?? value?.intrinsics?.K ?? value?.intrinsics;
-	if (!Array.isArray(K) || K.length !== 3 || K.some(row => !Array.isArray(row) || row.length !== 3)) {
-		throw new Error("camera must contain a 3x3 K array");
-	}
-	return K;
-}
-
-async function run(program, args, log) {
-	return (injectedExec ?? realExec)(program, args, log);
-}
-
-export async function extractObs({ host, video, camera, output, detector = "palette", keypoints = "vitpose", betas, log, force = false }) {
-	if (!host) throw new Error("missing host");
-	if (!video || !camera || !output) throw new Error("video, camera, and output are required");
-	if (!["palette", "yolo"].includes(detector)) throw new Error(`invalid detector: ${detector}`);
-	if (!["vitpose", "hybrid"].includes(keypoints)) throw new Error(`invalid keypoints: ${keypoints}`);
-	const exec = (program, args) => run(program, args, log);
-	if (!injectedExec) await assertGpuIdle(host);
-	else await exec("ssh", [...sshFlags, host, "python3 -c 'import sys; print(\"GPU idle for GVHMR\")'"]);
-
-	const remote = `/tmp/cclay-obs-${Date.now()}-${randomBytes(5).toString("hex")}`;
-	const localK = `${output}.K-${process.pid}.json`;
-	const K = cameraK(camera);
-	writeFileSync(localK, JSON.stringify(K));
-	let failure;
-	try {
-		await exec("ssh", [...sshFlags, host, `umask 077 && mkdir ${quote(remote)}`]);
-		await exec("scp", [...scpFlags, `${here}/cclay_bench_extract_obs.py`, `${host}:${remote}/cclay_bench_extract_obs.py`]);
-		await exec("scp", [...scpFlags, video, `${host}:${remote}/video.mp4`]);
-		await exec("scp", [...scpFlags, localK, `${host}:${remote}/K.json`]);
-		const args = [
-			`${remote}/cclay_bench_extract_obs.py`,
-			"cclay_gvhmr_extract.py",
-			`${remote}/video.mp4`, `${remote}/obs.npz`, "--K-json", `${remote}/K.json`,
-			"--detector", detector, "--keypoints", keypoints, "--out-root", `${remote}/cache`, "--check-pp",
-		];
-		if (betas) {
-			await exec("scp", [...scpFlags, betas, `${host}:${remote}/betas.json`]);
-			args.push("--betas-json", `${remote}/betas.json`);
-		}
-		await exec("ssh", [...sshFlags, host, `set -e; cd ~/cclay-ingest/GVHMR; .venv/bin/python ${args.map(quote).join(" ")}`]);
-		await exec("scp", [...scpFlags, `${host}:${remote}/obs.npz`, output]);
-	} catch (error) {
-		failure = error;
-	} finally {
-		try { unlinkSync(localK); } catch {}
-		try { await exec("ssh", [...sshFlags, host, `rm -rf ${quote(remote)}`]); }
-		catch (error) { failure = failure ? new AggregateError([failure, error], `remote cleanup failed: ${remote}`) : error; }
-	}
-	if (failure) throw failure;
-	return { output, remote };
-}

package/tools/bench/obs/rest_joints.py

@@ -1,52 +0,0 @@
-#!/usr/bin/env python3
-"""SMPL rest joints for one betas vector (box helper for tools/bench/obs-bench.mjs).
-
-Run from ~/cclay-ingest/GVHMR with its venv:
-    .venv/bin/python rest_joints.py '<json list of 10 betas>'
-
-Reproduces cclay_gvhmr_extract.py main()'s ``smpl_rest_joints`` exactly:
-SMPL-X ``make_smplx("supermotion")`` zero pose for ``betas`` -> smplx2smpl
-sparse vertices -> neutral SMPL J-regressor, pelvis moved to the origin.
-Also reports the rest mesh's lowest vertex height (pelvis-relative), which
-gives the body's sole clearance under each foot joint. CPU only: the GPU is
-reserved for serial GVHMR extraction, so CUDA is hidden before torch loads.
-"""
-import json
-import os
-import sys
-
-os.environ["CUDA_VISIBLE_DEVICES"] = ""
-
-import numpy as np
-import torch
-
-from hmr4d.utils.smplx_utils import make_smplx
-
-
-def main():
-    assert not torch.cuda.is_available(), "rest_joints.py must not see the GPU"
-    betas = np.asarray(json.loads(sys.argv[1]), dtype=np.float32)
-    if betas.shape != (10,) or not np.isfinite(betas).all():
-        raise ValueError("expected 10 finite betas")
-    smplx = make_smplx("supermotion")
-    sparse = torch.load("hmr4d/utils/body_model/smplx2smpl_sparse.pt", map_location="cpu")
-    regressor = torch.load("hmr4d/utils/body_model/smpl_neutral_J_regressor.pt", map_location="cpu")
-    with torch.no_grad():
-        rest = smplx(betas=torch.from_numpy(betas)[None], body_pose=torch.zeros(1, 63),
-                     global_orient=torch.zeros(1, 3), transl=torch.zeros(1, 3))
-        verts = torch.matmul(sparse, rest.vertices[0])
-        joints = torch.einsum("jv,vi->ji", regressor, verts)
-        pelvis = joints[0].clone()
-        joints = joints - pelvis
-        min_y = float((verts[:, 1] - pelvis[1]).min())
-    print(json.dumps({
-        "schema": "obs.rest-joints.v1",
-        "betas": [float(v) for v in betas],
-        "restJoints": joints.numpy().astype(np.float32).tolist(),
-        "restMinVertexY": min_y,
-        "model": "make_smplx('supermotion') zero pose -> smplx2smpl_sparse -> smpl_neutral_J_regressor; pelvis at origin",
-    }))
-
-
-if __name__ == "__main__":
-    main()

package/tools/bench/obs/ybot-targets.mjs

@@ -1,49 +0,0 @@
-#!/usr/bin/env node
-// Body-size targets for fit_mannequin_betas.py, measured on the character that
-// is actually rendered: the y-bot FBX rig in its rest (T) pose, at the Studio's
-// 0.01 armature scale. Only the character model is read (no motion, no video).
-// Usage: node tools/bench/obs/ybot-targets.mjs [model] > targets.json
-import { existsSync, readFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
-import * as THREE from "three";
-import { FBXLoader } from "three/examples/jsm/loaders/FBXLoader.js";
-
-const ROOT = join(dirname(fileURLToPath(import.meta.url)), "../../..");
-
-// Segment name -> Mixamo bone pair, matching fit_mannequin_betas.py SEGMENTS
-// (SMPL joint pairs there). Upper arm: the SMPL spine3 -> shoulder span maps to
-// Spine2 -> Arm on the rig, the same visible shoulder reach.
-export const RIG_SEGMENTS = {
-	thigh_left: ["LeftUpLeg", "LeftLeg"], thigh_right: ["RightUpLeg", "RightLeg"],
-	shin_left: ["LeftLeg", "LeftFoot"], shin_right: ["RightLeg", "RightFoot"],
-	foot_left: ["LeftFoot", "LeftToeBase"], foot_right: ["RightFoot", "RightToeBase"],
-	pelvis_width: ["LeftUpLeg", "RightUpLeg"], spine_length: ["Hips", "Spine2"],
-	upper_arm_left: ["Spine2", "LeftArm"], upper_arm_right: ["Spine2", "RightArm"],
-	forearm_left: ["LeftForeArm", "LeftHand"], forearm_right: ["RightForeArm", "RightHand"],
-	shoulder_width: ["LeftArm", "RightArm"],
-};
-
-export function ybotTargets(model = "y-bot-tpose") {
-	const path = ["public", "dist"].map((d) => join(ROOT, d, "models", `${model}.fbx`)).find(existsSync);
-	if (!path) throw new Error(`character model ${model}.fbx not found under public/ or dist/models`);
-	const bytes = readFileSync(path);
-	const rig = new FBXLoader().parse(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength), "");
-	rig.scale.setScalar(0.01);
-	rig.updateMatrixWorld(true);
-	const bones = new Map();
-	rig.traverse((o) => { if (o.isBone) bones.set(o.name.replace(/^mixamorig:?/, ""), o); });
-	const at = (name) => {
-		const bone = bones.get(name);
-		if (!bone) throw new Error(`${model}: no bone ${name}`);
-		return new THREE.Vector3().setFromMatrixPosition(bone.matrixWorld);
-	};
-	const targets = Object.fromEntries(Object.entries(RIG_SEGMENTS).map(([name, [a, b]]) => [name, at(a).distanceTo(at(b))]));
-	// Standing height: the skinned mesh's rest bounding box (sole to crown).
-	const box = new THREE.Box3();
-	rig.traverse((o) => { if (o.isMesh) { o.geometry.computeBoundingBox(); box.union(o.geometry.boundingBox.clone().applyMatrix4(o.matrixWorld)); } });
-	targets.total_height = box.max.y - box.min.y;
-	return { schema: "obs.ybot-targets.v1", model: path.slice(ROOT.length + 1), scale: 0.01, pose: "FBX rest (T) pose", targets };
-}
-
-if (import.meta.url === `file://${process.argv[1]}`) console.log(JSON.stringify(ybotTargets(process.argv[2]), null, 1));