@rig-c/pose 0.12.0 → 0.13.0

package/README.md

@@ -190,6 +190,8 @@ Exit codes: `0` written, `1` refused (every reason printed, nothing written), `2
 **From COCO-17** — the layout most pose models and datasets use — `rig-pose convert --coco in.json --out pose.json`, or `retarget --coco`. The input is `{ "format": "coco17-sequence/1", "width", "height", "source", "frames": [{ "t", "keypoints" }] }`, keypoints in COCO's order as 17 `[x, y, score]` or 51 numbers, or `null` for a frame with nobody. The mapping: COCO's `left_*`/`right_*` → body-18's `l_*`/`r_*` (both are the subject's sides), a score of 0 → `missing`, and body-18's `neck`, which COCO lacks, is the shoulders' midpoint scored the lower of the two (OpenPose's definition).
 
 
+**The camera** ([#63](https://github.com/rig-c/pose/issues/63)). A sequence may carry an optional `camera`, one `{ "s", "tx", "ty", "inliers" }` per frame: the picture's scale and translation against frame 0's, so a point p₀ of frame 0's picture appears at `s · p₀ + (tx, ty)` on that frame. Its joints are then already normalised by it, `p′ = (p − t) / s`, and `inliers` is how many background features the fit kept. `s` must be > 0, `tx` and `ty` finite, `inliers` an integer ≥ 0, and there is one entry per frame; anything else is refused by name (`POSE_CAMERA`). The retarget does not apply it again: it prints a `CAMERA` line with the range of `s` and the three frames where it changes fastest. With `--plant-feet` and no `camera`, the `CAMERA` line says the floor may step with the picture, since a zoom or a pan in the video moves the floor with it. An optional `cameraSource` string says where the camera came from: `"tracked"` or `"hand-labelled, N keys"`. It is refused without a `camera`, or when it is not a non-empty string (`POSE_CAMERA_SOURCE`), and the `CAMERA` line starts with it (`CAMERA tracked: …`; a sequence with a `camera` and no `cameraSource` prints `CAMERA (source not stated): …`). `tools/scan.py` (not shipped) reads the camera off the video's background and writes the normalised sequence with `"cameraSource": "tracked"`. A frame whose fit keeps too few background features is refused by name, and nothing is written. Where the background cannot be tracked, `tools/scan.py --camera-file` takes a hand-written camera instead, a `rig-pose-camera/1` file: `{ "spec": "rig-pose-camera/1", "keys": [ { "frame", "s", "tx", "ty" } ], "note"? }`. Its frames are integers of the pose and strictly increase, `s` is > 0, and `tx` and `ty` are finite. Values are linear between keys and held before the first key and after the last. `inliers` is 0 on every frame, since no feature was fitted, and anything malformed is refused by name (`CAMERA_FILE`). `tools/scan_selftest.py` is the synthetic check of both paths.
+
 **The older name.** `rig-c-pose-frames/0`, written by the extraction jobs this package's demos were made from, is the same file under another spec string. It has `space.units` "painting-px" and an `fps` key; each frame's `t` is the time read. It is read as it is.
 ## The map: `rig-pose-map/1`
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rig-c/pose",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "Puts motion from a body-18 pose sequence onto a rig built with rig-c: bone keys in rig-c's motion format, checked through rig-c's gate before anything is written. A CLI for agents that cannot see the rig.",
   "type": "module",
   "bin": {

package/src/pose.ts

@@ -39,16 +39,35 @@ export interface PoseFrame {
   t: number;
   people: Person[];
 }
+/**
+ * The picture's camera on one frame against frame 0's (rig-c/pose#63), as
+ * `tools/scan.py` reads it off the background: a point p₀ of frame 0's picture
+ * appears at `s · p₀ + (tx, ty)` on this frame. The joints of a sequence that
+ * carries a camera are already normalised by it, `p′ = (p − t) / s`. `inliers`
+ * is the number of background features the fit kept.
+ */
+export interface CameraFrame {
+  s: number;
+  tx: number;
+  ty: number;
+  inliers: number;
+}
 export interface PoseSequence {
   spec: typeof POSE_SPEC;
   space: typeof POSE_SPACE;
   width: number;
   height: number;
   source: string;
+  /** Optional, only with `camera`: where it came from — "tracked", or "hand-labelled, N keys" (#63). */
+  cameraSource?: string;
+  /** Optional: one per frame, when a scan read the camera (#63). */
+  camera?: CameraFrame[];
   frames: PoseFrame[];
 }
 
 const TOP_KEYS = ['spec', 'space', 'width', 'height', 'source', 'frames'];
+const OPTIONAL_KEYS = ['cameraSource', 'camera'];
+const CAMERA_KEYS = ['s', 'tx', 'ty', 'inliers'];
 
 /**
  * The older name the same file went by: `rig-c-pose-frames/0`, written by the
@@ -70,7 +89,7 @@ export function readPoseSequence(text: string, where: string): PoseSequence {
     throw new Error('unreachable');
   }
   const frames0 = doc.spec === POSE_SPEC_FRAMES0;
-  refuseUnknownKeys(p, 'POSE_KEYS', where, doc, frames0 ? [...TOP_KEYS, 'fps'] : TOP_KEYS);
+  refuseUnknownKeys(p, 'POSE_KEYS', where, doc, frames0 ? [...TOP_KEYS, ...OPTIONAL_KEYS, 'fps'] : [...TOP_KEYS, ...OPTIONAL_KEYS]);
   for (const k of TOP_KEYS) if (!(k in doc)) p.add('POSE_KEYS', `${where} has no "${k}"; a ${POSE_SPEC} document carries ${TOP_KEYS.join(', ')}`);
   if ('spec' in doc && doc.spec !== POSE_SPEC && !frames0) p.add('POSE_SPEC', `${where} "spec" is ${typeName(doc.spec)}; this reader reads "${POSE_SPEC}" (and the older name "${POSE_SPEC_FRAMES0}")`);
   const space = frames0 ? FRAMES0_SPACE : POSE_SPACE;
@@ -127,8 +146,52 @@ export function readPoseSequence(text: string, where: string): PoseSequence {
       });
     }
   }
+  const camera = 'camera' in doc ? readCamera(p, where, doc.camera, Array.isArray(doc.frames) ? doc.frames.length : null) : undefined;
+  if ('cameraSource' in doc) {
+    if (typeof doc.cameraSource !== 'string' || doc.cameraSource.trim() === '') p.add('POSE_CAMERA_SOURCE', `${where} "cameraSource" is ${typeName(doc.cameraSource)}; it must be a non-empty string saying where the camera came from ("tracked", "hand-labelled, N keys")`);
+    else if (!('camera' in doc)) p.add('POSE_CAMERA_SOURCE', `${where} has "cameraSource" ${JSON.stringify(doc.cameraSource)} and no "camera"; it states where a camera came from, so it needs one`);
+  }
   p.throwIfAny();
-  return { spec: POSE_SPEC, space: POSE_SPACE, width: doc.width as number, height: doc.height as number, source: doc.source as string, frames };
+  const cameraSource = typeof doc.cameraSource === 'string' ? doc.cameraSource : undefined;
+  return { spec: POSE_SPEC, space: POSE_SPACE, width: doc.width as number, height: doc.height as number, source: doc.source as string, ...(cameraSource !== undefined ? { cameraSource } : {}), ...(camera !== undefined ? { camera } : {}), frames };
+}
+
+/** `camera`: one `{ s > 0, tx, ty, inliers ≥ 0 }` per frame, as many as `frames`. */
+function readCamera(p: Problems, where: string, camera: unknown, frames: number | null): CameraFrame[] {
+  const out: CameraFrame[] = [];
+  if (!Array.isArray(camera)) {
+    p.add('POSE_CAMERA', `${where} "camera" is ${typeName(camera)}; it must be an array with one { s, tx, ty, inliers } per frame`);
+    return out;
+  }
+  if (frames !== null && camera.length !== frames) p.add('POSE_CAMERA', `${where} "camera" has ${camera.length} entr${camera.length === 1 ? 'y' : 'ies'} and "frames" ${frames}; it must have one per frame`);
+  camera.forEach((c: unknown, i: number) => {
+    const cw = `${where} camera[${i}]`;
+    if (!isObject(c)) {
+      p.add('POSE_CAMERA', `${cw} is ${typeName(c)}; a camera entry is { s, tx, ty, inliers }`);
+      return;
+    }
+    refuseUnknownKeys(p, 'POSE_CAMERA', cw, c, CAMERA_KEYS);
+    let ok = true;
+    for (const k of CAMERA_KEYS) if (!(k in c)) {
+      p.add('POSE_CAMERA', `${cw} has no "${k}"; a camera entry carries ${CAMERA_KEYS.join(', ')}`);
+      ok = false;
+    }
+    const finite = (v: unknown): v is number => typeof v === 'number' && Number.isFinite(v);
+    if ('s' in c && !(finite(c.s) && c.s > 0)) {
+      p.add('POSE_CAMERA', `${cw} "s" is ${typeName(c.s)}; the scale must be a finite number > 0`);
+      ok = false;
+    }
+    for (const k of ['tx', 'ty'] as const) if (k in c && !finite(c[k])) {
+      p.add('POSE_CAMERA', `${cw} "${k}" is ${typeName(c[k])}; it must be a finite number of picture px`);
+      ok = false;
+    }
+    if ('inliers' in c && !(Number.isInteger(c.inliers) && (c.inliers as number) >= 0)) {
+      p.add('POSE_CAMERA', `${cw} "inliers" is ${typeName(c.inliers)}; it must be an integer ≥ 0`);
+      ok = false;
+    }
+    if (ok) out.push({ s: c.s as number, tx: c.tx as number, ty: c.ty as number, inliers: c.inliers as number });
+  });
+  return out;
 }
 
 function readJoints(p: Problems, pw: string, joints: unknown): Partial<Record<Body18, Joint>> {
@@ -312,5 +375,6 @@ export function writePoseSequence(seq: PoseSequence): string {
       ),
     })),
   }));
-  return `${JSON.stringify({ spec: seq.spec, space: seq.space, width: seq.width, height: seq.height, source: seq.source, frames }, null, 2)}\n`;
+  const camera = seq.camera?.map((c) => ({ s: round(c.s, 6), tx: round(c.tx, 3), ty: round(c.ty, 3), inliers: c.inliers }));
+  return `${JSON.stringify({ spec: seq.spec, space: seq.space, width: seq.width, height: seq.height, source: seq.source, ...(seq.cameraSource !== undefined ? { cameraSource: seq.cameraSource } : {}), ...(camera !== undefined ? { camera } : {}), frames }, null, 2)}\n`;
 }

package/src/run.ts

@@ -8,7 +8,7 @@ import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeF
 import { tmpdir } from 'node:os';
 import { dirname, join, resolve } from 'node:path';
 import { readMap, resolveMap, type JointMap, type Rest } from './map.ts';
-import { convertCoco17, personTrack, readPoseSequence } from './pose.ts';
+import { convertCoco17, personTrack, readPoseSequence, type CameraFrame } from './pose.ts';
 import { isObject, lineOf, parseJson, Problems, Refusal, type Problem } from './problems.ts';
 import { boneIdentity, droppedLine, keptOutLine, mergeProblems, readFragment, readIdle, targetOf, trackLabel, type IdleReading, type RigcTrack, type Target } from './idle.ts';
 import { bendLine, endOnLine, flipLine, holdLine, rangeLine, retarget, shortLine, turnLine, type MotionTrack, type RetargetOptions } from './retarget.ts';
@@ -106,6 +106,8 @@ export function runRetarget(a: RetargetArgs, log: (line: string) => void): RunRe
     const r = retarget(rest, map, frames, times, a, slots);
     say(`rig-pose retarget: ${seq.frames.length} frame(s), ${r.animation.duration.toFixed(3)}s, onto "${input.name}" as "${a.name}" — ${r.animation.tracks.length} track(s); min-score ${a.minScore}, smooth ${a.smooth}, flip ${a.flip === 'off' ? 'off' : `${a.flip}°`}, short ${a.short}, bends ${a.bends} (tolerance ${a.bendTolerance}°)${a.legScale !== undefined && a.legScale !== 'off' ? `, leg-scale ${a.legScale[0]}–${a.legScale[1]}` : ''}${a.plantFeet === true ? ', plant-feet' : ''}${a.stance !== undefined && a.stance !== 'off' ? `, stance ${a.stance}` : ''}${a.carryIdle === true ? ', carry-idle' : ''}${a.merge !== undefined ? `, merge ${a.merge}` : ''}${a.reduce !== undefined ? `, reduce ${a.reduce}°${a.reduceHip !== undefined ? ` (hip ${a.reduceHip} units)` : ''}` : ''}`);
     say(`  RIGC   rig-c ${rigc.version} (${rigc.entry}) gates and poses this run`);
+    if (seq.camera !== undefined) say(`  CAMERA ${cameraLine(seq.camera, times, seq.cameraSource)}`);
+    else if (a.plantFeet === true) say('  CAMERA none in the sequence: --plant-feet reads the picture as a fixed camera, so a zoom or a pan in the video moves the floor with it and the floor may step with the picture (tools/scan.py reads the camera, rig-c/pose#63)');
     if (r.scale !== null) say(`  SCALE  ${r.scale.factor.toFixed(5)} rig units per picture px (rig ${r.scale.rig.toFixed(2)} / pose median ${r.scale.pose.toFixed(2)})`);
     for (const h of r.holds) say(`  HOLD   ${holdLine(h, r.times)}`);
     if (a.short !== 'off') {
@@ -390,6 +392,22 @@ function windowLines(map: JointMap, switches: ReadonlyArray<{ set: string; frame
   return out;
 }
 
+/**
+ * The CAMERA line (#63): where the camera came from (the sequence's
+ * `cameraSource`, or "(source not stated)"), the scale's range over the sequence, and the three
+ * frame-to-frame changes of s largest in size (ties to the earlier frame).
+ */
+export function cameraLine(camera: readonly CameraFrame[], times: readonly number[], source: string | undefined): string {
+  const ss = camera.map((c) => c.s);
+  const lo = ss.reduce((m, x, i) => (x < ss[m]! ? i : m), 0);
+  const hi = ss.reduce((m, x, i) => (x > ss[m]! ? i : m), 0);
+  const steps = ss.slice(1).map((x, k) => ({ to: k + 1, d: x - ss[k]! }));
+  const fastest = [...steps].sort((x, y) => Math.abs(y.d) - Math.abs(x.d) || x.to - y.to).slice(0, 3);
+  const sign = (d: number) => `${d < 0 ? '−' : '+'}${Math.abs(d).toFixed(4)}`;
+  const inl = camera.reduce((m, c) => Math.min(m, c.inliers), Infinity);
+  return `${source ?? '(source not stated)'}: the sequence carries the picture's camera against frame 0's; its joints are normalised by it (p′ = (p − t) / s): s ${ss[lo]!.toFixed(4)} (frame ${lo}) … ${ss[hi]!.toFixed(4)} (frame ${hi}) over ${camera.length} frames; s changes fastest at ${fastest.map((x) => `frame ${x.to - 1}→${x.to} (t=${times[x.to]!.toFixed(3)}s) ${sign(x.d)}`).join(', ') || 'no frame (one frame only)'}${source?.startsWith('hand-labelled') === true ? '' : `; fewest background inliers ${inl}`}`;
+}
+
 /** Each leg's first bone (its root, the thigh) and last bone (whose tip is the ankle), when the map has both. */
 function legBonesOf(map: JointMap): { r: { root: string; ankle: string }; l: { root: string; ankle: string } } | null {
   const l = legsOf(map, 'reading the feet');