@vincentt-xr/sdk 2.0.0-alpha.0 → 2.0.0-alpha.10

package/dist/body/bodyRigFit.d.ts

@@ -0,0 +1,148 @@
+import { BodyPose } from './bodyPose';
+import { BodyRigBinding, BodyRigBone } from './bodyRigSolver';
+import * as THREE from "three";
+/** The four limbs a body rig reaches with (two bones and an end each). */
+export type BodyRigLimb = "leftArm" | "rightArm" | "leftLeg" | "rightLeg";
+export declare const BODY_RIG_LIMBS: readonly BodyRigLimb[];
+/** Each limb's upper bone, lower bone and end bone. */
+export declare const BODY_RIG_LIMB_BONES: Readonly<Record<BodyRigLimb, readonly [BodyRigBone, BodyRigBone, BodyRigBone]>>;
+/** Each limb's pose joints: the root, the middle and the end. */
+export declare const BODY_RIG_LIMB_JOINTS: Readonly<Record<BodyRigLimb, readonly [number, number, number]>>;
+export type BodySceneJoints = {
+    /** The 33 joints in scene space, indexed like `BODY_JOINT_NAMES`. */
+    points: THREE.Vector3[];
+    /** Scene units per centimetre of the pose's world joints. */
+    scale: number;
+};
+/**
+ * A tracked body's joints in scene (world) space: each on the camera ray
+ * through its image point, at a depth from the metric world joints — scaled
+ * by how the world x / y best map onto the image's (fitted on the `z = 0`
+ * plane) — with the hips' midpoint on that plane.
+ */
+export declare const bodyPoseToScene: (pose: Pick<BodyPose, "image" | "world" | "visibility">, camera: THREE.Camera, out?: BodySceneJoints) => BodySceneJoints;
+/** A bound bone's current position in the rig root's space (needs fresh world matrices). */
+export declare const bodyRigJointPosition: (binding: BodyRigBinding, role: BodyRigBone, out?: THREE.Vector3) => THREE.Vector3;
+/** Segments whose length the rig can match to the person (`applyBodyRigLengths`). */
+export type BodyRigSegment = "neck" | "leftClavicle" | "rightClavicle" | "leftUpperArm" | "leftForearm" | "rightUpperArm" | "rightForearm" | "leftThigh" | "leftShin" | "rightThigh" | "rightShin";
+export declare const BODY_RIG_SEGMENTS: BodyRigSegment[];
+/** Clamp for segment length ratios: tracking noise and odd proportions stay sane. */
+export declare const BODY_RIG_LENGTH_RANGE: readonly [number, number];
+/** The clavicles may narrow or widen the shoulders more (models differ most there). */
+export declare const BODY_RIG_CLAVICLE_RANGE: readonly [number, number];
+/** The neck may shorten or lengthen more still (a stylised head sits high or low). */
+export declare const BODY_RIG_NECK_RANGE: readonly [number, number];
+/**
+ * Where a model's head centre sits above its head joint (the skull's base),
+ * in neck lengths (neck joint → head joint) along the neck, for a model whose
+ * head can't be measured from its mesh (`bodyRigModelHead`). The person's
+ * head centre is between their ears.
+ */
+export declare const BODY_RIG_HEAD_CENTER = 0.9;
+/**
+ * The model's size against the person's: scene units per model unit, from the
+ * torso (shoulders' midpoint to hips' midpoint), the steadiest length the
+ * pose has.
+ */
+export declare const bodyRigSceneScale: (binding: BodyRigBinding, scene: BodySceneJoints) => number;
+/**
+ * The model's size against the person's from the shoulder width alone (scene
+ * units per model unit), for when the hips are out of frame. People's
+ * shoulders sit narrower against their torso than most models' do: multiply
+ * by `BODY_RIG_SHOULDER_TO_TORSO` (or a factor measured on this person while
+ * both were seen) to match `bodyRigSceneScale`.
+ */
+export declare const bodyRigShoulderScale: (binding: BodyRigBinding, scene: BodySceneJoints) => number;
+/** `bodyRigSceneScale / bodyRigShoulderScale` for a typical person and model (measured: ≈ 1.1). */
+export declare const BODY_RIG_SHOULDER_TO_TORSO = 1.1;
+/**
+ * Clamp for the chest's scale: models are often built broader in the chest
+ * than people, and it thins (or fills out) within this range.
+ */
+export declare const BODY_RIG_CHEST_RANGE: readonly [number, number];
+/**
+ * How much narrower (< 1) or broader the person's chest is than the model's
+ * at `modelScale` (scene units per model unit): their shoulder width against
+ * the model's, clamped to `BODY_RIG_CHEST_RANGE`.
+ */
+export declare const bodyRigChestScale: (binding: BodyRigBinding, scene: BodySceneJoints, modelScale: number) => number;
+/**
+ * How much longer the person's segments are than the model's at `modelScale`
+ * (scene units per model unit), clamped; `1` keeps the model's own length.
+ */
+export declare const bodyRigLengthRatios: (binding: BodyRigBinding, scene: BodySceneJoints, modelScale: number, out?: Partial<Record<BodyRigSegment, number>>) => Partial<Record<BodyRigSegment, number>>;
+/**
+ * Gives segments their length ratios by moving each segment's far joint out
+ * along it (its rest local position × ratio): the skin between stretches
+ * through its weights, its thickness stays the model's. `chest`
+ * (`bodyRigChestScale`) scales the skin of the chest and the clavicles about
+ * their joints, so it narrows and its top comes down; every joint stays where
+ * the ratios put it, and the neck and arms keep their own size. `null`
+ * restores every rest length and the chest.
+ */
+export declare const applyBodyRigLengths: (binding: BodyRigBinding, ratios: Partial<Record<BodyRigSegment, number>> | null, chest?: number) => void;
+/** A person's head breadth over their ear-to-ear distance (the tracker's ear points). */
+export declare const BODY_RIG_EAR_TO_HEAD = 1.1;
+/** Clamp for the head's scale: a stylised head stays recognisable. */
+export declare const BODY_RIG_HEAD_RANGE: readonly [number, number];
+export type BodyRigModelHead = {
+    /** Breadth along the model's left–right axis, model units. */
+    width: number;
+    /** The head's centre (the middle of its box) in the head bone's own space, at rest. */
+    center: THREE.Vector3;
+};
+/**
+ * The model's head as its mesh has it: the box of the skinned vertices the
+ * head bone (or a bone below it) moves most. Read once, at rest; `null`
+ * without skinned head vertices.
+ */
+export declare const bodyRigModelHead: (binding: BodyRigBinding) => BodyRigModelHead | null;
+/**
+ * How much bigger the person's head is than the model's at `modelScale`
+ * (scene units per model unit): their ear-to-ear distance ×
+ * `BODY_RIG_EAR_TO_HEAD` against the model's head breadth, clamped to
+ * `BODY_RIG_HEAD_RANGE`. Close to the camera a head looks big against the
+ * shoulders, and the ratio follows. `null` when the model's head can't be
+ * measured.
+ */
+export declare const bodyRigHeadScale: (binding: BodyRigBinding, scene: BodySceneJoints, modelScale: number) => number | null;
+/** Scales the head bone by `ratio` from its rest scale (`null` = rest). */
+export declare const applyBodyRigHeadScale: (binding: BodyRigBinding, ratio: number | null) => void;
+/**
+ * Two-bone reach: turns a limb's upper and lower bones so its end joint lands
+ * on `target` (the rig root's space), `weight` of the way from where it is.
+ * The limb keeps the side its middle joint bends toward; the end bone (hand,
+ * foot) keeps its orientation. A target out of reach straightens the limb
+ * toward it. Needs fresh world matrices; leaves them fresh.
+ */
+export declare const reachBodyRigLimb: (binding: BodyRigBinding, limb: BodyRigLimb, target: THREE.Vector3, weight?: number) => void;
+/**
+ * Where a person's head centre (crown to chin, hair included) sits above
+ * their ears' midpoint, in ear-to-ear distances (measured on webcam and phone
+ * takes): the head's reach aims there, so a model's head covers the person's
+ * and leaves their neck free.
+ */
+export declare const BODY_RIG_HEAD_ABOVE_EARS = 0.2;
+/**
+ * The person's head centre in the rig root's space, from their ears (root
+ * space): between them, raised `BODY_RIG_HEAD_ABOVE_EARS` ear-to-ear
+ * distances along the model head's up as it is turned now (its rest up is the
+ * root's +Y). Needs fresh world matrices.
+ */
+export declare const bodyRigHeadTarget: (binding: BodyRigBinding, leftEar: THREE.Vector3, rightEar: THREE.Vector3, out?: THREE.Vector3) => THREE.Vector3;
+/** The model's head centre now, in the rig root's space (needs fresh world matrices). */
+export declare const bodyRigHeadCenter: (binding: BodyRigBinding, out?: THREE.Vector3) => THREE.Vector3 | null;
+/**
+ * The head's reach: turns the neck and sets its length (the head joint's rest
+ * local position × a ratio clamped to `BODY_RIG_NECK_RANGE`) so the model's
+ * head centre lands on `target` (the rig root's space; the person's: between
+ * their ears), `weight` of the way from where it is. The head keeps its
+ * orientation. Needs fresh world matrices; leaves them fresh.
+ */
+export declare const reachBodyRigHead: (binding: BodyRigBinding, target: THREE.Vector3, weight?: number) => void;
+/**
+ * Each bound bone's local rotation at the start of `clip` (an idle or rest
+ * pose clip, e.g. the example Y Bot's `idle`); bones the clip does not key
+ * keep their rest rotation.
+ */
+export declare const bodyRigClipPose: (binding: BodyRigBinding, clip: THREE.AnimationClip | null | undefined) => Partial<Record<BodyRigBone, THREE.Quaternion>>;

package/dist/body/bodyRigSolver.d.ts

@@ -0,0 +1,114 @@
+import { BodyPose } from './bodyPose';
+import * as THREE from "three";
+/**
+ * Body rig contract (v1).
+ *
+ * A compatible model is a humanoid skeleton with a node per role below,
+ * found by its Mixamo name (`Hips`, `LeftForeArm` …, an optional
+ * `mixamorig` prefix is accepted) or by `names` passed to `bindBodyRig`.
+ * Required: the hips, at least one spine bone, the neck or head, and each
+ * arm's and leg's three bones. Any rest pose (T or A), units, bone roll and
+ * authoring axes work: the solver measures every rotation from the model's
+ * own rest pose. The model must face +Z in its root's space (glTF's
+ * convention), its own left toward +X.
+ */
+export type BodyRigBone = "hips" | "spine" | "spine1" | "spine2" | "neck" | "head" | "leftShoulder" | "leftArm" | "leftForeArm" | "leftHand" | "rightShoulder" | "rightArm" | "rightForeArm" | "rightHand" | "leftUpLeg" | "leftLeg" | "leftFoot" | "leftToeBase" | "rightUpLeg" | "rightLeg" | "rightFoot" | "rightToeBase";
+/** Every role, parents before children. */
+export declare const BODY_RIG_BONES: readonly BodyRigBone[];
+/** The default node name of each role: Mixamo's, without the `mixamorig:` prefix. */
+export declare const BODY_RIG_MIXAMO_NAMES: Readonly<Record<BodyRigBone, string>>;
+/** Roles a rig must have (`bindBodyRig` throws without them). */
+export declare const BODY_RIG_REQUIRED_BONES: readonly BodyRigBone[];
+/** Parts of the body, each with its own tracking confidence. */
+export type BodyRigRegion = "torso" | "head" | "leftArm" | "rightArm" | "leftLeg" | "rightLeg";
+export declare const BODY_RIG_REGIONS: readonly BodyRigRegion[];
+/** The pose joints each region's confidence is read from (the lowest visibility wins). */
+export declare const BODY_RIG_REGION_JOINTS: Readonly<Record<BodyRigRegion, readonly number[]>>;
+/** The region each role belongs to (its rotation fades with that region's confidence). */
+export declare const BODY_RIG_BONE_REGION: Readonly<Record<BodyRigBone, BodyRigRegion>>;
+export type BodyRigBinding = {
+    /** The object whose matrix places the whole rig (the solver works in its space). */
+    root: THREE.Object3D;
+    bones: Record<BodyRigBone, THREE.Object3D | null>;
+    /** Rest joint position of each bone in root space (model units). */
+    restPositions: Partial<Record<BodyRigBone, THREE.Vector3>>;
+    /** Rest orientation of each bone in root space. */
+    restQuaternions: Partial<Record<BodyRigBone, THREE.Quaternion>>;
+    /** Rest local quaternion (`bone.quaternion`) of each bone. */
+    restLocal: Partial<Record<BodyRigBone, THREE.Quaternion>>;
+    /** Rest local position (`bone.position`) of each bone (segment lengths scale it). */
+    restLocalPositions: Partial<Record<BodyRigBone, THREE.Vector3>>;
+    /** Rest local scale (`bone.scale`) of each bone (the head's is fitted). */
+    restLocalScales: Partial<Record<BodyRigBone, THREE.Vector3>>;
+    /** Rest root-space orientation of each bone's parent node. */
+    restParentQuaternions: Partial<Record<BodyRigBone, THREE.Quaternion>>;
+    /** The nearest bound ancestor of each bone (`null` for the hips). */
+    boundParent: Partial<Record<BodyRigBone, BodyRigBone | null>>;
+    /** The spine bones the rig has, hips side first. */
+    spine: BodyRigBone[];
+    /**
+     * Each hand's rest knuckle frame in root space, when the rig has the index
+     * and pinky finger bones (`…HandIndex1`, `…HandPinky1`): from the wrist
+     * toward the knuckles, and the pinky → index line.
+     */
+    restHands: Partial<Record<"leftHand" | "rightHand", {
+        forward: THREE.Vector3;
+        across: THREE.Vector3;
+    }>>;
+};
+/**
+ * Reads a compatible rig's rest pose. `root` is the model's top object (for a
+ * GLB, the cloned `gltf.scene`); call before posing it. `names` overrides the
+ * node name of any role. Throws when a required role is missing.
+ */
+export declare const bindBodyRig: (root: THREE.Object3D, names?: Partial<Record<BodyRigBone, string>>) => BodyRigBinding;
+export type BodyRigSolution = {
+    /** Local quaternion for each bound bone. */
+    local: Partial<Record<BodyRigBone, THREE.Quaternion>>;
+    /** Mirror the root in X: the body is shown mirrored (the selfie view). */
+    mirrored: boolean;
+    /** Per region, the lowest visibility of the pose joints it is read from (0..1). */
+    confidence: Record<BodyRigRegion, number>;
+};
+/**
+ * How far the head may turn against the chest, degrees: `yaw` either way,
+ * `pitch` looking down (+) and up (−), `roll` toward either shoulder.
+ */
+export declare const BODY_RIG_HEAD_LIMITS: Readonly<{
+    yaw: 80;
+    down: 65;
+    up: 50;
+    roll: 45;
+}>;
+/**
+ * Clamps a head turn (against the chest, the model's rest axes: +X its left,
+ * +Y up, +Z forward) to `BODY_RIG_HEAD_LIMITS`, as yaw, then pitch, then roll.
+ */
+export declare const limitHeadTurn: (turn: THREE.Quaternion) => THREE.Quaternion;
+/** Below this visibility of the wrist and knuckle points the hand follows the forearm. */
+export declare const BODY_RIG_HAND_SEEN = 0.3;
+/** The share of the head's turn the neck takes (the head takes the rest). */
+export declare const BODY_RIG_NECK_SHARE = 0.4;
+/**
+ * The share of the hand's roll about the forearm the forearm bone takes, so
+ * the skin twists along the forearm instead of all at the wrist.
+ */
+export declare const BODY_RIG_FOREARM_TWIST_SHARE = 0.5;
+/**
+ * An elbow or knee this straight (sine of the bend) gives no usable hinge: the
+ * limb's roll then follows its parent. Full trust from `[1]` up.
+ */
+export declare const BODY_RIG_HINGE_BEND: readonly [number, number];
+/**
+ * Solves the rig for one frame: every bound bone's local rotation, measured
+ * from the model's rest pose, from the pose's metric world joints. Only
+ * directions are read (the model keeps its own bone lengths). On a mirrored
+ * feed the shape is un-mirrored first and `mirrored` asks the caller to
+ * mirror the root, so each side drives its own side of the model. Pure; reuse
+ * `out`.
+ */
+export declare const solveBodyRig: (binding: BodyRigBinding, pose: Pick<BodyPose, "world" | "visibility">, feedMirrored: boolean, out?: BodyRigSolution) => BodyRigSolution;
+/** Writes a solution onto the bound bones (no smoothing). */
+export declare const applyBodyRigSolution: (binding: BodyRigBinding, solution: BodyRigSolution) => void;
+/** Puts every bound bone back in its rest rotation, position and scale. */
+export declare const resetBodyRig: (binding: BodyRigBinding) => void;

package/dist/body/bodySubject.d.ts

@@ -0,0 +1,46 @@
+import { XRSubjectMemory, XRSubjectPolicy } from '../subjects/types';
+import { XRModelNodeBodyTracking } from '../types/nodes';
+import { BodyPose } from './bodyPose';
+/**
+ * Which body a body feature follows. `bodyId` wins over `subject`, which wins
+ * over `bodyIndex`.
+ */
+export type BodySubjectOptions = {
+    /** Result-array position (not a persistent identity). Default 0. */
+    bodyIndex?: number;
+    /** Follow the body with this stable id (`node.getBodies()`, `useBodyIds()`). */
+    bodyId?: number;
+    /**
+     * Pick a body by policy (`largest`, `nearest-center`, `leftmost`,
+     * `rightmost`, `index` over the bodies by id, or `sticky`, which keeps the
+     * picked person by id).
+     */
+    subject?: XRSubjectPolicy;
+    /**
+     * How many bodies to ask the shared model for (1–2). Default 2 with
+     * `bodyId` or `subject` (a policy needs candidates), else 1.
+     */
+    numBodies?: number;
+};
+/** The body a feature follows this frame. */
+export type ResolvedBody = {
+    pose: BodyPose;
+    /** Its stable id; `null` when read by `bodyIndex` before the body has one. */
+    id: number | null;
+    held: boolean;
+    /**
+     * Its position in the latest detection (`getLandmarks2D(index)`,
+     * `getVisibility(index)`); `null` while it is held.
+     */
+    index: number | null;
+};
+/** Bodies asked for: as given, else 2 when picking by id or policy. */
+export declare const bodySubjectCount: ({ bodyId, subject, numBodies }: BodySubjectOptions) => number;
+/**
+ * Picks the body `options` asks for from the model node. Pure: feed
+ * `result.memory` back on the next call (it carries `sticky`'s pick).
+ */
+export declare const resolveBody: (node: Pick<XRModelNodeBodyTracking, "getBodyPose" | "getBodies"> | null | undefined, { bodyIndex, bodyId, subject }: BodySubjectOptions, memory?: XRSubjectMemory) => {
+    body: ResolvedBody | null;
+    memory: XRSubjectMemory;
+};