@convai/embody 1.0.1 → 1.1.0-beta.0

package/dist/clips/retarget.d.ts

@@ -0,0 +1,168 @@
+/**
+ * Load-time retargeting: a clip authored on another skeleton (Mixamo, the
+ * Unreal mannequin, Rigify) → a clip on the body's own skeleton.
+ *
+ * Ported from the coffee-shop studio's camera-tuned retargeter and
+ * generalised: every rig is read through one table of canonical joints
+ * (Mixamo's names), so neither side's bone names are hard-coded into the
+ * method. The posture constants are the ones tuned on convai-mha-1 bodies.
+ *
+ * Method, per sampled frame:
+ *  1. Both rigs are put in one canonical frame (left +X, up +Y, forward +Z),
+ *     measured from their own hip line and the file's vertical.
+ *  2. Rest-pose fix: Mixamo rests in a T-pose, a MetaHuman binds in an
+ *     A-pose. Each target bone gets the shortest-arc rotation that lays its
+ *     rest direction (bone → mapped child) onto the source's. Feet are left
+ *     flat, the neck and head keep their own build, and a clavicle built
+ *     pointing back (Rigify) contributes only its elevation.
+ *  3. Each mapped bone takes the source bone's WORLD rotation change since
+ *     rest; unmapped bones (twists, correctives, extra spine) keep their
+ *     bind-pose local rotation.
+ *  4. The hips move by the source hips' offset, scaled by hip height.
+ *  5. Elbows and hands are kept outside the body's own torso and thighs,
+ *     measured from its skinned vertices (a clip made for a slimmer body
+ *     otherwise rests its hands inside this one).
+ *
+ * The top bone (`pelvis` on convai-mha-1) has a parent outside the skin
+ * (`root002`). It is solved against that parent's bind-frame transform,
+ * `bindMatrix · meshWorld⁻¹ · parent.matrixWorld`; solving it from skin bones
+ * alone writes a constant pelvis and breaks every clip that bends at the hips.
+ */
+import * as THREE from 'three';
+export type RigKind = 'mixamo' | 'unreal' | 'rigify';
+/** A skeleton's bones read as canonical joints. */
+export interface RigNames {
+    kind: RigKind;
+    /** Bone name → canonical joint. */
+    toCanon: Map<string, string>;
+    /** Canonical joint → bone name. */
+    fromCanon: Map<string, string>;
+}
+/**
+ * Reads a skeleton's bone names as canonical joints: Mixamo (any
+ * `mixamorig` prefix, or none), Unreal (UE4/UE5 mannequin, MetaHuman,
+ * convai-mha-1) and Rigify deform bones. Spines are matched bottom, middle,
+ * top: a 3-bone spine maps 1/2/3, a 5-bone one 1/3/5. `null` when the hips,
+ * thighs and feet cannot all be found.
+ */
+export declare function readRigNames(names: readonly string[]): RigNames | null;
+/** The skeleton under `root` that `readRigNames` reads, or null. */
+export declare function detectRig(root: THREE.Object3D): RigNames | null;
+interface Pose {
+    /** World rotation of each bone, in the rig's own frame. */
+    rot: Map<string, THREE.Quaternion>;
+    pos: Map<string, THREE.Vector3>;
+}
+/**
+ * One body segment's surface, read from the bind mesh: around the bone's
+ * axis (bone → tip), the radius the skin and clothing reach in each of
+ * SECTORS directions, over the stretch of the axis its vertices cover.
+ */
+interface Collider {
+    bone: string;
+    tip: string;
+    /** Radius per sector, bind units. */
+    radius: Float32Array;
+    /** Covered stretch of the axis, as a fraction of its length. */
+    t0: number;
+    t1: number;
+    /** Sector 0's direction, perpendicular to the axis, bind frame. */
+    e1: THREE.Vector3;
+}
+/** The body's skeleton, bind pose and surface, read once and shared by every clip retargeted onto it. */
+export interface TargetRig {
+    names: RigNames;
+    /** Every skeleton bone, by name. */
+    bones: Map<string, THREE.Bone>;
+    /** Bind pose, in the skinned mesh's bind frame. */
+    bind: Pose;
+    /** Each bone's full bind-frame matrix (with scale). */
+    bindMatrix: Map<string, THREE.Matrix4>;
+    /** Bind-pose local rotation of each bone. */
+    local: Map<string, THREE.Quaternion>;
+    /** Bind frame → canonical. */
+    toCanon: THREE.Quaternion;
+    /** Every skeleton bone, parents before children. */
+    order: THREE.Bone[];
+    /** Hips above the lower ankle, bind units. */
+    hipHeight: number;
+    /** Bind units per metre, from hip height. */
+    unitsPerM: number;
+    /**
+     * Bind-frame rotation of each top bone's parent: a node outside the skin
+     * (`root002` above `pelvis` on convai-mha-1), never animated.
+     */
+    topParent: Map<string, THREE.Quaternion>;
+    /** Bind frame → the hips' parent's local frame (with its scale). */
+    hipsParentInv: THREE.Matrix4;
+    /** Each bone's bind offset from its parent, in the parent's frame. */
+    offset: Map<string, THREE.Vector3>;
+    /** Torso and thighs as measured from the skinned meshes, for keeping hands out. */
+    colliders: Collider[];
+    /** Each bone and every bone below it. */
+    subtree: Map<string, string[]>;
+}
+/**
+ * Reads the body under `root` as a retarget target: the skinned mesh whose
+ * skeleton `readRigNames` reads (the largest, when several are), its bind
+ * pose from the inverse bind matrices, and its torso and thighs from every
+ * skinned mesh. Nothing is read from the current pose, so a body that is
+ * already animating (or scaled and grounded) gives the same rig. `null` when
+ * no skeleton is recognised.
+ */
+export declare function createTargetRig(root: THREE.Object3D): TargetRig | null;
+/** A loaded animation file: its skeleton (in its rest pose) and every clip in it. */
+export interface AnimationSource {
+    root: THREE.Object3D;
+    clips: THREE.AnimationClip[];
+    names: RigNames;
+    /** Each node's rest transform, put back before every retarget. */
+    rest: [THREE.Object3D, THREE.Vector3, THREE.Quaternion, THREE.Vector3][];
+}
+/**
+ * Wraps a loaded file (an FBXLoader group, or a GLTF's `scene`) as a
+ * retarget source, recording its rest pose. Call before anything animates
+ * it. `null` when its skeleton is not one `readRigNames` reads.
+ */
+export declare function createAnimationSource(root: THREE.Object3D, clips: THREE.AnimationClip[]): AnimationSource | null;
+/**
+ * Trunk lean (hips → neck, degrees, + = forward) the convai-mha-1 pack idle
+ * holds. Mixamo's idles and talks stand 1–4° further back than Mixamo's rest
+ * pose, which reads as a backward lean on these bodies.
+ */
+export declare const UPRIGHT_DEG = -3;
+export interface RetargetOptions {
+    /** Name of the produced clip. Default: the source clip's. */
+    name?: string;
+    /** Samples per second. Default 30. */
+    fps?: number;
+    /**
+     * Standing clips: pitch the spine forward so the clip's neutral pose
+     * (median lean over its standing frames) leans this much, by at most 6°,
+     * keeping the motion around it; knees straightened to about 8° of bend and
+     * low shoulders lifted the same way. `null` keeps the source's posture.
+     * Clips where the hips drop (kneel, sit) are never corrected. Default
+     * `UPRIGHT_DEG`.
+     */
+    upright?: number | null;
+    /** Keep elbows and hands outside the torso and thighs. Default `true`. */
+    clearHands?: boolean;
+}
+/** What the retarget changed, on `clip.userData.retarget`. */
+export interface RetargetReport {
+    source: RigKind;
+    neutralLeanDeg: number;
+    trunkCorrectionDeg: number;
+    kneeBendDeg: number;
+    kneeScale: number;
+    shoulderLiftDeg: Record<string, number>;
+    /** Deepest an elbow or hand went into the body before clearing, cm. */
+    handsClearedCm: number;
+}
+/**
+ * Retargets one clip of `source` onto `rig`: a clip of local rotations for
+ * the target's bones (only those that ever leave their rest) and the hips'
+ * position, sampled at `fps`.
+ */
+export declare function retargetClip(source: AnimationSource, src: THREE.AnimationClip, rig: TargetRig, opts?: RetargetOptions): THREE.AnimationClip;
+export {};