@vincentt-xr/sdk 2.0.0-alpha.7 → 2.0.0-alpha.9

package/dist/face-atlas/surface.d.ts

@@ -0,0 +1,155 @@
+import { FaceAtlasRegionSelection } from './regions';
+import * as THREE from "three";
+/**
+ * `FaceAtlasSurface`: one live face in the face atlas topology (468
+ * landmarks, `FACE_ATLAS_TRIANGLES`), shared by every layer drawn on that
+ * face (makeup regions, LipEffect, face swap, faceless, custom layers).
+ *
+ * Its vertex attributes are written once per frame and shared by every
+ * layer's geometry (each layer only has its own index buffer):
+ *
+ * | Attribute | |
+ * | --- | --- |
+ * | `position` | the live point (canonical cm, in the rig's smoothed-pose frame) plus any `FaceWarp3D` / `FaceMorph` offset |
+ * | `basePosition` | the live point without offsets: where the person's skin is in the camera image |
+ * | `normal` | live normals over all 898 triangles, smoothed over time |
+ * | `uv` | `FACE_ATLAS_UVS` |
+ * | `restPosition`, `restNormal`, `edge`, `lipEdge`, `eyeRing` | `getFaceAtlasAttributes()` |
+ * | `faceUv` | the camera-image uv of `basePosition`, written only while requested |
+ *
+ * Draw it through a group posed by the rig (`FaceRigState.pose`); the
+ * registry does this for you. Pure three.js, no React: `update` is
+ * allocation-free.
+ */
+/** Tuning of `FaceAtlasSurface`. */
+export declare const FACE_ATLAS_SURFACE: Readonly<{
+    /** Per-frame blend of the new normals into the smoothed ones (1 = none). */
+    normalSmoothing: 0.5;
+    /** Fade in (ms) when the face is found. */
+    fadeInMs: 150;
+    /** Fade out (ms) once the face is lost (after the rig's hold). */
+    fadeOutMs: 120;
+}>;
+/** Where the rig stands: `FaceRigTracking`. A held face keeps its last points. */
+export type FaceAtlasSurfaceTracking = "tracked" | "held" | "lost";
+/** One frame's input to `FaceAtlasSurface.update`. */
+export type FaceAtlasSurfaceInput = {
+    /**
+     * The live landmarks in the drawn frame, canonical cm, flat 468 × 3:
+     * `FaceRigState.view` (the smoothed pose's frame, so they sit on the
+     * camera image when drawn through the pose).
+     */
+    view: ArrayLike<number>;
+    /** Per-landmark displacement of the surface (`FaceSurfaceState.offsets`), or `null`. */
+    offsets?: ArrayLike<number> | null;
+    tracking: FaceAtlasSurfaceTracking;
+    /** Time since the last update (ms), for the fades. */
+    dtMs: number;
+};
+/**
+ * Which triangles a layer draws: atlas regions (one or a union), or explicit
+ * triangle indices into `FACE_ATLAS_TRIANGLES`, drawn in the order given
+ * (back-to-front lists for layers without depth writes).
+ */
+export type FaceAtlasTriangleSelection = FaceAtlasRegionSelection | {
+    triangles: ArrayLike<number>;
+};
+/** Names of the attributes every surface geometry shares. */
+export declare const FACE_ATLAS_SURFACE_ATTRIBUTES: readonly ["position", "basePosition", "normal", "uv", "restPosition", "restNormal", "edge", "lipEdge", "eyeRing", "faceUv"];
+export type FaceAtlasSurfaceAttributeName = (typeof FACE_ATLAS_SURFACE_ATTRIBUTES)[number];
+/** Flat landmark indices of a triangle selection. */
+export declare const faceAtlasSelectionIndices: (selection: FaceAtlasTriangleSelection) => Uint16Array;
+/**
+ * Writes unit vertex normals of `positions` over `FACE_ATLAS_TRIANGLES`
+ * (area-weighted) into `out`. Allocation-free.
+ */
+export declare const writeFaceAtlasNormals: (positions: ArrayLike<number>, out: Float32Array) => void;
+/**
+ * The live face in the face atlas topology. Create one per face (the
+ * face-atlas registry does), `update` it once per frame, and draw any number
+ * of layers from `createGeometry`.
+ */
+export declare class FaceAtlasSurface {
+    /** The shared attributes (see the table above). Read-only for layers. */
+    readonly attributes: Readonly<Record<FaceAtlasSurfaceAttributeName, THREE.BufferAttribute>>;
+    /** The whole face (every triangle, fills included), for custom effects. Owned by the surface. */
+    readonly geometry: THREE.BufferGeometry;
+    /** 0..1: fades in when the face is found, out once it is lost. */
+    presence: number;
+    /** Whether the face shows at all (`presence > 0`): hide layers when false. */
+    visible: boolean;
+    /** Bumped on every update that moved the points. */
+    version: number;
+    /** Where the rig stood at the last update. */
+    tracking: FaceAtlasSurfaceTracking;
+    private readonly normalScratch;
+    private normalsReady;
+    private faceUvRequests;
+    private readonly geometries;
+    private isDisposed;
+    /** The last update moved `position` off `basePosition` (a warp's offsets). */
+    private offsetsApplied;
+    /** `position` while `setUnwarped(true)` holds `basePosition` in it (made on first use). */
+    private warpedPositions;
+    /** `normal` while `setUnwarped(true)` holds the unwarped normals in it (made on first use). */
+    private warpedNormals;
+    /** Smoothed normals of `basePosition`, kept only while unwarped passes run (made on first use). */
+    private baseNormals;
+    /** The `version` `baseNormals` were last smoothed at (-1: never). */
+    private baseNormalsVersion;
+    private unwarped;
+    constructor();
+    /** True once `dispose` ran (the face's last layer went): make no more geometries from it. */
+    get disposed(): boolean;
+    /** Whether some layer asked for `faceUv` (`requestFaceUv`). */
+    get wantsFaceUv(): boolean;
+    /**
+     * A new geometry for a layer: the shared attributes plus its own index
+     * (regions, or explicit triangles in draw order) and any `extra`
+     * per-landmark attributes (468 items each, e.g. a face swap's mirror uv).
+     * Give it back with `releaseGeometry`, never `dispose()` it yourself: the
+     * attributes are shared with the other layers.
+     */
+    createGeometry(selection?: FaceAtlasTriangleSelection, extra?: Record<string, THREE.BufferAttribute>): THREE.BufferGeometry;
+    /**
+     * Releases a geometry from `createGeometry`. Its own index and extra
+     * attributes are freed; the shared attributes stay on the GPU while
+     * another geometry that has been drawn still uses them.
+     */
+    releaseGeometry(geometry: THREE.BufferGeometry): void;
+    /**
+     * The next update takes the fresh normals instead of blending them into
+     * the smoothed ones (a new camera source: another face).
+     */
+    snapNormals(): void;
+    /** Counts a layer that reads `faceUv`; returns its release. */
+    requestFaceUv(): () => void;
+    /**
+     * One frame: writes `position` (view + offsets) and `basePosition` (view),
+     * smooths the live normals, and steps the presence fade. A held face keeps
+     * its points; a lost face keeps its last points while it fades out; a face
+     * found again snaps its normals.
+     */
+    update({ view, offsets, tracking, dtMs }: FaceAtlasSurfaceInput): void;
+    /**
+     * For a pass that draws the face where the camera sees it (the made-up
+     * camera): while on, `position` holds `basePosition` (no warp offsets);
+     * off puts the warped points back. Does nothing when no offsets apply, so
+     * an unwarped face costs no upload. The next `update` ends it.
+     */
+    setUnwarped(on: boolean): void;
+    /**
+     * The normals of `basePosition`, smoothed as the live ones are: once per
+     * surface update, snapped when the last unwarped pass was not on the
+     * previous update (a made-up camera held again) or the face was found again.
+     */
+    private unwarpedNormals;
+    /**
+     * Writes `faceUv` (the camera-image uv of each `basePosition`) through
+     * `clipFromLocal` = projection × view × the pose group's world matrix.
+     * Only while requested; returns whether it wrote.
+     */
+    updateFaceUv(clipFromLocal: THREE.Matrix4): boolean;
+    /** Frees every geometry it made and the shared attributes. The surface is unusable after. */
+    dispose(): void;
+}

package/dist/face-atlas/topology.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Pure graph helpers over a triangle list (vertex indices, three per
+ * triangle). No atlas data is imported here, so `parseCanonicalFaceModel`
+ * can use them without pulling the atlas tables into a bundle; the atlas's
+ * own static attributes are in `attributes.ts`.
+ */
+/**
+ * The triangles between two closed vertex rings (an annulus such as the lip
+ * band between the outer and inner lip rings), as sorted triangle indices.
+ * Flood-fills the triangle adjacency from a triangle on the outside of the
+ * inner ring without crossing an edge of either ring.
+ *
+ * Fails closed: when the filled region is not sealed by exactly the two
+ * rings (different topology), it returns no triangles rather than leaking
+ * across the face.
+ */
+export declare const collectRingBandTriangles: (triangles: ArrayLike<number>, outerRing: readonly number[], innerRing: readonly number[]) => number[];
+/**
+ * Hop-count position of every vertex of a ring band between its two rings:
+ * `hopsToOuter / (hopsToOuter + hopsToInner)`, so outer-ring vertices are 0,
+ * inner-ring vertices 1 and a middle row 0.5; vertices outside the band are
+ * 1. `bandTriangles` is flat vertex indices (three per triangle). This is
+ * the `lipEdge` attribute for the lip rings, and the same measure for any
+ * other ring pair.
+ */
+export declare const computeRingDistance: (bandTriangles: ArrayLike<number>, vertexCount: number, outerRing: readonly number[], innerRing: readonly number[]) => Float32Array;
+/**
+ * Breadth-first hop count from `sources` over the edges of `triangles`.
+ * Unreachable vertices get `Infinity`.
+ */
+export declare const computeHopDistance: (triangles: ArrayLike<number>, vertexCount: number, sources: readonly number[]) => Float32Array;
+/**
+ * Shortest distance along mesh edges from `sources`, in the units of
+ * `positions` (Dijkstra; edge length = straight-line distance). Unreachable
+ * vertices get `Infinity`.
+ */
+export declare const computeEdgeDistance: (positions: ArrayLike<number>, triangles: ArrayLike<number>, vertexCount: number, sources: readonly number[]) => Float32Array;
+/**
+ * Area-weighted vertex normals of an outward-wound triangle list
+ * (counter-clockwise seen from outside), normalised; `[x, y, z, …]`.
+ */
+export declare const computeVertexNormals: (positions: ArrayLike<number>, triangles: ArrayLike<number>, vertexCount: number) => Float32Array;
+/** Flat vertex indices (three per triangle) of the given triangle indices. */
+export declare const triangleIdsToVertices: (triangleIds: ArrayLike<number>, triangles: ArrayLike<number>) => Uint16Array;

package/dist/face-character/__fixtures__/faceImageDetections.d.ts

@@ -0,0 +1,25 @@
+import { FaceImageDetection } from '../facePhoto';
+export type FaceImageFixtureOptions = {
+    /** Box centre, normalized. Default 0.5, 0.5. */
+    cx?: number;
+    cy?: number;
+    /** Box width and height, normalized. Default 0.3. */
+    size?: number;
+    /** Iris-centre distance in pixels of `image`. Default 120. */
+    interocularPixels?: number;
+    /** The photo canvas the pixels are measured in. Default 1000 × 1000. */
+    image?: {
+        width: number;
+        height: number;
+    };
+    /** Head yaw in degrees against the optical axis (positive = canonical +Z toward +X). Default 0. */
+    yawDegrees?: number;
+    /** The matrix translation in cm (camera at the origin looking down −Z). Default 0, 0, −60. */
+    position?: [number, number, number];
+    /** Omit the matrix (null). */
+    noMatrix?: boolean;
+    blendshapes?: Record<string, number>;
+    /** Landmark count. Default 478 (with irises). */
+    count?: number;
+};
+export declare const faceImageFixture: ({ cx, cy, size, interocularPixels, image, yawDegrees, position, noMatrix, blendshapes, count, }?: FaceImageFixtureOptions) => FaceImageDetection;

package/dist/face-character/__fixtures__/faceNode.d.ts

@@ -0,0 +1,22 @@
+import { NormalizedLandmark } from '@mediapipe/tasks-vision';
+import { XRModelNodeFaceTracking } from '../../types/nodes';
+import * as THREE from "three";
+/**
+ * Test fixtures: a face model node reporting the canonical face on the feed,
+ * and the scene camera that unprojects it.
+ */
+/** The canonical face as MediaPipe would report it: 478 normalised points around (cx, cy). */
+export declare const canonicalFaceLandmarks: ({ cx, cy, scale, aspect, }?: {
+    cx?: number;
+    cy?: number;
+    scale?: number;
+    aspect?: number;
+}) => NormalizedLandmark[];
+/** A face node whose `detectionResults` you set per frame (`faces` = one landmark list per face). */
+export declare const createFaceNode: (faces?: NormalizedLandmark[][], blendshapes?: Record<string, number>) => {
+    node: XRModelNodeFaceTracking;
+    /** A new inference: a new results object (and new landmark arrays). */
+    setFaces: (list: NormalizedLandmark[][]) => void;
+};
+/** The SDK's scene camera: perspective, at z = 5, looking down −Z, 4:3. */
+export declare const createSceneCamera: (aspect?: number) => THREE.PerspectiveCamera;

package/dist/face-character/bindFaceModel.d.ts

@@ -0,0 +1,106 @@
+import { FaceMeshBinding, FaceMeshBindingMode } from './faceMeshBinding';
+import { FaceMorphRegion } from './faceMorphs';
+import { FaceFeatureAlignment } from './featureWarp';
+import * as THREE from "three";
+/** One of a mesh's morph targets that an expression channel drives. */
+export type FaceMorphTarget = {
+    /** Index into the mesh's `morphTargetInfluences`. */
+    index: number;
+    /** The expression channel (`FACE_MORPH_NAMES` index). */
+    channel: number;
+    region: FaceMorphRegion;
+};
+/**
+ * How one mesh of a face model follows the face's expression:
+ * - `"surface"` — every vertex follows its own bound point (a face skin);
+ * - `"follow"` — the whole mesh moves as one with the point under its centre,
+ *   keeping its shape (eyeballs, teeth, a nose ring);
+ * - `"rigid"` — only the head pose (a helmet, hair).
+ */
+export type ExternalFaceMeshPart = "surface" | "follow" | "rigid";
+/** One bound mesh of a face model. */
+export type ExternalFaceMeshBinding = {
+    mesh: THREE.Mesh;
+    part: ExternalFaceMeshPart;
+    binding: FaceMeshBinding;
+};
+/** A bound mesh plus what the per-frame deformation needs. */
+export type BoundFaceMesh = ExternalFaceMeshBinding & {
+    /** The mesh's rest positions (its own frame, plain floats). */
+    rest: Float32Array;
+    /** The position attribute the deformation writes (replaces the original). */
+    position: THREE.BufferAttribute;
+    /** Canonical cm → mesh-local vector, row-major 3×3. */
+    toLocal: Float32Array;
+    /** For `"follow"`: the binding of the mesh's centre point. */
+    centre: FaceMeshBinding | null;
+    /** The mesh's morph targets that expression channels drive (by name). */
+    morphs: FaceMorphTarget[];
+    /**
+     * For a `"follow"` part named like an eye (`eye…`): which eye it is (by
+     * side of the face: +X = `"left"`, landmark 263's side), so the gaze can
+     * turn it; else `null`.
+     */
+    eye: "left" | "right" | null;
+    /** The mesh's centre in its own frame (the eyeball's pivot). */
+    pivot: THREE.Vector3;
+    /** Mesh-local → canonical vector, row-major 3×3 (the inverse of `toLocal`). */
+    toModel: Float32Array;
+    /** The mesh's rest vertices in the canonical head frame (cm, flat). */
+    canonical: Float32Array;
+    /**
+     * The positions the mesh is bound from: `canonical` moved by the model's
+     * feature alignment (`BindFaceModelOptions.align`), or `canonical` itself
+     * when there is none. `FaceReprojection` binds the worn face from these too.
+     */
+    aligned: Float32Array;
+    /**
+     * Per vertex, how much of the head-size fit it takes (0 on the face, 1 from
+     * `FACE_HEAD_FIT_RAMP[1]` cm away: the ears and the back of the head).
+     */
+    widen: Float32Array;
+};
+/** cm from the face over which the head-size fit ramps in (none on the face, full at the ears). */
+export declare const FACE_HEAD_FIT_RAMP: readonly [number, number];
+/**
+ * A face model's width at its ears (canonical cm): the widest extent, across
+ * its surface and rigid meshes, of the vertices off the face (past
+ * `FACE_HEAD_FIT_RAMP[0]`) within 1 cm of the face side's height (landmark
+ * 234). `0` when it has none (a face-only mask).
+ */
+export declare const measureFaceModelEarWidth: (meshes: readonly BoundFaceMesh[]) => number;
+/** How a face model's own features are aligned to the canonical face's before binding. */
+export type FaceModelAlignment = FaceFeatureAlignment | "auto" | false;
+/** Auto alignment leaves eyeballs farther than this (cm) from their eye opening alone (a stylised eye). */
+export declare const FACE_AUTO_ALIGN_MAX = 1.5;
+/**
+ * The feature alignment `"auto"` finds for a bound model: when it has both
+ * eyeballs (`"follow"` eye parts) near the canonical eye openings, each eye
+ * opening is moved onto the model's eye (its eyeball centre in x and y), and
+ * the mouth, nose and chin stay where they are. So the lids, the eyeballs and
+ * a worn face all bind to the model's own eyes. `null` when it has no
+ * eyeballs, one is farther than `FACE_AUTO_ALIGN_MAX`, or both already sit on
+ * their openings (within 0.5 mm).
+ */
+export declare const autoFaceModelAlignment: (meshes: readonly Pick<BoundFaceMesh, "eye" | "canonical">[]) => FaceFeatureAlignment | null;
+export type BindFaceModelOptions = {
+    mode?: FaceMeshBindingMode;
+    falloff?: readonly [number, number];
+    /** Per mesh (object name): its part. Default: inside the head → `"follow"`, else `"surface"`. */
+    parts?: Record<string, ExternalFaceMeshPart>;
+    /**
+     * The model's own features (canonical cm) to bind as if they sat on the
+     * canonical face's (see `FaceFeatureAlignment`); `"auto"` (default) takes
+     * the eyes from the model's eyeballs (`autoFaceModelAlignment`); `false`
+     * binds the model as it is.
+     */
+    align?: FaceModelAlignment;
+};
+/**
+ * Binds every mesh under `root` (a face model in the canonical head frame) to
+ * the canonical face, giving each mesh its own geometry copy with a plain
+ * float position attribute the deformation writes. Pure apart from that; run
+ * once per model (`ExternalFaceMesh` does). The root's own transform is
+ * ignored: meshes are measured relative to it.
+ */
+export declare const bindFaceModel: (root: THREE.Object3D, { mode, falloff, parts, align }?: BindFaceModelOptions) => BoundFaceMesh[];

package/dist/face-character/canonicalFace.d.ts

@@ -0,0 +1,17 @@
+import { CANONICAL_FACE_POSITIONS } from './canonicalFacePositions';
+export { CANONICAL_FACE_POSITIONS };
+/** Landmarks on the canonical face surface (the iris points 468–477 are not on it). */
+export declare const CANONICAL_FACE_LANDMARK_COUNT = 468;
+/**
+ * The canonical face surface: MediaPipe's face mesh triangles (landmark
+ * indices, flat). The mouth opening and the eye openings are holes — no
+ * triangle spans the inner lip ring or an eye ring — so a lip binds to its
+ * own lip, never across the mouth.
+ */
+export declare const CANONICAL_FACE_TRIANGLES: Uint16Array;
+/**
+ * The canonical face UV layout (flat `[u, v, …]` per landmark, v up: the chin
+ * near 0, the forehead near 1). A model whose face UVs follow it can bind by
+ * UV; glTF stores v down, which the binding detects.
+ */
+export declare const CANONICAL_FACE_UVS: Float32Array;

package/dist/face-character/canonicalFaceGeometry.d.ts

@@ -0,0 +1,11 @@
+import * as THREE from "three";
+/**
+ * The canonical face as a mesh to sculpt on: the 468 landmarks as vertices
+ * (in landmark order), the canonical triangles, the canonical UV layout and
+ * vertex normals — in the canonical head frame (cm, +Y up, +Z out of the
+ * face). Shape keys sculpted on it (keeping its vertices and their order) are
+ * `FaceWarp3D` blend shapes that apply exactly; export it with three's
+ * `GLTFExporter` to sculpt in Blender (the SDK repo ships it as
+ * `assets-src/face-character/canonical-face-reference.glb`).
+ */
+export declare const createCanonicalFaceGeometry: () => THREE.BufferGeometry;

package/dist/face-character/canonicalFacePositions.d.ts

@@ -0,0 +1,7 @@
+/**
+ * The 468 MediaPipe canonical face landmarks in the canonical head frame:
+ * centimetres, +X to the screen's right on a face looking at the camera, +Y up,
+ * +Z out of the face toward the camera (landmark 4 at `(0, -0.46, 7.59)`).
+ * Flat `[x, y, z, …]` in landmark order.
+ */
+export declare const CANONICAL_FACE_POSITIONS: Float32Array;

package/dist/face-character/driveFaceModel.d.ts

@@ -0,0 +1,57 @@
+import { BoundFaceMesh } from './bindFaceModel';
+import { FaceMorphRegion } from './faceMorphs';
+import { FaceRigState } from './faceRig';
+/**
+ * What moves a face model with the expression:
+ * - `"landmarks"` — the tracked landmarks bend the mesh (its own shape follows
+ *   the face: jaw, lips, cheeks, brows, lids);
+ * - `"morphs"` — the model's own morph targets, driven by the face's
+ *   blendshape scores (ARKit names); the mesh keeps the head pose only;
+ * - `"hybrid"` — landmarks for the jaw, lips and cheeks, the model's morphs
+ *   for the eyes and brows (thin, sculpted parts landmarks cannot close
+ *   cleanly — a blink rolls the authored lid over the eye);
+ * - `"auto"` (default) — `"morphs"` when the model has mouth morphs too (a
+ *   full ARKit set: the authored shapes look best), `"hybrid"` when it has
+ *   only eye / brow morphs, else `"landmarks"`;
+ * - `"none"` — the head pose only.
+ */
+export type FaceDriveMode = "auto" | "hybrid" | "morphs" | "landmarks" | "none";
+/** What a resolved drive mode does for one model. */
+export type FaceDrivePlan = {
+    mode: Exclude<FaceDriveMode, "auto">;
+    /** Whether the landmarks bend the mesh. */
+    landmarks: boolean;
+    /** The face regions whose morph targets are driven. */
+    morphRegions: ReadonlySet<FaceMorphRegion>;
+    /**
+     * Per landmark, how much of its offset to drop (1 = all): the regions the
+     * morphs take over in `"hybrid"`, so the two never fight. `null` = none.
+     */
+    mask: Float32Array | null;
+    /** Whether eyeball parts turn with the gaze. */
+    gaze: boolean;
+};
+/** Resolves a drive mode for a bound model (`"auto"` looks at its morph targets). */
+export declare const planFaceDrive: (meshes: readonly BoundFaceMesh[], mode: FaceDriveMode) => FaceDrivePlan;
+export type DriveFaceModelOptions = {
+    /** Expression strength: 0 = rest, 1 = the tracked face, > 1 exaggerates. Default 1. */
+    expression?: number;
+    /** Per expression channel (by `FACE_MORPH_NAMES` index): a gain on its morph. */
+    morphGain?: ArrayLike<number> | null;
+    /** How far a full look turns an eyeball: [sideways, up/down], degrees. Default [25, 20]. */
+    gazeDegrees?: readonly [number, number];
+    /**
+     * The head-size fit: the model's width at the ears and back of the head as
+     * a multiple of its own (1 = as modelled), ramping in from the face
+     * (`BoundFaceMesh.widen`). Surface and rigid parts; the face itself keeps its fit.
+     */
+    headWidth?: number;
+};
+/**
+ * One frame of a bound face model: bends the meshes by the landmarks and sets
+ * the morph targets and eyeball turns from the rig, as `plan` says. `rig`
+ * `null` or lost returns everything to rest. Writes the meshes' position
+ * attributes and `morphTargetInfluences`; leaves morph targets no expression
+ * channel drives untouched (so your own animation keeps them).
+ */
+export declare const driveFaceModel: (meshes: readonly BoundFaceMesh[], plan: FaceDrivePlan, rig: Pick<FaceRigState, "residual" | "morphs" | "tracking"> | null, { expression, morphGain, gazeDegrees, headWidth }?: DriveFaceModelOptions) => void;

package/dist/face-character/expressionOutputs.d.ts

@@ -0,0 +1,45 @@
+import { ExpressionRule, ExpressionState } from './faceExpressions';
+import * as THREE from "three";
+/**
+ * Where a controller rule writes its value (all optional, any combination),
+ * on the object named `target` in the controller's subtree (default: the
+ * whole subtree):
+ * - `morph` — morph target(s) by name on every mesh under it (an ARKit name
+ *   also matches its Face Cap / Blender spelling: `eyeBlinkLeft` ↔ `eyeBlink_L`);
+ * - `property` — a material uniform (`uniforms[name].value`) or numeric
+ *   property (`emissiveIntensity`, `opacity`) on every material under it;
+ * - `visible` — shown while the value is ≥ `above` (default 0.5) and/or ≤ `below`;
+ * - `transform` — an offset from the target's rest at value 1 (`position` in
+ *   its parent's units, `rotation` in degrees, `scale` as a factor), scaled by
+ *   the value; several rules on one target add up;
+ * - `set` — your own setter, called every frame with the value.
+ */
+export type ExpressionOutputs = {
+    target?: string;
+    morph?: string | readonly string[];
+    property?: string;
+    visible?: {
+        above?: number;
+        below?: number;
+    };
+    transform?: {
+        position?: readonly [number, number, number];
+        rotation?: readonly [number, number, number];
+        scale?: number;
+    };
+    set?: (value: number) => void;
+};
+/** A rule with outputs (what `FaceExpressionController` takes). */
+export type ExpressionOutputRule = ExpressionRule & ExpressionOutputs;
+/**
+ * The outputs of a rule set on a subtree: binds each rule's targets (morph
+ * slots, materials, the transform target's rest) and writes the values every
+ * frame. `resolve()` again when the subtree changes (`FaceExpressionController`
+ * does it every second, so late-loading children are picked up).
+ */
+export declare const createExpressionOutputs: (root: THREE.Object3D) => {
+    /** Binds `rules` to the subtree; with `warn`, reports targets it cannot find (once each). */
+    resolve(rules: readonly ExpressionOutputRule[], warn?: boolean): void;
+    /** Writes this frame's values (`state` from `stepExpressions`). */
+    apply(rules: readonly ExpressionOutputRule[], state: ExpressionState): void;
+};

package/dist/face-character/faceExpressions.d.ts

@@ -0,0 +1,104 @@
+import { FaceActionName } from '../face-action';
+import { FaceInfo } from '../face-info';
+import { FaceMorphName } from './faceMorphs';
+import { FaceRigState } from './faceRig';
+/** What a face rig exposes to expression rules, per frame. */
+export type ExpressionFrame = {
+    /** The 51 blendshape channels, 0..1, the person's neutral removed (`FaceRigState.morphs`). */
+    morphs: ArrayLike<number>;
+    /** The raw face (blendshapes as MediaPipe reports them, head rotation), or `null`. */
+    face: FaceInfo | null;
+    /** Whether a face drives the rules this frame (tracked or held). */
+    live: boolean;
+};
+/**
+ * What a rule reads:
+ * - a blendshape channel by ARKit name (`"jawOpen"`), neutral removed;
+ * - several combined: `{ channels: ["mouthSmileLeft", "mouthSmileRight"], combine: "avg" }`
+ *   (`"max"` default, `"min"`, `"avg"`, `"sum"`);
+ * - a Face Action's value (`{ action: "mouthOpen" }`, as `useFaceAction` reads it:
+ *   raw blendshapes, head turns in radians; left / right are the person's own);
+ * - your own function of the frame.
+ */
+export type ExpressionInput = FaceMorphName | {
+    channels: readonly FaceMorphName[];
+    combine?: "max" | "min" | "avg" | "sum";
+} | {
+    action: FaceActionName;
+} | ((frame: ExpressionFrame) => number);
+/**
+ * How a rule turns its input into a value, every frame:
+ * `input → range (→ 0..1) → curve → × multiplier → clamp → smoothing`.
+ */
+export type ExpressionShape = {
+    /**
+     * The input span mapped onto 0..1: below the first value reads 0, above the
+     * second reads 1 (a dead zone and a saturation). Default `[0, 1]`.
+     */
+    range?: readonly [number, number];
+    /** Exponent on the 0..1 value: > 1 eases in (subtle until strong), < 1 eases out. Default 1. */
+    curve?: number;
+    /** Scale after the curve (exaggerate > 1, tone down < 1). Default 1. */
+    multiplier?: number;
+    /** Limits after the multiplier. Default `[0, max(1, multiplier)]`. */
+    clamp?: readonly [number, number];
+    /**
+     * Per-frame EMA toward the target, `1` = none: one factor, or separate
+     * `attack` (rising) and `release` (falling) — e.g. a snappy blink that
+     * reopens softly. Default 0.6.
+     */
+    smoothing?: number | {
+        attack: number;
+        release: number;
+    };
+    /** The value at rest: on face loss (with `onLost: "rest"`) and after a reset. Default 0. */
+    rest?: number;
+    /** On face loss: ease back to `rest` (default) or `"hold"` the last value. */
+    onLost?: "rest" | "hold";
+};
+/**
+ * One rule: an input, its shaping, and (in `FaceExpressionController`) the
+ * outputs it drives. `id` names it in readouts and `useExpressionValues`.
+ */
+export type ExpressionRule = ExpressionShape & {
+    id?: string;
+    input: ExpressionInput;
+    /**
+     * Edge events with hysteresis: `onStart` when the value rises to
+     * `trigger[0]`, `onEnd` when it falls back to `trigger[1]`. Default
+     * `[0.5, 0.35]`.
+     */
+    trigger?: readonly [number, number];
+    onStart?: () => void;
+    onEnd?: () => void;
+};
+/** A rule's live state. */
+export type ExpressionValue = {
+    /** The shaped, smoothed value. */
+    value: number;
+    /** The shaped value before smoothing (this frame's target). */
+    target: number;
+    /** Whether the trigger is on (between `onStart` and `onEnd`). */
+    active: boolean;
+};
+export type ExpressionState = {
+    values: ExpressionValue[];
+};
+/** A rule's input, compiled to a reader (unknown names read 0 and warn once). */
+type Reader = (frame: ExpressionFrame) => number;
+export declare const compileExpressionInput: (input: ExpressionInput) => Reader;
+/** Shapes a raw input value (no smoothing): range → curve → multiplier → clamp. */
+export declare const shapeExpressionValue: (raw: number, shape?: ExpressionShape) => number;
+export declare const createExpressionState: (rules: readonly ExpressionRule[]) => ExpressionState;
+/** Puts every rule back at rest (and ends any active trigger, firing `onEnd`). */
+export declare const resetExpressions: (state: ExpressionState, rules: readonly ExpressionRule[]) => void;
+/**
+ * One frame of a rule set: reads each input from `frame`, shapes it, smooths
+ * it (attack / release) and fires its trigger edges. Readers come from
+ * `compileExpressionInput` (cache them with the rules). Pure apart from
+ * the callbacks; allocation-free.
+ */
+export declare const stepExpressions: (state: ExpressionState, rules: readonly ExpressionRule[], readers: readonly Reader[], frame: ExpressionFrame) => ExpressionState;
+/** Builds the frame a rule set reads from a face rig (and its raw face). */
+export declare const expressionFrame: (rig: Pick<FaceRigState, "morphs" | "tracking" | "face"> | null, out?: ExpressionFrame) => ExpressionFrame;
+export {};