@vincentt-xr/sdk 2.0.0-alpha.8 → 2.0.0-alpha.9

package/dist/face-atlas/probe/selfTest.d.ts

@@ -0,0 +1,87 @@
+import { FaceImageLandmark } from '../../face-character/faceReprojection';
+import { FaceSkinStats } from './stats';
+import { FaceSkinProbeFormat, FaceSkinStatsFormat } from './targets';
+import * as THREE from "three";
+/**
+ * A GPU self-test of `FaceSkinProbe` on a synthetic face whose answers are
+ * known: a canonical face drawn on the CPU (Lambert shading from a known
+ * light, a known skin, lip and brow colour), unwrapped and measured on this
+ * device's GPU, then read back. It exercises the paths phones disagree on:
+ * RGBA16F MRT rendering, ConstantAlpha blending, mipmap generation, the
+ * stats ping-pong and its precision, and the RGBA8 fallback. The headless
+ * smoke run drives it (`example/smoke/face-skin-probe.spec.ts`); open
+ * `/smoke/face-skin-probe.html` on the dev server to run it on a device.
+ */
+type Vec3 = [number, number, number];
+/** The synthetic face's truths. */
+export declare const FACE_SKIN_SELF_TEST_SCENE: Readonly<{
+    size: 256;
+    /** px per canonical cm. */
+    scale: 9;
+    yawDeg: 8;
+    skin: Vec3;
+    lip: Vec3;
+    brow: Vec3;
+    background: 0.2;
+    ambient: 0.3;
+    diffuse: 0.7;
+    /** View space (x right, y up, z toward the camera). */
+    light: Vec3;
+    /** The second image's gain (the EMA step). */
+    gain: 1.25;
+    /** Seconds of the second update. */
+    dt: 0.25;
+    /**
+     * The hold: from a still at gain 1, `holdUpdates` live updates `holdDt`
+     * apart on an image at `holdGain`. A 2 % step is under what half-float
+     * stats can store per update at 15 Hz (the slow skin stalls short of it).
+     */
+    holdGain: 1.02;
+    holdUpdates: 300;
+    holdDt: number;
+    /** After the hold, `Rs` within this of `Rf` (20 s is 5 τ: 0.7 % of the step is left). */
+    holdTolerance: 0.005;
+}>;
+export type FaceSkinSelfTestImage = {
+    width: number;
+    height: number;
+    /** RGBA8, sRGB-encoded, row 0 = image top (a `DataTexture` with `flipY` false). */
+    pixels: Uint8Array;
+    landmarks: FaceImageLandmark[];
+    /** The light, unit, view space. */
+    light: Vec3;
+};
+/** Draws the synthetic face (pure, no GPU). `gain` scales its linear colour. */
+export declare const createFaceSkinSelfTestImage: (gain?: number) => FaceSkinSelfTestImage;
+/** The stats read back at each step of one run. */
+export type FaceSkinSelfTestReadings = {
+    /** After the still image. */
+    still: FaceSkinStats;
+    /** After one update on the `gain` image. */
+    next: FaceSkinStats;
+    /** After a second still at gain 1 (the hold's start). */
+    holdStart: FaceSkinStats;
+    /** After `holdUpdates` updates on the `holdGain` image. */
+    held: FaceSkinStats;
+};
+/** One format's outcome. */
+export type FaceSkinSelfTestRun = {
+    format: FaceSkinProbeFormat;
+    statsFormat: FaceSkinStatsFormat;
+    ok: boolean;
+    /** What failed, or a summary. */
+    detail: string;
+    readings: FaceSkinSelfTestReadings | null;
+};
+/** The checks on one probe's readbacks (pure; tested without a GPU). */
+export declare const judgeFaceSkinSelfTest: ({ still, next, holdStart, held }: FaceSkinSelfTestReadings, light: Vec3) => string[];
+/**
+ * Runs the self-test on `renderer` (or a new 1 × 1 one, disposed after) for
+ * each format: the device's choice and the RGBA8 fallback. Never throws.
+ */
+export declare const runFaceSkinProbeSelfTest: (renderer?: THREE.WebGLRenderer) => {
+    ok: boolean;
+    runs: FaceSkinSelfTestRun[];
+    error: string | null;
+};
+export {};

package/dist/face-atlas/probe/stats.d.ts

@@ -0,0 +1,143 @@
+import { MakeupSkinTexel } from '../shading';
+import { FaceSkinStepParams } from './constants';
+/**
+ * The probe's stats: one small texture (`lightingMap`) of 8 texels the
+ * stats pass rewrites on every update (ping-pong), read by shaders with
+ * `texelFetch` (the `FACE_SKIN_STATS_GLSL` reader). This file holds its
+ * layout, the probe's sample points, and `stepFaceSkinStats`, the
+ * line-for-line TypeScript mirror of the stats pass (tests and docs).
+ */
+/** The stats texels (`lightingMap`, read with `faceSkinStat(texel)`). Colours are linear. */
+export declare const FACE_SKIN_STATS: Readonly<{
+    /** Texels in the layout (texels 5–7 are reserved, 0). */
+    readonly texels: 8;
+    /** rgb `Rs`: the person's skin under the settled light (τ 4 s). a: `ready`, 0 → 1 over 0.3 s of data. */
+    readonly skinSlow: 0;
+    /** rgb `Rf`: the skin under the current light (τ 0.25 s). a: skin coverage now (the skin map's top-mip weight). */
+    readonly skinFast: 1;
+    /**
+     * The light, `lum = a + b · n` over the skin: rgb the gradient `b` in VIEW
+     * space (x right, y up, z toward the camera), a the ambient `a`. Direction
+     * `normalize(b)`, confidence `|b| / a` (under 0.15 shading uses the key light).
+     */
+    readonly light: 2;
+    /** rgb: the person's own lip colour. a: how much lip the taps saw (0..1). */
+    readonly lip: 3;
+    /**
+     * rgb: the person's own brow-hair colour, `skin + (brow − skin) / 0.6` over
+     * the whole brow band, the extrapolation stopped at half the band's colour.
+     * a: how much brow the taps saw.
+     */
+    readonly brow: 4;
+}>;
+/** Where the stats pass samples the maps (atlas UV, v up). */
+export type FaceSkinTaps = {
+    /** `LIGHT_FIT_TAPS`: UV (2 per tap), rest normal (3 per tap), `LIGHT_FIT_GROUP` flags. */
+    light: {
+        uv: Float32Array;
+        normal: Float32Array;
+        group: Int32Array;
+    };
+    /**
+     * The middle of the lip vermilion: halfway between each outer-ring landmark
+     * and its inner-ring partner, away from the corners (2 per tap).
+     */
+    lip: Float32Array;
+    /**
+     * Across each brow, upper edge to lower edge, at `FACE_SKIN_BROW_TAP_ROWS`
+     * (a sixth, half and five sixths down), both sides (2 per tap).
+     */
+    brow: Float32Array;
+};
+/**
+ * Where the brow taps sit between each upper-edge landmark (0) and its
+ * lower-edge partner (1): the centres of three equal strips, so they average
+ * the whole band, whose mean holds the hair share
+ * `FACE_SKIN_PROBE.browHairShare` assumes (the middle alone is nearly all hair).
+ */
+export declare const FACE_SKIN_BROW_TAP_ROWS: readonly number[];
+/** The stats pass's sample points (built once, shared: do not mutate). */
+export declare const getFaceSkinTaps: () => FaceSkinTaps;
+type Vec3 = [number, number, number];
+/** What the stats pass samples, as raw texels of the maps (`(c·w, w)`, or `(w·√c, w)` on RGBA8). */
+export type FaceSkinStatsInput = {
+    /** The skin map's top mip (1 × 1): the weighted mean of all the skin seen. */
+    skinTop: MakeupSkinTexel;
+    /** The skin map at the low-pass mip (`makeupSkinLod(size)`) at each light tap. */
+    lightSkin: readonly MakeupSkinTexel[];
+    /** The feature map (mip 0) at each lip tap. */
+    lipFeature: readonly MakeupSkinTexel[];
+    /** The feature map (mip 0) at each brow tap. */
+    browFeature: readonly MakeupSkinTexel[];
+    /** The skin map at the low-pass mip at each brow tap: the local skin around the brow. */
+    browSkin: readonly MakeupSkinTexel[];
+    /** The skin map `lowpassCoarseLods` mips coarser at each brow tap (where the low-pass has no skin). */
+    browSkinCoarse: readonly MakeupSkinTexel[];
+    /** Canonical head frame → view at this update, 3 × 3 column-major (`THREE.Matrix3.elements`). */
+    rotation: ArrayLike<number>;
+};
+/** The stats pass's blend factors and flags, plus the maps' encoding. */
+export type FaceSkinStatsParams = FaceSkinStepParams & {
+    /** The maps hold `(w·√c, w)` (RGBA8). */
+    sqrt: boolean;
+};
+/**
+ * The light fit over the light taps, against the rest normals rotated into
+ * view space: `[a, bx, by, bz]`, or null when less than `minLightTaps` of
+ * skin was seen. Exactly `fitLight` (the CPU reference the GLSL ports) with
+ * the tap groups.
+ */
+export declare const fitFaceSkinLight: (lightSkin: readonly MakeupSkinTexel[], rotation: ArrayLike<number>, sqrt: boolean) => [number, number, number, number] | null;
+/**
+ * The person's brow-hair colour from the brow band's mean colour and the
+ * skin around it: the band is `browHairShare` hair, so hair =
+ * `skin + (brow − skin) / share`, along the skin → brow line. Dense brows
+ * hold more hair than that, and the line would run past black, so the
+ * factor stops where any channel reaches `browHairFloor` of the band's
+ * colour (or 1 for brows lighter than the skin): a dark brown stays brown.
+ */
+export declare const faceSkinBrowHair: (brow: Readonly<Vec3>, skin: Readonly<Vec3>) => Vec3;
+/**
+ * One stats update (the stats pass, texel by texel): `previous` and `out`
+ * are `FACE_SKIN_STATS.texels × 4` floats in the `FACE_SKIN_STATS` layout.
+ *
+ * - `Rf` follows the skin map's top mip (already averaged over time on half
+ *   float; `fastBlend` averages it here on RGBA8), `Rs` follows `Rf` slowly.
+ * - Every slow average restarts (factor 1) on a snap, and also when `Rf`'s
+ *   luminance differs from `Rs`'s by more than `snapJump` (40 %).
+ * - With too little skin seen (`minCoverage`), the values hold; on a snap
+ *   they clear (not ready). Too few light, lip or brow taps hold that value.
+ * - Brow hair is `faceSkinBrowHair` of the brow band's mean and the skin
+ *   around it.
+ */
+export declare const stepFaceSkinStats: (previous: ArrayLike<number>, input: FaceSkinStatsInput, params: FaceSkinStatsParams, out?: Float32Array) => Float32Array;
+/** The stats, decoded. Colours are linear RGB. */
+export type FaceSkinStats = {
+    /** `Rs`. */
+    skinSlow: Vec3;
+    /** 0 until the probe has data, then 1. */
+    ready: number;
+    /** `Rf`. */
+    skinFast: Vec3;
+    /** The skin coverage at the last update (share of the atlas seen as skin). */
+    coverage: number;
+    light: {
+        /** `b` of `lum = a + b · n`, view space. */
+        gradient: Vec3;
+        /** `a`. */
+        ambient: number;
+        /** `normalize(b)` (`[0, 0, 1]` when b is 0). */
+        direction: Vec3;
+        /** `|b| / a`, capped at 100. Under 0.15 shading uses the key light. */
+        confidence: number;
+    };
+    lip: Vec3;
+    lipSeen: number;
+    brow: Vec3;
+    browSeen: number;
+};
+/** The fit's confidence `|b| / a` (`fitLight`'s rule: capped, and safe for a ≤ 0). */
+export declare const faceSkinLightConfidence: (gradient: ArrayLike<number>, ambient: number) => number;
+/** Decodes `FACE_SKIN_STATS.texels × 4` floats (see `FaceSkinProbe.readStats`). */
+export declare const readFaceSkinStats: (data: ArrayLike<number>) => FaceSkinStats;
+export {};

package/dist/face-atlas/probe/targets.d.ts

@@ -0,0 +1,70 @@
+import * as THREE from "three";
+/**
+ * The probe's render targets and their formats.
+ *
+ * Maps (`FaceSkinProbeFormat`):
+ * - `"half-float"`: RGBA16F, when the GPU can render to it
+ *   (`EXT_color_buffer_half_float` or `EXT_color_buffer_float`). The skin and
+ *   feature maps average over time with ConstantAlpha blending.
+ * - `"rgba8"`: the fallback. The maps hold `(w·√c, w)` (the square root
+ *   before premultiplying keeps dark skin precise; `uSkinSqrt` = 1) and are
+ *   replaced on every update (8-bit EMAs stall).
+ *
+ * Stats (`FaceSkinStatsFormat`): the slow averages move by about 1.7 % of
+ * the difference per update, which half floats round away (a 2 % step never
+ * lands), so the stats are never RGBA16F:
+ * - `"float"`: RGBA32F, when the GPU can render to it (`EXT_color_buffer_float`;
+ *   the stats pass neither blends nor filters, so nothing else is needed);
+ * - `"packed"`: RGBA8, each value in three texels (24 bits).
+ */
+export type FaceSkinProbeFormat = "half-float" | "rgba8";
+/** How the stats ping-pong stores its values (see `FaceSkinProbeFormat`). */
+export type FaceSkinStatsFormat = "float" | "packed";
+type ExtensionReader = {
+    extensions: {
+        has: (name: string) => boolean;
+    };
+};
+/** Whether `renderer` can render to (and blend into) RGBA16F targets. */
+export declare const canRenderHalfFloat: (renderer: ExtensionReader) => boolean;
+/** Whether `renderer` can render to RGBA32F targets (no blending or filtering needed). */
+export declare const canRenderFloat: (renderer: ExtensionReader) => boolean;
+/** The probe format for a `halfFloat` option: `"auto"` asks the renderer. */
+export declare const resolveFaceSkinProbeFormat: (halfFloat: boolean | "auto", renderer: ExtensionReader | null) => FaceSkinProbeFormat;
+/**
+ * The stats format: RGBA32F where the renderer can render to it, else packed.
+ * The RGBA8 fallback (`"rgba8"` maps) always packs. Without a renderer it is
+ * provisional (`"float"`), settled on the first render.
+ */
+export declare const resolveFaceSkinStatsFormat: (format: FaceSkinProbeFormat, renderer: ExtensionReader | null) => FaceSkinStatsFormat;
+/** Logical texels in the stats texture. */
+export declare const FACE_SKIN_STATS_TEXELS = 8;
+/** Texels per value in the packed stats layout (8 bits each: high, middle, low). */
+export declare const FACE_SKIN_STATS_PACKED_TEXELS = 3;
+/** The stats texture's width: 8 texels, or 24 when each value is packed into three. */
+export declare const faceSkinStatsWidth: (format: FaceSkinStatsFormat) => number;
+/**
+ * The skin map (texture 0) and the feature map (texture 1), one MRT target:
+ * `size`², mipmapped (trilinear), linear data (no colour space), no depth.
+ */
+export declare const createFaceSkinMapTarget: (size: number, format: FaceSkinProbeFormat) => THREE.WebGLRenderTarget<THREE.Texture>;
+/** One side of the stats ping-pong: `faceSkinStatsWidth(format)` × 1, nearest, no mips. */
+export declare const createFaceSkinStatsTarget: (format: FaceSkinStatsFormat) => THREE.WebGLRenderTarget<THREE.Texture>;
+/** The stats texel whose rgb is signed (the light gradient), stored as `(x + 1) / 2` when packed. */
+export declare const FACE_SKIN_STATS_SIGNED_TEXEL = 2;
+/**
+ * Packs one value in 0..1 into three bytes (high, middle, low), as the
+ * shader writes it to an RGBA8 target (which rounds each byte).
+ */
+export declare const packFaceSkinValue: (value: number) => [number, number, number];
+/**
+ * Decodes stats read back from the GPU into `FACE_SKIN_STATS_TEXELS × 4`
+ * floats: float data as is, packed data (bytes, 24 texels) unpacked.
+ */
+export declare const decodeFaceSkinStats: (data: ArrayLike<number>, format: FaceSkinStatsFormat, out?: Float32Array) => Float32Array;
+/**
+ * Encodes `FACE_SKIN_STATS_TEXELS × 4` floats as the packed stats texture
+ * holds them (24 texels of bytes): the inverse of `decodeFaceSkinStats`.
+ */
+export declare const encodeFaceSkinStatsPacked: (stats: ArrayLike<number>, out?: Uint8Array) => Uint8Array;
+export {};

package/dist/face-atlas/probe/unwrap.d.ts

@@ -0,0 +1,76 @@
+import { FaceImageLandmark } from '../../face-character/faceReprojection';
+import * as THREE from "three";
+export type FaceSkinUnwrapImage = {
+    /** The image's width / height. */
+    aspect: number;
+    /**
+     * The texture keeps the image's top row at `v = 1` (`faceSkinTextureTopAtV1`),
+     * so a landmark samples at `(x, 1 − y)`; else at `(x, y)`.
+     */
+    flipY: boolean;
+};
+/**
+ * Where a texture keeps its image's top row: `true` at `v = 1` (three's
+ * default `flipY` upload of an image, canvas or video, and anything three
+ * rendered into a render target), `false` at `v = 0` (`flipY` false, as a
+ * `DataTexture` built top row first, and any `ImageBitmap`: WebGL ignores
+ * `flipY` for those uploads).
+ */
+export declare const faceSkinTextureTopAtV1: (texture: THREE.Texture) => boolean;
+/**
+ * One update of the unwrap: `cameraUv` (2 per landmark) is where each atlas
+ * vertex samples the image (texture space), `weight` (1 per landmark) how
+ * much it counts: how squarely it faces the camera (each triangle's area on
+ * the image over its 3D area, averaged per vertex, the landmarks' own z)
+ * times a fade near the image edge. Landmarks are normalized image
+ * coordinates (y down), z in x units, as the tracker and the still-image
+ * detector give them. Allocation-free.
+ */
+export declare const writeFaceSkinUnwrapFrame: (landmarks: ArrayLike<FaceImageLandmark>, image: FaceSkinUnwrapImage, cameraUv: Float32Array, weight: Float32Array) => void;
+/** The head's pose as the probe measures it from landmarks. */
+export type FaceSkinPose = {
+    /**
+     * Canonical head frame → the face's line-of-sight frame (x right, y up, z
+     * from the face toward the camera), as a three.js `Matrix3` (column-major
+     * `elements`, ready for a `mat3` uniform). Weak perspective: the landmarks
+     * show the head's turn against the ray to the camera, so a face off the
+     * optical axis that looks into the lens reads as frontal. The camera's view
+     * frame is `lineOfSightToView(ray) · rotation`.
+     */
+    rotation: THREE.Matrix3;
+    /** Head turn left/right against the line of sight, degrees (0 looking into the lens). */
+    yawDeg: number;
+    /** Head tilt up/down against the line of sight, degrees (0 looking into the lens). */
+    pitchDeg: number;
+    /** Landmark 10 (forehead) to 152 (chin) on the image, px. */
+    faceHeightPx: number;
+};
+export declare const createFaceSkinPose: () => FaceSkinPose;
+/**
+ * The rotation from a face's line-of-sight frame to the camera's view frame:
+ * the smallest turn of +Z onto the direction from the face back to the
+ * camera. `ray` is the direction from the camera to the face in view space
+ * (the camera looks down −Z), any length. Identity for a missing, zero or
+ * backward ray. Writes `out`; allocation-free.
+ */
+export declare const lineOfSightToView: (ray: THREE.Vector3Like | null | undefined, out?: THREE.Matrix3) => THREE.Matrix3;
+/**
+ * The view-space direction from a perspective `camera` to a feed point
+ * (0..1, y down; the feed fills the camera's frustum, as `feedToWorld`
+ * assumes): the `lineOfSight` of `FaceSkinProbe.render`. Null for a camera
+ * without perspective (every line of sight is the optical axis there).
+ * Writes `out`; allocation-free.
+ */
+export declare const feedLineOfSight: (point: {
+    x: number;
+    y: number;
+}, camera: THREE.Camera, out: THREE.Vector3) => THREE.Vector3 | null;
+/**
+ * Fits the head's rotation to the landmarks: the similarity from the
+ * canonical rigid landmarks (no mouth or jaw) onto the landmarks in image
+ * units, `(x · aspect, −y, −z · aspect)` (MediaPipe's z is smaller toward
+ * the camera). Any image works, mirrored or not: the tracker reports a
+ * proper rotation of its canonical face either way. `width` and `height`
+ * are the image's pixel size.
+ */
+export declare const fitFaceSkinPose: (landmarks: ArrayLike<FaceImageLandmark>, width: number, height: number, out?: FaceSkinPose) => FaceSkinPose;

package/dist/face-atlas/quality.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Quality tiers of the face-atlas surface, by the session's adaptive quality
+ * tier (`useXRQuality()`: first tier = high, last = low, any between =
+ * medium; no controller = high).
+ *
+ * - `leadCapMs`: the landmark lead of a rig the surface owns, about one
+ *   inference interval at the tier's rate (30 / 24 / 15 Hz). A borrowed rig
+ *   (`FaceRig`, `FaceWarp3D`, `FaceMorph`) keeps its own `leadMs`.
+ * - `selfOcclusion`: the depth pre-pass and restore (the far cheek stops at
+ *   the nose); off on the low tier, where only the facing fade applies.
+ */
+export declare const FACE_ATLAS_QUALITY: Readonly<{
+    readonly high: {
+        readonly leadCapMs: 33;
+        readonly selfOcclusion: true;
+    };
+    readonly medium: {
+        readonly leadCapMs: 42;
+        readonly selfOcclusion: true;
+    };
+    readonly low: {
+        readonly leadCapMs: 66;
+        readonly selfOcclusion: false;
+    };
+}>;
+export type FaceAtlasQualityKey = keyof typeof FACE_ATLAS_QUALITY;
+/** The face-atlas tier for the session's tier index (`useXRQuality().index`) of `tierCount`. */
+export declare const resolveFaceAtlasQualityKey: (tierIndex: number, tierCount: number) => FaceAtlasQualityKey;

package/dist/face-atlas/regions.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Named parts of the face atlas. `eyes` and `mouth` are the fill triangles
+ * over the eye and mouth openings; every other region leaves them and the
+ * nostrils out. `lids` is the whole eye surround: from the under-eye row
+ * (landmarks 111, 117–121, three rings below the lower lash line) up to the
+ * brow's lower edge.
+ */
+export declare const FACE_ATLAS_REGIONS: readonly ["face", "skin", "lips", "mouth", "eyes", "lids", "brows", "cheeks", "nose", "forehead", "jaw"];
+export type FaceAtlasRegion = (typeof FACE_ATLAS_REGIONS)[number];
+/** One region or several (their union). */
+export type FaceAtlasRegionSelection = FaceAtlasRegion | readonly FaceAtlasRegion[];
+/** A region's extent in atlas UV (v up). */
+export type FaceAtlasUvBounds = {
+    minU: number;
+    maxU: number;
+    minV: number;
+    maxV: number;
+};
+/**
+ * Triangle indices (into `FACE_ATLAS_TRIANGLES`) of a region or a union of
+ * regions, sorted and without duplicates. Shared and cached: do not mutate.
+ */
+export declare const getFaceAtlasRegionTriangles: (selection: FaceAtlasRegionSelection) => Uint16Array;
+/**
+ * An index buffer (landmark indices, three per triangle) for a region or a
+ * union of regions, ready for `geometry.setIndex(new THREE.BufferAttribute(…, 1))`
+ * on a 468-vertex atlas mesh. A new array on every call.
+ */
+export declare const getFaceAtlasRegionIndices: (selection: FaceAtlasRegionSelection) => Uint16Array;
+/** UV extent (v up) of a region or a union of regions. */
+export declare const getFaceAtlasRegionBounds: (selection: FaceAtlasRegionSelection) => FaceAtlasUvBounds;