@vincentt-xr/sdk 2.0.0-alpha.7 → 2.0.0-alpha.9

package/dist/face-atlas/registry.d.ts

@@ -0,0 +1,252 @@
+import { FaceRigDriver, FaceRigOptions, FaceRigState } from '../face-character/faceRig';
+import { XRModelNodeFaceTracking } from '../types/nodes';
+import { FaceAtlasSurface } from './surface';
+import * as THREE from "three";
+/**
+ * The face-atlas registry: one live surface per face and per R3F root,
+ * shared by every layer drawn on that face (makeup regions, LipEffect, face
+ * swap, faceless, custom `FaceAtlasLayer`s), refcounted.
+ *
+ * An entry either BORROWS a rig (the layers are nested in a `FaceRig`,
+ * `FaceWarp3D` or `FaceMorph`: they ride that face, with its lead and its
+ * moved surface) or OWNS one (no rig above: it steps a rig of its own from
+ * the face model with `stepFaceRigFromModel`). Its per-frame work runs once
+ * per frame whichever layer asks first (frame-stamped), so N layers on a
+ * face cost one rig step and one surface update.
+ *
+ * Each entry owns a pose group (added to the scene root while in use): the
+ * layers' meshes and the self-occlusion pre-pass / restore live in it, posed
+ * by the rig, out of any tracker group's render order.
+ */
+/** A ref to something published per frame (a `FaceRig`'s state, a face surface's offsets). */
+export type FaceAtlasRef<T> = {
+    current: T;
+};
+/** The moved surface a `FaceWarp3D` / `FaceMorph` publishes (`FaceSurfaceState`). */
+export type FaceAtlasOffsetsSource = FaceAtlasRef<{
+    offsets: Float32Array;
+    active: boolean;
+}>;
+/** How an entry is found: by the rig it borrows (and the surface it rides), or by face index. */
+export type FaceAtlasEntryKey = {
+    kind: "owned";
+    faceIndex: number;
+} | {
+    kind: "borrowed";
+    rig: FaceAtlasRef<FaceRigState>;
+    offsets: FaceAtlasOffsetsSource | null;
+};
+/**
+ * Uniforms every face-atlas layer of one face shares (the same `{ value }`
+ * objects), updated once per frame. Spread them into a layer's material.
+ */
+export type FaceAtlasUniforms = {
+    /** The camera feed (sample it at `basePosition`'s screen point). */
+    uCameraMap: {
+        value: THREE.Texture | null;
+    };
+    /** 0..1: fades in when the face is found, out once it is lost. */
+    uPresence: {
+        value: number;
+    };
+    /** #298 `personLeftSign(mirrored)`: +1 unmirrored, −1 on the mirrored webcam (set per layer). */
+    uPersonLeftSign: {
+        value: number;
+    };
+    /**
+     * The skin probe's statistics (`FACE_SKIN_STATS_TEXELS`): copied each
+     * frame from the face's probe while one runs (a makeup region, or a
+     * `useFaceSkin` call for this face); `null` while none does.
+     */
+    uSkinStats: {
+        value: THREE.Texture | null;
+    };
+    /** 1 when `uSkinStats` is the packed RGBA8 layout, 0 for float stats. */
+    uSkinStatsPacked: {
+        value: number;
+    };
+    /** The skin probe's atlas skin map; `null` while no probe runs. */
+    uSkinMap: {
+        value: THREE.Texture | null;
+    };
+    /** The skin map's low-pass mip. */
+    uSkinLod: {
+        value: number;
+    };
+    /** 1 when the skin map uses the RGBA8 square-root fallback. */
+    uSkinSqrt: {
+        value: number;
+    };
+};
+/** The skin probe's uniform values an entry copies (`FaceSkinProbe.uniforms`). */
+export type FaceAtlasSkinSource = {
+    uSkinStats: {
+        value: THREE.Texture | null;
+    };
+    uSkinStatsPacked: {
+        value: number;
+    };
+    uSkinMap: {
+        value: THREE.Texture | null;
+    };
+    uSkinLod: {
+        value: number;
+    };
+    uSkinSqrt: {
+        value: number;
+    };
+};
+export declare const createFaceAtlasUniforms: () => FaceAtlasUniforms;
+/** What a layer hands the entry each frame. */
+export type FaceAtlasFrameInput = {
+    /** The frame's stamp: the same value for every caller in one frame (R3F's `clock.elapsedTime`). */
+    stamp: number;
+    nowMs: number;
+    /** The camera feed texture (`useXRCameraTexture()`); a new one is a new face source. */
+    cameraTexture: THREE.Texture | null;
+    /** Whether the quality tier allows the self-occlusion passes. */
+    selfOcclusion: boolean;
+    /** For `faceUv`: the scene camera (projection × view). */
+    camera?: THREE.Camera;
+};
+/** One face's shared surface. Created by `FaceAtlasRegistry`; never construct it yourself. */
+export declare class FaceAtlasEntry {
+    readonly key: FaceAtlasEntryKey;
+    readonly surface: FaceAtlasSurface;
+    /** The pose group: layer meshes go in here (a portal target). */
+    readonly group: THREE.Group<THREE.Object3DEventMap>;
+    /** The rig the surface follows (borrowed, or this entry's own). */
+    readonly rig: FaceAtlasRef<FaceRigState>;
+    readonly uniforms: FaceAtlasUniforms;
+    /** Layers holding the entry (`retain` / `release`). */
+    refs: number;
+    /** The owned rig's driver; `null` when borrowing. */
+    readonly driver: FaceRigDriver | null;
+    readonly prepass: THREE.Mesh;
+    readonly restore: THREE.Mesh;
+    private rigStamp;
+    private surfaceStamp;
+    private lastNowMs;
+    private selfOcclusionRequests;
+    /** Each layer's `holdMs` (keyed by its request token), and their largest. */
+    private readonly holdRequests;
+    private requestedHoldMs;
+    /** The options `stepRig` hands the rig: the caller's, with the shared hold (no allocation per frame). */
+    private readonly stepOptions;
+    private scene;
+    /** The last camera feed seen (never set back to `null`). */
+    private lastCameraTexture;
+    constructor(key: FaceAtlasEntryKey);
+    /** Whether this entry steps a rig of its own. */
+    get owned(): boolean;
+    /** The face this entry follows. */
+    get faceIndex(): number;
+    /** Whether some layer asked for the self-occlusion passes. */
+    get wantsSelfOcclusion(): boolean;
+    /** Counts a layer that wants the self-occlusion passes; returns its release. */
+    requestSelfOcclusion(): () => void;
+    /**
+     * The hold of the owned rig through dropouts: the largest `holdMs` any
+     * layer on the face asks for (`requestHold`), or `undefined` (the rig's
+     * default, 250 ms) when none does. A borrowed rig keeps its own hold.
+     */
+    get holdMs(): number | undefined;
+    /**
+     * Counts a layer's `holdMs` (ms, at least 0) toward the face's shared
+     * hold, where the largest value wins; returns its release. A value that
+     * is not a finite number counts as no request.
+     */
+    requestHold(holdMs: number): () => void;
+    private updateHold;
+    /**
+     * The owned rig's step (frame priority -1): `stepFaceRigFromModel` once per
+     * `stamp`, whoever calls first. A no-op when borrowing (the `FaceRig` steps
+     * its own). `cameraTexture` is the camera feed (`useXRCameraTexture()`):
+     * a new feed restarts the rig first (`followCameraTexture`). The hold is
+     * the face's shared one (`holdMs`) when a layer asked for one, else
+     * `options.holdMs`. Returns the tracking edge of this frame's step, if it ran.
+     */
+    stepRig(stamp: number, node: XRModelNodeFaceTracking | null | undefined, camera: THREE.Camera, nowMs: number, options?: FaceRigOptions, cameraTexture?: THREE.Texture | null): "found" | "lost" | null;
+    /**
+     * The surface update (frame priority -0.4, after the rig and after warps
+     * publish their offsets): positions, normals, presence, the pose group and
+     * the shared uniforms, once per `stamp`. Returns false when it already ran.
+     */
+    update({ stamp, nowMs, cameraTexture, selfOcclusion, camera }: FaceAtlasFrameInput): boolean;
+    /**
+     * Copies the face's skin probe values into the shared uniforms (each
+     * frame, after the probe's update), or clears them with `null` (no probe
+     * runs for this face).
+     */
+    updateSkin(source: FaceAtlasSkinSource | null): void;
+    /**
+     * A new camera feed (another camera, another video) is another face, maybe
+     * another person: an owned rig restarts (`resetFaceRigDriver`: the neutral
+     * is captured again, the pose and the landmark lead snap), as `FaceRig`
+     * does for its own, and the surface's normals snap instead of blending
+     * from the old face. The first feed, and a feed briefly gone (`null`),
+     * restart nothing.
+     */
+    private followCameraTexture;
+    /** Adds the pose group to `scene` (the R3F root) while the entry is in use. */
+    attach(scene: THREE.Object3D): void;
+    detach(): void;
+    /** Frees the GPU resources; the entry is unusable after. */
+    dispose(): void;
+    /** The pose group's matrix = the rig's pose (canonical cm → world), under any scene transform. */
+    private pose;
+    private depthMesh;
+}
+/**
+ * The entries of one R3F root. `get` finds or creates an entry (not yet
+ * counted); `retain` / `release` count the layers using it, and the last
+ * release disposes it.
+ */
+export declare class FaceAtlasRegistry {
+    private readonly owned;
+    private readonly borrowed;
+    /** The entry for `key`, created on first use. Count it with `retain` (in an effect). */
+    get(key: FaceAtlasEntryKey): FaceAtlasEntry;
+    /** Counts one more layer on `entry`, and puts its pose group in `scene`. */
+    retain(entry: FaceAtlasEntry, scene: THREE.Object3D): void;
+    /** One layer fewer; the last one disposes the entry. */
+    release(entry: FaceAtlasEntry): void;
+    /** The callback `forEachEntry` hands the borrowed entries to (no closure per call). */
+    private visit;
+    private readonly visitBorrowed;
+    /** Calls `visit` for every entry (owned, then borrowed). */
+    forEachEntry(visit: (entry: FaceAtlasEntry) => void): void;
+    /** Entries in use or created (for tests and diagnostics). */
+    get size(): number;
+}
+/** Where a face-atlas layer sits, for `resolveFaceAtlasBinding`. */
+export type FaceAtlasBindingInput = {
+    /** The `faceIndex` prop, if given (any value; resolved like `FaceRetouch`'s). */
+    faceIndex?: unknown;
+    /** The `numFaces` prop, if given. */
+    numFaces?: unknown;
+    /** The face of the nearest `FaceRig` (`FaceWarp3D`, `FaceMorph`) above, or `null` without one. */
+    rigFaceIndex: number | null;
+    /** The enclosing `FaceTracker`'s face, or `null` outside a face tracker. */
+    trackerFaceIndex: number | null;
+};
+/** How a layer binds to a face: which face, whether it rides the rig above, whether it starts the model. */
+export type FaceAtlasBinding = {
+    faceIndex: number;
+    /** Ride the `FaceRig` above (its pose, lead and warp offsets). */
+    borrow: boolean;
+    /** Start the face model itself: only with no face tracker and no rig above. */
+    acquire: boolean;
+    /** The face count to start the model with (`resolveNumFaces`). */
+    numFaces: number;
+};
+/**
+ * A layer rides the `FaceRig` above when it asks for no face or for that
+ * rig's face; otherwise it owns a rig for its face (the `faceIndex` prop,
+ * else the `FaceTracker`'s, else 0). It starts the face model only when no
+ * tracker or rig above does: a tracker starts it with its own `numFaces`,
+ * and a child acquiring first would set the count (effects run child-first).
+ */
+export declare const resolveFaceAtlasBinding: ({ faceIndex, numFaces, rigFaceIndex, trackerFaceIndex, }: FaceAtlasBindingInput) => FaceAtlasBinding;
+/** The registry of an R3F root, keyed by any per-root object (its scene). */
+export declare const getFaceAtlasRegistry: (root: object) => FaceAtlasRegistry;

package/dist/face-atlas/renderState.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Draw order and blending of everything drawn on the face atlas surface
+ * (makeup, LipEffect, custom face-atlas layers) and of `FaceRetouch`.
+ * Re-exported from `@vincentt-xr/sdk/face-effects`.
+ *
+ * | Order | Pass |
+ * | --- | --- |
+ * | -1000 | camera background |
+ * | -900 | `HandCutout` (depth at the near plane) |
+ * | -2.5 | `FaceRetouch` (opaque, blended, depth-tested) |
+ * | -2 | the face itself: `FaceWarp3D`, `FaceMorph`, characters, face swap |
+ * | -1.99 | face pre-pass: depth + stencil bit 0x80, colour off |
+ * | -1.9 … -1.64 | makeup regions, then lips |
+ * | -1.55 | restore: depth back to the far plane where the pre-pass wrote, bit 0x80 cleared |
+ * | -1.5 | `FaceInset` |
+ * | -1 | head occluder |
+ * | 0+ | accessories |
+ */
+/**
+ * Draw order of `FaceRetouch`: in the opaque face layer, after the camera
+ * background (-1000) and `HandCutout` (-900, so hands over the face stay
+ * sharp), before everything that is the face (`FACE_SURFACE_RENDER_ORDER`,
+ * -2) and the makeup.
+ */
+export declare const FACE_RETOUCH_RENDER_ORDER = -2.5;
+/**
+ * Draw orders of the makeup layers, inside the face layer: after the face
+ * effects (`FACE_SURFACE_RENDER_ORDER`, -2), before `FaceInset` (-1.5), the
+ * head occluder (-1) and accessories, so hands (`HandCutout`) and glasses
+ * draw over makeup.
+ *
+ * - `prepass`: the face's depth and the face stencil bit (self-occlusion),
+ *   colour off; high and medium quality tiers only.
+ * - `contour` … `lips`: the region layers, back to front.
+ * - `restore`: puts the depth back where the pre-pass wrote it (far plane)
+ *   and clears the stencil bit, so later layers see the depth as before.
+ *
+ * Put a custom layer between two regions with a small offset:
+ * `MAKEUP_RENDER_ORDER.blush + 0.01`.
+ */
+export declare const MAKEUP_RENDER_ORDER: Readonly<{
+    prepass: -1.99;
+    contour: -1.9;
+    blush: -1.86;
+    highlighter: -1.83;
+    eyeshadow: -1.8;
+    eyeliner: -1.76;
+    brows: -1.72;
+    design: -1.68;
+    lips: -1.64;
+    restore: -1.55;
+}>;
+/**
+ * Whether a face-atlas layer gains from the self-occlusion passes: it is
+ * shown and draws between `MAKEUP_RENDER_ORDER.prepass` and `.restore`. A
+ * layer before the pre-pass (a face swap at -2) or after the restore never
+ * sees the face's depth, so the passes would only cost two draws.
+ */
+export declare const faceAtlasLayerSelfOccludes: (renderOrder: number, visible?: boolean) => boolean;
+/**
+ * Blending for makeup and other face-atlas layers: premultiplied "over"
+ * (`rgb·1 + dst·(1 − a)`), applied in the opaque pass (`transparent: false`)
+ * so the render order above holds against the head occluder. The canvas
+ * alpha stays opaque. Spread it into a custom material:
+ * `<shaderMaterial {...MAKEUP_BLENDING} depthWrite={false} … />`, and write
+ * premultiplied colour from the fragment shader.
+ */
+export declare const MAKEUP_BLENDING: Readonly<{
+    readonly blending: 5;
+    readonly blendEquation: 100;
+    readonly blendSrc: 201;
+    readonly blendDst: 205;
+    readonly blendSrcAlpha: 201;
+    readonly blendDstAlpha: 205;
+    readonly transparent: false;
+}>;
+/**
+ * Stencil bits the face layers reserve. `mouth` (0x01): the mouth interior,
+ * written and tested by `LipEffect`'s mask modes. `surface` (0x80): the face
+ * pre-pass marks the face's visible pixels and the restore clears them.
+ * Custom stencil effects should use other bits (`stencilWriteMask`).
+ */
+export declare const FACE_ATLAS_STENCIL: Readonly<{
+    mouth: 1;
+    surface: 128;
+}>;

package/dist/face-atlas/selfOcclusion.d.ts

@@ -0,0 +1,62 @@
+import * as THREE from "three";
+/** Material state of the face pre-pass (tests check it; `createFaceAtlasPrepassMaterial` uses it). */
+export declare const FACE_ATLAS_PREPASS_STATE: Readonly<{
+    colorWrite: false;
+    depthTest: true;
+    depthWrite: true;
+    depthFunc: 3;
+    polygonOffset: true;
+    polygonOffsetFactor: 1;
+    polygonOffsetUnits: 1;
+    stencilWrite: true;
+    stencilRef: 128;
+    stencilFunc: 519;
+    stencilFuncMask: 255;
+    stencilWriteMask: 128;
+    stencilFail: 7680;
+    stencilZFail: 7680;
+    stencilZPass: 7681;
+    side: 0;
+    toneMapped: false;
+}>;
+/** Material state of the face restore. */
+export declare const FACE_ATLAS_RESTORE_STATE: Readonly<{
+    colorWrite: false;
+    depthTest: true;
+    depthWrite: true;
+    depthFunc: 1;
+    polygonOffset: false;
+    stencilWrite: true;
+    stencilRef: 128;
+    stencilFunc: 514;
+    stencilFuncMask: 128;
+    stencilWriteMask: 128;
+    stencilFail: 7680;
+    stencilZFail: 7680;
+    stencilZPass: 0;
+    side: 0;
+    toneMapped: false;
+}>;
+/** The face pre-pass material: depth and stencil bit 0x80, no colour. */
+export declare const createFaceAtlasPrepassMaterial: () => THREE.ShaderMaterial;
+/** The face restore material: far-plane depth where bit 0x80 is set, which it clears. */
+export declare const createFaceAtlasRestoreMaterial: () => THREE.ShaderMaterial;
+/** The shader sources, for the static tests. */
+export declare const FACE_ATLAS_SELF_OCCLUSION_GLSL: Readonly<{
+    prepassVertex: "\nvoid main() {\n  gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);\n}\n";
+    restoreVertex: "\nvoid main() {\n  vec4 clip = projectionMatrix * modelViewMatrix * vec4(position, 1.0);\n  // On the far plane: depth 1, the cleared value, with no gl_FragDepth.\n  clip.z = clip.w;\n  gl_Position = clip;\n}\n";
+    fragment: "\nvoid main() {\n  gl_FragColor = vec4(0.0);\n}\n";
+}>;
+/**
+ * Whether the renderer's current target has a stencil buffer: the canvas
+ * (its context attributes, read once) or the bound render target (a stencil
+ * renderbuffer, or a `DepthStencilFormat` depth texture).
+ */
+export declare const faceAtlasTargetHasStencil: (renderer: THREE.WebGLRenderer) => boolean;
+/**
+ * Makes a pre-pass or restore mesh draw nothing on a target without a
+ * stencil buffer (its depth function becomes `Never` for that draw): without
+ * the stencil the restore could not tell the face from a hand and would wipe
+ * the hand cut-out's depth. Installs an `onBeforeRender` on `mesh`.
+ */
+export declare const guardFaceAtlasStencilDraw: (mesh: THREE.Mesh, depthFunc: THREE.DepthModes) => void;