@vincentt-xr/sdk 2.0.0-alpha.7 → 2.0.0-alpha.9

package/dist/face-character/faceTextureStats.d.ts

@@ -0,0 +1,298 @@
+import { FaceImageLandmark } from './faceReprojection';
+import * as THREE from "three";
+/**
+ * Skin colour statistics of a face (FACE-2 face swap; meant to serve FACE-1
+ * makeup's skin probe too). Everything is in the face atlas
+ * (`FACE_ATLAS_UVS`), linear light, premultiplied by skin coverage:
+ *
+ * - **skin pass** (`FaceSkinPass`, 128²): the unwrap mesh draws the image ×
+ *   the eroded skin mask × how squarely each point faces the camera ×
+ *   in-frame, so only skin that was really seen counts. Every face with
+ *   landmarks (live or a photo) takes this path: `updateFromImage`.
+ * - **full map** (`update` with `mask: true`): a face map that has no
+ *   landmarks (a painted face) is read at `faceStatsInputLod(mapSize)` with
+ *   the mask applied instead. Everything with alpha counts as seen there,
+ *   mirror-filled and padded texels included, so never use it for a live
+ *   face's map.
+ * - **H** (64²): a 13-tap horizontal Gaussian (σ 0.8 cm of face, taps one
+ *   texel apart). Each tap reads the 2×2 input texels it covers at their
+ *   centres and weighs each against the previous local skin (a robust weight
+ *   that rejects speculars, beard, glasses frames, hair), so a highlight's
+ *   rim is not blended into skin before it is judged. The coverage itself is
+ *   never down-weighted, so real shading is kept.
+ * - **V** (64², `low`): the vertical Gaussian, blended into the persistent
+ *   `low` over time (`ConstantAlpha`, k = 1 − e^(−dt/τ), τ 250 ms; k = 1 on a
+ *   reset). `low.rgb / low.a` is the local skin colour, the features filled
+ *   from the skin around them (normalised convolution); `low.a` is how much
+ *   skin was seen there.
+ * - **M** (1×1, `mean`): the coverage-weighted mean of `low` (so it is
+ *   smoothed over time exactly like `low`); its alpha is the share of the
+ *   face's skin that was seen.
+ *
+ * Targets are RGBA16F when the GPU renders to half floats, else SRGB8_ALPHA8
+ * (WebGL2 blends and filters sRGB targets in linear light). They are never
+ * mipmapped, so no float mipmap generation is needed. Nothing is read back.
+ */
+export declare const FACE_TEXTURE_STATS: {
+    /** The skin pass (px, square). */
+    readonly skinSize: 128;
+    /** `low` (px, square). */
+    readonly size: 64;
+    /** σ of the `low` Gaussian, cm of face. */
+    readonly sigmaCm: 0.8;
+    /** Taps per direction (odd), one `low` texel apart: ±6 texels = ±2.75 σ at 64². */
+    readonly taps: 13;
+    /** Robust weight `exp(−((lum − ref) / (spread · ref))²)`, ref = the previous local skin. */
+    readonly robustSpread: 0.35;
+    /**
+     * Share of the robust weight left in a tap window (robust / plain) below
+     * which the plain average is used instead (every tap an outlier: the
+     * reference is stale, e.g. the light changed), blended between the two.
+     */
+    readonly robustKeep: readonly [0.05, 0.15];
+    /** Time constant of the smoothing, ms. */
+    readonly tauMs: 250;
+    /** The mean reads a grid of this many taps per side of `low`. */
+    readonly meanGrid: 16;
+    /** The skin pass's `facing` (cosines, as `FaceUnwrapFrameOptions.facing`). */
+    readonly facing: readonly [0.1, 0.4];
+};
+/** The statistics targets' format: `"float"` RGBA16F, `"srgb8"` SRGB8_ALPHA8. */
+export type FaceStatsFormat = "float" | "srgb8";
+/** `low`'s Gaussian: tap offsets (in `low` texels) and normalised weights. */
+export type FaceStatsKernel = {
+    offsets: Float32Array;
+    weights: Float32Array;
+};
+/**
+ * The Gaussian of a `low` of `size` px: σ = `sigmaCm` of face, `taps` taps
+ * at whole texels (so each tap covers whole input texels at twice the size).
+ */
+export declare const faceStatsKernel: (size?: number) => FaceStatsKernel;
+/**
+ * The share of a new measurement in the smoothed statistics:
+ * `1 − e^(−dt/τ)`, 1 on a reset, 0 for no time step.
+ */
+export declare const faceStatsBlend: (dtMs: number, reset?: boolean, tauMs?: number) => number;
+/**
+ * The mip level to read a full face map of `mapSize` px at, so it matches
+ * the skin pass (twice `low`'s `size`: 128² by default). Whole levels only:
+ * the map must be that size times a power of two (128, 256, 512, 1024,
+ * 2048), so every sample of the H pass lands on one texel's centre. Bake any
+ * other size into such a target first. Throws otherwise.
+ */
+export declare const faceStatsInputLod: (mapSize: number, size?: number) => number;
+/** The robust weight of a sample of luminance `lum` against the reference `ref` (1 without a reference). */
+export declare const faceStatsRobustWeight: (lum: number, ref: number) => number;
+/**
+ * Whether this renderer can render (and blend) into RGBA16F:
+ * `EXT_color_buffer_float` or `EXT_color_buffer_half_float`, and a 1×1
+ * half-float target that is framebuffer-complete. Checked once per context.
+ */
+export declare const canRenderHalfFloat: (renderer: THREE.WebGLRenderer) => boolean;
+/** `"float"` when wanted and renderable, else `"srgb8"`. */
+export declare const resolveFaceStatsFormat: (renderer: THREE.WebGLRenderer, float: boolean) => FaceStatsFormat;
+/** The statistics shaders (GLSL ES 3.00, for `glslVersion: THREE.GLSL3` ShaderMaterials). */
+export declare const FACE_TEXTURE_STATS_GLSL: {
+    readonly quadVertex: "\nout vec2 vUv;\nvoid main() {\n  vUv = uv;\n  gl_Position = vec4(position.xy, 0.0, 1.0);\n}\n";
+    readonly skinVertex: "\nin vec2 cameraUv;\nin float weight;\nout vec2 vCameraUv;\nout float vWeight;\nout vec2 vAtlasUv;\nvoid main() {\n  vCameraUv = cameraUv;\n  vWeight = weight;\n  vAtlasUv = position.xy;\n  gl_Position = vec4(position.xy * 2.0 - 1.0, 0.0, 1.0);\n}\n";
+    readonly skinFragment: string;
+    readonly horizontal: string;
+    readonly vertical: string;
+    readonly mean: string;
+};
+export type FaceSkinPassOptions = {
+    /** Size of the pass (px, square). Default 128. */
+    size?: number;
+    /** RGBA16F when the GPU renders to it (default), else SRGB8_ALPHA8. */
+    float?: boolean;
+    /** How squarely skin must face the camera to count (cosines). Default `FACE_TEXTURE_STATS.facing`. */
+    facing?: readonly [number, number];
+    /**
+     * Which atlas texels count: a single-channel mask in the face atlas (read
+     * `.r`, rows from v = 0). Default the eroded skin mask
+     * (`getFaceSkinMaskTexture`). A lips or brows mask
+     * (`createFaceAtlasMaskTexture`) measures those instead.
+     */
+    mask?: THREE.Texture | null;
+};
+/**
+ * The direct skin pass: the face unwrapped into the face atlas at 128²,
+ * premultiplied `(image × w, w)` with `w` = mask × facing × in-frame, so
+ * texels that were not seen are transparent. It works on any image with its
+ * landmarks: the camera, a photo, a video frame. No React needed.
+ */
+export declare class FaceSkinPass {
+    readonly target: THREE.WebGLRenderTarget;
+    private readonly geometry;
+    private readonly material;
+    private readonly scene;
+    private readonly camera;
+    private readonly state;
+    private readonly uniforms;
+    private options;
+    /** The unwrap frame options (one object, reused): landmarks outside the image are not skin. */
+    private readonly frame;
+    private resolvedFormat;
+    constructor(options?: FaceSkinPassOptions);
+    /** The last pass: atlas layout, linear, premultiplied by the skin weight. */
+    get texture(): THREE.Texture;
+    /** The format in use (`null` before the first `render`). */
+    get format(): FaceStatsFormat | null;
+    setOptions(options: FaceSkinPassOptions): void;
+    /**
+     * Unwraps `image` (sRGB images are read in linear light) at `landmarks`
+     * (≥ 468, normalized image space, y down; as the tracker or
+     * `detectFaceImage` gives them). `aspect` defaults to the image's own.
+     * Leaves the renderer's target, clear colour and `autoClear` as they were.
+     */
+    render(renderer: THREE.WebGLRenderer, image: THREE.Texture, landmarks: ArrayLike<FaceImageLandmark>, aspect?: number): void;
+    dispose(): void;
+}
+export type FaceTextureStatsOptions = {
+    /** `low`'s size (px, square). Default 64. */
+    size?: number;
+    /** RGBA16F when the GPU renders to it (default), else SRGB8_ALPHA8. */
+    float?: boolean;
+    /** Smoothing time constant, ms. Default 250. */
+    tauMs?: number;
+};
+export type FaceTextureStatsUpdate = {
+    /** ms since the previous update (the smoothing's time step). */
+    dtMs: number;
+    /**
+     * Restart the smoothing (k = 1): a face (re)acquired, the camera texture
+     * replaced, a jump. The first update always restarts.
+     */
+    reset?: boolean;
+    /**
+     * The input is a full face map rather than a skin pass: multiply it by the
+     * eroded skin mask. Default `false`. For maps without landmarks only (a
+     * painted face): everything with alpha counts as seen skin, mirror-filled
+     * and padded texels included, and nothing is gated by facing or the image
+     * edge. A face with landmarks (live or a photo) uses `updateFromImage`.
+     */
+    mask?: boolean;
+    /**
+     * The input's mip level, a whole number: `faceStatsInputLod(mapSize)` for a
+     * full map (mipmapped, 128 × 2^k px). The level read must be twice `low`'s
+     * size (128² by default). Default 0.
+     */
+    lod?: number;
+};
+/**
+ * The skin statistics of one face (see the module comment): `low` (64²) and
+ * `mean` (1×1), linear light, premultiplied by skin coverage, in the face
+ * atlas, smoothed over time in place. The textures keep their identity for
+ * the object's life, so bind them once. Run `update` at the statistics rate
+ * (and only on new landmarks); there is no readback. No React needed.
+ */
+export declare class FaceTextureStats {
+    /** Local skin colour: `rgb / a`, filled across the features; `a` = skin seen there. */
+    readonly low: THREE.Texture;
+    /** Mean skin colour: `rgb / a`; `a` = the share of the face's skin seen. */
+    readonly mean: THREE.Texture;
+    private readonly lowTarget;
+    private readonly meanTarget;
+    private readonly horizontalTarget;
+    private skinPass;
+    private readonly scene;
+    private readonly quad;
+    private readonly camera;
+    private readonly state;
+    private readonly horizontalUniforms;
+    private readonly verticalUniforms;
+    private readonly meanUniforms;
+    private readonly horizontal;
+    private readonly vertical;
+    private readonly meanMaterial;
+    private size;
+    private float;
+    private tauMs;
+    private resolvedFormat;
+    /** Statistics exist (bound textures hold a result). */
+    private valid;
+    /** `reset()` was called: the next update restarts the smoothing. */
+    private restartPending;
+    /** `updateFromImage`'s update (one object, reused). */
+    private readonly imageUpdate;
+    constructor(options?: FaceTextureStatsOptions);
+    /** The format in use (`null` before the first update). */
+    get format(): FaceStatsFormat | null;
+    /**
+     * True once statistics have been computed (since construction or a format
+     * change). `reset()` keeps it: the last statistics stay bound and usable
+     * until the next update restarts them.
+     */
+    get ready(): boolean;
+    /** The size the input is read at (px, square): twice `low`'s, the skin pass's size. */
+    get readSize(): number;
+    /** The last skin pass of `updateFromImage` (`null` before one). */
+    get skinTexture(): THREE.Texture | null;
+    /** `float` (re-made on next update if the format changes) and `tauMs`. */
+    setOptions(options: Pick<FaceTextureStatsOptions, "float" | "tauMs">): void;
+    /**
+     * The next update restarts the smoothing (k = 1). `ready` stays as it is:
+     * the last statistics stay bound until then.
+     */
+    reset(): void;
+    /**
+     * Updates the statistics from `input`: a skin pass (`FaceSkinPass`) or,
+     * with `mask: true`, a full face map in the face atlas (premultiplied,
+     * read at `lod`). Leaves the renderer's target, clear colour and
+     * `autoClear` as they were. Throws when `lod` is not a whole level, or
+     * when the input (its size known) is not `readSize` px at that level or is
+     * read below level 0 without mipmaps.
+     */
+    update(renderer: THREE.WebGLRenderer, input: THREE.Texture, update: FaceTextureStatsUpdate): void;
+    /**
+     * A skin pass of `image` at `landmarks` (see `FaceSkinPass.render`), then
+     * `update` from it. The path for every face with landmarks, live or a
+     * photo: only skin that was seen (facing the camera, inside the image)
+     * counts.
+     */
+    updateFromImage(renderer: THREE.WebGLRenderer, image: THREE.Texture, landmarks: ArrayLike<FaceImageLandmark>, update: Omit<FaceTextureStatsUpdate, "mask" | "lod"> & {
+        aspect?: number;
+    }): void;
+    private checkInput;
+    private ensureFormat;
+    private passes;
+    dispose(): void;
+}
+/** A float image, rows from v = 0 (as a render target reads back). */
+export type FaceStatsImage = {
+    width: number;
+    height: number;
+    /** 1 (a mask: `.r`) or 4 (premultiplied RGBA). */
+    channels: 1 | 4;
+    data: Float32Array;
+};
+/** Bilinear, clamped to the edge, as `texture()` with linear filtering samples it. */
+export declare const sampleFaceStatsImage: (image: FaceStatsImage, u: number, v: number, out?: [number, number, number, number]) => [number, number, number, number];
+export type FaceTextureStatsReferenceInput = {
+    /** The skin pass or full map (premultiplied RGBA), read at LOD 0. */
+    input: FaceStatsImage;
+    /** The eroded skin mask (1 channel), e.g. `createFaceSkinMask(128, "bottom")` / 255. */
+    skinMask: FaceStatsImage;
+    /** Multiply the input by the mask (a full map). */
+    mask?: boolean;
+    /** The previous `low` and `mean`, or null for a restart (plain, then robust). */
+    previous: {
+        low: FaceStatsImage;
+        mean: readonly [number, number, number, number];
+    } | null;
+    /** The blend of the new measurement (ignored on a restart). */
+    k?: number;
+    /** `low`'s size. Default 64. */
+    size?: number;
+};
+/**
+ * The statistics passes on the CPU: `{ horizontal, low, mean }` as
+ * `FaceTextureStats.update` computes them (a restart runs the plain pass,
+ * then the robust pass against it).
+ */
+export declare const faceTextureStatsReference: ({ input, skinMask, mask, previous, k, size, }: FaceTextureStatsReferenceInput) => {
+    horizontal: FaceStatsImage;
+    low: FaceStatsImage;
+    mean: [number, number, number, number];
+};