@vincentt-xr/sdk 2.0.0-alpha.8 → 2.0.0-alpha.9

package/dist/face-character/faceSwapPresence.d.ts

@@ -0,0 +1,85 @@
+/**
+ * When a face swap shows (FACE-2). A two-person swap starts by itself once
+ * two faces have been tracked for a few inference results in a row, fades
+ * in, holds through a short dropout, then fades back to the plain camera.
+ * Presence reads only how many faces are tracked, never which index is
+ * which, so MediaPipe reordering the faces does not toggle it (the draw
+ * rank below is per index: see `rankFacesBySize`). Pure and
+ * allocation-free: the state is mutated in place, once per rendered frame.
+ */
+export declare const FACE_SWAP_PRESENCE: {
+    /** Faces needed: 2 for a two-person swap. */
+    readonly required: 2;
+    /** Consecutive inference results with enough faces before it starts. */
+    readonly frames: 5;
+    /** Fade in and out (ms). */
+    readonly fadeMs: 200;
+    /** How long it holds from the first result with too few faces (ms), before it fades out. */
+    readonly holdMs: 250;
+    /**
+     * With no new result at all for this long after the last one with enough
+     * faces (ms), it fades out: the model stopped. Slow inference (a weak
+     * device running two faces) stays on.
+     */
+    readonly staleMs: 1000;
+};
+export type FaceSwapPresenceOptions = Partial<Record<keyof typeof FACE_SWAP_PRESENCE, number>>;
+export type FaceSwapPresence = {
+    /** Consecutive inference results with enough faces. */
+    streak: number;
+    /** Time (ms) of the last inference result with enough faces; null before any. */
+    lastSeenMs: number | null;
+    /** Time (ms) of the first result with too few faces since the last good one; null while none. */
+    lostSinceMs: number | null;
+    /** Wanted: on once the streak is long enough, off after the hold. */
+    on: boolean;
+    /** The fade, 0..1: the alpha the swap draws with (multiply into `amount`). */
+    fade: number;
+    /** Time (ms) of the last step; null before the first. */
+    lastStepMs: number | null;
+};
+export declare const createFaceSwapPresence: () => FaceSwapPresence;
+export type FaceSwapPresenceInput = {
+    /** Now (ms), e.g. `performance.now()`. */
+    nowMs: number;
+    /** A new inference result arrived since the last step. Camera-only frames pass false. */
+    inference: boolean;
+    /** Faces tracked in the newest result (read only when `inference` is true). */
+    faces: number;
+    /** Everything it draws with exists (the source is loaded, the first statistics are done). Default true. */
+    ready?: boolean;
+};
+/**
+ * Advance one rendered frame. The streak counts inference results, not
+ * frames. A result with too few faces breaks the streak but not the swap: it
+ * keeps showing for `holdMs` from that result, so a 200 ms dropout is
+ * invisible, then fades out. Results that are merely slow (further apart
+ * than `holdMs`) do not count as a dropout; only `staleMs` without any new
+ * result does. It never shows while `ready` is false. Returns `state`.
+ */
+export declare const stepFaceSwapPresence: (state: FaceSwapPresence, input: FaceSwapPresenceInput, options?: FaceSwapPresenceOptions) => FaceSwapPresence;
+/** Drawn now: fading in, shown or fading out. */
+export declare const isFaceSwapPresenceActive: (state: FaceSwapPresence) => boolean;
+/** Drop everything (the source changed, the component re-mounted): hidden, streak 0. */
+export declare const resetFaceSwapPresence: (state: FaceSwapPresence) => FaceSwapPresence;
+/**
+ * A face keeps its place in the draw order until another face is this much
+ * larger, so two faces of about the same size don't flicker.
+ */
+export declare const FACE_SWAP_RANK_MARGIN = 0.05;
+/**
+ * Draw rank per face: 0 draws first, `n − 1` last (on top). Larger faces
+ * draw later, so where two faces overlap the larger (nearer) one is on top.
+ * `sizes` is any size measure per face (face height, box area); a missing or
+ * non-finite size counts as 0. With `previous` ranks, each rank step is worth
+ * `margin`, so the order only changes once a face is clearly larger. Ties go
+ * to the lower index first.
+ *
+ * Ranks are per face INDEX. When MediaPipe reorders two faces (the registry
+ * reports reason `"jump"` on both), swap their entries in `previous` before
+ * ranking, so the margin stays with the person and not the index.
+ *
+ * Pass `out` every frame (it may be `previous`, updated in place): without
+ * it a new array is allocated per call.
+ */
+export declare const rankFacesBySize: (sizes: ArrayLike<number>, previous?: ArrayLike<number> | null, out?: number[], margin?: number) => number[];

package/dist/face-character/faceTexture.d.ts

@@ -0,0 +1,169 @@
+import { FacePhotoSource } from './faceImageLandmarker';
+import { FacePhotoPick } from './facePhoto';
+import { FaceImageLandmark } from './faceReprojection';
+import type * as THREE from "three";
+/**
+ * Where a face texture came from:
+ * - `live`: a tracked face, unwrapped from the camera on every new landmark
+ *   result (`useLiveFaceTexture`);
+ * - `photo`: a face found in a still photo, baked once (`createFaceTexture`);
+ * - `canonical`: a face painted in the face atlas, baked once.
+ */
+export type FaceTextureKind = "live" | "photo" | "canonical";
+/**
+ * One face laid flat in the face atlas, ready to be drawn on any face
+ * (`FaceTransfer`, `<FaceTextureLayer>`). The textures keep their identity
+ * for the object's life (bind them once); a live face texture is updated in
+ * place and its `version` increases.
+ */
+export type FaceTexture = {
+    readonly kind: FaceTextureKind;
+    /**
+     * The face in the face atlas (`FACE_ATLAS_UVS`, v up): premultiplied by
+     * coverage (alpha 0 where the face was not seen), sRGB, mipmapped, the eye
+     * and mouth openings filled. `null`: statistics only.
+     */
+    readonly map: THREE.Texture | null;
+    /**
+     * Skin-only low-frequency colour (`FaceTextureStats.low`): 64², linear,
+     * premultiplied by how much skin was seen, features filled from the skin
+     * around them.
+     */
+    readonly low: THREE.Texture;
+    /** Mean skin colour (`FaceTextureStats.mean`): 1×1, linear, premultiplied. */
+    readonly mean: THREE.Texture;
+    /**
+     * true when canonical +X (landmark 263, atlas u > 0.5) is the person's own
+     * RIGHT: a live face on the mirrored webcam, or a selfie saved mirrored.
+     * false for a photo as stored, a painted face and a live face on an
+     * unmirrored feed. See `resolveFaceSourceMirror`.
+     */
+    readonly mirrored: boolean;
+    /** Photo: the detected points (478, normalized image space, y down). Otherwise null. */
+    readonly landmarks: ReadonlyArray<FaceImageLandmark> | null;
+    /** Live: the tracked face's index. Otherwise null. */
+    readonly faceIndex: number | null;
+    /** 0 until the texture holds a face; increases with every update. */
+    readonly version: number;
+    /**
+     * The map (when there is one) and the statistics hold this face. False
+     * before the first update and after `dispose`.
+     */
+    readonly ready: boolean;
+};
+/** A face's skin statistics (the target side of a transfer). */
+export type FaceTextureStatistics = Pick<FaceTexture, "low" | "mean">;
+/**
+ * A face to bake into a `FaceTexture` (`createFaceTexture`, `useFaceTexture`):
+ * - `photo`: any photo (URL, File or Blob, `<img>`, canvas, ImageBitmap).
+ *   The face is found once (`detectFacePhoto`), unwrapped into the face
+ *   atlas, the hidden half of a turned face filled from the other side.
+ *   `mirrored`: the photo itself is mirrored (some phones save selfies so);
+ *   `pick`: `"largest"` (default) or the n-th face from the left; `maxSize`:
+ *   the photo is analysed at most this large (default 1280).
+ * - `canonicalTexture`: a face already painted in the face atlas
+ *   (`FACE_ATLAS_UVS`): upright, image left = the person's own right,
+ *   straight alpha. A URL, or a texture as `TextureLoader` gives it (baked
+ *   once its image has loaded; untagged, `NoColorSpace`, it is read as
+ *   sRGB). `mirrored`: painted as seen in a mirror (default false).
+ */
+export type FaceTextureSource = {
+    kind: "photo";
+    src: FacePhotoSource;
+    mirrored?: boolean;
+    pick?: FacePhotoPick;
+    maxSize?: number;
+} | {
+    kind: "canonicalTexture";
+    src: string | THREE.Texture;
+    mirrored?: boolean;
+};
+/**
+ * Face textures (FACE-2): one face laid flat in the face atlas
+ * (`FACE_ATLAS_UVS`, the canonical face OBJ's `vt` layout that LipEffect and
+ * makeup designs use; not `CANONICAL_FACE_UVS`), whether it was unwrapped
+ * from the live camera, from a photo, or painted.
+ *
+ * This file is the side half: which side of a face texture is which, and how
+ * a face is mirrored. Sides are never mirrored by `u → 1 − u` (the atlas is
+ * symmetric only to about 0.001 in u) and no canvas is flipped: a face is
+ * mirrored through the exact landmark mirror map, and a source's sides are
+ * matched to a target's through `resolveFaceSourceMirror`.
+ *
+ * A face texture's `mirrored` flag says which canonical side is the person's
+ * own left:
+ * - `false`: canonical +X (landmark 263, atlas u > 0.5) is the person's own
+ *   LEFT. A photo as stored, a painted face (the atlas guide's "person's
+ *   LEFT" half), and a live face on an unmirrored feed (rear camera, video).
+ * - `true`: canonical +X is the person's own RIGHT. A live face on the
+ *   mirrored front webcam, or a selfie the phone saved mirrored.
+ *
+ * It is the same convention as a feed's mirroring: the person's own left lies
+ * along canonical X × `personLeftSign(mirrored)` (lib/coordinates/side.ts).
+ */
+/**
+ * The unwrap vertices of the canonical face (`getFaceUnwrapMesh()`: the 468
+ * surface landmarks, then the centres of the +X eye, the −X eye and the
+ * mouth openings).
+ */
+export declare const FACE_UNWRAP_VERTEX_COUNT: number;
+/**
+ * `FACE_UNWRAP_MIRROR[i]`: the unwrap vertex at `(−x, y, z)` of unwrap vertex
+ * `i` (471 entries). Exact: the canonical face is symmetric to 0 cm, so a
+ * mirrored point keeps its mirror twin's barycentric weights. Its own
+ * inverse; 263 ↔ 33, 61 ↔ 291; the 28 midline landmarks and the mouth centre
+ * map to themselves; the two eye centres swap.
+ */
+export declare const FACE_UNWRAP_MIRROR: Int32Array;
+/** The tracker's landmark count with irises: 468 surface points, then 468–472 and 473–477. */
+export declare const FACE_LANDMARK_COUNT_WITH_IRISES = 478;
+/**
+ * `FACE_LANDMARK_MIRROR[i]`: the tracker landmark at `(−x, y, z)` of landmark
+ * `i` on the canonical face (478 entries: the 468 surface landmarks through
+ * the atlas's exact mirror map, then the irises by role). Its own inverse.
+ * Index space of the tracker's landmarks, not of the unwrap vertices
+ * (`FACE_UNWRAP_MIRROR`), whose 468–470 are opening centres.
+ */
+export declare const FACE_LANDMARK_MIRROR: Int32Array;
+/**
+ * The face's landmarks with every point moved to its mirror twin's place:
+ * `out[i]` is a copy of `landmarks[FACE_LANDMARK_MIRROR[i]]`. Positions stay
+ * in image space; only which landmark sits where changes, so an unwrap of the
+ * result reads each point of the face from its twin on the other side (the
+ * mirror fill of a turned face, `leftLeft` / `rightRight` sources). Applying
+ * it twice gives the landmarks back.
+ *
+ * `landmarks` has 468 points or more (478 with irises); with fewer than 478
+ * only the surface points are mirrored, and points past the mirrored range
+ * are copied as they are. `out` is reused when given, so a per-frame caller
+ * allocates nothing once it is filled; it must not be `landmarks`, nor share
+ * point objects with it.
+ */
+export declare const mirrorFaceLandmarks: (landmarks: ArrayLike<FaceImageLandmark>, out?: FaceImageLandmark[]) => FaceImageLandmark[];
+/**
+ * Whether a source face texture must be read mirrored (at `uvMirror`) so
+ * that its person's own left lands on the target person's own left. True
+ * when the two place the person's own left on different canonical sides:
+ * `personLeftSign(sourceMirrored) !== personLeftSign(targetFeedMirrored)`.
+ *
+ * | Source → target | Mirrored read |
+ * |---|---|
+ * | photo or painted (`false`) → mirrored webcam (`true`) | yes |
+ * | photo or painted → rear camera, video or picture (`false`) | no |
+ * | live face → its own feed | no |
+ * | live webcam face (`true`) → a picture (`false`, FACE-6) | yes |
+ *
+ * `sourceMirrored` is the face texture's `mirrored` flag; `targetFeedMirrored`
+ * is the mirroring of the feed (or picture) the face is drawn on
+ * (`useXRMirrored()`).
+ */
+export declare const resolveFaceSourceMirror: (sourceMirrored: boolean, targetFeedMirrored: boolean) => boolean;
+/**
+ * The face atlas UV (v up) of every unwrap vertex (471 × 2): `FACE_ATLAS_UVS`
+ * for the 468 landmarks, and each opening's centre at the average of its
+ * ring's UVs. An unwrap drawn with these UVs (FaceReprojector's atlas
+ * layout) and the transfer mesh's `uv` (`getFaceTransferGeometry`) are the
+ * same piecewise-linear map, so a live face drawn back with `asIs` lands
+ * where it was. Shared: do not mutate.
+ */
+export declare const getFaceUnwrapAtlasUvs: () => Float32Array;

package/dist/face-character/faceTextureBaker.d.ts

@@ -0,0 +1,86 @@
+import { XRAssetCache } from '../assets/cache';
+import { FacePhotoDetection } from './faceImageLandmarker';
+import { FacePhotoWarning } from './facePhoto';
+import { FaceTexture, FaceTextureSource } from './faceTexture';
+import * as THREE from "three";
+/**
+ * Still face textures (FACE-2): a photo or a painted face baked once into
+ * the face atlas (`FACE_ATLAS_UVS`), with its skin statistics, ready for
+ * `FaceTransfer` / `<FaceTextureLayer>`.
+ *
+ * - **Photo:** decoded upright into one canvas (at most 1280 px), the face
+ *   found by the still-image detector (`detectFacePhoto`: the same face
+ *   model, IMAGE mode), unwrapped by `FaceReprojector` with the hidden half
+ *   of a turned face filled from the other side (`mirrorFill`) and padded,
+ *   copied into the texture's own mipmapped target; the statistics come from
+ *   the skin pass over the photo (only skin that was seen counts). The
+ *   canvas and its texture are freed after the bake.
+ * - **Painted** (`canonicalTexture`): a face already painted in the face
+ *   atlas, straight alpha. Premultiplied and mipmapped into the texture's
+ *   own target; the statistics read that map through the skin mask. A URL
+ *   loads through the SDK asset cache (a failed download rejects with
+ *   `cors`, bytes that are not an image with `decode`). A texture is baked
+ *   once its image has loaded (`TextureLoader.load` fills it in later); an
+ *   untagged one (`NoColorSpace`) is read as sRGB.
+ */
+/** Sizes a still face texture can be baked at (px, square): a 128 × 2^k size the statistics read whole levels of. */
+export declare const FACE_TEXTURE_RESOLUTIONS: readonly [512, 1024];
+export type FaceTextureResolution = (typeof FACE_TEXTURE_RESOLUTIONS)[number];
+/** How a photo is unwrapped. */
+export declare const FACE_PHOTO_BAKE: {
+    /** A photo's turned-away skin is dropped below these facing cosines, then filled from the other side. */
+    readonly facing: readonly [0.2, 0.5];
+    /** Edge padding of the unwrap (px), so the mipmaps of the face's rim do not darken. */
+    readonly unwrapPadding: 8;
+};
+/** A baked face texture: its warnings and face count, and `dispose()` (yours to call). */
+export type BakedFaceTexture = FaceTexture & {
+    /** Photo warnings (`FACE_PHOTO_WARNINGS` order); empty for a painted face. */
+    readonly warnings: readonly FacePhotoWarning[];
+    /** Faces found in the photo (1 for a painted face). */
+    readonly faceCount: number;
+    /** Frees the map and the statistics; `ready` turns false. Safe to call twice. */
+    dispose(): void;
+};
+export type CreateFaceTextureOptions = {
+    /** The map's size (px, square). Default 1024. */
+    resolution?: FaceTextureResolution;
+    /** Cancels the load; the promise then rejects with code `aborted`. */
+    signal?: AbortSignal;
+    /** The asset cache a painted face's URL loads through. Default the SDK's shared cache. */
+    assetCache?: XRAssetCache;
+};
+/** What the photo bake needs of a detection (`FacePhotoDetection`). */
+export type FacePhotoBakeInput = Pick<FacePhotoDetection, "canvas" | "width" | "height" | "detection" | "detections" | "warnings" | "mirrored">;
+/**
+ * The photo bake after detection: unwraps `photo.canvas` at the detected
+ * landmarks into a new face texture. Frees the canvas (its size becomes 0).
+ * Internal (the GPU self-test drives it with known landmarks).
+ */
+export declare const bakeFacePhoto: (renderer: THREE.WebGLRenderer, photo: FacePhotoBakeInput, resolution?: FaceTextureResolution) => BakedFaceTexture;
+/** How long a painted face's texture may wait for its image (ms), and how often it looks. */
+export declare const FACE_TEXTURE_IMAGE_WAIT: {
+    readonly timeoutMs: 30000;
+    readonly pollMs: 50;
+};
+/**
+ * Bakes a photo or a painted face into a `FaceTexture` (see the module
+ * comment). Needs the renderer; no React. The result is yours to
+ * `dispose()`. Rejects with a `FaceTextureError`-shaped error: `noFace`,
+ * `decode`, `cors`, `model` or `aborted`.
+ *
+ * ```ts
+ * const face = await createFaceTexture(renderer, { kind: "photo", src: file });
+ * transfer.setSource(face);
+ * // later
+ * face.dispose();
+ * ```
+ */
+export declare const createFaceTexture: (renderer: THREE.WebGLRenderer, source: FaceTextureSource, options?: CreateFaceTextureOptions) => Promise<BakedFaceTexture>;
+/**
+ * A stable key for a source, so an inline `{ kind: "photo", src: file }`
+ * re-made on every render bakes once: the kind, the URL (or the identity of
+ * the File, image, canvas, bitmap or texture), the options and the
+ * resolution. `null` for no source.
+ */
+export declare const faceTextureSourceKey: (source: FaceTextureSource | null | undefined, resolution?: number) => string | null;

package/dist/face-character/faceTextureGuide.d.ts

@@ -0,0 +1,106 @@
+import { FaceSwapPresetKey } from '../components/FaceSwap/settings';
+export type FaceTextureGuideOptions = {
+    /** Pixel size of the square canvas the guide represents. Default 1024. */
+    size?: number;
+    /** Fill the canvas with a colour. Omit for a transparent background. */
+    background?: string;
+    /** Draw the triangle wireframe. Default true. */
+    wireframe?: boolean;
+    /** Draw the title, legend and side labels. Default true. */
+    labels?: boolean;
+    /** UV grid spacing (0.1 = every tenth of the atlas). 0 draws no grid. Default 0.1. */
+    grid?: number;
+    /**
+     * The FaceSwap preset whose edge fade the guide shows, named in the legend.
+     * Default `"keepColors"`, the preset painted faces start from.
+     */
+    preset?: FaceSwapPresetKey;
+    /** cm: where the edge fade starts inside the outline. Default FaceSwap's `edgeInset` (0.3). */
+    edgeInset?: number;
+    /** cm: width of the edge fade. Default the preset's `edgeSoftness` (keepColors: 1). */
+    edgeSoftness?: number;
+    /**
+     * cm: the rim outside the eye and inner-lip rings over which the paint fades
+     * into the wearer's own eyes and mouth (`ownEyes` / `ownMouth`, on by
+     * default). Default FaceSwap's `featureSoftness` (0.25).
+     */
+    featureSoftness?: number;
+};
+/** Line colours of the face-texture guide (the atlas guide's colours for the rest). */
+export declare const FACE_TEXTURE_GUIDE_COLORS: {
+    readonly outline: "#0f172a";
+    readonly fade: "#dc2626";
+    readonly band: "#ea580c";
+    readonly own: "#2563eb";
+    readonly midline: "#475569";
+    readonly grid: "#e2e8f0";
+    readonly gridText: "#94a3b8";
+};
+/** The distances FaceSwap fades by, in cm. */
+export type FaceTextureFadeField = 
+/** Inside the smoothed face outline (negative outside it): the edge fade and colour band. */
+"edge"
+/** Outside the nearer smoothed eye ring (negative inside the opening): the rim around kept eyes. */
+ | "eye"
+/** Outside the smoothed inner-lip ring (negative inside the opening): the rim around a kept mouth. */
+ | "mouth";
+/**
+ * Where one of FaceSwap's fade distances (`FaceTextureFadeField`) equals
+ * `levelCm`, as polylines in atlas UV: `[u0, v0, u1, v1, …]`, a closed line
+ * ending on its first point. The distances are measured like FaceSwap's
+ * transfer geometry (straight-line 3D distance to the smoothed outline, eye
+ * rings and inner-lip ring) on the same level-2 subdivision, so the lines lie
+ * where the shader's fades are.
+ */
+export declare const faceTextureFadeContour: (field: FaceTextureFadeField, levelCm: number) => number[][];
+/**
+ * The smoothed face outline FaceSwap measures its edge fade from, in atlas
+ * UV: `[u0, v0, u1, v1, …]`, closed (the last sample connects to the first).
+ */
+export declare const faceTextureOutline: () => Float64Array;
+/**
+ * The creator template for a painted face (face swap's `canonicalTexture`
+ * source) as an SVG string, in the face atlas layout. On top of the atlas
+ * guide (wireframe, skin statistics area, lips, eye and mouth fills) it draws
+ * FaceSwap's fades for the `preset` (default `keepColors`, the preset painted
+ * faces start from): the smoothed face outline, where the edge fade starts
+ * (`edgeInset`) and where the paint is fully opaque
+ * (`edgeInset + edgeSoftness`), the colour band, the eye and inner-lip rings
+ * (the wearer's own eyes and mouth show inside them by default) and, at
+ * `featureSoftness` outside them, where the paint is opaque again. It adds
+ * the midline, a UV grid and the person-side labels. Every fade line is
+ * measured as FaceSwap measures it (`faceTextureFadeContour`).
+ */
+export declare const createFaceTextureGuideSvg: (options?: FaceTextureGuideOptions) => string;
+/** The face-texture guide as an `image/svg+xml` data URL (an `<img>` or `THREE.TextureLoader`). */
+export declare const createFaceTextureGuideDataUrl: (options?: FaceTextureGuideOptions) => string;
+/**
+ * Rasterize the face-texture guide to a PNG `Blob` in the browser, for a
+ * download button or as a paint layer. Transparent unless `background` is set.
+ */
+export declare const createFaceTextureGuidePngBlob: (options?: FaceTextureGuideOptions) => Promise<Blob>;
+export type FaceTextureSideTestOptions = {
+    /** Pixel size of the square texture. Default 1024. */
+    size?: number;
+};
+/** Colours of the side-test face. */
+export declare const FACE_TEXTURE_SIDE_TEST_COLORS: {
+    readonly skin: "#dcae92";
+    readonly brows: "#5b4034";
+    readonly lips: "#b4535e";
+    readonly sclera: "#f4f0ea";
+    readonly iris: "#3f2c22";
+    readonly mouth: "#3a1c1f";
+    readonly left: "#1d4ed8";
+    readonly right: "#c81e1e";
+};
+/**
+ * A face texture for checking sides on every feed, in the face atlas layout:
+ * flat skin, brows, lips, eyes and mouth, a large "L" on the person's own
+ * left cheek (below landmark 263, image right in the atlas) and "R" on their
+ * right cheek (below 33), and an arrow along the left brow pointing to the
+ * person's left. Worn as a face swap source, the letters read like a sticker
+ * on that cheek: correctly on the rear camera, images and videos, mirrored in
+ * the mirrored webcam view.
+ */
+export declare const createFaceTextureSideTestSvg: (options?: FaceTextureSideTestOptions) => string;