@vincentt-xr/sdk 2.0.0-alpha.7 → 2.0.0-alpha.9

package/dist/hooks/useCalibrationFaceTracker.d.ts

@@ -18,6 +18,6 @@ export declare const useCalibrationFaceTracker: ({ enabled, displayMirrored, ima
 }) => {
     activeDelegate: "GPU";
     faces: import('../types').FaceTrackingResult[];
-    status: "idle" | "ready" | "error" | "initializing";
+    status: "ready" | "error" | "idle" | "initializing";
     error: string | null;
 };

package/dist/hooks/useFaceInfo.d.ts

@@ -1,7 +1,19 @@
 import { FaceInfo } from '../face-info';
+import { XRSubjectPolicy } from '../subjects/types';
 import { UseFaceInfosOptions } from './useFaceInfos';
 export type UseFaceInfoOptions = UseFaceInfosOptions & {
+    /** Result-array position to read (not a persistent identity). Default 0. Ignored when `subject` is set. */
     faceIndex?: number;
+    /**
+     * Multi-subject policy: pick the face by `largest`, `nearest-center`,
+     * `leftmost`, `rightmost`, or `sticky` (follows the previously selected
+     * face by proximity). With a policy and no `numFaces`, the model is asked
+     * for `DEFAULT_POLICY_NUM_FACES` (4) faces so the policy has candidates;
+     * pass `numFaces` to choose the count.
+     */
+    subject?: XRSubjectPolicy;
 };
+/** Faces requested when a `subject` policy is set without `numFaces`. */
+export declare const DEFAULT_POLICY_NUM_FACES = 4;
 /** Single-face selector over `useFaceInfos`; it does not create another reader loop. */
-export declare const useFaceInfo: ({ faceIndex, numFaces, ...options }?: UseFaceInfoOptions) => FaceInfo | null;
+export declare const useFaceInfo: ({ faceIndex, numFaces, subject, ...options }?: UseFaceInfoOptions) => FaceInfo | null;

package/dist/hooks/useFaceInfos.d.ts

@@ -2,7 +2,7 @@ import { FaceInfo } from '../face-info';
 export type UseFaceInfosOptions = {
     /** Pauses this consumer's model interest when false. Default true. */
     active?: boolean;
-    /** Maximum faces requested from the shared model. Default 1. */
+    /** Maximum faces requested from the shared model. Default 1, capped at 10 (`XR_MAX_SUBJECTS.face.max`). */
     numFaces?: number;
     /** Optional inference-rate cap. */
     targetFps?: number;

package/dist/hooks/useFaceModelAcquisition.d.ts

@@ -0,0 +1,38 @@
+import { XRModelNodeFaceTracking } from '../types/nodes';
+/**
+ * The face-model start rule shared by every component that starts the face
+ * model itself when no `FaceTracker` (or face rig) is above it: the
+ * face-atlas surfaces (makeup) and the live faces (face swap). Pair with
+ * `resolveNumFaces` (`lib/utils/resolveNumFaces.ts`) for the count.
+ */
+/**
+ * Starts the shared face model (`useXRModel`'s acquire, without its
+ * re-render on readiness) while `enabled`, asking for `numFaces` faces.
+ * The first component to start the model sets its count; later ones share
+ * it. The acquire waits for the end of the commit's effects (a microtask),
+ * so a `FaceTracker` mounted beside the component in the same commit
+ * acquires first and its `numFaces` / `targetFps` win, as they did when face
+ * layers never started the model (alpha.8). Nothing outside an `XRProvider`.
+ */
+export declare const useFaceModelAcquisition: (enabled: boolean, numFaces: number) => void;
+/**
+ * True when the face model behind `node` was started with too few faces to
+ * ever report face `faceIndex` (`XRModelNodeFaceTracking.numFaces`). False
+ * while the node or its count is unknown.
+ */
+export declare const faceModelMissesFace: (node: Pick<XRModelNodeFaceTracking, "numFaces"> | null | undefined, faceIndex: number) => boolean;
+/**
+ * Warns once per face model and face index when the face model was started with
+ * fewer faces than `faceIndex + 1`: the face is then never tracked and the
+ * component draws nothing. `who` names the component in the message.
+ * Returns whether it warned now. Not gated on dev builds: a library build
+ * replaces `import.meta.env.DEV` with false, so a gated warning would never
+ * reach an app.
+ */
+export declare const warnFaceModelMissesFace: (node: Pick<XRModelNodeFaceTracking, "numFaces"> | null | undefined, faceIndex: number, who: string) => boolean;
+/**
+ * `warnFaceModelMissesFace` from React: checks whenever the face model's
+ * node (a new model) or the face index changes, while `enabled`. No per-frame
+ * cost: the node's `numFaces` is fixed once the node is published.
+ */
+export declare const useFaceModelFaceCheck: (node: Pick<XRModelNodeFaceTracking, "numFaces"> | null | undefined, faceIndex: number, who: string, enabled?: boolean) => void;

package/dist/hooks/useGesture.d.ts

@@ -0,0 +1,32 @@
+import { GestureDetectorOptions, GestureState } from '../gestures/detector';
+import { GestureRef } from '../gestures/definitions';
+export type UseGestureOptions = Partial<GestureDetectorOptions> & {
+    /** Pause measuring (the tracker interest is released too). Default true. */
+    active?: boolean;
+    /** Face gestures: result-array index of the face. Default 0. */
+    faceIndex?: number;
+    /** Face gestures: faces to request from the shared model (first acquirer wins). */
+    numFaces?: number;
+    /** Hand gestures: prefer this hand when present. */
+    hand?: "left" | "right";
+    /** Hand gestures: positional fallback. Default 0. */
+    handIndex?: number;
+    /** Body gestures: pose index. Default 0. */
+    bodyIndex?: number;
+    /** Optional inference cap for the underlying model. */
+    targetFps?: number;
+    /** Lifecycle events; called from the frame loop with the state of that frame. */
+    onBegin?: (state: GestureState) => void;
+    onEnd?: (state: GestureState) => void;
+    onChange?: (state: GestureState) => void;
+};
+/**
+ * Detect one gesture from the tracker it needs. Acquires the shared face,
+ * hand or body model (no second detector), measures the definition every
+ * frame, and runs the gesture engine: enter/exit hysteresis, hold,
+ * cooldown, a confidence gate and a loss grace window. Returns React state
+ * that changes only when the gesture's state changes; `onBegin` / `onEnd` /
+ * `onChange` fire from the frame loop. Must run inside `XRProvider` and the
+ * R3F canvas.
+ */
+export declare const useGesture: (gesture: GestureRef, options?: UseGestureOptions) => GestureState;

package/dist/hooks/useIrisTracking.d.ts

@@ -0,0 +1,45 @@
+import { IrisResult } from '../iris/types';
+export type IrisTrackingStatus = "loading" | "ready" | "error";
+export type UseIrisTrackingOptions = {
+    /** Result-array index of the face to read. Default 0. */
+    faceIndex?: number;
+    /** Maximum faces requested from the shared face model (first acquirer wins). Default `faceIndex + 1`. */
+    numFaces?: number;
+    /** Pauses this consumer's model interest when false. Default true. */
+    active?: boolean;
+    /** Optional inference-rate cap for the shared face model. */
+    targetFps?: number;
+    /** Keep the last iris data for this long (ms) after the face drops. Default 250. */
+    holdMs?: number;
+    /** While held, extrapolate along the last velocity for up to this long (ms). Default 0. */
+    predictMs?: number;
+    /**
+     * One Euro smoothing of the iris centers and radius, same scale as
+     * `TrackingAnchor`'s `smoothing` (~15 is a good default; 0 = raw). Resets on
+     * re-acquire so the iris never slides in from a stale position.
+     */
+    smoothing?: number;
+    /**
+     * Inference runs below the render rate (24 fps by default against 60+
+     * rendered), so the iris would hold then jump on each sample. Between
+     * samples the centers are advanced along their last velocity for at most
+     * this long (ms), which keeps fast head motion continuous. Default 80;
+     * 0 disables.
+     */
+    leadMs?: number;
+};
+export type IrisTrackingHandle = {
+    /** Stable object, updated in place every frame. Read it inside `useFrame`. */
+    result: IrisResult;
+    /** True once the shared face model is registered without error. */
+    ready: boolean;
+    status: IrisTrackingStatus;
+    error: Error | null;
+};
+/**
+ * Per-frame iris reader over the shared face tracker: no second detector,
+ * the same `faceIndex` / `numFaces` selection as `useFaceInfo`, the SDK's
+ * one dropout bridge (`FaceTrackMemory`, hold 250 / predict 0) and optional
+ * One Euro smoothing. Must run inside `XRProvider` and the R3F canvas.
+ */
+export declare const useIrisTracking: ({ faceIndex, numFaces, active, targetFps, holdMs, predictMs, smoothing, leadMs, }?: UseIrisTrackingOptions) => IrisTrackingHandle;

package/dist/hooks/useTrackerContext.d.ts

@@ -25,6 +25,12 @@ export type TrackerContextValue = {
      * `useFrame`.
      */
     resolveLandmark: TrackerLandmarkResolver;
+    /**
+     * The tracked subject this tracker binds to (`FaceTracker`'s `faceIndex`).
+     * Fitted children (`CanonicalHeadFrame`, `HeadAttachment`, `HeadOccluder`)
+     * default to it, so a subtree under `faceIndex={1}` follows face 1.
+     */
+    faceIndex?: number;
 };
 export declare const TrackerContext: import('react').Context<TrackerContextValue | null>;
 /**

package/dist/hooks/useXRDiagnostics.d.ts

@@ -0,0 +1,81 @@
+import { XRModel, XRMediaSource } from '../constants';
+import { XRModelDiagnostics } from '../models/Base';
+import { XRSessionSourceKind } from '../types/compositor';
+import { SessionFacade } from './useXRSession';
+import { SessionLifecycleState, XRSessionError } from './useXRSession/store';
+/** One tracker's health, merged from the registry and the model instance. */
+export type XRModelDiagnosticsEntry = XRModelDiagnostics & {
+    model: XRModel;
+    /** Short human label (`face`, `hand`, …). */
+    label: string;
+    /**
+     * `unavailable`: nothing acquired it. `loading`: acquired, initializing.
+     * `ready`: running. `paused`: initialized but no active owner (or the
+     * session is stopped / paused). `error`: initialize or inference failed.
+     */
+    state: "unavailable" | "loading" | "ready" | "paused" | "error";
+    /** Inference-rate cap requested for this model (`null` = uncapped / unknown). */
+    targetFps: number | null;
+    /** Owners holding the model and those that want it running. */
+    owners: {
+        refs: string[];
+        active: string[];
+    };
+};
+export type XRSessionDiagnostics = {
+    lifecycle: SessionLifecycleState;
+    ready: boolean;
+    error: XRSessionError | undefined;
+    source: XRMediaSource | undefined;
+    sourceKind: XRSessionSourceKind | undefined;
+    /** Native media size. */
+    sourceWidth: number;
+    sourceHeight: number;
+    /** Tracker feed / render size. */
+    renderWidth: number;
+    renderHeight: number;
+    /** Frames the session blitted into the tracker feed per second (0 while paused). */
+    feedFps: number;
+    /** Total feed frames since the provider mounted. */
+    feedFrames: number;
+};
+export type XRDiagnostics = {
+    session: XRSessionDiagnostics;
+    /** Adaptive-quality tier (`null` when no controller is mounted). */
+    quality: {
+        tier: string;
+        index: number;
+        locked: boolean;
+        reason: string;
+    } | null;
+    models: XRModelDiagnosticsEntry[];
+    /** Sum of the last inference time of every running model (ms). */
+    inferenceMsTotal: number;
+    /** `performance.now()` when this sample was taken. */
+    sampledAt: number;
+};
+export type UseXRDiagnosticsOptions = {
+    /** Sampling period in ms (default 250). Diagnostics never sample per frame. */
+    intervalMs?: number;
+    /** Models to report; default every `XRModel`, so unacquired ones show as `unavailable`. */
+    models?: XRModel[];
+    /** `false` stops sampling (the last sample stays). Default `true`. */
+    enabled?: boolean;
+};
+export declare const XR_MODEL_LABELS: Record<XRModel, string>;
+/**
+ * Take one diagnostics sample from a session. Pure over the session's current
+ * state; `previous` supplies the frame baseline for the feed fps.
+ */
+export declare const sampleXRDiagnostics: (session: SessionFacade, models: XRModel[], previous: {
+    feedFrames: number;
+    sampledAt: number;
+} | null, now?: number) => XRDiagnostics;
+/**
+ * Opt-in tracking diagnostics: session lifecycle, source and feed size, feed
+ * frame rate, and per-model readiness, inference time and rate, skipped
+ * frames, detections, confidence and errors. Sampled on an interval (never per
+ * frame) only while a consumer is mounted, so it costs nothing in production
+ * until you render it. Requires an `<XRProvider>`.
+ */
+export declare const useXRDiagnostics: (options?: UseXRDiagnosticsOptions) => XRDiagnostics;

package/dist/hooks/useXRModel.d.ts

@@ -1,4 +1,5 @@
 import { XRModel } from '../constants';
+import { XRFallbackReason } from '../capabilities/types';
 /**
  * Serialize model options to a value-stable key. Exported for tests: the
  * acquire effect must key on this, not on the options object's identity.
@@ -8,6 +9,12 @@ export type UseXRModelResult<T extends Record<string, unknown>> = {
     ready: boolean;
     node: T | null;
     error: Error | null;
+    /**
+     * Structured cause when `error` is set (`model-unreachable`,
+     * `model-integrity`, `model-init-failed`, …); `null` while healthy. Camera
+     * failures are session-level: read `useXRError()` / `useXRTrackerStatus()`.
+     */
+    fallback: XRFallbackReason | null;
 };
 export declare function useXRModel<T extends Record<string, unknown>>(modelName: XRModel, options?: Record<string, unknown>, 
 /**

package/dist/hooks/useXRSession/facade.d.ts

@@ -1,6 +1,8 @@
 import { Camera, Scene, WebGLRenderer } from 'three';
 import { XRModel } from '../../constants';
-import { RegisterXRPipelineParams, SetFrameSourceParams, SetMediaSourceParams, XRPerfConfig } from '../../types';
+import { XRCameraFacing, XRCameraRequest } from '../../types/compositor';
+import { XRQualityConfig, XRQualityState } from '../../quality/types';
+import { RegisterXRPipelineParams, SetFrameSourceParams, SetMediaSourceParams, XRLifecycleConfig, XRPerfConfig } from '../../types';
 import { ModelRegistry } from './registry';
 import { SessionStore } from './store';
 export type SessionFacade = {
@@ -11,6 +13,15 @@ export type SessionFacade = {
     start: () => Promise<void>;
     stop: () => Promise<void>;
     reset: () => Promise<void>;
+    pause: () => void;
+    resume: () => Promise<void>;
+    /**
+     * Open another camera (facing or device) on a live session, keeping every
+     * model and the lifecycle: a `setMediaSource({ source: WEBCAM, ...request })`
+     * that also starts a `media-bound` / `stopped` session. Resolves to the
+     * facing the session ended up with.
+     */
+    switchCamera: (request?: XRCameraRequest) => Promise<XRCameraFacing>;
     /** @deprecated prefer `useXRModel` */
     registerXRPipeline: (params: RegisterXRPipelineParams) => Promise<void>;
     getModelNode: <T extends Record<string, unknown>>(modelCode: XRModel) => T;
@@ -31,7 +42,36 @@ export type SessionFacade = {
     /** Update the session perf config (targetFps / camRes). Applied to models on
      *  their next run; camRes takes effect on the next media (re)bind. */
     _setPerf: (perf: XRPerfConfig | undefined) => void;
+    /** Update the lifecycle config (visibility handling). */
+    _setLifecycle: (lifecycle: XRLifecycleConfig | undefined) => void;
+    /**
+     * Re-open the current webcam with the current perf config (a changed
+     * `camRes`). Keeps lifecycle and models; no-op for stream / canvas sources.
+     */
+    _rebindMedia: () => Promise<void>;
+    /** Publish adaptive-quality state (the controller owns it). */
+    _setQuality: (quality: XRQualityState | null) => void;
+    /** Publish the controller's resolved config so `useXRQuality()` resolves tiers against it. */
+    _setQualityConfig: (config: XRQualityConfig | null) => void;
+    /** Current perf config as applied (`_setPerf` merges tier knobs into it). */
+    _getPerf: () => XRPerfConfig | undefined;
+    /**
+     * Final teardown: `reset()` plus listener removal. The provider calls this
+     * on unmount; afterwards every bind/start is rejected until `rearm()`.
+     * Idempotent.
+     */
+    dispose: () => Promise<void>;
+    /**
+     * Undo `dispose()`'s rejection so the same facade can bind again. The
+     * provider calls it on every mount: React StrictMode runs mount → unmount →
+     * mount against one memoized facade, so a sticky `disposed` would poison it.
+     */
+    rearm: () => void;
+    /** True after `dispose()` (until `rearm()`). */
+    isDisposed: () => boolean;
 };
+/** The front camera is mirrored unless the caller says otherwise; the rear camera is drawn straight. */
+export declare const resolveWebcamMirror: (facingMode: XRCameraFacing, mirrored?: boolean) => boolean;
 export declare const resolveSourceCanvasSize: ({ renderHeight, renderWidth, videoHeight, videoWidth, }: {
     renderHeight: number;
     renderWidth: number;
@@ -41,4 +81,4 @@ export declare const resolveSourceCanvasSize: ({ renderHeight, renderWidth, vide
     height: number;
     width: number;
 };
-export declare const createSessionFacade: (perf?: XRPerfConfig) => SessionFacade;
+export declare const createSessionFacade: (perf?: XRPerfConfig, lifecycle?: XRLifecycleConfig) => SessionFacade;

package/dist/hooks/useXRSession/index.d.ts

@@ -1,4 +1,4 @@
-import { XRPerfConfig } from '../../types';
+import { XRLifecycleConfig, XRPerfConfig } from '../../types';
 import { SessionFacade } from './facade';
 export type { SessionFacade } from './facade';
-export declare const useXRSession: (perf?: XRPerfConfig) => SessionFacade;
+export declare const useXRSession: (perf?: XRPerfConfig, lifecycle?: XRLifecycleConfig) => SessionFacade;

package/dist/hooks/useXRSession/reason.d.ts

@@ -14,6 +14,8 @@
  * Depends on: nothing.
  * Depended on by: `facade.ts`, `store.ts`, `../selectors.ts`.
  */
-export type XRErrorReason = "permission-denied" | "camera-in-use" | "no-camera" | "unsupported" | "unknown";
+export type XRErrorReason = "permission-denied" | "camera-in-use" | "no-camera" | "unsupported"
+/** The camera stopped mid-session: unplugged, revoked, or taken by another app. */
+ | "camera-lost" | "unknown";
 export declare const mapDomExceptionToReason: (err: unknown) => XRErrorReason;
 export declare const MESSAGE_FOR: Record<XRErrorReason, string>;

package/dist/hooks/useXRSession/registry.d.ts

@@ -2,6 +2,13 @@ import { XRModel } from '../../constants';
 import { BaseXRModel } from '../../models/Base';
 import { Class } from '../../types';
 import { SessionStore } from './store';
+/**
+ * Where a model is in its life: `idle` (never acquired, or released),
+ * `initializing` (acquired, `initialize()` in flight), `ready` (initialized
+ * and published), `error` (the last initialize rejected; the next acquire
+ * retries).
+ */
+export type ModelRegistryState = "idle" | "initializing" | "ready" | "error";
 export type ModelRegistry = {
     /**
      * Acquire a model by name on behalf of `ownerId`. Increments the refcount,
@@ -50,6 +57,13 @@ export type ModelRegistry = {
      * Force-release every model. Used by `session.reset()`.
      */
     releaseAll: () => void;
+    /** Lifecycle state of a model (see `ModelRegistryState`). */
+    getState: (modelName: XRModel) => ModelRegistryState;
+    /** Owner ids currently holding the model, and those that want it running. */
+    getOwners: (modelName: XRModel) => {
+        refs: string[];
+        active: string[];
+    };
 };
 export declare const createModelRegistry: (store: SessionStore, modelCodeMap: Record<XRModel, Class<BaseXRModel>>, 
 /**

package/dist/hooks/useXRSession/store.d.ts

@@ -1,9 +1,17 @@
 import { CanvasTexture } from 'three';
 import { XRMediaSource, XRModel } from '../../constants';
-import { XRMediaSourceKind } from '../../types/compositor';
+import { XRCameraFacing, XROrientation, XRSessionSourceKind } from '../../types/compositor';
 import { BaseXRModel } from '../../models/Base';
+import { XRFallbackReason } from '../../capabilities/types';
+import { XRCameraPermissionState } from '../../camera/permission';
+import { XRQualityConfig, XRQualityState } from '../../quality/types';
 import { XRErrorReason } from './reason';
-export type SessionLifecycleState = "idle" | "media-bound" | "running" | "stopped";
+/**
+ * `paused` is a running session whose blit loop and models are suspended by
+ * `session.pause()` (or a hidden tab with `lifecycle.pauseWhenHidden`); the
+ * media stream stays open and `resume()` continues without a rebind.
+ */
+export type SessionLifecycleState = "idle" | "media-bound" | "running" | "paused" | "stopped";
 /**
  * `message` is human prose and may be reworded at any time; `reason` is the
  * stable machine-readable identity. `error` is a legacy constant kept for
@@ -14,6 +22,13 @@ export type XRSessionError = {
     message: string;
     reason: XRErrorReason;
 };
+/** Why a model failed to initialize; cleared when the model is acquired again. */
+export type XRModelError = {
+    model: XRModel;
+    error: Error;
+    /** `performance.now()` when the failure was recorded. */
+    at: number;
+};
 export type SessionState = {
     lifecycle: SessionLifecycleState;
     ready: boolean;
@@ -22,13 +37,47 @@ export type SessionState = {
     renderWidth: number;
     renderHeight: number;
     source: XRMediaSource | undefined;
-    sourceKind: "webcam" | XRMediaSourceKind | undefined;
+    sourceKind: XRSessionSourceKind | undefined;
+    /** Facing of the bound live camera; `null` for stream / canvas sources. */
+    cameraFacing: XRCameraFacing | null;
+    /** Device id of the bound live camera when one was requested or reported by the track. */
+    cameraDeviceId: string | null;
+    /** Current device orientation; updated on `screen.orientation` / resize changes. */
+    orientation: XROrientation;
+    /** Adaptive-quality controller state (`null` until a controller publishes). */
+    quality: XRQualityState | null;
+    /** The resolved config the mounted controller runs with (tiers, budget, bounds); `null` without one. */
+    qualityConfig: XRQualityConfig | null;
+    /** Native pixel size of the bound media frame (0 until bound). */
+    sourceWidth: number;
+    sourceHeight: number;
+    /** Framing zoom the blit loop applies when cover-cropping the source into the feed. */
+    cameraZoom: number;
     /**
      * Active model instances by `XRModel` code. The Map identity is replaced
      * (not mutated in place) on every add/remove so subscribers comparing
      * snapshots see a change.
      */
     pipeline: ReadonlyMap<XRModel, BaseXRModel>;
+    /**
+     * Models whose last `initialize()` rejected. Replaced (not mutated) on every
+     * change so subscribers comparing snapshots see it. A successful re-acquire
+     * removes the entry.
+     */
+    modelErrors: ReadonlyMap<XRModel, XRModelError>;
+    /**
+     * Live camera permission as the session experiences it: `requesting`
+     * while a webcam `getUserMedia` is pending, then `granted`, or the
+     * classified failure. Stays `unknown` for stream / canvas sources.
+     */
+    cameraPermission: XRCameraPermissionState;
+    cameraPermissionReason: XRFallbackReason | null;
+    /**
+     * Monotonic count of frames the session blitted into the tracker feed.
+     * Diagnostics derive the camera frame rate from its delta; it is not a
+     * per-frame React signal (published at most a few times per second).
+     */
+    feedFrames: number;
 };
 export type SessionStore = {
     getSnapshot: () => SessionState;