@vincentt-xr/sdk 2.0.0-alpha.0

package/dist/tracking.js

@@ -0,0 +1,344 @@
+import { u as pr } from "./useFaceResults-cpsSmKKc.js";
+import { u as g, T as z, a as K } from "./index-BD-JWMjE.js";
+import { b as yr, F as mr } from "./index-BD-JWMjE.js";
+import { jsx as E, Fragment as G } from "react/jsx-runtime";
+import { useThree as N, useFrame as v } from "@react-three/fiber";
+import { useRef as T, useCallback as C, useMemo as k, useEffect as x } from "react";
+import * as R from "three";
+import { X as S, R as w } from "./index-CYMj26Ej.js";
+const L = {
+  "body.head": 0,
+  "body.leftShoulder": 11,
+  "body.rightShoulder": 12,
+  "body.leftElbow": 13,
+  "body.rightElbow": 14,
+  "body.leftWrist": 15,
+  "body.rightWrist": 16,
+  "body.leftHip": 23,
+  "body.rightHip": 24,
+  "body.leftKnee": 25,
+  "body.rightKnee": 26,
+  "body.leftAnkle": 27,
+  "body.rightAnkle": 28
+}, er = ({ bodyIndex: r = 0, active: n = !0, children: t }) => {
+  const e = T(null), { ready: o, node: a } = g(
+    S.BODY_TRACKER,
+    void 0,
+    n
+  ), { camera: s } = N(), d = T(s);
+  d.current !== s && (d.current = s);
+  const i = T(null);
+  v(() => {
+    if (!o || !a || !e.current) return;
+    const f = a;
+    i.current = f;
+    const c = f.getLandmarks2D(r), u = c == null ? void 0 : c[0];
+    if (!u) {
+      e.current.visible = !1;
+      return;
+    }
+    const p = D(u, d.current);
+    e.current.visible = !0, e.current.position.copy(p);
+  });
+  const l = C(
+    (f) => {
+      const c = i.current;
+      if (!c) return null;
+      const u = c.getLandmarks2D(r);
+      if (!u) return null;
+      if (f === "body.neck") {
+        const h = u[11], b = u[12];
+        if (!h || !b) return null;
+        const A = {
+          x: (h.x + b.x) / 2,
+          y: (h.y + b.y) / 2,
+          z: (h.z + b.z) / 2
+        };
+        return { position: D(A, d.current) };
+      }
+      const p = L[f];
+      if (p === void 0) return null;
+      const m = u[p];
+      return m ? { position: D(m, d.current) } : null;
+    },
+    [r]
+  ), y = k(
+    () => ({ type: "body", resolveLandmark: l }),
+    [l]
+  );
+  return /* @__PURE__ */ E(z.Provider, { value: y, children: /* @__PURE__ */ E("group", { ref: e, renderOrder: w.TRACKED, children: t }) });
+}, D = (r, n) => {
+  const t = new R.Vector3(r.x, r.y, r.z);
+  t.unproject(n);
+  const e = t.sub(n.position).normalize(), o = -n.position.z / e.z;
+  return n.position.clone().add(e.multiplyScalar(o));
+}, W = {
+  "hand.wrist": 0,
+  "hand.thumbTip": 4,
+  "hand.indexTip": 8,
+  "hand.middleTip": 12,
+  "hand.ringTip": 16,
+  "hand.pinkyTip": 20,
+  "hand.indexBase": 5,
+  "hand.middleBase": 9,
+  "hand.ringBase": 13,
+  "hand.pinkyBase": 17
+}, tr = ({ children: r, targetFps: n, active: t = !0 }) => {
+  const e = T(null), o = k(
+    () => n === void 0 ? void 0 : { targetFps: n },
+    [n]
+  ), { ready: a, node: s } = g(
+    S.GESTURE_TRACKER,
+    o,
+    t
+  ), { camera: d } = N(), i = T(d);
+  i.current !== d && (i.current = d);
+  const l = T(null);
+  v(() => {
+    var p;
+    if (!a || !s || !e.current) return;
+    l.current = s;
+    const c = (p = s == null ? void 0 : s.coordinates) == null ? void 0 : p[0];
+    if (!c) {
+      e.current.visible = !1;
+      return;
+    }
+    const u = _(c, i.current);
+    e.current.visible = !0, e.current.position.copy(u);
+  });
+  const y = C((c) => {
+    var m, h;
+    const u = W[c];
+    if (u === void 0) return null;
+    const p = (h = (m = l.current) == null ? void 0 : m.coordinates) == null ? void 0 : h[u];
+    return p ? { position: _(p, i.current) } : null;
+  }, []), f = k(
+    () => ({ type: "hand", resolveLandmark: y }),
+    [y]
+  );
+  return /* @__PURE__ */ E(z.Provider, { value: f, children: /* @__PURE__ */ E("group", { ref: e, renderOrder: w.TRACKED, children: r }) });
+}, _ = (r, n) => {
+  const t = new R.Vector3(r.x, r.y, r.z);
+  t.unproject(n);
+  const e = t.sub(n.position).normalize(), o = -n.position.z / e.z;
+  return n.position.clone().add(e.multiplyScalar(o));
+}, P = 800, V = 200, $ = () => ({
+  armed: !1,
+  armStart: null,
+  holdStart: null,
+  fired: !1
+}), j = (r, {
+  isTrigger: n,
+  gesture: t,
+  now: e,
+  mode: o,
+  debounceMs: a,
+  holdMs: s
+}) => {
+  if (r.fired) return null;
+  if (o === "hold") {
+    if (n && t) {
+      if (r.holdStart === null && (r.holdStart = e), e - r.holdStart >= s)
+        return r.fired = !0, t;
+    } else
+      r.holdStart = null;
+    return null;
+  }
+  return r.armed ? n && t ? (r.fired = !0, t) : null : (n ? r.armStart = null : (r.armStart === null && (r.armStart = e), e - r.armStart >= a && (r.armed = !0)), null);
+};
+function F({
+  gestures: r,
+  mode: n = "arm",
+  debounceMs: t = P,
+  holdMs: e = V,
+  onTrigger: o,
+  enabled: a = !0
+}) {
+  const { ready: s, node: d } = g(S.GESTURE_TRACKER), i = T(o);
+  x(() => {
+    i.current = o;
+  }, [o]);
+  const l = T(/* @__PURE__ */ new Set());
+  x(() => {
+    l.current = new Set(Array.isArray(r) ? r : [r]);
+  }, [r]);
+  const y = T($());
+  v(() => {
+    if (!a || !s || !d) return;
+    const f = d.gesture, c = !!(f && l.current.has(f)), u = j(y.current, {
+      isTrigger: c,
+      gesture: f,
+      now: performance.now(),
+      mode: n,
+      debounceMs: t,
+      holdMs: e
+    });
+    u && i.current(u);
+  });
+}
+const or = (r) => (F(r), null), q = {
+  "hand.wrist": 0,
+  "hand.thumbTip": 4,
+  "hand.indexTip": 8,
+  "hand.middleTip": 12,
+  "hand.ringTip": 16,
+  "hand.pinkyTip": 20,
+  "hand.indexBase": 5,
+  "hand.middleBase": 9,
+  "hand.ringBase": 13,
+  "hand.pinkyBase": 17
+}, ir = ({ children: r, targetFps: n, active: t = !0 }) => {
+  const e = T(null), o = k(
+    () => n === void 0 ? void 0 : { targetFps: n },
+    [n]
+  ), { ready: a, node: s } = g(
+    S.HAND_TRACKER,
+    o,
+    t
+  ), { camera: d } = N(), i = T(d);
+  i.current !== d && (i.current = d);
+  const l = T(null);
+  v(() => {
+    var m;
+    if (!a || !s || !e.current) return;
+    const c = s;
+    l.current = c;
+    const u = (m = c == null ? void 0 : c.coordinates) == null ? void 0 : m[0];
+    if (!u) {
+      e.current.visible = !1;
+      return;
+    }
+    const p = B(u, i.current);
+    e.current.visible = !0, e.current.position.copy(p);
+  });
+  const y = C((c) => {
+    var m, h;
+    const u = q[c];
+    if (u === void 0) return null;
+    const p = (h = (m = l.current) == null ? void 0 : m.coordinates) == null ? void 0 : h[u];
+    return p ? { position: B(p, i.current) } : null;
+  }, []), f = k(
+    () => ({ type: "hand", resolveLandmark: y }),
+    [y]
+  );
+  return /* @__PURE__ */ E(z.Provider, { value: f, children: /* @__PURE__ */ E("group", { ref: e, renderOrder: w.TRACKED, children: r }) });
+}, B = (r, n) => {
+  const t = new R.Vector3(r.x, r.y, r.z);
+  t.unproject(n);
+  const e = t.sub(n.position).normalize(), o = -n.position.z / e.z;
+  return n.position.clone().add(e.multiplyScalar(o));
+}, H = ["portrait"], cr = ({ type: r, active: n = !0, children: t }) => (g(S.BACKGROUND_SEGMENTATION, { type: r }, n), x(() => {
+  H.includes(r) || console.warn(
+    `[Segmentation] unsupported type "${r}"; valid values: ${H.join(", ")}`
+  );
+}, [r]), /* @__PURE__ */ E(G, { children: t })), sr = [
+  "face.forehead",
+  "face.noseTip",
+  "face.chin",
+  "face.leftEyeOuter",
+  "face.rightEyeOuter",
+  "face.leftCheek",
+  "face.rightCheek",
+  "face.mouthCenter"
+], ur = [
+  "hand.wrist",
+  "hand.thumbTip",
+  "hand.indexTip",
+  "hand.middleTip",
+  "hand.ringTip",
+  "hand.pinkyTip",
+  "hand.indexBase",
+  "hand.middleBase",
+  "hand.ringBase",
+  "hand.pinkyBase"
+], ar = [
+  "body.head",
+  "body.neck",
+  "body.leftShoulder",
+  "body.rightShoulder",
+  "body.leftElbow",
+  "body.rightElbow",
+  "body.leftWrist",
+  "body.rightWrist",
+  "body.leftHip",
+  "body.rightHip",
+  "body.leftKnee",
+  "body.rightKnee",
+  "body.leftAnkle",
+  "body.rightAnkle"
+], U = (r) => r.startsWith("face.") ? "face" : r.startsWith("hand.") ? "hand" : r.startsWith("body.") ? "body" : null, M = (r, n, t) => n + t * (r - n), O = (r, n) => 1 / (1 + 1 / (2 * Math.PI * r) / n), X = (r, n, t, e, o, a) => {
+  if (!r.initialized || t <= 0)
+    return r.pos.copy(n), r.prevRaw.copy(n), r.dxHat.set(0, 0, 0), r.initialized = !0, r;
+  const s = O(a, t), d = ["x", "y", "z"];
+  for (const i of d) {
+    const l = (n[i] - r.prevRaw[i]) / t, y = M(l, r.dxHat[i], s);
+    r.dxHat[i] = y;
+    const f = e + o * Math.abs(y), c = O(f, t);
+    r.pos[i] = M(n[i], r.pos[i], c), r.prevRaw[i] = n[i];
+  }
+  return r;
+}, dr = ({
+  target: r,
+  offset: n,
+  smoothing: t = 0,
+  children: e
+}) => {
+  const o = K(), a = T(null), s = T(!1), d = T(new R.Vector3()), i = T({
+    pos: new R.Vector3(),
+    prevRaw: new R.Vector3(),
+    dxHat: new R.Vector3(),
+    initialized: !1
+  }), l = U(r), y = o !== null && l !== null && o.type !== l, f = o === null, c = f || y || l === null;
+  return x(() => {
+    f ? console.warn(
+      `[TrackingAnchor] target "${r}" requires a FaceTracker / HandTracker / BodyTracker ancestor; rendering null.`
+    ) : y ? console.warn(
+      `[TrackingAnchor] target "${r}" namespace does not match parent tracker type "${o == null ? void 0 : o.type}"; rendering null.`
+    ) : l === null && console.warn(
+      `[TrackingAnchor] target "${r}" is not a recognized face/hand/body landmark; rendering null.`
+    );
+  }, [r, l, f, y, o == null ? void 0 : o.type]), v((u, p) => {
+    if (c || !a.current || !o) return;
+    const m = o.resolveLandmark(r);
+    if (!m) {
+      a.current.visible = !1, s.current = !1, i.current.initialized = !1;
+      return;
+    }
+    a.current.visible = !0;
+    const h = d.current.copy(m.position), b = a.current.parent;
+    if (b) {
+      b.updateMatrixWorld();
+      const A = new R.Matrix4().copy(b.matrixWorld).invert();
+      h.applyMatrix4(A);
+    }
+    if (n && (h.x += n[0], h.y += n[1], h.z += n[2]), t > 0) {
+      const A = t * 0.1;
+      X(
+        i.current,
+        h,
+        s.current ? p : 0,
+        A,
+        0.5,
+        1
+      ), a.current.position.copy(i.current.pos);
+    } else
+      a.current.position.copy(h);
+    s.current = !0, m.quaternion && a.current.quaternion.copy(m.quaternion);
+  }), c ? null : /* @__PURE__ */ E("group", { ref: a, renderOrder: w.TRACKED, children: e });
+};
+export {
+  ar as BODY_TARGETS,
+  er as BodyTracker,
+  sr as FACE_TARGETS,
+  yr as FaceMesh,
+  mr as FaceTracker,
+  tr as GestureTracker,
+  or as GestureTrigger,
+  ur as HAND_TARGETS,
+  ir as HandTracker,
+  H as SEGMENTATION_TYPES,
+  cr as Segmentation,
+  dr as TrackingAnchor,
+  U as getTargetNamespace,
+  pr as useFaceResults
+};

package/dist/types/body.d.ts

@@ -0,0 +1,3 @@
+export type BodyModelOptions = {
+    numPoses?: number;
+};

package/dist/types/client.d.ts

@@ -0,0 +1,3 @@
+export type XRClientConfig = {
+    apiKey: string;
+};

package/dist/types/common.d.ts

@@ -0,0 +1,9 @@
+export type Coordinate = {
+    x: number;
+    y: number;
+    z: number;
+};
+export type PerformanceData = {
+    name: string;
+    processingTime: number | null;
+};

package/dist/types/compositor.d.ts

@@ -0,0 +1,23 @@
+export type XRLayerConfig = {
+    name: string;
+    clearColor: string;
+    hidden?: boolean;
+    pointerEvents?: "none" | "auto";
+    zIndex: number;
+};
+export type XRMediaSourceKind = "image" | "video";
+export type XRMediaLayout = {
+    source: {
+        width: number;
+        height: number;
+    };
+    destination: {
+        width: number;
+        height: number;
+    };
+    drawWidth: number;
+    drawHeight: number;
+    offsetX: number;
+    offsetY: number;
+    scale: number;
+};

package/dist/types/debugger.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Debugger type definitions for the XR SDK
+ * Provides real-time performance insights for WebAR experiences
+ */
+/**
+ * Per-model performance data
+ */
+export interface ModelPerformanceData {
+    /** Model name */
+    name: string;
+    /** Processing time in milliseconds */
+    processingTime: number;
+}
+/**
+ * Core metrics collected by the debugger
+ */
+export interface DebuggerMetrics {
+    /** Camera frames per second (updated every 500ms) */
+    cameraFPS: number;
+    /** Total detection time across all models in milliseconds (per frame) */
+    detectionTime: number;
+    /** Per-model detection times */
+    modelPerformance: ModelPerformanceData[];
+    /** Rendering time per frame in milliseconds */
+    renderTime: number;
+    /** Effective FPS combining detection + render (updated every 1s) */
+    effectiveFPS: number;
+    /** Memory usage information */
+    memory: {
+        /** Used heap size in MB */
+        used: number;
+        /** Total heap size in MB */
+        total: number;
+        /** Usage percentage (0-100) */
+        percentage: number;
+    } | null;
+    /** Camera resolution as "widthxheight" string (e.g., "1280x720") */
+    resolution: string;
+}
+/**
+ * Debugger state managed by DebuggerProvider
+ */
+export interface DebuggerState {
+    /** Whether the debugger is enabled (visible) */
+    enabled: boolean;
+    /** Whether the debugger UI is collapsed */
+    collapsed: boolean;
+    /** Current metrics */
+    metrics: DebuggerMetrics;
+}
+/**
+ * Configuration options for the DebuggerOverlay component
+ */
+export interface DebuggerConfig {
+    /** Starting state: collapsed or expanded */
+    defaultCollapsed?: boolean;
+    /** Keyboard shortcut to toggle debugger (default: "ctrl+shift+d") */
+    keyboardShortcut?: string;
+    /** Position of the overlay (default: "top-left") */
+    position?: "top-left" | "top-right" | "bottom-left" | "bottom-right";
+}
+/**
+ * Context value provided by DebuggerProvider
+ */
+export interface DebuggerContextValue {
+    /** Current debugger state */
+    state: DebuggerState;
+    /** Toggle debugger visibility */
+    toggle: () => void;
+    /** Set collapsed state */
+    setCollapsed: (collapsed: boolean) => void;
+    /** Enable debugger */
+    enable: () => void;
+    /** Disable debugger */
+    disable: () => void;
+}
+/**
+ * Props for DebuggerOverlay component
+ */
+export interface DebuggerOverlayProps extends DebuggerConfig {
+    /** Optional className for custom styling */
+    className?: string;
+}

package/dist/types/face.d.ts

@@ -0,0 +1,67 @@
+import { Category, Matrix, NormalizedLandmark } from '@mediapipe/tasks-vision';
+import { Vector3 } from 'three';
+export type FaceModelOptions = {
+    numFaces?: number;
+};
+export type Face = {
+    transform: {
+        position: Vector3;
+        rotation: Vector3;
+        scale: Vector3;
+    };
+    gestures: Record<string, number>;
+};
+export type FaceBounds = {
+    minX: number;
+    minY: number;
+    maxX: number;
+    maxY: number;
+    width: number;
+    height: number;
+    centerX: number;
+    centerY: number;
+};
+export type FaceTrackingResult = {
+    landmarks: NormalizedLandmark[];
+    bounds: FaceBounds;
+};
+/**
+ * Raw MediaPipe face-tracking payload for one detected face. This stays
+ * close to the upstream output so custom effects and AI-assisted tooling
+ * can inspect low-level tracking data without reconstructing it from the
+ * higher-level `Face` transform abstraction.
+ */
+export type FaceRawTrackingData = {
+    landmarks?: NormalizedLandmark[];
+    blendshapes?: Category[];
+    transformationMatrix?: Matrix;
+};
+/**
+ * 2D face landmark in normalized image space (0..1 across both axes).
+ */
+export type FaceLandmark2D = {
+    x: number;
+    y: number;
+};
+/**
+ * Normalized bounding box (0..1 across both axes). Multiply by container
+ * dimensions to position in CSS or screen space.
+ */
+export type BoundingBox = {
+    left: number;
+    top: number;
+    width: number;
+    height: number;
+};
+/**
+ * Elliptical region used by the face-retouch shader. All coordinates are
+ * in normalized image space (0..1).
+ */
+export type FaceRetouchRegion = {
+    centerX: number;
+    centerY: number;
+    radiusX: number;
+    radiusY: number;
+    strength: number;
+    kind: "face" | "feature";
+};

package/dist/types/gesture-tracking.d.ts

@@ -0,0 +1,15 @@
+import { Coordinate } from './common';
+export type GestureData = {
+    coordinates?: Coordinate[];
+    gesture?: "closed_fist" | "open_palm" | "pointing_up" | "thumb_down" | "thumb_up" | "victory" | "i_love_you" | "none";
+};
+export declare enum Gestures {
+    ClosedFist = "closed_fist",
+    OpenPalm = "open_palm",
+    PointingUp = "pointing_up",
+    ThumbDown = "thumb_down",
+    ThumbUp = "thumb_up",
+    Victory = "victory",
+    ILoveYou = "i_love_you",
+    None = "none"
+}

package/dist/types/hand-tracking.d.ts

@@ -0,0 +1,5 @@
+import { Coordinate } from './common';
+export type HandData = {
+    coordinates?: Coordinate[];
+    gesture?: "closed_fist" | "open_palm";
+};

package/dist/types/headBinding.d.ts

@@ -0,0 +1,67 @@
+export type HeadBindingShape = {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    centerX: number;
+    centerY: number;
+};
+export type HeadBindingAdaptiveValues = {
+    aspectRatio: number;
+    normalizedAspect: number;
+    scaleY: number;
+    offsetY: number;
+};
+export type HeadBindingCubePose = {
+    boxCenterZ: number;
+    centerX: number;
+    centerY: number;
+    depth: number;
+    height: number;
+    rotationX: number;
+    rotationY: number;
+    rotationZ: number;
+    width: number;
+};
+export type HeadBindingCubeModel = "v1" | "v2";
+export type HeadBindingDebugMetrics = {
+    center: {
+        x: number;
+        y: number;
+        z: number;
+    };
+    depthPx: number;
+    face: {
+        centerX: number;
+        centerY: number;
+        height: number;
+        width: number;
+    };
+    normalized: {
+        rawPitch: number;
+        rawPitchFromBalance: number;
+        rawPitchFromCenter: number;
+        rawYaw: number;
+        roll: number;
+        verticalBalance: number;
+    };
+    pose: {
+        pitch: number;
+        rotationX: number;
+        rotationY: number;
+        rotationZ: number;
+        yaw: number;
+    };
+    screen: {
+        heightPx: number;
+        widthPx: number;
+    };
+    skullAnchor: {
+        x: number;
+        y: number;
+    };
+};
+export type HeadBindingSolveResult = {
+    pose: HeadBindingCubePose;
+    debugMetrics: HeadBindingDebugMetrics;
+};

package/dist/types/index.d.ts

@@ -0,0 +1,12 @@
+export * from './client';
+export * from './nodes';
+export * from './session';
+export * from './body';
+export * from './face';
+export * from './headBinding';
+export * from './model';
+export * from './segmentation';
+export * from './common';
+export * from './compositor';
+export * from './logger';
+export * from './util-types';

package/dist/types/logger.d.ts

@@ -0,0 +1,4 @@
+export type LoggerOptions = {
+    verbose: boolean;
+    prefix: string;
+};

package/dist/types/model.d.ts

@@ -0,0 +1,46 @@
+import { XRMediaSourceKind } from './compositor';
+/**
+ * Session-level performance config, set via `<XRProvider perf={...}>`. Trades
+ * tracking fidelity for lower GPU/CPU load and heat. Every field is optional;
+ * omitted fields keep the SDK's built-in behavior.
+ */
+export type XRPerfConfig = {
+    /**
+     * Default inference-rate cap (fps) applied to every tracker model that does
+     * not set its own `targetFps` prop. Undefined / <= 0 → uncapped for Face and
+     * Hand (Gesture keeps its own 15fps default). A per-tracker `targetFps` prop
+     * overrides this for that tracker.
+     */
+    targetFps?: number;
+    /**
+     * Target width (px) for the camera-inference source canvas — the image the
+     * ML models analyze, NOT the visible camera background the user sees. Lower
+     * = less ML work. Undefined keeps the SDK default. This does not change the
+     * displayed feed quality.
+     */
+    camRes?: number;
+    /**
+     * Camera framing zoom applied when cover-cropping the webcam into the
+     * viewport. `1` = the full native frame (widest field of view); `> 1` crops
+     * in and enlarges the subject (a `1.2` fills the frame like a native selfie /
+     * Instagram front camera). This is a FRAMING control, not a perf one — it
+     * changes the visible feed (and the captured photo and what the trackers see,
+     * since all three read the same cover-cropped source). Default `1.2` to match
+     * the tight native-selfie framing users expect; set `1` for the widest FOV.
+     */
+    cameraZoom?: number;
+};
+export type RunModelParams = {
+    sourceCanvas: OffscreenCanvas | HTMLCanvasElement;
+    renderWidth: number;
+    renderHeight: number;
+    getSource?: () => "webcam" | XRMediaSourceKind | undefined;
+    /**
+     * Cap the model's inference rate to this many frames per second. The model's
+     * RAF loop still runs at display rate, but the (expensive) MediaPipe detect
+     * call is skipped between intervals. Undefined / <= 0 means uncapped (detect
+     * every frame). Lower values cut GPU/CPU load and heat at the cost of tracking
+     * smoothness. See `session.perf.targetFps` and the per-tracker `targetFps`.
+     */
+    targetFps?: number;
+};