@vincentt-xr/sdk 2.0.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/GROUNDING.md +537 -0
  2. package/LICENSE +7 -0
  3. package/README.md +85 -0
  4. package/dist/DebuggerUI-CEfdddkZ.cjs +1 -0
  5. package/dist/DebuggerUI-Dgop6Q6o.js +269 -0
  6. package/dist/audio/AudioEngine.d.ts +98 -0
  7. package/dist/audio/index.d.ts +34 -0
  8. package/dist/components/AspectRatioContainer/index.d.ts +10 -0
  9. package/dist/components/BodyTracker/index.d.ts +15 -0
  10. package/dist/components/Debugger/DebuggerProvider.d.ts +22 -0
  11. package/dist/components/Debugger/DebuggerUI.d.ts +13 -0
  12. package/dist/components/Debugger/index.d.ts +38 -0
  13. package/dist/components/Debugger/styles.d.ts +19 -0
  14. package/dist/components/FaceMesh/FaceMesh.d.ts +26 -0
  15. package/dist/components/FaceMesh/canonical/CanonicalFaceMeshRenderer.d.ts +34 -0
  16. package/dist/components/FaceMesh/canonical/parseCanonicalFaceModel.d.ts +16 -0
  17. package/dist/components/FaceMesh/canonical/useCanonicalFaceMesh.d.ts +15 -0
  18. package/dist/components/FaceMesh/deform.d.ts +36 -0
  19. package/dist/components/FaceMesh/index.d.ts +6 -0
  20. package/dist/components/FaceMesh/indices.d.ts +1 -0
  21. package/dist/components/FaceMesh/material/blendModes.d.ts +2 -0
  22. package/dist/components/FaceMesh/material/defaults.d.ts +3 -0
  23. package/dist/components/FaceMesh/material/index.d.ts +5 -0
  24. package/dist/components/FaceMesh/material/options.d.ts +25 -0
  25. package/dist/components/FaceMesh/material/support.d.ts +3 -0
  26. package/dist/components/FaceMesh/material/types.d.ts +23 -0
  27. package/dist/components/FaceMesh/uv.d.ts +1 -0
  28. package/dist/components/FaceRetouch/Background.d.ts +16 -0
  29. package/dist/components/FaceRetouch/index.d.ts +6 -0
  30. package/dist/components/FaceRetouch/landmarkMask.d.ts +23 -0
  31. package/dist/components/FaceRetouch/retouchRegions.d.ts +23 -0
  32. package/dist/components/FaceRetouch/useFaceRetouchMask.d.ts +14 -0
  33. package/dist/components/FaceRetouch/useFaceRetouchRegions.d.ts +12 -0
  34. package/dist/components/FaceTracker/BoundingBoxOverlay.d.ts +20 -0
  35. package/dist/components/FaceTracker/index.d.ts +54 -0
  36. package/dist/components/GestureTracker/index.d.ts +31 -0
  37. package/dist/components/GestureTrigger/index.d.ts +3 -0
  38. package/dist/components/HandTracker/index.d.ts +24 -0
  39. package/dist/components/HeadBinding/index.d.ts +31 -0
  40. package/dist/components/HeadBindingOccluder/index.d.ts +36 -0
  41. package/dist/components/HeadOccluder/index.d.ts +13 -0
  42. package/dist/components/Panel/index.d.ts +71 -0
  43. package/dist/components/PerspectiveCamera/index.d.ts +14 -0
  44. package/dist/components/ScreenSpaceUI/ScreenTransform.d.ts +54 -0
  45. package/dist/components/ScreenSpaceUI/context.d.ts +29 -0
  46. package/dist/components/ScreenSpaceUI/index.d.ts +40 -0
  47. package/dist/components/Segmentation/index.d.ts +44 -0
  48. package/dist/components/SpriteAnimation/index.d.ts +52 -0
  49. package/dist/components/TextLabel/index.d.ts +62 -0
  50. package/dist/components/TrackingAnchor/index.d.ts +74 -0
  51. package/dist/components/VideoBackground/index.d.ts +38 -0
  52. package/dist/components/XRCompositor/index.d.ts +5 -0
  53. package/dist/components/XRDebugView/index.d.ts +6 -0
  54. package/dist/components/XRLayer/index.d.ts +6 -0
  55. package/dist/components/XRMediaSurface/index.d.ts +27 -0
  56. package/dist/components/XRProvider/index.d.ts +13 -0
  57. package/dist/components/XRScene/defaults.d.ts +4 -0
  58. package/dist/components/XRScene/index.d.ts +15 -0
  59. package/dist/components/XRViewport/index.d.ts +101 -0
  60. package/dist/components/index.d.ts +26 -0
  61. package/dist/config/index.d.ts +8 -0
  62. package/dist/constants/index.d.ts +25 -0
  63. package/dist/constants/renderLayers.d.ts +25 -0
  64. package/dist/entries/all.d.ts +15 -0
  65. package/dist/entries/core.d.ts +21 -0
  66. package/dist/entries/face-effects.d.ts +19 -0
  67. package/dist/entries/internal.d.ts +16 -0
  68. package/dist/entries/low-level.d.ts +30 -0
  69. package/dist/entries/tracking.d.ts +15 -0
  70. package/dist/face-effects.cjs +252 -0
  71. package/dist/face-effects.js +3542 -0
  72. package/dist/hasFullUnitUVBounds-BGvcFsQM.js +26 -0
  73. package/dist/hasFullUnitUVBounds-p1Dl0ejX.cjs +1 -0
  74. package/dist/hooks/faceResults.d.ts +39 -0
  75. package/dist/hooks/selectors.d.ts +33 -0
  76. package/dist/hooks/useCalibrationFaceTracker.d.ts +23 -0
  77. package/dist/hooks/useDebugger/index.d.ts +25 -0
  78. package/dist/hooks/useDebugger/types.d.ts +20 -0
  79. package/dist/hooks/useFaceBoundingBox.d.ts +11 -0
  80. package/dist/hooks/useFaceResults.d.ts +31 -0
  81. package/dist/hooks/useGestureEdge.d.ts +53 -0
  82. package/dist/hooks/useMediaRecorder.d.ts +40 -0
  83. package/dist/hooks/useTrackerContext.d.ts +35 -0
  84. package/dist/hooks/useXRModel.d.ts +15 -0
  85. package/dist/hooks/useXRSession/constants.d.ts +4 -0
  86. package/dist/hooks/useXRSession/facade.d.ts +39 -0
  87. package/dist/hooks/useXRSession/fns.d.ts +5 -0
  88. package/dist/hooks/useXRSession/index.d.ts +4 -0
  89. package/dist/hooks/useXRSession/registry.d.ts +52 -0
  90. package/dist/hooks/useXRSession/store.d.ts +40 -0
  91. package/dist/index-B5J4gzn2.cjs +1 -0
  92. package/dist/index-BD-JWMjE.js +1991 -0
  93. package/dist/index-BVBVg96G.cjs +1 -0
  94. package/dist/index-CYMj26Ej.js +1331 -0
  95. package/dist/index-DtwHElmG.js +157 -0
  96. package/dist/index-PtFmD6B8.cjs +1 -0
  97. package/dist/internal-DH5d1Xnq.cjs +1 -0
  98. package/dist/internal-KOKJDxCG.js +519 -0
  99. package/dist/internal.cjs +1 -0
  100. package/dist/internal.js +14 -0
  101. package/dist/low-level.cjs +1 -0
  102. package/dist/low-level.js +941 -0
  103. package/dist/main.cjs +1 -0
  104. package/dist/main.js +612 -0
  105. package/dist/models/Base/index.d.ts +25 -0
  106. package/dist/models/Body/index.d.ts +22 -0
  107. package/dist/models/Face/constants.d.ts +2 -0
  108. package/dist/models/Face/index.d.ts +45 -0
  109. package/dist/models/Face/landmarksData.d.ts +2 -0
  110. package/dist/models/Hand/GestureTracking.d.ts +21 -0
  111. package/dist/models/Hand/HandTracking.d.ts +21 -0
  112. package/dist/models/Segmentation/index.d.ts +75 -0
  113. package/dist/models/index.d.ts +5 -0
  114. package/dist/modules/XRClient/index.d.ts +9 -0
  115. package/dist/modules/index.d.ts +1 -0
  116. package/dist/selectors-BHO6TnoC.js +59 -0
  117. package/dist/selectors-Bub0o3qy.cjs +1 -0
  118. package/dist/services/ApiService.d.ts +12 -0
  119. package/dist/services/MetricsCollector/CameraFPSTracker.d.ts +25 -0
  120. package/dist/services/MetricsCollector/MemoryTracker.d.ts +43 -0
  121. package/dist/services/MetricsCollector/RenderTimeTracker.d.ts +10 -0
  122. package/dist/services/MetricsCollector/ResolutionTracker.d.ts +26 -0
  123. package/dist/services/MetricsCollector/index.d.ts +59 -0
  124. package/dist/services/MetricsCollector/types.d.ts +47 -0
  125. package/dist/services/MetricsCollector/utils.d.ts +65 -0
  126. package/dist/texture/computeContainScale.d.ts +25 -0
  127. package/dist/texture/hasFullUnitUVBounds.d.ts +37 -0
  128. package/dist/texture/types.d.ts +20 -0
  129. package/dist/tracking.cjs +1 -0
  130. package/dist/tracking.js +344 -0
  131. package/dist/types/body.d.ts +3 -0
  132. package/dist/types/client.d.ts +3 -0
  133. package/dist/types/common.d.ts +9 -0
  134. package/dist/types/compositor.d.ts +23 -0
  135. package/dist/types/debugger.d.ts +83 -0
  136. package/dist/types/face.d.ts +67 -0
  137. package/dist/types/gesture-tracking.d.ts +15 -0
  138. package/dist/types/hand-tracking.d.ts +5 -0
  139. package/dist/types/headBinding.d.ts +67 -0
  140. package/dist/types/index.d.ts +12 -0
  141. package/dist/types/logger.d.ts +4 -0
  142. package/dist/types/model.d.ts +46 -0
  143. package/dist/types/nodes.d.ts +35 -0
  144. package/dist/types/segmentation.d.ts +31 -0
  145. package/dist/types/session.d.ts +102 -0
  146. package/dist/types/tracking.d.ts +30 -0
  147. package/dist/types/util-types.d.ts +1 -0
  148. package/dist/useFaceResults-cpsSmKKc.js +81 -0
  149. package/dist/useFaceResults-hVUJYSEY.cjs +1 -0
  150. package/dist/utils/calibrationDebug.d.ts +9 -0
  151. package/dist/utils/dataURL.d.ts +6 -0
  152. package/dist/utils/drawMediaCover.d.ts +18 -0
  153. package/dist/utils/errors.d.ts +3 -0
  154. package/dist/utils/fns.d.ts +48 -0
  155. package/dist/utils/headBinding.d.ts +85 -0
  156. package/dist/utils/integrity.d.ts +43 -0
  157. package/dist/utils/landmarkBounds.d.ts +11 -0
  158. package/dist/utils/logger.d.ts +7 -0
  159. package/package.json +113 -0
@@ -0,0 +1,44 @@
1
+ import { ReactNode } from 'react';
2
+ import { SegmentationType } from '../../types/segmentation';
3
+ type SegmentationProps = {
4
+ /** Which segmentation pipeline to enable. v1 supports `"portrait"`. */
5
+ type: SegmentationType;
6
+ /**
7
+ * When `false`, the segmentation pipeline is still acquired (for instant
8
+ * re-enable) but the mask is not exposed to consumers like VideoBackground.
9
+ * Children always render — Segmentation is an effect provider, not a
10
+ * visual container.
11
+ */
12
+ visible?: boolean;
13
+ /**
14
+ * Whether the segmentation pipeline should run. `false` pauses the shared
15
+ * model's per-frame inference (and its expensive GPU→CPU readback) while
16
+ * keeping it mounted and initialized, so gating it off costs nothing yet
17
+ * flipping back to `true` resumes instantly — no unmount, no registry
18
+ * re-acquire race. Distinct from `visible`, which only gates mask exposure
19
+ * while inference keeps running. When several consumers share the model, it
20
+ * runs while any is active. Defaults to `true`.
21
+ */
22
+ active?: boolean;
23
+ /**
24
+ * Children are rendered as-is via a fragment. Segmentation has no
25
+ * geometry or transform of its own — it only registers the pipeline —
26
+ * so nesting children here is equivalent to placing them as siblings.
27
+ * This allows scene trees where a VideoBackground is grouped under its
28
+ * Segmentation provider to still render correctly.
29
+ */
30
+ children?: ReactNode;
31
+ };
32
+ /**
33
+ * Declarative scene-level component that enables a segmentation
34
+ * pipeline. Renders no visible geometry — its only effect is to
35
+ * declare a refcount on the BACKGROUND_SEGMENTATION model for as long
36
+ * as it is mounted via `useXRModel`. Multiple `<Segmentation>` instances
37
+ * (or a `<Segmentation>` plus a `<VideoBackground segmentation="…">`)
38
+ * share one underlying model instance.
39
+ *
40
+ * `<VideoBackground segmentation="…">` consumes the resulting mask by
41
+ * subscribing to the same model node via `useXRModelNode`.
42
+ */
43
+ export declare const Segmentation: ({ type, active, children }: SegmentationProps) => import("react/jsx-runtime").JSX.Element;
44
+ export {};
@@ -0,0 +1,52 @@
1
+ import { ScreenAnchorEdges } from '../ScreenSpaceUI/ScreenTransform';
2
+ /**
3
+ * The UV offset for frame `idx` of a `cols`x`rows` atlas. Frames read
4
+ * left-to-right, top-to-bottom. three UVs are bottom-left origin but atlas rows
5
+ * read top-down, so the row is flipped: the FIRST frame (row 0) sits at the TOP
6
+ * of the sheet, which is the HIGHEST v-offset. Pair with `repeat = (1/cols,
7
+ * 1/rows)`. Exported for the unit test that guards the row-flip.
8
+ */
9
+ export declare const spriteCellUV: (idx: number, cols: number, rows: number) => {
10
+ x: number;
11
+ y: number;
12
+ };
13
+ /**
14
+ * One transparent WebP sprite-sheet atlas describing an animation: `cols`x`rows`
15
+ * cells, the first `frames` read left-to-right, top-to-bottom, played at `fps`.
16
+ *
17
+ * This is the platform's animated-transparent-overlay primitive. It is a single
18
+ * static GPU texture with a per-frame UV offset, so there is no video decode:
19
+ * it works in every native and in-app browser, has no autoplay policy, and costs
20
+ * ~zero decode/heat. Prefer it over transparent video, which has no universal
21
+ * codec (HEVC-alpha is iOS-only; VP8/VP9-alpha WebM renders blank on iOS Safari)
22
+ * and overheats the device when decoded alongside face tracking. Pair opaque
23
+ * H.264 video only for full-frame backgrounds.
24
+ */
25
+ export type SpriteSheet = {
26
+ /** URL of the transparent WebP atlas (real 8-bit alpha; export with cwebp, not
27
+ * ffmpeg's libwebp which drops alpha). */
28
+ url: string;
29
+ cols: number;
30
+ rows: number;
31
+ /** Frames to play (may be fewer than cols*rows if the last cells are blank). */
32
+ frames: number;
33
+ fps: number;
34
+ };
35
+ export type SpriteAnimationProps = {
36
+ sheet: SpriteSheet;
37
+ /** Where to place the overlay. Defaults to full-screen. */
38
+ anchors?: ScreenAnchorEdges;
39
+ /** Frame width/height. When set, the sheet is contain-fit into the anchor rect
40
+ * (no stretch); omit for a plane that fills the rect exactly. */
41
+ contentAspect?: number;
42
+ renderOrder?: number;
43
+ name?: string;
44
+ };
45
+ /**
46
+ * Plays a sprite-sheet animation as a screen-space overlay (a frame border, a
47
+ * gesture-instruction graphic, an animated sticker). Transparent, animated,
48
+ * GPU-cheap, and identical in every browser. Defaults to full-screen; pass
49
+ * `anchors` to place it and `contentAspect` (frame width/height) to contain-fit
50
+ * without stretching. Must live inside a `<ScreenSpaceUI>`.
51
+ */
52
+ export declare const SpriteAnimation: ({ sheet, anchors, contentAspect, renderOrder, name, }: SpriteAnimationProps) => import("react/jsx-runtime").JSX.Element;
@@ -0,0 +1,62 @@
1
+ /**
2
+ * `<TextLabel>` — a text label drawn into a CanvasTexture and rendered as
3
+ * a single mesh. First-class scene-tree primitive so lifecycle scripts
4
+ * don't have to hand-paint canvas textures for every quiz question,
5
+ * progress indicator, or floating answer.
6
+ *
7
+ * Sizes itself 1×1 in world units; use `scale` on the JSX node or a
8
+ * parent `<ScreenTransform>` / `<TrackingAnchor>` to place it.
9
+ *
10
+ * Word-wraps automatically inside the panel's width; when there are too
11
+ * many lines for the chosen font size, the font shrinks one step before
12
+ * the text is allowed to clip.
13
+ */
14
+ export type TextLabelProps = {
15
+ /** The text to render. Required. */
16
+ text: string;
17
+ /** Text color. Default `#ffffff`. */
18
+ color?: string;
19
+ /** Font weight. Default `bold`. */
20
+ fontWeight?: "normal" | "bold" | number;
21
+ /** Font family. Default `Arial, sans-serif`. */
22
+ fontFamily?: string;
23
+ /**
24
+ * Font size in canvas px at the default 512px resolution. Roughly maps
25
+ * to "10pt" → 28, "12pt" → 36, "16pt" → 48, etc. Default 48.
26
+ */
27
+ fontSize?: number;
28
+ /** Panel background color. Use `'transparent'` for text-only. Default `rgba(20,40,110,0.88)`. */
29
+ bgColor?: string;
30
+ /** Optional border color. Omit for no border. */
31
+ borderColor?: string;
32
+ /** Border thickness. Default 0. */
33
+ borderWidth?: number;
34
+ /** Corner radius. Use 0 for square. Default 24. */
35
+ borderRadius?: number;
36
+ /** Inner padding in canvas px. Default 32. */
37
+ padding?: number;
38
+ /**
39
+ * Aspect ratio (width / height). The mesh itself is always 1×1; this
40
+ * controls how the texture canvas is shaped so wrapping and corner
41
+ * radius read correctly when the label isn't square. Default 2.5
42
+ * (roughly banner-shaped).
43
+ */
44
+ aspect?: number;
45
+ /** Canvas resolution (height in px). Default 512. */
46
+ resolution?: number;
47
+ /** Material opacity multiplier. Default 1. */
48
+ opacity?: number;
49
+ /** Optional `name` for runtime `getObjectByName` lookups. */
50
+ name?: string;
51
+ /** Ignore depth — appropriate for overlay UI. Default true. */
52
+ overlay?: boolean;
53
+ /** Local position offset, forwarded to the underlying mesh. */
54
+ position?: [number, number, number];
55
+ /** Local rotation (Euler radians), forwarded to the underlying mesh. */
56
+ rotation?: [number, number, number];
57
+ /** Local scale, forwarded to the underlying mesh. */
58
+ scale?: [number, number, number] | number;
59
+ /** Mesh visibility, forwarded so descendants inherit correctly. */
60
+ visible?: boolean;
61
+ };
62
+ export declare function TextLabel(props: TextLabelProps): import("react/jsx-runtime").JSX.Element;
@@ -0,0 +1,74 @@
1
+ import { PropsWithChildren } from 'react';
2
+ import { TrackingTarget } from '../../types/tracking';
3
+ import * as THREE from "three";
4
+ /**
5
+ * Frame-rate-independent lerp factor for a given smoothing rate and frame
6
+ * delta: `alpha = 1 - e^(-smoothing*dt)`. Same rate produces the same visual
7
+ * smoothing whether the render loop ticks at 30 or 120fps (a fixed per-frame
8
+ * alpha would smooth twice as fast at double the frame rate). `smoothing <= 0`
9
+ * returns 1 (snap). Exported for the unit test.
10
+ */
11
+ export declare const smoothingAlpha: (smoothing: number, dt: number) => number;
12
+ /**
13
+ * One Euro Filter — the standard adaptive low-pass for smoothing noisy,
14
+ * low-rate interactive tracking (it's what MediaPipe's own hand/face demos use).
15
+ *
16
+ * Why not velocity extrapolation: inference is throttled (~24fps) but rendering
17
+ * runs at 60-120fps, so the raw sample holds then jumps. Extrapolating forward
18
+ * on the per-sample velocity *overshoots* — the estimate is noisy, so the anchor
19
+ * coasts past the target and the next sample yanks it back, a visible sawtooth
20
+ * wobble. This filter takes the opposite approach: it never predicts ahead (so
21
+ * it cannot overshoot); instead it varies its own smoothing by speed. When the
22
+ * landmark moves slowly the cutoff is low → heavy smoothing kills jitter; when
23
+ * it moves fast the cutoff rises → light smoothing kills lag. That adaptivity is
24
+ * what a fixed lerp can't do — a fixed lerp must trade jitter against lag, and at
25
+ * 24fps→60fps neither setting is good.
26
+ *
27
+ * `minCutoff` sets the floor smoothing (lower = smoother at rest, more lag);
28
+ * `beta` sets how much speed loosens it (higher = less lag when moving, but more
29
+ * jitter passes). `dcutoff` smooths the internal speed estimate. These map from
30
+ * the single `smoothing` prop below. Operates per-axis on a Vec3.
31
+ */
32
+ export type OneEuroState = {
33
+ /** Filtered position (the value to render). */
34
+ pos: THREE.Vector3;
35
+ /** Previous raw sample, for the derivative. */
36
+ prevRaw: THREE.Vector3;
37
+ /** Filtered derivative (units/sec), per axis. */
38
+ dxHat: THREE.Vector3;
39
+ initialized: boolean;
40
+ };
41
+ /**
42
+ * One frame of the One Euro Filter, pure so it can be unit-tested without a
43
+ * scene. Mutates and returns `state`. `dt` is the render frame delta (seconds).
44
+ * `minCutoff`/`beta`/`dcutoff` are the filter params (see type doc). Filters
45
+ * each axis independently. On the first sample it seeds without smoothing.
46
+ */
47
+ export declare const stepOneEuro: (state: OneEuroState, sample: THREE.Vector3, dt: number, minCutoff: number, beta: number, dcutoff: number) => OneEuroState;
48
+ type TrackingAnchorProps = PropsWithChildren & {
49
+ /**
50
+ * Landmark/joint to attach to. The namespace prefix (`face.`, `hand.`,
51
+ * `body.`) MUST match the type of the nearest ancestor tracker —
52
+ * mismatched anchors render `null` and emit a console warning.
53
+ */
54
+ target: TrackingTarget;
55
+ /** Optional offset applied in the tracker's local frame after resolution. */
56
+ offset?: [number, number, number];
57
+ /**
58
+ * Position-smoothing amount. Inference is throttled (the `perf` API caps it at
59
+ * 24fps by default) while rendering runs at 60-120fps, so a raw anchor holds
60
+ * then jumps on each new sample — visible stepping under head/hand movement. A
61
+ * plain lerp only hides it at rest; under motion it still steps. With
62
+ * `smoothing > 0` the anchor runs a **One Euro filter** (the standard adaptive
63
+ * low-pass for interactive tracking): it smooths hard when the landmark moves
64
+ * slowly (kills jitter) and loosens as it speeds up (kills lag), and never
65
+ * extrapolates so it can't overshoot. Higher `smoothing` is snappier (less
66
+ * lag, slightly more jitter); lower is smoother; ~12-20 is a good range. It
67
+ * snaps (resets the filter) on first placement and on re-acquire after the
68
+ * face/hand was lost, so it never slides in from a stale position. `0`
69
+ * (default) keeps the raw snap-every-frame behavior. Rotation is not smoothed.
70
+ */
71
+ smoothing?: number;
72
+ };
73
+ export declare const TrackingAnchor: ({ target, offset, smoothing, children, }: TrackingAnchorProps) => import("react/jsx-runtime").JSX.Element | null;
74
+ export {};
@@ -0,0 +1,38 @@
1
+ import { Texture } from 'three';
2
+ import { SegmentationType } from '../../types/segmentation';
3
+ export type VideoBackgroundProps = {
4
+ /**
5
+ * Optional segmentation mask texture.
6
+ * When provided, the person area becomes transparent (alpha=0)
7
+ * and the background area remains opaque (alpha=255).
8
+ *
9
+ * Prefer the higher-level `segmentation` prop, which auto-resolves
10
+ * the mask from a `<Segmentation>` component mounted in the scene.
11
+ * `segmentationMask` is retained for advanced/imperative use.
12
+ */
13
+ segmentationMask?: Texture;
14
+ /**
15
+ * Declarative segmentation type to consume. When set, `VideoBackground`
16
+ * automatically resolves the matching mask texture from the xr-sdk
17
+ * runtime — there must be a `<Segmentation type="…">` component of the
18
+ * same type mounted somewhere in the scene tree.
19
+ */
20
+ segmentation?: SegmentationType;
21
+ /**
22
+ * When true, inverts the segmentation mask so the person area is
23
+ * replaced by the custom background color while the real-world
24
+ * background remains visible from the camera feed.
25
+ */
26
+ invert?: boolean;
27
+ /**
28
+ * Optional custom background texture or color.
29
+ * Shows through the transparent person area when segmentation is enabled.
30
+ */
31
+ customBackground?: Texture | string;
32
+ /**
33
+ * Render order for the video background layer.
34
+ * Default: `RENDER_LAYERS.BACKGROUND` (`-1000`).
35
+ */
36
+ renderOrder?: number;
37
+ };
38
+ export declare const VideoBackground: ({ segmentationMask, segmentation, invert, customBackground, renderOrder, }: VideoBackgroundProps) => import("react/jsx-runtime").JSX.Element | null;
@@ -0,0 +1,5 @@
1
+ import { CSSProperties, PropsWithChildren } from 'react';
2
+ export declare const XRCompositor: ({ background, children, style, }: PropsWithChildren<{
3
+ background?: CSSProperties["background"];
4
+ style?: CSSProperties;
5
+ }>) => import("react/jsx-runtime").JSX.Element;
@@ -0,0 +1,6 @@
1
+ import { PropsWithChildren } from 'react';
2
+ export declare const XRDebugView: ({ children, defaultCollapsed, title, width, }: PropsWithChildren<{
3
+ defaultCollapsed?: boolean;
4
+ title: string;
5
+ width?: number;
6
+ }>) => import("react/jsx-runtime").JSX.Element;
@@ -0,0 +1,6 @@
1
+ import { PropsWithChildren } from 'react';
2
+ import { XRLayerConfig } from '../../types/compositor';
3
+ export declare const XRLayer: ({ children, config, style, }: PropsWithChildren<{
4
+ config: XRLayerConfig;
5
+ style?: React.CSSProperties;
6
+ }>) => import("react/jsx-runtime").JSX.Element;
@@ -0,0 +1,27 @@
1
+ import { RefObject } from 'react';
2
+ import { XRLayerConfig, XRMediaLayout, XRMediaSourceKind } from '../../types/compositor';
3
+ export declare const XRMediaSurface: ({ canvasRef, imageRef, layer, layout, mirrored, sourceKind, videoRef, }: {
4
+ canvasRef: RefObject<HTMLCanvasElement>;
5
+ imageRef: RefObject<HTMLImageElement>;
6
+ layer: XRLayerConfig;
7
+ layout: XRMediaLayout | null;
8
+ mirrored: boolean;
9
+ sourceKind: XRMediaSourceKind;
10
+ videoRef: RefObject<HTMLVideoElement>;
11
+ }) => import("react/jsx-runtime").JSX.Element;
12
+ export declare const XRTrackingOverlay: ({ canvasRef, controlInsetTop, debugVisible, fps, imageRef, layer, layout, mirrored, sourceKind, videoRef, viewportSize, }: {
13
+ canvasRef: RefObject<HTMLCanvasElement>;
14
+ controlInsetTop?: number;
15
+ debugVisible: boolean;
16
+ fps?: number;
17
+ imageRef: RefObject<HTMLImageElement>;
18
+ layer: XRLayerConfig;
19
+ layout: XRMediaLayout | null;
20
+ mirrored: boolean;
21
+ sourceKind: XRMediaSourceKind;
22
+ videoRef: RefObject<HTMLVideoElement>;
23
+ viewportSize: {
24
+ width: number;
25
+ height: number;
26
+ };
27
+ }) => import("react/jsx-runtime").JSX.Element;
@@ -0,0 +1,13 @@
1
+ import { PropsWithChildren } from 'react';
2
+ import { SessionFacade } from '../../hooks/useXRSession';
3
+ import { XRPerfConfig } from '../../types';
4
+ export type XRContextType = {
5
+ session: SessionFacade;
6
+ };
7
+ export declare const useXRContext: () => XRContextType;
8
+ export type XRProviderProps = PropsWithChildren<{
9
+ /** Performance config: cap ML inference fps and camera resolution to reduce
10
+ * GPU/CPU load and heat. See {@link XRPerfConfig}. */
11
+ perf?: XRPerfConfig;
12
+ }>;
13
+ export declare const XRProvider: ({ children, perf }: XRProviderProps) => import("react/jsx-runtime").JSX.Element;
@@ -0,0 +1,4 @@
1
+ export declare const DefaultLoadingComponent: () => import("react/jsx-runtime").JSX.Element;
2
+ export declare const DefaultErrorComponent: (props: {
3
+ error: string;
4
+ }) => import("react/jsx-runtime").JSX.Element;
@@ -0,0 +1,15 @@
1
+ import { CanvasProps } from '@react-three/fiber';
2
+ import { default as React, PropsWithChildren } from 'react';
3
+ type XRSceneProps = PropsWithChildren & {
4
+ style?: React.CSSProperties;
5
+ canvasProps?: CanvasProps & React.RefAttributes<HTMLCanvasElement>;
6
+ loadingComponent?: React.ReactNode;
7
+ loadingTransitionDuration?: number;
8
+ errorComponent?: React.ReactNode;
9
+ renderTargetSize?: {
10
+ width: number;
11
+ height: number;
12
+ };
13
+ };
14
+ export declare const XRScene: ({ style, canvasProps, children, loadingComponent, loadingTransitionDuration, errorComponent, renderTargetSize, }: XRSceneProps) => import("react/jsx-runtime").JSX.Element;
15
+ export {};
@@ -0,0 +1,101 @@
1
+ import { CSSProperties, ReactNode } from 'react';
2
+ import { CalibrationMediaSourceKind } from '../../hooks/useCalibrationFaceTracker';
3
+ import { XRMediaLayout } from '../../types/compositor';
4
+ import { FaceTrackingResult } from '../../types/face';
5
+ import { XRModelNodeFaceTracking } from '../../types/nodes';
6
+ export type XRViewportMode = "dom" | "r3f" | "hybrid";
7
+ export type XRViewportSourceKind = "webcam" | "image" | "video";
8
+ export type XRViewportFitMode = "auto" | "cover" | "contain";
9
+ export type XRViewportMediaStrategy = "internal" | "external";
10
+ export type XRViewportMediaAssetOption = {
11
+ id: string;
12
+ label: string;
13
+ src: string;
14
+ };
15
+ export type XRViewportSourceConfig = {
16
+ kind: "webcam";
17
+ } | {
18
+ kind: "image";
19
+ assetId?: string;
20
+ } | {
21
+ kind: "video";
22
+ assetId?: string;
23
+ };
24
+ export type XRViewportSize = {
25
+ width: number;
26
+ height: number;
27
+ };
28
+ export type XRViewportRenderState = {
29
+ activeDelegate: "GPU" | "CPU" | null;
30
+ faces: FaceTrackingResult[];
31
+ imageOptions: XRViewportMediaAssetOption[];
32
+ layout: XRMediaLayout | null;
33
+ mediaError: string | null;
34
+ mediaFitMode: Exclude<XRViewportFitMode, "auto">;
35
+ mediaLabel: string;
36
+ mediaStatus: "idle" | "requesting" | "ready" | "error";
37
+ mirrored: boolean;
38
+ selectedImageId: string;
39
+ selectedSource: XRViewportSourceKind;
40
+ selectedVideoId: string;
41
+ setMirrored: (value: boolean) => void;
42
+ setSelectedImageId: (value: string) => void;
43
+ setSelectedSource: (value: XRViewportSourceKind) => void;
44
+ setSelectedVideoId: (value: string) => void;
45
+ setShowMediaControls: (value: boolean) => void;
46
+ showMediaControls: boolean;
47
+ trackerError: string | null;
48
+ trackerStatus: "idle" | "initializing" | "ready" | "error";
49
+ videoOptions: XRViewportMediaAssetOption[];
50
+ viewportSize: XRViewportSize;
51
+ mediaSize: XRViewportSize;
52
+ };
53
+ type XRViewportProps = {
54
+ children?: ReactNode | ((state: XRViewportRenderState) => ReactNode);
55
+ fitMode?: XRViewportFitMode;
56
+ imageOptions?: XRViewportMediaAssetOption[];
57
+ initialImageId?: string;
58
+ initialMirrored?: boolean;
59
+ initialSource?: XRViewportSourceKind;
60
+ initialVideoId?: string;
61
+ mediaStrategy?: XRViewportMediaStrategy;
62
+ mediaDock?: "top-left" | "bottom-left";
63
+ mode?: XRViewportMode;
64
+ showMediaControlsByDefault?: boolean;
65
+ source?: XRViewportSourceConfig;
66
+ style?: CSSProperties;
67
+ videoOptions?: XRViewportMediaAssetOption[];
68
+ };
69
+ type XRViewportSlotProps = {
70
+ children?: ReactNode | ((state: XRViewportRenderState) => ReactNode);
71
+ };
72
+ type XRViewportComponent = ((props: XRViewportProps) => JSX.Element) & {
73
+ Scene: (props: XRViewportSlotProps) => JSX.Element;
74
+ Overlay: (props: XRViewportSlotProps) => JSX.Element;
75
+ };
76
+ export declare const resolveViewportFitMode: (fitMode: XRViewportFitMode, selectedSource: XRViewportSourceKind) => Exclude<XRViewportFitMode, "auto">;
77
+ export declare const resolveViewportSourceSelection: ({ imageOptions, initialImageId, initialSource, initialVideoId, source, videoOptions, }: {
78
+ imageOptions: XRViewportMediaAssetOption[];
79
+ initialImageId?: string;
80
+ initialSource?: XRViewportSourceKind;
81
+ initialVideoId?: string;
82
+ source?: XRViewportSourceConfig;
83
+ videoOptions: XRViewportMediaAssetOption[];
84
+ }) => {
85
+ selectedSource: XRViewportSourceKind;
86
+ selectedImageId: string;
87
+ selectedVideoId: string;
88
+ };
89
+ export declare const computeViewportLayout: (source: XRViewportSize, destination: XRViewportSize, fitMode: Exclude<XRViewportFitMode, "auto">) => XRMediaLayout | null;
90
+ export declare const resolveViewportSlots: (mode: XRViewportMode, children: ReactNode) => {
91
+ sceneContent: ReactNode;
92
+ overlayContent: ReactNode;
93
+ };
94
+ export declare const resolveViewportSlotContent: (content: ReactNode | ((state: XRViewportRenderState) => ReactNode), state: XRViewportRenderState) => ReactNode;
95
+ export declare const resolveViewportExternalFaces: ({ detectionResults, selectedSource, }: {
96
+ detectionResults: ReturnType<XRModelNodeFaceTracking["getDetectionResults"]> | null | undefined;
97
+ selectedSource: CalibrationMediaSourceKind;
98
+ }) => FaceTrackingResult[];
99
+ export declare const useXRViewportState: () => XRViewportRenderState | null;
100
+ export declare const XRViewport: XRViewportComponent;
101
+ export {};
@@ -0,0 +1,26 @@
1
+ export * from './AspectRatioContainer';
2
+ export * from './BodyTracker';
3
+ export * from './FaceTracker';
4
+ export * from './FaceMesh';
5
+ export * from './FaceRetouch';
6
+ export * from './GestureTracker';
7
+ export * from './GestureTrigger';
8
+ export * from './HandTracker';
9
+ export * from './HeadBinding';
10
+ export * from './HeadBindingOccluder';
11
+ export * from './HeadOccluder';
12
+ export * from './Panel';
13
+ export * from './PerspectiveCamera';
14
+ export * from './ScreenSpaceUI';
15
+ export * from './Segmentation';
16
+ export * from './SpriteAnimation';
17
+ export * from './TextLabel';
18
+ export * from './TrackingAnchor';
19
+ export * from './VideoBackground';
20
+ export * from './XRProvider';
21
+ export * from './XRCompositor';
22
+ export * from './XRDebugView';
23
+ export * from './XRLayer';
24
+ export * from './XRMediaSurface';
25
+ export * from './XRViewport';
26
+ export * from './XRScene';
@@ -0,0 +1,8 @@
1
+ export declare const libConfig: {
2
+ apiBaseUrl: string;
3
+ assetStorageEndpoint: string;
4
+ sdkAssetsBaseUrl: string;
5
+ loggerPrefix: string;
6
+ sessionCheckInterval: number;
7
+ };
8
+ export type LibConfig = typeof libConfig;
@@ -0,0 +1,25 @@
1
+ export declare enum XRMediaSource {
2
+ WEBCAM = "webcam",
3
+ STREAM = "stream",
4
+ CANVAS = "canvas"
5
+ }
6
+ export declare enum XRModel {
7
+ HAND_TRACKER = "vincentt.xr.hand-tracker",
8
+ GESTURE_TRACKER = "vincentt.xr.gesture-tracker",
9
+ FACE_TRACKER = "vincentt.xr.face-tracker",
10
+ BODY_TRACKER = "vincentt.xr.body-tracker",
11
+ BACKGROUND_SEGMENTATION = "vincentt.xr.background-segmentation"
12
+ }
13
+ export declare enum XRComponentEnum {
14
+ CAMERA = "vincentt.component.camera",
15
+ FACE_GEOMETRY = "vincentt.component.face-geometry"
16
+ }
17
+ export declare const SDKEvents: {
18
+ session: {
19
+ started: string;
20
+ stopped: string;
21
+ readiness: string;
22
+ startProgress: string;
23
+ error: string;
24
+ };
25
+ };
@@ -0,0 +1,25 @@
1
+ /**
2
+ * Fixed render-layer convention for AR scenes.
3
+ *
4
+ * The AR element model defines four conceptual layers stacked from back
5
+ * to front. Each container component sets `renderOrder` (and the
6
+ * appropriate `material-depthTest`) on its descendants so the design
7
+ * agent and creators do not need to think about Z order.
8
+ *
9
+ * Within a single layer, sibling order in the scene tree determines
10
+ * draw order — later siblings draw on top of earlier siblings.
11
+ *
12
+ * Children may override the container default by passing an explicit
13
+ * `renderOrder` prop, which takes precedence via R3F prop merging.
14
+ */
15
+ export declare const RENDER_LAYERS: {
16
+ /** Camera background and segmentation mask. `depthTest` disabled. */
17
+ readonly BACKGROUND: -1000;
18
+ /** Free world-space content. `depthTest` enabled. */
19
+ readonly WORLD: 0;
20
+ /** Descendants of `FaceTracker` / `HandTracker` / `BodyTracker`. `depthTest` enabled. */
21
+ readonly TRACKED: 100;
22
+ /** Descendants of `ScreenSpaceUI`. `depthTest` disabled. */
23
+ readonly SCREEN_UI: 1000;
24
+ };
25
+ export type RenderLayer = (typeof RENDER_LAYERS)[keyof typeof RENDER_LAYERS];
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Aggregate of every domain — NOT a package export.
3
+ *
4
+ * This exists only so the internal example playground can keep resolving a
5
+ * flat `@vincentt-xr/sdk` alias without editing its demo files. It is
6
+ * never listed in `package.json` `exports`; real apps import from the split
7
+ * subpaths (`.`, `/tracking`, `/face-effects`, `/low-level`). `/internal` is
8
+ * included here because the playground demos the debugger — which is exactly
9
+ * why this aggregate is example-only and never shipped.
10
+ */
11
+ export * from './core';
12
+ export * from './tracking';
13
+ export * from './face-effects';
14
+ export * from './low-level';
15
+ export * from './internal';
@@ -0,0 +1,21 @@
1
+ /**
2
+ * `@vincentt-xr/sdk` (core) — the always-there surface.
3
+ *
4
+ * Session root, scene, screen-space layout, and content primitives — the
5
+ * frame every AR app has, with no tracker and no effect. Task-specific
6
+ * surface lives behind subpaths: `/tracking`, `/face-effects`, `/low-level`.
7
+ */
8
+ export { XRProvider, XRScene, ScreenSpaceUI, ScreenTransform, useScreenRectAspect, TextLabel, Panel, VideoBackground, AspectRatioContainer, SpriteAnimation, } from '../components';
9
+ export type { PanelProps } from '../components/Panel';
10
+ export type { TextLabelProps } from '../components/TextLabel';
11
+ export type { ScreenTransformProps } from '../components/ScreenSpaceUI';
12
+ export type { SpriteSheet, SpriteAnimationProps } from '../components/SpriteAnimation';
13
+ export { computeContainScale } from '../texture/computeContainScale';
14
+ export { useMediaRecorder } from '../hooks/useMediaRecorder';
15
+ export type { UseMediaRecorderOptions } from '../hooks/useMediaRecorder';
16
+ export { dataURLtoFile } from '../utils/dataURL';
17
+ export { AudioEngine, audio, prefetchMusic, registerSound, startMusic, stopMusic, playOneShot, setMuted, toggleMuted, isMuted, onMuteChange, useMuted, } from '../audio';
18
+ export type { AudioEngineOptions } from '../audio';
19
+ export { XRMediaSource, XRModel } from '../constants';
20
+ export type { XRSession } from '../types';
21
+ export type { XRPerfConfig } from '../types/model';
@@ -0,0 +1,19 @@
1
+ /**
2
+ * `@vincentt-xr/sdk/face-effects` — face-filter power-user surface.
3
+ *
4
+ * FaceMesh material configuration, landmark-driven retouch, the canonical
5
+ * face mesh, and the head-binding solver + occluders. The bare
6
+ * `<FaceMesh appearance={{ map }} />` is in `/tracking`; this is its deep
7
+ * configuration.
8
+ */
9
+ export { blendModeIndices, DEFAULT_APPEARANCE, DEFAULT_MASKING, materialTypeOptions, shaderPresetOptions, sideOptions, textureMapOptions, alphaMapOptions, blendModeOptions, supportsTexturePanel, supportsBlendMode, } from '../components/FaceMesh';
10
+ export type { FaceMeshMaterialType, FaceMeshShaderPreset, FaceMeshTextureMap, FaceMeshAlphaMap, FaceMeshMaskTextureSource, FaceMeshSide, FaceMeshBlendMode, FaceMeshAppearance, FaceMeshMasking, } from '../components/FaceMesh';
11
+ export { CanonicalFaceMeshRenderer, useCanonicalFaceMesh, parseCanonicalFaceModel, } from '../components';
12
+ export type { CanonicalFaceMeshRendererProps, ParsedCanonicalMesh } from '../components/FaceMesh';
13
+ export { FaceRetouchBackground, drawFaceRetouchMask, drawMouthRetouchMask, FACE_MASK_SIZE, FACE_OVAL, LEFT_EYE, RIGHT_EYE, OUTER_LIPS, INNER_MOUTH, LEFT_EYEBROW, RIGHT_EYEBROW, createFaceRetouchRegions, DEFAULT_FEATURE_GROUPS, DEFAULT_FEATURE_POINT_INDICES, useFaceRetouchMask, useFaceRetouchRegions, } from '../components/FaceRetouch';
14
+ export type { FaceRetouchBackgroundProps, CreateFaceRetouchRegionsOptions, } from '../components/FaceRetouch';
15
+ export { HeadBinding, HeadBindingOccluder, HeadOccluder, HEAD_OCCLUDER_DEFAULT_OPACITY, HEAD_OCCLUDER_DEFAULT_POSITION, HEAD_OCCLUDER_DEFAULT_SCALE, } from '../components';
16
+ export type { HeadBindingOccluderOptions, HeadBindingProps } from '../components';
17
+ export { computeAdaptiveVerticalFit, computeHeadBindingCubePose, computeHeadBindingDebugMetrics, computeHeadBindingShape, computeUniversalHeadDepthOffset, computeUniversalOccluderScale, computeUniversalPitchOffsetY, computeUniversalYawOffsetX, solveHeadBinding, } from '../utils/headBinding';
18
+ export type { HeadBindingAdaptiveValues, HeadBindingCubeModel, HeadBindingCubePose, HeadBindingDebugMetrics, HeadBindingSolveResult, HeadBindingShape, } from '../types/headBinding';
19
+ export type { FaceLandmark2D, BoundingBox, FaceRawTrackingData, FaceRetouchRegion, FaceBounds, FaceTrackingResult, } from '../types/face';
@@ -0,0 +1,16 @@
1
+ /**
2
+ * `@vincentt-xr/sdk/internal` — not for app authors.
3
+ *
4
+ * Debugger plumbing, private helpers, calibration-debug utilities, and
5
+ * low-level compositor/render types. Present for the SDK's own tooling and
6
+ * the editor; app code should not import from here.
7
+ */
8
+ export { XRDebugView } from '../components';
9
+ export { DebuggerProvider } from '../components/Debugger/DebuggerProvider';
10
+ export { DebuggerOverlay } from '../components/Debugger';
11
+ export { useDebugger } from '../hooks/useDebugger';
12
+ export type { DebuggerMetrics, DebuggerConfig } from '../types/debugger';
13
+ export { logUVBoundsWarningOnce, _resetUVBoundsWarnings } from '../texture/hasFullUnitUVBounds';
14
+ export { readCalibrationDebugEvents, recordCalibrationDebugEvent } from '../utils/calibrationDebug';
15
+ export { RENDER_LAYERS } from '../constants/renderLayers';
16
+ export type { RenderLayer } from '../constants/renderLayers';