@vincentt-xr/sdk 2.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GROUNDING.md +537 -0
- package/LICENSE +7 -0
- package/README.md +85 -0
- package/dist/DebuggerUI-CEfdddkZ.cjs +1 -0
- package/dist/DebuggerUI-Dgop6Q6o.js +269 -0
- package/dist/audio/AudioEngine.d.ts +98 -0
- package/dist/audio/index.d.ts +34 -0
- package/dist/components/AspectRatioContainer/index.d.ts +10 -0
- package/dist/components/BodyTracker/index.d.ts +15 -0
- package/dist/components/Debugger/DebuggerProvider.d.ts +22 -0
- package/dist/components/Debugger/DebuggerUI.d.ts +13 -0
- package/dist/components/Debugger/index.d.ts +38 -0
- package/dist/components/Debugger/styles.d.ts +19 -0
- package/dist/components/FaceMesh/FaceMesh.d.ts +26 -0
- package/dist/components/FaceMesh/canonical/CanonicalFaceMeshRenderer.d.ts +34 -0
- package/dist/components/FaceMesh/canonical/parseCanonicalFaceModel.d.ts +16 -0
- package/dist/components/FaceMesh/canonical/useCanonicalFaceMesh.d.ts +15 -0
- package/dist/components/FaceMesh/deform.d.ts +36 -0
- package/dist/components/FaceMesh/index.d.ts +6 -0
- package/dist/components/FaceMesh/indices.d.ts +1 -0
- package/dist/components/FaceMesh/material/blendModes.d.ts +2 -0
- package/dist/components/FaceMesh/material/defaults.d.ts +3 -0
- package/dist/components/FaceMesh/material/index.d.ts +5 -0
- package/dist/components/FaceMesh/material/options.d.ts +25 -0
- package/dist/components/FaceMesh/material/support.d.ts +3 -0
- package/dist/components/FaceMesh/material/types.d.ts +23 -0
- package/dist/components/FaceMesh/uv.d.ts +1 -0
- package/dist/components/FaceRetouch/Background.d.ts +16 -0
- package/dist/components/FaceRetouch/index.d.ts +6 -0
- package/dist/components/FaceRetouch/landmarkMask.d.ts +23 -0
- package/dist/components/FaceRetouch/retouchRegions.d.ts +23 -0
- package/dist/components/FaceRetouch/useFaceRetouchMask.d.ts +14 -0
- package/dist/components/FaceRetouch/useFaceRetouchRegions.d.ts +12 -0
- package/dist/components/FaceTracker/BoundingBoxOverlay.d.ts +20 -0
- package/dist/components/FaceTracker/index.d.ts +54 -0
- package/dist/components/GestureTracker/index.d.ts +31 -0
- package/dist/components/GestureTrigger/index.d.ts +3 -0
- package/dist/components/HandTracker/index.d.ts +24 -0
- package/dist/components/HeadBinding/index.d.ts +31 -0
- package/dist/components/HeadBindingOccluder/index.d.ts +36 -0
- package/dist/components/HeadOccluder/index.d.ts +13 -0
- package/dist/components/Panel/index.d.ts +71 -0
- package/dist/components/PerspectiveCamera/index.d.ts +14 -0
- package/dist/components/ScreenSpaceUI/ScreenTransform.d.ts +54 -0
- package/dist/components/ScreenSpaceUI/context.d.ts +29 -0
- package/dist/components/ScreenSpaceUI/index.d.ts +40 -0
- package/dist/components/Segmentation/index.d.ts +44 -0
- package/dist/components/SpriteAnimation/index.d.ts +52 -0
- package/dist/components/TextLabel/index.d.ts +62 -0
- package/dist/components/TrackingAnchor/index.d.ts +74 -0
- package/dist/components/VideoBackground/index.d.ts +38 -0
- package/dist/components/XRCompositor/index.d.ts +5 -0
- package/dist/components/XRDebugView/index.d.ts +6 -0
- package/dist/components/XRLayer/index.d.ts +6 -0
- package/dist/components/XRMediaSurface/index.d.ts +27 -0
- package/dist/components/XRProvider/index.d.ts +13 -0
- package/dist/components/XRScene/defaults.d.ts +4 -0
- package/dist/components/XRScene/index.d.ts +15 -0
- package/dist/components/XRViewport/index.d.ts +101 -0
- package/dist/components/index.d.ts +26 -0
- package/dist/config/index.d.ts +8 -0
- package/dist/constants/index.d.ts +25 -0
- package/dist/constants/renderLayers.d.ts +25 -0
- package/dist/entries/all.d.ts +15 -0
- package/dist/entries/core.d.ts +21 -0
- package/dist/entries/face-effects.d.ts +19 -0
- package/dist/entries/internal.d.ts +16 -0
- package/dist/entries/low-level.d.ts +30 -0
- package/dist/entries/tracking.d.ts +15 -0
- package/dist/face-effects.cjs +252 -0
- package/dist/face-effects.js +3542 -0
- package/dist/hasFullUnitUVBounds-BGvcFsQM.js +26 -0
- package/dist/hasFullUnitUVBounds-p1Dl0ejX.cjs +1 -0
- package/dist/hooks/faceResults.d.ts +39 -0
- package/dist/hooks/selectors.d.ts +33 -0
- package/dist/hooks/useCalibrationFaceTracker.d.ts +23 -0
- package/dist/hooks/useDebugger/index.d.ts +25 -0
- package/dist/hooks/useDebugger/types.d.ts +20 -0
- package/dist/hooks/useFaceBoundingBox.d.ts +11 -0
- package/dist/hooks/useFaceResults.d.ts +31 -0
- package/dist/hooks/useGestureEdge.d.ts +53 -0
- package/dist/hooks/useMediaRecorder.d.ts +40 -0
- package/dist/hooks/useTrackerContext.d.ts +35 -0
- package/dist/hooks/useXRModel.d.ts +15 -0
- package/dist/hooks/useXRSession/constants.d.ts +4 -0
- package/dist/hooks/useXRSession/facade.d.ts +39 -0
- package/dist/hooks/useXRSession/fns.d.ts +5 -0
- package/dist/hooks/useXRSession/index.d.ts +4 -0
- package/dist/hooks/useXRSession/registry.d.ts +52 -0
- package/dist/hooks/useXRSession/store.d.ts +40 -0
- package/dist/index-B5J4gzn2.cjs +1 -0
- package/dist/index-BD-JWMjE.js +1991 -0
- package/dist/index-BVBVg96G.cjs +1 -0
- package/dist/index-CYMj26Ej.js +1331 -0
- package/dist/index-DtwHElmG.js +157 -0
- package/dist/index-PtFmD6B8.cjs +1 -0
- package/dist/internal-DH5d1Xnq.cjs +1 -0
- package/dist/internal-KOKJDxCG.js +519 -0
- package/dist/internal.cjs +1 -0
- package/dist/internal.js +14 -0
- package/dist/low-level.cjs +1 -0
- package/dist/low-level.js +941 -0
- package/dist/main.cjs +1 -0
- package/dist/main.js +612 -0
- package/dist/models/Base/index.d.ts +25 -0
- package/dist/models/Body/index.d.ts +22 -0
- package/dist/models/Face/constants.d.ts +2 -0
- package/dist/models/Face/index.d.ts +45 -0
- package/dist/models/Face/landmarksData.d.ts +2 -0
- package/dist/models/Hand/GestureTracking.d.ts +21 -0
- package/dist/models/Hand/HandTracking.d.ts +21 -0
- package/dist/models/Segmentation/index.d.ts +75 -0
- package/dist/models/index.d.ts +5 -0
- package/dist/modules/XRClient/index.d.ts +9 -0
- package/dist/modules/index.d.ts +1 -0
- package/dist/selectors-BHO6TnoC.js +59 -0
- package/dist/selectors-Bub0o3qy.cjs +1 -0
- package/dist/services/ApiService.d.ts +12 -0
- package/dist/services/MetricsCollector/CameraFPSTracker.d.ts +25 -0
- package/dist/services/MetricsCollector/MemoryTracker.d.ts +43 -0
- package/dist/services/MetricsCollector/RenderTimeTracker.d.ts +10 -0
- package/dist/services/MetricsCollector/ResolutionTracker.d.ts +26 -0
- package/dist/services/MetricsCollector/index.d.ts +59 -0
- package/dist/services/MetricsCollector/types.d.ts +47 -0
- package/dist/services/MetricsCollector/utils.d.ts +65 -0
- package/dist/texture/computeContainScale.d.ts +25 -0
- package/dist/texture/hasFullUnitUVBounds.d.ts +37 -0
- package/dist/texture/types.d.ts +20 -0
- package/dist/tracking.cjs +1 -0
- package/dist/tracking.js +344 -0
- package/dist/types/body.d.ts +3 -0
- package/dist/types/client.d.ts +3 -0
- package/dist/types/common.d.ts +9 -0
- package/dist/types/compositor.d.ts +23 -0
- package/dist/types/debugger.d.ts +83 -0
- package/dist/types/face.d.ts +67 -0
- package/dist/types/gesture-tracking.d.ts +15 -0
- package/dist/types/hand-tracking.d.ts +5 -0
- package/dist/types/headBinding.d.ts +67 -0
- package/dist/types/index.d.ts +12 -0
- package/dist/types/logger.d.ts +4 -0
- package/dist/types/model.d.ts +46 -0
- package/dist/types/nodes.d.ts +35 -0
- package/dist/types/segmentation.d.ts +31 -0
- package/dist/types/session.d.ts +102 -0
- package/dist/types/tracking.d.ts +30 -0
- package/dist/types/util-types.d.ts +1 -0
- package/dist/useFaceResults-cpsSmKKc.js +81 -0
- package/dist/useFaceResults-hVUJYSEY.cjs +1 -0
- package/dist/utils/calibrationDebug.d.ts +9 -0
- package/dist/utils/dataURL.d.ts +6 -0
- package/dist/utils/drawMediaCover.d.ts +18 -0
- package/dist/utils/errors.d.ts +3 -0
- package/dist/utils/fns.d.ts +48 -0
- package/dist/utils/headBinding.d.ts +85 -0
- package/dist/utils/integrity.d.ts +43 -0
- package/dist/utils/landmarkBounds.d.ts +11 -0
- package/dist/utils/logger.d.ts +7 -0
- package/package.json +113 -0
|
@@ -0,0 +1,1331 @@
|
|
|
1
|
+
var fe = Object.defineProperty;
|
|
2
|
+
var pe = (i, s, e) => s in i ? fe(i, s, { enumerable: !0, configurable: !0, writable: !0, value: e }) : i[s] = e;
|
|
3
|
+
var h = (i, s, e) => pe(i, typeof s != "symbol" ? s + "" : s, e);
|
|
4
|
+
import { jsx as we } from "react/jsx-runtime";
|
|
5
|
+
import { useMemo as ee, useEffect as ve, createContext as Ce, useContext as Se } from "react";
|
|
6
|
+
import { FilesetResolver as U, PoseLandmarker as ke, FaceLandmarker as ye, GestureRecognizer as Me, HandLandmarker as Te, ImageSegmenter as Re, DrawingUtils as Fe } from "@mediapipe/tasks-vision";
|
|
7
|
+
import * as T from "three";
|
|
8
|
+
import { RGBAFormat as te, LinearFilter as B, CanvasTexture as se, SRGBColorSpace as xe } from "three";
|
|
9
|
+
var y = /* @__PURE__ */ ((i) => (i.WEBCAM = "webcam", i.STREAM = "stream", i.CANVAS = "canvas", i))(y || {}), _ = /* @__PURE__ */ ((i) => (i.HAND_TRACKER = "vincentt.xr.hand-tracker", i.GESTURE_TRACKER = "vincentt.xr.gesture-tracker", i.FACE_TRACKER = "vincentt.xr.face-tracker", i.BODY_TRACKER = "vincentt.xr.body-tracker", i.BACKGROUND_SEGMENTATION = "vincentt.xr.background-segmentation", i))(_ || {});
|
|
10
|
+
const ft = {
|
|
11
|
+
/** Camera background and segmentation mask. `depthTest` disabled. */
|
|
12
|
+
BACKGROUND: -1e3,
|
|
13
|
+
/** Free world-space content. `depthTest` enabled. */
|
|
14
|
+
WORLD: 0,
|
|
15
|
+
/** Descendants of `FaceTracker` / `HandTracker` / `BodyTracker`. `depthTest` enabled. */
|
|
16
|
+
TRACKED: 100,
|
|
17
|
+
/** Descendants of `ScreenSpaceUI`. `depthTest` disabled. */
|
|
18
|
+
SCREEN_UI: 1e3
|
|
19
|
+
};
|
|
20
|
+
class F extends Error {
|
|
21
|
+
constructor(s) {
|
|
22
|
+
super(s), this.name = "VincenttXRError";
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
const x = {
|
|
26
|
+
// Pinned to the CDN, NOT build-env-driven: a stale VITE_ASSET_STORAGE_ENDPOINT
|
|
27
|
+
// baked a now-deleted CloudFront domain into every published bundle, breaking
|
|
28
|
+
// WASM/model loads at runtime. The host must never come from a mutable env value.
|
|
29
|
+
assetStorageEndpoint: "https://cdn.vincentt.studio",
|
|
30
|
+
// Versioned bucket for SDK-shipped reference assets (canonical face model, face textures, etc.).
|
|
31
|
+
sdkAssetsBaseUrl: "https://cdn.vincentt.studio/assets/sdk/face/v1",
|
|
32
|
+
loggerPrefix: "[vincentt-xr]:"
|
|
33
|
+
};
|
|
34
|
+
class Ae {
|
|
35
|
+
constructor(s) {
|
|
36
|
+
h(this, "options");
|
|
37
|
+
const e = {
|
|
38
|
+
verbose: !1,
|
|
39
|
+
prefix: x.loggerPrefix
|
|
40
|
+
};
|
|
41
|
+
this.options = { ...e, ...s };
|
|
42
|
+
}
|
|
43
|
+
log(...s) {
|
|
44
|
+
const { verbose: e, prefix: t } = this.options;
|
|
45
|
+
e && console.log(t, ...s);
|
|
46
|
+
}
|
|
47
|
+
error(...s) {
|
|
48
|
+
const { verbose: e, prefix: t } = this.options;
|
|
49
|
+
e && console.error(t, ...s);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const D = (i) => i * 2 - 1, K = new T.Vector3(), W = new T.Vector3(), V = ({ x: i, y: s, z: e }, t, n) => {
|
|
53
|
+
K.set((i - 0.5) * 2, (0.5 - s) * 2, -e), K.unproject(t), W.copy(K).sub(t.position).normalize();
|
|
54
|
+
const r = -t.position.z / W.z;
|
|
55
|
+
return n.copy(t.position).addScaledVector(W, r), n;
|
|
56
|
+
}, pt = (i, s) => V(i, s, new T.Vector3()), Ee = (i, s, e = 0.2) => e * s + (1 - e) * i;
|
|
57
|
+
class G {
|
|
58
|
+
constructor(s) {
|
|
59
|
+
h(this, "canvas");
|
|
60
|
+
this.maxDimension = s;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Returns a downscaled copy of `source` capped to `maxDimension` on its
|
|
64
|
+
* longest side (or `source` unchanged if it's already within the cap, or if a
|
|
65
|
+
* 2D context can't be obtained). The returned canvas is reused across calls.
|
|
66
|
+
*/
|
|
67
|
+
draw(s) {
|
|
68
|
+
const e = s.width, t = s.height, n = Math.min(1, this.maxDimension / Math.max(e, t));
|
|
69
|
+
if (n >= 1) return s;
|
|
70
|
+
const r = Math.max(1, Math.round(e * n)), c = Math.max(1, Math.round(t * n));
|
|
71
|
+
(!this.canvas || this.canvas.width !== r || this.canvas.height !== c) && (this.canvas = new OffscreenCanvas(r, c));
|
|
72
|
+
const d = this.canvas.getContext("2d");
|
|
73
|
+
return d ? (d.setTransform(1, 0, 0, 1, 0, 0), d.clearRect(0, 0, r, c), d.drawImage(s, 0, 0, r, c), this.canvas) : s;
|
|
74
|
+
}
|
|
75
|
+
dispose() {
|
|
76
|
+
this.canvas = void 0;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const be = {};
|
|
80
|
+
async function Ie(i) {
|
|
81
|
+
const s = await fetch(i);
|
|
82
|
+
if (!s.ok)
|
|
83
|
+
throw new Error(`fetchAndVerify: ${s.status} ${i}`);
|
|
84
|
+
const e = await s.arrayBuffer(), t = De(i);
|
|
85
|
+
if (!t)
|
|
86
|
+
return console.warn(`[xr-sdk integrity] no SRI hash for ${i} — passing through`), e;
|
|
87
|
+
const n = await _e(e);
|
|
88
|
+
if (n !== t)
|
|
89
|
+
throw new Error(
|
|
90
|
+
`[xr-sdk integrity] SRI mismatch for ${i}: expected ${t}, got ${n}`
|
|
91
|
+
);
|
|
92
|
+
return e;
|
|
93
|
+
}
|
|
94
|
+
async function z(i) {
|
|
95
|
+
const s = await Ie(i);
|
|
96
|
+
return new Uint8Array(s);
|
|
97
|
+
}
|
|
98
|
+
function De(i) {
|
|
99
|
+
try {
|
|
100
|
+
const s = new URL(i);
|
|
101
|
+
return be[s.pathname];
|
|
102
|
+
} catch {
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async function _e(i) {
|
|
107
|
+
const s = await crypto.subtle.digest("SHA-384", i), e = new Uint8Array(s);
|
|
108
|
+
let t = "";
|
|
109
|
+
for (let n = 0; n < e.byteLength; n++)
|
|
110
|
+
t += String.fromCharCode(e[n]);
|
|
111
|
+
return `sha384-${btoa(t)}`;
|
|
112
|
+
}
|
|
113
|
+
class L {
|
|
114
|
+
constructor() {
|
|
115
|
+
h(this, "disposed", !1);
|
|
116
|
+
h(this, "paused", !1);
|
|
117
|
+
h(this, "rafId", null);
|
|
118
|
+
// Inference-rate throttle. `targetFrameIntervalMs` is 0 when uncapped
|
|
119
|
+
// (detect every frame); a positive value skips detection until that many ms
|
|
120
|
+
// have elapsed since the last inference. `run()` sets this from
|
|
121
|
+
// `params.targetFps`; subclasses call `shouldRunInference()` in their RAF
|
|
122
|
+
// loop to gate the expensive MediaPipe call.
|
|
123
|
+
h(this, "targetFrameIntervalMs", 0);
|
|
124
|
+
h(this, "lastInferenceMs", 0);
|
|
125
|
+
}
|
|
126
|
+
/** Set the per-frame inference cap. fps <= 0 (or undefined) → uncapped. */
|
|
127
|
+
setTargetFps(s) {
|
|
128
|
+
this.targetFrameIntervalMs = s && s > 0 ? 1e3 / s : 0;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* True if enough time has elapsed to run inference this frame (and records
|
|
132
|
+
* the timestamp). Always true when uncapped. Call once per RAF tick, right
|
|
133
|
+
* before the detect call; skip the detect (but keep rescheduling RAF) when
|
|
134
|
+
* it returns false.
|
|
135
|
+
*/
|
|
136
|
+
shouldRunInference() {
|
|
137
|
+
if (this.targetFrameIntervalMs <= 0) return !0;
|
|
138
|
+
const s = performance.now();
|
|
139
|
+
return s - this.lastInferenceMs < this.targetFrameIntervalMs ? !1 : (this.lastInferenceMs = s, !0);
|
|
140
|
+
}
|
|
141
|
+
// load models
|
|
142
|
+
async initialize(s) {
|
|
143
|
+
}
|
|
144
|
+
// setup for running + start running
|
|
145
|
+
async run(s) {
|
|
146
|
+
}
|
|
147
|
+
// get model's data while processing
|
|
148
|
+
getModelNode() {
|
|
149
|
+
return {};
|
|
150
|
+
}
|
|
151
|
+
async getPerformanceData() {
|
|
152
|
+
return {
|
|
153
|
+
name: this.constructor.name,
|
|
154
|
+
processingTime: null
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
// Cancel the per-frame RAF loop without releasing MediaPipe resources or
|
|
158
|
+
// dropping refs. Subclasses' process* methods check `paused` and return
|
|
159
|
+
// without rescheduling. Idempotent.
|
|
160
|
+
pause() {
|
|
161
|
+
this.paused = !0, this.rafId !== null && (cancelAnimationFrame(this.rafId), this.rafId = null);
|
|
162
|
+
}
|
|
163
|
+
// Resume a paused loop by running the process function once. Subclasses
|
|
164
|
+
// override to kick their specific process* method. Idempotent — no-op if
|
|
165
|
+
// not paused. Calling resume() before run() is a no-op.
|
|
166
|
+
resume() {
|
|
167
|
+
this.paused = !1;
|
|
168
|
+
}
|
|
169
|
+
// Release MediaPipe landmarkers, cancel rAF loops, drop refs.
|
|
170
|
+
// Subclasses MUST override and call super.dispose() at the end.
|
|
171
|
+
dispose() {
|
|
172
|
+
this.disposed = !0, this.rafId !== null && (cancelAnimationFrame(this.rafId), this.rafId = null);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
const Pe = 768;
|
|
176
|
+
class Oe extends L {
|
|
177
|
+
constructor() {
|
|
178
|
+
super();
|
|
179
|
+
h(this, "landmarker", null);
|
|
180
|
+
h(this, "processingTime", null);
|
|
181
|
+
h(this, "coordinates");
|
|
182
|
+
h(this, "sourceCanvas");
|
|
183
|
+
h(this, "downscaler", new G(Pe));
|
|
184
|
+
}
|
|
185
|
+
async getBodyData() {
|
|
186
|
+
if (!this.landmarker || !this.sourceCanvas)
|
|
187
|
+
return;
|
|
188
|
+
const e = this.downscaler.draw(this.sourceCanvas), t = performance.now(), n = this.landmarker.detectForVideo(e, t), r = performance.now();
|
|
189
|
+
this.processingTime = r - t;
|
|
190
|
+
const c = n.landmarks, d = [];
|
|
191
|
+
if (c)
|
|
192
|
+
return c.forEach((m, u) => {
|
|
193
|
+
d[u] || (d[u] = []), m.forEach((g) => {
|
|
194
|
+
const f = D(g.x), w = D(1 - g.y), p = D(g.z);
|
|
195
|
+
d[u].push({ x: f, y: w, z: p });
|
|
196
|
+
});
|
|
197
|
+
}), {
|
|
198
|
+
coordinatesList: d
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
async processBodyTracking() {
|
|
202
|
+
if (this.disposed || !this.landmarker)
|
|
203
|
+
return;
|
|
204
|
+
const e = await this.getBodyData();
|
|
205
|
+
e && (this.coordinates = e.coordinatesList), !(this.disposed || this.paused) && (this.rafId = requestAnimationFrame(this.processBodyTracking.bind(this)));
|
|
206
|
+
}
|
|
207
|
+
resume() {
|
|
208
|
+
this.paused && (super.resume(), this.processBodyTracking());
|
|
209
|
+
}
|
|
210
|
+
dispose() {
|
|
211
|
+
if (super.dispose(), this.landmarker) {
|
|
212
|
+
try {
|
|
213
|
+
this.landmarker.close();
|
|
214
|
+
} catch {
|
|
215
|
+
}
|
|
216
|
+
this.landmarker = null;
|
|
217
|
+
}
|
|
218
|
+
this.sourceCanvas = void 0, this.downscaler.dispose(), this.coordinates = void 0;
|
|
219
|
+
}
|
|
220
|
+
getLandmarks2D(e = 0) {
|
|
221
|
+
if (this.coordinates)
|
|
222
|
+
return this.coordinates[e];
|
|
223
|
+
}
|
|
224
|
+
async initialize(e) {
|
|
225
|
+
const t = await U.forVisionTasks(
|
|
226
|
+
new URL("/assets/xr/wasm", x.assetStorageEndpoint).toString()
|
|
227
|
+
), n = await z(
|
|
228
|
+
new URL("/assets/xr/models/body_tracking.task", x.assetStorageEndpoint).toString()
|
|
229
|
+
);
|
|
230
|
+
this.landmarker = await ke.createFromOptions(t, {
|
|
231
|
+
baseOptions: {
|
|
232
|
+
modelAssetBuffer: n,
|
|
233
|
+
delegate: "GPU"
|
|
234
|
+
},
|
|
235
|
+
runningMode: "VIDEO",
|
|
236
|
+
numPoses: (e == null ? void 0 : e.numPoses) || 1
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
async run({ sourceCanvas: e }) {
|
|
240
|
+
this.sourceCanvas = e, this.rafId === null && await this.processBodyTracking();
|
|
241
|
+
}
|
|
242
|
+
getModelNode() {
|
|
243
|
+
return {
|
|
244
|
+
getLandmarks2D: this.getLandmarks2D.bind(this)
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
async getPerformanceData() {
|
|
248
|
+
return {
|
|
249
|
+
name: this.constructor.name,
|
|
250
|
+
processingTime: this.processingTime
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
const Ue = 0.11, ze = 4e-3;
|
|
255
|
+
class Le extends L {
|
|
256
|
+
constructor() {
|
|
257
|
+
super();
|
|
258
|
+
h(this, "landmarker", null);
|
|
259
|
+
h(this, "processingTime", null);
|
|
260
|
+
h(this, "sourceCanvas");
|
|
261
|
+
h(this, "analysisCanvas");
|
|
262
|
+
h(this, "getSource");
|
|
263
|
+
h(this, "detectionResults");
|
|
264
|
+
h(this, "numFaces", 1);
|
|
265
|
+
h(this, "cameras", []);
|
|
266
|
+
// camera object to unproject each face's translation
|
|
267
|
+
h(this, "faces", []);
|
|
268
|
+
// Reused per-face work vectors. Allocated once per face index and mutated
|
|
269
|
+
// in place each frame, so the published Face[] doesn't churn the GC at
|
|
270
|
+
// 30+ fps. Per-face indexing means two faces don't alias each other's
|
|
271
|
+
// transforms; consumers see "current value" each frame.
|
|
272
|
+
h(this, "leftCheekScratch", []);
|
|
273
|
+
h(this, "rightCheekScratch", []);
|
|
274
|
+
// Reused per-frame landmark used to feed unprojectToZZeroInto with an
|
|
275
|
+
// x-mirrored copy for non-webcam sources, without allocating a new
|
|
276
|
+
// landmark array (~468 entries) each frame.
|
|
277
|
+
h(this, "landmarkScratch", { x: 0, y: 0, z: 0, visibility: 0 });
|
|
278
|
+
// Reused per-frame rotation scratch (matrix -> quaternion -> mirror -> euler),
|
|
279
|
+
// allocated once to avoid per-frame churn at 30+ fps.
|
|
280
|
+
h(this, "rotMatrixScratch", new T.Matrix4());
|
|
281
|
+
h(this, "rotQuatScratch", new T.Quaternion());
|
|
282
|
+
h(this, "rotEulerScratch", new T.Euler());
|
|
283
|
+
}
|
|
284
|
+
// For non-webcam sources MediaPipe landmarks read as mirrored; flip x in
|
|
285
|
+
// place into the reused scratch so downstream unprojection sees the
|
|
286
|
+
// corrected position. Webcam input is already correct, so it's returned
|
|
287
|
+
// by reference.
|
|
288
|
+
landmarkForSource(e, t) {
|
|
289
|
+
return t ? (this.landmarkScratch.x = 1 - e.x, this.landmarkScratch.y = e.y, this.landmarkScratch.z = e.z, this.landmarkScratch.visibility = e.visibility, this.landmarkScratch) : e;
|
|
290
|
+
}
|
|
291
|
+
extractAndUpdateFaceData(e) {
|
|
292
|
+
var n;
|
|
293
|
+
if (!e.faceLandmarks.length) {
|
|
294
|
+
this.faces = [];
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
const t = e.faceLandmarks.length;
|
|
298
|
+
for (this.faces.length > t && (this.faces.length = t, this.leftCheekScratch.length = t, this.rightCheekScratch.length = t); this.faces.length < t; )
|
|
299
|
+
this.faces.push({
|
|
300
|
+
transform: {
|
|
301
|
+
position: new T.Vector3(),
|
|
302
|
+
rotation: new T.Vector3(),
|
|
303
|
+
scale: new T.Vector3()
|
|
304
|
+
},
|
|
305
|
+
gestures: {}
|
|
306
|
+
}), this.leftCheekScratch.push(new T.Vector3()), this.rightCheekScratch.push(new T.Vector3());
|
|
307
|
+
for (let r = 0; r < t; r += 1) {
|
|
308
|
+
if (!this.cameras[r])
|
|
309
|
+
continue;
|
|
310
|
+
const c = this.faces[r], d = (n = this.getSource) == null ? void 0 : n.call(this), m = !(d === "webcam" || !d), u = e.faceLandmarks[r], g = e.facialTransformationMatrixes[r];
|
|
311
|
+
this.rotMatrixScratch.fromArray(g.data), this.rotQuatScratch.setFromRotationMatrix(this.rotMatrixScratch), this.rotQuatScratch.set(
|
|
312
|
+
-this.rotQuatScratch.x,
|
|
313
|
+
this.rotQuatScratch.y,
|
|
314
|
+
this.rotQuatScratch.z,
|
|
315
|
+
this.rotQuatScratch.w
|
|
316
|
+
), this.rotEulerScratch.setFromQuaternion(this.rotQuatScratch, "XYZ"), c.transform.rotation.set(
|
|
317
|
+
-this.rotEulerScratch.x,
|
|
318
|
+
this.rotEulerScratch.y,
|
|
319
|
+
this.rotEulerScratch.z
|
|
320
|
+
), V(
|
|
321
|
+
this.landmarkForSource(u[4], m),
|
|
322
|
+
this.cameras[r],
|
|
323
|
+
c.transform.position
|
|
324
|
+
), V(
|
|
325
|
+
this.landmarkForSource(u[50], m),
|
|
326
|
+
this.cameras[r],
|
|
327
|
+
this.leftCheekScratch[r]
|
|
328
|
+
), V(
|
|
329
|
+
this.landmarkForSource(u[280], m),
|
|
330
|
+
this.cameras[r],
|
|
331
|
+
this.rightCheekScratch[r]
|
|
332
|
+
);
|
|
333
|
+
const w = this.leftCheekScratch[r].distanceTo(this.rightCheekScratch[r]) * Ue, p = Math.abs(T.MathUtils.radToDeg(this.rotEulerScratch.y)), k = w + p * ze, C = c.transform.scale.x, R = Ee(
|
|
334
|
+
isNaN(C) ? 0 : C,
|
|
335
|
+
k
|
|
336
|
+
);
|
|
337
|
+
c.transform.scale.set(R, R, R);
|
|
338
|
+
const E = c.gestures, b = e.faceBlendshapes[r].categories;
|
|
339
|
+
for (const A of b)
|
|
340
|
+
E[A.categoryName] = A.score;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
async processFaceTracking() {
|
|
344
|
+
if (this.disposed || this.paused || !this.landmarker || !this.sourceCanvas)
|
|
345
|
+
return null;
|
|
346
|
+
if (!this.shouldRunInference()) {
|
|
347
|
+
this.rafId = requestAnimationFrame(this.processFaceTracking.bind(this));
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
const e = Ne(this.numFaces), t = this.sourceCanvas.width, n = this.sourceCanvas.height, r = Math.min(1, e / Math.max(t, n)), c = Math.max(1, Math.round(t * r)), d = Math.max(1, Math.round(n * r));
|
|
351
|
+
(!this.analysisCanvas || this.analysisCanvas.width !== c || this.analysisCanvas.height !== d) && (this.analysisCanvas = new OffscreenCanvas(c, d));
|
|
352
|
+
const m = this.analysisCanvas.getContext("2d");
|
|
353
|
+
if (!m)
|
|
354
|
+
return;
|
|
355
|
+
m.setTransform(1, 0, 0, 1, 0, 0), m.clearRect(0, 0, c, d), m.drawImage(this.sourceCanvas, 0, 0, c, d);
|
|
356
|
+
const u = performance.now(), g = this.landmarker.detectForVideo(this.analysisCanvas, u), f = performance.now();
|
|
357
|
+
this.processingTime = f - u, this.detectionResults = g, this.extractAndUpdateFaceData(g), !(this.disposed || this.paused) && (this.rafId = requestAnimationFrame(this.processFaceTracking.bind(this)));
|
|
358
|
+
}
|
|
359
|
+
resume() {
|
|
360
|
+
this.paused && (super.resume(), this.processFaceTracking());
|
|
361
|
+
}
|
|
362
|
+
getFace(e = 0) {
|
|
363
|
+
return this.faces[e];
|
|
364
|
+
}
|
|
365
|
+
getDetectionResults() {
|
|
366
|
+
return this.detectionResults;
|
|
367
|
+
}
|
|
368
|
+
getRawFaceTrackingData(e = 0) {
|
|
369
|
+
var t, n, r, c;
|
|
370
|
+
if (this.detectionResults)
|
|
371
|
+
return {
|
|
372
|
+
landmarks: (t = this.detectionResults.faceLandmarks) == null ? void 0 : t[e],
|
|
373
|
+
blendshapes: (r = (n = this.detectionResults.faceBlendshapes) == null ? void 0 : n[e]) == null ? void 0 : r.categories,
|
|
374
|
+
transformationMatrix: (c = this.detectionResults.facialTransformationMatrixes) == null ? void 0 : c[e]
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
async initialize(e) {
|
|
378
|
+
this.numFaces = (e == null ? void 0 : e.numFaces) || 1;
|
|
379
|
+
const t = await U.forVisionTasks(
|
|
380
|
+
new URL("/assets/xr/wasm", x.assetStorageEndpoint).toString()
|
|
381
|
+
), n = await z(
|
|
382
|
+
new URL("/assets/xr/models/face_tracking.task", x.assetStorageEndpoint).toString()
|
|
383
|
+
), r = await ye.createFromOptions(t, {
|
|
384
|
+
baseOptions: {
|
|
385
|
+
modelAssetBuffer: n,
|
|
386
|
+
delegate: "GPU"
|
|
387
|
+
},
|
|
388
|
+
// Keep detection permissive for calibration workflows with dark webcam input.
|
|
389
|
+
minFaceDetectionConfidence: 0.3,
|
|
390
|
+
minFacePresenceConfidence: 0.3,
|
|
391
|
+
minTrackingConfidence: 0.3,
|
|
392
|
+
outputFaceBlendshapes: !0,
|
|
393
|
+
outputFacialTransformationMatrixes: !0,
|
|
394
|
+
runningMode: "VIDEO",
|
|
395
|
+
numFaces: this.numFaces
|
|
396
|
+
});
|
|
397
|
+
this.landmarker = r;
|
|
398
|
+
}
|
|
399
|
+
async run({ sourceCanvas: e, getSource: t, targetFps: n }) {
|
|
400
|
+
this.sourceCanvas = e, this.getSource = t, this.setTargetFps(n), this.analysisCanvas = void 0, this.rafId === null && this.processFaceTracking();
|
|
401
|
+
}
|
|
402
|
+
getModelNode() {
|
|
403
|
+
const e = this, t = {
|
|
404
|
+
get detectionResults() {
|
|
405
|
+
return e.detectionResults;
|
|
406
|
+
},
|
|
407
|
+
getDetectionResults: this.getDetectionResults.bind(this),
|
|
408
|
+
getFace: this.getFace.bind(this),
|
|
409
|
+
getRawFaceTrackingData: this.getRawFaceTrackingData.bind(this)
|
|
410
|
+
};
|
|
411
|
+
return Object.assign(t, {
|
|
412
|
+
_setCamera: (n, r) => {
|
|
413
|
+
this.cameras[r] = n;
|
|
414
|
+
}
|
|
415
|
+
}), t;
|
|
416
|
+
}
|
|
417
|
+
async getPerformanceData() {
|
|
418
|
+
return {
|
|
419
|
+
name: this.constructor.name,
|
|
420
|
+
processingTime: this.processingTime
|
|
421
|
+
};
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
const Ne = (i) => i > 1 ? 1024 : 512;
|
|
425
|
+
var I = /* @__PURE__ */ ((i) => (i.ClosedFist = "closed_fist", i.OpenPalm = "open_palm", i.PointingUp = "pointing_up", i.ThumbDown = "thumb_down", i.ThumbUp = "thumb_up", i.Victory = "victory", i.ILoveYou = "i_love_you", i.None = "none", i))(I || {});
|
|
426
|
+
const Ve = 512, Be = {
|
|
427
|
+
Closed_Fist: I.ClosedFist,
|
|
428
|
+
Open_Palm: I.OpenPalm,
|
|
429
|
+
Pointing_Up: I.PointingUp,
|
|
430
|
+
Thumb_Down: I.ThumbDown,
|
|
431
|
+
Thumb_Up: I.ThumbUp,
|
|
432
|
+
Victory: I.Victory,
|
|
433
|
+
ILoveYou: I.ILoveYou,
|
|
434
|
+
None: I.None
|
|
435
|
+
}, He = 15;
|
|
436
|
+
class qe extends L {
|
|
437
|
+
constructor() {
|
|
438
|
+
super();
|
|
439
|
+
h(this, "landmarker", null);
|
|
440
|
+
h(this, "processingTime", null);
|
|
441
|
+
h(this, "coordinates");
|
|
442
|
+
h(this, "gesture");
|
|
443
|
+
h(this, "sourceCanvas");
|
|
444
|
+
h(this, "downscaler", new G(Ve));
|
|
445
|
+
}
|
|
446
|
+
async getGestureData() {
|
|
447
|
+
if (!this.landmarker || !this.sourceCanvas)
|
|
448
|
+
return;
|
|
449
|
+
const e = this.downscaler.draw(this.sourceCanvas), t = performance.now(), n = this.landmarker.recognizeForVideo(e, t), r = performance.now();
|
|
450
|
+
this.processingTime = r - t;
|
|
451
|
+
const c = n.landmarks[0], d = [];
|
|
452
|
+
if (c) {
|
|
453
|
+
const m = n.gestures[0][0].categoryName;
|
|
454
|
+
return c.forEach((u) => {
|
|
455
|
+
d.push({
|
|
456
|
+
x: D(u.x),
|
|
457
|
+
y: D(1 - u.y),
|
|
458
|
+
z: -u.z
|
|
459
|
+
});
|
|
460
|
+
}), {
|
|
461
|
+
coordinates: d,
|
|
462
|
+
gesture: Be[m]
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
return { coordinates: void 0, gesture: void 0 };
|
|
466
|
+
}
|
|
467
|
+
async processGestureTracking() {
|
|
468
|
+
if (!this.disposed) {
|
|
469
|
+
if (this.shouldRunInference()) {
|
|
470
|
+
const e = await this.getGestureData();
|
|
471
|
+
e && (this.coordinates = e.coordinates, this.gesture = e.gesture);
|
|
472
|
+
}
|
|
473
|
+
this.disposed || this.paused || (this.rafId = requestAnimationFrame(this.processGestureTracking.bind(this)));
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
resume() {
|
|
477
|
+
this.paused && (super.resume(), this.processGestureTracking());
|
|
478
|
+
}
|
|
479
|
+
dispose() {
|
|
480
|
+
if (super.dispose(), this.landmarker) {
|
|
481
|
+
try {
|
|
482
|
+
this.landmarker.close();
|
|
483
|
+
} catch {
|
|
484
|
+
}
|
|
485
|
+
this.landmarker = null;
|
|
486
|
+
}
|
|
487
|
+
this.sourceCanvas = void 0, this.downscaler.dispose(), this.coordinates = void 0, this.gesture = void 0;
|
|
488
|
+
}
|
|
489
|
+
async initialize() {
|
|
490
|
+
const e = await U.forVisionTasks(
|
|
491
|
+
new URL("/assets/xr/wasm", x.assetStorageEndpoint).toString()
|
|
492
|
+
), t = await z(
|
|
493
|
+
new URL("/assets/xr/models/gesture_tracking.task", x.assetStorageEndpoint).toString()
|
|
494
|
+
), n = await Me.createFromOptions(e, {
|
|
495
|
+
baseOptions: {
|
|
496
|
+
modelAssetBuffer: t,
|
|
497
|
+
delegate: "GPU"
|
|
498
|
+
},
|
|
499
|
+
runningMode: "VIDEO",
|
|
500
|
+
numHands: 1
|
|
501
|
+
});
|
|
502
|
+
this.landmarker = n;
|
|
503
|
+
}
|
|
504
|
+
async run({ sourceCanvas: e, targetFps: t }) {
|
|
505
|
+
this.sourceCanvas = e, this.setTargetFps(t ?? He), this.rafId === null && await this.processGestureTracking();
|
|
506
|
+
}
|
|
507
|
+
getModelNode() {
|
|
508
|
+
const e = this;
|
|
509
|
+
return {
|
|
510
|
+
get coordinates() {
|
|
511
|
+
return e.coordinates;
|
|
512
|
+
},
|
|
513
|
+
get gesture() {
|
|
514
|
+
return e.gesture;
|
|
515
|
+
}
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
async getPerformanceData() {
|
|
519
|
+
return {
|
|
520
|
+
name: this.constructor.name,
|
|
521
|
+
processingTime: this.processingTime
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
const Ke = 512;
|
|
526
|
+
class We extends L {
|
|
527
|
+
constructor() {
|
|
528
|
+
super();
|
|
529
|
+
h(this, "landmarker", null);
|
|
530
|
+
h(this, "processingTime", null);
|
|
531
|
+
h(this, "coordinates");
|
|
532
|
+
h(this, "gesture");
|
|
533
|
+
h(this, "sourceCanvas");
|
|
534
|
+
h(this, "downscaler", new G(Ke));
|
|
535
|
+
}
|
|
536
|
+
async getHandData() {
|
|
537
|
+
if (!this.landmarker || !this.sourceCanvas)
|
|
538
|
+
return;
|
|
539
|
+
const e = this.downscaler.draw(this.sourceCanvas), t = performance.now(), n = this.landmarker.detectForVideo(e, t), r = performance.now();
|
|
540
|
+
this.processingTime = r - t;
|
|
541
|
+
const c = n.landmarks[0], d = [];
|
|
542
|
+
if (c) {
|
|
543
|
+
const m = n.landmarks[0][0].z < 0 ? "closed_fist" : "open_palm";
|
|
544
|
+
return c.forEach((u) => {
|
|
545
|
+
d.push({
|
|
546
|
+
x: D(u.x),
|
|
547
|
+
y: D(1 - u.y),
|
|
548
|
+
z: -u.z
|
|
549
|
+
});
|
|
550
|
+
}), {
|
|
551
|
+
coordinates: d,
|
|
552
|
+
gesture: m
|
|
553
|
+
};
|
|
554
|
+
}
|
|
555
|
+
return { coordinates: void 0, gesture: void 0 };
|
|
556
|
+
}
|
|
557
|
+
async processHandTracking() {
|
|
558
|
+
if (!this.disposed) {
|
|
559
|
+
if (this.shouldRunInference()) {
|
|
560
|
+
const e = await this.getHandData();
|
|
561
|
+
e && (this.coordinates = e.coordinates, this.gesture = e.gesture);
|
|
562
|
+
}
|
|
563
|
+
this.disposed || this.paused || (this.rafId = requestAnimationFrame(this.processHandTracking.bind(this)));
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
resume() {
|
|
567
|
+
this.paused && (super.resume(), this.processHandTracking());
|
|
568
|
+
}
|
|
569
|
+
dispose() {
|
|
570
|
+
if (super.dispose(), this.landmarker) {
|
|
571
|
+
try {
|
|
572
|
+
this.landmarker.close();
|
|
573
|
+
} catch {
|
|
574
|
+
}
|
|
575
|
+
this.landmarker = null;
|
|
576
|
+
}
|
|
577
|
+
this.sourceCanvas = void 0, this.downscaler.dispose(), this.coordinates = void 0, this.gesture = void 0;
|
|
578
|
+
}
|
|
579
|
+
async initialize() {
|
|
580
|
+
const e = await U.forVisionTasks(
|
|
581
|
+
new URL("/assets/xr/wasm", x.assetStorageEndpoint).toString()
|
|
582
|
+
), t = await z(
|
|
583
|
+
new URL("/assets/xr/models/hand_tracking.task", x.assetStorageEndpoint).toString()
|
|
584
|
+
), n = await Te.createFromOptions(e, {
|
|
585
|
+
baseOptions: {
|
|
586
|
+
modelAssetBuffer: t,
|
|
587
|
+
delegate: "GPU"
|
|
588
|
+
},
|
|
589
|
+
runningMode: "VIDEO",
|
|
590
|
+
numHands: 1
|
|
591
|
+
});
|
|
592
|
+
this.landmarker = n;
|
|
593
|
+
}
|
|
594
|
+
async run({ sourceCanvas: e, targetFps: t }) {
|
|
595
|
+
this.sourceCanvas = e, this.setTargetFps(t), this.rafId === null && await this.processHandTracking();
|
|
596
|
+
}
|
|
597
|
+
getModelNode() {
|
|
598
|
+
const e = this;
|
|
599
|
+
return {
|
|
600
|
+
get coordinates() {
|
|
601
|
+
return e.coordinates;
|
|
602
|
+
},
|
|
603
|
+
get gesture() {
|
|
604
|
+
return e.gesture;
|
|
605
|
+
}
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
async getPerformanceData() {
|
|
609
|
+
return {
|
|
610
|
+
name: this.constructor.name,
|
|
611
|
+
processingTime: this.processingTime
|
|
612
|
+
};
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
const Ge = 24, Xe = 720, $e = [
|
|
616
|
+
[255, 255, 255, 255],
|
|
617
|
+
// 0: background → green 255, kept
|
|
618
|
+
[0, 0, 0, 0]
|
|
619
|
+
// 1: person → green 0, discarded
|
|
620
|
+
];
|
|
621
|
+
class Ye extends L {
|
|
622
|
+
constructor() {
|
|
623
|
+
super();
|
|
624
|
+
h(this, "segmenter", null);
|
|
625
|
+
h(this, "processingTime", null);
|
|
626
|
+
h(this, "sourceCanvas");
|
|
627
|
+
// input source canvas
|
|
628
|
+
h(this, "processCanvas");
|
|
629
|
+
// downscaled canvas of sourceCanvas for processing
|
|
630
|
+
h(this, "filterCanvas");
|
|
631
|
+
// post-processing (blur) canvas
|
|
632
|
+
// Optional high-resolution input override. The shared session feed canvas is
|
|
633
|
+
// sized to the on-screen display layout (often a small, display-fit crop),
|
|
634
|
+
// which starves the segmenter of the resolution it needs to capture the whole
|
|
635
|
+
// body. When a consumer supplies a native-resolution source here (e.g. the raw
|
|
636
|
+
// <video>/<img>), the segmenter runs on THAT instead — letterboxed into the
|
|
637
|
+
// process canvas so nothing is cropped. Falls back to `sourceCanvas` when unset.
|
|
638
|
+
h(this, "inputSource");
|
|
639
|
+
h(this, "inputSourceSize");
|
|
640
|
+
// The segmenter's own WebGL2 context. DrawingUtils renders the inverted mask
|
|
641
|
+
// into this context on the GPU; we then blit that GL canvas straight into the
|
|
642
|
+
// mask canvas with a single `drawImage` — a GPU→GPU transfer, no CPU readback
|
|
643
|
+
// and no per-pixel JS loop. (A spike proved `drawImage(glCanvas)` reads the
|
|
644
|
+
// full DrawingUtils output correctly; the earlier "reads empty" wall that once
|
|
645
|
+
// forced a readPixels path was a false alarm.)
|
|
646
|
+
h(this, "glCanvas");
|
|
647
|
+
h(this, "drawingUtils");
|
|
648
|
+
h(this, "useGpuMaskPath", !1);
|
|
649
|
+
// CPU-fallback reused ImageData, allocated on size change only. Used ONLY when
|
|
650
|
+
// WebGL2 is unavailable and we must read the single-channel mask back by hand.
|
|
651
|
+
h(this, "maskImageData");
|
|
652
|
+
h(this, "maskCanvas", null);
|
|
653
|
+
h(this, "segmentationMaskTexture");
|
|
654
|
+
h(this, "isMaskSetup", !1);
|
|
655
|
+
h(this, "isProcessing", !1);
|
|
656
|
+
// Live clone of the most recent category mask, kept so getRawSegmentationData()
|
|
657
|
+
// can read the single-channel Uint8Array lazily — that readback is paid ONLY
|
|
658
|
+
// when a consumer actually asks for raw data. Cloned because the mask returned
|
|
659
|
+
// by segmentForVideo is invalidated on the next inference call.
|
|
660
|
+
h(this, "latestMask", null);
|
|
661
|
+
// Per-frame cache of the read-back array, so repeated getRawSegmentationData()
|
|
662
|
+
// calls within one frame don't re-run the GPU→CPU transfer.
|
|
663
|
+
h(this, "rawCache", null);
|
|
664
|
+
h(this, "quality", "medium");
|
|
665
|
+
h(this, "stagingCanvas");
|
|
666
|
+
}
|
|
667
|
+
getQualityScaleFactor() {
|
|
668
|
+
switch (this.quality) {
|
|
669
|
+
case "low":
|
|
670
|
+
return 0.5;
|
|
671
|
+
case "medium":
|
|
672
|
+
return 0.75;
|
|
673
|
+
case "high":
|
|
674
|
+
return 1;
|
|
675
|
+
default:
|
|
676
|
+
return 0.5;
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
/**
|
|
680
|
+
* Set (or clear) a high-resolution input override. Pass the raw <video>/<img>
|
|
681
|
+
* (or any CanvasImageSource with intrinsic dimensions) to run segmentation on
|
|
682
|
+
* native-resolution, uncropped frames instead of the small display-fit session
|
|
683
|
+
* feed. Pass `undefined` to revert to the shared feed canvas.
|
|
684
|
+
*/
|
|
685
|
+
setInputSource(e) {
|
|
686
|
+
this.inputSource = e, this.inputSourceSize = e ? this.readIntrinsicSize(e) : void 0;
|
|
687
|
+
}
|
|
688
|
+
readIntrinsicSize(e) {
|
|
689
|
+
if (e instanceof HTMLVideoElement)
|
|
690
|
+
return e.videoWidth > 0 ? { width: e.videoWidth, height: e.videoHeight } : void 0;
|
|
691
|
+
if (e instanceof HTMLImageElement)
|
|
692
|
+
return e.naturalWidth > 0 ? { width: e.naturalWidth, height: e.naturalHeight } : void 0;
|
|
693
|
+
const t = e.width, n = e.height;
|
|
694
|
+
return typeof t == "number" && t > 0 && typeof n == "number" && n > 0 ? { width: t, height: n } : void 0;
|
|
695
|
+
}
|
|
696
|
+
/**
|
|
697
|
+
* Size the process canvas to the override input's aspect ratio, bounded by
|
|
698
|
+
* PROCESS_MAX_DIM on the long edge, and keep the GL canvas in sync so the GPU
|
|
699
|
+
* mask renders at the same resolution. No-op when there is no override, or when
|
|
700
|
+
* the intrinsic size hasn't changed since the last frame.
|
|
701
|
+
*/
|
|
702
|
+
ensureProcessCanvasForInput() {
|
|
703
|
+
if (!this.processCanvas || !this.inputSource) return;
|
|
704
|
+
const e = this.readIntrinsicSize(this.inputSource);
|
|
705
|
+
if (!e) return;
|
|
706
|
+
this.inputSourceSize = e;
|
|
707
|
+
const t = Math.min(1, Xe / Math.max(e.width, e.height)), n = Math.max(1, Math.round(e.width * t)), r = Math.max(1, Math.round(e.height * t));
|
|
708
|
+
this.processCanvas.width === n && this.processCanvas.height === r || (this.processCanvas.width = n, this.processCanvas.height = r, this.glCanvas && (this.glCanvas.width = n, this.glCanvas.height = r), this.maskImageData = void 0);
|
|
709
|
+
}
|
|
710
|
+
async processSegmentation() {
|
|
711
|
+
if (this.disposed || !this.isProcessing || !this.maskCanvas || !this.processCanvas)
|
|
712
|
+
return;
|
|
713
|
+
if (!this.shouldRunInference()) {
|
|
714
|
+
if (this.disposed || this.paused) return;
|
|
715
|
+
this.rafId = requestAnimationFrame(this.processSegmentation.bind(this));
|
|
716
|
+
return;
|
|
717
|
+
}
|
|
718
|
+
const e = this.inputSource ?? this.sourceCanvas;
|
|
719
|
+
if (!this.segmenter || !e) {
|
|
720
|
+
if (this.disposed || this.paused) return;
|
|
721
|
+
this.rafId = requestAnimationFrame(this.processSegmentation.bind(this));
|
|
722
|
+
return;
|
|
723
|
+
}
|
|
724
|
+
const t = performance.now();
|
|
725
|
+
this.ensureProcessCanvasForInput();
|
|
726
|
+
const n = this.processCanvas.getContext("2d");
|
|
727
|
+
if (n) {
|
|
728
|
+
const d = this.processCanvas.width, m = this.processCanvas.height;
|
|
729
|
+
if (n.clearRect(0, 0, d, m), this.inputSource && this.inputSourceSize) {
|
|
730
|
+
const { width: u, height: g } = this.inputSourceSize, f = Math.min(d / u, m / g), w = u * f, p = g * f;
|
|
731
|
+
n.drawImage(this.inputSource, (d - w) / 2, (m - p) / 2, w, p);
|
|
732
|
+
} else
|
|
733
|
+
n.drawImage(e, 0, 0, d, m);
|
|
734
|
+
}
|
|
735
|
+
const c = this.segmenter.segmentForVideo(
|
|
736
|
+
this.processCanvas,
|
|
737
|
+
performance.now()
|
|
738
|
+
).categoryMask;
|
|
739
|
+
if (!c) {
|
|
740
|
+
if (this.disposed || this.paused) return;
|
|
741
|
+
this.rafId = requestAnimationFrame(this.processSegmentation.bind(this));
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
if (this.rawCache = null, this.latestMask && (this.latestMask.close(), this.latestMask = null), this.useGpuMaskPath && this.drawingUtils && this.glCanvas)
|
|
745
|
+
this.drawingUtils.drawCategoryMask(c, $e), this.latestMask = c.clone(), c.close(), this.blitGlToMask();
|
|
746
|
+
else {
|
|
747
|
+
this.latestMask = c.clone();
|
|
748
|
+
const d = this.readMaskToImageData(c);
|
|
749
|
+
c.close(), d && this.blitImageDataToMask(d);
|
|
750
|
+
}
|
|
751
|
+
this.segmentationMaskTexture && (this.isMaskSetup || (this.segmentationMaskTexture.format = te, this.segmentationMaskTexture.minFilter = B, this.segmentationMaskTexture.magFilter = B, this.segmentationMaskTexture.generateMipmaps = !1, this.isMaskSetup = !0), this.segmentationMaskTexture.needsUpdate = !0), this.processingTime = performance.now() - t, !(this.disposed || this.paused) && (this.rafId = requestAnimationFrame(this.processSegmentation.bind(this)));
|
|
752
|
+
}
|
|
753
|
+
/**
|
|
754
|
+
* Blit the GPU-rendered mask (already RGBA-inverted by drawCategoryMask)
|
|
755
|
+
* straight from the GL canvas into the mask canvas, cover-fitted through the
|
|
756
|
+
* soft-edge blur. `drawImage(glCanvas)` is a GPU→GPU transfer — no readPixels,
|
|
757
|
+
* no CPU array, no ImageData, no row flip (the 2D canvas presents the WebGL
|
|
758
|
+
* canvas right-side-up). This is the zero-readback path.
|
|
759
|
+
*/
|
|
760
|
+
blitGlToMask() {
|
|
761
|
+
if (!this.maskCanvas || !this.filterCanvas || !this.glCanvas) return;
|
|
762
|
+
const e = this.maskCanvas.getContext("2d"), t = this.filterCanvas.getContext("2d");
|
|
763
|
+
!e || !t || (t.filter = "blur(0.5px)", t.clearRect(0, 0, this.filterCanvas.width, this.filterCanvas.height), t.drawImage(this.glCanvas, 0, 0, this.filterCanvas.width, this.filterCanvas.height), e.clearRect(0, 0, this.maskCanvas.width, this.maskCanvas.height), e.drawImage(this.filterCanvas, 0, 0, this.maskCanvas.width, this.maskCanvas.height));
|
|
764
|
+
}
|
|
765
|
+
/**
|
|
766
|
+
* CPU-fallback inverted-alpha ImageData build (only when the GPU path is off).
|
|
767
|
+
* The single-channel mask is expanded to RGBA with the person→transparent,
|
|
768
|
+
* background→opaque convention (green carries the signal for the alphaMap).
|
|
769
|
+
*/
|
|
770
|
+
readMaskToImageData(e) {
|
|
771
|
+
const t = e.getAsUint8Array(), { width: n, height: r } = e;
|
|
772
|
+
if (!t || n <= 0 || r <= 0) return null;
|
|
773
|
+
(!this.maskImageData || this.maskImageData.width !== n || this.maskImageData.height !== r) && (this.maskImageData = new ImageData(n, r));
|
|
774
|
+
const c = this.maskImageData.data;
|
|
775
|
+
for (let d = 0; d < t.length; d += 1) {
|
|
776
|
+
const m = t[d] === 255 ? 0 : 255;
|
|
777
|
+
c[d * 4] = 255, c[d * 4 + 1] = m, c[d * 4 + 2] = 255, c[d * 4 + 3] = m;
|
|
778
|
+
}
|
|
779
|
+
return this.maskImageData;
|
|
780
|
+
}
|
|
781
|
+
/**
|
|
782
|
+
* CPU-fallback blit: put the inverted-alpha mask ImageData into maskCanvas
|
|
783
|
+
* with a soft-edge blur, cover-fitted to the render target. Only used when the
|
|
784
|
+
* GPU path is off; the GPU path blits the GL canvas directly (blitGlToMask).
|
|
785
|
+
*/
|
|
786
|
+
blitImageDataToMask(e) {
|
|
787
|
+
if (!this.maskCanvas || !this.filterCanvas) return;
|
|
788
|
+
const t = this.maskCanvas.getContext("2d"), n = this.filterCanvas.getContext("2d");
|
|
789
|
+
if (!t || !n) return;
|
|
790
|
+
const r = this.getStagingCanvas(e.width, e.height), c = r.getContext("2d");
|
|
791
|
+
c && (c.putImageData(e, 0, 0), n.filter = "blur(0.5px)", n.clearRect(0, 0, this.filterCanvas.width, this.filterCanvas.height), n.drawImage(r, 0, 0, this.filterCanvas.width, this.filterCanvas.height), t.clearRect(0, 0, this.maskCanvas.width, this.maskCanvas.height), t.drawImage(this.filterCanvas, 0, 0, this.maskCanvas.width, this.maskCanvas.height));
|
|
792
|
+
}
|
|
793
|
+
getStagingCanvas(e, t) {
|
|
794
|
+
return this.stagingCanvas || (this.stagingCanvas = document.createElement("canvas")), this.stagingCanvas.width !== e && (this.stagingCanvas.width = e), this.stagingCanvas.height !== t && (this.stagingCanvas.height = t), this.stagingCanvas;
|
|
795
|
+
}
|
|
796
|
+
resume() {
|
|
797
|
+
this.paused && (super.resume(), this.processSegmentation());
|
|
798
|
+
}
|
|
799
|
+
dispose() {
|
|
800
|
+
if (super.dispose(), this.isProcessing = !1, this.latestMask) {
|
|
801
|
+
try {
|
|
802
|
+
this.latestMask.close();
|
|
803
|
+
} catch {
|
|
804
|
+
}
|
|
805
|
+
this.latestMask = null;
|
|
806
|
+
}
|
|
807
|
+
if (this.rawCache = null, this.segmenter) {
|
|
808
|
+
try {
|
|
809
|
+
this.segmenter.close();
|
|
810
|
+
} catch {
|
|
811
|
+
}
|
|
812
|
+
this.segmenter = null;
|
|
813
|
+
}
|
|
814
|
+
this.drawingUtils = void 0, this.glCanvas = void 0, this.maskImageData = void 0, this.segmentationMaskTexture && (this.segmentationMaskTexture.dispose(), this.segmentationMaskTexture = void 0), this.sourceCanvas = void 0, this.inputSource = void 0, this.inputSourceSize = void 0, this.processCanvas = void 0, this.filterCanvas = void 0, this.stagingCanvas = void 0, this.maskCanvas = null;
|
|
815
|
+
}
|
|
816
|
+
async initialize(e) {
|
|
817
|
+
this.quality = (e == null ? void 0 : e.quality) ?? "low";
|
|
818
|
+
const t = await U.forVisionTasks(
|
|
819
|
+
new URL("/assets/xr/wasm", x.assetStorageEndpoint).toString()
|
|
820
|
+
), n = await z(
|
|
821
|
+
new URL("/assets/xr/models/segmentation.tflite", x.assetStorageEndpoint).toString()
|
|
822
|
+
);
|
|
823
|
+
this.glCanvas = new OffscreenCanvas(1, 1);
|
|
824
|
+
const r = this.glCanvas.getContext("webgl2", {
|
|
825
|
+
preserveDrawingBuffer: !0
|
|
826
|
+
});
|
|
827
|
+
this.segmenter = await Re.createFromOptions(t, {
|
|
828
|
+
baseOptions: {
|
|
829
|
+
modelAssetBuffer: n,
|
|
830
|
+
delegate: "GPU"
|
|
831
|
+
},
|
|
832
|
+
...r ? { canvas: this.glCanvas } : {},
|
|
833
|
+
outputCategoryMask: !0,
|
|
834
|
+
outputConfidenceMasks: !1,
|
|
835
|
+
runningMode: "VIDEO"
|
|
836
|
+
}), r ? (this.drawingUtils = new Fe(r), this.useGpuMaskPath = !0) : this.useGpuMaskPath = !1;
|
|
837
|
+
}
|
|
838
|
+
async run({
|
|
839
|
+
sourceCanvas: e,
|
|
840
|
+
renderWidth: t,
|
|
841
|
+
renderHeight: n,
|
|
842
|
+
targetFps: r
|
|
843
|
+
}) {
|
|
844
|
+
if (!e)
|
|
845
|
+
return;
|
|
846
|
+
this.setTargetFps(r ?? Ge), this.isProcessing = !1, this.rafId !== null && (cancelAnimationFrame(this.rafId), this.rafId = null);
|
|
847
|
+
const c = this.getQualityScaleFactor();
|
|
848
|
+
this.sourceCanvas = e, this.processCanvas = document.createElement("canvas"), this.processCanvas.width = e.width * c, this.processCanvas.height = e.height * c, this.glCanvas && (this.glCanvas.width = this.processCanvas.width, this.glCanvas.height = this.processCanvas.height), this.maskImageData = void 0, this.maskCanvas = document.createElement("canvas"), this.maskCanvas.width = t, this.maskCanvas.height = n, this.filterCanvas = document.createElement("canvas"), this.filterCanvas.width = t, this.filterCanvas.height = n, this.segmentationMaskTexture = new se(this.maskCanvas), this.isMaskSetup = !1, this.rawCache = null, this.isProcessing = !0, this.lastInferenceMs = 0, this.rafId = requestAnimationFrame(this.processSegmentation.bind(this));
|
|
849
|
+
}
|
|
850
|
+
/**
|
|
851
|
+
* Lazily read the single-channel mask back to a Uint8Array. Only invoked when
|
|
852
|
+
* a consumer calls getRawSegmentationData(); the default texture path never
|
|
853
|
+
* touches it, so apps that only use <VideoBackground> pay no extra readback.
|
|
854
|
+
*/
|
|
855
|
+
getRawSegmentationData() {
|
|
856
|
+
if (this.rawCache) return this.rawCache;
|
|
857
|
+
if (!this.latestMask) return null;
|
|
858
|
+
const e = this.latestMask.getAsUint8Array();
|
|
859
|
+
return this.rawCache = {
|
|
860
|
+
maskData: e,
|
|
861
|
+
width: this.latestMask.width,
|
|
862
|
+
height: this.latestMask.height
|
|
863
|
+
}, this.rawCache;
|
|
864
|
+
}
|
|
865
|
+
getModelNode() {
|
|
866
|
+
const e = this;
|
|
867
|
+
return {
|
|
868
|
+
get backgroundSegmentationMask() {
|
|
869
|
+
return e.segmentationMaskTexture;
|
|
870
|
+
},
|
|
871
|
+
getMaskCanvas: () => e.maskCanvas,
|
|
872
|
+
getRawSegmentationData: () => e.getRawSegmentationData(),
|
|
873
|
+
setInputSource: (t) => e.setInputSource(t ?? void 0)
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
async getPerformanceData() {
|
|
877
|
+
return {
|
|
878
|
+
name: this.constructor.name,
|
|
879
|
+
processingTime: this.processingTime
|
|
880
|
+
};
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
const Z = {
|
|
884
|
+
[_.BACKGROUND_SEGMENTATION]: Ye,
|
|
885
|
+
[_.HAND_TRACKER]: We,
|
|
886
|
+
[_.GESTURE_TRACKER]: qe,
|
|
887
|
+
[_.FACE_TRACKER]: Le,
|
|
888
|
+
[_.BODY_TRACKER]: Oe
|
|
889
|
+
}, Qe = async (i) => {
|
|
890
|
+
const s = document.createElement("video");
|
|
891
|
+
s.srcObject = i, s.muted = !0, s.playsInline = !0, s.autoplay = !0, await new Promise((t) => {
|
|
892
|
+
s.onloadedmetadata = () => {
|
|
893
|
+
s.play(), t();
|
|
894
|
+
};
|
|
895
|
+
});
|
|
896
|
+
const e = s;
|
|
897
|
+
return await new Promise((t) => {
|
|
898
|
+
const n = setTimeout(t, 150), r = () => {
|
|
899
|
+
clearTimeout(n), t();
|
|
900
|
+
};
|
|
901
|
+
typeof e.requestVideoFrameCallback == "function" ? e.requestVideoFrameCallback(r) : requestAnimationFrame(() => requestAnimationFrame(() => r()));
|
|
902
|
+
}), s;
|
|
903
|
+
}, J = (i) => {
|
|
904
|
+
const s = new se(i);
|
|
905
|
+
return s.colorSpace = xe, s.format = te, s.minFilter = B, s.magFilter = B, s.generateMipmaps = !1, s.needsUpdate = !0, s;
|
|
906
|
+
}, je = (i, s) => {
|
|
907
|
+
const e = i.videoWidth, t = i.videoHeight;
|
|
908
|
+
e > 0 && t > 0 && (s.width !== e && (s.width = e), s.height !== t && (s.height = t));
|
|
909
|
+
const n = s.getContext("2d", {
|
|
910
|
+
colorSpace: "srgb"
|
|
911
|
+
});
|
|
912
|
+
n.save(), n.scale(-1, 1), n.drawImage(i, -s.width, 0, s.width, s.height), n.restore();
|
|
913
|
+
}, Ze = (i, s, e = 1) => {
|
|
914
|
+
const t = s.getContext("2d", {
|
|
915
|
+
colorSpace: "srgb"
|
|
916
|
+
}), n = s.width / s.height, r = i.width / i.height;
|
|
917
|
+
let c, d;
|
|
918
|
+
r > n ? (d = s.height, c = i.width * (s.height / i.height)) : (c = s.width, d = i.height * (s.width / i.width));
|
|
919
|
+
const m = Number.isFinite(e) && e > 0 ? e : 1;
|
|
920
|
+
c *= m, d *= m;
|
|
921
|
+
const u = (s.width - c) / 2, g = (s.height - d) / 2;
|
|
922
|
+
t.setTransform(1, 0, 0, 1, 0, 0), t.clearRect(0, 0, s.width, s.height), t.drawImage(
|
|
923
|
+
i,
|
|
924
|
+
0,
|
|
925
|
+
0,
|
|
926
|
+
i.width,
|
|
927
|
+
i.height,
|
|
928
|
+
u,
|
|
929
|
+
g,
|
|
930
|
+
c,
|
|
931
|
+
d
|
|
932
|
+
);
|
|
933
|
+
}, Je = (i, s, e) => {
|
|
934
|
+
const t = /* @__PURE__ */ new Map();
|
|
935
|
+
return { acquire: async (g, f, w) => {
|
|
936
|
+
let p = t.get(g);
|
|
937
|
+
if (p)
|
|
938
|
+
return p.refs.add(f), p.activeRefs.add(f), p.initPromise;
|
|
939
|
+
const k = s[g];
|
|
940
|
+
if (!k)
|
|
941
|
+
throw new Error(`Invalid model: ${g}`);
|
|
942
|
+
const C = new k(), R = /* @__PURE__ */ new Set([f]), E = /* @__PURE__ */ new Set([f]), b = (async () => {
|
|
943
|
+
await C.initialize(w), e && await e(g, C);
|
|
944
|
+
const A = t.get(g);
|
|
945
|
+
return A && A.refs.size > 0 && i.setPipeline((N) => N.set(g, C)), C;
|
|
946
|
+
})();
|
|
947
|
+
return p = { model: C, refs: R, activeRefs: E, initPromise: b, options: w }, t.set(g, p), b.catch(() => {
|
|
948
|
+
t.get(g) === p && (t.delete(g), i.setPipeline((N) => N.delete(g)));
|
|
949
|
+
}), b;
|
|
950
|
+
}, release: (g, f) => {
|
|
951
|
+
var k;
|
|
952
|
+
const w = t.get(g);
|
|
953
|
+
if (!w || (w.refs.delete(f), w.activeRefs.delete(f) && w.refs.size > 0 && w.activeRefs.size === 0 && w.model.pause(), w.refs.size > 0)) return;
|
|
954
|
+
t.delete(g);
|
|
955
|
+
const p = w.model;
|
|
956
|
+
try {
|
|
957
|
+
(k = p.dispose) == null || k.call(p);
|
|
958
|
+
} catch (C) {
|
|
959
|
+
console.warn(`[xr-sdk registry] dispose threw for ${g}`, C);
|
|
960
|
+
}
|
|
961
|
+
i.setPipeline((C) => C.delete(g));
|
|
962
|
+
}, setActive: (g, f, w) => {
|
|
963
|
+
const p = t.get(g);
|
|
964
|
+
if (!p || !p.refs.has(f)) return;
|
|
965
|
+
const k = p.activeRefs.size > 0;
|
|
966
|
+
w ? p.activeRefs.add(f) : p.activeRefs.delete(f);
|
|
967
|
+
const C = p.activeRefs.size > 0;
|
|
968
|
+
k !== C && (C ? p.model.resume() : p.model.pause());
|
|
969
|
+
}, get: (g) => {
|
|
970
|
+
var f;
|
|
971
|
+
return (f = t.get(g)) == null ? void 0 : f.model;
|
|
972
|
+
}, getOptions: (g) => {
|
|
973
|
+
var f;
|
|
974
|
+
return (f = t.get(g)) == null ? void 0 : f.options;
|
|
975
|
+
}, releaseAll: () => {
|
|
976
|
+
var g;
|
|
977
|
+
for (const f of Array.from(t.keys())) {
|
|
978
|
+
const w = t.get(f);
|
|
979
|
+
if (!w) continue;
|
|
980
|
+
w.refs.clear(), t.delete(f);
|
|
981
|
+
const p = w.model;
|
|
982
|
+
try {
|
|
983
|
+
(g = p.dispose) == null || g.call(p);
|
|
984
|
+
} catch (k) {
|
|
985
|
+
console.warn(`[xr-sdk registry] dispose threw for ${f}`, k);
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
i.setPipeline((f) => f.clear());
|
|
989
|
+
} };
|
|
990
|
+
}, et = {
|
|
991
|
+
lifecycle: "idle",
|
|
992
|
+
ready: !1,
|
|
993
|
+
error: void 0,
|
|
994
|
+
cameraTexture: void 0,
|
|
995
|
+
renderWidth: 0,
|
|
996
|
+
renderHeight: 0,
|
|
997
|
+
source: void 0,
|
|
998
|
+
sourceKind: void 0,
|
|
999
|
+
pipeline: /* @__PURE__ */ new Map()
|
|
1000
|
+
}, tt = () => {
|
|
1001
|
+
let i = { ...et, pipeline: /* @__PURE__ */ new Map() };
|
|
1002
|
+
const s = /* @__PURE__ */ new Set(), e = () => {
|
|
1003
|
+
s.forEach((t) => t());
|
|
1004
|
+
};
|
|
1005
|
+
return {
|
|
1006
|
+
getSnapshot: () => i,
|
|
1007
|
+
set: (t) => {
|
|
1008
|
+
let n = !1;
|
|
1009
|
+
for (const r of Object.keys(t))
|
|
1010
|
+
if (t[r] !== i[r]) {
|
|
1011
|
+
n = !0;
|
|
1012
|
+
break;
|
|
1013
|
+
}
|
|
1014
|
+
n && (i = { ...i, ...t }, e());
|
|
1015
|
+
},
|
|
1016
|
+
setPipeline: (t) => {
|
|
1017
|
+
const n = new Map(i.pipeline);
|
|
1018
|
+
return t(n), i = { ...i, pipeline: n }, e(), n;
|
|
1019
|
+
},
|
|
1020
|
+
subscribe: (t) => (s.add(t), () => {
|
|
1021
|
+
s.delete(t);
|
|
1022
|
+
})
|
|
1023
|
+
};
|
|
1024
|
+
}, st = ({
|
|
1025
|
+
renderHeight: i,
|
|
1026
|
+
renderWidth: s,
|
|
1027
|
+
videoHeight: e,
|
|
1028
|
+
videoWidth: t
|
|
1029
|
+
}) => ({
|
|
1030
|
+
height: i || e,
|
|
1031
|
+
width: s || t
|
|
1032
|
+
}), it = (i) => {
|
|
1033
|
+
const s = new Ae({ verbose: !0 }), e = tt();
|
|
1034
|
+
let t = i;
|
|
1035
|
+
const n = (a) => {
|
|
1036
|
+
var l;
|
|
1037
|
+
const o = (l = P.getOptions(a)) == null ? void 0 : l.targetFps;
|
|
1038
|
+
return typeof o == "number" ? o : t == null ? void 0 : t.targetFps;
|
|
1039
|
+
};
|
|
1040
|
+
let r, c, d, m, u, g, f, w, p, k, C = !1, R = !1;
|
|
1041
|
+
const E = /* @__PURE__ */ new Set(), b = /* @__PURE__ */ new Set(), A = async (a, o) => {
|
|
1042
|
+
if (!E.has(a) && e.getSnapshot().lifecycle === "running" && u) {
|
|
1043
|
+
E.add(a);
|
|
1044
|
+
try {
|
|
1045
|
+
await o.run({
|
|
1046
|
+
sourceCanvas: u,
|
|
1047
|
+
renderWidth: e.getSnapshot().renderWidth,
|
|
1048
|
+
renderHeight: e.getSnapshot().renderHeight,
|
|
1049
|
+
getSource: () => e.getSnapshot().sourceKind,
|
|
1050
|
+
targetFps: n(a)
|
|
1051
|
+
}), R && (await new Promise((l) => requestAnimationFrame(() => l())), await new Promise((l) => requestAnimationFrame(() => l())), o.pause());
|
|
1052
|
+
} catch (l) {
|
|
1053
|
+
throw E.delete(a), l;
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
}, P = Je(e, Z, (a, o) => A(a, o));
|
|
1057
|
+
e.subscribe(() => {
|
|
1058
|
+
if (e.getSnapshot().lifecycle !== "running") return;
|
|
1059
|
+
const a = e.getSnapshot().pipeline;
|
|
1060
|
+
for (const [o, l] of a.entries())
|
|
1061
|
+
E.has(o) || A(o, l);
|
|
1062
|
+
});
|
|
1063
|
+
const O = () => {
|
|
1064
|
+
if (k !== void 0 && (cancelAnimationFrame(k), k = void 0), c) {
|
|
1065
|
+
try {
|
|
1066
|
+
c.pause();
|
|
1067
|
+
} catch {
|
|
1068
|
+
}
|
|
1069
|
+
try {
|
|
1070
|
+
c.srcObject = null;
|
|
1071
|
+
} catch {
|
|
1072
|
+
}
|
|
1073
|
+
c = void 0;
|
|
1074
|
+
}
|
|
1075
|
+
if (d) {
|
|
1076
|
+
try {
|
|
1077
|
+
d.getTracks().forEach((a) => a.stop());
|
|
1078
|
+
} catch {
|
|
1079
|
+
}
|
|
1080
|
+
d = void 0;
|
|
1081
|
+
}
|
|
1082
|
+
}, X = () => {
|
|
1083
|
+
if (C || !c || !m || !u) return;
|
|
1084
|
+
const a = window.__xr_debugger_metrics;
|
|
1085
|
+
if (a != null && a.trackCameraFrame && a.trackCameraFrame(), (e.getSnapshot().sourceKind ?? "webcam") === "webcam")
|
|
1086
|
+
je(c, m);
|
|
1087
|
+
else {
|
|
1088
|
+
const v = m.getContext("2d");
|
|
1089
|
+
if (!v) return;
|
|
1090
|
+
const S = c.videoWidth, M = c.videoHeight;
|
|
1091
|
+
m.width !== S && (m.width = S), m.height !== M && (m.height = M), v.setTransform(1, 0, 0, 1, 0, 0), v.clearRect(0, 0, S, M), v.drawImage(c, 0, 0, S, M);
|
|
1092
|
+
}
|
|
1093
|
+
Ze(m, u, (t == null ? void 0 : t.cameraZoom) ?? 1.2);
|
|
1094
|
+
const l = e.getSnapshot().cameraTexture;
|
|
1095
|
+
l && (l.needsUpdate = !0);
|
|
1096
|
+
}, $ = () => {
|
|
1097
|
+
if (!c || !m || !u) return;
|
|
1098
|
+
if (R) {
|
|
1099
|
+
X();
|
|
1100
|
+
return;
|
|
1101
|
+
}
|
|
1102
|
+
const a = () => {
|
|
1103
|
+
k = requestAnimationFrame(a), X();
|
|
1104
|
+
};
|
|
1105
|
+
k = requestAnimationFrame(a);
|
|
1106
|
+
}, ae = async (a) => {
|
|
1107
|
+
if (a.source === y.WEBCAM)
|
|
1108
|
+
try {
|
|
1109
|
+
const o = t != null && t.camRes && t.camRes > 0 ? t.camRes : 960;
|
|
1110
|
+
return await navigator.mediaDevices.getUserMedia({
|
|
1111
|
+
video: {
|
|
1112
|
+
facingMode: "user",
|
|
1113
|
+
frameRate: { min: 24, ideal: 30, max: 60 },
|
|
1114
|
+
width: { ideal: o }
|
|
1115
|
+
}
|
|
1116
|
+
});
|
|
1117
|
+
} catch (o) {
|
|
1118
|
+
const l = o instanceof DOMException ? o.name : "", v = l === "NotReadableError" || l === "TrackStartError" ? "Camera is already in use by another app or browser tab. Close it and refresh." : l === "NotFoundError" || l === "DevicesNotFoundError" ? "No camera was found on this device." : l === "NotAllowedError" || l === "PermissionDeniedError" ? "Camera access was blocked. Allow camera permissions in your browser, then refresh." : "Unable to access camera. Allow camera permissions in your browser, then refresh.";
|
|
1119
|
+
throw e.set({ error: { error: "PermissionError", message: v } }), new F(o instanceof Error ? o.message : "getUserMedia failed");
|
|
1120
|
+
}
|
|
1121
|
+
return a.stream;
|
|
1122
|
+
}, H = async (a) => {
|
|
1123
|
+
const o = await ae(a);
|
|
1124
|
+
d = o;
|
|
1125
|
+
const l = await Qe(o);
|
|
1126
|
+
c = l, m = new OffscreenCanvas(l.videoWidth, l.videoHeight);
|
|
1127
|
+
const { renderWidth: v, renderHeight: S } = e.getSnapshot(), M = st({
|
|
1128
|
+
renderWidth: v,
|
|
1129
|
+
renderHeight: S,
|
|
1130
|
+
videoWidth: l.videoWidth,
|
|
1131
|
+
videoHeight: l.videoHeight
|
|
1132
|
+
});
|
|
1133
|
+
if (u)
|
|
1134
|
+
(u.width !== M.width || u.height !== M.height) && (u.width = M.width, u.height = M.height);
|
|
1135
|
+
else {
|
|
1136
|
+
u = new OffscreenCanvas(M.width, M.height);
|
|
1137
|
+
const me = J(u);
|
|
1138
|
+
e.set({ cameraTexture: me });
|
|
1139
|
+
}
|
|
1140
|
+
const j = a.source === y.STREAM ? a.static === !0 : !1;
|
|
1141
|
+
R = j, r = a.source === y.WEBCAM ? { source: y.WEBCAM, sourceKind: "webcam" } : {
|
|
1142
|
+
source: y.STREAM,
|
|
1143
|
+
stream: a.stream,
|
|
1144
|
+
static: j,
|
|
1145
|
+
sourceKind: a.sourceKind ?? "webcam"
|
|
1146
|
+
}, e.set({
|
|
1147
|
+
source: a.source,
|
|
1148
|
+
sourceKind: r.sourceKind,
|
|
1149
|
+
error: void 0
|
|
1150
|
+
});
|
|
1151
|
+
}, q = async (a) => {
|
|
1152
|
+
const o = a.sourceKind ?? "webcam";
|
|
1153
|
+
u = a.canvas;
|
|
1154
|
+
const l = J(u);
|
|
1155
|
+
e.set({
|
|
1156
|
+
cameraTexture: l,
|
|
1157
|
+
source: y.CANVAS,
|
|
1158
|
+
sourceKind: o,
|
|
1159
|
+
renderWidth: u.width,
|
|
1160
|
+
renderHeight: u.height,
|
|
1161
|
+
error: void 0
|
|
1162
|
+
}), r = {
|
|
1163
|
+
source: y.CANVAS,
|
|
1164
|
+
canvas: a.canvas,
|
|
1165
|
+
sourceKind: o
|
|
1166
|
+
};
|
|
1167
|
+
}, ne = (a, o) => !o || a.source !== o.source ? !1 : a.source === y.STREAM && o.source === y.STREAM ? a.stream === o.stream && (a.sourceKind ?? "webcam") === o.sourceKind : !0, re = async () => {
|
|
1168
|
+
if (!u) return;
|
|
1169
|
+
const a = e.getSnapshot().pipeline;
|
|
1170
|
+
for (const [, o] of a.entries())
|
|
1171
|
+
o.resume();
|
|
1172
|
+
await new Promise((o) => requestAnimationFrame(() => o())), await new Promise((o) => requestAnimationFrame(() => o()));
|
|
1173
|
+
for (const [, o] of a.entries())
|
|
1174
|
+
o.pause();
|
|
1175
|
+
}, oe = (a, o) => !o || o.source !== y.CANVAS ? !1 : a.canvas === o.canvas && (a.sourceKind ?? "webcam") === o.sourceKind, ce = async (a) => {
|
|
1176
|
+
const o = a.source;
|
|
1177
|
+
if (o !== y.WEBCAM && o !== y.STREAM)
|
|
1178
|
+
throw new F(`Invalid media source: ${String(o)}`);
|
|
1179
|
+
if (ne(a, r)) {
|
|
1180
|
+
s.log("Media source unchanged, skipping rebind.");
|
|
1181
|
+
return;
|
|
1182
|
+
}
|
|
1183
|
+
const { lifecycle: l } = e.getSnapshot();
|
|
1184
|
+
if (l === "idle" || l === "stopped") {
|
|
1185
|
+
await H(a), e.set({ lifecycle: "media-bound" }), s.log("Media source set to:", a.source);
|
|
1186
|
+
return;
|
|
1187
|
+
}
|
|
1188
|
+
if (l === "media-bound") {
|
|
1189
|
+
O(), await H(a), s.log("Media source rebound to:", a.source);
|
|
1190
|
+
return;
|
|
1191
|
+
}
|
|
1192
|
+
if (C = !0, await new Promise((v) => requestAnimationFrame(() => v())), O(), await H(a), C = !1, $(), R)
|
|
1193
|
+
await re();
|
|
1194
|
+
else {
|
|
1195
|
+
const v = e.getSnapshot().pipeline;
|
|
1196
|
+
for (const [, S] of v.entries()) S.resume();
|
|
1197
|
+
}
|
|
1198
|
+
s.log("Media source swapped to:", a.source);
|
|
1199
|
+
}, he = async (a) => {
|
|
1200
|
+
if (oe(a, r)) {
|
|
1201
|
+
s.log("Frame source unchanged, skipping rebind.");
|
|
1202
|
+
return;
|
|
1203
|
+
}
|
|
1204
|
+
const { lifecycle: o } = e.getSnapshot();
|
|
1205
|
+
if (o === "idle" || o === "stopped") {
|
|
1206
|
+
await q(a), e.set({ lifecycle: "media-bound" }), s.log("Frame source set to canvas.");
|
|
1207
|
+
return;
|
|
1208
|
+
}
|
|
1209
|
+
if (o === "media-bound") {
|
|
1210
|
+
O(), await q(a), s.log("Frame source rebound to canvas.");
|
|
1211
|
+
return;
|
|
1212
|
+
}
|
|
1213
|
+
C = !0, await new Promise((l) => requestAnimationFrame(() => l())), O(), await q(a), C = !1, s.log("Frame source swapped to canvas.");
|
|
1214
|
+
}, de = async () => {
|
|
1215
|
+
const { lifecycle: a } = e.getSnapshot();
|
|
1216
|
+
if (a === "running") return;
|
|
1217
|
+
if (a === "idle")
|
|
1218
|
+
throw new F("session.start() called before setMediaSource — call setMediaSource first");
|
|
1219
|
+
if (!u)
|
|
1220
|
+
throw new F("session.start(): media source not bound");
|
|
1221
|
+
const o = (r == null ? void 0 : r.source) === y.CANVAS;
|
|
1222
|
+
if (!o && (!c || !m))
|
|
1223
|
+
throw a === "stopped" ? new F(
|
|
1224
|
+
"session.start() called from stopped state without a media source — call setMediaSource first"
|
|
1225
|
+
) : new F("session.start(): media source not bound");
|
|
1226
|
+
o || $(), e.set({ lifecycle: "running" }), await new Promise((v) => requestAnimationFrame(() => v()));
|
|
1227
|
+
const l = e.getSnapshot().pipeline;
|
|
1228
|
+
for (const [v, S] of l.entries())
|
|
1229
|
+
await A(v, S);
|
|
1230
|
+
e.set({ ready: !0 }), s.log("XR session started.");
|
|
1231
|
+
}, Y = async () => {
|
|
1232
|
+
const { lifecycle: a } = e.getSnapshot();
|
|
1233
|
+
a === "stopped" || a === "idle" || (O(), e.set({ lifecycle: "stopped", ready: !1 }), s.log("XR session stopped."));
|
|
1234
|
+
}, ue = async () => {
|
|
1235
|
+
await Y(), P.releaseAll(), b.clear(), E.clear(), r = void 0, u = void 0, m = void 0, R = !1, e.set({
|
|
1236
|
+
lifecycle: "idle",
|
|
1237
|
+
cameraTexture: void 0,
|
|
1238
|
+
source: void 0
|
|
1239
|
+
}), s.log("XR session reset.");
|
|
1240
|
+
}, le = async (a) => {
|
|
1241
|
+
const o = "legacy:registerXRPipeline", l = [];
|
|
1242
|
+
await Promise.all(
|
|
1243
|
+
a.map(async (v) => {
|
|
1244
|
+
const { model: S } = v;
|
|
1245
|
+
if (!Z[S])
|
|
1246
|
+
throw new F(`Invalid model: ${S}`);
|
|
1247
|
+
b.has(S) || l.push(S);
|
|
1248
|
+
const M = "options" in v ? v.options : void 0;
|
|
1249
|
+
await P.acquire(S, o, M);
|
|
1250
|
+
})
|
|
1251
|
+
), l.length > 0 && (l.forEach((v) => b.add(v)), s.log("XR pipeline registered."));
|
|
1252
|
+
}, ge = (a) => {
|
|
1253
|
+
const o = P.get(a);
|
|
1254
|
+
if (!o)
|
|
1255
|
+
throw new F(`Model ${a} is not registered in the pipeline`);
|
|
1256
|
+
return o.getModelNode();
|
|
1257
|
+
}, Q = async (a, o) => {
|
|
1258
|
+
a <= 0 || o <= 0 || (e.set({ renderWidth: a, renderHeight: o }), u && (u.width !== a || u.height !== o) && (u.width = a, u.height = o), s.log(`Source canvas set. Render size: ${a}x${o}`));
|
|
1259
|
+
};
|
|
1260
|
+
return {
|
|
1261
|
+
store: e,
|
|
1262
|
+
registry: P,
|
|
1263
|
+
setMediaSource: ce,
|
|
1264
|
+
start: de,
|
|
1265
|
+
stop: Y,
|
|
1266
|
+
reset: ue,
|
|
1267
|
+
setFrameSource: he,
|
|
1268
|
+
registerXRPipeline: le,
|
|
1269
|
+
getModelNode: ge,
|
|
1270
|
+
captureFrame: () => {
|
|
1271
|
+
if (!g)
|
|
1272
|
+
throw new F(
|
|
1273
|
+
"session.captureFrame() called before the XRScene render target was bound — make sure <XRScene> is mounted"
|
|
1274
|
+
);
|
|
1275
|
+
if (!f || !w || !p)
|
|
1276
|
+
throw new F(
|
|
1277
|
+
"session.captureFrame() called before the R3F state was bound — make sure <XRScene> is mounted"
|
|
1278
|
+
);
|
|
1279
|
+
return f.render(w, p), g.toDataURL("image/png");
|
|
1280
|
+
},
|
|
1281
|
+
_setRenderTarget: async (a) => {
|
|
1282
|
+
var S;
|
|
1283
|
+
g = a;
|
|
1284
|
+
const o = (S = a.getBoundingClientRect) == null ? void 0 : S.call(a), l = o && o.width > 0 ? Math.round(o.width) : a.clientWidth, v = o && o.height > 0 ? Math.round(o.height) : a.clientHeight;
|
|
1285
|
+
await Q(l, v);
|
|
1286
|
+
},
|
|
1287
|
+
_setRenderSize: Q,
|
|
1288
|
+
_setR3FState: (a) => {
|
|
1289
|
+
f = a.renderer, w = a.scene, p = a.camera;
|
|
1290
|
+
},
|
|
1291
|
+
_getVideoElement: () => c,
|
|
1292
|
+
_setPerf: (a) => {
|
|
1293
|
+
if (t = a, e.getSnapshot().lifecycle !== "running" || !u) return;
|
|
1294
|
+
const o = e.getSnapshot().pipeline;
|
|
1295
|
+
for (const [l, v] of o.entries())
|
|
1296
|
+
v.run({
|
|
1297
|
+
sourceCanvas: u,
|
|
1298
|
+
renderWidth: e.getSnapshot().renderWidth,
|
|
1299
|
+
renderHeight: e.getSnapshot().renderHeight,
|
|
1300
|
+
getSource: () => e.getSnapshot().sourceKind,
|
|
1301
|
+
targetFps: n(l)
|
|
1302
|
+
});
|
|
1303
|
+
}
|
|
1304
|
+
};
|
|
1305
|
+
}, at = (i) => {
|
|
1306
|
+
const s = ee(() => it(i), []), e = JSON.stringify(i ?? {});
|
|
1307
|
+
return ve(() => {
|
|
1308
|
+
s._setPerf(i);
|
|
1309
|
+
}, [s, e]), s;
|
|
1310
|
+
}, ie = Ce(void 0), wt = () => {
|
|
1311
|
+
const i = Se(ie);
|
|
1312
|
+
if (!i)
|
|
1313
|
+
throw new F(
|
|
1314
|
+
"XR context can only be used inside XRProvider, did you forget to wrap your app in <XRProvider>?"
|
|
1315
|
+
);
|
|
1316
|
+
return i;
|
|
1317
|
+
}, vt = ({ children: i, perf: s }) => {
|
|
1318
|
+
const e = at(s), t = ee(() => ({ session: e }), [e]);
|
|
1319
|
+
return /* @__PURE__ */ we(ie.Provider, { value: t, children: i });
|
|
1320
|
+
};
|
|
1321
|
+
export {
|
|
1322
|
+
ft as R,
|
|
1323
|
+
_ as X,
|
|
1324
|
+
vt as a,
|
|
1325
|
+
y as b,
|
|
1326
|
+
V as c,
|
|
1327
|
+
pt as d,
|
|
1328
|
+
Ie as f,
|
|
1329
|
+
x as l,
|
|
1330
|
+
wt as u
|
|
1331
|
+
};
|