@omnidotdev/rdk 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/build/assets/onnxWorker-B_g0sj2t.js +1 -0
  2. package/build/{fiducial-CqsG94R9.js → fiducial-Bi9lhbyN.js} +1 -1
  3. package/build/{fiducial-dapuWTXg.cjs → fiducial-DdfnVsvF.cjs} +1 -1
  4. package/build/fiducial.cjs +1 -1
  5. package/build/fiducial.d.ts +1 -1
  6. package/build/fiducial.js +2 -2
  7. package/build/geolocation/geolocationBackend.d.ts +1 -1
  8. package/build/geolocation/geolocationBackend.d.ts.map +1 -1
  9. package/build/{geolocation-ClkbtHic.cjs → geolocation-D_rCGA-C.cjs} +1 -1
  10. package/build/{geolocation-DxPzqQPN.js → geolocation-DqxybM8d.js} +47 -37
  11. package/build/geolocation.cjs +1 -1
  12. package/build/geolocation.d.ts +1 -1
  13. package/build/geolocation.js +2 -2
  14. package/build/{immersive-Dp2-ZCRC.js → immersive-BTwiipnT.js} +1 -1
  15. package/build/{immersive-8m33PLug.cjs → immersive-DKHCJlkN.cjs} +1 -1
  16. package/build/immersive.cjs +1 -1
  17. package/build/immersive.d.ts +1 -1
  18. package/build/immersive.js +2 -2
  19. package/build/index.cjs +1 -1
  20. package/build/index.d.ts +1 -0
  21. package/build/index.d.ts.map +1 -1
  22. package/build/index.js +10 -9
  23. package/build/lib/types/engine/backend/Backend.interface.d.ts +1 -0
  24. package/build/lib/types/engine/backend/Backend.interface.d.ts.map +1 -1
  25. package/build/{magic-DAQdGGEO.js → magic-CPIBm_8-.js} +1 -1
  26. package/build/{magic-DznuBYgG.cjs → magic-CWhj_WTi.cjs} +1 -1
  27. package/build/magic.cjs +1 -1
  28. package/build/magic.d.ts +1 -1
  29. package/build/magic.js +2 -2
  30. package/build/useXRStore-8Vl3wsP5.cjs +1 -0
  31. package/build/{useXRStore-B3-MaaCu.js → useXRStore-Dsm1C1fI.js} +2 -1
  32. package/build/vision/CameraBackground.d.ts +15 -0
  33. package/build/vision/CameraBackground.d.ts.map +1 -0
  34. package/build/vision/HandTracker.d.ts +23 -0
  35. package/build/vision/HandTracker.d.ts.map +1 -0
  36. package/build/vision/VisionAnchor.d.ts +28 -0
  37. package/build/vision/VisionAnchor.d.ts.map +1 -0
  38. package/build/vision/VisionDebug.d.ts +18 -0
  39. package/build/vision/VisionDebug.d.ts.map +1 -0
  40. package/build/vision/VisionOverlay.d.ts +28 -0
  41. package/build/vision/VisionOverlay.d.ts.map +1 -0
  42. package/build/vision/VisionSession.d.ts +18 -0
  43. package/build/vision/VisionSession.d.ts.map +1 -0
  44. package/build/vision/VisionStatus.d.ts +15 -0
  45. package/build/vision/VisionStatus.d.ts.map +1 -0
  46. package/build/vision/gestures/builtinGestures.d.ts +9 -0
  47. package/build/vision/gestures/builtinGestures.d.ts.map +1 -0
  48. package/build/vision/gestures/detectGesture.d.ts +9 -0
  49. package/build/vision/gestures/detectGesture.d.ts.map +1 -0
  50. package/build/vision/gestures/index.d.ts +4 -0
  51. package/build/vision/gestures/index.d.ts.map +1 -0
  52. package/build/vision/gestures/types.d.ts +7 -0
  53. package/build/vision/gestures/types.d.ts.map +1 -0
  54. package/build/vision/index.d.ts +27 -0
  55. package/build/vision/index.d.ts.map +1 -0
  56. package/build/vision/landmarkMapping.d.ts +25 -0
  57. package/build/vision/landmarkMapping.d.ts.map +1 -0
  58. package/build/vision/presets.d.ts +25 -0
  59. package/build/vision/presets.d.ts.map +1 -0
  60. package/build/vision/providers/createVisionProvider.d.ts +5 -0
  61. package/build/vision/providers/createVisionProvider.d.ts.map +1 -0
  62. package/build/vision/providers/index.d.ts +2 -0
  63. package/build/vision/providers/index.d.ts.map +1 -0
  64. package/build/vision/providers/mediapipeProvider.d.ts +23 -0
  65. package/build/vision/providers/mediapipeProvider.d.ts.map +1 -0
  66. package/build/vision/providers/onnxProvider.d.ts +30 -0
  67. package/build/vision/providers/onnxProvider.d.ts.map +1 -0
  68. package/build/vision/types.d.ts +92 -0
  69. package/build/vision/types.d.ts.map +1 -0
  70. package/build/vision/useVisionBackend.d.ts +8 -0
  71. package/build/vision/useVisionBackend.d.ts.map +1 -0
  72. package/build/vision/useVisionFrame.d.ts +13 -0
  73. package/build/vision/useVisionFrame.d.ts.map +1 -0
  74. package/build/vision/visionBackend.d.ts +17 -0
  75. package/build/vision/visionBackend.d.ts.map +1 -0
  76. package/build/vision/worker/decoders/index.d.ts +6 -0
  77. package/build/vision/worker/decoders/index.d.ts.map +1 -0
  78. package/build/vision/worker/decoders/nms.d.ts +23 -0
  79. package/build/vision/worker/decoders/nms.d.ts.map +1 -0
  80. package/build/vision/worker/decoders/rfDetr.d.ts +10 -0
  81. package/build/vision/worker/decoders/rfDetr.d.ts.map +1 -0
  82. package/build/vision/worker/decoders/types.d.ts +40 -0
  83. package/build/vision/worker/decoders/types.d.ts.map +1 -0
  84. package/build/vision/worker/decoders/yolo.d.ts +11 -0
  85. package/build/vision/worker/decoders/yolo.d.ts.map +1 -0
  86. package/build/vision/worker/onnxWorker.d.ts +2 -0
  87. package/build/vision/worker/onnxWorker.d.ts.map +1 -0
  88. package/build/vision/worker/preprocess.d.ts +22 -0
  89. package/build/vision/worker/preprocess.d.ts.map +1 -0
  90. package/build/vision/worker/types.d.ts +50 -0
  91. package/build/vision/worker/types.d.ts.map +1 -0
  92. package/build/vision-BKOfvNSR.js +836 -0
  93. package/build/vision-CRUCg7uc.cjs +1 -0
  94. package/build/vision.cjs +1 -0
  95. package/build/vision.d.ts +2 -0
  96. package/build/vision.js +3 -0
  97. package/package.json +23 -8
  98. package/build/useXRStore-7Gy83DEg.cjs +0 -1
@@ -0,0 +1,836 @@
1
+ import { i as e, r as t } from "./useXRStore-Dsm1C1fI.js";
2
+ import { useCallback as n, useEffect as r, useMemo as i, useRef as a, useState as o } from "react";
3
+ import { Canvas as s, useFrame as c, useThree as l } from "@react-three/fiber";
4
+ import { Fragment as u, jsx as d, jsxs as f } from "react/jsx-runtime";
5
+ import { Object3D as p, Vector3 as m } from "three";
6
+ //#region src/vision/CameraBackground.tsx
7
+ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
8
+ let s = a(null), [c, l] = o(!1), u = a(t);
9
+ u.current = t;
10
+ let f = a(n);
11
+ return f.current = n, r(() => {
12
+ let t = !1;
13
+ return (async () => {
14
+ try {
15
+ let n = await navigator.mediaDevices.getUserMedia({ video: e || {
16
+ width: { ideal: 1280 },
17
+ height: { ideal: 720 },
18
+ facingMode: "user"
19
+ } });
20
+ if (t || !s.current) {
21
+ for (let e of n.getTracks()) e.stop();
22
+ return;
23
+ }
24
+ if (s.current.srcObject = n, await new Promise((e, t) => {
25
+ let n = s.current;
26
+ if (!n) {
27
+ t(/* @__PURE__ */ Error("Video ref lost"));
28
+ return;
29
+ }
30
+ let r = () => {
31
+ n.removeEventListener("canplay", r), n.removeEventListener("error", i), e();
32
+ }, i = () => {
33
+ n.removeEventListener("canplay", r), n.removeEventListener("error", i), t(/* @__PURE__ */ Error("Video failed to load"));
34
+ };
35
+ n.addEventListener("canplay", r, { once: !0 }), n.addEventListener("error", i, { once: !0 });
36
+ }), t) return;
37
+ await s.current.play(), l(!0), u.current?.(s.current);
38
+ } catch (e) {
39
+ if (t) return;
40
+ let n = e instanceof Error ? e : /* @__PURE__ */ Error("Failed to access camera");
41
+ console.error("Camera error:", n), f.current?.(n);
42
+ }
43
+ })(), () => {
44
+ if (t = !0, s.current?.srcObject) {
45
+ let e = s.current.srcObject;
46
+ for (let t of e.getTracks()) t.stop();
47
+ }
48
+ };
49
+ }, [e]), /* @__PURE__ */ d("video", {
50
+ ref: s,
51
+ style: {
52
+ position: "absolute",
53
+ inset: 0,
54
+ width: "100%",
55
+ height: "100%",
56
+ objectFit: "cover",
57
+ transform: i ? "scaleX(-1)" : void 0,
58
+ opacity: c ? 1 : 0,
59
+ transition: "opacity 0.5s ease"
60
+ },
61
+ autoPlay: !0,
62
+ playsInline: !0,
63
+ muted: !0
64
+ });
65
+ }, g = (e, t) => e.y < t.y, _ = (e) => [
66
+ Math.abs(e[4].x - e[9].x) > Math.abs(e[3].x - e[9].x),
67
+ g(e[8], e[6]),
68
+ g(e[12], e[10]),
69
+ g(e[16], e[14]),
70
+ g(e[20], e[18])
71
+ ], v = (e) => {
72
+ if (e.length < 21) return null;
73
+ let t = _(e);
74
+ return t[0] && !t[1] && !t[2] && !t[3] && !t[4] ? {
75
+ gesture: "thumbs_up",
76
+ confidence: .9
77
+ } : null;
78
+ }, y = (e) => {
79
+ if (e.length < 21) return null;
80
+ let t = _(e);
81
+ return !t[0] && t[1] && t[2] && !t[3] && !t[4] ? {
82
+ gesture: "peace",
83
+ confidence: .8
84
+ } : null;
85
+ }, b = (e) => {
86
+ if (e.length < 21) return null;
87
+ let t = _(e);
88
+ return Math.sqrt((e[4].x - e[8].x) ** 2 + (e[4].y - e[8].y) ** 2) < .05 && t[2] && t[3] && t[4] ? {
89
+ gesture: "ok",
90
+ confidence: .8
91
+ } : null;
92
+ }, x = (e) => e.length < 21 ? null : _(e).filter(Boolean).length === 5 ? {
93
+ gesture: "open_hand",
94
+ confidence: .7
95
+ } : null, S = (e) => e.length < 21 ? null : _(e).filter(Boolean).length <= 1 ? {
96
+ gesture: "fist",
97
+ confidence: .7
98
+ } : null, C = [
99
+ v,
100
+ y,
101
+ b,
102
+ x,
103
+ S
104
+ ], w = {
105
+ gesture: "none",
106
+ confidence: 0
107
+ }, T = (e, t = C) => {
108
+ let n = w;
109
+ for (let r of t) {
110
+ let t = r(e);
111
+ t && t.confidence > n.confidence && (n = t);
112
+ }
113
+ return n;
114
+ }, E = (e, t, n, r = !0) => {
115
+ let i = t.width / t.height, a = n.width / n.height, o = a > i ? a / i : 1, s = a > i ? 1 : i / a, c = .5 + (e.x - .5) * o, l = .5 + (e.y - .5) * s;
116
+ return [
117
+ (r ? .5 - c : c - .5) * t.width,
118
+ (.5 - l) * t.height,
119
+ 0
120
+ ];
121
+ }, D = (e) => {
122
+ let t = e.length || 1, n = 0, r = 0, i = 0;
123
+ for (let t of e) n += t.x, r += t.y, i += t.z ?? 0;
124
+ return {
125
+ x: n / t,
126
+ y: r / t,
127
+ z: i / t
128
+ };
129
+ }, O = {
130
+ isPending: !0,
131
+ isSuccess: !1,
132
+ provider: null,
133
+ videoElement: null,
134
+ onDetection: () => () => {}
135
+ }, k = () => {
136
+ let n = t((e) => e.backends).get(e.VISION);
137
+ return n ? {
138
+ ...n.getInternal(),
139
+ isPending: !1,
140
+ isSuccess: !0
141
+ } : O;
142
+ }, A = ({ onHandMove: e, onGesture: t, landmarkIndex: n = 9, mirror: i = !0, children: o }) => {
143
+ let { onDetection: s, isSuccess: u } = k(), { viewport: f } = l(), p = a(null), h = a(new m()), g = a("");
144
+ return r(() => {
145
+ if (u) return s((e) => {
146
+ p.current = e;
147
+ });
148
+ }, [u, s]), c(() => {
149
+ let r = p.current;
150
+ if (!r) return;
151
+ let a = r.hands[0];
152
+ if (!a || a.landmarks.length === 0) return;
153
+ let [o, s, c] = E(a.landmarks[n] ?? a.landmarks[0], f, r.frameSize, i);
154
+ h.current.set(o, s, c), e?.(h.current);
155
+ let l = T(a.landmarks);
156
+ l.gesture !== g.current && (g.current = l.gesture, t?.(l.gesture, l.confidence));
157
+ }), /* @__PURE__ */ d("group", { children: o });
158
+ }, j = /* @__PURE__ */ "person.bicycle.car.motorcycle.airplane.bus.train.truck.boat.traffic light.fire hydrant.stop sign.parking meter.bench.bird.cat.dog.horse.sheep.cow.elephant.bear.zebra.giraffe.backpack.umbrella.handbag.tie.suitcase.frisbee.skis.snowboard.sports ball.kite.baseball bat.baseball glove.skateboard.surfboard.tennis racket.bottle.wine glass.cup.fork.knife.spoon.bowl.banana.apple.sandwich.orange.broccoli.carrot.hot dog.pizza.donut.cake.chair.couch.potted plant.bed.dining table.toilet.tv.laptop.mouse.remote.keyboard.cell phone.microwave.oven.toaster.sink.refrigerator.book.clock.vase.scissors.teddy bear.hair drier.toothbrush".split("."), M = (e, t = {}) => ({
159
+ name: "yolo",
160
+ inputSize: 640,
161
+ labels: [...j],
162
+ ...t,
163
+ path: e,
164
+ decoder: "yolo"
165
+ }), N = (e, t = {}) => ({
166
+ name: "rf-detr",
167
+ inputSize: 560,
168
+ labels: [...j],
169
+ ...t,
170
+ path: e,
171
+ decoder: "rfdetr"
172
+ }), P = (e) => {
173
+ let { isSuccess: t, onDetection: n } = k(), [i, s] = o(null), c = e?.throttle ?? 16, l = a(0);
174
+ return r(() => {
175
+ if (!t) {
176
+ s(null);
177
+ return;
178
+ }
179
+ return n((e) => {
180
+ let t = performance.now();
181
+ t - l.current < c || (l.current = t, s(e));
182
+ });
183
+ }, [
184
+ t,
185
+ n,
186
+ c
187
+ ]), i;
188
+ }, F = 478, I = (e, t) => t === "faces" ? e.faces : t === "poses" ? e.poses : e.hands, L = ({ task: e, index: t = 0, debug: n = !1, mirror: o = !0, smoothing: s = .4, children: u }) => {
189
+ let { onDetection: h, isSuccess: g } = k(), { viewport: _ } = l(), v = a(null), y = a(null), b = a(null), x = i(() => new p(), []), S = i(() => new m(), []);
190
+ return r(() => {
191
+ y.current && (y.current.visible = !1), b.current && (b.current.count = 0);
192
+ }, []), r(() => {
193
+ if (g) return h((e) => {
194
+ v.current = e;
195
+ });
196
+ }, [g, h]), c(() => {
197
+ let r = v.current, i = y.current;
198
+ if (!r || !i) return;
199
+ let a = I(r, e)[t], c = b.current;
200
+ if (!a || a.landmarks.length === 0) {
201
+ i.visible = !1, c && (c.count = 0);
202
+ return;
203
+ }
204
+ let l = i.visible;
205
+ i.visible = !0;
206
+ let [u, d, f] = E(D(a.landmarks), _, r.frameSize, o);
207
+ if (S.set(u, d, f), !l || s >= 1 ? i.position.copy(S) : i.position.lerp(S, s), n && c) {
208
+ let e = i.position, t = Math.min(a.landmarks.length, F);
209
+ for (let n = 0; n < t; n++) {
210
+ let [t, i, s] = E(a.landmarks[n], _, r.frameSize, o);
211
+ x.position.set(t - e.x, i - e.y, s - e.z), x.updateMatrix(), c.setMatrixAt(n, x.matrix);
212
+ }
213
+ c.count = t, c.instanceMatrix.needsUpdate = !0;
214
+ }
215
+ }), /* @__PURE__ */ f("group", {
216
+ ref: y,
217
+ children: [u, n && /* @__PURE__ */ f("instancedMesh", {
218
+ ref: b,
219
+ args: [
220
+ void 0,
221
+ void 0,
222
+ F
223
+ ],
224
+ children: [/* @__PURE__ */ d("sphereGeometry", { args: [
225
+ .02,
226
+ 6,
227
+ 6
228
+ ] }), /* @__PURE__ */ d("meshBasicMaterial", { color: "#00ff88" })]
229
+ })]
230
+ });
231
+ }, R = ({ tasks: e, width: t = 320, height: n = 240, style: r }) => {
232
+ let i = P({ throttle: 50 });
233
+ return /* @__PURE__ */ f("div", {
234
+ style: {
235
+ background: "rgba(0, 0, 0, 0.8)",
236
+ color: "#00ff88",
237
+ padding: "8px",
238
+ borderRadius: "8px",
239
+ fontFamily: "monospace",
240
+ fontSize: "0.75rem",
241
+ width: t,
242
+ height: n,
243
+ overflow: "auto",
244
+ ...r
245
+ },
246
+ children: [/* @__PURE__ */ f("div", { children: ["Frame: ", i ? "active" : "none"] }), i && /* @__PURE__ */ f(u, { children: [
247
+ /* @__PURE__ */ f("div", { children: [
248
+ "Size: ",
249
+ i.frameSize.width,
250
+ "x",
251
+ i.frameSize.height
252
+ ] }),
253
+ e.includes("hands") && /* @__PURE__ */ f("div", { children: ["Hands: ", i.hands.length] }),
254
+ e.includes("faces") && /* @__PURE__ */ f("div", { children: ["Faces: ", i.faces.length] }),
255
+ e.includes("poses") && /* @__PURE__ */ f("div", { children: ["Poses: ", i.poses.length] }),
256
+ e.includes("objects") && /* @__PURE__ */ f("div", { children: ["Objects: ", i.objects.length] })
257
+ ] })]
258
+ });
259
+ }, z = class {
260
+ type = "mediapipe";
261
+ models = {};
262
+ isInitialized = !1;
263
+ isProcessing = !1;
264
+ isDisposed = !1;
265
+ videoElement = null;
266
+ animationId = null;
267
+ lastProcessTime = 0;
268
+ callbacks = [];
269
+ options;
270
+ constructor(e) {
271
+ this.options = e;
272
+ }
273
+ async initialize(e) {
274
+ this.videoElement = e;
275
+ let { FilesetResolver: t, HandLandmarker: n, FaceLandmarker: r, PoseLandmarker: i } = await import("@mediapipe/tasks-vision"), a = await t.forVisionTasks("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10/wasm"), o = this.options.useGpu ?? !0 ? "GPU" : "CPU", s = async (e) => {
276
+ if (o === "CPU") return e("CPU");
277
+ try {
278
+ return await e("GPU");
279
+ } catch {
280
+ return console.warn("MediaPipe GPU delegate failed, falling back to CPU"), e("CPU");
281
+ }
282
+ }, c = new Set(this.options.tasks ?? [
283
+ "hands",
284
+ "faces",
285
+ "poses"
286
+ ]), l = [
287
+ "hands",
288
+ "faces",
289
+ "poses"
290
+ ].filter((e) => c.has(e)), u = 0;
291
+ c.has("hands") && (this.options.onProgress?.({
292
+ step: ++u,
293
+ total: l.length,
294
+ label: "hand landmarker"
295
+ }), this.models.handLandmarker = await s((e) => n.createFromOptions(a, {
296
+ baseOptions: {
297
+ modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
298
+ delegate: e
299
+ },
300
+ runningMode: "VIDEO",
301
+ numHands: 2,
302
+ minHandDetectionConfidence: .7,
303
+ minHandPresenceConfidence: .5,
304
+ minTrackingConfidence: .5
305
+ }))), c.has("faces") && (this.options.onProgress?.({
306
+ step: ++u,
307
+ total: l.length,
308
+ label: "face landmarker"
309
+ }), this.models.faceLandmarker = await s((e) => r.createFromOptions(a, {
310
+ baseOptions: {
311
+ modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",
312
+ delegate: e
313
+ },
314
+ runningMode: "VIDEO",
315
+ numFaces: 1,
316
+ minFaceDetectionConfidence: .7,
317
+ minFacePresenceConfidence: .5,
318
+ minTrackingConfidence: .5,
319
+ outputFaceBlendshapes: !1,
320
+ outputFacialTransformationMatrixes: !1
321
+ }))), c.has("poses") && (this.options.onProgress?.({
322
+ step: ++u,
323
+ total: l.length,
324
+ label: "pose landmarker"
325
+ }), this.models.poseLandmarker = await s((e) => i.createFromOptions(a, {
326
+ baseOptions: {
327
+ modelAssetPath: "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task",
328
+ delegate: e
329
+ },
330
+ runningMode: "VIDEO",
331
+ numPoses: 1,
332
+ minPoseDetectionConfidence: .7,
333
+ minPosePresenceConfidence: .5,
334
+ minTrackingConfidence: .5,
335
+ outputSegmentationMasks: this.options.segmentation ?? !1
336
+ }))), this.isInitialized = !0, this.options.onProgress?.(null);
337
+ }
338
+ startDetection() {
339
+ !this.isInitialized || !this.videoElement || this.isProcessing || (this.isProcessing = !0, this.processFrame());
340
+ }
341
+ stopDetection() {
342
+ this.isProcessing = !1, this.animationId &&= (cancelAnimationFrame(this.animationId), null);
343
+ }
344
+ onDetection(e) {
345
+ return this.callbacks.push(e), () => {
346
+ let t = this.callbacks.indexOf(e);
347
+ t > -1 && this.callbacks.splice(t, 1);
348
+ };
349
+ }
350
+ dispose() {
351
+ this.isDisposed = !0, this.stopDetection(), this.models.handLandmarker && this.models.handLandmarker.close(), this.models.faceLandmarker && this.models.faceLandmarker.close(), this.models.poseLandmarker && this.models.poseLandmarker.close(), this.models = {}, this.videoElement = null, this.callbacks = [], this.isInitialized = !1, this.isProcessing = !1;
352
+ }
353
+ processFrame = () => {
354
+ if (!this.isProcessing || !this.videoElement || this.isDisposed) return;
355
+ let e = performance.now(), t = this.options.throttle ?? 16;
356
+ if (e - this.lastProcessTime < t) {
357
+ this.animationId = requestAnimationFrame(this.processFrame);
358
+ return;
359
+ }
360
+ let { videoWidth: n, videoHeight: r } = this.videoElement;
361
+ if (n === 0 || r === 0) {
362
+ this.animationId = requestAnimationFrame(this.processFrame);
363
+ return;
364
+ }
365
+ this.lastProcessTime = e;
366
+ let i = e;
367
+ try {
368
+ let e = {
369
+ hands: [],
370
+ faces: [],
371
+ poses: [],
372
+ objects: [],
373
+ timestamp: i,
374
+ frameSize: {
375
+ width: n,
376
+ height: r
377
+ }
378
+ };
379
+ if (this.models.handLandmarker) {
380
+ let t = this.models.handLandmarker.detectForVideo(this.videoElement, i);
381
+ e.hands = t.landmarks.map((e, n) => ({
382
+ landmarks: e.map((e) => ({
383
+ x: e.x,
384
+ y: e.y,
385
+ z: e.z || 0
386
+ })),
387
+ confidence: t.handednesses?.[n]?.[0]?.score || .5
388
+ }));
389
+ }
390
+ if (this.models.faceLandmarker && (e.faces = this.models.faceLandmarker.detectForVideo(this.videoElement, i).faceLandmarks.map((e) => {
391
+ let t = e.map((e) => e.x), n = e.map((e) => e.y), r = (Math.max(...t) - Math.min(...t)) * (Math.max(...n) - Math.min(...n)), i = Math.min(.95, Math.max(.6, r * 4));
392
+ return {
393
+ landmarks: e.map((e) => ({
394
+ x: e.x,
395
+ y: e.y,
396
+ z: e.z || 0
397
+ })),
398
+ confidence: i
399
+ };
400
+ })), this.models.poseLandmarker) {
401
+ let t = this.models.poseLandmarker.detectForVideo(this.videoElement, i);
402
+ e.poses = t.landmarks.map((e) => {
403
+ let t = 0, n = 0;
404
+ for (let r of e) typeof r.visibility == "number" && (t += r.visibility, n++);
405
+ let r = n > 0 ? t / n : .5;
406
+ return {
407
+ landmarks: e.map((e) => ({
408
+ x: e.x,
409
+ y: e.y,
410
+ z: e.z || 0
411
+ })),
412
+ confidence: r
413
+ };
414
+ });
415
+ let n = t.segmentationMasks?.[0];
416
+ if (n) {
417
+ let t = new OffscreenCanvas(n.width, n.height).getContext("2d"), r = n.getAsFloat32Array(), i = t.createImageData(n.width, n.height);
418
+ for (let e = 0; e < r.length; e++) {
419
+ let t = r[e] * 255;
420
+ i.data[e * 4] = t, i.data[e * 4 + 1] = t, i.data[e * 4 + 2] = t, i.data[e * 4 + 3] = 255;
421
+ }
422
+ e.segmentationMask = i;
423
+ }
424
+ }
425
+ this.handleResult(e);
426
+ } catch (e) {
427
+ console.error("Vision processing error:", e);
428
+ }
429
+ this.animationId = requestAnimationFrame(this.processFrame);
430
+ };
431
+ handleResult(e) {
432
+ let t = this.options.minConfidence ?? .7, n = this.options.maxResults ?? 2, r = {
433
+ ...e,
434
+ hands: e.hands.filter((e) => e.confidence >= t).slice(0, n),
435
+ faces: e.faces.filter((e) => e.confidence >= t).slice(0, n),
436
+ poses: e.poses.filter((e) => e.confidence >= t).slice(0, n),
437
+ objects: []
438
+ };
439
+ for (let e of this.callbacks) try {
440
+ e(r);
441
+ } catch (e) {
442
+ console.error("Error in vision callback:", e);
443
+ }
444
+ }
445
+ }, B = 5e3, V = class {
446
+ type = "onnx";
447
+ worker = null;
448
+ isInitialized = !1;
449
+ isDisposed = !1;
450
+ isProcessing = !1;
451
+ videoElement = null;
452
+ animationId = null;
453
+ loadedModels = /* @__PURE__ */ new Map();
454
+ lastProcessTime = 0;
455
+ processingStartTime = 0;
456
+ callbacks = [];
457
+ options;
458
+ messageHandler = null;
459
+ pendingModelCallbacks = /* @__PURE__ */ new Map();
460
+ constructor(e) {
461
+ this.options = e;
462
+ }
463
+ post(e, t) {
464
+ t ? this.worker?.postMessage(e, t) : this.worker?.postMessage(e);
465
+ }
466
+ async initialize(e) {
467
+ this.videoElement = e, this.worker = new Worker(new URL(
468
+ /* @vite-ignore */
469
+ "/assets/onnxWorker-B_g0sj2t.js",
470
+ "" + import.meta.url
471
+ ), { type: "module" }), await new Promise((e, t) => {
472
+ let n = setTimeout(() => {
473
+ t(/* @__PURE__ */ Error("ONNX worker initialization timeout"));
474
+ }, 1e4);
475
+ this.messageHandler = (r) => {
476
+ let i = r.data;
477
+ switch (i.type) {
478
+ case "initialized":
479
+ clearTimeout(n), this.isInitialized = !0, e();
480
+ break;
481
+ case "result":
482
+ this.isProcessing = !1, this.handleResult(i.result);
483
+ break;
484
+ case "modelLoaded": {
485
+ let e = this.pendingModelCallbacks.get(i.modelName);
486
+ e && (this.pendingModelCallbacks.delete(i.modelName), e.resolve());
487
+ break;
488
+ }
489
+ case "modelError": {
490
+ let e = this.pendingModelCallbacks.get(i.modelName);
491
+ e && (this.pendingModelCallbacks.delete(i.modelName), e.reject(Error(i.error)));
492
+ break;
493
+ }
494
+ case "error":
495
+ this.isProcessing = !1, this.isInitialized ? console.error("ONNX worker error:", i.error) : (clearTimeout(n), t(Error(i.error)));
496
+ break;
497
+ }
498
+ }, this.worker?.addEventListener("message", this.messageHandler), this.post({ type: "init" });
499
+ });
500
+ let t = this.options.onnx?.models ?? [];
501
+ for (let e of t) await this.loadModel(e);
502
+ }
503
+ startDetection() {
504
+ if (!this.isInitialized || !this.videoElement) {
505
+ console.warn("ONNX provider not initialized");
506
+ return;
507
+ }
508
+ let e = () => {
509
+ if (!this.videoElement || !this.isInitialized) return;
510
+ let t = Date.now(), n = this.options.throttle ?? 100;
511
+ if (t - this.lastProcessTime < n) {
512
+ this.animationId = requestAnimationFrame(e);
513
+ return;
514
+ }
515
+ this.isProcessing && this.processingStartTime > 0 && t - this.processingStartTime > B && (console.warn("ONNX processing stall detected, resetting"), this.isProcessing = !1), this.isProcessing || this.processCurrentFrame(), this.animationId = requestAnimationFrame(e);
516
+ };
517
+ e();
518
+ }
519
+ stopDetection() {
520
+ this.animationId &&= (cancelAnimationFrame(this.animationId), null), this.isProcessing = !1;
521
+ }
522
+ onDetection(e) {
523
+ return this.callbacks.push(e), () => {
524
+ let t = this.callbacks.indexOf(e);
525
+ t > -1 && this.callbacks.splice(t, 1);
526
+ };
527
+ }
528
+ dispose() {
529
+ this.isDisposed = !0, this.stopDetection(), this.worker &&= (this.messageHandler &&= (this.worker.removeEventListener("message", this.messageHandler), null), this.post({ type: "dispose" }), this.worker.terminate(), null), this.videoElement = null, this.callbacks.length = 0, this.loadedModels.clear(), this.pendingModelCallbacks.clear(), this.isInitialized = !1;
530
+ }
531
+ async loadModel(e) {
532
+ if (!this.worker) throw Error("Worker not initialized");
533
+ let t = new Promise((t, n) => {
534
+ this.pendingModelCallbacks.set(e.name, {
535
+ resolve: t,
536
+ reject: n
537
+ });
538
+ });
539
+ this.post({
540
+ type: "loadModel",
541
+ model: e
542
+ }), await t, this.loadedModels.set(e.name, e);
543
+ }
544
+ processCurrentFrame() {
545
+ if (!this.videoElement || !this.worker || this.isProcessing) return;
546
+ let e = Date.now();
547
+ this.isProcessing = !0, this.lastProcessTime = e, this.processingStartTime = e;
548
+ let { videoWidth: t, videoHeight: n } = this.videoElement;
549
+ createImageBitmap(this.videoElement).then((e) => {
550
+ if (this.isDisposed) {
551
+ e.close();
552
+ return;
553
+ }
554
+ this.post({
555
+ type: "process",
556
+ imageBitmap: e,
557
+ sourceWidth: t,
558
+ sourceHeight: n,
559
+ options: {
560
+ minConfidence: this.options.minConfidence ?? .5,
561
+ maxResults: this.options.maxResults ?? 100
562
+ }
563
+ }, [e]);
564
+ }, (e) => {
565
+ this.isProcessing = !1, console.error("Error creating ImageBitmap:", e);
566
+ });
567
+ }
568
+ handleResult(e) {
569
+ let t = this.options.minConfidence ?? .5, n = this.options.maxResults ?? 100, r = {
570
+ hands: [],
571
+ faces: [],
572
+ poses: [],
573
+ objects: e.detections.filter((e) => e.confidence >= t).sort((e, t) => t.confidence - e.confidence).slice(0, n),
574
+ timestamp: e.timestamp,
575
+ frameSize: e.frameSize,
576
+ processingTime: e.processingTime
577
+ };
578
+ for (let e of this.callbacks) try {
579
+ e(r);
580
+ } catch (e) {
581
+ console.error("Error in ONNX detection callback:", e);
582
+ }
583
+ }
584
+ }, H = (e) => e.provider === "onnx" ? new V(e) : new z(e), U = async (e) => {
585
+ let t = document.createElement("video");
586
+ t.style.display = "none", document.body.appendChild(t);
587
+ try {
588
+ return t.srcObject = await navigator.mediaDevices.getUserMedia({ video: {
589
+ width: {
590
+ ideal: 1280,
591
+ min: 640
592
+ },
593
+ height: {
594
+ ideal: 720,
595
+ min: 480
596
+ },
597
+ facingMode: e.provider === "onnx" ? "environment" : "user",
598
+ frameRate: {
599
+ ideal: 30,
600
+ min: 15
601
+ }
602
+ } }), t.playsInline = !0, t.muted = !0, await new Promise((e, n) => {
603
+ t.onloadedmetadata = () => {
604
+ t.play().then(e).catch(n);
605
+ }, t.onerror = n;
606
+ }), t;
607
+ } catch (e) {
608
+ throw document.body.removeChild(t), e;
609
+ }
610
+ }, W = (e) => {
611
+ if (!e) return;
612
+ let t = e.srcObject;
613
+ if (t) for (let e of t.getTracks()) e.stop();
614
+ e.parentNode && e.parentNode.removeChild(e);
615
+ }, G = (t) => {
616
+ let n = null, r = null, i = !1, a = null, o = [], s = (e) => (o.push(e), () => {
617
+ let t = o.indexOf(e);
618
+ t > -1 && o.splice(t, 1);
619
+ });
620
+ return {
621
+ type: e.VISION,
622
+ async init(e) {
623
+ t.videoElement ? (r = t.videoElement, i = !1) : (r = await U(t), i = !0), n = H(t), await n.initialize(r), a = n.onDetection((e) => {
624
+ for (let t of o) try {
625
+ t(e);
626
+ } catch (e) {
627
+ console.error("Error in vision detection callback:", e);
628
+ }
629
+ }), n.startDetection();
630
+ },
631
+ update(e) {},
632
+ dispose() {
633
+ a?.(), a = null, n?.dispose(), i && W(r), r = null, n = null, o.length = 0;
634
+ },
635
+ getInternal: () => ({
636
+ provider: n,
637
+ videoElement: r,
638
+ onDetection: s
639
+ })
640
+ };
641
+ }, K = ({ options: e, children: n, onError: i }) => {
642
+ let { scene: o, camera: s, gl: c } = l(), { registerBackend: f, unregisterBackend: p } = t(), m = a(null), h = a(e);
643
+ h.current = e;
644
+ let g = a(i);
645
+ return g.current = i, r(() => {
646
+ if (m.current) return;
647
+ let e = !1;
648
+ return (async () => {
649
+ try {
650
+ let t = G(h.current);
651
+ if (e) return;
652
+ if (await f(t, {
653
+ scene: o,
654
+ camera: s,
655
+ renderer: c
656
+ }), e) {
657
+ p(t);
658
+ return;
659
+ }
660
+ m.current = t;
661
+ } catch (e) {
662
+ let t = e instanceof Error ? e : Error(String(e));
663
+ console.error("[VisionSession] Failed to initialize:", t), g.current?.(t);
664
+ }
665
+ })(), () => {
666
+ e = !0, m.current &&= (p(m.current), null);
667
+ };
668
+ }, [
669
+ o,
670
+ s,
671
+ c,
672
+ f,
673
+ p
674
+ ]), /* @__PURE__ */ d(u, { children: n });
675
+ }, q = ({ tasks: e, progress: t, showDebug: n, onToggleDebug: r }) => {
676
+ let i = P({ throttle: 100 }), a = (e) => {
677
+ if (!i) return !1;
678
+ switch (e) {
679
+ case "hands": return i.hands.length > 0;
680
+ case "faces": return i.faces.length > 0;
681
+ case "poses": return i.poses.length > 0;
682
+ case "objects": return i.objects.length > 0;
683
+ default: return !1;
684
+ }
685
+ }, o = e.some(a), s = i !== null;
686
+ return /* @__PURE__ */ f(u, { children: [/* @__PURE__ */ f("div", {
687
+ style: {
688
+ position: "fixed",
689
+ top: "20px",
690
+ left: "20px",
691
+ background: "rgba(0, 0, 0, 0.7)",
692
+ color: "white",
693
+ padding: "12px",
694
+ borderRadius: "8px",
695
+ fontFamily: "system-ui, sans-serif",
696
+ fontSize: "0.9rem",
697
+ zIndex: 1e3
698
+ },
699
+ children: [/* @__PURE__ */ f("div", {
700
+ style: {
701
+ display: "flex",
702
+ alignItems: "center",
703
+ gap: "8px",
704
+ marginBottom: "8px"
705
+ },
706
+ children: [/* @__PURE__ */ d("div", { style: {
707
+ width: "8px",
708
+ height: "8px",
709
+ borderRadius: "50%",
710
+ background: s ? "#00ff88" : "#ffaa00"
711
+ } }), /* @__PURE__ */ d("span", { children: o ? "Active" : s ? "Ready" : t ? `Loading models (${t.step}/${t.total}) ${t.label}...` : "Starting..." })]
712
+ }), e.map((e) => /* @__PURE__ */ f("div", {
713
+ style: {
714
+ fontSize: "0.8rem",
715
+ opacity: .8,
716
+ display: "flex",
717
+ justifyContent: "space-between",
718
+ gap: "12px"
719
+ },
720
+ children: [/* @__PURE__ */ d("span", { children: e }), /* @__PURE__ */ d("span", { children: a(e) ? "active" : s ? "idle" : "..." })]
721
+ }, e))]
722
+ }), /* @__PURE__ */ f("button", {
723
+ onClick: r,
724
+ type: "button",
725
+ style: {
726
+ position: "fixed",
727
+ top: "20px",
728
+ right: "20px",
729
+ background: "rgba(0, 0, 0, 0.7)",
730
+ color: "white",
731
+ border: "1px solid #333",
732
+ padding: "8px 12px",
733
+ borderRadius: "6px",
734
+ cursor: "pointer",
735
+ fontFamily: "system-ui, sans-serif",
736
+ fontSize: "0.8rem",
737
+ zIndex: 1e3
738
+ },
739
+ children: [n ? "Hide" : "Show", " Debug"]
740
+ })] });
741
+ }, J = {}, Y = ({ tasks: e = ["hands"], options: t, children: r, showDebug: c = !1, cameraConstraints: l, canvasStyle: u = J, onReady: p, onError: m }) => {
742
+ let [g, _] = o(c), [v, y] = o(null), [b, x] = o(null), S = a(p);
743
+ S.current = p;
744
+ let C = a(x);
745
+ C.current = x;
746
+ let w = n((e) => C.current(e), []), T = n((e) => {
747
+ y(e), S.current?.();
748
+ }, []), E = i(() => ({
749
+ tasks: e,
750
+ minConfidence: .6,
751
+ maxResults: 2,
752
+ useGpu: !0,
753
+ ...t,
754
+ videoElement: v ?? void 0,
755
+ onProgress: w
756
+ }), [
757
+ e,
758
+ t,
759
+ v,
760
+ w
761
+ ]);
762
+ return /* @__PURE__ */ f("div", {
763
+ style: {
764
+ position: "relative",
765
+ width: "100%",
766
+ height: "100%"
767
+ },
768
+ children: [
769
+ /* @__PURE__ */ d(h, {
770
+ constraints: l,
771
+ onReady: T,
772
+ onError: m
773
+ }),
774
+ v && /* @__PURE__ */ d(s, {
775
+ style: {
776
+ position: "absolute",
777
+ inset: 0,
778
+ pointerEvents: "none",
779
+ ...u
780
+ },
781
+ camera: {
782
+ position: [
783
+ 0,
784
+ 0,
785
+ 5
786
+ ],
787
+ fov: 75
788
+ },
789
+ gl: {
790
+ alpha: !0,
791
+ premultipliedAlpha: !1
792
+ },
793
+ children: /* @__PURE__ */ d(X, {
794
+ options: E,
795
+ onError: m,
796
+ children: r
797
+ })
798
+ }),
799
+ /* @__PURE__ */ d(q, {
800
+ tasks: e,
801
+ progress: b,
802
+ showDebug: g,
803
+ onToggleDebug: () => _((e) => !e)
804
+ }),
805
+ g && /* @__PURE__ */ d(R, {
806
+ tasks: e,
807
+ width: 320,
808
+ height: 240,
809
+ style: {
810
+ position: "fixed",
811
+ bottom: "80px",
812
+ right: "20px",
813
+ maxWidth: "calc(100vw - 40px)",
814
+ overflow: "hidden"
815
+ }
816
+ })
817
+ ]
818
+ });
819
+ }, X = ({ options: e, onError: t, children: n }) => /* @__PURE__ */ f(K, {
820
+ options: e,
821
+ onError: t,
822
+ children: [
823
+ /* @__PURE__ */ d("ambientLight", { intensity: .6 }),
824
+ /* @__PURE__ */ d("directionalLight", {
825
+ position: [
826
+ 10,
827
+ 10,
828
+ 10
829
+ ],
830
+ intensity: .8
831
+ }),
832
+ n
833
+ ]
834
+ });
835
+ //#endregion
836
+ export { h as S, S as _, R as a, y as b, j as c, A as d, k as f, C as g, T as h, G as i, N as l, D as m, q as n, L as o, E as p, K as r, P as s, Y as t, M as u, b as v, v as x, x as y };