@omnidotdev/rdk 0.10.1 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/build/assets/onnxWorker-BNjoneWr.js +1 -0
  2. package/build/{fiducial-CqsG94R9.js → fiducial-Bi9lhbyN.js} +1 -1
  3. package/build/{fiducial-dapuWTXg.cjs → fiducial-DdfnVsvF.cjs} +1 -1
  4. package/build/fiducial.cjs +1 -1
  5. package/build/fiducial.js +2 -2
  6. package/build/{geolocation-tCLtH8CU.cjs → geolocation-D_rCGA-C.cjs} +1 -1
  7. package/build/{geolocation-BIUFKLO_.js → geolocation-DqxybM8d.js} +1 -1
  8. package/build/geolocation.cjs +1 -1
  9. package/build/geolocation.js +2 -2
  10. package/build/{immersive-Dp2-ZCRC.js → immersive-BTwiipnT.js} +1 -1
  11. package/build/{immersive-8m33PLug.cjs → immersive-DKHCJlkN.cjs} +1 -1
  12. package/build/immersive.cjs +1 -1
  13. package/build/immersive.js +2 -2
  14. package/build/index.cjs +1 -1
  15. package/build/index.d.ts +1 -0
  16. package/build/index.d.ts.map +1 -1
  17. package/build/index.js +10 -9
  18. package/build/lib/types/engine/backend/Backend.interface.d.ts +1 -0
  19. package/build/lib/types/engine/backend/Backend.interface.d.ts.map +1 -1
  20. package/build/{magic-DAQdGGEO.js → magic-CPIBm_8-.js} +1 -1
  21. package/build/{magic-DznuBYgG.cjs → magic-CWhj_WTi.cjs} +1 -1
  22. package/build/magic.cjs +1 -1
  23. package/build/magic.js +2 -2
  24. package/build/useXRStore-8Vl3wsP5.cjs +1 -0
  25. package/build/{useXRStore-B3-MaaCu.js → useXRStore-Dsm1C1fI.js} +2 -1
  26. package/build/vision/CameraBackground.d.ts +15 -0
  27. package/build/vision/CameraBackground.d.ts.map +1 -0
  28. package/build/vision/HandTracker.d.ts +23 -0
  29. package/build/vision/HandTracker.d.ts.map +1 -0
  30. package/build/vision/VisionAnchor.d.ts +28 -0
  31. package/build/vision/VisionAnchor.d.ts.map +1 -0
  32. package/build/vision/VisionDebug.d.ts +18 -0
  33. package/build/vision/VisionDebug.d.ts.map +1 -0
  34. package/build/vision/VisionOverlay.d.ts +28 -0
  35. package/build/vision/VisionOverlay.d.ts.map +1 -0
  36. package/build/vision/VisionSession.d.ts +18 -0
  37. package/build/vision/VisionSession.d.ts.map +1 -0
  38. package/build/vision/VisionStatus.d.ts +15 -0
  39. package/build/vision/VisionStatus.d.ts.map +1 -0
  40. package/build/vision/gestures/builtinGestures.d.ts +9 -0
  41. package/build/vision/gestures/builtinGestures.d.ts.map +1 -0
  42. package/build/vision/gestures/detectGesture.d.ts +9 -0
  43. package/build/vision/gestures/detectGesture.d.ts.map +1 -0
  44. package/build/vision/gestures/index.d.ts +4 -0
  45. package/build/vision/gestures/index.d.ts.map +1 -0
  46. package/build/vision/gestures/types.d.ts +7 -0
  47. package/build/vision/gestures/types.d.ts.map +1 -0
  48. package/build/vision/index.d.ts +27 -0
  49. package/build/vision/index.d.ts.map +1 -0
  50. package/build/vision/landmarkMapping.d.ts +25 -0
  51. package/build/vision/landmarkMapping.d.ts.map +1 -0
  52. package/build/vision/presets.d.ts +34 -0
  53. package/build/vision/presets.d.ts.map +1 -0
  54. package/build/vision/providers/createVisionProvider.d.ts +5 -0
  55. package/build/vision/providers/createVisionProvider.d.ts.map +1 -0
  56. package/build/vision/providers/index.d.ts +2 -0
  57. package/build/vision/providers/index.d.ts.map +1 -0
  58. package/build/vision/providers/mediapipeProvider.d.ts +23 -0
  59. package/build/vision/providers/mediapipeProvider.d.ts.map +1 -0
  60. package/build/vision/providers/onnxProvider.d.ts +30 -0
  61. package/build/vision/providers/onnxProvider.d.ts.map +1 -0
  62. package/build/vision/types.d.ts +110 -0
  63. package/build/vision/types.d.ts.map +1 -0
  64. package/build/vision/useVisionBackend.d.ts +8 -0
  65. package/build/vision/useVisionBackend.d.ts.map +1 -0
  66. package/build/vision/useVisionFrame.d.ts +13 -0
  67. package/build/vision/useVisionFrame.d.ts.map +1 -0
  68. package/build/vision/visionBackend.d.ts +17 -0
  69. package/build/vision/visionBackend.d.ts.map +1 -0
  70. package/build/vision/worker/decoders/index.d.ts +6 -0
  71. package/build/vision/worker/decoders/index.d.ts.map +1 -0
  72. package/build/vision/worker/decoders/nms.d.ts +23 -0
  73. package/build/vision/worker/decoders/nms.d.ts.map +1 -0
  74. package/build/vision/worker/decoders/rfDetr.d.ts +10 -0
  75. package/build/vision/worker/decoders/rfDetr.d.ts.map +1 -0
  76. package/build/vision/worker/decoders/types.d.ts +45 -0
  77. package/build/vision/worker/decoders/types.d.ts.map +1 -0
  78. package/build/vision/worker/decoders/yolo.d.ts +11 -0
  79. package/build/vision/worker/decoders/yolo.d.ts.map +1 -0
  80. package/build/vision/worker/decoders/yoloSeg.d.ts +11 -0
  81. package/build/vision/worker/decoders/yoloSeg.d.ts.map +1 -0
  82. package/build/vision/worker/onnxWorker.d.ts +2 -0
  83. package/build/vision/worker/onnxWorker.d.ts.map +1 -0
  84. package/build/vision/worker/preprocess.d.ts +22 -0
  85. package/build/vision/worker/preprocess.d.ts.map +1 -0
  86. package/build/vision/worker/types.d.ts +51 -0
  87. package/build/vision/worker/types.d.ts.map +1 -0
  88. package/build/vision-Bdhf3N-w.js +844 -0
  89. package/build/vision-sMKG7-na.cjs +1 -0
  90. package/build/vision.cjs +1 -0
  91. package/build/vision.d.ts +2 -0
  92. package/build/vision.js +3 -0
  93. package/package.json +16 -1
  94. package/build/useXRStore-7Gy83DEg.cjs +0 -1
@@ -0,0 +1,844 @@
1
+ import { i as e, r as t } from "./useXRStore-Dsm1C1fI.js";
2
+ import { useCallback as n, useEffect as r, useMemo as i, useRef as a, useState as o } from "react";
3
+ import { Canvas as s, useFrame as c, useThree as l } from "@react-three/fiber";
4
+ import { Fragment as u, jsx as d, jsxs as f } from "react/jsx-runtime";
5
+ import { Object3D as p, Vector3 as m } from "three";
6
+ //#region src/vision/CameraBackground.tsx
7
+ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
8
+ let s = a(null), [c, l] = o(!1), u = a(t);
9
+ u.current = t;
10
+ let f = a(n);
11
+ return f.current = n, r(() => {
12
+ let t = !1;
13
+ return (async () => {
14
+ try {
15
+ let n = await navigator.mediaDevices.getUserMedia({ video: e || {
16
+ width: { ideal: 1280 },
17
+ height: { ideal: 720 },
18
+ facingMode: "user"
19
+ } });
20
+ if (t || !s.current) {
21
+ for (let e of n.getTracks()) e.stop();
22
+ return;
23
+ }
24
+ if (s.current.srcObject = n, await new Promise((e, t) => {
25
+ let n = s.current;
26
+ if (!n) {
27
+ t(/* @__PURE__ */ Error("Video ref lost"));
28
+ return;
29
+ }
30
+ let r = () => {
31
+ n.removeEventListener("canplay", r), n.removeEventListener("error", i), e();
32
+ }, i = () => {
33
+ n.removeEventListener("canplay", r), n.removeEventListener("error", i), t(/* @__PURE__ */ Error("Video failed to load"));
34
+ };
35
+ n.addEventListener("canplay", r, { once: !0 }), n.addEventListener("error", i, { once: !0 });
36
+ }), t) return;
37
+ await s.current.play(), l(!0), u.current?.(s.current);
38
+ } catch (e) {
39
+ if (t) return;
40
+ let n = e instanceof Error ? e : /* @__PURE__ */ Error("Failed to access camera");
41
+ console.error("Camera error:", n), f.current?.(n);
42
+ }
43
+ })(), () => {
44
+ if (t = !0, s.current?.srcObject) {
45
+ let e = s.current.srcObject;
46
+ for (let t of e.getTracks()) t.stop();
47
+ }
48
+ };
49
+ }, [e]), /* @__PURE__ */ d("video", {
50
+ ref: s,
51
+ style: {
52
+ position: "absolute",
53
+ inset: 0,
54
+ width: "100%",
55
+ height: "100%",
56
+ objectFit: "cover",
57
+ transform: i ? "scaleX(-1)" : void 0,
58
+ opacity: c ? 1 : 0,
59
+ transition: "opacity 0.5s ease"
60
+ },
61
+ autoPlay: !0,
62
+ playsInline: !0,
63
+ muted: !0
64
+ });
65
+ }, g = (e, t) => e.y < t.y, _ = (e) => [
66
+ Math.abs(e[4].x - e[9].x) > Math.abs(e[3].x - e[9].x),
67
+ g(e[8], e[6]),
68
+ g(e[12], e[10]),
69
+ g(e[16], e[14]),
70
+ g(e[20], e[18])
71
+ ], v = (e) => {
72
+ if (e.length < 21) return null;
73
+ let t = _(e);
74
+ return t[0] && !t[1] && !t[2] && !t[3] && !t[4] ? {
75
+ gesture: "thumbs_up",
76
+ confidence: .9
77
+ } : null;
78
+ }, y = (e) => {
79
+ if (e.length < 21) return null;
80
+ let t = _(e);
81
+ return !t[0] && t[1] && t[2] && !t[3] && !t[4] ? {
82
+ gesture: "peace",
83
+ confidence: .8
84
+ } : null;
85
+ }, b = (e) => {
86
+ if (e.length < 21) return null;
87
+ let t = _(e);
88
+ return Math.sqrt((e[4].x - e[8].x) ** 2 + (e[4].y - e[8].y) ** 2) < .05 && t[2] && t[3] && t[4] ? {
89
+ gesture: "ok",
90
+ confidence: .8
91
+ } : null;
92
+ }, x = (e) => e.length < 21 ? null : _(e).filter(Boolean).length === 5 ? {
93
+ gesture: "open_hand",
94
+ confidence: .7
95
+ } : null, S = (e) => e.length < 21 ? null : _(e).filter(Boolean).length <= 1 ? {
96
+ gesture: "fist",
97
+ confidence: .7
98
+ } : null, C = [
99
+ v,
100
+ y,
101
+ b,
102
+ x,
103
+ S
104
+ ], w = {
105
+ gesture: "none",
106
+ confidence: 0
107
+ }, T = (e, t = C) => {
108
+ let n = w;
109
+ for (let r of t) {
110
+ let t = r(e);
111
+ t && t.confidence > n.confidence && (n = t);
112
+ }
113
+ return n;
114
+ }, E = (e, t, n, r = !0) => {
115
+ let i = t.width / t.height, a = n.width / n.height, o = a > i ? a / i : 1, s = a > i ? 1 : i / a, c = .5 + (e.x - .5) * o, l = .5 + (e.y - .5) * s;
116
+ return [
117
+ (r ? .5 - c : c - .5) * t.width,
118
+ (.5 - l) * t.height,
119
+ 0
120
+ ];
121
+ }, D = (e) => {
122
+ let t = e.length || 1, n = 0, r = 0, i = 0;
123
+ for (let t of e) n += t.x, r += t.y, i += t.z ?? 0;
124
+ return {
125
+ x: n / t,
126
+ y: r / t,
127
+ z: i / t
128
+ };
129
+ }, O = {
130
+ isPending: !0,
131
+ isSuccess: !1,
132
+ provider: null,
133
+ videoElement: null,
134
+ onDetection: () => () => {}
135
+ }, k = () => {
136
+ let n = t((e) => e.backends).get(e.VISION);
137
+ return n ? {
138
+ ...n.getInternal(),
139
+ isPending: !1,
140
+ isSuccess: !0
141
+ } : O;
142
+ }, A = ({ onHandMove: e, onGesture: t, landmarkIndex: n = 9, mirror: i = !0, children: o }) => {
143
+ let { onDetection: s, isSuccess: u } = k(), { viewport: f } = l(), p = a(null), h = a(new m()), g = a("");
144
+ return r(() => {
145
+ if (u) return s((e) => {
146
+ p.current = e;
147
+ });
148
+ }, [u, s]), c(() => {
149
+ let r = p.current;
150
+ if (!r) return;
151
+ let a = r.hands[0];
152
+ if (!a || a.landmarks.length === 0) return;
153
+ let [o, s, c] = E(a.landmarks[n] ?? a.landmarks[0], f, r.frameSize, i);
154
+ h.current.set(o, s, c), e?.(h.current);
155
+ let l = T(a.landmarks);
156
+ l.gesture !== g.current && (g.current = l.gesture, t?.(l.gesture, l.confidence));
157
+ }), /* @__PURE__ */ d("group", { children: o });
158
+ }, j = /* @__PURE__ */ "person.bicycle.car.motorcycle.airplane.bus.train.truck.boat.traffic light.fire hydrant.stop sign.parking meter.bench.bird.cat.dog.horse.sheep.cow.elephant.bear.zebra.giraffe.backpack.umbrella.handbag.tie.suitcase.frisbee.skis.snowboard.sports ball.kite.baseball bat.baseball glove.skateboard.surfboard.tennis racket.bottle.wine glass.cup.fork.knife.spoon.bowl.banana.apple.sandwich.orange.broccoli.carrot.hot dog.pizza.donut.cake.chair.couch.potted plant.bed.dining table.toilet.tv.laptop.mouse.remote.keyboard.cell phone.microwave.oven.toaster.sink.refrigerator.book.clock.vase.scissors.teddy bear.hair drier.toothbrush".split("."), M = (e, t = {}) => ({
159
+ name: "yolo",
160
+ inputSize: 640,
161
+ labels: [...j],
162
+ ...t,
163
+ path: e,
164
+ decoder: "yolo"
165
+ }), N = (e, t = {}) => ({
166
+ name: "rf-detr",
167
+ inputSize: 560,
168
+ labels: [...j],
169
+ ...t,
170
+ path: e,
171
+ decoder: "rfdetr"
172
+ }), P = (e, t = {}) => ({
173
+ name: "yolo-seg",
174
+ inputSize: 640,
175
+ labels: [...j],
176
+ ...t,
177
+ path: e,
178
+ decoder: "yoloseg"
179
+ }), F = (e) => {
180
+ let { isSuccess: t, onDetection: n } = k(), [i, s] = o(null), c = e?.throttle ?? 16, l = a(0);
181
+ return r(() => {
182
+ if (!t) {
183
+ s(null);
184
+ return;
185
+ }
186
+ return n((e) => {
187
+ let t = performance.now();
188
+ t - l.current < c || (l.current = t, s(e));
189
+ });
190
+ }, [
191
+ t,
192
+ n,
193
+ c
194
+ ]), i;
195
+ }, I = 478, L = (e, t) => t === "faces" ? e.faces : t === "poses" ? e.poses : e.hands, R = ({ task: e, index: t = 0, debug: n = !1, mirror: o = !0, smoothing: s = .4, children: u }) => {
196
+ let { onDetection: h, isSuccess: g } = k(), { viewport: _ } = l(), v = a(null), y = a(null), b = a(null), x = i(() => new p(), []), S = i(() => new m(), []);
197
+ return r(() => {
198
+ y.current && (y.current.visible = !1), b.current && (b.current.count = 0);
199
+ }, []), r(() => {
200
+ if (g) return h((e) => {
201
+ v.current = e;
202
+ });
203
+ }, [g, h]), c(() => {
204
+ let r = v.current, i = y.current;
205
+ if (!r || !i) return;
206
+ let a = L(r, e)[t], c = b.current;
207
+ if (!a || a.landmarks.length === 0) {
208
+ i.visible = !1, c && (c.count = 0);
209
+ return;
210
+ }
211
+ let l = i.visible;
212
+ i.visible = !0;
213
+ let [u, d, f] = E(D(a.landmarks), _, r.frameSize, o);
214
+ if (S.set(u, d, f), !l || s >= 1 ? i.position.copy(S) : i.position.lerp(S, s), n && c) {
215
+ let e = i.position, t = Math.min(a.landmarks.length, I);
216
+ for (let n = 0; n < t; n++) {
217
+ let [t, i, s] = E(a.landmarks[n], _, r.frameSize, o);
218
+ x.position.set(t - e.x, i - e.y, s - e.z), x.updateMatrix(), c.setMatrixAt(n, x.matrix);
219
+ }
220
+ c.count = t, c.instanceMatrix.needsUpdate = !0;
221
+ }
222
+ }), /* @__PURE__ */ f("group", {
223
+ ref: y,
224
+ children: [u, n && /* @__PURE__ */ f("instancedMesh", {
225
+ ref: b,
226
+ args: [
227
+ void 0,
228
+ void 0,
229
+ I
230
+ ],
231
+ children: [/* @__PURE__ */ d("sphereGeometry", { args: [
232
+ .02,
233
+ 6,
234
+ 6
235
+ ] }), /* @__PURE__ */ d("meshBasicMaterial", { color: "#00ff88" })]
236
+ })]
237
+ });
238
+ }, z = ({ tasks: e, width: t = 320, height: n = 240, style: r }) => {
239
+ let i = F({ throttle: 50 });
240
+ return /* @__PURE__ */ f("div", {
241
+ style: {
242
+ background: "rgba(0, 0, 0, 0.8)",
243
+ color: "#00ff88",
244
+ padding: "8px",
245
+ borderRadius: "8px",
246
+ fontFamily: "monospace",
247
+ fontSize: "0.75rem",
248
+ width: t,
249
+ height: n,
250
+ overflow: "auto",
251
+ ...r
252
+ },
253
+ children: [/* @__PURE__ */ f("div", { children: ["Frame: ", i ? "active" : "none"] }), i && /* @__PURE__ */ f(u, { children: [
254
+ /* @__PURE__ */ f("div", { children: [
255
+ "Size: ",
256
+ i.frameSize.width,
257
+ "x",
258
+ i.frameSize.height
259
+ ] }),
260
+ e.includes("hands") && /* @__PURE__ */ f("div", { children: ["Hands: ", i.hands.length] }),
261
+ e.includes("faces") && /* @__PURE__ */ f("div", { children: ["Faces: ", i.faces.length] }),
262
+ e.includes("poses") && /* @__PURE__ */ f("div", { children: ["Poses: ", i.poses.length] }),
263
+ e.includes("objects") && /* @__PURE__ */ f("div", { children: ["Objects: ", i.objects.length] })
264
+ ] })]
265
+ });
266
+ }, B = class {
267
+ type = "mediapipe";
268
+ models = {};
269
+ isInitialized = !1;
270
+ isProcessing = !1;
271
+ isDisposed = !1;
272
+ videoElement = null;
273
+ animationId = null;
274
+ lastProcessTime = 0;
275
+ callbacks = [];
276
+ options;
277
+ constructor(e) {
278
+ this.options = e;
279
+ }
280
+ async initialize(e) {
281
+ this.videoElement = e;
282
+ let { FilesetResolver: t, HandLandmarker: n, FaceLandmarker: r, PoseLandmarker: i } = await import("@mediapipe/tasks-vision"), a = await t.forVisionTasks("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10/wasm"), o = this.options.useGpu ?? !0 ? "GPU" : "CPU", s = async (e) => {
283
+ if (o === "CPU") return e("CPU");
284
+ try {
285
+ return await e("GPU");
286
+ } catch {
287
+ return console.warn("MediaPipe GPU delegate failed, falling back to CPU"), e("CPU");
288
+ }
289
+ }, c = new Set(this.options.tasks ?? [
290
+ "hands",
291
+ "faces",
292
+ "poses"
293
+ ]), l = [
294
+ "hands",
295
+ "faces",
296
+ "poses"
297
+ ].filter((e) => c.has(e)), u = 0;
298
+ c.has("hands") && (this.options.onProgress?.({
299
+ step: ++u,
300
+ total: l.length,
301
+ label: "hand landmarker"
302
+ }), this.models.handLandmarker = await s((e) => n.createFromOptions(a, {
303
+ baseOptions: {
304
+ modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
305
+ delegate: e
306
+ },
307
+ runningMode: "VIDEO",
308
+ numHands: 2,
309
+ minHandDetectionConfidence: .7,
310
+ minHandPresenceConfidence: .5,
311
+ minTrackingConfidence: .5
312
+ }))), c.has("faces") && (this.options.onProgress?.({
313
+ step: ++u,
314
+ total: l.length,
315
+ label: "face landmarker"
316
+ }), this.models.faceLandmarker = await s((e) => r.createFromOptions(a, {
317
+ baseOptions: {
318
+ modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",
319
+ delegate: e
320
+ },
321
+ runningMode: "VIDEO",
322
+ numFaces: 1,
323
+ minFaceDetectionConfidence: .7,
324
+ minFacePresenceConfidence: .5,
325
+ minTrackingConfidence: .5,
326
+ outputFaceBlendshapes: !1,
327
+ outputFacialTransformationMatrixes: !1
328
+ }))), c.has("poses") && (this.options.onProgress?.({
329
+ step: ++u,
330
+ total: l.length,
331
+ label: "pose landmarker"
332
+ }), this.models.poseLandmarker = await s((e) => i.createFromOptions(a, {
333
+ baseOptions: {
334
+ modelAssetPath: "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task",
335
+ delegate: e
336
+ },
337
+ runningMode: "VIDEO",
338
+ numPoses: 1,
339
+ minPoseDetectionConfidence: .7,
340
+ minPosePresenceConfidence: .5,
341
+ minTrackingConfidence: .5,
342
+ outputSegmentationMasks: this.options.segmentation ?? !1
343
+ }))), this.isInitialized = !0, this.options.onProgress?.(null);
344
+ }
345
+ startDetection() {
346
+ !this.isInitialized || !this.videoElement || this.isProcessing || (this.isProcessing = !0, this.processFrame());
347
+ }
348
+ stopDetection() {
349
+ this.isProcessing = !1, this.animationId &&= (cancelAnimationFrame(this.animationId), null);
350
+ }
351
+ onDetection(e) {
352
+ return this.callbacks.push(e), () => {
353
+ let t = this.callbacks.indexOf(e);
354
+ t > -1 && this.callbacks.splice(t, 1);
355
+ };
356
+ }
357
+ dispose() {
358
+ this.isDisposed = !0, this.stopDetection(), this.models.handLandmarker && this.models.handLandmarker.close(), this.models.faceLandmarker && this.models.faceLandmarker.close(), this.models.poseLandmarker && this.models.poseLandmarker.close(), this.models = {}, this.videoElement = null, this.callbacks = [], this.isInitialized = !1, this.isProcessing = !1;
359
+ }
360
+ processFrame = () => {
361
+ if (!this.isProcessing || !this.videoElement || this.isDisposed) return;
362
+ let e = performance.now(), t = this.options.throttle ?? 16;
363
+ if (e - this.lastProcessTime < t) {
364
+ this.animationId = requestAnimationFrame(this.processFrame);
365
+ return;
366
+ }
367
+ let { videoWidth: n, videoHeight: r } = this.videoElement;
368
+ if (n === 0 || r === 0) {
369
+ this.animationId = requestAnimationFrame(this.processFrame);
370
+ return;
371
+ }
372
+ this.lastProcessTime = e;
373
+ let i = e;
374
+ try {
375
+ let e = {
376
+ hands: [],
377
+ faces: [],
378
+ poses: [],
379
+ objects: [],
380
+ timestamp: i,
381
+ frameSize: {
382
+ width: n,
383
+ height: r
384
+ }
385
+ };
386
+ if (this.models.handLandmarker) {
387
+ let t = this.models.handLandmarker.detectForVideo(this.videoElement, i);
388
+ e.hands = t.landmarks.map((e, n) => ({
389
+ landmarks: e.map((e) => ({
390
+ x: e.x,
391
+ y: e.y,
392
+ z: e.z || 0
393
+ })),
394
+ confidence: t.handednesses?.[n]?.[0]?.score || .5
395
+ }));
396
+ }
397
+ if (this.models.faceLandmarker && (e.faces = this.models.faceLandmarker.detectForVideo(this.videoElement, i).faceLandmarks.map((e) => {
398
+ let t = e.map((e) => e.x), n = e.map((e) => e.y), r = (Math.max(...t) - Math.min(...t)) * (Math.max(...n) - Math.min(...n)), i = Math.min(.95, Math.max(.6, r * 4));
399
+ return {
400
+ landmarks: e.map((e) => ({
401
+ x: e.x,
402
+ y: e.y,
403
+ z: e.z || 0
404
+ })),
405
+ confidence: i
406
+ };
407
+ })), this.models.poseLandmarker) {
408
+ let t = this.models.poseLandmarker.detectForVideo(this.videoElement, i);
409
+ e.poses = t.landmarks.map((e) => {
410
+ let t = 0, n = 0;
411
+ for (let r of e) typeof r.visibility == "number" && (t += r.visibility, n++);
412
+ let r = n > 0 ? t / n : .5;
413
+ return {
414
+ landmarks: e.map((e) => ({
415
+ x: e.x,
416
+ y: e.y,
417
+ z: e.z || 0
418
+ })),
419
+ confidence: r
420
+ };
421
+ });
422
+ let n = t.segmentationMasks?.[0];
423
+ if (n) {
424
+ let t = new OffscreenCanvas(n.width, n.height).getContext("2d"), r = n.getAsFloat32Array(), i = t.createImageData(n.width, n.height);
425
+ for (let e = 0; e < r.length; e++) {
426
+ let t = r[e] * 255;
427
+ i.data[e * 4] = t, i.data[e * 4 + 1] = t, i.data[e * 4 + 2] = t, i.data[e * 4 + 3] = 255;
428
+ }
429
+ e.segmentationMask = i;
430
+ }
431
+ }
432
+ this.handleResult(e);
433
+ } catch (e) {
434
+ console.error("Vision processing error:", e);
435
+ }
436
+ this.animationId = requestAnimationFrame(this.processFrame);
437
+ };
438
+ handleResult(e) {
439
+ let t = this.options.minConfidence ?? .7, n = this.options.maxResults ?? 2, r = {
440
+ ...e,
441
+ hands: e.hands.filter((e) => e.confidence >= t).slice(0, n),
442
+ faces: e.faces.filter((e) => e.confidence >= t).slice(0, n),
443
+ poses: e.poses.filter((e) => e.confidence >= t).slice(0, n),
444
+ objects: []
445
+ };
446
+ for (let e of this.callbacks) try {
447
+ e(r);
448
+ } catch (e) {
449
+ console.error("Error in vision callback:", e);
450
+ }
451
+ }
452
+ }, V = 5e3, H = class {
453
+ type = "onnx";
454
+ worker = null;
455
+ isInitialized = !1;
456
+ isDisposed = !1;
457
+ isProcessing = !1;
458
+ videoElement = null;
459
+ animationId = null;
460
+ loadedModels = /* @__PURE__ */ new Map();
461
+ lastProcessTime = 0;
462
+ processingStartTime = 0;
463
+ callbacks = [];
464
+ options;
465
+ messageHandler = null;
466
+ pendingModelCallbacks = /* @__PURE__ */ new Map();
467
+ constructor(e) {
468
+ this.options = e;
469
+ }
470
+ post(e, t) {
471
+ t ? this.worker?.postMessage(e, t) : this.worker?.postMessage(e);
472
+ }
473
+ async initialize(e) {
474
+ this.videoElement = e, this.worker = new Worker(new URL(
475
+ /* @vite-ignore */
476
+ "/assets/onnxWorker-BNjoneWr.js",
477
+ "" + import.meta.url
478
+ ), { type: "module" }), await new Promise((e, t) => {
479
+ let n = setTimeout(() => {
480
+ t(/* @__PURE__ */ Error("ONNX worker initialization timeout"));
481
+ }, 1e4);
482
+ this.messageHandler = (r) => {
483
+ let i = r.data;
484
+ switch (i.type) {
485
+ case "initialized":
486
+ clearTimeout(n), this.isInitialized = !0, e();
487
+ break;
488
+ case "result":
489
+ this.isProcessing = !1, this.handleResult(i.result);
490
+ break;
491
+ case "modelLoaded": {
492
+ let e = this.pendingModelCallbacks.get(i.modelName);
493
+ e && (this.pendingModelCallbacks.delete(i.modelName), e.resolve());
494
+ break;
495
+ }
496
+ case "modelError": {
497
+ let e = this.pendingModelCallbacks.get(i.modelName);
498
+ e && (this.pendingModelCallbacks.delete(i.modelName), e.reject(Error(i.error)));
499
+ break;
500
+ }
501
+ case "error":
502
+ this.isProcessing = !1, this.isInitialized ? console.error("ONNX worker error:", i.error) : (clearTimeout(n), t(Error(i.error)));
503
+ break;
504
+ }
505
+ }, this.worker?.addEventListener("message", this.messageHandler), this.post({ type: "init" });
506
+ });
507
+ let t = this.options.onnx?.models ?? [];
508
+ for (let e of t) await this.loadModel(e);
509
+ }
510
+ startDetection() {
511
+ if (!this.isInitialized || !this.videoElement) {
512
+ console.warn("ONNX provider not initialized");
513
+ return;
514
+ }
515
+ let e = () => {
516
+ if (!this.videoElement || !this.isInitialized) return;
517
+ let t = Date.now(), n = this.options.throttle ?? 100;
518
+ if (t - this.lastProcessTime < n) {
519
+ this.animationId = requestAnimationFrame(e);
520
+ return;
521
+ }
522
+ this.isProcessing && this.processingStartTime > 0 && t - this.processingStartTime > V && (console.warn("ONNX processing stall detected, resetting"), this.isProcessing = !1), this.isProcessing || this.processCurrentFrame(), this.animationId = requestAnimationFrame(e);
523
+ };
524
+ e();
525
+ }
526
+ stopDetection() {
527
+ this.animationId &&= (cancelAnimationFrame(this.animationId), null), this.isProcessing = !1;
528
+ }
529
+ onDetection(e) {
530
+ return this.callbacks.push(e), () => {
531
+ let t = this.callbacks.indexOf(e);
532
+ t > -1 && this.callbacks.splice(t, 1);
533
+ };
534
+ }
535
+ dispose() {
536
+ this.isDisposed = !0, this.stopDetection(), this.worker &&= (this.messageHandler &&= (this.worker.removeEventListener("message", this.messageHandler), null), this.post({ type: "dispose" }), this.worker.terminate(), null), this.videoElement = null, this.callbacks.length = 0, this.loadedModels.clear(), this.pendingModelCallbacks.clear(), this.isInitialized = !1;
537
+ }
538
+ async loadModel(e) {
539
+ if (!this.worker) throw Error("Worker not initialized");
540
+ let t = new Promise((t, n) => {
541
+ this.pendingModelCallbacks.set(e.name, {
542
+ resolve: t,
543
+ reject: n
544
+ });
545
+ });
546
+ this.post({
547
+ type: "loadModel",
548
+ model: e
549
+ }), await t, this.loadedModels.set(e.name, e);
550
+ }
551
+ processCurrentFrame() {
552
+ if (!this.videoElement || !this.worker || this.isProcessing) return;
553
+ let e = Date.now();
554
+ this.isProcessing = !0, this.lastProcessTime = e, this.processingStartTime = e;
555
+ let { videoWidth: t, videoHeight: n } = this.videoElement;
556
+ createImageBitmap(this.videoElement).then((e) => {
557
+ if (this.isDisposed) {
558
+ e.close();
559
+ return;
560
+ }
561
+ this.post({
562
+ type: "process",
563
+ imageBitmap: e,
564
+ sourceWidth: t,
565
+ sourceHeight: n,
566
+ options: {
567
+ minConfidence: this.options.minConfidence ?? .5,
568
+ maxResults: this.options.maxResults ?? 100
569
+ }
570
+ }, [e]);
571
+ }, (e) => {
572
+ this.isProcessing = !1, console.error("Error creating ImageBitmap:", e);
573
+ });
574
+ }
575
+ handleResult(e) {
576
+ let t = this.options.minConfidence ?? .5, n = this.options.maxResults ?? 100, r = {
577
+ hands: [],
578
+ faces: [],
579
+ poses: [],
580
+ objects: e.detections.filter((e) => e.confidence >= t).sort((e, t) => t.confidence - e.confidence).slice(0, n),
581
+ masks: e.masks.length > 0 ? e.masks : void 0,
582
+ timestamp: e.timestamp,
583
+ frameSize: e.frameSize,
584
+ processingTime: e.processingTime
585
+ };
586
+ for (let e of this.callbacks) try {
587
+ e(r);
588
+ } catch (e) {
589
+ console.error("Error in ONNX detection callback:", e);
590
+ }
591
+ }
592
+ }, U = (e) => e.provider === "onnx" ? new H(e) : new B(e), W = async (e) => {
593
+ let t = document.createElement("video");
594
+ t.style.display = "none", document.body.appendChild(t);
595
+ try {
596
+ return t.srcObject = await navigator.mediaDevices.getUserMedia({ video: {
597
+ width: {
598
+ ideal: 1280,
599
+ min: 640
600
+ },
601
+ height: {
602
+ ideal: 720,
603
+ min: 480
604
+ },
605
+ facingMode: e.provider === "onnx" ? "environment" : "user",
606
+ frameRate: {
607
+ ideal: 30,
608
+ min: 15
609
+ }
610
+ } }), t.playsInline = !0, t.muted = !0, await new Promise((e, n) => {
611
+ t.onloadedmetadata = () => {
612
+ t.play().then(e).catch(n);
613
+ }, t.onerror = n;
614
+ }), t;
615
+ } catch (e) {
616
+ throw document.body.removeChild(t), e;
617
+ }
618
+ }, G = (e) => {
619
+ if (!e) return;
620
+ let t = e.srcObject;
621
+ if (t) for (let e of t.getTracks()) e.stop();
622
+ e.parentNode && e.parentNode.removeChild(e);
623
+ }, K = (t) => {
624
+ let n = null, r = null, i = !1, a = null, o = [], s = (e) => (o.push(e), () => {
625
+ let t = o.indexOf(e);
626
+ t > -1 && o.splice(t, 1);
627
+ });
628
+ return {
629
+ type: e.VISION,
630
+ async init(e) {
631
+ t.videoElement ? (r = t.videoElement, i = !1) : (r = await W(t), i = !0), n = U(t), await n.initialize(r), a = n.onDetection((e) => {
632
+ for (let t of o) try {
633
+ t(e);
634
+ } catch (e) {
635
+ console.error("Error in vision detection callback:", e);
636
+ }
637
+ }), n.startDetection();
638
+ },
639
+ update(e) {},
640
+ dispose() {
641
+ a?.(), a = null, n?.dispose(), i && G(r), r = null, n = null, o.length = 0;
642
+ },
643
+ getInternal: () => ({
644
+ provider: n,
645
+ videoElement: r,
646
+ onDetection: s
647
+ })
648
+ };
649
+ }, q = ({ options: e, children: n, onError: i }) => {
650
+ let { scene: o, camera: s, gl: c } = l(), { registerBackend: f, unregisterBackend: p } = t(), m = a(null), h = a(e);
651
+ h.current = e;
652
+ let g = a(i);
653
+ return g.current = i, r(() => {
654
+ if (m.current) return;
655
+ let e = !1;
656
+ return (async () => {
657
+ try {
658
+ let t = K(h.current);
659
+ if (e) return;
660
+ if (await f(t, {
661
+ scene: o,
662
+ camera: s,
663
+ renderer: c
664
+ }), e) {
665
+ p(t);
666
+ return;
667
+ }
668
+ m.current = t;
669
+ } catch (e) {
670
+ let t = e instanceof Error ? e : Error(String(e));
671
+ console.error("[VisionSession] Failed to initialize:", t), g.current?.(t);
672
+ }
673
+ })(), () => {
674
+ e = !0, m.current &&= (p(m.current), null);
675
+ };
676
+ }, [
677
+ o,
678
+ s,
679
+ c,
680
+ f,
681
+ p
682
+ ]), /* @__PURE__ */ d(u, { children: n });
683
+ }, J = ({ tasks: e, progress: t, showDebug: n, onToggleDebug: r }) => {
684
+ let i = F({ throttle: 100 }), a = (e) => {
685
+ if (!i) return !1;
686
+ switch (e) {
687
+ case "hands": return i.hands.length > 0;
688
+ case "faces": return i.faces.length > 0;
689
+ case "poses": return i.poses.length > 0;
690
+ case "objects": return i.objects.length > 0;
691
+ default: return !1;
692
+ }
693
+ }, o = e.some(a), s = i !== null;
694
+ return /* @__PURE__ */ f(u, { children: [/* @__PURE__ */ f("div", {
695
+ style: {
696
+ position: "fixed",
697
+ top: "20px",
698
+ left: "20px",
699
+ background: "rgba(0, 0, 0, 0.7)",
700
+ color: "white",
701
+ padding: "12px",
702
+ borderRadius: "8px",
703
+ fontFamily: "system-ui, sans-serif",
704
+ fontSize: "0.9rem",
705
+ zIndex: 1e3
706
+ },
707
+ children: [/* @__PURE__ */ f("div", {
708
+ style: {
709
+ display: "flex",
710
+ alignItems: "center",
711
+ gap: "8px",
712
+ marginBottom: "8px"
713
+ },
714
+ children: [/* @__PURE__ */ d("div", { style: {
715
+ width: "8px",
716
+ height: "8px",
717
+ borderRadius: "50%",
718
+ background: s ? "#00ff88" : "#ffaa00"
719
+ } }), /* @__PURE__ */ d("span", { children: o ? "Active" : s ? "Ready" : t ? `Loading models (${t.step}/${t.total}) ${t.label}...` : "Starting..." })]
720
+ }), e.map((e) => /* @__PURE__ */ f("div", {
721
+ style: {
722
+ fontSize: "0.8rem",
723
+ opacity: .8,
724
+ display: "flex",
725
+ justifyContent: "space-between",
726
+ gap: "12px"
727
+ },
728
+ children: [/* @__PURE__ */ d("span", { children: e }), /* @__PURE__ */ d("span", { children: a(e) ? "active" : s ? "idle" : "..." })]
729
+ }, e))]
730
+ }), /* @__PURE__ */ f("button", {
731
+ onClick: r,
732
+ type: "button",
733
+ style: {
734
+ position: "fixed",
735
+ top: "20px",
736
+ right: "20px",
737
+ background: "rgba(0, 0, 0, 0.7)",
738
+ color: "white",
739
+ border: "1px solid #333",
740
+ padding: "8px 12px",
741
+ borderRadius: "6px",
742
+ cursor: "pointer",
743
+ fontFamily: "system-ui, sans-serif",
744
+ fontSize: "0.8rem",
745
+ zIndex: 1e3
746
+ },
747
+ children: [n ? "Hide" : "Show", " Debug"]
748
+ })] });
749
+ }, Y = {}, X = ({ tasks: e = ["hands"], options: t, children: r, showDebug: c = !1, cameraConstraints: l, canvasStyle: u = Y, onReady: p, onError: m }) => {
750
+ let [g, _] = o(c), [v, y] = o(null), [b, x] = o(null), S = a(p);
751
+ S.current = p;
752
+ let C = a(x);
753
+ C.current = x;
754
+ let w = n((e) => C.current(e), []), T = n((e) => {
755
+ y(e), S.current?.();
756
+ }, []), E = i(() => ({
757
+ tasks: e,
758
+ minConfidence: .6,
759
+ maxResults: 2,
760
+ useGpu: !0,
761
+ ...t,
762
+ videoElement: v ?? void 0,
763
+ onProgress: w
764
+ }), [
765
+ e,
766
+ t,
767
+ v,
768
+ w
769
+ ]);
770
+ return /* @__PURE__ */ f("div", {
771
+ style: {
772
+ position: "relative",
773
+ width: "100%",
774
+ height: "100%"
775
+ },
776
+ children: [
777
+ /* @__PURE__ */ d(h, {
778
+ constraints: l,
779
+ onReady: T,
780
+ onError: m
781
+ }),
782
+ v && /* @__PURE__ */ d(s, {
783
+ style: {
784
+ position: "absolute",
785
+ inset: 0,
786
+ pointerEvents: "none",
787
+ ...u
788
+ },
789
+ camera: {
790
+ position: [
791
+ 0,
792
+ 0,
793
+ 5
794
+ ],
795
+ fov: 75
796
+ },
797
+ gl: {
798
+ alpha: !0,
799
+ premultipliedAlpha: !1
800
+ },
801
+ children: /* @__PURE__ */ d(Z, {
802
+ options: E,
803
+ onError: m,
804
+ children: r
805
+ })
806
+ }),
807
+ /* @__PURE__ */ d(J, {
808
+ tasks: e,
809
+ progress: b,
810
+ showDebug: g,
811
+ onToggleDebug: () => _((e) => !e)
812
+ }),
813
+ g && /* @__PURE__ */ d(z, {
814
+ tasks: e,
815
+ width: 320,
816
+ height: 240,
817
+ style: {
818
+ position: "fixed",
819
+ bottom: "80px",
820
+ right: "20px",
821
+ maxWidth: "calc(100vw - 40px)",
822
+ overflow: "hidden"
823
+ }
824
+ })
825
+ ]
826
+ });
827
+ }, Z = ({ options: e, onError: t, children: n }) => /* @__PURE__ */ f(q, {
828
+ options: e,
829
+ onError: t,
830
+ children: [
831
+ /* @__PURE__ */ d("ambientLight", { intensity: .6 }),
832
+ /* @__PURE__ */ d("directionalLight", {
833
+ position: [
834
+ 10,
835
+ 10,
836
+ 10
837
+ ],
838
+ intensity: .8
839
+ }),
840
+ n
841
+ ]
842
+ });
843
+ //#endregion
844
+ export { h as C, v as S, C as _, z as a, x as b, j as c, P as d, A as f, T as g, D as h, K as i, N as l, E as m, J as n, R as o, k as p, q as r, F as s, X as t, M as u, S as v, y as x, b as y };