@omnidotdev/rdk 0.14.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/assets/onnxWorker-3jptKfHs.js +1 -0
- package/build/engine-B_SO7XR5.cjs +1 -0
- package/build/{engine-NEJZ-OgD.js → engine-DeBLXgW-.js} +2 -3
- package/build/engine.cjs +1 -1
- package/build/engine.js +2 -2
- package/build/fiducial/FiducialAnchor.d.ts +1 -1
- package/build/fiducial/FiducialAnchor.d.ts.map +1 -1
- package/build/{fiducial-7pF-ErgE.js → fiducial-DUn6sFBT.js} +17 -13
- package/build/fiducial-IzVogwhD.cjs +1 -0
- package/build/fiducial.cjs +1 -1
- package/build/fiducial.js +1 -2
- package/build/geolocation/GeoLine.d.ts +3 -15
- package/build/geolocation/GeoLine.d.ts.map +1 -1
- package/build/geolocation/GeoPolygon.d.ts +3 -15
- package/build/geolocation/GeoPolygon.d.ts.map +1 -1
- package/build/geolocation/GeolocationAnchor.d.ts +1 -1
- package/build/geolocation/useMaterialStore.d.ts +20 -0
- package/build/geolocation/useMaterialStore.d.ts.map +1 -0
- package/build/geolocation-BK0FDlmT.cjs +1 -0
- package/build/geolocation-DmJvz9om.js +376 -0
- package/build/geolocation.cjs +1 -1
- package/build/geolocation.js +1 -2
- package/build/immersive/ImmersiveSession.d.ts +1 -1
- package/build/immersive/ImmersiveSession.d.ts.map +1 -1
- package/build/{immersive-CoSRZ58s.js → immersive-BNYjOBFO.js} +2 -3
- package/build/{immersive-DMYGUy_v.cjs → immersive-C76xlUbB.cjs} +1 -1
- package/build/immersive.cjs +1 -1
- package/build/immersive.js +1 -2
- package/build/index.cjs +1 -1
- package/build/index.js +7 -7
- package/build/magic/WorldAnchor.d.ts +1 -1
- package/build/magic/WorldAnchor.d.ts.map +1 -1
- package/build/magic-BbyHQVOH.cjs +1 -0
- package/build/{magic-CjnX9lWp.js → magic-COfaC4at.js} +10 -9
- package/build/magic.cjs +1 -1
- package/build/magic.js +1 -2
- package/build/{useXRStore-BTnI1ZM2.js → useXRStore-CrglOU9_.js} +5 -8
- package/build/useXRStore-DfyQv43d.cjs +1 -0
- package/build/vision/HandTracker.d.ts +1 -1
- package/build/vision/HandTracker.d.ts.map +1 -1
- package/build/vision/VisionAnchor.d.ts +1 -1
- package/build/vision/VisionAnchor.d.ts.map +1 -1
- package/build/vision/VisionSession.d.ts.map +1 -1
- package/build/vision/providers/mediapipeProvider.d.ts +6 -0
- package/build/vision/providers/mediapipeProvider.d.ts.map +1 -1
- package/build/vision/types.d.ts +12 -0
- package/build/vision/types.d.ts.map +1 -1
- package/build/vision/visionBackend.d.ts +1 -1
- package/build/vision/visionBackend.d.ts.map +1 -1
- package/build/{vision-Vp2iuVNJ.js → vision-1_qdyG-O.js} +60 -49
- package/build/vision-B9EQHClV.cjs +1 -0
- package/build/vision.cjs +1 -1
- package/build/vision.js +1 -2
- package/package.json +5 -5
- package/build/assets/onnxWorker-BNjoneWr.js +0 -1
- package/build/engine-EvzyijbY.cjs +0 -1
- package/build/fiducial-Tris7XAY.cjs +0 -1
- package/build/geolocation-B7zCO7n6.js +0 -346
- package/build/geolocation-ypey8aeT.cjs +0 -1
- package/build/magic-B9pQnRYJ.cjs +0 -1
- package/build/useXRStore-Cf_XL0aI.cjs +0 -1
- package/build/vision-BLWMU9Ys.cjs +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mediapipeProvider.d.ts","sourceRoot":"","sources":["../../../src/vision/providers/mediapipeProvider.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAEV,WAAW,EACX,cAAc,EACd,oBAAoB,EACrB,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"mediapipeProvider.d.ts","sourceRoot":"","sources":["../../../src/vision/providers/mediapipeProvider.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAEV,WAAW,EACX,cAAc,EACd,oBAAoB,EACrB,MAAM,UAAU,CAAC;AA6BlB,cAAM,iBAAkB,YAAW,cAAc;IAC/C,QAAQ,CAAC,IAAI,EAAG,WAAW,CAAU;IAErC,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,YAAY,CAAiC;IACrD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,SAAS,CAA2C;IAC5D,OAAO,CAAC,cAAc,CAAqC;IAC3D,OAAO,CAAC,OAAO,CAAuB;gBAE1B,OAAO,EAAE,oBAAoB;IAInC,UAAU,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAyGxD,cAAc,IAAI,IAAI;IAOtB,aAAa,IAAI,IAAI;IAQrB,WAAW,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,MAAM,IAAI;IAS/D,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,MAAM,IAAI;IASrD,OAAO,IAAI,IAAI;IAoBf,OAAO,CAAC,YAAY,CAyIlB;IAEF;;OAEG;IACH,OAAO,CAAC,IAAI;IAYZ,OAAO,CAAC,YAAY;CA0BrB;AAED,eAAe,iBAAiB,CAAC"}
|
package/build/vision/types.d.ts
CHANGED
|
@@ -93,6 +93,16 @@ export type VisionSessionOptions = {
|
|
|
93
93
|
videoElement?: HTMLVideoElement;
|
|
94
94
|
/** Progress callback for model loading */
|
|
95
95
|
onProgress?: (progress: VisionProgress | null) => void;
|
|
96
|
+
/** MediaPipe-specific options */
|
|
97
|
+
mediapipe?: {
|
|
98
|
+
/**
|
|
99
|
+
* Base URL of the MediaPipe Tasks Vision wasm files. Defaults to the
|
|
100
|
+
* jsDelivr CDN build matching the supported major version. Set this to
|
|
101
|
+
* self-host the files or pin an exact version (it must match the installed
|
|
102
|
+
* `@mediapipe/tasks-vision` version)
|
|
103
|
+
*/
|
|
104
|
+
wasmBasePath?: string;
|
|
105
|
+
};
|
|
96
106
|
/** ONNX-specific options */
|
|
97
107
|
onnx?: {
|
|
98
108
|
models: ONNXModelConfig[];
|
|
@@ -105,6 +115,8 @@ export type VisionProvider = {
|
|
|
105
115
|
startDetection(): void;
|
|
106
116
|
stopDetection(): void;
|
|
107
117
|
onDetection(callback: (frame: VisionFrame) => void): () => void;
|
|
118
|
+
/** Subscribe to errors that stop detection for good */
|
|
119
|
+
onError?(callback: (error: Error) => void): () => void;
|
|
108
120
|
dispose(): void;
|
|
109
121
|
};
|
|
110
122
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/vision/types.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,MAAM,MAAM,cAAc,GAAG;IAC3B,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX,CAAC;AAEF,oEAAoE;AACpE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,wCAAwC;AACxC,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE;QACJ,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH,CAAC;AAEF,oEAAoE;AACpE,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,IAAI,EAAE;QACJ,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,qFAAqF;IACrF,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,iDAAiD;AACjD,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,+EAA+E;IAC/E,KAAK,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,SAAS,CAAC;CAC9B,CAAC;AAEF,6BAA6B;AAC7B,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;AAEjE,qDAAqD;AACrD,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE5D,+BAA+B;AAC/B,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,oFAAoF;IACpF,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B,CAAC;AAEF,gCAAgC;AAChC,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,4CAA4C;AAC5C,MAAM,MAAM,oBAAoB,GAAG;IACjC,4BAA4B;IAC5B,QAAQ,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC;IAChC,6BAA6B;IAC7B,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IACrB,mCAAmC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,+BAA+B;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sCAAsC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2BAA2B;IAC3B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,0CAA0C;IAC1C,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI,KAAK,IAAI,CAAC;IACvD,4BAA4B;IAC5B,IAAI,CAAC,EAAE;QACL,MAAM,EAAE,eAAe,EAAE,CAAC;KAC3B,CAAC;CACH,CAAC;AAEF,gCAAgC;AAChC,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnD,cAAc,IAAI,IAAI,CAAC;IACvB,aAAa,IAAI,IAAI,CAAC;IACtB,WAAW,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,MAAM,IAAI,CAAC;IAChE,OAAO,IAAI,IAAI,CAAC;CACjB,CAAC"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/vision/types.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,MAAM,MAAM,cAAc,GAAG;IAC3B,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX,CAAC;AAEF,oEAAoE;AACpE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,wCAAwC;AACxC,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE;QACJ,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH,CAAC;AAEF,oEAAoE;AACpE,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,IAAI,EAAE;QACJ,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,qFAAqF;IACrF,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,iDAAiD;AACjD,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,+EAA+E;IAC/E,KAAK,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,SAAS,CAAC;CAC9B,CAAC;AAEF,6BAA6B;AAC7B,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;AAEjE,qDAAqD;AACrD,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE5D,+BAA+B;AAC/B,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,oFAAoF;IACpF,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B,CAAC;AAEF,gCAAgC;AAChC,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,4CAA4C;AAC5C,MAAM,MAAM,oBAAoB,GAAG;IACjC,4BAA4B;IAC5B,QAAQ,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC;IAChC,6BAA6B;IAC7B,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IACrB,mCAAmC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,+BAA+B;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sCAAsC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2BAA2B;IAC3B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,0CAA0C;IAC1C,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI,KAAK,IAAI,CAAC;IACvD,iCAAiC;IACjC,SAAS,CAAC,EAAE;QACV;;;;;WAKG;QACH,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,4BAA4B;IAC5B,IAAI,CAAC,EAAE;QACL,MAAM,EAAE,eAAe,EAAE,CAAC;KAC3B,CAAC;CACH,CAAC;AAEF,gCAAgC;AAChC,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnD,cAAc,IAAI,IAAI,CAAC;IACvB,aAAa,IAAI,IAAI,CAAC;IACtB,WAAW,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,MAAM,IAAI,CAAC;IAChE,uDAAuD;IACvD,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,MAAM,IAAI,CAAC;IACvD,OAAO,IAAI,IAAI,CAAC;CACjB,CAAC"}
|
|
@@ -12,6 +12,6 @@ export type VisionBackendState = VisionInternal & {
|
|
|
12
12
|
isSuccess: boolean;
|
|
13
13
|
};
|
|
14
14
|
/** Create a vision backend following the Backend interface contract */
|
|
15
|
-
declare const createVisionBackend: (options: VisionSessionOptions) => Backend<VisionInternal>;
|
|
15
|
+
declare const createVisionBackend: (options: VisionSessionOptions, onError?: (error: Error) => void) => Backend<VisionInternal>;
|
|
16
16
|
export default createVisionBackend;
|
|
17
17
|
//# sourceMappingURL=visionBackend.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"visionBackend.d.ts","sourceRoot":"","sources":["../../src/vision/visionBackend.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAmB,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EACV,WAAW,EACX,cAAc,EACd,oBAAoB,EACrB,MAAM,SAAS,CAAC;AAEjB,mDAAmD;AACnD,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,cAAc,GAAG,IAAI,CAAC;IAChC,YAAY,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACtC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,KAAK,MAAM,IAAI,CAAC;CAC/D,CAAC;AAEF,gDAAgD;AAChD,MAAM,MAAM,kBAAkB,GAAG,cAAc,GAAG;IAChD,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;CACpB,CAAC;AAsDF,uEAAuE;AACvE,QAAA,MAAM,mBAAmB,GACvB,SAAS,oBAAoB,
|
|
1
|
+
{"version":3,"file":"visionBackend.d.ts","sourceRoot":"","sources":["../../src/vision/visionBackend.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAmB,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EACV,WAAW,EACX,cAAc,EACd,oBAAoB,EACrB,MAAM,SAAS,CAAC;AAEjB,mDAAmD;AACnD,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,cAAc,GAAG,IAAI,CAAC;IAChC,YAAY,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACtC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,KAAK,MAAM,IAAI,CAAC;CAC/D,CAAC;AAEF,gDAAgD;AAChD,MAAM,MAAM,kBAAkB,GAAG,cAAc,GAAG;IAChD,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;CACpB,CAAC;AAsDF,uEAAuE;AACvE,QAAA,MAAM,mBAAmB,GACvB,SAAS,oBAAoB,EAC7B,UAAU,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,KAC/B,OAAO,CAAC,cAAc,CAqExB,CAAC;AAEF,eAAe,mBAAmB,CAAC"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { i as e, r as t } from "./useXRStore-
|
|
1
|
+
import { i as e, r as t } from "./useXRStore-CrglOU9_.js";
|
|
2
2
|
import { useCallback as n, useEffect as r, useMemo as i, useRef as a, useState as o } from "react";
|
|
3
3
|
import { Canvas as s, useFrame as c, useThree as l } from "@react-three/fiber";
|
|
4
4
|
import { Fragment as u, jsx as d, jsxs as f } from "react/jsx-runtime";
|
|
5
5
|
import { Object3D as p, Vector3 as m } from "three";
|
|
6
|
-
|
|
6
|
+
// src/vision/CameraBackground.tsx
|
|
7
7
|
var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
8
8
|
let s = a(null), [c, l] = o(!1), u = a(t);
|
|
9
9
|
u.current = t;
|
|
@@ -55,7 +55,7 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
55
55
|
height: "100%",
|
|
56
56
|
objectFit: "cover",
|
|
57
57
|
transform: i ? "scaleX(-1)" : void 0,
|
|
58
|
-
opacity: c
|
|
58
|
+
opacity: +!!c,
|
|
59
59
|
transition: "opacity 0.5s ease"
|
|
60
60
|
},
|
|
61
61
|
autoPlay: !0,
|
|
@@ -179,11 +179,8 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
179
179
|
}), F = (e) => {
|
|
180
180
|
let { isSuccess: t, onDetection: n } = k(), [i, s] = o(null), c = e?.throttle ?? 16, l = a(0);
|
|
181
181
|
return r(() => {
|
|
182
|
-
if (!t)
|
|
183
|
-
|
|
184
|
-
return;
|
|
185
|
-
}
|
|
186
|
-
return n((e) => {
|
|
182
|
+
if (!t) s(null);
|
|
183
|
+
else return n((e) => {
|
|
187
184
|
let t = performance.now();
|
|
188
185
|
t - l.current < c || (l.current = t, s(e));
|
|
189
186
|
});
|
|
@@ -263,7 +260,12 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
263
260
|
e.includes("objects") && /* @__PURE__ */ f("div", { children: ["Objects: ", i.objects.length] })
|
|
264
261
|
] })]
|
|
265
262
|
});
|
|
266
|
-
}, B = class {
|
|
263
|
+
}, B = class extends Error {
|
|
264
|
+
cause;
|
|
265
|
+
constructor(e, t) {
|
|
266
|
+
super(e), this.name = "VisionStoppedError", this.cause = t;
|
|
267
|
+
}
|
|
268
|
+
}, V = "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1/wasm", H = class {
|
|
267
269
|
type = "mediapipe";
|
|
268
270
|
models = {};
|
|
269
271
|
isInitialized = !1;
|
|
@@ -273,13 +275,14 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
273
275
|
animationId = null;
|
|
274
276
|
lastProcessTime = 0;
|
|
275
277
|
callbacks = [];
|
|
278
|
+
errorCallbacks = [];
|
|
276
279
|
options;
|
|
277
280
|
constructor(e) {
|
|
278
281
|
this.options = e;
|
|
279
282
|
}
|
|
280
283
|
async initialize(e) {
|
|
281
284
|
this.videoElement = e;
|
|
282
|
-
let { FilesetResolver: t, HandLandmarker: n, FaceLandmarker: r, PoseLandmarker: i } = await import("@mediapipe/tasks-vision"), a = await t.forVisionTasks(
|
|
285
|
+
let { FilesetResolver: t, HandLandmarker: n, FaceLandmarker: r, PoseLandmarker: i } = await import("@mediapipe/tasks-vision"), a = await t.forVisionTasks(this.options.mediapipe?.wasmBasePath ?? V), o = this.options.useGpu ?? !0 ? "GPU" : "CPU", s = async (e) => {
|
|
283
286
|
if (o === "CPU") return e("CPU");
|
|
284
287
|
try {
|
|
285
288
|
return await e("GPU");
|
|
@@ -343,7 +346,7 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
343
346
|
}))), this.isInitialized = !0, this.options.onProgress?.(null);
|
|
344
347
|
}
|
|
345
348
|
startDetection() {
|
|
346
|
-
|
|
349
|
+
this.isInitialized && this.videoElement && !this.isProcessing && (this.isProcessing = !0, this.processFrame());
|
|
347
350
|
}
|
|
348
351
|
stopDetection() {
|
|
349
352
|
this.isProcessing = !1, this.animationId &&= (cancelAnimationFrame(this.animationId), null);
|
|
@@ -354,8 +357,18 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
354
357
|
t > -1 && this.callbacks.splice(t, 1);
|
|
355
358
|
};
|
|
356
359
|
}
|
|
360
|
+
onError(e) {
|
|
361
|
+
return this.errorCallbacks.push(e), () => {
|
|
362
|
+
let t = this.errorCallbacks.indexOf(e);
|
|
363
|
+
t > -1 && this.errorCallbacks.splice(t, 1);
|
|
364
|
+
};
|
|
365
|
+
}
|
|
357
366
|
dispose() {
|
|
358
|
-
this.isDisposed = !0, this.stopDetection()
|
|
367
|
+
this.isDisposed = !0, this.stopDetection();
|
|
368
|
+
for (let e of Object.values(this.models)) try {
|
|
369
|
+
e?.close();
|
|
370
|
+
} catch {}
|
|
371
|
+
this.models = {}, this.videoElement = null, this.callbacks = [], this.errorCallbacks = [], this.isInitialized = !1, this.isProcessing = !1;
|
|
359
372
|
}
|
|
360
373
|
processFrame = () => {
|
|
361
374
|
if (!this.isProcessing || !this.videoElement || this.isDisposed) return;
|
|
@@ -431,10 +444,19 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
431
444
|
}
|
|
432
445
|
this.handleResult(e);
|
|
433
446
|
} catch (e) {
|
|
447
|
+
if (e instanceof WebAssembly.RuntimeError) {
|
|
448
|
+
this.fail(e);
|
|
449
|
+
return;
|
|
450
|
+
}
|
|
434
451
|
console.error("Vision processing error:", e);
|
|
435
452
|
}
|
|
436
453
|
this.animationId = requestAnimationFrame(this.processFrame);
|
|
437
454
|
};
|
|
455
|
+
fail(e) {
|
|
456
|
+
this.stopDetection(), console.error("Vision processing stopped:", e);
|
|
457
|
+
let t = new B("Vision tracking stopped because this browser blocked a graphics feature it needs. Strict fingerprinting protection is a common cause.", e);
|
|
458
|
+
for (let e of this.errorCallbacks) e(t);
|
|
459
|
+
}
|
|
438
460
|
handleResult(e) {
|
|
439
461
|
let t = this.options.minConfidence ?? .7, n = this.options.maxResults ?? 2, r = {
|
|
440
462
|
...e,
|
|
@@ -449,7 +471,7 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
449
471
|
console.error("Error in vision callback:", e);
|
|
450
472
|
}
|
|
451
473
|
}
|
|
452
|
-
},
|
|
474
|
+
}, U = 5e3, W = class {
|
|
453
475
|
type = "onnx";
|
|
454
476
|
worker = null;
|
|
455
477
|
isInitialized = !1;
|
|
@@ -473,7 +495,7 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
473
495
|
async initialize(e) {
|
|
474
496
|
this.videoElement = e, this.worker = new Worker(new URL(
|
|
475
497
|
/* @vite-ignore */
|
|
476
|
-
"/assets/onnxWorker-
|
|
498
|
+
"/assets/onnxWorker-3jptKfHs.js",
|
|
477
499
|
"" + import.meta.url
|
|
478
500
|
), { type: "module" }), await new Promise((e, t) => {
|
|
479
501
|
let n = setTimeout(() => {
|
|
@@ -498,9 +520,7 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
498
520
|
e && (this.pendingModelCallbacks.delete(i.modelName), e.reject(Error(i.error)));
|
|
499
521
|
break;
|
|
500
522
|
}
|
|
501
|
-
case "error":
|
|
502
|
-
this.isProcessing = !1, this.isInitialized ? console.error("ONNX worker error:", i.error) : (clearTimeout(n), t(Error(i.error)));
|
|
503
|
-
break;
|
|
523
|
+
case "error": this.isProcessing = !1, this.isInitialized ? console.error("ONNX worker error:", i.error) : (clearTimeout(n), t(Error(i.error)));
|
|
504
524
|
}
|
|
505
525
|
}, this.worker?.addEventListener("message", this.messageHandler), this.post({ type: "init" });
|
|
506
526
|
});
|
|
@@ -515,11 +535,7 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
515
535
|
let e = () => {
|
|
516
536
|
if (!this.videoElement || !this.isInitialized) return;
|
|
517
537
|
let t = Date.now(), n = this.options.throttle ?? 100;
|
|
518
|
-
|
|
519
|
-
this.animationId = requestAnimationFrame(e);
|
|
520
|
-
return;
|
|
521
|
-
}
|
|
522
|
-
this.isProcessing && this.processingStartTime > 0 && t - this.processingStartTime > V && (console.warn("ONNX processing stall detected, resetting"), this.isProcessing = !1), this.isProcessing || this.processCurrentFrame(), this.animationId = requestAnimationFrame(e);
|
|
538
|
+
t - this.lastProcessTime < n || (this.isProcessing && this.processingStartTime > 0 && t - this.processingStartTime > U && (console.warn("ONNX processing stall detected, resetting"), this.isProcessing = !1), this.isProcessing || this.processCurrentFrame()), this.animationId = requestAnimationFrame(e);
|
|
523
539
|
};
|
|
524
540
|
e();
|
|
525
541
|
}
|
|
@@ -554,11 +570,7 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
554
570
|
this.isProcessing = !0, this.lastProcessTime = e, this.processingStartTime = e;
|
|
555
571
|
let { videoWidth: t, videoHeight: n } = this.videoElement;
|
|
556
572
|
createImageBitmap(this.videoElement).then((e) => {
|
|
557
|
-
|
|
558
|
-
e.close();
|
|
559
|
-
return;
|
|
560
|
-
}
|
|
561
|
-
this.post({
|
|
573
|
+
this.isDisposed ? e.close() : this.post({
|
|
562
574
|
type: "process",
|
|
563
575
|
imageBitmap: e,
|
|
564
576
|
sourceWidth: t,
|
|
@@ -589,7 +601,7 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
589
601
|
console.error("Error in ONNX detection callback:", e);
|
|
590
602
|
}
|
|
591
603
|
}
|
|
592
|
-
},
|
|
604
|
+
}, G = (e) => e.provider === "onnx" ? new W(e) : new H(e), K = async (e) => {
|
|
593
605
|
let t = document.createElement("video");
|
|
594
606
|
t.style.display = "none", document.body.appendChild(t);
|
|
595
607
|
try {
|
|
@@ -615,38 +627,38 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
615
627
|
} catch (e) {
|
|
616
628
|
throw document.body.removeChild(t), e;
|
|
617
629
|
}
|
|
618
|
-
},
|
|
630
|
+
}, q = (e) => {
|
|
619
631
|
if (!e) return;
|
|
620
632
|
let t = e.srcObject;
|
|
621
633
|
if (t) for (let e of t.getTracks()) e.stop();
|
|
622
634
|
e.parentNode && e.parentNode.removeChild(e);
|
|
623
|
-
},
|
|
624
|
-
let
|
|
625
|
-
let t =
|
|
626
|
-
t > -1 &&
|
|
635
|
+
}, J = (t, n) => {
|
|
636
|
+
let r = null, i = null, a = !1, o = null, s = null, c = [], l = (e) => (c.push(e), () => {
|
|
637
|
+
let t = c.indexOf(e);
|
|
638
|
+
t > -1 && c.splice(t, 1);
|
|
627
639
|
});
|
|
628
640
|
return {
|
|
629
641
|
type: e.VISION,
|
|
630
642
|
async init(e) {
|
|
631
|
-
t.videoElement ? (
|
|
632
|
-
for (let t of
|
|
643
|
+
t.videoElement ? (i = t.videoElement, a = !1) : (i = await K(t), a = !0), r = G(t), await r.initialize(i), o = r.onDetection((e) => {
|
|
644
|
+
for (let t of c) try {
|
|
633
645
|
t(e);
|
|
634
646
|
} catch (e) {
|
|
635
647
|
console.error("Error in vision detection callback:", e);
|
|
636
648
|
}
|
|
637
|
-
}), n.startDetection();
|
|
649
|
+
}), s = r.onError?.((e) => n?.(e)) ?? null, r.startDetection();
|
|
638
650
|
},
|
|
639
651
|
update(e) {},
|
|
640
652
|
dispose() {
|
|
641
|
-
|
|
653
|
+
o?.(), o = null, s?.(), s = null, r?.dispose(), a && q(i), i = null, r = null, c.length = 0;
|
|
642
654
|
},
|
|
643
655
|
getInternal: () => ({
|
|
644
|
-
provider:
|
|
645
|
-
videoElement:
|
|
646
|
-
onDetection:
|
|
656
|
+
provider: r,
|
|
657
|
+
videoElement: i,
|
|
658
|
+
onDetection: l
|
|
647
659
|
})
|
|
648
660
|
};
|
|
649
|
-
},
|
|
661
|
+
}, Y = ({ options: e, children: n, onError: i }) => {
|
|
650
662
|
let { scene: o, camera: s, gl: c, size: f } = l(), { registerBackend: p, unregisterBackend: m } = t(), h = a(null), g = a(e);
|
|
651
663
|
g.current = e;
|
|
652
664
|
let _ = a(i);
|
|
@@ -655,7 +667,7 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
655
667
|
let e = !1;
|
|
656
668
|
return (async () => {
|
|
657
669
|
try {
|
|
658
|
-
let t =
|
|
670
|
+
let t = J(g.current, (e) => _.current?.(e));
|
|
659
671
|
if (e) return;
|
|
660
672
|
if (await p(t, {
|
|
661
673
|
scene: o,
|
|
@@ -681,7 +693,7 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
681
693
|
p,
|
|
682
694
|
m
|
|
683
695
|
]), /* @__PURE__ */ d(u, { children: n });
|
|
684
|
-
},
|
|
696
|
+
}, X = ({ tasks: e, progress: t, showDebug: n, onToggleDebug: r }) => {
|
|
685
697
|
let i = F({ throttle: 100 }), a = (e) => {
|
|
686
698
|
if (!i) return !1;
|
|
687
699
|
switch (e) {
|
|
@@ -747,7 +759,7 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
747
759
|
},
|
|
748
760
|
children: [n ? "Hide" : "Show", " Debug"]
|
|
749
761
|
})] });
|
|
750
|
-
},
|
|
762
|
+
}, Z = {}, Q = ({ tasks: e = ["hands"], options: t, children: r, showDebug: c = !1, cameraConstraints: l, canvasStyle: u = Z, onReady: p, onError: m }) => {
|
|
751
763
|
let [g, _] = o(c), [v, y] = o(null), [b, x] = o(null), S = a(p);
|
|
752
764
|
S.current = p;
|
|
753
765
|
let C = a(x);
|
|
@@ -799,13 +811,13 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
799
811
|
alpha: !0,
|
|
800
812
|
premultipliedAlpha: !1
|
|
801
813
|
},
|
|
802
|
-
children: /* @__PURE__ */ d(
|
|
814
|
+
children: /* @__PURE__ */ d($, {
|
|
803
815
|
options: E,
|
|
804
816
|
onError: m,
|
|
805
817
|
children: r
|
|
806
818
|
})
|
|
807
819
|
}),
|
|
808
|
-
/* @__PURE__ */ d(
|
|
820
|
+
/* @__PURE__ */ d(X, {
|
|
809
821
|
tasks: e,
|
|
810
822
|
progress: b,
|
|
811
823
|
showDebug: g,
|
|
@@ -825,7 +837,7 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
825
837
|
})
|
|
826
838
|
]
|
|
827
839
|
});
|
|
828
|
-
},
|
|
840
|
+
}, $ = ({ options: e, onError: t, children: n }) => /* @__PURE__ */ f(Y, {
|
|
829
841
|
options: e,
|
|
830
842
|
onError: t,
|
|
831
843
|
children: [
|
|
@@ -841,5 +853,4 @@ var h = ({ constraints: e, onReady: t, onError: n, mirrored: i = !0 }) => {
|
|
|
841
853
|
n
|
|
842
854
|
]
|
|
843
855
|
});
|
|
844
|
-
|
|
845
|
-
export { h as C, v as S, C as _, z as a, x as b, j as c, P as d, A as f, T as g, D as h, K as i, N as l, E as m, J as n, R as o, k as p, q as r, F as s, X as t, M as u, S as v, y as x, b as y };
|
|
856
|
+
export { h as C, v as S, C as _, z as a, x as b, j as c, P as d, A as f, T as g, D as h, J as i, N as l, E as m, X as n, R as o, k as p, Y as r, F as s, Q as t, M as u, S as v, y as x, b as y };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const e=require("./useXRStore-DfyQv43d.cjs");let t=require("react"),n=require("@react-three/fiber"),r=require("react/jsx-runtime"),i=require("three");var a=({constraints:e,onReady:n,onError:i,mirrored:a=!0})=>{let o=(0,t.useRef)(null),[s,c]=(0,t.useState)(!1),l=(0,t.useRef)(n);l.current=n;let u=(0,t.useRef)(i);return u.current=i,(0,t.useEffect)(()=>{let t=!1;return(async()=>{try{let n=await navigator.mediaDevices.getUserMedia({video:e||{width:{ideal:1280},height:{ideal:720},facingMode:`user`}});if(t||!o.current){for(let e of n.getTracks())e.stop();return}if(o.current.srcObject=n,await new Promise((e,t)=>{let n=o.current;if(!n){t(Error(`Video ref lost`));return}let r=()=>{n.removeEventListener(`canplay`,r),n.removeEventListener(`error`,i),e()},i=()=>{n.removeEventListener(`canplay`,r),n.removeEventListener(`error`,i),t(Error(`Video failed to load`))};n.addEventListener(`canplay`,r,{once:!0}),n.addEventListener(`error`,i,{once:!0})}),t)return;await o.current.play(),c(!0),l.current?.(o.current)}catch(e){if(t)return;let n=e instanceof Error?e:Error(`Failed to access camera`);console.error(`Camera error:`,n),u.current?.(n)}})(),()=>{if(t=!0,o.current?.srcObject){let e=o.current.srcObject;for(let t of e.getTracks())t.stop()}}},[e]),(0,r.jsx)(`video`,{ref:o,style:{position:`absolute`,inset:0,width:`100%`,height:`100%`,objectFit:`cover`,transform:a?`scaleX(-1)`:void 0,opacity:+!!s,transition:`opacity 0.5s ease`},autoPlay:!0,playsInline:!0,muted:!0})},o=(e,t)=>e.y<t.y,s=e=>[Math.abs(e[4].x-e[9].x)>Math.abs(e[3].x-e[9].x),o(e[8],e[6]),o(e[12],e[10]),o(e[16],e[14]),o(e[20],e[18])],c=e=>{if(e.length<21)return null;let t=s(e);return t[0]&&!t[1]&&!t[2]&&!t[3]&&!t[4]?{gesture:`thumbs_up`,confidence:.9}:null},l=e=>{if(e.length<21)return null;let t=s(e);return!t[0]&&t[1]&&t[2]&&!t[3]&&!t[4]?{gesture:`peace`,confidence:.8}:null},u=e=>{if(e.length<21)return null;let t=s(e);return Math.sqrt((e[4].x-e[8].x)**2+(e[4].y-e[8].y)**2)<.05&&t[2]&&t[3]&&t[4]?{gesture:`ok`,confidence:.8}:null},d=e=>e.length<21?null:s(e).filter(Boolean).length===5?{gesture:`open_hand`,confidence:.7}:null,f=e=>e.length<21?null:s(e).filter(Boolean).length<=1?{gesture:`fist`,confidence:.7}:null,p=[c,l,u,d,f],m={gesture:`none`,confidence:0},h=(e,t=p)=>{let n=m;for(let r of t){let t=r(e);t&&t.confidence>n.confidence&&(n=t)}return n},g=(e,t,n,r=!0)=>{let i=t.width/t.height,a=n.width/n.height,o=a>i?a/i:1,s=a>i?1:i/a,c=.5+(e.x-.5)*o,l=.5+(e.y-.5)*s;return[(r?.5-c:c-.5)*t.width,(.5-l)*t.height,0]},_=e=>{let t=e.length||1,n=0,r=0,i=0;for(let t of e)n+=t.x,r+=t.y,i+=t.z??0;return{x:n/t,y:r/t,z:i/t}},v={isPending:!0,isSuccess:!1,provider:null,videoElement:null,onDetection:()=>()=>{}},y=()=>{let t=e.r(e=>e.backends).get(e.i.VISION);return t?{...t.getInternal(),isPending:!1,isSuccess:!0}:v},b=({onHandMove:e,onGesture:a,landmarkIndex:o=9,mirror:s=!0,children:c})=>{let{onDetection:l,isSuccess:u}=y(),{viewport:d}=(0,n.useThree)(),f=(0,t.useRef)(null),p=(0,t.useRef)(new i.Vector3),m=(0,t.useRef)(``);return(0,t.useEffect)(()=>{if(u)return l(e=>{f.current=e})},[u,l]),(0,n.useFrame)(()=>{let t=f.current;if(!t)return;let n=t.hands[0];if(!n||n.landmarks.length===0)return;let[r,i,c]=g(n.landmarks[o]??n.landmarks[0],d,t.frameSize,s);p.current.set(r,i,c),e?.(p.current);let l=h(n.landmarks);l.gesture!==m.current&&(m.current=l.gesture,a?.(l.gesture,l.confidence))}),(0,r.jsx)(`group`,{children:c})},x=`person.bicycle.car.motorcycle.airplane.bus.train.truck.boat.traffic light.fire hydrant.stop sign.parking meter.bench.bird.cat.dog.horse.sheep.cow.elephant.bear.zebra.giraffe.backpack.umbrella.handbag.tie.suitcase.frisbee.skis.snowboard.sports ball.kite.baseball bat.baseball glove.skateboard.surfboard.tennis racket.bottle.wine glass.cup.fork.knife.spoon.bowl.banana.apple.sandwich.orange.broccoli.carrot.hot dog.pizza.donut.cake.chair.couch.potted plant.bed.dining table.toilet.tv.laptop.mouse.remote.keyboard.cell phone.microwave.oven.toaster.sink.refrigerator.book.clock.vase.scissors.teddy bear.hair drier.toothbrush`.split(`.`),S=(e,t={})=>({name:`yolo`,inputSize:640,labels:[...x],...t,path:e,decoder:`yolo`}),C=(e,t={})=>({name:`rf-detr`,inputSize:560,labels:[...x],...t,path:e,decoder:`rfdetr`}),w=(e,t={})=>({name:`yolo-seg`,inputSize:640,labels:[...x],...t,path:e,decoder:`yoloseg`}),T=e=>{let{isSuccess:n,onDetection:r}=y(),[i,a]=(0,t.useState)(null),o=e?.throttle??16,s=(0,t.useRef)(0);return(0,t.useEffect)(()=>{if(!n)a(null);else return r(e=>{let t=performance.now();t-s.current<o||(s.current=t,a(e))})},[n,r,o]),i},E=478,D=(e,t)=>t===`faces`?e.faces:t===`poses`?e.poses:e.hands,O=({task:e,index:a=0,debug:o=!1,mirror:s=!0,smoothing:c=.4,children:l})=>{let{onDetection:u,isSuccess:d}=y(),{viewport:f}=(0,n.useThree)(),p=(0,t.useRef)(null),m=(0,t.useRef)(null),h=(0,t.useRef)(null),v=(0,t.useMemo)(()=>new i.Object3D,[]),b=(0,t.useMemo)(()=>new i.Vector3,[]);return(0,t.useEffect)(()=>{m.current&&(m.current.visible=!1),h.current&&(h.current.count=0)},[]),(0,t.useEffect)(()=>{if(d)return u(e=>{p.current=e})},[d,u]),(0,n.useFrame)(()=>{let t=p.current,n=m.current;if(!t||!n)return;let r=D(t,e)[a],i=h.current;if(!r||r.landmarks.length===0){n.visible=!1,i&&(i.count=0);return}let l=n.visible;n.visible=!0;let[u,d,y]=g(_(r.landmarks),f,t.frameSize,s);if(b.set(u,d,y),!l||c>=1?n.position.copy(b):n.position.lerp(b,c),o&&i){let e=n.position,a=Math.min(r.landmarks.length,E);for(let n=0;n<a;n++){let[a,o,c]=g(r.landmarks[n],f,t.frameSize,s);v.position.set(a-e.x,o-e.y,c-e.z),v.updateMatrix(),i.setMatrixAt(n,v.matrix)}i.count=a,i.instanceMatrix.needsUpdate=!0}}),(0,r.jsxs)(`group`,{ref:m,children:[l,o&&(0,r.jsxs)(`instancedMesh`,{ref:h,args:[void 0,void 0,E],children:[(0,r.jsx)(`sphereGeometry`,{args:[.02,6,6]}),(0,r.jsx)(`meshBasicMaterial`,{color:`#00ff88`})]})]})},k=({tasks:e,width:t=320,height:n=240,style:i})=>{let a=T({throttle:50});return(0,r.jsxs)(`div`,{style:{background:`rgba(0, 0, 0, 0.8)`,color:`#00ff88`,padding:`8px`,borderRadius:`8px`,fontFamily:`monospace`,fontSize:`0.75rem`,width:t,height:n,overflow:`auto`,...i},children:[(0,r.jsxs)(`div`,{children:[`Frame: `,a?`active`:`none`]}),a&&(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)(`div`,{children:[`Size: `,a.frameSize.width,`x`,a.frameSize.height]}),e.includes(`hands`)&&(0,r.jsxs)(`div`,{children:[`Hands: `,a.hands.length]}),e.includes(`faces`)&&(0,r.jsxs)(`div`,{children:[`Faces: `,a.faces.length]}),e.includes(`poses`)&&(0,r.jsxs)(`div`,{children:[`Poses: `,a.poses.length]}),e.includes(`objects`)&&(0,r.jsxs)(`div`,{children:[`Objects: `,a.objects.length]})]})]})},A=class extends Error{cause;constructor(e,t){super(e),this.name=`VisionStoppedError`,this.cause=t}},j=`https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1/wasm`,M=class{type=`mediapipe`;models={};isInitialized=!1;isProcessing=!1;isDisposed=!1;videoElement=null;animationId=null;lastProcessTime=0;callbacks=[];errorCallbacks=[];options;constructor(e){this.options=e}async initialize(e){this.videoElement=e;let{FilesetResolver:t,HandLandmarker:n,FaceLandmarker:r,PoseLandmarker:i}=await import(`@mediapipe/tasks-vision`),a=await t.forVisionTasks(this.options.mediapipe?.wasmBasePath??j),o=this.options.useGpu??!0?`GPU`:`CPU`,s=async e=>{if(o===`CPU`)return e(`CPU`);try{return await e(`GPU`)}catch{return console.warn(`MediaPipe GPU delegate failed, falling back to CPU`),e(`CPU`)}},c=new Set(this.options.tasks??[`hands`,`faces`,`poses`]),l=[`hands`,`faces`,`poses`].filter(e=>c.has(e)),u=0;c.has(`hands`)&&(this.options.onProgress?.({step:++u,total:l.length,label:`hand landmarker`}),this.models.handLandmarker=await s(e=>n.createFromOptions(a,{baseOptions:{modelAssetPath:`https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,delegate:e},runningMode:`VIDEO`,numHands:2,minHandDetectionConfidence:.7,minHandPresenceConfidence:.5,minTrackingConfidence:.5}))),c.has(`faces`)&&(this.options.onProgress?.({step:++u,total:l.length,label:`face landmarker`}),this.models.faceLandmarker=await s(e=>r.createFromOptions(a,{baseOptions:{modelAssetPath:`https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`,delegate:e},runningMode:`VIDEO`,numFaces:1,minFaceDetectionConfidence:.7,minFacePresenceConfidence:.5,minTrackingConfidence:.5,outputFaceBlendshapes:!1,outputFacialTransformationMatrixes:!1}))),c.has(`poses`)&&(this.options.onProgress?.({step:++u,total:l.length,label:`pose landmarker`}),this.models.poseLandmarker=await s(e=>i.createFromOptions(a,{baseOptions:{modelAssetPath:`https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task`,delegate:e},runningMode:`VIDEO`,numPoses:1,minPoseDetectionConfidence:.7,minPosePresenceConfidence:.5,minTrackingConfidence:.5,outputSegmentationMasks:this.options.segmentation??!1}))),this.isInitialized=!0,this.options.onProgress?.(null)}startDetection(){this.isInitialized&&this.videoElement&&!this.isProcessing&&(this.isProcessing=!0,this.processFrame())}stopDetection(){this.isProcessing=!1,this.animationId&&=(cancelAnimationFrame(this.animationId),null)}onDetection(e){return this.callbacks.push(e),()=>{let t=this.callbacks.indexOf(e);t>-1&&this.callbacks.splice(t,1)}}onError(e){return this.errorCallbacks.push(e),()=>{let t=this.errorCallbacks.indexOf(e);t>-1&&this.errorCallbacks.splice(t,1)}}dispose(){this.isDisposed=!0,this.stopDetection();for(let e of Object.values(this.models))try{e?.close()}catch{}this.models={},this.videoElement=null,this.callbacks=[],this.errorCallbacks=[],this.isInitialized=!1,this.isProcessing=!1}processFrame=()=>{if(!this.isProcessing||!this.videoElement||this.isDisposed)return;let e=performance.now(),t=this.options.throttle??16;if(e-this.lastProcessTime<t){this.animationId=requestAnimationFrame(this.processFrame);return}let{videoWidth:n,videoHeight:r}=this.videoElement;if(n===0||r===0){this.animationId=requestAnimationFrame(this.processFrame);return}this.lastProcessTime=e;let i=e;try{let e={hands:[],faces:[],poses:[],objects:[],timestamp:i,frameSize:{width:n,height:r}};if(this.models.handLandmarker){let t=this.models.handLandmarker.detectForVideo(this.videoElement,i);e.hands=t.landmarks.map((e,n)=>({landmarks:e.map(e=>({x:e.x,y:e.y,z:e.z||0})),confidence:t.handednesses?.[n]?.[0]?.score||.5}))}if(this.models.faceLandmarker&&(e.faces=this.models.faceLandmarker.detectForVideo(this.videoElement,i).faceLandmarks.map(e=>{let t=e.map(e=>e.x),n=e.map(e=>e.y),r=(Math.max(...t)-Math.min(...t))*(Math.max(...n)-Math.min(...n)),i=Math.min(.95,Math.max(.6,r*4));return{landmarks:e.map(e=>({x:e.x,y:e.y,z:e.z||0})),confidence:i}})),this.models.poseLandmarker){let t=this.models.poseLandmarker.detectForVideo(this.videoElement,i);e.poses=t.landmarks.map(e=>{let t=0,n=0;for(let r of e)typeof r.visibility==`number`&&(t+=r.visibility,n++);let r=n>0?t/n:.5;return{landmarks:e.map(e=>({x:e.x,y:e.y,z:e.z||0})),confidence:r}});let n=t.segmentationMasks?.[0];if(n){let t=new OffscreenCanvas(n.width,n.height).getContext(`2d`),r=n.getAsFloat32Array(),i=t.createImageData(n.width,n.height);for(let e=0;e<r.length;e++){let t=r[e]*255;i.data[e*4]=t,i.data[e*4+1]=t,i.data[e*4+2]=t,i.data[e*4+3]=255}e.segmentationMask=i}}this.handleResult(e)}catch(e){if(e instanceof WebAssembly.RuntimeError){this.fail(e);return}console.error(`Vision processing error:`,e)}this.animationId=requestAnimationFrame(this.processFrame)};fail(e){this.stopDetection(),console.error(`Vision processing stopped:`,e);let t=new A(`Vision tracking stopped because this browser blocked a graphics feature it needs. Strict fingerprinting protection is a common cause.`,e);for(let e of this.errorCallbacks)e(t)}handleResult(e){let t=this.options.minConfidence??.7,n=this.options.maxResults??2,r={...e,hands:e.hands.filter(e=>e.confidence>=t).slice(0,n),faces:e.faces.filter(e=>e.confidence>=t).slice(0,n),poses:e.poses.filter(e=>e.confidence>=t).slice(0,n),objects:[]};for(let e of this.callbacks)try{e(r)}catch(e){console.error(`Error in vision callback:`,e)}}},N=5e3,P=class{type=`onnx`;worker=null;isInitialized=!1;isDisposed=!1;isProcessing=!1;videoElement=null;animationId=null;loadedModels=new Map;lastProcessTime=0;processingStartTime=0;callbacks=[];options;messageHandler=null;pendingModelCallbacks=new Map;constructor(e){this.options=e}post(e,t){t?this.worker?.postMessage(e,t):this.worker?.postMessage(e)}async initialize(e){this.videoElement=e,this.worker=new Worker(new URL(`/assets/onnxWorker-3jptKfHs.js`,``+{}.url),{type:`module`}),await new Promise((e,t)=>{let n=setTimeout(()=>{t(Error(`ONNX worker initialization timeout`))},1e4);this.messageHandler=r=>{let i=r.data;switch(i.type){case`initialized`:clearTimeout(n),this.isInitialized=!0,e();break;case`result`:this.isProcessing=!1,this.handleResult(i.result);break;case`modelLoaded`:{let e=this.pendingModelCallbacks.get(i.modelName);e&&(this.pendingModelCallbacks.delete(i.modelName),e.resolve());break}case`modelError`:{let e=this.pendingModelCallbacks.get(i.modelName);e&&(this.pendingModelCallbacks.delete(i.modelName),e.reject(Error(i.error)));break}case`error`:this.isProcessing=!1,this.isInitialized?console.error(`ONNX worker error:`,i.error):(clearTimeout(n),t(Error(i.error)))}},this.worker?.addEventListener(`message`,this.messageHandler),this.post({type:`init`})});let t=this.options.onnx?.models??[];for(let e of t)await this.loadModel(e)}startDetection(){if(!this.isInitialized||!this.videoElement){console.warn(`ONNX provider not initialized`);return}let e=()=>{if(!this.videoElement||!this.isInitialized)return;let t=Date.now(),n=this.options.throttle??100;t-this.lastProcessTime<n||(this.isProcessing&&this.processingStartTime>0&&t-this.processingStartTime>N&&(console.warn(`ONNX processing stall detected, resetting`),this.isProcessing=!1),this.isProcessing||this.processCurrentFrame()),this.animationId=requestAnimationFrame(e)};e()}stopDetection(){this.animationId&&=(cancelAnimationFrame(this.animationId),null),this.isProcessing=!1}onDetection(e){return this.callbacks.push(e),()=>{let t=this.callbacks.indexOf(e);t>-1&&this.callbacks.splice(t,1)}}dispose(){this.isDisposed=!0,this.stopDetection(),this.worker&&=(this.messageHandler&&=(this.worker.removeEventListener(`message`,this.messageHandler),null),this.post({type:`dispose`}),this.worker.terminate(),null),this.videoElement=null,this.callbacks.length=0,this.loadedModels.clear(),this.pendingModelCallbacks.clear(),this.isInitialized=!1}async loadModel(e){if(!this.worker)throw Error(`Worker not initialized`);let t=new Promise((t,n)=>{this.pendingModelCallbacks.set(e.name,{resolve:t,reject:n})});this.post({type:`loadModel`,model:e}),await t,this.loadedModels.set(e.name,e)}processCurrentFrame(){if(!this.videoElement||!this.worker||this.isProcessing)return;let e=Date.now();this.isProcessing=!0,this.lastProcessTime=e,this.processingStartTime=e;let{videoWidth:t,videoHeight:n}=this.videoElement;createImageBitmap(this.videoElement).then(e=>{this.isDisposed?e.close():this.post({type:`process`,imageBitmap:e,sourceWidth:t,sourceHeight:n,options:{minConfidence:this.options.minConfidence??.5,maxResults:this.options.maxResults??100}},[e])},e=>{this.isProcessing=!1,console.error(`Error creating ImageBitmap:`,e)})}handleResult(e){let t=this.options.minConfidence??.5,n=this.options.maxResults??100,r={hands:[],faces:[],poses:[],objects:e.detections.filter(e=>e.confidence>=t).sort((e,t)=>t.confidence-e.confidence).slice(0,n),masks:e.masks.length>0?e.masks:void 0,timestamp:e.timestamp,frameSize:e.frameSize,processingTime:e.processingTime};for(let e of this.callbacks)try{e(r)}catch(e){console.error(`Error in ONNX detection callback:`,e)}}},F=e=>e.provider===`onnx`?new P(e):new M(e),I=async e=>{let t=document.createElement(`video`);t.style.display=`none`,document.body.appendChild(t);try{return t.srcObject=await navigator.mediaDevices.getUserMedia({video:{width:{ideal:1280,min:640},height:{ideal:720,min:480},facingMode:e.provider===`onnx`?`environment`:`user`,frameRate:{ideal:30,min:15}}}),t.playsInline=!0,t.muted=!0,await new Promise((e,n)=>{t.onloadedmetadata=()=>{t.play().then(e).catch(n)},t.onerror=n}),t}catch(e){throw document.body.removeChild(t),e}},L=e=>{if(!e)return;let t=e.srcObject;if(t)for(let e of t.getTracks())e.stop();e.parentNode&&e.parentNode.removeChild(e)},R=(t,n)=>{let r=null,i=null,a=!1,o=null,s=null,c=[],l=e=>(c.push(e),()=>{let t=c.indexOf(e);t>-1&&c.splice(t,1)});return{type:e.i.VISION,async init(e){t.videoElement?(i=t.videoElement,a=!1):(i=await I(t),a=!0),r=F(t),await r.initialize(i),o=r.onDetection(e=>{for(let t of c)try{t(e)}catch(e){console.error(`Error in vision detection callback:`,e)}}),s=r.onError?.(e=>n?.(e))??null,r.startDetection()},update(e){},dispose(){o?.(),o=null,s?.(),s=null,r?.dispose(),a&&L(i),i=null,r=null,c.length=0},getInternal:()=>({provider:r,videoElement:i,onDetection:l})}},z=({options:i,children:a,onError:o})=>{let{scene:s,camera:c,gl:l,size:u}=(0,n.useThree)(),{registerBackend:d,unregisterBackend:f}=e.r(),p=(0,t.useRef)(null),m=(0,t.useRef)(i);m.current=i;let h=(0,t.useRef)(o);return h.current=o,(0,t.useEffect)(()=>{if(p.current)return;let e=!1;return(async()=>{try{let t=R(m.current,e=>h.current?.(e));if(e)return;if(await d(t,{scene:s,camera:c,renderer:l,size:u}),e){f(t);return}p.current=t}catch(e){let t=e instanceof Error?e:Error(String(e));console.error(`[VisionSession] Failed to initialize:`,t),h.current?.(t)}})(),()=>{e=!0,p.current&&=(f(p.current),null)}},[s,c,l,d,f]),(0,r.jsx)(r.Fragment,{children:a})},B=({tasks:e,progress:t,showDebug:n,onToggleDebug:i})=>{let a=T({throttle:100}),o=e=>{if(!a)return!1;switch(e){case`hands`:return a.hands.length>0;case`faces`:return a.faces.length>0;case`poses`:return a.poses.length>0;case`objects`:return a.objects.length>0;default:return!1}},s=e.some(o),c=a!==null;return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)(`div`,{style:{position:`fixed`,top:`20px`,left:`20px`,background:`rgba(0, 0, 0, 0.7)`,color:`white`,padding:`12px`,borderRadius:`8px`,fontFamily:`system-ui, sans-serif`,fontSize:`0.9rem`,zIndex:1e3},children:[(0,r.jsxs)(`div`,{style:{display:`flex`,alignItems:`center`,gap:`8px`,marginBottom:`8px`},children:[(0,r.jsx)(`div`,{style:{width:`8px`,height:`8px`,borderRadius:`50%`,background:c?`#00ff88`:`#ffaa00`}}),(0,r.jsx)(`span`,{children:s?`Active`:c?`Ready`:t?`Loading models (${t.step}/${t.total}) ${t.label}...`:`Starting...`})]}),e.map(e=>(0,r.jsxs)(`div`,{style:{fontSize:`0.8rem`,opacity:.8,display:`flex`,justifyContent:`space-between`,gap:`12px`},children:[(0,r.jsx)(`span`,{children:e}),(0,r.jsx)(`span`,{children:o(e)?`active`:c?`idle`:`...`})]},e))]}),(0,r.jsxs)(`button`,{onClick:i,type:`button`,style:{position:`fixed`,top:`20px`,right:`20px`,background:`rgba(0, 0, 0, 0.7)`,color:`white`,border:`1px solid #333`,padding:`8px 12px`,borderRadius:`6px`,cursor:`pointer`,fontFamily:`system-ui, sans-serif`,fontSize:`0.8rem`,zIndex:1e3},children:[n?`Hide`:`Show`,` Debug`]})]})},V={},H=({tasks:e=[`hands`],options:i,children:o,showDebug:s=!1,cameraConstraints:c,canvasStyle:l=V,onReady:u,onError:d})=>{let[f,p]=(0,t.useState)(s),[m,h]=(0,t.useState)(null),[g,_]=(0,t.useState)(null),v=(0,t.useRef)(u);v.current=u;let y=(0,t.useRef)(_);y.current=_;let b=(0,t.useCallback)(e=>y.current(e),[]),x=(0,t.useCallback)(e=>{h(e),v.current?.()},[]),S=(0,t.useMemo)(()=>({tasks:e,minConfidence:.6,maxResults:2,useGpu:!0,...i,videoElement:m??void 0,onProgress:b}),[e,i,m,b]);return(0,r.jsxs)(`div`,{style:{position:`relative`,width:`100%`,height:`100%`},children:[(0,r.jsx)(a,{constraints:c,onReady:x,onError:d}),m&&(0,r.jsx)(n.Canvas,{style:{position:`absolute`,inset:0,pointerEvents:`none`,...l},camera:{position:[0,0,5],fov:75},gl:{alpha:!0,premultipliedAlpha:!1},children:(0,r.jsx)(U,{options:S,onError:d,children:o})}),(0,r.jsx)(B,{tasks:e,progress:g,showDebug:f,onToggleDebug:()=>p(e=>!e)}),f&&(0,r.jsx)(k,{tasks:e,width:320,height:240,style:{position:`fixed`,bottom:`80px`,right:`20px`,maxWidth:`calc(100vw - 40px)`,overflow:`hidden`}})]})},U=({options:e,onError:t,children:n})=>(0,r.jsxs)(z,{options:e,onError:t,children:[(0,r.jsx)(`ambientLight`,{intensity:.6}),(0,r.jsx)(`directionalLight`,{position:[10,10,10],intensity:.8}),n]});Object.defineProperty(exports,"C",{enumerable:!0,get:function(){return a}}),Object.defineProperty(exports,"S",{enumerable:!0,get:function(){return c}}),Object.defineProperty(exports,"_",{enumerable:!0,get:function(){return p}}),Object.defineProperty(exports,"a",{enumerable:!0,get:function(){return k}}),Object.defineProperty(exports,"b",{enumerable:!0,get:function(){return d}}),Object.defineProperty(exports,"c",{enumerable:!0,get:function(){return x}}),Object.defineProperty(exports,"d",{enumerable:!0,get:function(){return w}}),Object.defineProperty(exports,"f",{enumerable:!0,get:function(){return b}}),Object.defineProperty(exports,"g",{enumerable:!0,get:function(){return h}}),Object.defineProperty(exports,"h",{enumerable:!0,get:function(){return _}}),Object.defineProperty(exports,"i",{enumerable:!0,get:function(){return R}}),Object.defineProperty(exports,"l",{enumerable:!0,get:function(){return C}}),Object.defineProperty(exports,"m",{enumerable:!0,get:function(){return g}}),Object.defineProperty(exports,"n",{enumerable:!0,get:function(){return B}}),Object.defineProperty(exports,"o",{enumerable:!0,get:function(){return O}}),Object.defineProperty(exports,"p",{enumerable:!0,get:function(){return y}}),Object.defineProperty(exports,"r",{enumerable:!0,get:function(){return z}}),Object.defineProperty(exports,"s",{enumerable:!0,get:function(){return T}}),Object.defineProperty(exports,"t",{enumerable:!0,get:function(){return H}}),Object.defineProperty(exports,"u",{enumerable:!0,get:function(){return S}}),Object.defineProperty(exports,"v",{enumerable:!0,get:function(){return f}}),Object.defineProperty(exports,"x",{enumerable:!0,get:function(){return l}}),Object.defineProperty(exports,"y",{enumerable:!0,get:function(){return u}});
|
package/build/vision.cjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});const e=require(
|
|
1
|
+
Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});const e=require("./vision-B9EQHClV.cjs");exports.COCO_LABELS=e.c,exports.CameraBackground=e.C,exports.DEFAULT_GESTURES=e._,exports.HandTracker=e.f,exports.VisionAnchor=e.o,exports.VisionDebug=e.a,exports.VisionOverlay=e.t,exports.VisionSession=e.r,exports.VisionStatus=e.n,exports.createVisionBackend=e.i,exports.detectGesture=e.g,exports.fist=e.v,exports.landmarkToWorld=e.m,exports.landmarksCentroid=e.h,exports.ok=e.y,exports.openHand=e.b,exports.peace=e.x,exports.rfDetr=e.l,exports.thumbsUp=e.S,exports.useVisionBackend=e.p,exports.useVisionFrame=e.s,exports.yolo=e.u,exports.yoloSeg=e.d;
|
package/build/vision.js
CHANGED
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
export { a as COCO_LABELS, e as CameraBackground, n as DEFAULT_GESTURES, s as HandTracker, m as VisionAnchor, r as VisionDebug, v as VisionOverlay, g as VisionSession, p as VisionStatus, u as createVisionBackend, c as detectGesture, b as fist, f as landmarkToWorld, l as landmarksCentroid, S as ok, i as openHand, x as peace, d as rfDetr, t as thumbsUp, h as useVisionBackend, _ as useVisionFrame, y as yolo, o as yoloSeg };
|
|
1
|
+
export { c as COCO_LABELS, C as CameraBackground, _ as DEFAULT_GESTURES, f as HandTracker, o as VisionAnchor, a as VisionDebug, t as VisionOverlay, r as VisionSession, n as VisionStatus, i as createVisionBackend, g as detectGesture, v as fist, m as landmarkToWorld, h as landmarksCentroid, y as ok, b as openHand, x as peace, l as rfDetr, S as thumbsUp, p as useVisionBackend, s as useVisionFrame, u as yolo, d as yoloSeg } from "./vision-1_qdyG-O.js";
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@omnidotdev/rdk",
|
|
3
3
|
"description": "Reality Development Kit, a unified React-first XR engine",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.15.0",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"sideEffects": false,
|
|
@@ -79,7 +79,7 @@
|
|
|
79
79
|
},
|
|
80
80
|
"peerDependencies": {
|
|
81
81
|
"@ar-js-org/ar.js": "^3.4.7",
|
|
82
|
-
"@mediapipe/tasks-vision": "^0.
|
|
82
|
+
"@mediapipe/tasks-vision": "^1.0.0",
|
|
83
83
|
"@react-three/fiber": "^9.5.0",
|
|
84
84
|
"@react-three/xr": "^6.6.29",
|
|
85
85
|
"locar": "^0.2.12",
|
|
@@ -108,21 +108,21 @@
|
|
|
108
108
|
"devDependencies": {
|
|
109
109
|
"@ar-js-org/ar.js": "^3.4.7",
|
|
110
110
|
"@biomejs/biome": "^2.4.8",
|
|
111
|
-
"@mediapipe/tasks-vision": "^
|
|
111
|
+
"@mediapipe/tasks-vision": "^1.2.3",
|
|
112
112
|
"@react-three/fiber": "^9.5.0",
|
|
113
113
|
"@testing-library/jest-dom": "^7.0.0",
|
|
114
114
|
"@testing-library/react": "^16.3.1",
|
|
115
115
|
"@types/node": "^26.0.0",
|
|
116
116
|
"@types/react": "^19.2.7",
|
|
117
117
|
"@types/react-dom": "^19.2.3",
|
|
118
|
-
"@types/three": "^0.
|
|
118
|
+
"@types/three": "^0.187.0",
|
|
119
119
|
"@typescript/typescript6": "^6.0.2",
|
|
120
120
|
"jsdom": "^30.0.0",
|
|
121
121
|
"locar": "^0.2.12",
|
|
122
122
|
"onnxruntime-web": "^1.27.0",
|
|
123
123
|
"react": "^19.2.3",
|
|
124
124
|
"react-dom": "^19.2.3",
|
|
125
|
-
"three": "^0.
|
|
125
|
+
"three": "^0.187.0",
|
|
126
126
|
"typescript": "^7.0.0",
|
|
127
127
|
"vite": "^8.0.0",
|
|
128
128
|
"vite-plugin-dts": "^5.0.0",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
const e=(e,t)=>{let n=e.x+e.width,r=e.y+e.height,i=t.x+t.width,a=t.y+t.height,o=Math.max(e.x,t.x),s=Math.max(e.y,t.y),c=Math.min(n,i),l=Math.min(r,a),u=Math.max(0,c-o)*Math.max(0,l-s);if(u===0)return 0;let d=e.width*e.height+t.width*t.height-u;return d<=0?0:u/d},t=(t,n)=>{let r=[...t].sort((e,t)=>t.confidence-e.confidence),i=[];for(let t of r)i.some(r=>r.label===t.label&&e(r.bbox,t.bbox)>n)||i.push(t);return i},n=(e,t,n,r,i)=>{let a=(e-i.padX)/i.scale,o=(t-i.padY)/i.scale,s=(n-i.padX)/i.scale,c=(r-i.padY)/i.scale,l=Math.max(0,Math.min(a,i.sourceWidth)),u=Math.max(0,Math.min(o,i.sourceHeight)),d=Math.max(0,Math.min(s,i.sourceWidth)),f=Math.max(0,Math.min(c,i.sourceHeight)),p=d-l,m=f-u;return p<=0||m<=0?null:{x:l,y:u,width:p,height:m}},r=e=>1/(1+Math.exp(-e)),i=e=>{let t=Object.values(e).filter(e=>e.dims.length===3),n=t.find(e=>e.dims[2]===4),r=t.find(e=>e!==n);if(!n||!r)throw Error(`RF-DETR decoder: expected a [1, Q, 4] boxes tensor and a [1, Q, C] logits tensor`);return{boxes:n,logits:r}},a={name:`rfdetr`,decode(e,t){let{boxes:a,logits:o}=i(e),s=a.dims[1],c=o.dims[2],l=a.data,u=o.data,d=[];for(let e=0;e<s;e++){let i=0,a=-1;for(let t=0;t<c;t++){let n=r(u[e*c+t]);n>i&&(i=n,a=t)}if(a<0||i<t.minConfidence)continue;let o=e*4,s=l[o]*t.inputSize,f=l[o+1]*t.inputSize,p=l[o+2]*t.inputSize,m=l[o+3]*t.inputSize,h=n(s-p/2,f-m/2,s+p/2,f+m/2,t);h&&d.push({label:t.labels[a]??String(a),confidence:i,bbox:h})}return{objects:d.sort((e,t)=>t.confidence-e.confidence).slice(0,t.maxResults)}}},o=e=>{let t=Object.values(e);if(t.length===0)throw Error(`YOLO decoder: no output tensors`);return t[0]},s={name:`yolo`,decode(e,r){let{dims:i,data:a}=o(e);if(i.length!==3||i[0]!==1)throw Error(`YOLO decoder: expected [1, C, A] output, got [${i.join(`, `)}]`);let s=i[1],c=i[2],l=s-4;if(l<=0)throw Error(`YOLO decoder: too few channels (${s})`);let u=[];for(let e=0;e<c;e++){let t=0,i=-1;for(let n=0;n<l;n++){let r=a[(4+n)*c+e];r>t&&(t=r,i=n)}if(i<0||t<r.minConfidence)continue;let o=a[e],s=a[c+e],d=a[2*c+e],f=a[3*c+e],p=n(o-d/2,s-f/2,o+d/2,s+f/2,r);p&&u.push({label:r.labels[i]??String(i),confidence:t,bbox:p})}return{objects:t(u,.45).sort((e,t)=>t.confidence-e.confidence).slice(0,r.maxResults)}}},c=e=>1/(1+Math.exp(-e)),l={yolo:s,rfdetr:a,yoloseg:{name:`yoloseg`,decode(t,r){let i=Object.values(t),a=i.find(e=>e.dims.length===4),o=i.find(e=>e.dims.length===3);if(!a||!o)throw Error(`YOLO-seg decoder: expected a [1, C, A] detection tensor and a [1, M, H, W] prototype tensor`);let s=a.dims[1],l=a.dims[2],u=a.dims[3],d=o.dims[1],f=o.dims[2],p=d-4-s;if(p<=0)throw Error(`YOLO-seg decoder: bad channel count (${d})`);let m=o.data,h=[];for(let e=0;e<f;e++){let t=0,i=-1;for(let n=0;n<p;n++){let r=m[(4+n)*f+e];r>t&&(t=r,i=n)}if(i<0||t<r.minConfidence)continue;let a=m[e],o=m[f+e],c=m[2*f+e],l=m[3*f+e],u=n(a-c/2,o-l/2,a+c/2,o+l/2,r);if(!u)continue;let d=[];for(let t=0;t<s;t++)d.push(m[(4+p+t)*f+e]);h.push({label:r.labels[i]??String(i),confidence:t,bbox:u,coeffs:d})}h.sort((e,t)=>t.confidence-e.confidence);let g=[];for(let t of h)if(g.some(n=>n.label===t.label&&e(n.bbox,t.bbox)>.45)||g.push(t),g.length>=r.maxResults)break;let _=a.data,v=l*u,y=u/r.inputSize,b=l/r.inputSize,x=g.map(e=>{let{x:t,y:n,width:i,height:a}=e.bbox,o=t*r.scale+r.padX,d=n*r.scale+r.padY,f=(t+i)*r.scale+r.padX,p=(n+a)*r.scale+r.padY,m=Math.max(0,Math.floor(o*y)),h=Math.max(0,Math.floor(d*b)),g=Math.min(u,Math.ceil(f*y)),x=Math.min(l,Math.ceil(p*b)),S=Math.max(1,g-m),C=Math.max(1,x-h),w=new Uint8Array(S*C);for(let t=h;t<x;t++)for(let n=m;n<g;n++){let r=0;for(let i=0;i<s;i++)r+=e.coeffs[i]*_[i*v+t*u+n];c(r)>.5&&(w[(t-h)*S+(n-m)]=255)}return{label:e.label,confidence:e.confidence,bbox:e.bbox,mask:w,maskWidth:S,maskHeight:C}});return{objects:g.map(e=>({label:e.label,confidence:e.confidence,bbox:e.bbox})),masks:x}}}},u=(e=`yolo`)=>l[e]??s,d=(e,t,n)=>{let r=Math.min(n/e,n/t),i=e*r,a=t*r;return{scale:r,drawWidth:i,drawHeight:a,padX:(n-i)/2,padY:(n-a)/2}},f=(e,t)=>{let n=t*t,r=new Float32Array(3*n);for(let t=0;t<n;t++)r[t]=e[t*4]/255,r[n+t]=e[t*4+1]/255,r[2*n+t]=e[t*4+2]/255;return r};let p=null;const m=new Map,h=()=>(p||=import(`onnxruntime-web`).then(e=>(e.env.wasm.wasmPaths=`https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/`,e)),p),g=(e,t)=>{t&&t.length>0?self.postMessage(e,{transfer:t}):self.postMessage(e)},_=async(e,t,n,r,i,a)=>{let o=t.config.inputSize??640,{scale:s,padX:c,padY:l,drawWidth:p,drawHeight:m}=d(r,i,o),h=new OffscreenCanvas(o,o).getContext(`2d`);if(!h)throw Error(`Failed to acquire OffscreenCanvas 2D context`);h.fillStyle=`rgb(114, 114, 114)`,h.fillRect(0,0,o,o),h.drawImage(n,c,l,p,m);let{data:g}=h.getImageData(0,0,o,o),_=new e.Tensor(`float32`,f(g,o),[1,3,o,o]),v={[t.session.inputNames[0]]:_},y=await t.session.run(v),b={};for(let[e,t]of Object.entries(y))b[e]={data:t.data,dims:t.dims};let x={inputSize:o,sourceWidth:r,sourceHeight:i,scale:s,padX:c,padY:l,labels:t.config.labels??[],minConfidence:a.minConfidence,maxResults:a.maxResults};return u(t.config.decoder).decode(b,x)};self.addEventListener(`message`,async e=>{let t=e.data;switch(t.type){case`init`:g({type:`initialized`});break;case`loadModel`:{let{model:e}=t;try{let t=await(await h()).InferenceSession.create(e.path,{executionProviders:[`webgpu`,`wasm`]});m.set(e.name,{config:e,session:t}),g({type:`modelLoaded`,modelName:e.name})}catch(t){g({type:`modelError`,modelName:e.name,error:t instanceof Error?t.message:`Unknown error`})}break}case`process`:{let{imageBitmap:e,sourceWidth:n,sourceHeight:r,options:i}=t;try{let t=await h(),a=performance.now(),o=[],s=[];for(let a of m.values()){let c=await _(t,a,e,n,r,i);c.objects&&o.push(...c.objects),c.masks&&s.push(...c.masks)}let c=s.map(e=>e.mask.buffer);g({type:`result`,result:{detections:o,masks:s,frameSize:{width:n,height:r},timestamp:Date.now(),processingTime:performance.now()-a}},c)}catch(e){g({type:`error`,error:e instanceof Error?e.message:`Unknown error`})}finally{e.close()}break}case`dispose`:for(let{session:e}of m.values())e.release();m.clear();break}});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
const e=require(`./useXRStore-Cf_XL0aI.cjs`);let t=require(`@react-three/fiber`);var n=({children:n})=>{let{updateBackends:r}=e.r();return(0,t.useFrame)((e,t)=>{r(t)}),n};Object.defineProperty(exports,`t`,{enumerable:!0,get:function(){return n}});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
const e=require(`./useXRStore-Cf_XL0aI.cjs`);let t=require(`react`),n=require(`@react-three/fiber`),r=require(`@ar-js-org/ar.js/three.js/build/ar-threex`),i=require(`react/jsx-runtime`);var a={isPending:!0,isSuccess:!1,arSource:null,arContext:null},o=()=>{let t=e.r(e=>e.backends).get(e.i.FIDUCIAL);return t?{...t.getInternal(),isPending:!1,isSuccess:!0}:a},s=({patternUrl:e,barcodeValue:a,params:s,onMarkerFound:c,onMarkerLost:l,children:u})=>{let d=(0,t.useRef)(null),f=o(),p=(0,t.useRef)(!1),m=(0,t.useRef)(!1),h=(0,t.useRef)(null);return(0,t.useEffect)(()=>{if(!f.isSuccess)return;let t=!1;return(()=>{try{let{arContext:n}=f,i=d.current;if(!n||!i||t)return;if(i.visible=!1,m.current=!1,!n.arController){requestAnimationFrame(()=>{!d.current||t||!f.arContext?.arController||(h.current=new r.ArMarkerControls(f.arContext,d.current,{type:e?`pattern`:a?`barcode`:`unknown`,patternUrl:e,barcodeValue:a,...s}),m.current=!0)});return}h.current=new r.ArMarkerControls(n,i,{type:e?`pattern`:a?`barcode`:`unknown`,patternUrl:e,barcodeValue:a,...s}),m.current=!0}catch(e){console.error(`Failed to initialize fiducial anchor:`,e)}})(),()=>{t=!0;let e=h.current;typeof e?.dispose==`function`&&e.dispose(),m.current=!1}},[f.isSuccess,f.arContext,e,a,s,f]),(0,n.useFrame)(()=>{let e=d.current;if(!e||!m.current){e&&(e.visible=!1);return}let t=e.visible;t&&!p.current?(p.current=!0,c?.()):!t&&p.current&&(p.current=!1,l?.())}),(0,i.jsx)(`group`,{ref:d,visible:!1,children:u})},c=({options:r={},children:i})=>{let{scene:a,camera:o,gl:s,size:c}=(0,n.useThree)(),{registerBackend:u,unregisterBackend:d}=e.r(),f=(0,t.useRef)(null),p=(0,t.useRef)(r);return p.current=r,(0,t.useEffect)(()=>{if(f.current)return;let e=!1;return(async()=>{try{let t=l(p.current);if(e)return;await u(t,{scene:a,camera:o,renderer:s,size:c}),f.current=t}catch(e){console.error(`[FiducialSession] Failed to initialize:`,e)}})(),()=>{e=!0,f.current&&=(d(f.current),null)}},[a,o,s,u,d]),i},l=t=>{let n=null,i=null,a=null,o;return{type:e.i.FIDUCIAL,async init({camera:e,renderer:s}){n=new r.ArToolkitSource({sourceType:t?.sourceType??`webcam`}),await new Promise((e,t)=>{let r=setTimeout(()=>{t(Error(`AR source initialization timeout`))},1e4);n?.init(()=>{clearTimeout(r);let t=n?.domElement;t&&(t.style.position=`fixed`,t.style.top=`0`,t.style.left=`0`,t.style.width=`100vw`,t.style.height=`100vh`,t.style.objectFit=`cover`,t.style.zIndex=`-1`,t.style.pointerEvents=`none`),e()})}),i=new r.ArToolkitContext({cameraParametersUrl:t?.cameraParametersUrl??new URL(`data:application/octet-stream;base64,AAACgAAAAeBAgwrsW6bUSwAAAAAAAAAAQHQ3KqAAAAAAAAAAAAAAAAAAAAAAAAAAQIL0K3dHyf9AbbNowAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/wAAAAAAAAAAAAAAAAAAA/uWNa4AAAAL+3lTLAAAAAv17YFWAAAAA/VYLXIAAAAECCe0YgAAAAQIJlMOAAAABAdDcqoAAAAEBts2jAAAAAP+8OmzqkDy4=`,``+{}.url).toString(),detectionMode:t?.detectionMode??`mono`,patternRatio:t?.patternRatio??.5,matrixCodeType:t?.matrixCodeType??`3x3`}),a=e;let c=()=>{n?.onResizeElement();let e=window.innerWidth,t=window.innerHeight;s.setSize(e,t,!1),n?.copyElementSizeTo(s.domElement),i?.arController&&n?.copyElementSizeTo(i.arController.canvas)};await new Promise((t,n)=>{let r=setTimeout(()=>{n(Error(`AR context initialization timeout`))},1e4);i?.init(()=>{clearTimeout(r);try{let n=i?.getProjectionMatrix();n&&e.projectionMatrix.copy(n),c(),t()}catch(e){n(e)}})}),window.addEventListener(`resize`,c),o=c},update(){if(!(!n||!i)&&!(!n.ready||!n.domElement))try{i.update(n.domElement);let e=i.getProjectionMatrix?.();a&&e&&a.projectionMatrix.copy(e)}catch(e){console.warn(`AR.js update error:`,e)}},dispose(){if(o&&=(window.removeEventListener(`resize`,o),void 0),i&&typeof i.dispose==`function`&&i.dispose(),n?.domElement){let e=n.domElement.parentNode;e&&e.removeChild(n.domElement)}n=null,i=null,a=null},getInternal:()=>({arSource:n,arContext:i})}};Object.defineProperty(exports,`i`,{enumerable:!0,get:function(){return o}}),Object.defineProperty(exports,`n`,{enumerable:!0,get:function(){return c}}),Object.defineProperty(exports,`r`,{enumerable:!0,get:function(){return s}}),Object.defineProperty(exports,`t`,{enumerable:!0,get:function(){return l}});
|