@omnidotdev/rdk 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/build/assets/onnxWorker-B_g0sj2t.js +1 -0
  2. package/build/{fiducial-CqsG94R9.js → fiducial-Bi9lhbyN.js} +1 -1
  3. package/build/{fiducial-dapuWTXg.cjs → fiducial-DdfnVsvF.cjs} +1 -1
  4. package/build/fiducial.cjs +1 -1
  5. package/build/fiducial.d.ts +1 -1
  6. package/build/fiducial.js +2 -2
  7. package/build/geolocation/geolocationBackend.d.ts +1 -1
  8. package/build/geolocation/geolocationBackend.d.ts.map +1 -1
  9. package/build/{geolocation-ClkbtHic.cjs → geolocation-D_rCGA-C.cjs} +1 -1
  10. package/build/{geolocation-DxPzqQPN.js → geolocation-DqxybM8d.js} +47 -37
  11. package/build/geolocation.cjs +1 -1
  12. package/build/geolocation.d.ts +1 -1
  13. package/build/geolocation.js +2 -2
  14. package/build/{immersive-Dp2-ZCRC.js → immersive-BTwiipnT.js} +1 -1
  15. package/build/{immersive-8m33PLug.cjs → immersive-DKHCJlkN.cjs} +1 -1
  16. package/build/immersive.cjs +1 -1
  17. package/build/immersive.d.ts +1 -1
  18. package/build/immersive.js +2 -2
  19. package/build/index.cjs +1 -1
  20. package/build/index.d.ts +1 -0
  21. package/build/index.d.ts.map +1 -1
  22. package/build/index.js +10 -9
  23. package/build/lib/types/engine/backend/Backend.interface.d.ts +1 -0
  24. package/build/lib/types/engine/backend/Backend.interface.d.ts.map +1 -1
  25. package/build/{magic-DAQdGGEO.js → magic-CPIBm_8-.js} +1 -1
  26. package/build/{magic-DznuBYgG.cjs → magic-CWhj_WTi.cjs} +1 -1
  27. package/build/magic.cjs +1 -1
  28. package/build/magic.d.ts +1 -1
  29. package/build/magic.js +2 -2
  30. package/build/useXRStore-8Vl3wsP5.cjs +1 -0
  31. package/build/{useXRStore-B3-MaaCu.js → useXRStore-Dsm1C1fI.js} +2 -1
  32. package/build/vision/CameraBackground.d.ts +15 -0
  33. package/build/vision/CameraBackground.d.ts.map +1 -0
  34. package/build/vision/HandTracker.d.ts +23 -0
  35. package/build/vision/HandTracker.d.ts.map +1 -0
  36. package/build/vision/VisionAnchor.d.ts +28 -0
  37. package/build/vision/VisionAnchor.d.ts.map +1 -0
  38. package/build/vision/VisionDebug.d.ts +18 -0
  39. package/build/vision/VisionDebug.d.ts.map +1 -0
  40. package/build/vision/VisionOverlay.d.ts +28 -0
  41. package/build/vision/VisionOverlay.d.ts.map +1 -0
  42. package/build/vision/VisionSession.d.ts +18 -0
  43. package/build/vision/VisionSession.d.ts.map +1 -0
  44. package/build/vision/VisionStatus.d.ts +15 -0
  45. package/build/vision/VisionStatus.d.ts.map +1 -0
  46. package/build/vision/gestures/builtinGestures.d.ts +9 -0
  47. package/build/vision/gestures/builtinGestures.d.ts.map +1 -0
  48. package/build/vision/gestures/detectGesture.d.ts +9 -0
  49. package/build/vision/gestures/detectGesture.d.ts.map +1 -0
  50. package/build/vision/gestures/index.d.ts +4 -0
  51. package/build/vision/gestures/index.d.ts.map +1 -0
  52. package/build/vision/gestures/types.d.ts +7 -0
  53. package/build/vision/gestures/types.d.ts.map +1 -0
  54. package/build/vision/index.d.ts +27 -0
  55. package/build/vision/index.d.ts.map +1 -0
  56. package/build/vision/landmarkMapping.d.ts +25 -0
  57. package/build/vision/landmarkMapping.d.ts.map +1 -0
  58. package/build/vision/presets.d.ts +25 -0
  59. package/build/vision/presets.d.ts.map +1 -0
  60. package/build/vision/providers/createVisionProvider.d.ts +5 -0
  61. package/build/vision/providers/createVisionProvider.d.ts.map +1 -0
  62. package/build/vision/providers/index.d.ts +2 -0
  63. package/build/vision/providers/index.d.ts.map +1 -0
  64. package/build/vision/providers/mediapipeProvider.d.ts +23 -0
  65. package/build/vision/providers/mediapipeProvider.d.ts.map +1 -0
  66. package/build/vision/providers/onnxProvider.d.ts +30 -0
  67. package/build/vision/providers/onnxProvider.d.ts.map +1 -0
  68. package/build/vision/types.d.ts +92 -0
  69. package/build/vision/types.d.ts.map +1 -0
  70. package/build/vision/useVisionBackend.d.ts +8 -0
  71. package/build/vision/useVisionBackend.d.ts.map +1 -0
  72. package/build/vision/useVisionFrame.d.ts +13 -0
  73. package/build/vision/useVisionFrame.d.ts.map +1 -0
  74. package/build/vision/visionBackend.d.ts +17 -0
  75. package/build/vision/visionBackend.d.ts.map +1 -0
  76. package/build/vision/worker/decoders/index.d.ts +6 -0
  77. package/build/vision/worker/decoders/index.d.ts.map +1 -0
  78. package/build/vision/worker/decoders/nms.d.ts +23 -0
  79. package/build/vision/worker/decoders/nms.d.ts.map +1 -0
  80. package/build/vision/worker/decoders/rfDetr.d.ts +10 -0
  81. package/build/vision/worker/decoders/rfDetr.d.ts.map +1 -0
  82. package/build/vision/worker/decoders/types.d.ts +40 -0
  83. package/build/vision/worker/decoders/types.d.ts.map +1 -0
  84. package/build/vision/worker/decoders/yolo.d.ts +11 -0
  85. package/build/vision/worker/decoders/yolo.d.ts.map +1 -0
  86. package/build/vision/worker/onnxWorker.d.ts +2 -0
  87. package/build/vision/worker/onnxWorker.d.ts.map +1 -0
  88. package/build/vision/worker/preprocess.d.ts +22 -0
  89. package/build/vision/worker/preprocess.d.ts.map +1 -0
  90. package/build/vision/worker/types.d.ts +50 -0
  91. package/build/vision/worker/types.d.ts.map +1 -0
  92. package/build/vision-BKOfvNSR.js +836 -0
  93. package/build/vision-CRUCg7uc.cjs +1 -0
  94. package/build/vision.cjs +1 -0
  95. package/build/vision.d.ts +2 -0
  96. package/build/vision.js +3 -0
  97. package/package.json +23 -8
  98. package/build/useXRStore-7Gy83DEg.cjs +0 -1
@@ -0,0 +1 @@
1
+ const e=require(`./useXRStore-8Vl3wsP5.cjs`);let t=require(`react`),n=require(`@react-three/fiber`),r=require(`react/jsx-runtime`),i=require(`three`);var a=({constraints:e,onReady:n,onError:i,mirrored:a=!0})=>{let o=(0,t.useRef)(null),[s,c]=(0,t.useState)(!1),l=(0,t.useRef)(n);l.current=n;let u=(0,t.useRef)(i);return u.current=i,(0,t.useEffect)(()=>{let t=!1;return(async()=>{try{let n=await navigator.mediaDevices.getUserMedia({video:e||{width:{ideal:1280},height:{ideal:720},facingMode:`user`}});if(t||!o.current){for(let e of n.getTracks())e.stop();return}if(o.current.srcObject=n,await new Promise((e,t)=>{let n=o.current;if(!n){t(Error(`Video ref lost`));return}let r=()=>{n.removeEventListener(`canplay`,r),n.removeEventListener(`error`,i),e()},i=()=>{n.removeEventListener(`canplay`,r),n.removeEventListener(`error`,i),t(Error(`Video failed to load`))};n.addEventListener(`canplay`,r,{once:!0}),n.addEventListener(`error`,i,{once:!0})}),t)return;await o.current.play(),c(!0),l.current?.(o.current)}catch(e){if(t)return;let n=e instanceof Error?e:Error(`Failed to access camera`);console.error(`Camera error:`,n),u.current?.(n)}})(),()=>{if(t=!0,o.current?.srcObject){let e=o.current.srcObject;for(let t of e.getTracks())t.stop()}}},[e]),(0,r.jsx)(`video`,{ref:o,style:{position:`absolute`,inset:0,width:`100%`,height:`100%`,objectFit:`cover`,transform:a?`scaleX(-1)`:void 0,opacity:s?1:0,transition:`opacity 0.5s ease`},autoPlay:!0,playsInline:!0,muted:!0})},o=(e,t)=>e.y<t.y,s=e=>[Math.abs(e[4].x-e[9].x)>Math.abs(e[3].x-e[9].x),o(e[8],e[6]),o(e[12],e[10]),o(e[16],e[14]),o(e[20],e[18])],c=e=>{if(e.length<21)return null;let t=s(e);return t[0]&&!t[1]&&!t[2]&&!t[3]&&!t[4]?{gesture:`thumbs_up`,confidence:.9}:null},l=e=>{if(e.length<21)return null;let t=s(e);return!t[0]&&t[1]&&t[2]&&!t[3]&&!t[4]?{gesture:`peace`,confidence:.8}:null},u=e=>{if(e.length<21)return null;let t=s(e);return Math.sqrt((e[4].x-e[8].x)**2+(e[4].y-e[8].y)**2)<.05&&t[2]&&t[3]&&t[4]?{gesture:`ok`,confidence:.8}:null},d=e=>e.length<21?null:s(e).filter(Boolean).length===5?{gesture:`open_hand`,confidence:.7}:null,f=e=>e.length<21?null:s(e).filter(Boolean).length<=1?{gesture:`fist`,confidence:.7}:null,p=[c,l,u,d,f],m={gesture:`none`,confidence:0},h=(e,t=p)=>{let n=m;for(let r of t){let t=r(e);t&&t.confidence>n.confidence&&(n=t)}return n},g=(e,t,n,r=!0)=>{let i=t.width/t.height,a=n.width/n.height,o=a>i?a/i:1,s=a>i?1:i/a,c=.5+(e.x-.5)*o,l=.5+(e.y-.5)*s;return[(r?.5-c:c-.5)*t.width,(.5-l)*t.height,0]},_=e=>{let t=e.length||1,n=0,r=0,i=0;for(let t of e)n+=t.x,r+=t.y,i+=t.z??0;return{x:n/t,y:r/t,z:i/t}},v={isPending:!0,isSuccess:!1,provider:null,videoElement:null,onDetection:()=>()=>{}},y=()=>{let t=e.r(e=>e.backends).get(e.i.VISION);return t?{...t.getInternal(),isPending:!1,isSuccess:!0}:v},b=({onHandMove:e,onGesture:a,landmarkIndex:o=9,mirror:s=!0,children:c})=>{let{onDetection:l,isSuccess:u}=y(),{viewport:d}=(0,n.useThree)(),f=(0,t.useRef)(null),p=(0,t.useRef)(new i.Vector3),m=(0,t.useRef)(``);return(0,t.useEffect)(()=>{if(u)return l(e=>{f.current=e})},[u,l]),(0,n.useFrame)(()=>{let t=f.current;if(!t)return;let n=t.hands[0];if(!n||n.landmarks.length===0)return;let[r,i,c]=g(n.landmarks[o]??n.landmarks[0],d,t.frameSize,s);p.current.set(r,i,c),e?.(p.current);let l=h(n.landmarks);l.gesture!==m.current&&(m.current=l.gesture,a?.(l.gesture,l.confidence))}),(0,r.jsx)(`group`,{children:c})},x=`person.bicycle.car.motorcycle.airplane.bus.train.truck.boat.traffic light.fire hydrant.stop sign.parking meter.bench.bird.cat.dog.horse.sheep.cow.elephant.bear.zebra.giraffe.backpack.umbrella.handbag.tie.suitcase.frisbee.skis.snowboard.sports ball.kite.baseball bat.baseball glove.skateboard.surfboard.tennis racket.bottle.wine glass.cup.fork.knife.spoon.bowl.banana.apple.sandwich.orange.broccoli.carrot.hot dog.pizza.donut.cake.chair.couch.potted plant.bed.dining table.toilet.tv.laptop.mouse.remote.keyboard.cell phone.microwave.oven.toaster.sink.refrigerator.book.clock.vase.scissors.teddy bear.hair drier.toothbrush`.split(`.`),S=(e,t={})=>({name:`yolo`,inputSize:640,labels:[...x],...t,path:e,decoder:`yolo`}),C=(e,t={})=>({name:`rf-detr`,inputSize:560,labels:[...x],...t,path:e,decoder:`rfdetr`}),w=e=>{let{isSuccess:n,onDetection:r}=y(),[i,a]=(0,t.useState)(null),o=e?.throttle??16,s=(0,t.useRef)(0);return(0,t.useEffect)(()=>{if(!n){a(null);return}return r(e=>{let t=performance.now();t-s.current<o||(s.current=t,a(e))})},[n,r,o]),i},T=478,E=(e,t)=>t===`faces`?e.faces:t===`poses`?e.poses:e.hands,D=({task:e,index:a=0,debug:o=!1,mirror:s=!0,smoothing:c=.4,children:l})=>{let{onDetection:u,isSuccess:d}=y(),{viewport:f}=(0,n.useThree)(),p=(0,t.useRef)(null),m=(0,t.useRef)(null),h=(0,t.useRef)(null),v=(0,t.useMemo)(()=>new i.Object3D,[]),b=(0,t.useMemo)(()=>new i.Vector3,[]);return(0,t.useEffect)(()=>{m.current&&(m.current.visible=!1),h.current&&(h.current.count=0)},[]),(0,t.useEffect)(()=>{if(d)return u(e=>{p.current=e})},[d,u]),(0,n.useFrame)(()=>{let t=p.current,n=m.current;if(!t||!n)return;let r=E(t,e)[a],i=h.current;if(!r||r.landmarks.length===0){n.visible=!1,i&&(i.count=0);return}let l=n.visible;n.visible=!0;let[u,d,y]=g(_(r.landmarks),f,t.frameSize,s);if(b.set(u,d,y),!l||c>=1?n.position.copy(b):n.position.lerp(b,c),o&&i){let e=n.position,a=Math.min(r.landmarks.length,T);for(let n=0;n<a;n++){let[a,o,c]=g(r.landmarks[n],f,t.frameSize,s);v.position.set(a-e.x,o-e.y,c-e.z),v.updateMatrix(),i.setMatrixAt(n,v.matrix)}i.count=a,i.instanceMatrix.needsUpdate=!0}}),(0,r.jsxs)(`group`,{ref:m,children:[l,o&&(0,r.jsxs)(`instancedMesh`,{ref:h,args:[void 0,void 0,T],children:[(0,r.jsx)(`sphereGeometry`,{args:[.02,6,6]}),(0,r.jsx)(`meshBasicMaterial`,{color:`#00ff88`})]})]})},O=({tasks:e,width:t=320,height:n=240,style:i})=>{let a=w({throttle:50});return(0,r.jsxs)(`div`,{style:{background:`rgba(0, 0, 0, 0.8)`,color:`#00ff88`,padding:`8px`,borderRadius:`8px`,fontFamily:`monospace`,fontSize:`0.75rem`,width:t,height:n,overflow:`auto`,...i},children:[(0,r.jsxs)(`div`,{children:[`Frame: `,a?`active`:`none`]}),a&&(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)(`div`,{children:[`Size: `,a.frameSize.width,`x`,a.frameSize.height]}),e.includes(`hands`)&&(0,r.jsxs)(`div`,{children:[`Hands: `,a.hands.length]}),e.includes(`faces`)&&(0,r.jsxs)(`div`,{children:[`Faces: `,a.faces.length]}),e.includes(`poses`)&&(0,r.jsxs)(`div`,{children:[`Poses: `,a.poses.length]}),e.includes(`objects`)&&(0,r.jsxs)(`div`,{children:[`Objects: `,a.objects.length]})]})]})},k=class{type=`mediapipe`;models={};isInitialized=!1;isProcessing=!1;isDisposed=!1;videoElement=null;animationId=null;lastProcessTime=0;callbacks=[];options;constructor(e){this.options=e}async initialize(e){this.videoElement=e;let{FilesetResolver:t,HandLandmarker:n,FaceLandmarker:r,PoseLandmarker:i}=await import(`@mediapipe/tasks-vision`),a=await t.forVisionTasks(`https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10/wasm`),o=this.options.useGpu??!0?`GPU`:`CPU`,s=async e=>{if(o===`CPU`)return e(`CPU`);try{return await e(`GPU`)}catch{return console.warn(`MediaPipe GPU delegate failed, falling back to CPU`),e(`CPU`)}},c=new Set(this.options.tasks??[`hands`,`faces`,`poses`]),l=[`hands`,`faces`,`poses`].filter(e=>c.has(e)),u=0;c.has(`hands`)&&(this.options.onProgress?.({step:++u,total:l.length,label:`hand landmarker`}),this.models.handLandmarker=await s(e=>n.createFromOptions(a,{baseOptions:{modelAssetPath:`https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,delegate:e},runningMode:`VIDEO`,numHands:2,minHandDetectionConfidence:.7,minHandPresenceConfidence:.5,minTrackingConfidence:.5}))),c.has(`faces`)&&(this.options.onProgress?.({step:++u,total:l.length,label:`face landmarker`}),this.models.faceLandmarker=await s(e=>r.createFromOptions(a,{baseOptions:{modelAssetPath:`https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`,delegate:e},runningMode:`VIDEO`,numFaces:1,minFaceDetectionConfidence:.7,minFacePresenceConfidence:.5,minTrackingConfidence:.5,outputFaceBlendshapes:!1,outputFacialTransformationMatrixes:!1}))),c.has(`poses`)&&(this.options.onProgress?.({step:++u,total:l.length,label:`pose landmarker`}),this.models.poseLandmarker=await s(e=>i.createFromOptions(a,{baseOptions:{modelAssetPath:`https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task`,delegate:e},runningMode:`VIDEO`,numPoses:1,minPoseDetectionConfidence:.7,minPosePresenceConfidence:.5,minTrackingConfidence:.5,outputSegmentationMasks:this.options.segmentation??!1}))),this.isInitialized=!0,this.options.onProgress?.(null)}startDetection(){!this.isInitialized||!this.videoElement||this.isProcessing||(this.isProcessing=!0,this.processFrame())}stopDetection(){this.isProcessing=!1,this.animationId&&=(cancelAnimationFrame(this.animationId),null)}onDetection(e){return this.callbacks.push(e),()=>{let t=this.callbacks.indexOf(e);t>-1&&this.callbacks.splice(t,1)}}dispose(){this.isDisposed=!0,this.stopDetection(),this.models.handLandmarker&&this.models.handLandmarker.close(),this.models.faceLandmarker&&this.models.faceLandmarker.close(),this.models.poseLandmarker&&this.models.poseLandmarker.close(),this.models={},this.videoElement=null,this.callbacks=[],this.isInitialized=!1,this.isProcessing=!1}processFrame=()=>{if(!this.isProcessing||!this.videoElement||this.isDisposed)return;let e=performance.now(),t=this.options.throttle??16;if(e-this.lastProcessTime<t){this.animationId=requestAnimationFrame(this.processFrame);return}let{videoWidth:n,videoHeight:r}=this.videoElement;if(n===0||r===0){this.animationId=requestAnimationFrame(this.processFrame);return}this.lastProcessTime=e;let i=e;try{let e={hands:[],faces:[],poses:[],objects:[],timestamp:i,frameSize:{width:n,height:r}};if(this.models.handLandmarker){let t=this.models.handLandmarker.detectForVideo(this.videoElement,i);e.hands=t.landmarks.map((e,n)=>({landmarks:e.map(e=>({x:e.x,y:e.y,z:e.z||0})),confidence:t.handednesses?.[n]?.[0]?.score||.5}))}if(this.models.faceLandmarker&&(e.faces=this.models.faceLandmarker.detectForVideo(this.videoElement,i).faceLandmarks.map(e=>{let t=e.map(e=>e.x),n=e.map(e=>e.y),r=(Math.max(...t)-Math.min(...t))*(Math.max(...n)-Math.min(...n)),i=Math.min(.95,Math.max(.6,r*4));return{landmarks:e.map(e=>({x:e.x,y:e.y,z:e.z||0})),confidence:i}})),this.models.poseLandmarker){let t=this.models.poseLandmarker.detectForVideo(this.videoElement,i);e.poses=t.landmarks.map(e=>{let t=0,n=0;for(let r of e)typeof r.visibility==`number`&&(t+=r.visibility,n++);let r=n>0?t/n:.5;return{landmarks:e.map(e=>({x:e.x,y:e.y,z:e.z||0})),confidence:r}});let n=t.segmentationMasks?.[0];if(n){let t=new OffscreenCanvas(n.width,n.height).getContext(`2d`),r=n.getAsFloat32Array(),i=t.createImageData(n.width,n.height);for(let e=0;e<r.length;e++){let t=r[e]*255;i.data[e*4]=t,i.data[e*4+1]=t,i.data[e*4+2]=t,i.data[e*4+3]=255}e.segmentationMask=i}}this.handleResult(e)}catch(e){console.error(`Vision processing error:`,e)}this.animationId=requestAnimationFrame(this.processFrame)};handleResult(e){let t=this.options.minConfidence??.7,n=this.options.maxResults??2,r={...e,hands:e.hands.filter(e=>e.confidence>=t).slice(0,n),faces:e.faces.filter(e=>e.confidence>=t).slice(0,n),poses:e.poses.filter(e=>e.confidence>=t).slice(0,n),objects:[]};for(let e of this.callbacks)try{e(r)}catch(e){console.error(`Error in vision callback:`,e)}}},A=5e3,j=class{type=`onnx`;worker=null;isInitialized=!1;isDisposed=!1;isProcessing=!1;videoElement=null;animationId=null;loadedModels=new Map;lastProcessTime=0;processingStartTime=0;callbacks=[];options;messageHandler=null;pendingModelCallbacks=new Map;constructor(e){this.options=e}post(e,t){t?this.worker?.postMessage(e,t):this.worker?.postMessage(e)}async initialize(e){this.videoElement=e,this.worker=new Worker(new URL(`/assets/onnxWorker-B_g0sj2t.js`,``+{}.url),{type:`module`}),await new Promise((e,t)=>{let n=setTimeout(()=>{t(Error(`ONNX worker initialization timeout`))},1e4);this.messageHandler=r=>{let i=r.data;switch(i.type){case`initialized`:clearTimeout(n),this.isInitialized=!0,e();break;case`result`:this.isProcessing=!1,this.handleResult(i.result);break;case`modelLoaded`:{let e=this.pendingModelCallbacks.get(i.modelName);e&&(this.pendingModelCallbacks.delete(i.modelName),e.resolve());break}case`modelError`:{let e=this.pendingModelCallbacks.get(i.modelName);e&&(this.pendingModelCallbacks.delete(i.modelName),e.reject(Error(i.error)));break}case`error`:this.isProcessing=!1,this.isInitialized?console.error(`ONNX worker error:`,i.error):(clearTimeout(n),t(Error(i.error)));break}},this.worker?.addEventListener(`message`,this.messageHandler),this.post({type:`init`})});let t=this.options.onnx?.models??[];for(let e of t)await this.loadModel(e)}startDetection(){if(!this.isInitialized||!this.videoElement){console.warn(`ONNX provider not initialized`);return}let e=()=>{if(!this.videoElement||!this.isInitialized)return;let t=Date.now(),n=this.options.throttle??100;if(t-this.lastProcessTime<n){this.animationId=requestAnimationFrame(e);return}this.isProcessing&&this.processingStartTime>0&&t-this.processingStartTime>A&&(console.warn(`ONNX processing stall detected, resetting`),this.isProcessing=!1),this.isProcessing||this.processCurrentFrame(),this.animationId=requestAnimationFrame(e)};e()}stopDetection(){this.animationId&&=(cancelAnimationFrame(this.animationId),null),this.isProcessing=!1}onDetection(e){return this.callbacks.push(e),()=>{let t=this.callbacks.indexOf(e);t>-1&&this.callbacks.splice(t,1)}}dispose(){this.isDisposed=!0,this.stopDetection(),this.worker&&=(this.messageHandler&&=(this.worker.removeEventListener(`message`,this.messageHandler),null),this.post({type:`dispose`}),this.worker.terminate(),null),this.videoElement=null,this.callbacks.length=0,this.loadedModels.clear(),this.pendingModelCallbacks.clear(),this.isInitialized=!1}async loadModel(e){if(!this.worker)throw Error(`Worker not initialized`);let t=new Promise((t,n)=>{this.pendingModelCallbacks.set(e.name,{resolve:t,reject:n})});this.post({type:`loadModel`,model:e}),await t,this.loadedModels.set(e.name,e)}processCurrentFrame(){if(!this.videoElement||!this.worker||this.isProcessing)return;let e=Date.now();this.isProcessing=!0,this.lastProcessTime=e,this.processingStartTime=e;let{videoWidth:t,videoHeight:n}=this.videoElement;createImageBitmap(this.videoElement).then(e=>{if(this.isDisposed){e.close();return}this.post({type:`process`,imageBitmap:e,sourceWidth:t,sourceHeight:n,options:{minConfidence:this.options.minConfidence??.5,maxResults:this.options.maxResults??100}},[e])},e=>{this.isProcessing=!1,console.error(`Error creating ImageBitmap:`,e)})}handleResult(e){let t=this.options.minConfidence??.5,n=this.options.maxResults??100,r={hands:[],faces:[],poses:[],objects:e.detections.filter(e=>e.confidence>=t).sort((e,t)=>t.confidence-e.confidence).slice(0,n),timestamp:e.timestamp,frameSize:e.frameSize,processingTime:e.processingTime};for(let e of this.callbacks)try{e(r)}catch(e){console.error(`Error in ONNX detection callback:`,e)}}},M=e=>e.provider===`onnx`?new j(e):new k(e),N=async e=>{let t=document.createElement(`video`);t.style.display=`none`,document.body.appendChild(t);try{return t.srcObject=await navigator.mediaDevices.getUserMedia({video:{width:{ideal:1280,min:640},height:{ideal:720,min:480},facingMode:e.provider===`onnx`?`environment`:`user`,frameRate:{ideal:30,min:15}}}),t.playsInline=!0,t.muted=!0,await new Promise((e,n)=>{t.onloadedmetadata=()=>{t.play().then(e).catch(n)},t.onerror=n}),t}catch(e){throw document.body.removeChild(t),e}},P=e=>{if(!e)return;let t=e.srcObject;if(t)for(let e of t.getTracks())e.stop();e.parentNode&&e.parentNode.removeChild(e)},F=t=>{let n=null,r=null,i=!1,a=null,o=[],s=e=>(o.push(e),()=>{let t=o.indexOf(e);t>-1&&o.splice(t,1)});return{type:e.i.VISION,async init(e){t.videoElement?(r=t.videoElement,i=!1):(r=await N(t),i=!0),n=M(t),await n.initialize(r),a=n.onDetection(e=>{for(let t of o)try{t(e)}catch(e){console.error(`Error in vision detection callback:`,e)}}),n.startDetection()},update(e){},dispose(){a?.(),a=null,n?.dispose(),i&&P(r),r=null,n=null,o.length=0},getInternal:()=>({provider:n,videoElement:r,onDetection:s})}},I=({options:i,children:a,onError:o})=>{let{scene:s,camera:c,gl:l}=(0,n.useThree)(),{registerBackend:u,unregisterBackend:d}=e.r(),f=(0,t.useRef)(null),p=(0,t.useRef)(i);p.current=i;let m=(0,t.useRef)(o);return m.current=o,(0,t.useEffect)(()=>{if(f.current)return;let e=!1;return(async()=>{try{let t=F(p.current);if(e)return;if(await u(t,{scene:s,camera:c,renderer:l}),e){d(t);return}f.current=t}catch(e){let t=e instanceof Error?e:Error(String(e));console.error(`[VisionSession] Failed to initialize:`,t),m.current?.(t)}})(),()=>{e=!0,f.current&&=(d(f.current),null)}},[s,c,l,u,d]),(0,r.jsx)(r.Fragment,{children:a})},L=({tasks:e,progress:t,showDebug:n,onToggleDebug:i})=>{let a=w({throttle:100}),o=e=>{if(!a)return!1;switch(e){case`hands`:return a.hands.length>0;case`faces`:return a.faces.length>0;case`poses`:return a.poses.length>0;case`objects`:return a.objects.length>0;default:return!1}},s=e.some(o),c=a!==null;return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)(`div`,{style:{position:`fixed`,top:`20px`,left:`20px`,background:`rgba(0, 0, 0, 0.7)`,color:`white`,padding:`12px`,borderRadius:`8px`,fontFamily:`system-ui, sans-serif`,fontSize:`0.9rem`,zIndex:1e3},children:[(0,r.jsxs)(`div`,{style:{display:`flex`,alignItems:`center`,gap:`8px`,marginBottom:`8px`},children:[(0,r.jsx)(`div`,{style:{width:`8px`,height:`8px`,borderRadius:`50%`,background:c?`#00ff88`:`#ffaa00`}}),(0,r.jsx)(`span`,{children:s?`Active`:c?`Ready`:t?`Loading models (${t.step}/${t.total}) ${t.label}...`:`Starting...`})]}),e.map(e=>(0,r.jsxs)(`div`,{style:{fontSize:`0.8rem`,opacity:.8,display:`flex`,justifyContent:`space-between`,gap:`12px`},children:[(0,r.jsx)(`span`,{children:e}),(0,r.jsx)(`span`,{children:o(e)?`active`:c?`idle`:`...`})]},e))]}),(0,r.jsxs)(`button`,{onClick:i,type:`button`,style:{position:`fixed`,top:`20px`,right:`20px`,background:`rgba(0, 0, 0, 0.7)`,color:`white`,border:`1px solid #333`,padding:`8px 12px`,borderRadius:`6px`,cursor:`pointer`,fontFamily:`system-ui, sans-serif`,fontSize:`0.8rem`,zIndex:1e3},children:[n?`Hide`:`Show`,` Debug`]})]})},R={},z=({tasks:e=[`hands`],options:i,children:o,showDebug:s=!1,cameraConstraints:c,canvasStyle:l=R,onReady:u,onError:d})=>{let[f,p]=(0,t.useState)(s),[m,h]=(0,t.useState)(null),[g,_]=(0,t.useState)(null),v=(0,t.useRef)(u);v.current=u;let y=(0,t.useRef)(_);y.current=_;let b=(0,t.useCallback)(e=>y.current(e),[]),x=(0,t.useCallback)(e=>{h(e),v.current?.()},[]),S=(0,t.useMemo)(()=>({tasks:e,minConfidence:.6,maxResults:2,useGpu:!0,...i,videoElement:m??void 0,onProgress:b}),[e,i,m,b]);return(0,r.jsxs)(`div`,{style:{position:`relative`,width:`100%`,height:`100%`},children:[(0,r.jsx)(a,{constraints:c,onReady:x,onError:d}),m&&(0,r.jsx)(n.Canvas,{style:{position:`absolute`,inset:0,pointerEvents:`none`,...l},camera:{position:[0,0,5],fov:75},gl:{alpha:!0,premultipliedAlpha:!1},children:(0,r.jsx)(B,{options:S,onError:d,children:o})}),(0,r.jsx)(L,{tasks:e,progress:g,showDebug:f,onToggleDebug:()=>p(e=>!e)}),f&&(0,r.jsx)(O,{tasks:e,width:320,height:240,style:{position:`fixed`,bottom:`80px`,right:`20px`,maxWidth:`calc(100vw - 40px)`,overflow:`hidden`}})]})},B=({options:e,onError:t,children:n})=>(0,r.jsxs)(I,{options:e,onError:t,children:[(0,r.jsx)(`ambientLight`,{intensity:.6}),(0,r.jsx)(`directionalLight`,{position:[10,10,10],intensity:.8}),n]});Object.defineProperty(exports,`S`,{enumerable:!0,get:function(){return a}}),Object.defineProperty(exports,`_`,{enumerable:!0,get:function(){return f}}),Object.defineProperty(exports,`a`,{enumerable:!0,get:function(){return O}}),Object.defineProperty(exports,`b`,{enumerable:!0,get:function(){return l}}),Object.defineProperty(exports,`c`,{enumerable:!0,get:function(){return x}}),Object.defineProperty(exports,`d`,{enumerable:!0,get:function(){return b}}),Object.defineProperty(exports,`f`,{enumerable:!0,get:function(){return y}}),Object.defineProperty(exports,`g`,{enumerable:!0,get:function(){return p}}),Object.defineProperty(exports,`h`,{enumerable:!0,get:function(){return h}}),Object.defineProperty(exports,`i`,{enumerable:!0,get:function(){return F}}),Object.defineProperty(exports,`l`,{enumerable:!0,get:function(){return C}}),Object.defineProperty(exports,`m`,{enumerable:!0,get:function(){return _}}),Object.defineProperty(exports,`n`,{enumerable:!0,get:function(){return L}}),Object.defineProperty(exports,`o`,{enumerable:!0,get:function(){return D}}),Object.defineProperty(exports,`p`,{enumerable:!0,get:function(){return g}}),Object.defineProperty(exports,`r`,{enumerable:!0,get:function(){return I}}),Object.defineProperty(exports,`s`,{enumerable:!0,get:function(){return w}}),Object.defineProperty(exports,`t`,{enumerable:!0,get:function(){return z}}),Object.defineProperty(exports,`u`,{enumerable:!0,get:function(){return S}}),Object.defineProperty(exports,`v`,{enumerable:!0,get:function(){return u}}),Object.defineProperty(exports,`x`,{enumerable:!0,get:function(){return c}}),Object.defineProperty(exports,`y`,{enumerable:!0,get:function(){return d}});
@@ -0,0 +1 @@
1
+ Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`}),require(`./useXRStore-8Vl3wsP5.cjs`);const e=require(`./vision-CRUCg7uc.cjs`);exports.COCO_LABELS=e.c,exports.CameraBackground=e.S,exports.DEFAULT_GESTURES=e.g,exports.HandTracker=e.d,exports.VisionAnchor=e.o,exports.VisionDebug=e.a,exports.VisionOverlay=e.t,exports.VisionSession=e.r,exports.VisionStatus=e.n,exports.createVisionBackend=e.i,exports.detectGesture=e.h,exports.fist=e._,exports.landmarkToWorld=e.p,exports.landmarksCentroid=e.m,exports.ok=e.v,exports.openHand=e.y,exports.peace=e.b,exports.rfDetr=e.l,exports.thumbsUp=e.x,exports.useVisionBackend=e.f,exports.useVisionFrame=e.s,exports.yolo=e.u;
@@ -0,0 +1,2 @@
1
+ export * from './vision/index.js'
2
+ export {}
@@ -0,0 +1,3 @@
1
+ import "./useXRStore-Dsm1C1fI.js";
2
+ import { S as e, _ as t, a as n, b as r, c as i, d as a, f as o, g as s, h as c, i as l, l as u, m as d, n as f, o as p, p as m, r as h, s as g, t as _, u as v, v as y, x as b, y as x } from "./vision-BKOfvNSR.js";
3
+ export { i as COCO_LABELS, e as CameraBackground, s as DEFAULT_GESTURES, a as HandTracker, p as VisionAnchor, n as VisionDebug, _ as VisionOverlay, h as VisionSession, f as VisionStatus, l as createVisionBackend, c as detectGesture, t as fist, m as landmarkToWorld, d as landmarksCentroid, y as ok, x as openHand, r as peace, u as rfDetr, b as thumbsUp, o as useVisionBackend, g as useVisionFrame, v as yolo };
package/package.json CHANGED
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "name": "@omnidotdev/rdk",
3
3
  "description": "Reality Development Kit, a unified React-first XR engine",
4
- "version": "0.10.0",
5
- "license": "MIT",
4
+ "version": "0.11.0",
5
+ "license": "Apache-2.0",
6
6
  "type": "module",
7
7
  "main": "build/index.js",
8
8
  "module": "build/index.js",
@@ -54,6 +54,11 @@
54
54
  "types": "./build/magic.d.ts",
55
55
  "import": "./build/magic.js",
56
56
  "require": "./build/magic.cjs"
57
+ },
58
+ "./vision": {
59
+ "types": "./build/vision.d.ts",
60
+ "import": "./build/vision.js",
61
+ "require": "./build/vision.cjs"
57
62
  }
58
63
  },
59
64
  "scripts": {
@@ -68,42 +73,52 @@
68
73
  },
69
74
  "peerDependencies": {
70
75
  "@ar-js-org/ar.js": "^3.4.7",
76
+ "@mediapipe/tasks-vision": "^0.10.0",
71
77
  "@react-three/fiber": "^9.5.0",
72
78
  "@react-three/xr": "^6.6.29",
73
- "locar": "^0.1.8",
79
+ "locar": "^0.2.3",
80
+ "onnxruntime-web": "^1.22.0",
74
81
  "react": "^19.2.0",
75
82
  "react-dom": "^19.2.0",
76
- "three": "^0.183.0"
83
+ "three": "^0.185.0"
77
84
  },
78
85
  "peerDependenciesMeta": {
79
86
  "@ar-js-org/ar.js": {
80
87
  "optional": true
81
88
  },
89
+ "@mediapipe/tasks-vision": {
90
+ "optional": true
91
+ },
82
92
  "@react-three/xr": {
83
93
  "optional": true
84
94
  },
85
95
  "locar": {
86
96
  "optional": true
97
+ },
98
+ "onnxruntime-web": {
99
+ "optional": true
87
100
  }
88
101
  },
89
102
  "devDependencies": {
90
103
  "@ar-js-org/ar.js": "^3.4.7",
91
104
  "@biomejs/biome": "^2.4.8",
105
+ "@mediapipe/tasks-vision": "^0.10.35",
92
106
  "@react-three/fiber": "^9.5.0",
93
107
  "@testing-library/jest-dom": "^6.9.1",
94
108
  "@testing-library/react": "^16.3.1",
95
109
  "@types/node": "^25.0.3",
96
110
  "@types/react": "^19.2.7",
97
111
  "@types/react-dom": "^19.2.3",
98
- "@types/three": "^0.183.0",
112
+ "@types/three": "^0.185.0",
99
113
  "jsdom": "^29.0.0",
100
- "locar": "^0.1.8",
114
+ "locar": "^0.2.3",
115
+ "onnxruntime-web": "^1.27.0",
101
116
  "react": "^19.2.3",
102
117
  "react-dom": "^19.2.3",
103
- "three": "^0.183.0",
118
+ "three": "^0.185.0",
104
119
  "typescript": "^5.9.3",
105
120
  "vite": "^8.0.0",
106
- "vite-plugin-dts": "^4.5.4",
121
+ "vite-plugin-dts": "^5.0.0",
107
122
  "vite-tsconfig-paths": "^6.0.3",
108
123
  "vitest": "^4.0.16"
109
124
  },
@@ -1 +0,0 @@
1
- var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},s=(n,r,a)=>(a=n==null?{}:e(i(n)),o(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n));let c=require(`react`);c=s(c);var l={FIDUCIAL:`fiducial`,GEOLOCATION:`geolocation`,IMMERSIVE:`immersive`,MAGIC:`magic`},u=e=>{let t,n=new Set,r=(e,r)=>{let i=typeof e==`function`?e(t):e;if(!Object.is(i,t)){let e=t;t=r??(typeof i!=`object`||!i)?i:Object.assign({},t,i),n.forEach(n=>n(t,e))}},i=()=>t,a={setState:r,getState:i,getInitialState:()=>o,subscribe:e=>(n.add(e),()=>n.delete(e))},o=t=e(r,i,a);return a},d=(e=>e?u(e):u),f=e=>e;function p(e,t=f){let n=c.default.useSyncExternalStore(e.subscribe,c.default.useCallback(()=>t(e.getState()),[e,t]),c.default.useCallback(()=>t(e.getInitialState()),[e,t]));return c.default.useDebugValue(n),n}var m=e=>{let t=d(e),n=e=>p(t,e);return Object.assign(n,t),n},h=(e=>e?m(e):m),g=e=>(t,n,r)=>{let i=r.subscribe;return r.subscribe=((e,t,n)=>{let a=e;if(t){let i=n?.equalityFn||Object.is,o=e(r.getState());a=n=>{let r=e(n);if(!i(o,r)){let e=o;t(o=r,e)}},n?.fireImmediately&&t(o,o)}return i(a)}),e(t,n,r)},_=e=>Symbol.iterator in e,v=e=>`entries`in e,y=(e,t)=>{let n=e instanceof Map?e:new Map(e.entries()),r=t instanceof Map?t:new Map(t.entries());if(n.size!==r.size)return!1;for(let[e,t]of n)if(!r.has(e)||!Object.is(t,r.get(e)))return!1;return!0},b=(e,t)=>{let n=e[Symbol.iterator](),r=t[Symbol.iterator](),i=n.next(),a=r.next();for(;!i.done&&!a.done;){if(!Object.is(i.value,a.value))return!1;i=n.next(),a=r.next()}return!!i.done&&!!a.done};function x(e,t){return Object.is(e,t)?!0:typeof e!=`object`||!e||typeof t!=`object`||!t||Object.getPrototypeOf(e)!==Object.getPrototypeOf(t)?!1:_(e)&&_(t)?v(e)&&v(t)?y(e,t):b(e,t):y({entries:()=>Object.entries(e)},{entries:()=>Object.entries(t)})}function S(e){let t=c.default.useRef(void 0);return n=>{let r=e(n);return x(t.current,r)?t.current:t.current=r}}var C=h()(g((e,t)=>({video:null,backends:new Map,immersiveStore:null,registerBackend:async(n,r)=>{try{let i=t(),a=new Set(i.backends.keys()),o=a.has(l.FIDUCIAL)||n.type===l.FIDUCIAL,s=a.has(l.GEOLOCATION)||n.type===l.GEOLOCATION;if(o&&s){let e=`❌ [RDK] INCOMPATIBLE SESSIONS: fiducial and geolocation backends cannot be used together due to camera/video conflicts between AR.js and LocAR.js libraries. Use only one session type per app.`;throw console.error(e),Error(e)}await n.init({scene:r.scene,camera:r.camera,renderer:r.renderer}),e(e=>{let t=new Map(e.backends);return t.set(n.type,n),{backends:t}})}catch(e){throw console.error(`[XRStore] Failed to register backend:`,e),e}},unregisterBackend:t=>{e(e=>{let n=new Map(e.backends);n.delete(t.type);try{t.dispose?.()}catch(e){console.error(`[XRStore] Error disposing backend:`,e)}return{backends:n}})},setVideo:t=>{e({video:t})},updateBackends:e=>{let{backends:n}=t();for(let t of n.values())try{t.update?.(e)}catch(e){console.error(`[XRStore] Backend update error:`,e)}},setImmersiveStore:t=>{e({immersiveStore:t})}}))),w=()=>C.getState(),T=C.subscribe,E=e=>C(S(t=>{let n={...t,isImmersive:t.backends.has(l.IMMERSIVE),immersive:t.immersiveStore||null};return e?e(n):n}));Object.defineProperty(exports,`a`,{enumerable:!0,get:function(){return s}}),Object.defineProperty(exports,`i`,{enumerable:!0,get:function(){return l}}),Object.defineProperty(exports,`n`,{enumerable:!0,get:function(){return T}}),Object.defineProperty(exports,`r`,{enumerable:!0,get:function(){return E}}),Object.defineProperty(exports,`t`,{enumerable:!0,get:function(){return w}});