@omnidotdev/rdk 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/assets/onnxWorker-B_g0sj2t.js +1 -0
- package/build/{fiducial-CqsG94R9.js → fiducial-Bi9lhbyN.js} +1 -1
- package/build/{fiducial-dapuWTXg.cjs → fiducial-DdfnVsvF.cjs} +1 -1
- package/build/fiducial.cjs +1 -1
- package/build/fiducial.d.ts +1 -1
- package/build/fiducial.js +2 -2
- package/build/geolocation/geolocationBackend.d.ts +1 -1
- package/build/geolocation/geolocationBackend.d.ts.map +1 -1
- package/build/{geolocation-ClkbtHic.cjs → geolocation-D_rCGA-C.cjs} +1 -1
- package/build/{geolocation-DxPzqQPN.js → geolocation-DqxybM8d.js} +47 -37
- package/build/geolocation.cjs +1 -1
- package/build/geolocation.d.ts +1 -1
- package/build/geolocation.js +2 -2
- package/build/{immersive-Dp2-ZCRC.js → immersive-BTwiipnT.js} +1 -1
- package/build/{immersive-8m33PLug.cjs → immersive-DKHCJlkN.cjs} +1 -1
- package/build/immersive.cjs +1 -1
- package/build/immersive.d.ts +1 -1
- package/build/immersive.js +2 -2
- package/build/index.cjs +1 -1
- package/build/index.d.ts +1 -0
- package/build/index.d.ts.map +1 -1
- package/build/index.js +10 -9
- package/build/lib/types/engine/backend/Backend.interface.d.ts +1 -0
- package/build/lib/types/engine/backend/Backend.interface.d.ts.map +1 -1
- package/build/{magic-DAQdGGEO.js → magic-CPIBm_8-.js} +1 -1
- package/build/{magic-DznuBYgG.cjs → magic-CWhj_WTi.cjs} +1 -1
- package/build/magic.cjs +1 -1
- package/build/magic.d.ts +1 -1
- package/build/magic.js +2 -2
- package/build/useXRStore-8Vl3wsP5.cjs +1 -0
- package/build/{useXRStore-B3-MaaCu.js → useXRStore-Dsm1C1fI.js} +2 -1
- package/build/vision/CameraBackground.d.ts +15 -0
- package/build/vision/CameraBackground.d.ts.map +1 -0
- package/build/vision/HandTracker.d.ts +23 -0
- package/build/vision/HandTracker.d.ts.map +1 -0
- package/build/vision/VisionAnchor.d.ts +28 -0
- package/build/vision/VisionAnchor.d.ts.map +1 -0
- package/build/vision/VisionDebug.d.ts +18 -0
- package/build/vision/VisionDebug.d.ts.map +1 -0
- package/build/vision/VisionOverlay.d.ts +28 -0
- package/build/vision/VisionOverlay.d.ts.map +1 -0
- package/build/vision/VisionSession.d.ts +18 -0
- package/build/vision/VisionSession.d.ts.map +1 -0
- package/build/vision/VisionStatus.d.ts +15 -0
- package/build/vision/VisionStatus.d.ts.map +1 -0
- package/build/vision/gestures/builtinGestures.d.ts +9 -0
- package/build/vision/gestures/builtinGestures.d.ts.map +1 -0
- package/build/vision/gestures/detectGesture.d.ts +9 -0
- package/build/vision/gestures/detectGesture.d.ts.map +1 -0
- package/build/vision/gestures/index.d.ts +4 -0
- package/build/vision/gestures/index.d.ts.map +1 -0
- package/build/vision/gestures/types.d.ts +7 -0
- package/build/vision/gestures/types.d.ts.map +1 -0
- package/build/vision/index.d.ts +27 -0
- package/build/vision/index.d.ts.map +1 -0
- package/build/vision/landmarkMapping.d.ts +25 -0
- package/build/vision/landmarkMapping.d.ts.map +1 -0
- package/build/vision/presets.d.ts +25 -0
- package/build/vision/presets.d.ts.map +1 -0
- package/build/vision/providers/createVisionProvider.d.ts +5 -0
- package/build/vision/providers/createVisionProvider.d.ts.map +1 -0
- package/build/vision/providers/index.d.ts +2 -0
- package/build/vision/providers/index.d.ts.map +1 -0
- package/build/vision/providers/mediapipeProvider.d.ts +23 -0
- package/build/vision/providers/mediapipeProvider.d.ts.map +1 -0
- package/build/vision/providers/onnxProvider.d.ts +30 -0
- package/build/vision/providers/onnxProvider.d.ts.map +1 -0
- package/build/vision/types.d.ts +92 -0
- package/build/vision/types.d.ts.map +1 -0
- package/build/vision/useVisionBackend.d.ts +8 -0
- package/build/vision/useVisionBackend.d.ts.map +1 -0
- package/build/vision/useVisionFrame.d.ts +13 -0
- package/build/vision/useVisionFrame.d.ts.map +1 -0
- package/build/vision/visionBackend.d.ts +17 -0
- package/build/vision/visionBackend.d.ts.map +1 -0
- package/build/vision/worker/decoders/index.d.ts +6 -0
- package/build/vision/worker/decoders/index.d.ts.map +1 -0
- package/build/vision/worker/decoders/nms.d.ts +23 -0
- package/build/vision/worker/decoders/nms.d.ts.map +1 -0
- package/build/vision/worker/decoders/rfDetr.d.ts +10 -0
- package/build/vision/worker/decoders/rfDetr.d.ts.map +1 -0
- package/build/vision/worker/decoders/types.d.ts +40 -0
- package/build/vision/worker/decoders/types.d.ts.map +1 -0
- package/build/vision/worker/decoders/yolo.d.ts +11 -0
- package/build/vision/worker/decoders/yolo.d.ts.map +1 -0
- package/build/vision/worker/onnxWorker.d.ts +2 -0
- package/build/vision/worker/onnxWorker.d.ts.map +1 -0
- package/build/vision/worker/preprocess.d.ts +22 -0
- package/build/vision/worker/preprocess.d.ts.map +1 -0
- package/build/vision/worker/types.d.ts +50 -0
- package/build/vision/worker/types.d.ts.map +1 -0
- package/build/vision-BKOfvNSR.js +836 -0
- package/build/vision-CRUCg7uc.cjs +1 -0
- package/build/vision.cjs +1 -0
- package/build/vision.d.ts +2 -0
- package/build/vision.js +3 -0
- package/package.json +23 -8
- package/build/useXRStore-7Gy83DEg.cjs +0 -1
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { default as React } from 'react';
|
|
2
|
+
import { VisionSessionOptions } from './types';
|
|
3
|
+
export type VisionSessionProps = {
|
|
4
|
+
/** Vision session configuration */
|
|
5
|
+
options: VisionSessionOptions;
|
|
6
|
+
/** Children to render inside the session */
|
|
7
|
+
children?: React.ReactNode;
|
|
8
|
+
/** Error callback */
|
|
9
|
+
onError?: (error: Error) => void;
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* Vision session component that creates and registers the vision backend
|
|
13
|
+
* with the XR store, providing detection context to child components.
|
|
14
|
+
* Must render inside a `@react-three/fiber` Canvas (uses `useThree`).
|
|
15
|
+
*/
|
|
16
|
+
declare const VisionSession: React.FC<VisionSessionProps>;
|
|
17
|
+
export default VisionSession;
|
|
18
|
+
//# sourceMappingURL=VisionSession.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"VisionSession.d.ts","sourceRoot":"","sources":["../../src/vision/VisionSession.tsx"],"names":[],"mappings":"AAOA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAEpD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,mCAAmC;IACnC,OAAO,EAAE,oBAAoB,CAAC;IAC9B,4CAA4C;IAC5C,QAAQ,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC;IAC3B,qBAAqB;IACrB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAClC,CAAC;AAEF;;;;GAIG;AACH,QAAA,MAAM,aAAa,EAAE,KAAK,CAAC,EAAE,CAAC,kBAAkB,CAyD/C,CAAC;AAEF,eAAe,aAAa,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { default as React } from 'react';
|
|
2
|
+
import { VisionTask } from './types';
|
|
3
|
+
import { VisionProgress } from './worker/types';
|
|
4
|
+
export type VisionStatusProps = {
|
|
5
|
+
tasks: VisionTask[];
|
|
6
|
+
progress?: VisionProgress | null;
|
|
7
|
+
showDebug: boolean;
|
|
8
|
+
onToggleDebug: () => void;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Vision status overlay showing detection state for each task
|
|
12
|
+
*/
|
|
13
|
+
declare const VisionStatus: React.FC<VisionStatusProps>;
|
|
14
|
+
export default VisionStatus;
|
|
15
|
+
//# sourceMappingURL=VisionStatus.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"VisionStatus.d.ts","sourceRoot":"","sources":["../../src/vision/VisionStatus.tsx"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAErD,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,QAAQ,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IACjC,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,IAAI,CAAC;CAC3B,CAAC;AAEF;;GAEG;AACH,QAAA,MAAM,YAAY,EAAE,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAkH7C,CAAC;AAEF,eAAe,YAAY,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { GestureDetector } from './types';
|
|
2
|
+
declare const thumbsUp: GestureDetector;
|
|
3
|
+
declare const peace: GestureDetector;
|
|
4
|
+
declare const ok: GestureDetector;
|
|
5
|
+
declare const openHand: GestureDetector;
|
|
6
|
+
declare const fist: GestureDetector;
|
|
7
|
+
declare const DEFAULT_GESTURES: GestureDetector[];
|
|
8
|
+
export { DEFAULT_GESTURES, fist, ok, openHand, peace, thumbsUp };
|
|
9
|
+
//# sourceMappingURL=builtinGestures.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"builtinGestures.d.ts","sourceRoot":"","sources":["../../../src/vision/gestures/builtinGestures.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAiB,MAAM,SAAS,CAAC;AAiB9D,QAAA,MAAM,QAAQ,EAAE,eASf,CAAC;AAEF,QAAA,MAAM,KAAK,EAAE,eASZ,CAAC;AAEF,QAAA,MAAM,EAAE,EAAE,eAeT,CAAC;AAEF,QAAA,MAAM,QAAQ,EAAE,eAUf,CAAC;AAEF,QAAA,MAAM,IAAI,EAAE,eAUX,CAAC;AAEF,QAAA,MAAM,gBAAgB,EAAE,eAAe,EAMtC,CAAC;AAEF,OAAO,EAAE,gBAAgB,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { VisionLandmark } from '../types';
|
|
2
|
+
import { GestureDetector, GestureResult } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* Detect gestures from hand landmarks using the provided detectors.
|
|
5
|
+
* Returns the highest-confidence match, or `{ gesture: "none", confidence: 0 }`.
|
|
6
|
+
*/
|
|
7
|
+
declare const detectGesture: (landmarks: VisionLandmark[], detectors?: GestureDetector[]) => GestureResult;
|
|
8
|
+
export default detectGesture;
|
|
9
|
+
//# sourceMappingURL=detectGesture.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"detectGesture.d.ts","sourceRoot":"","sources":["../../../src/vision/gestures/detectGesture.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC/C,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAI9D;;;GAGG;AACH,QAAA,MAAM,aAAa,GACjB,WAAW,cAAc,EAAE,EAC3B,YAAW,eAAe,EAAqB,KAC9C,aAWF,CAAC;AAEF,eAAe,aAAa,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/vision/gestures/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,IAAI,EACJ,EAAE,EACF,QAAQ,EACR,KAAK,EACL,QAAQ,GACT,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAE3D,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/vision/gestures/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/C,MAAM,MAAM,aAAa,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpE,MAAM,MAAM,eAAe,GAAG,CAC5B,SAAS,EAAE,cAAc,EAAE,KACxB,aAAa,GAAG,IAAI,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export { default as CameraBackground } from './CameraBackground';
|
|
2
|
+
export { DEFAULT_GESTURES, detectGesture, fist, ok, openHand, peace, thumbsUp, } from './gestures';
|
|
3
|
+
export { default as HandTracker } from './HandTracker';
|
|
4
|
+
export { landmarksCentroid, landmarkToWorld } from './landmarkMapping';
|
|
5
|
+
export { COCO_LABELS, rfDetr, yolo } from './presets';
|
|
6
|
+
export { default as useVisionBackend } from './useVisionBackend';
|
|
7
|
+
export { default as useVisionFrame } from './useVisionFrame';
|
|
8
|
+
export { default as VisionAnchor } from './VisionAnchor';
|
|
9
|
+
export { default as VisionDebug } from './VisionDebug';
|
|
10
|
+
export { default as VisionOverlay } from './VisionOverlay';
|
|
11
|
+
export { default as VisionSession } from './VisionSession';
|
|
12
|
+
export { default as VisionStatus } from './VisionStatus';
|
|
13
|
+
export { default as createVisionBackend } from './visionBackend';
|
|
14
|
+
export type { CameraBackgroundProps } from './CameraBackground';
|
|
15
|
+
export type { GestureDetector, GestureResult } from './gestures';
|
|
16
|
+
export type { HandTrackerProps } from './HandTracker';
|
|
17
|
+
export type { FrameSize, Viewport } from './landmarkMapping';
|
|
18
|
+
export type { PresetOverrides } from './presets';
|
|
19
|
+
export type { LandmarkDetection, ObjectDetection, ONNXDecoderName, ONNXModelConfig, VisionFrame, VisionLandmark, VisionProgress, VisionProvider, VisionSessionOptions, VisionTask, } from './types';
|
|
20
|
+
export type { UseVisionFrameOptions } from './useVisionFrame';
|
|
21
|
+
export type { VisionAnchorProps } from './VisionAnchor';
|
|
22
|
+
export type { VisionDebugProps } from './VisionDebug';
|
|
23
|
+
export type { VisionOverlayProps } from './VisionOverlay';
|
|
24
|
+
export type { VisionSessionProps } from './VisionSession';
|
|
25
|
+
export type { VisionStatusProps } from './VisionStatus';
|
|
26
|
+
export type { VisionBackendState, VisionInternal, } from './visionBackend';
|
|
27
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/vision/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,IAAI,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EACL,gBAAgB,EAChB,aAAa,EACb,IAAI,EACJ,EAAE,EACF,QAAQ,EACR,KAAK,EACL,QAAQ,GACT,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtD,OAAO,EAAE,OAAO,IAAI,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,OAAO,IAAI,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,OAAO,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACzD,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,OAAO,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACzD,OAAO,EAAE,OAAO,IAAI,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEjE,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAChE,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AACjE,YAAY,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACtD,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7D,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,YAAY,EACV,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,eAAe,EACf,WAAW,EACX,cAAc,EACd,cAAc,EACd,cAAc,EACd,oBAAoB,EACpB,UAAU,GACX,MAAM,SAAS,CAAC;AACjB,YAAY,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAC9D,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACtD,YAAY,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,YAAY,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACxD,YAAY,EACV,kBAAkB,EAClB,cAAc,GACf,MAAM,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { VisionLandmark } from './types';
|
|
2
|
+
/** World-space dimensions of the camera focal plane (from R3F `useThree().viewport`) */
|
|
3
|
+
export type Viewport = {
|
|
4
|
+
width: number;
|
|
5
|
+
height: number;
|
|
6
|
+
};
|
|
7
|
+
/** Pixel dimensions of the source video frame (from `VisionFrame.frameSize`) */
|
|
8
|
+
export type FrameSize = {
|
|
9
|
+
width: number;
|
|
10
|
+
height: number;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* Map a normalized MediaPipe landmark (x, y in [0, 1] with y top-down) into R3F
|
|
14
|
+
* world coordinates on the camera focal plane (z = 0), accounting for the
|
|
15
|
+
* `object-fit: cover` center-crop used to display the camera feed. `mirror`
|
|
16
|
+
* flips x to match a mirrored (selfie) preview.
|
|
17
|
+
*
|
|
18
|
+
* The camera feed is displayed with `cover`, so the axis whose aspect overflows
|
|
19
|
+
* the canvas is cropped; landmarks are normalized to the full frame, so that
|
|
20
|
+
* crop must be undone or overlays drift toward the edges (badly on portrait).
|
|
21
|
+
*/
|
|
22
|
+
export declare const landmarkToWorld: (landmark: VisionLandmark, viewport: Viewport, frameSize: FrameSize, mirror?: boolean) => [number, number, number];
|
|
23
|
+
/** Average a set of landmarks into a single centroid landmark */
|
|
24
|
+
export declare const landmarksCentroid: (landmarks: VisionLandmark[]) => VisionLandmark;
|
|
25
|
+
//# sourceMappingURL=landmarkMapping.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"landmarkMapping.d.ts","sourceRoot":"","sources":["../../src/vision/landmarkMapping.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE9C,wFAAwF;AACxF,MAAM,MAAM,QAAQ,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzD,gFAAgF;AAChF,MAAM,MAAM,SAAS,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1D;;;;;;;;;GASG;AACH,eAAO,MAAM,eAAe,GAC1B,UAAU,cAAc,EACxB,UAAU,QAAQ,EAClB,WAAW,SAAS,EACpB,gBAAa,KACZ,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAezB,CAAC;AAEF,iEAAiE;AACjE,eAAO,MAAM,iBAAiB,GAC5B,WAAW,cAAc,EAAE,KAC1B,cAWF,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { ONNXModelConfig } from './types';
|
|
2
|
+
/**
|
|
3
|
+
* COCO 80-class labels in canonical index order. Default label set for the
|
|
4
|
+
* built-in detection presets.
|
|
5
|
+
*/
|
|
6
|
+
export declare const COCO_LABELS: readonly ["person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"];
|
|
7
|
+
/** Overridable fields when constructing a preset model config */
|
|
8
|
+
export type PresetOverrides = Partial<Omit<ONNXModelConfig, "path" | "decoder">>;
|
|
9
|
+
/**
|
|
10
|
+
* Build a config for a YOLOv8/v11-style detector. Expects a single
|
|
11
|
+
* `[1, 4 + numClasses, numAnchors]` output (decoded with NMS).
|
|
12
|
+
*
|
|
13
|
+
* @param path URL or same-origin path to the exported `.onnx` weights
|
|
14
|
+
* (e.g. from `yolo export format=onnx`). Weights are not bundled.
|
|
15
|
+
*/
|
|
16
|
+
export declare const yolo: (path: string, overrides?: PresetOverrides) => ONNXModelConfig;
|
|
17
|
+
/**
|
|
18
|
+
* Build a config for an RF-DETR / DETR-style detector. Expects a `[1, Q, 4]`
|
|
19
|
+
* boxes output and a `[1, Q, C]` logits output (NMS-free set prediction).
|
|
20
|
+
*
|
|
21
|
+
* @param path URL or same-origin path to the exported `.onnx` weights.
|
|
22
|
+
* Weights are not bundled.
|
|
23
|
+
*/
|
|
24
|
+
export declare const rfDetr: (path: string, overrides?: PresetOverrides) => ONNXModelConfig;
|
|
25
|
+
//# sourceMappingURL=presets.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"presets.d.ts","sourceRoot":"","sources":["../../src/vision/presets.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C;;;GAGG;AACH,eAAO,MAAM,WAAW,w2BAiFd,CAAC;AAEX,iEAAiE;AACjE,MAAM,MAAM,eAAe,GAAG,OAAO,CACnC,IAAI,CAAC,eAAe,EAAE,MAAM,GAAG,SAAS,CAAC,CAC1C,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,IAAI,GACf,MAAM,MAAM,EACZ,YAAW,eAAoB,KAC9B,eAOD,CAAC;AAEH;;;;;;GAMG;AACH,eAAO,MAAM,MAAM,GACjB,MAAM,MAAM,EACZ,YAAW,eAAoB,KAC9B,eAOD,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { VisionProvider, VisionSessionOptions } from '../types';
|
|
2
|
+
/** Create a vision provider based on the session options */
|
|
3
|
+
declare const createVisionProvider: (options: VisionSessionOptions) => VisionProvider;
|
|
4
|
+
export default createVisionProvider;
|
|
5
|
+
//# sourceMappingURL=createVisionProvider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createVisionProvider.d.ts","sourceRoot":"","sources":["../../../src/vision/providers/createVisionProvider.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAErE,4DAA4D;AAC5D,QAAA,MAAM,oBAAoB,GACxB,SAAS,oBAAoB,KAC5B,cAMF,CAAC;AAEF,eAAe,oBAAoB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/vision/providers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,IAAI,oBAAoB,EAAE,MAAM,wBAAwB,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { VisionFrame, VisionProvider, VisionSessionOptions } from '../types';
|
|
2
|
+
declare class MediaPipeProvider implements VisionProvider {
|
|
3
|
+
readonly type: "mediapipe";
|
|
4
|
+
private models;
|
|
5
|
+
private isInitialized;
|
|
6
|
+
private isProcessing;
|
|
7
|
+
private isDisposed;
|
|
8
|
+
private videoElement;
|
|
9
|
+
private animationId;
|
|
10
|
+
private lastProcessTime;
|
|
11
|
+
private callbacks;
|
|
12
|
+
private options;
|
|
13
|
+
constructor(options: VisionSessionOptions);
|
|
14
|
+
initialize(video: HTMLVideoElement): Promise<void>;
|
|
15
|
+
startDetection(): void;
|
|
16
|
+
stopDetection(): void;
|
|
17
|
+
onDetection(callback: (frame: VisionFrame) => void): () => void;
|
|
18
|
+
dispose(): void;
|
|
19
|
+
private processFrame;
|
|
20
|
+
private handleResult;
|
|
21
|
+
}
|
|
22
|
+
export default MediaPipeProvider;
|
|
23
|
+
//# sourceMappingURL=mediapipeProvider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mediapipeProvider.d.ts","sourceRoot":"","sources":["../../../src/vision/providers/mediapipeProvider.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAEV,WAAW,EACX,cAAc,EACd,oBAAoB,EACrB,MAAM,UAAU,CAAC;AAWlB,cAAM,iBAAkB,YAAW,cAAc;IAC/C,QAAQ,CAAC,IAAI,EAAG,WAAW,CAAU;IAErC,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,YAAY,CAAiC;IACrD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,SAAS,CAA2C;IAC5D,OAAO,CAAC,OAAO,CAAuB;gBAE1B,OAAO,EAAE,oBAAoB;IAInC,UAAU,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAyGxD,cAAc,IAAI,IAAI;IAOtB,aAAa,IAAI,IAAI;IAQrB,WAAW,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,MAAM,IAAI;IAS/D,OAAO,IAAI,IAAI;IAef,OAAO,CAAC,YAAY,CAkIlB;IAEF,OAAO,CAAC,YAAY;CA0BrB;AAED,eAAe,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { VisionFrame, VisionProvider, VisionSessionOptions } from '../types';
|
|
2
|
+
declare class ONNXProvider implements VisionProvider {
|
|
3
|
+
readonly type: "onnx";
|
|
4
|
+
private worker;
|
|
5
|
+
private isInitialized;
|
|
6
|
+
private isDisposed;
|
|
7
|
+
private isProcessing;
|
|
8
|
+
private videoElement;
|
|
9
|
+
private animationId;
|
|
10
|
+
private loadedModels;
|
|
11
|
+
private lastProcessTime;
|
|
12
|
+
private processingStartTime;
|
|
13
|
+
private callbacks;
|
|
14
|
+
private options;
|
|
15
|
+
private messageHandler;
|
|
16
|
+
private pendingModelCallbacks;
|
|
17
|
+
constructor(options: VisionSessionOptions);
|
|
18
|
+
/** Post a typed request to the worker, with an optional transfer list */
|
|
19
|
+
private post;
|
|
20
|
+
initialize(video: HTMLVideoElement): Promise<void>;
|
|
21
|
+
startDetection(): void;
|
|
22
|
+
stopDetection(): void;
|
|
23
|
+
onDetection(callback: (frame: VisionFrame) => void): () => void;
|
|
24
|
+
dispose(): void;
|
|
25
|
+
private loadModel;
|
|
26
|
+
private processCurrentFrame;
|
|
27
|
+
private handleResult;
|
|
28
|
+
}
|
|
29
|
+
export default ONNXProvider;
|
|
30
|
+
//# sourceMappingURL=onnxProvider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"onnxProvider.d.ts","sourceRoot":"","sources":["../../../src/vision/providers/onnxProvider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAEV,WAAW,EACX,cAAc,EACd,oBAAoB,EACrB,MAAM,UAAU,CAAC;AAUlB,cAAM,YAAa,YAAW,cAAc;IAC1C,QAAQ,CAAC,IAAI,EAAG,MAAM,CAAU;IAEhC,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,YAAY,CAAiC;IACrD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,YAAY,CAAsC;IAC1D,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,SAAS,CAA2C;IAC5D,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,cAAc,CAAgD;IACtE,OAAO,CAAC,qBAAqB,CAGzB;gBAEQ,OAAO,EAAE,oBAAoB;IAIzC,yEAAyE;IACzE,OAAO,CAAC,IAAI;IAQN,UAAU,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsExD,cAAc,IAAI,IAAI;IAoCtB,aAAa,IAAI,IAAI;IAQrB,WAAW,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,MAAM,IAAI;IAS/D,OAAO,IAAI,IAAI;YAqBD,SAAS;IAavB,OAAO,CAAC,mBAAmB;IAwC3B,OAAO,CAAC,YAAY;CA4BrB;AAED,eAAe,YAAY,CAAC"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/** Single 3D landmark point */
|
|
2
|
+
export type VisionLandmark = {
|
|
3
|
+
x: number;
|
|
4
|
+
y: number;
|
|
5
|
+
z: number;
|
|
6
|
+
};
|
|
7
|
+
/** Detected landmark group (hand, face, or pose) with confidence */
|
|
8
|
+
export type LandmarkDetection = {
|
|
9
|
+
landmarks: VisionLandmark[];
|
|
10
|
+
confidence: number;
|
|
11
|
+
};
|
|
12
|
+
/** Detected object with bounding box */
|
|
13
|
+
export type ObjectDetection = {
|
|
14
|
+
label: string;
|
|
15
|
+
confidence: number;
|
|
16
|
+
bbox: {
|
|
17
|
+
x: number;
|
|
18
|
+
y: number;
|
|
19
|
+
width: number;
|
|
20
|
+
height: number;
|
|
21
|
+
};
|
|
22
|
+
};
|
|
23
|
+
/** A single frame of vision detection results */
|
|
24
|
+
export type VisionFrame = {
|
|
25
|
+
hands: LandmarkDetection[];
|
|
26
|
+
faces: LandmarkDetection[];
|
|
27
|
+
poses: LandmarkDetection[];
|
|
28
|
+
objects: ObjectDetection[];
|
|
29
|
+
timestamp: number;
|
|
30
|
+
frameSize: {
|
|
31
|
+
width: number;
|
|
32
|
+
height: number;
|
|
33
|
+
};
|
|
34
|
+
processingTime?: number;
|
|
35
|
+
segmentationMask?: ImageData;
|
|
36
|
+
};
|
|
37
|
+
/** Supported vision tasks */
|
|
38
|
+
export type VisionTask = "hands" | "faces" | "poses" | "objects";
|
|
39
|
+
/** Identifier of a registered ONNX output decoder */
|
|
40
|
+
export type ONNXDecoderName = "yolo" | "rfdetr";
|
|
41
|
+
/** ONNX model configuration */
|
|
42
|
+
export type ONNXModelConfig = {
|
|
43
|
+
name: string;
|
|
44
|
+
path: string;
|
|
45
|
+
/** Square input side length the model expects (default 640) */
|
|
46
|
+
inputSize?: number;
|
|
47
|
+
/** Class labels indexed by class id (defaults to COCO-80) */
|
|
48
|
+
labels?: string[];
|
|
49
|
+
/** Which registered decoder interprets this model's raw outputs (default "yolo") */
|
|
50
|
+
decoder?: ONNXDecoderName;
|
|
51
|
+
};
|
|
52
|
+
/** Progress callback payload */
|
|
53
|
+
export type VisionProgress = {
|
|
54
|
+
step: number;
|
|
55
|
+
total: number;
|
|
56
|
+
label: string;
|
|
57
|
+
};
|
|
58
|
+
/** Options for creating a vision session */
|
|
59
|
+
export type VisionSessionOptions = {
|
|
60
|
+
/** Which provider to use */
|
|
61
|
+
provider?: "mediapipe" | "onnx";
|
|
62
|
+
/** Vision tasks to enable */
|
|
63
|
+
tasks?: VisionTask[];
|
|
64
|
+
/** Minimum confidence threshold */
|
|
65
|
+
minConfidence?: number;
|
|
66
|
+
/** Maximum results per task */
|
|
67
|
+
maxResults?: number;
|
|
68
|
+
/** Frame processing throttle in ms */
|
|
69
|
+
throttle?: number;
|
|
70
|
+
/** Use GPU acceleration */
|
|
71
|
+
useGpu?: boolean;
|
|
72
|
+
/** Compute a pose segmentation mask (MediaPipe pose only; costly, default off) */
|
|
73
|
+
segmentation?: boolean;
|
|
74
|
+
/** External video element (skip internal camera setup) */
|
|
75
|
+
videoElement?: HTMLVideoElement;
|
|
76
|
+
/** Progress callback for model loading */
|
|
77
|
+
onProgress?: (progress: VisionProgress | null) => void;
|
|
78
|
+
/** ONNX-specific options */
|
|
79
|
+
onnx?: {
|
|
80
|
+
models: ONNXModelConfig[];
|
|
81
|
+
};
|
|
82
|
+
};
|
|
83
|
+
/** Vision provider interface */
|
|
84
|
+
export type VisionProvider = {
|
|
85
|
+
readonly type: string;
|
|
86
|
+
initialize(video: HTMLVideoElement): Promise<void>;
|
|
87
|
+
startDetection(): void;
|
|
88
|
+
stopDetection(): void;
|
|
89
|
+
onDetection(callback: (frame: VisionFrame) => void): () => void;
|
|
90
|
+
dispose(): void;
|
|
91
|
+
};
|
|
92
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/vision/types.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,MAAM,MAAM,cAAc,GAAG;IAC3B,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX,CAAC;AAEF,oEAAoE;AACpE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,wCAAwC;AACxC,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE;QACJ,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH,CAAC;AAEF,iDAAiD;AACjD,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,SAAS,CAAC;CAC9B,CAAC;AAEF,6BAA6B;AAC7B,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;AAEjE,qDAAqD;AACrD,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEhD,+BAA+B;AAC/B,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,oFAAoF;IACpF,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B,CAAC;AAEF,gCAAgC;AAChC,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,4CAA4C;AAC5C,MAAM,MAAM,oBAAoB,GAAG;IACjC,4BAA4B;IAC5B,QAAQ,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC;IAChC,6BAA6B;IAC7B,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IACrB,mCAAmC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,+BAA+B;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sCAAsC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2BAA2B;IAC3B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,kFAAkF;IAClF,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,0CAA0C;IAC1C,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,cAAc,GAAG,IAAI,KAAK,IAAI,CAAC;IACvD,4BAA4B;IAC5B,IAAI,CAAC,EAAE;QACL,MAAM,EAAE,eAAe,EAAE,CAAC;KAC3B,CAAC;CACH,CAAC;AAEF,gCAAgC;AAChC,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnD,cAAc,IAAI,IAAI,CAAC;IACvB,aAAa,IAAI,IAAI,CAAC;IACtB,WAAW,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,MAAM,IAAI,CAAC;IAChE,OAAO,IAAI,IAAI,CAAC;CACjB,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { VisionBackendState } from './visionBackend';
|
|
2
|
+
/**
|
|
3
|
+
* Access the vision backend internals.
|
|
4
|
+
* @returns vision backend state with `isSuccess` to verify readiness
|
|
5
|
+
*/
|
|
6
|
+
declare const useVisionBackend: () => VisionBackendState;
|
|
7
|
+
export default useVisionBackend;
|
|
8
|
+
//# sourceMappingURL=useVisionBackend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useVisionBackend.d.ts","sourceRoot":"","sources":["../../src/vision/useVisionBackend.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,kBAAkB,EAAkB,MAAM,iBAAiB,CAAC;AAW1E;;;GAGG;AACH,QAAA,MAAM,gBAAgB,QAAO,kBAiB5B,CAAC;AAEF,eAAe,gBAAgB,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { VisionFrame } from './types';
|
|
2
|
+
export type UseVisionFrameOptions = {
|
|
3
|
+
/** Minimum interval between state updates (ms) */
|
|
4
|
+
throttle?: number;
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Subscribe to full vision frames with a single backend subscription.
|
|
8
|
+
* Use this instead of multiple `useVisionDetection` calls when you need
|
|
9
|
+
* data from several tasks simultaneously.
|
|
10
|
+
*/
|
|
11
|
+
declare const useVisionFrame: (options?: UseVisionFrameOptions) => VisionFrame | null;
|
|
12
|
+
export default useVisionFrame;
|
|
13
|
+
//# sourceMappingURL=useVisionFrame.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useVisionFrame.d.ts","sourceRoot":"","sources":["../../src/vision/useVisionFrame.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,MAAM,MAAM,qBAAqB,GAAG;IAClC,kDAAkD;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;;GAIG;AACH,QAAA,MAAM,cAAc,GAClB,UAAU,qBAAqB,KAC9B,WAAW,GAAG,IAuBhB,CAAC;AAEF,eAAe,cAAc,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Backend } from 'lib/types/engine';
|
|
2
|
+
import { VisionFrame, VisionProvider, VisionSessionOptions } from './types';
|
|
3
|
+
/** Internal state exposed by the vision backend */
|
|
4
|
+
export type VisionInternal = {
|
|
5
|
+
provider: VisionProvider | null;
|
|
6
|
+
videoElement: HTMLVideoElement | null;
|
|
7
|
+
onDetection: (cb: (frame: VisionFrame) => void) => () => void;
|
|
8
|
+
};
|
|
9
|
+
/** State returned by the vision backend hook */
|
|
10
|
+
export type VisionBackendState = VisionInternal & {
|
|
11
|
+
isPending: boolean;
|
|
12
|
+
isSuccess: boolean;
|
|
13
|
+
};
|
|
14
|
+
/** Create a vision backend following the Backend interface contract */
|
|
15
|
+
declare const createVisionBackend: (options: VisionSessionOptions) => Backend<VisionInternal>;
|
|
16
|
+
export default createVisionBackend;
|
|
17
|
+
//# sourceMappingURL=visionBackend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"visionBackend.d.ts","sourceRoot":"","sources":["../../src/vision/visionBackend.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAmB,MAAM,kBAAkB,CAAC;AACjE,OAAO,KAAK,EACV,WAAW,EACX,cAAc,EACd,oBAAoB,EACrB,MAAM,SAAS,CAAC;AAEjB,mDAAmD;AACnD,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,cAAc,GAAG,IAAI,CAAC;IAChC,YAAY,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACtC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,KAAK,MAAM,IAAI,CAAC;CAC/D,CAAC;AAEF,gDAAgD;AAChD,MAAM,MAAM,kBAAkB,GAAG,cAAc,GAAG;IAChD,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;CACpB,CAAC;AAsDF,uEAAuE;AACvE,QAAA,MAAM,mBAAmB,GACvB,SAAS,oBAAoB,KAC5B,OAAO,CAAC,cAAc,CAgExB,CAAC;AAEF,eAAe,mBAAmB,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { ONNXDecoderName } from '../../types';
|
|
2
|
+
import { ONNXDecoder } from './types';
|
|
3
|
+
/** Resolve a decoder by name, defaulting to YOLO */
|
|
4
|
+
export declare const getDecoder: (name?: ONNXDecoderName) => ONNXDecoder;
|
|
5
|
+
export type { DecodeContext, ONNXDecoder, TensorLike } from './types';
|
|
6
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/vision/worker/decoders/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAQ3C,oDAAoD;AACpD,eAAO,MAAM,UAAU,GAAI,OAAM,eAAwB,KAAG,WAC7B,CAAC;AAEhC,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { ObjectDetection } from '../../types';
|
|
2
|
+
import { DecodeContext } from './types';
|
|
3
|
+
/** Axis-aligned box in [x, y, width, height] form */
|
|
4
|
+
export type Box = {
|
|
5
|
+
x: number;
|
|
6
|
+
y: number;
|
|
7
|
+
width: number;
|
|
8
|
+
height: number;
|
|
9
|
+
};
|
|
10
|
+
/** Intersection-over-union of two axis-aligned boxes */
|
|
11
|
+
export declare const iou: (a: Box, b: Box) => number;
|
|
12
|
+
/**
|
|
13
|
+
* Greedy non-maximum suppression. Detections are sorted by confidence, then
|
|
14
|
+
* lower-scoring boxes overlapping a kept box (of the same class) beyond
|
|
15
|
+
* `iouThreshold` are discarded.
|
|
16
|
+
*/
|
|
17
|
+
export declare const nms: (detections: ObjectDetection[], iouThreshold: number) => ObjectDetection[];
|
|
18
|
+
/**
|
|
19
|
+
* Map a box from letterboxed input space (xyxy) back to source pixels,
|
|
20
|
+
* clamped to the frame, returning null if degenerate.
|
|
21
|
+
*/
|
|
22
|
+
export declare const letterboxToSource: (x1: number, y1: number, x2: number, y2: number, ctx: DecodeContext) => Box | null;
|
|
23
|
+
//# sourceMappingURL=nms.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"nms.d.ts","sourceRoot":"","sources":["../../../../src/vision/worker/decoders/nms.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAE7C,qDAAqD;AACrD,MAAM,MAAM,GAAG,GAAG;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1E,wDAAwD;AACxD,eAAO,MAAM,GAAG,GAAI,GAAG,GAAG,EAAE,GAAG,GAAG,KAAG,MAkBpC,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,GAAG,GACd,YAAY,eAAe,EAAE,EAC7B,cAAc,MAAM,KACnB,eAAe,EAcjB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,iBAAiB,GAC5B,IAAI,MAAM,EACV,IAAI,MAAM,EACV,IAAI,MAAM,EACV,IAAI,MAAM,EACV,KAAK,aAAa,KACjB,GAAG,GAAG,IAgBR,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { ONNXDecoder } from './types';
|
|
2
|
+
/**
|
|
3
|
+
* Decoder for RF-DETR / DETR-style set-prediction heads.
|
|
4
|
+
*
|
|
5
|
+
* Two outputs: boxes `[1, Q, 4]` in normalized `cxcywh` (0-1 over the square
|
|
6
|
+
* input) and logits `[1, Q, C]`. Scores are `sigmoid(logits)`; the best class
|
|
7
|
+
* per query is thresholded and top-k selected. NMS-free by construction.
|
|
8
|
+
*/
|
|
9
|
+
export declare const rfDetrDecoder: ONNXDecoder;
|
|
10
|
+
//# sourceMappingURL=rfDetr.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rfDetr.d.ts","sourceRoot":"","sources":["../../../../src/vision/worker/decoders/rfDetr.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAiB,WAAW,EAAc,MAAM,SAAS,CAAC;AAoBtE;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,EAAE,WAyD3B,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { ObjectDetection } from '../../types';
|
|
2
|
+
/**
|
|
3
|
+
* Minimal structural view of an onnxruntime-web tensor. Decoders depend on this
|
|
4
|
+
* rather than the ort types so they stay pure and unit-testable with fixtures.
|
|
5
|
+
*/
|
|
6
|
+
export type TensorLike = {
|
|
7
|
+
data: Float32Array | number[];
|
|
8
|
+
dims: readonly number[];
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Preprocessing transform + metadata needed to map model outputs (in
|
|
12
|
+
* letterboxed input space) back to source pixel coordinates.
|
|
13
|
+
*/
|
|
14
|
+
export type DecodeContext = {
|
|
15
|
+
/** Square input side length fed to the model (e.g. 640) */
|
|
16
|
+
inputSize: number;
|
|
17
|
+
/** Original source frame dimensions */
|
|
18
|
+
sourceWidth: number;
|
|
19
|
+
sourceHeight: number;
|
|
20
|
+
/** Uniform letterbox scale applied to the source: min(inputSize/sw, inputSize/sh) */
|
|
21
|
+
scale: number;
|
|
22
|
+
/** Letterbox padding (input-space pixels) added on each axis */
|
|
23
|
+
padX: number;
|
|
24
|
+
padY: number;
|
|
25
|
+
/** Class labels indexed by class id */
|
|
26
|
+
labels: string[];
|
|
27
|
+
/** Minimum confidence to keep a detection */
|
|
28
|
+
minConfidence: number;
|
|
29
|
+
/** Maximum detections to return */
|
|
30
|
+
maxResults: number;
|
|
31
|
+
};
|
|
32
|
+
/**
|
|
33
|
+
* Interprets a model's raw output tensors into detections in source pixel
|
|
34
|
+
* coordinates. Register concrete decoders by {@link import("../../types").ONNXDecoderName}.
|
|
35
|
+
*/
|
|
36
|
+
export type ONNXDecoder = {
|
|
37
|
+
readonly name: string;
|
|
38
|
+
decode(outputs: Record<string, TensorLike>, ctx: DecodeContext): ObjectDetection[];
|
|
39
|
+
};
|
|
40
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/vision/worker/decoders/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,YAAY,GAAG,MAAM,EAAE,CAAC;IAC9B,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CACzB,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,qFAAqF;IACrF,KAAK,EAAE,MAAM,CAAC;IACd,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,6CAA6C;IAC7C,aAAa,EAAE,MAAM,CAAC;IACtB,mCAAmC;IACnC,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,MAAM,CACJ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,EACnC,GAAG,EAAE,aAAa,GACjB,eAAe,EAAE,CAAC;CACtB,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { ONNXDecoder } from './types';
|
|
2
|
+
/**
|
|
3
|
+
* Decoder for YOLOv8/v11-style detection heads.
|
|
4
|
+
*
|
|
5
|
+
* Output tensor shape `[1, 4 + numClasses, numAnchors]` (transposed head):
|
|
6
|
+
* value(channel, anchor) = data[channel * numAnchors + anchor]. Channels 0-3
|
|
7
|
+
* are `cx, cy, w, h` in input-space pixels; the rest are per-class scores.
|
|
8
|
+
* Requires NMS since anchors are dense and unfiltered.
|
|
9
|
+
*/
|
|
10
|
+
export declare const yoloDecoder: ONNXDecoder;
|
|
11
|
+
//# sourceMappingURL=yolo.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"yolo.d.ts","sourceRoot":"","sources":["../../../../src/vision/worker/decoders/yolo.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAiB,WAAW,EAAc,MAAM,SAAS,CAAC;AAYtE;;;;;;;GAOG;AACH,eAAO,MAAM,WAAW,EAAE,WAgEzB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"onnxWorker.d.ts","sourceRoot":"","sources":["../../../src/vision/worker/onnxWorker.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/** Letterbox transform mapping a source frame into a square model input */
|
|
2
|
+
export type Letterbox = {
|
|
3
|
+
/** Uniform scale applied to the source */
|
|
4
|
+
scale: number;
|
|
5
|
+
/** Padding added on each axis (input-space pixels) */
|
|
6
|
+
padX: number;
|
|
7
|
+
padY: number;
|
|
8
|
+
/** Scaled source dimensions drawn onto the square canvas */
|
|
9
|
+
drawWidth: number;
|
|
10
|
+
drawHeight: number;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* Compute the aspect-preserving letterbox that fits a source frame into a
|
|
14
|
+
* `size x size` square, centered with equal padding.
|
|
15
|
+
*/
|
|
16
|
+
export declare const computeLetterbox: (sourceWidth: number, sourceHeight: number, size: number) => Letterbox;
|
|
17
|
+
/**
|
|
18
|
+
* Convert interleaved RGBA pixels into a planar NCHW float tensor
|
|
19
|
+
* (`[1, 3, size, size]`), normalizing each channel to 0-1. Alpha is dropped.
|
|
20
|
+
*/
|
|
21
|
+
export declare const rgbaToNchw: (rgba: Uint8ClampedArray | Uint8Array, size: number) => Float32Array;
|
|
22
|
+
//# sourceMappingURL=preprocess.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"preprocess.d.ts","sourceRoot":"","sources":["../../../src/vision/worker/preprocess.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,MAAM,MAAM,SAAS,GAAG;IACtB,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,gBAAgB,GAC3B,aAAa,MAAM,EACnB,cAAc,MAAM,EACpB,MAAM,MAAM,KACX,SAWF,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,UAAU,GACrB,MAAM,iBAAiB,GAAG,UAAU,EACpC,MAAM,MAAM,KACX,YASF,CAAC"}
|