@camstack/vision 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addons/animal-classifier/index.d.mts +25 -0
- package/dist/addons/animal-classifier/index.d.ts +25 -0
- package/dist/addons/animal-classifier/index.js +469 -0
- package/dist/addons/animal-classifier/index.js.map +1 -0
- package/dist/addons/animal-classifier/index.mjs +9 -0
- package/dist/addons/animal-classifier/index.mjs.map +1 -0
- package/dist/addons/audio-classification/index.d.mts +31 -0
- package/dist/addons/audio-classification/index.d.ts +31 -0
- package/dist/addons/audio-classification/index.js +411 -0
- package/dist/addons/audio-classification/index.js.map +1 -0
- package/dist/addons/audio-classification/index.mjs +8 -0
- package/dist/addons/audio-classification/index.mjs.map +1 -0
- package/dist/addons/bird-global-classifier/index.d.mts +26 -0
- package/dist/addons/bird-global-classifier/index.d.ts +26 -0
- package/dist/addons/bird-global-classifier/index.js +475 -0
- package/dist/addons/bird-global-classifier/index.js.map +1 -0
- package/dist/addons/bird-global-classifier/index.mjs +9 -0
- package/dist/addons/bird-global-classifier/index.mjs.map +1 -0
- package/dist/addons/bird-nabirds-classifier/index.d.mts +28 -0
- package/dist/addons/bird-nabirds-classifier/index.d.ts +28 -0
- package/dist/addons/bird-nabirds-classifier/index.js +517 -0
- package/dist/addons/bird-nabirds-classifier/index.js.map +1 -0
- package/dist/addons/bird-nabirds-classifier/index.mjs +9 -0
- package/dist/addons/bird-nabirds-classifier/index.mjs.map +1 -0
- package/dist/addons/camera-native-detection/index.d.mts +32 -0
- package/dist/addons/camera-native-detection/index.d.ts +32 -0
- package/dist/addons/camera-native-detection/index.js +99 -0
- package/dist/addons/camera-native-detection/index.js.map +1 -0
- package/dist/addons/camera-native-detection/index.mjs +7 -0
- package/dist/addons/camera-native-detection/index.mjs.map +1 -0
- package/dist/addons/face-detection/index.d.mts +24 -0
- package/dist/addons/face-detection/index.d.ts +24 -0
- package/dist/addons/face-detection/index.js +513 -0
- package/dist/addons/face-detection/index.js.map +1 -0
- package/dist/addons/face-detection/index.mjs +10 -0
- package/dist/addons/face-detection/index.mjs.map +1 -0
- package/dist/addons/face-recognition/index.d.mts +24 -0
- package/dist/addons/face-recognition/index.d.ts +24 -0
- package/dist/addons/face-recognition/index.js +437 -0
- package/dist/addons/face-recognition/index.js.map +1 -0
- package/dist/addons/face-recognition/index.mjs +9 -0
- package/dist/addons/face-recognition/index.mjs.map +1 -0
- package/dist/addons/motion-detection/index.d.mts +26 -0
- package/dist/addons/motion-detection/index.d.ts +26 -0
- package/dist/addons/motion-detection/index.js +273 -0
- package/dist/addons/motion-detection/index.js.map +1 -0
- package/dist/addons/motion-detection/index.mjs +8 -0
- package/dist/addons/motion-detection/index.mjs.map +1 -0
- package/dist/addons/object-detection/index.d.mts +25 -0
- package/dist/addons/object-detection/index.d.ts +25 -0
- package/dist/addons/object-detection/index.js +673 -0
- package/dist/addons/object-detection/index.js.map +1 -0
- package/dist/addons/object-detection/index.mjs +10 -0
- package/dist/addons/object-detection/index.mjs.map +1 -0
- package/dist/addons/plate-detection/index.d.mts +25 -0
- package/dist/addons/plate-detection/index.d.ts +25 -0
- package/dist/addons/plate-detection/index.js +477 -0
- package/dist/addons/plate-detection/index.js.map +1 -0
- package/dist/addons/plate-detection/index.mjs +10 -0
- package/dist/addons/plate-detection/index.mjs.map +1 -0
- package/dist/addons/plate-recognition/index.d.mts +25 -0
- package/dist/addons/plate-recognition/index.d.ts +25 -0
- package/dist/addons/plate-recognition/index.js +470 -0
- package/dist/addons/plate-recognition/index.js.map +1 -0
- package/dist/addons/plate-recognition/index.mjs +9 -0
- package/dist/addons/plate-recognition/index.mjs.map +1 -0
- package/dist/chunk-3BKYLBBH.mjs +229 -0
- package/dist/chunk-3BKYLBBH.mjs.map +1 -0
- package/dist/chunk-4PC262GU.mjs +203 -0
- package/dist/chunk-4PC262GU.mjs.map +1 -0
- package/dist/chunk-6OR5TE7A.mjs +101 -0
- package/dist/chunk-6OR5TE7A.mjs.map +1 -0
- package/dist/chunk-7SZAISGP.mjs +210 -0
- package/dist/chunk-7SZAISGP.mjs.map +1 -0
- package/dist/chunk-AD2TFYZA.mjs +235 -0
- package/dist/chunk-AD2TFYZA.mjs.map +1 -0
- package/dist/chunk-CGYSSHHM.mjs +363 -0
- package/dist/chunk-CGYSSHHM.mjs.map +1 -0
- package/dist/chunk-IYHMGYGP.mjs +79 -0
- package/dist/chunk-IYHMGYGP.mjs.map +1 -0
- package/dist/chunk-J3IUBPRE.mjs +187 -0
- package/dist/chunk-J3IUBPRE.mjs.map +1 -0
- package/dist/chunk-KFZDJPYL.mjs +190 -0
- package/dist/chunk-KFZDJPYL.mjs.map +1 -0
- package/dist/chunk-KUO2BVFY.mjs +90 -0
- package/dist/chunk-KUO2BVFY.mjs.map +1 -0
- package/dist/chunk-PXBY3QOA.mjs +152 -0
- package/dist/chunk-PXBY3QOA.mjs.map +1 -0
- package/dist/chunk-XUKDL23Y.mjs +216 -0
- package/dist/chunk-XUKDL23Y.mjs.map +1 -0
- package/dist/chunk-Z26BVC7S.mjs +214 -0
- package/dist/chunk-Z26BVC7S.mjs.map +1 -0
- package/dist/chunk-Z5AHZQEZ.mjs +258 -0
- package/dist/chunk-Z5AHZQEZ.mjs.map +1 -0
- package/dist/index.d.mts +152 -0
- package/dist/index.d.ts +152 -0
- package/dist/index.js +2775 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +205 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +43 -0
- package/python/coreml_inference.py +67 -0
- package/python/openvino_inference.py +76 -0
- package/python/pytorch_inference.py +74 -0
|
@@ -0,0 +1,363 @@
|
|
|
1
|
+
import {
|
|
2
|
+
nms,
|
|
3
|
+
yoloPostprocess
|
|
4
|
+
} from "./chunk-KUO2BVFY.mjs";
|
|
5
|
+
import {
|
|
6
|
+
letterbox
|
|
7
|
+
} from "./chunk-6OR5TE7A.mjs";
|
|
8
|
+
import {
|
|
9
|
+
resolveEngine
|
|
10
|
+
} from "./chunk-J3IUBPRE.mjs";
|
|
11
|
+
|
|
12
|
+
// src/addons/object-detection/index.ts
|
|
13
|
+
import {
|
|
14
|
+
OBJECT_DETECTION_MODELS,
|
|
15
|
+
SEGMENTATION_MODELS,
|
|
16
|
+
COCO_TO_MACRO,
|
|
17
|
+
MACRO_LABELS
|
|
18
|
+
} from "@camstack/types";
|
|
19
|
+
|
|
20
|
+
// src/shared/postprocess/yolo-seg.ts
|
|
21
|
+
function sigmoid(x) {
|
|
22
|
+
return 1 / (1 + Math.exp(-x));
|
|
23
|
+
}
|
|
24
|
+
function computeRawMask(coeffs, protos, numMaskCoeffs, maskH, maskW) {
|
|
25
|
+
const maskSize = maskH * maskW;
|
|
26
|
+
const rawMask = new Float32Array(maskSize);
|
|
27
|
+
for (let px = 0; px < maskSize; px++) {
|
|
28
|
+
let val = 0;
|
|
29
|
+
for (let k = 0; k < numMaskCoeffs; k++) {
|
|
30
|
+
val += (coeffs[k] ?? 0) * (protos[k * maskSize + px] ?? 0);
|
|
31
|
+
}
|
|
32
|
+
rawMask[px] = sigmoid(val);
|
|
33
|
+
}
|
|
34
|
+
return rawMask;
|
|
35
|
+
}
|
|
36
|
+
function cropAndThresholdMask(rawMask, maskH, maskW, bbox, maskThreshold, maskScale) {
|
|
37
|
+
const cropX1 = Math.max(0, Math.floor(bbox.x * maskScale));
|
|
38
|
+
const cropY1 = Math.max(0, Math.floor(bbox.y * maskScale));
|
|
39
|
+
const cropX2 = Math.min(maskW, Math.ceil((bbox.x + bbox.w) * maskScale));
|
|
40
|
+
const cropY2 = Math.min(maskH, Math.ceil((bbox.y + bbox.h) * maskScale));
|
|
41
|
+
const cropW = Math.max(1, cropX2 - cropX1);
|
|
42
|
+
const cropH = Math.max(1, cropY2 - cropY1);
|
|
43
|
+
const data = new Uint8Array(cropW * cropH);
|
|
44
|
+
for (let row = 0; row < cropH; row++) {
|
|
45
|
+
const srcRow = cropY1 + row;
|
|
46
|
+
for (let col = 0; col < cropW; col++) {
|
|
47
|
+
const srcCol = cropX1 + col;
|
|
48
|
+
const srcIdx = srcRow * maskW + srcCol;
|
|
49
|
+
data[row * cropW + col] = (rawMask[srcIdx] ?? 0) > maskThreshold ? 255 : 0;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return { data, width: cropW, height: cropH };
|
|
53
|
+
}
|
|
54
|
+
function yoloSegPostprocess(segOutput, options) {
|
|
55
|
+
const {
|
|
56
|
+
detectionOutput,
|
|
57
|
+
protoOutput,
|
|
58
|
+
numClasses,
|
|
59
|
+
numBoxes,
|
|
60
|
+
numMaskCoeffs,
|
|
61
|
+
maskHeight,
|
|
62
|
+
maskWidth
|
|
63
|
+
} = segOutput;
|
|
64
|
+
const {
|
|
65
|
+
confidence,
|
|
66
|
+
iouThreshold,
|
|
67
|
+
labels,
|
|
68
|
+
scale,
|
|
69
|
+
padX,
|
|
70
|
+
padY,
|
|
71
|
+
originalWidth,
|
|
72
|
+
originalHeight,
|
|
73
|
+
maskThreshold = 0.5
|
|
74
|
+
} = options;
|
|
75
|
+
const yoloInputSize = 640;
|
|
76
|
+
const maskScale = maskHeight / yoloInputSize;
|
|
77
|
+
const candidates = [];
|
|
78
|
+
for (let i = 0; i < numBoxes; i++) {
|
|
79
|
+
const cx = detectionOutput[0 * numBoxes + i] ?? 0;
|
|
80
|
+
const cy = detectionOutput[1 * numBoxes + i] ?? 0;
|
|
81
|
+
const w = detectionOutput[2 * numBoxes + i] ?? 0;
|
|
82
|
+
const h = detectionOutput[3 * numBoxes + i] ?? 0;
|
|
83
|
+
let bestScore = -Infinity;
|
|
84
|
+
let bestClass = 0;
|
|
85
|
+
for (let j = 0; j < numClasses; j++) {
|
|
86
|
+
const score = detectionOutput[(4 + j) * numBoxes + i] ?? 0;
|
|
87
|
+
if (score > bestScore) {
|
|
88
|
+
bestScore = score;
|
|
89
|
+
bestClass = j;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (bestScore < confidence) continue;
|
|
93
|
+
const bbox = {
|
|
94
|
+
x: cx - w / 2,
|
|
95
|
+
y: cy - h / 2,
|
|
96
|
+
w,
|
|
97
|
+
h
|
|
98
|
+
};
|
|
99
|
+
const coeffs = new Float32Array(numMaskCoeffs);
|
|
100
|
+
for (let k = 0; k < numMaskCoeffs; k++) {
|
|
101
|
+
coeffs[k] = detectionOutput[(4 + numClasses + k) * numBoxes + i] ?? 0;
|
|
102
|
+
}
|
|
103
|
+
candidates.push({ bbox, score: bestScore, classIdx: bestClass, coeffs });
|
|
104
|
+
}
|
|
105
|
+
if (candidates.length === 0) return [];
|
|
106
|
+
const keptIndices = nms(candidates, iouThreshold);
|
|
107
|
+
return keptIndices.map((idx) => {
|
|
108
|
+
const { bbox, score, classIdx, coeffs } = candidates[idx];
|
|
109
|
+
const label = labels[classIdx] ?? String(classIdx);
|
|
110
|
+
const x = Math.max(0, Math.min(originalWidth, (bbox.x - padX) / scale));
|
|
111
|
+
const y = Math.max(0, Math.min(originalHeight, (bbox.y - padY) / scale));
|
|
112
|
+
const x2 = Math.max(0, Math.min(originalWidth, (bbox.x + bbox.w - padX) / scale));
|
|
113
|
+
const y2 = Math.max(0, Math.min(originalHeight, (bbox.y + bbox.h - padY) / scale));
|
|
114
|
+
const finalBbox = { x, y, w: x2 - x, h: y2 - y };
|
|
115
|
+
const rawMask = computeRawMask(coeffs, protoOutput, numMaskCoeffs, maskHeight, maskWidth);
|
|
116
|
+
const { data: maskData, width: mW, height: mH } = cropAndThresholdMask(
|
|
117
|
+
rawMask,
|
|
118
|
+
maskHeight,
|
|
119
|
+
maskWidth,
|
|
120
|
+
bbox,
|
|
121
|
+
maskThreshold,
|
|
122
|
+
maskScale
|
|
123
|
+
);
|
|
124
|
+
return {
|
|
125
|
+
class: label,
|
|
126
|
+
originalClass: label,
|
|
127
|
+
score,
|
|
128
|
+
bbox: finalBbox,
|
|
129
|
+
mask: maskData,
|
|
130
|
+
maskWidth: mW,
|
|
131
|
+
maskHeight: mH
|
|
132
|
+
};
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// src/addons/object-detection/index.ts
|
|
137
|
+
function isSegModel(modelId) {
|
|
138
|
+
return modelId.includes("-seg");
|
|
139
|
+
}
|
|
140
|
+
var ALL_DETECTION_MODELS = [
|
|
141
|
+
...OBJECT_DETECTION_MODELS,
|
|
142
|
+
...SEGMENTATION_MODELS
|
|
143
|
+
];
|
|
144
|
+
function applyClassMap(detections, classMap) {
|
|
145
|
+
return detections.filter((d) => classMap.mapping[d.class] !== void 0).map((d) => ({
|
|
146
|
+
...d,
|
|
147
|
+
originalClass: d.class,
|
|
148
|
+
class: classMap.mapping[d.class]
|
|
149
|
+
}));
|
|
150
|
+
}
|
|
151
|
+
var ObjectDetectionAddon = class {
|
|
152
|
+
id = "object-detection";
|
|
153
|
+
slot = "detector";
|
|
154
|
+
inputClasses = null;
|
|
155
|
+
outputClasses = ["person", "vehicle", "animal"];
|
|
156
|
+
slotPriority = 0;
|
|
157
|
+
manifest = {
|
|
158
|
+
id: "object-detection",
|
|
159
|
+
name: "Object Detection",
|
|
160
|
+
version: "0.1.0",
|
|
161
|
+
description: "YOLO-based object detection \u2014 detects persons, vehicles, and animals",
|
|
162
|
+
packageName: "@camstack/vision",
|
|
163
|
+
slot: "detector",
|
|
164
|
+
inputClasses: void 0,
|
|
165
|
+
outputClasses: ["person", "vehicle", "animal"],
|
|
166
|
+
supportsCustomModels: true,
|
|
167
|
+
mayRequirePython: false,
|
|
168
|
+
defaultConfig: {
|
|
169
|
+
modelId: "yolov8n",
|
|
170
|
+
runtime: "auto",
|
|
171
|
+
backend: "cpu",
|
|
172
|
+
confidence: 0.5,
|
|
173
|
+
iouThreshold: 0.45
|
|
174
|
+
}
|
|
175
|
+
};
|
|
176
|
+
engine;
|
|
177
|
+
modelEntry;
|
|
178
|
+
confidence = 0.5;
|
|
179
|
+
iouThreshold = 0.45;
|
|
180
|
+
async initialize(ctx) {
|
|
181
|
+
const cfg = ctx.addonConfig;
|
|
182
|
+
const modelId = cfg["modelId"] ?? "yolov8n";
|
|
183
|
+
const runtime = cfg["runtime"] ?? "auto";
|
|
184
|
+
const backend = cfg["backend"] ?? "cpu";
|
|
185
|
+
this.confidence = cfg["confidence"] ?? 0.5;
|
|
186
|
+
this.iouThreshold = cfg["iouThreshold"] ?? 0.45;
|
|
187
|
+
const entry = ALL_DETECTION_MODELS.find((m) => m.id === modelId);
|
|
188
|
+
if (!entry) {
|
|
189
|
+
throw new Error(`ObjectDetectionAddon: unknown modelId "${modelId}"`);
|
|
190
|
+
}
|
|
191
|
+
this.modelEntry = entry;
|
|
192
|
+
const resolved = await resolveEngine({
|
|
193
|
+
runtime,
|
|
194
|
+
backend,
|
|
195
|
+
modelEntry: entry,
|
|
196
|
+
modelsDir: ctx.locationPaths.models
|
|
197
|
+
});
|
|
198
|
+
this.engine = resolved.engine;
|
|
199
|
+
}
|
|
200
|
+
async detect(frame) {
|
|
201
|
+
const start = Date.now();
|
|
202
|
+
const { width: inputW, height: inputH } = this.modelEntry.inputSize;
|
|
203
|
+
const targetSize = Math.max(inputW, inputH);
|
|
204
|
+
const lb = await letterbox(frame.data, targetSize);
|
|
205
|
+
const numClasses = this.modelEntry.labels.length;
|
|
206
|
+
const labels = this.modelEntry.labels.map((l) => l.id);
|
|
207
|
+
const postprocessOpts = {
|
|
208
|
+
confidence: this.confidence,
|
|
209
|
+
iouThreshold: this.iouThreshold,
|
|
210
|
+
labels,
|
|
211
|
+
scale: lb.scale,
|
|
212
|
+
padX: lb.padX,
|
|
213
|
+
padY: lb.padY,
|
|
214
|
+
originalWidth: lb.originalWidth,
|
|
215
|
+
originalHeight: lb.originalHeight
|
|
216
|
+
};
|
|
217
|
+
let rawDetections;
|
|
218
|
+
if (isSegModel(this.modelEntry.id)) {
|
|
219
|
+
const outputs = await this.engine.runMultiOutput(lb.data, [1, 3, targetSize, targetSize]);
|
|
220
|
+
const outputNames = Object.keys(outputs);
|
|
221
|
+
if (outputNames.length < 2) {
|
|
222
|
+
throw new Error(
|
|
223
|
+
`ObjectDetectionAddon: seg model "${this.modelEntry.id}" returned ${outputNames.length} output(s); expected 2`
|
|
224
|
+
);
|
|
225
|
+
}
|
|
226
|
+
const detectionOutput = outputs[outputNames[0]];
|
|
227
|
+
const protoOutput = outputs[outputNames[1]];
|
|
228
|
+
const numMaskCoeffs = 32;
|
|
229
|
+
const numBoxes = detectionOutput.length / (4 + numClasses + numMaskCoeffs);
|
|
230
|
+
const maskHeight = 160;
|
|
231
|
+
const maskWidth = 160;
|
|
232
|
+
rawDetections = yoloSegPostprocess(
|
|
233
|
+
{
|
|
234
|
+
detectionOutput,
|
|
235
|
+
protoOutput,
|
|
236
|
+
numClasses,
|
|
237
|
+
numBoxes,
|
|
238
|
+
numMaskCoeffs,
|
|
239
|
+
maskHeight,
|
|
240
|
+
maskWidth
|
|
241
|
+
},
|
|
242
|
+
postprocessOpts
|
|
243
|
+
);
|
|
244
|
+
} else {
|
|
245
|
+
const output = await this.engine.run(lb.data, [1, 3, targetSize, targetSize]);
|
|
246
|
+
const numBoxes = output.length / (4 + numClasses);
|
|
247
|
+
rawDetections = yoloPostprocess(output, numClasses, numBoxes, postprocessOpts);
|
|
248
|
+
}
|
|
249
|
+
const detections = applyClassMap(rawDetections, COCO_TO_MACRO);
|
|
250
|
+
return {
|
|
251
|
+
detections,
|
|
252
|
+
inferenceMs: Date.now() - start,
|
|
253
|
+
modelId: this.modelEntry.id
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
async shutdown() {
|
|
257
|
+
await this.engine?.dispose();
|
|
258
|
+
}
|
|
259
|
+
getConfigSchema() {
|
|
260
|
+
return {
|
|
261
|
+
sections: [
|
|
262
|
+
{
|
|
263
|
+
id: "model",
|
|
264
|
+
title: "Model",
|
|
265
|
+
columns: 2,
|
|
266
|
+
fields: [
|
|
267
|
+
{
|
|
268
|
+
key: "modelId",
|
|
269
|
+
label: "Model",
|
|
270
|
+
type: "model-selector",
|
|
271
|
+
catalog: [...ALL_DETECTION_MODELS],
|
|
272
|
+
allowCustom: true,
|
|
273
|
+
allowConversion: true,
|
|
274
|
+
acceptFormats: ["onnx", "coreml", "openvino", "tflite"],
|
|
275
|
+
requiredMetadata: ["inputSize", "labels", "outputFormat"],
|
|
276
|
+
outputFormatHint: "yolo"
|
|
277
|
+
}
|
|
278
|
+
]
|
|
279
|
+
},
|
|
280
|
+
{
|
|
281
|
+
id: "runtime",
|
|
282
|
+
title: "Runtime",
|
|
283
|
+
columns: 2,
|
|
284
|
+
fields: [
|
|
285
|
+
{
|
|
286
|
+
key: "runtime",
|
|
287
|
+
label: "Runtime",
|
|
288
|
+
type: "select",
|
|
289
|
+
options: [
|
|
290
|
+
{ value: "auto", label: "Auto (recommended)" },
|
|
291
|
+
{ value: "onnx", label: "ONNX Runtime" },
|
|
292
|
+
{ value: "coreml", label: "CoreML (Apple)" },
|
|
293
|
+
{ value: "openvino", label: "OpenVINO (Intel)" }
|
|
294
|
+
]
|
|
295
|
+
},
|
|
296
|
+
{
|
|
297
|
+
key: "backend",
|
|
298
|
+
label: "Backend",
|
|
299
|
+
type: "select",
|
|
300
|
+
dependsOn: { runtime: "onnx" },
|
|
301
|
+
options: [
|
|
302
|
+
{ value: "cpu", label: "CPU" },
|
|
303
|
+
{ value: "coreml", label: "CoreML" },
|
|
304
|
+
{ value: "cuda", label: "CUDA (NVIDIA)" },
|
|
305
|
+
{ value: "tensorrt", label: "TensorRT (NVIDIA)" }
|
|
306
|
+
]
|
|
307
|
+
}
|
|
308
|
+
]
|
|
309
|
+
},
|
|
310
|
+
{
|
|
311
|
+
id: "thresholds",
|
|
312
|
+
title: "Detection Thresholds",
|
|
313
|
+
columns: 2,
|
|
314
|
+
fields: [
|
|
315
|
+
{
|
|
316
|
+
key: "confidence",
|
|
317
|
+
label: "Confidence Threshold",
|
|
318
|
+
type: "slider",
|
|
319
|
+
min: 0.1,
|
|
320
|
+
max: 1,
|
|
321
|
+
step: 0.05,
|
|
322
|
+
default: 0.5
|
|
323
|
+
},
|
|
324
|
+
{
|
|
325
|
+
key: "iouThreshold",
|
|
326
|
+
label: "IoU Threshold (NMS)",
|
|
327
|
+
type: "slider",
|
|
328
|
+
min: 0.1,
|
|
329
|
+
max: 1,
|
|
330
|
+
step: 0.05,
|
|
331
|
+
default: 0.45
|
|
332
|
+
}
|
|
333
|
+
]
|
|
334
|
+
}
|
|
335
|
+
]
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
getClassMap() {
|
|
339
|
+
return COCO_TO_MACRO;
|
|
340
|
+
}
|
|
341
|
+
getModelCatalog() {
|
|
342
|
+
return [...ALL_DETECTION_MODELS];
|
|
343
|
+
}
|
|
344
|
+
getAvailableModels() {
|
|
345
|
+
return [];
|
|
346
|
+
}
|
|
347
|
+
getActiveLabels() {
|
|
348
|
+
return MACRO_LABELS;
|
|
349
|
+
}
|
|
350
|
+
async probe() {
|
|
351
|
+
return {
|
|
352
|
+
available: true,
|
|
353
|
+
runtime: this.engine?.runtime ?? "onnx",
|
|
354
|
+
device: this.engine?.device ?? "cpu",
|
|
355
|
+
capabilities: ["fp32"]
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
};
|
|
359
|
+
|
|
360
|
+
export {
|
|
361
|
+
ObjectDetectionAddon
|
|
362
|
+
};
|
|
363
|
+
//# sourceMappingURL=chunk-CGYSSHHM.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/addons/object-detection/index.ts","../src/shared/postprocess/yolo-seg.ts"],"sourcesContent":["import type {\n IDetectorProvider,\n IDetectionAddon,\n AddonManifest,\n AddonContext,\n FrameInput,\n DetectorOutput,\n SpatialDetection,\n ConfigUISchema,\n ClassMapDefinition,\n ProbeResult,\n ModelCatalogEntry,\n DetectionModel,\n LabelDefinition,\n IInferenceEngine,\n} from '@camstack/types'\nimport {\n OBJECT_DETECTION_MODELS,\n SEGMENTATION_MODELS,\n COCO_TO_MACRO,\n MACRO_LABELS,\n} from '@camstack/types'\nimport { letterbox } from '../../shared/image-utils.js'\nimport { yoloPostprocess } from '../../shared/postprocess/yolo.js'\nimport { yoloSegPostprocess } from '../../shared/postprocess/yolo-seg.js'\nimport { resolveEngine } from '../../shared/engine-resolver.js'\n\n/** Returns true when the model id identifies a YOLO segmentation model (e.g. yolov8n-seg, yolo11m-seg) */\nfunction isSegModel(modelId: string): boolean {\n return modelId.includes('-seg')\n}\n\n/** Combined catalog: regular detection models + segmentation models */\nconst ALL_DETECTION_MODELS: readonly ModelCatalogEntry[] = [\n ...OBJECT_DETECTION_MODELS,\n ...SEGMENTATION_MODELS,\n]\n\n/** Map COCO-class detections to macro categories (person/vehicle/animal) */\nfunction applyClassMap(\n detections: SpatialDetection[],\n classMap: ClassMapDefinition,\n): SpatialDetection[] {\n return detections\n .filter((d) => classMap.mapping[d.class] !== undefined)\n .map((d) => ({\n ...d,\n originalClass: d.class,\n class: classMap.mapping[d.class]!,\n }))\n}\n\nexport default class ObjectDetectionAddon implements IDetectorProvider, IDetectionAddon {\n readonly id = 'object-detection'\n readonly slot = 'detector' as const\n readonly inputClasses: readonly string[] | null = null\n readonly outputClasses = ['person', 'vehicle', 'animal'] as const\n readonly slotPriority = 0\n readonly manifest: AddonManifest = {\n id: 'object-detection',\n name: 'Object Detection',\n version: '0.1.0',\n description: 'YOLO-based object detection — detects persons, vehicles, and animals',\n packageName: '@camstack/vision',\n slot: 'detector',\n inputClasses: undefined,\n outputClasses: ['person', 'vehicle', 'animal'],\n supportsCustomModels: true,\n mayRequirePython: false,\n defaultConfig: {\n modelId: 'yolov8n',\n runtime: 'auto',\n backend: 'cpu',\n confidence: 0.5,\n iouThreshold: 0.45,\n },\n }\n\n private engine!: IInferenceEngine\n private modelEntry!: ModelCatalogEntry\n private confidence = 0.5\n private iouThreshold = 0.45\n\n async initialize(ctx: AddonContext): Promise<void> {\n const cfg = ctx.addonConfig\n const modelId = (cfg['modelId'] as string | undefined) ?? 'yolov8n'\n const runtime = (cfg['runtime'] as string | undefined) ?? 'auto'\n const backend = (cfg['backend'] as string | undefined) ?? 'cpu'\n this.confidence = (cfg['confidence'] as number | undefined) ?? 0.5\n this.iouThreshold = (cfg['iouThreshold'] as number | undefined) ?? 0.45\n\n const entry = ALL_DETECTION_MODELS.find((m) => m.id === modelId)\n if (!entry) {\n throw new Error(`ObjectDetectionAddon: unknown modelId \"${modelId}\"`)\n }\n this.modelEntry = entry\n\n const resolved = await resolveEngine({\n runtime: runtime as 'auto',\n backend,\n modelEntry: entry,\n modelsDir: ctx.locationPaths.models,\n })\n this.engine = resolved.engine\n }\n\n async detect(frame: FrameInput): Promise<DetectorOutput> {\n const start = Date.now()\n const { width: inputW, height: inputH } = this.modelEntry.inputSize\n const targetSize = Math.max(inputW, inputH)\n\n const lb = await letterbox(frame.data, targetSize)\n\n const numClasses = this.modelEntry.labels.length\n const labels = this.modelEntry.labels.map((l) => l.id)\n const postprocessOpts = {\n confidence: this.confidence,\n iouThreshold: this.iouThreshold,\n labels,\n scale: lb.scale,\n padX: lb.padX,\n padY: lb.padY,\n originalWidth: lb.originalWidth,\n originalHeight: lb.originalHeight,\n }\n\n let rawDetections: SpatialDetection[]\n\n if (isSegModel(this.modelEntry.id)) {\n // YOLO-seg models produce two outputs:\n // output0: [1, 4 + numClasses + 32, numBoxes] (detection + mask coefficients)\n // output1: [1, 32, 160, 160] (prototype masks)\n const outputs = await this.engine.runMultiOutput(lb.data, [1, 3, targetSize, targetSize])\n const outputNames = Object.keys(outputs)\n\n if (outputNames.length < 2) {\n throw new Error(\n `ObjectDetectionAddon: seg model \"${this.modelEntry.id}\" returned ${outputNames.length} output(s); expected 2`,\n )\n }\n\n // ONNX output order is deterministic; first output is the detection tensor, second is protos\n const detectionOutput = outputs[outputNames[0]!]!\n const protoOutput = outputs[outputNames[1]!]!\n\n // Infer dims from tensor sizes\n const numMaskCoeffs = 32\n const numBoxes = detectionOutput.length / (4 + numClasses + numMaskCoeffs)\n const maskHeight = 160\n const maskWidth = 160\n\n rawDetections = yoloSegPostprocess(\n {\n detectionOutput,\n protoOutput,\n numClasses,\n numBoxes,\n numMaskCoeffs,\n maskHeight,\n maskWidth,\n },\n postprocessOpts,\n )\n } else {\n // Standard YOLO output: [1, 4+numClasses, numBoxes]\n const output = await this.engine.run(lb.data, [1, 3, targetSize, targetSize])\n const numBoxes = output.length / (4 + numClasses)\n rawDetections = yoloPostprocess(output, numClasses, numBoxes, postprocessOpts)\n }\n\n const detections = applyClassMap(rawDetections, COCO_TO_MACRO)\n\n return {\n detections,\n inferenceMs: Date.now() - start,\n modelId: this.modelEntry.id,\n }\n }\n\n async shutdown(): Promise<void> {\n await this.engine?.dispose()\n }\n\n getConfigSchema(): ConfigUISchema {\n return {\n sections: [\n {\n id: 'model',\n title: 'Model',\n columns: 2,\n fields: [\n {\n key: 'modelId',\n label: 'Model',\n type: 'model-selector',\n catalog: [...ALL_DETECTION_MODELS],\n allowCustom: true,\n allowConversion: true,\n acceptFormats: ['onnx', 'coreml', 'openvino', 'tflite'],\n requiredMetadata: ['inputSize', 'labels', 'outputFormat'],\n outputFormatHint: 'yolo',\n },\n ],\n },\n {\n id: 'runtime',\n title: 'Runtime',\n columns: 2,\n fields: [\n {\n key: 'runtime',\n label: 'Runtime',\n type: 'select',\n options: [\n { value: 'auto', label: 'Auto (recommended)' },\n { value: 'onnx', label: 'ONNX Runtime' },\n { value: 'coreml', label: 'CoreML (Apple)' },\n { value: 'openvino', label: 'OpenVINO (Intel)' },\n ],\n },\n {\n key: 'backend',\n label: 'Backend',\n type: 'select',\n dependsOn: { runtime: 'onnx' },\n options: [\n { value: 'cpu', label: 'CPU' },\n { value: 'coreml', label: 'CoreML' },\n { value: 'cuda', label: 'CUDA (NVIDIA)' },\n { value: 'tensorrt', label: 'TensorRT (NVIDIA)' },\n ],\n },\n ],\n },\n {\n id: 'thresholds',\n title: 'Detection Thresholds',\n columns: 2,\n fields: [\n {\n key: 'confidence',\n label: 'Confidence Threshold',\n type: 'slider',\n min: 0.1,\n max: 1.0,\n step: 0.05,\n default: 0.5,\n },\n {\n key: 'iouThreshold',\n label: 'IoU Threshold (NMS)',\n type: 'slider',\n min: 0.1,\n max: 1.0,\n step: 0.05,\n default: 0.45,\n },\n ],\n },\n ],\n }\n }\n\n getClassMap(): ClassMapDefinition {\n return COCO_TO_MACRO\n }\n\n getModelCatalog(): ModelCatalogEntry[] {\n return [...ALL_DETECTION_MODELS]\n }\n\n getAvailableModels(): DetectionModel[] {\n // TODO: check downloaded models in modelsDir\n return []\n }\n\n getActiveLabels(): readonly LabelDefinition[] {\n return MACRO_LABELS\n }\n\n async probe(): Promise<ProbeResult> {\n return {\n available: true,\n runtime: this.engine?.runtime ?? 'onnx',\n device: this.engine?.device ?? 'cpu',\n capabilities: ['fp32'],\n }\n }\n}\n","import type { SpatialDetection, BoundingBox } from '@camstack/types'\nimport { nms, iou as _iou } from './yolo.js'\nimport type { YoloPostprocessOptions } from './yolo.js'\n\nexport interface YoloSegOutput {\n /** Detection output: [1, 4 + numClasses + numMaskCoeffs, numBoxes] (row-major, batch dim stripped) */\n readonly detectionOutput: Float32Array\n /** Prototype masks: [1, numMaskCoeffs, maskH, maskW] (batch dim stripped) */\n readonly protoOutput: Float32Array\n readonly numClasses: number\n readonly numBoxes: number\n /** Number of mask prototype coefficients per detection (typically 32) */\n readonly numMaskCoeffs: number\n /** Height of prototype mask grid (typically 160) */\n readonly maskHeight: number\n /** Width of prototype mask grid (typically 160) */\n readonly maskWidth: number\n}\n\nexport interface YoloSegPostprocessOptions extends YoloPostprocessOptions {\n /** Binary mask threshold — pixels above this value become 255, others 0 (default: 0.5) */\n readonly maskThreshold?: number\n}\n\n/** Sigmoid activation: σ(x) = 1 / (1 + e^-x) */\nexport function sigmoid(x: number): number {\n return 1 / (1 + Math.exp(-x))\n}\n\n/**\n * Compute instance segmentation mask for a single detection.\n *\n * @param coeffs [numMaskCoeffs] mask coefficients for this detection\n * @param protos [numMaskCoeffs × maskH × maskW] prototype masks (flat, row-major)\n * @param numMaskCoeffs number of prototype channels (e.g. 32)\n * @param maskH prototype grid height (e.g. 160)\n * @param maskW prototype grid width (e.g. 160)\n * @returns [maskH × maskW] sigmoid-activated raw mask values in [0, 1]\n */\nexport function computeRawMask(\n coeffs: Float32Array,\n protos: Float32Array,\n numMaskCoeffs: number,\n maskH: number,\n maskW: number,\n): Float32Array {\n const maskSize = maskH * maskW\n const rawMask = new Float32Array(maskSize)\n\n // mask_raw[y * maskW + x] = sum_k( coeffs[k] * protos[k * maskSize + y * maskW + x] )\n // Then apply sigmoid element-wise.\n for (let px = 0; px < maskSize; px++) {\n let val = 0\n for (let k = 0; k < numMaskCoeffs; k++) {\n val += (coeffs[k] ?? 0) * (protos[k * maskSize + px] ?? 0)\n }\n rawMask[px] = sigmoid(val)\n }\n\n return rawMask\n}\n\n/**\n * Crop a flat [maskH × maskW] mask to a bounding box region (in mask coordinate space),\n * threshold to binary, and return the cropped Uint8Array along with its dimensions.\n *\n * @param rawMask [maskH × maskW] sigmoid-activated values in [0, 1]\n * @param maskH full mask grid height\n * @param maskW full mask grid width\n * @param bbox detection bounding box in the inference input space (e.g. 640×640 letterboxed)\n * @param maskThreshold threshold above which a pixel is considered foreground (default 0.5)\n * @param maskScale ratio of mask resolution to inference resolution (e.g. 160/640 = 0.25)\n * @returns { data: Uint8Array, width, height } of the cropped binary mask\n */\nexport function cropAndThresholdMask(\n rawMask: Float32Array,\n maskH: number,\n maskW: number,\n bbox: BoundingBox,\n maskThreshold: number,\n maskScale: number,\n): { readonly data: Uint8Array; readonly width: number; readonly height: number } {\n const cropX1 = Math.max(0, Math.floor(bbox.x * maskScale))\n const cropY1 = Math.max(0, Math.floor(bbox.y * maskScale))\n const cropX2 = Math.min(maskW, Math.ceil((bbox.x + bbox.w) * maskScale))\n const cropY2 = Math.min(maskH, Math.ceil((bbox.y + bbox.h) * maskScale))\n\n const cropW = Math.max(1, cropX2 - cropX1)\n const cropH = Math.max(1, cropY2 - cropY1)\n\n const data = new Uint8Array(cropW * cropH)\n\n for (let row = 0; row < cropH; row++) {\n const srcRow = cropY1 + row\n for (let col = 0; col < cropW; col++) {\n const srcCol = cropX1 + col\n const srcIdx = srcRow * maskW + srcCol\n data[row * cropW + col] = (rawMask[srcIdx] ?? 0) > maskThreshold ? 255 : 0\n }\n }\n\n return { data, width: cropW, height: cropH }\n}\n\n/**\n * YOLO-seg postprocessing: run detection filtering + NMS, then decode instance masks.\n *\n * The seg detection tensor has layout [1, 4 + numClasses + numMaskCoeffs, numBoxes]:\n * rows 0–3 : cx, cy, w, h\n * rows 4 to 4+C-1 : class scores\n * rows 4+C to end : mask coefficients (32)\n *\n * Steps:\n * 1. Filter boxes by confidence threshold, collecting mask coefficients per candidate.\n * 2. Run NMS across all candidates.\n * 3. For each surviving detection, compute the instance mask and attach it.\n */\nexport function yoloSegPostprocess(\n segOutput: YoloSegOutput,\n options: YoloSegPostprocessOptions,\n): SpatialDetection[] {\n const {\n detectionOutput,\n protoOutput,\n numClasses,\n numBoxes,\n numMaskCoeffs,\n maskHeight,\n maskWidth,\n } = segOutput\n\n const {\n confidence,\n iouThreshold,\n labels,\n scale,\n padX,\n padY,\n originalWidth,\n originalHeight,\n maskThreshold = 0.5,\n } = options\n\n // The mask resolution to inference resolution scale factor (e.g. 160/640 = 0.25)\n // We derive the inference size from scale/pad context: the letterboxed input is typically 640×640\n // but we receive coordinates in that space already, so we use maskHeight / inferH.\n // Since all YOLO-seg models use 640×640 input and 160×160 proto, the ratio is always 160/640.\n // We compute it generically from the known prototype dims + fixed YOLO input of 640.\n const yoloInputSize = 640\n const maskScale = maskHeight / yoloInputSize\n\n interface Candidate {\n readonly bbox: BoundingBox\n readonly score: number\n readonly classIdx: number\n readonly coeffs: Float32Array\n }\n\n const candidates: Candidate[] = []\n\n for (let i = 0; i < numBoxes; i++) {\n const cx = detectionOutput[0 * numBoxes + i] ?? 0\n const cy = detectionOutput[1 * numBoxes + i] ?? 0\n const w = detectionOutput[2 * numBoxes + i] ?? 0\n const h = detectionOutput[3 * numBoxes + i] ?? 0\n\n let bestScore = -Infinity\n let bestClass = 0\n\n for (let j = 0; j < numClasses; j++) {\n const score = detectionOutput[(4 + j) * numBoxes + i] ?? 0\n if (score > bestScore) {\n bestScore = score\n bestClass = j\n }\n }\n\n if (bestScore < confidence) continue\n\n const bbox: BoundingBox = {\n x: cx - w / 2,\n y: cy - h / 2,\n w,\n h,\n }\n\n // Extract mask coefficients for this box\n const coeffs = new Float32Array(numMaskCoeffs)\n for (let k = 0; k < numMaskCoeffs; k++) {\n coeffs[k] = detectionOutput[(4 + numClasses + k) * numBoxes + i] ?? 0\n }\n\n candidates.push({ bbox, score: bestScore, classIdx: bestClass, coeffs })\n }\n\n if (candidates.length === 0) return []\n\n const keptIndices = nms(candidates, iouThreshold)\n\n return keptIndices.map((idx) => {\n const { bbox, score, classIdx, coeffs } = candidates[idx]!\n const label = labels[classIdx] ?? String(classIdx)\n\n // Scale bbox back from letterbox space to original image coords\n const x = Math.max(0, Math.min(originalWidth, (bbox.x - padX) / scale))\n const y = Math.max(0, Math.min(originalHeight, (bbox.y - padY) / scale))\n const x2 = Math.max(0, Math.min(originalWidth, (bbox.x + bbox.w - padX) / scale))\n const y2 = Math.max(0, Math.min(originalHeight, (bbox.y + bbox.h - padY) / scale))\n const finalBbox: BoundingBox = { x, y, w: x2 - x, h: y2 - y }\n\n // Decode instance mask using the letterbox-space bbox (before coord scaling)\n const rawMask = computeRawMask(coeffs, protoOutput, numMaskCoeffs, maskHeight, maskWidth)\n const { data: maskData, width: mW, height: mH } = cropAndThresholdMask(\n rawMask,\n maskHeight,\n maskWidth,\n bbox,\n maskThreshold,\n maskScale,\n )\n\n return {\n class: label,\n originalClass: label,\n score,\n bbox: finalBbox,\n mask: maskData,\n maskWidth: mW,\n maskHeight: mH,\n } satisfies SpatialDetection\n })\n}\n"],"mappings":";;;;;;;;;;;;AAgBA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;;;ACIA,SAAS,QAAQ,GAAmB;AACzC,SAAO,KAAK,IAAI,KAAK,IAAI,CAAC,CAAC;AAC7B;AAYO,SAAS,eACd,QACA,QACA,eACA,OACA,OACc;AACd,QAAM,WAAW,QAAQ;AACzB,QAAM,UAAU,IAAI,aAAa,QAAQ;AAIzC,WAAS,KAAK,GAAG,KAAK,UAAU,MAAM;AACpC,QAAI,MAAM;AACV,aAAS,IAAI,GAAG,IAAI,eAAe,KAAK;AACtC,cAAQ,OAAO,CAAC,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,KAAK;AAAA,IAC1D;AACA,YAAQ,EAAE,IAAI,QAAQ,GAAG;AAAA,EAC3B;AAEA,SAAO;AACT;AAcO,SAAS,qBACd,SACA,OACA,OACA,MACA,eACA,WACgF;AAChF,QAAM,SAAS,KAAK,IAAI,GAAG,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC;AACzD,QAAM,SAAS,KAAK,IAAI,GAAG,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC;AACzD,QAAM,SAAS,KAAK,IAAI,OAAO,KAAK,MAAM,KAAK,IAAI,KAAK,KAAK,SAAS,CAAC;AACvE,QAAM,SAAS,KAAK,IAAI,OAAO,KAAK,MAAM,KAAK,IAAI,KAAK,KAAK,SAAS,CAAC;AAEvE,QAAM,QAAQ,KAAK,IAAI,GAAG,SAAS,MAAM;AACzC,QAAM,QAAQ,KAAK,IAAI,GAAG,SAAS,MAAM;AAEzC,QAAM,OAAO,IAAI,WAAW,QAAQ,KAAK;AAEzC,WAAS,MAAM,GAAG,MAAM,OAAO,OAAO;AACpC,UAAM,SAAS,SAAS;AACxB,aAAS,MAAM,GAAG,MAAM,OAAO,OAAO;AACpC,YAAM,SAAS,SAAS;AACxB,YAAM,SAAS,SAAS,QAAQ;AAChC,WAAK,MAAM,QAAQ,GAAG,KAAK,QAAQ,MAAM,KAAK,KAAK,gBAAgB,MAAM;AAAA,IAC3E;AAAA,EACF;AAEA,SAAO,EAAE,MAAM,OAAO,OAAO,QAAQ,MAAM;AAC7C;AAeO,SAAS,mBACd,WACA,SACoB;AACpB,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,IAAI;AAEJ,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,gBAAgB;AAAA,EAClB,IAAI;AAOJ,QAAM,gBAAgB;AACtB,QAAM,YAAY,aAAa;AAS/B,QAAM,aAA0B,CAAC;AAEjC,WAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AACjC,UAAM,KAAK,gBAAgB,IAAI,WAAW,CAAC,KAAK;AAChD,UAAM,KAAK,gBAAgB,IAAI,WAAW,CAAC,KAAK;AAChD,UAAM,IAAI,gBAAgB,IAAI,WAAW,CAAC,KAAK;AAC/C,UAAM,IAAI,gBAAgB,IAAI,WAAW,CAAC,KAAK;AAE/C,QAAI,YAAY;AAChB,QAAI,YAAY;AAEhB,aAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,YAAM,QAAQ,iBAAiB,IAAI,KAAK,WAAW,CAAC,KAAK;AACzD,UAAI,QAAQ,WAAW;AACrB,oBAAY;AACZ,oBAAY;AAAA,MACd;AAAA,IACF;AAEA,QAAI,YAAY,WAAY;AAE5B,UAAM,OAAoB;AAAA,MACxB,GAAG,KAAK,IAAI;AAAA,MACZ,GAAG,KAAK,IAAI;AAAA,MACZ;AAAA,MACA;AAAA,IACF;AAGA,UAAM,SAAS,IAAI,aAAa,aAAa;AAC7C,aAAS,IAAI,GAAG,IAAI,eAAe,KAAK;AACtC,aAAO,CAAC,IAAI,iBAAiB,IAAI,aAAa,KAAK,WAAW,CAAC,KAAK;AAAA,IACtE;AAEA,eAAW,KAAK,EAAE,MAAM,OAAO,WAAW,UAAU,WAAW,OAAO,CAAC;AAAA,EACzE;AAEA,MAAI,WAAW,WAAW,EAAG,QAAO,CAAC;AAErC,QAAM,cAAc,IAAI,YAAY,YAAY;AAEhD,SAAO,YAAY,IAAI,CAAC,QAAQ;AAC9B,UAAM,EAAE,MAAM,OAAO,UAAU,OAAO,IAAI,WAAW,GAAG;AACxD,UAAM,QAAQ,OAAO,QAAQ,KAAK,OAAO,QAAQ;AAGjD,UAAM,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,gBAAgB,KAAK,IAAI,QAAQ,KAAK,CAAC;AACtE,UAAM,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,iBAAiB,KAAK,IAAI,QAAQ,KAAK,CAAC;AACvE,UAAM,KAAK,KAAK,IAAI,GAAG,KAAK,IAAI,gBAAgB,KAAK,IAAI,KAAK,IAAI,QAAQ,KAAK,CAAC;AAChF,UAAM,KAAK,KAAK,IAAI,GAAG,KAAK,IAAI,iBAAiB,KAAK,IAAI,KAAK,IAAI,QAAQ,KAAK,CAAC;AACjF,UAAM,YAAyB,EAAE,GAAG,GAAG,GAAG,KAAK,GAAG,GAAG,KAAK,EAAE;AAG5D,UAAM,UAAU,eAAe,QAAQ,aAAa,eAAe,YAAY,SAAS;AACxF,UAAM,EAAE,MAAM,UAAU,OAAO,IAAI,QAAQ,GAAG,IAAI;AAAA,MAChD;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,WAAO;AAAA,MACL,OAAO;AAAA,MACP,eAAe;AAAA,MACf;AAAA,MACA,MAAM;AAAA,MACN,MAAM;AAAA,MACN,WAAW;AAAA,MACX,YAAY;AAAA,IACd;AAAA,EACF,CAAC;AACH;;;AD3MA,SAAS,WAAW,SAA0B;AAC5C,SAAO,QAAQ,SAAS,MAAM;AAChC;AAGA,IAAM,uBAAqD;AAAA,EACzD,GAAG;AAAA,EACH,GAAG;AACL;AAGA,SAAS,cACP,YACA,UACoB;AACpB,SAAO,WACJ,OAAO,CAAC,MAAM,SAAS,QAAQ,EAAE,KAAK,MAAM,MAAS,EACrD,IAAI,CAAC,OAAO;AAAA,IACX,GAAG;AAAA,IACH,eAAe,EAAE;AAAA,IACjB,OAAO,SAAS,QAAQ,EAAE,KAAK;AAAA,EACjC,EAAE;AACN;AAEA,IAAqB,uBAArB,MAAwF;AAAA,EAC7E,KAAK;AAAA,EACL,OAAO;AAAA,EACP,eAAyC;AAAA,EACzC,gBAAgB,CAAC,UAAU,WAAW,QAAQ;AAAA,EAC9C,eAAe;AAAA,EACf,WAA0B;AAAA,IACjC,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,SAAS;AAAA,IACT,aAAa;AAAA,IACb,aAAa;AAAA,IACb,MAAM;AAAA,IACN,cAAc;AAAA,IACd,eAAe,CAAC,UAAU,WAAW,QAAQ;AAAA,IAC7C,sBAAsB;AAAA,IACtB,kBAAkB;AAAA,IAClB,eAAe;AAAA,MACb,SAAS;AAAA,MACT,SAAS;AAAA,MACT,SAAS;AAAA,MACT,YAAY;AAAA,MACZ,cAAc;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ;AAAA,EACA;AAAA,EACA,aAAa;AAAA,EACb,eAAe;AAAA,EAEvB,MAAM,WAAW,KAAkC;AACjD,UAAM,MAAM,IAAI;AAChB,UAAM,UAAW,IAAI,SAAS,KAA4B;AAC1D,UAAM,UAAW,IAAI,SAAS,KAA4B;AAC1D,UAAM,UAAW,IAAI,SAAS,KAA4B;AAC1D,SAAK,aAAc,IAAI,YAAY,KAA4B;AAC/D,SAAK,eAAgB,IAAI,cAAc,KAA4B;AAEnE,UAAM,QAAQ,qBAAqB,KAAK,CAAC,MAAM,EAAE,OAAO,OAAO;AAC/D,QAAI,CAAC,OAAO;AACV,YAAM,IAAI,MAAM,0CAA0C,OAAO,GAAG;AAAA,IACtE;AACA,SAAK,aAAa;AAElB,UAAM,WAAW,MAAM,cAAc;AAAA,MACnC;AAAA,MACA;AAAA,MACA,YAAY;AAAA,MACZ,WAAW,IAAI,cAAc;AAAA,IAC/B,CAAC;AACD,SAAK,SAAS,SAAS;AAAA,EACzB;AAAA,EAEA,MAAM,OAAO,OAA4C;AACvD,UAAM,QAAQ,KAAK,IAAI;AACvB,UAAM,EAAE,OAAO,QAAQ,QAAQ,OAAO,IAAI,KAAK,WAAW;AAC1D,UAAM,aAAa,KAAK,IAAI,QAAQ,MAAM;AAE1C,UAAM,KAAK,MAAM,UAAU,MAAM,MAAM,UAAU;AAEjD,UAAM,aAAa,KAAK,WAAW,OAAO;AAC1C,UAAM,SAAS,KAAK,WAAW,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE;AACrD,UAAM,kBAAkB;AAAA,MACtB,YAAY,KAAK;AAAA,MACjB,cAAc,KAAK;AAAA,MACnB;AAAA,MACA,OAAO,GAAG;AAAA,MACV,MAAM,GAAG;AAAA,MACT,MAAM,GAAG;AAAA,MACT,eAAe,GAAG;AAAA,MAClB,gBAAgB,GAAG;AAAA,IACrB;AAEA,QAAI;AAEJ,QAAI,WAAW,KAAK,WAAW,EAAE,GAAG;AAIlC,YAAM,UAAU,MAAM,KAAK,OAAO,eAAe,GAAG,MAAM,CAAC,GAAG,GAAG,YAAY,UAAU,CAAC;AACxF,YAAM,cAAc,OAAO,KAAK,OAAO;AAEvC,UAAI,YAAY,SAAS,GAAG;AAC1B,cAAM,IAAI;AAAA,UACR,oCAAoC,KAAK,WAAW,EAAE,cAAc,YAAY,MAAM;AAAA,QACxF;AAAA,MACF;AAGA,YAAM,kBAAkB,QAAQ,YAAY,CAAC,CAAE;AAC/C,YAAM,cAAc,QAAQ,YAAY,CAAC,CAAE;AAG3C,YAAM,gBAAgB;AACtB,YAAM,WAAW,gBAAgB,UAAU,IAAI,aAAa;AAC5D,YAAM,aAAa;AACnB,YAAM,YAAY;AAElB,sBAAgB;AAAA,QACd;AAAA,UACE;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACF;AAAA,QACA;AAAA,MACF;AAAA,IACF,OAAO;AAEL,YAAM,SAAS,MAAM,KAAK,OAAO,IAAI,GAAG,MAAM,CAAC,GAAG,GAAG,YAAY,UAAU,CAAC;AAC5E,YAAM,WAAW,OAAO,UAAU,IAAI;AACtC,sBAAgB,gBAAgB,QAAQ,YAAY,UAAU,eAAe;AAAA,IAC/E;AAEA,UAAM,aAAa,cAAc,eAAe,aAAa;AAE7D,WAAO;AAAA,MACL;AAAA,MACA,aAAa,KAAK,IAAI,IAAI;AAAA,MAC1B,SAAS,KAAK,WAAW;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,MAAM,WAA0B;AAC9B,UAAM,KAAK,QAAQ,QAAQ;AAAA,EAC7B;AAAA,EAEA,kBAAkC;AAChC,WAAO;AAAA,MACL,UAAU;AAAA,QACR;AAAA,UACE,IAAI;AAAA,UACJ,OAAO;AAAA,UACP,SAAS;AAAA,UACT,QAAQ;AAAA,YACN;AAAA,cACE,KAAK;AAAA,cACL,OAAO;AAAA,cACP,MAAM;AAAA,cACN,SAAS,CAAC,GAAG,oBAAoB;AAAA,cACjC,aAAa;AAAA,cACb,iBAAiB;AAAA,cACjB,eAAe,CAAC,QAAQ,UAAU,YAAY,QAAQ;AAAA,cACtD,kBAAkB,CAAC,aAAa,UAAU,cAAc;AAAA,cACxD,kBAAkB;AAAA,YACpB;AAAA,UACF;AAAA,QACF;AAAA,QACA;AAAA,UACE,IAAI;AAAA,UACJ,OAAO;AAAA,UACP,SAAS;AAAA,UACT,QAAQ;AAAA,YACN;AAAA,cACE,KAAK;AAAA,cACL,OAAO;AAAA,cACP,MAAM;AAAA,cACN,SAAS;AAAA,gBACP,EAAE,OAAO,QAAQ,OAAO,qBAAqB;AAAA,gBAC7C,EAAE,OAAO,QAAQ,OAAO,eAAe;AAAA,gBACvC,EAAE,OAAO,UAAU,OAAO,iBAAiB;AAAA,gBAC3C,EAAE,OAAO,YAAY,OAAO,mBAAmB;AAAA,cACjD;AAAA,YACF;AAAA,YACA;AAAA,cACE,KAAK;AAAA,cACL,OAAO;AAAA,cACP,MAAM;AAAA,cACN,WAAW,EAAE,SAAS,OAAO;AAAA,cAC7B,SAAS;AAAA,gBACP,EAAE,OAAO,OAAO,OAAO,MAAM;AAAA,gBAC7B,EAAE,OAAO,UAAU,OAAO,SAAS;AAAA,gBACnC,EAAE,OAAO,QAAQ,OAAO,gBAAgB;AAAA,gBACxC,EAAE,OAAO,YAAY,OAAO,oBAAoB;AAAA,cAClD;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,QACA;AAAA,UACE,IAAI;AAAA,UACJ,OAAO;AAAA,UACP,SAAS;AAAA,UACT,QAAQ;AAAA,YACN;AAAA,cACE,KAAK;AAAA,cACL,OAAO;AAAA,cACP,MAAM;AAAA,cACN,KAAK;AAAA,cACL,KAAK;AAAA,cACL,MAAM;AAAA,cACN,SAAS;AAAA,YACX;AAAA,YACA;AAAA,cACE,KAAK;AAAA,cACL,OAAO;AAAA,cACP,MAAM;AAAA,cACN,KAAK;AAAA,cACL,KAAK;AAAA,cACL,MAAM;AAAA,cACN,SAAS;AAAA,YACX;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,cAAkC;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,kBAAuC;AACrC,WAAO,CAAC,GAAG,oBAAoB;AAAA,EACjC;AAAA,EAEA,qBAAuC;AAErC,WAAO,CAAC;AAAA,EACV;AAAA,EAEA,kBAA8C;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,QAA8B;AAClC,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS,KAAK,QAAQ,WAAW;AAAA,MACjC,QAAQ,KAAK,QAAQ,UAAU;AAAA,MAC/B,cAAc,CAAC,MAAM;AAAA,IACvB;AAAA,EACF;AACF;","names":[]}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// src/addons/camera-native-detection/index.ts
|
|
2
|
+
var NATIVE_LABELS = [
|
|
3
|
+
{ id: "person", name: "Person" },
|
|
4
|
+
{ id: "vehicle", name: "Vehicle" },
|
|
5
|
+
{ id: "motion", name: "Motion" },
|
|
6
|
+
{ id: "face", name: "Face" }
|
|
7
|
+
];
|
|
8
|
+
var NATIVE_CLASS_MAP = { mapping: {}, preserveOriginal: true };
|
|
9
|
+
var CameraNativeDetectionAddon = class {
|
|
10
|
+
id = "camera-native-detection";
|
|
11
|
+
slot = "detector";
|
|
12
|
+
inputClasses = null;
|
|
13
|
+
outputClasses = ["person", "vehicle", "motion", "face"];
|
|
14
|
+
slotPriority = 5;
|
|
15
|
+
manifest = {
|
|
16
|
+
id: "camera-native-detection",
|
|
17
|
+
name: "Camera Native Detection",
|
|
18
|
+
version: "0.1.0",
|
|
19
|
+
description: "Passthrough adapter for camera-native events (Frigate, Scrypted, ONVIF) \u2014 no inference engine",
|
|
20
|
+
packageName: "@camstack/vision",
|
|
21
|
+
slot: "detector",
|
|
22
|
+
inputClasses: void 0,
|
|
23
|
+
outputClasses: ["person", "vehicle", "motion", "face"],
|
|
24
|
+
supportsCustomModels: false,
|
|
25
|
+
mayRequirePython: false,
|
|
26
|
+
defaultConfig: {}
|
|
27
|
+
};
|
|
28
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
29
|
+
async initialize(_ctx) {
|
|
30
|
+
}
|
|
31
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
32
|
+
async detect(_frame) {
|
|
33
|
+
return {
|
|
34
|
+
detections: [],
|
|
35
|
+
inferenceMs: 0,
|
|
36
|
+
modelId: "camera-native"
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
async shutdown() {
|
|
40
|
+
}
|
|
41
|
+
getConfigSchema() {
|
|
42
|
+
return {
|
|
43
|
+
sections: [
|
|
44
|
+
{
|
|
45
|
+
id: "info",
|
|
46
|
+
title: "Camera Native Detection",
|
|
47
|
+
description: "This addon forwards detections from native camera events (Frigate webhooks, Scrypted push notifications, ONVIF events). No configuration required.",
|
|
48
|
+
fields: []
|
|
49
|
+
}
|
|
50
|
+
]
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
getClassMap() {
|
|
54
|
+
return NATIVE_CLASS_MAP;
|
|
55
|
+
}
|
|
56
|
+
getModelCatalog() {
|
|
57
|
+
return [];
|
|
58
|
+
}
|
|
59
|
+
getAvailableModels() {
|
|
60
|
+
return [];
|
|
61
|
+
}
|
|
62
|
+
getActiveLabels() {
|
|
63
|
+
return NATIVE_LABELS;
|
|
64
|
+
}
|
|
65
|
+
async probe() {
|
|
66
|
+
return {
|
|
67
|
+
available: true,
|
|
68
|
+
runtime: "onnx",
|
|
69
|
+
// no runtime used; satisfies the type
|
|
70
|
+
device: "cpu",
|
|
71
|
+
capabilities: ["fp32"]
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
export {
|
|
77
|
+
CameraNativeDetectionAddon
|
|
78
|
+
};
|
|
79
|
+
//# sourceMappingURL=chunk-IYHMGYGP.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/addons/camera-native-detection/index.ts"],"sourcesContent":["import type {\n IDetectorProvider,\n IDetectionAddon,\n AddonManifest,\n AddonContext,\n FrameInput,\n DetectorOutput,\n ConfigUISchema,\n ClassMapDefinition,\n ProbeResult,\n ModelCatalogEntry,\n DetectionModel,\n LabelDefinition,\n} from '@camstack/types'\n\n// Labels reported by cameras natively (a superset across manufacturers)\nconst NATIVE_LABELS: readonly LabelDefinition[] = [\n { id: 'person', name: 'Person' },\n { id: 'vehicle', name: 'Vehicle' },\n { id: 'motion', name: 'Motion' },\n { id: 'face', name: 'Face' },\n] as const\n\nconst NATIVE_CLASS_MAP: ClassMapDefinition = { mapping: {}, preserveOriginal: true }\n\n/**\n * CameraNativeDetectionAddon\n *\n * A stub detector that wraps native camera event subscriptions (e.g. Frigate / Scrypted\n * webhooks). The detect() method always returns empty — real detections are delivered\n * asynchronously via external event subscriptions which should populate the pipeline\n * from outside this addon's detect() call.\n *\n * This addon exists so the pipeline can declare a 'detector' slot backed by camera events\n * without requiring any inference model.\n */\nexport default class CameraNativeDetectionAddon implements IDetectorProvider, IDetectionAddon {\n readonly id = 'camera-native-detection'\n readonly slot = 'detector' as const\n readonly inputClasses: readonly string[] | null = null\n readonly outputClasses = ['person', 'vehicle', 'motion', 'face'] as const\n readonly slotPriority = 5\n readonly manifest: AddonManifest = {\n id: 'camera-native-detection',\n name: 'Camera Native Detection',\n version: '0.1.0',\n description:\n 'Passthrough adapter for camera-native events (Frigate, Scrypted, ONVIF) — no inference engine',\n packageName: '@camstack/vision',\n slot: 'detector',\n inputClasses: undefined,\n outputClasses: ['person', 'vehicle', 'motion', 'face'],\n supportsCustomModels: false,\n mayRequirePython: false,\n defaultConfig: {},\n }\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n async initialize(_ctx: AddonContext): Promise<void> {\n // No engine to initialize — events come from external subscriptions\n }\n\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n async detect(_frame: FrameInput): Promise<DetectorOutput> {\n // Real detections are delivered via event subscriptions, not through detect().\n // This method intentionally returns empty.\n return {\n detections: [],\n inferenceMs: 0,\n modelId: 'camera-native',\n }\n }\n\n async shutdown(): Promise<void> {\n // Nothing to tear down\n }\n\n getConfigSchema(): ConfigUISchema {\n return {\n sections: [\n {\n id: 'info',\n title: 'Camera Native Detection',\n description:\n 'This addon forwards detections from native camera events (Frigate webhooks, ' +\n 'Scrypted push notifications, ONVIF events). No configuration required.',\n fields: [],\n },\n ],\n }\n }\n\n getClassMap(): ClassMapDefinition {\n return NATIVE_CLASS_MAP\n }\n\n getModelCatalog(): ModelCatalogEntry[] {\n return []\n }\n\n getAvailableModels(): DetectionModel[] {\n return []\n }\n\n getActiveLabels(): readonly LabelDefinition[] {\n return NATIVE_LABELS\n }\n\n async probe(): Promise<ProbeResult> {\n return {\n available: true,\n runtime: 'onnx', // no runtime used; satisfies the type\n device: 'cpu',\n capabilities: ['fp32'],\n }\n }\n}\n"],"mappings":";AAgBA,IAAM,gBAA4C;AAAA,EAChD,EAAE,IAAI,UAAU,MAAM,SAAS;AAAA,EAC/B,EAAE,IAAI,WAAW,MAAM,UAAU;AAAA,EACjC,EAAE,IAAI,UAAU,MAAM,SAAS;AAAA,EAC/B,EAAE,IAAI,QAAQ,MAAM,OAAO;AAC7B;AAEA,IAAM,mBAAuC,EAAE,SAAS,CAAC,GAAG,kBAAkB,KAAK;AAanF,IAAqB,6BAArB,MAA8F;AAAA,EACnF,KAAK;AAAA,EACL,OAAO;AAAA,EACP,eAAyC;AAAA,EACzC,gBAAgB,CAAC,UAAU,WAAW,UAAU,MAAM;AAAA,EACtD,eAAe;AAAA,EACf,WAA0B;AAAA,IACjC,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,SAAS;AAAA,IACT,aACE;AAAA,IACF,aAAa;AAAA,IACb,MAAM;AAAA,IACN,cAAc;AAAA,IACd,eAAe,CAAC,UAAU,WAAW,UAAU,MAAM;AAAA,IACrD,sBAAsB;AAAA,IACtB,kBAAkB;AAAA,IAClB,eAAe,CAAC;AAAA,EAClB;AAAA;AAAA,EAGA,MAAM,WAAW,MAAmC;AAAA,EAEpD;AAAA;AAAA,EAGA,MAAM,OAAO,QAA6C;AAGxD,WAAO;AAAA,MACL,YAAY,CAAC;AAAA,MACb,aAAa;AAAA,MACb,SAAS;AAAA,IACX;AAAA,EACF;AAAA,EAEA,MAAM,WAA0B;AAAA,EAEhC;AAAA,EAEA,kBAAkC;AAChC,WAAO;AAAA,MACL,UAAU;AAAA,QACR;AAAA,UACE,IAAI;AAAA,UACJ,OAAO;AAAA,UACP,aACE;AAAA,UAEF,QAAQ,CAAC;AAAA,QACX;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,cAAkC;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,kBAAuC;AACrC,WAAO,CAAC;AAAA,EACV;AAAA,EAEA,qBAAuC;AACrC,WAAO,CAAC;AAAA,EACV;AAAA,EAEA,kBAA8C;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,QAA8B;AAClC,WAAO;AAAA,MACL,WAAW;AAAA,MACX,SAAS;AAAA;AAAA,MACT,QAAQ;AAAA,MACR,cAAc,CAAC,MAAM;AAAA,IACvB;AAAA,EACF;AACF;","names":[]}
|