@camstack/addon-vision 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/dist/addons/animal-classifier/index.d.mts +25 -0
  2. package/dist/addons/animal-classifier/index.d.ts +25 -0
  3. package/dist/addons/animal-classifier/index.js +652 -0
  4. package/dist/addons/animal-classifier/index.js.map +1 -0
  5. package/dist/addons/animal-classifier/index.mjs +10 -0
  6. package/dist/addons/animal-classifier/index.mjs.map +1 -0
  7. package/dist/addons/audio-classification/index.d.mts +31 -0
  8. package/dist/addons/audio-classification/index.d.ts +31 -0
  9. package/dist/addons/audio-classification/index.js +572 -0
  10. package/dist/addons/audio-classification/index.js.map +1 -0
  11. package/dist/addons/audio-classification/index.mjs +8 -0
  12. package/dist/addons/audio-classification/index.mjs.map +1 -0
  13. package/dist/addons/bird-global-classifier/index.d.mts +26 -0
  14. package/dist/addons/bird-global-classifier/index.d.ts +26 -0
  15. package/dist/addons/bird-global-classifier/index.js +658 -0
  16. package/dist/addons/bird-global-classifier/index.js.map +1 -0
  17. package/dist/addons/bird-global-classifier/index.mjs +10 -0
  18. package/dist/addons/bird-global-classifier/index.mjs.map +1 -0
  19. package/dist/addons/bird-nabirds-classifier/index.d.mts +28 -0
  20. package/dist/addons/bird-nabirds-classifier/index.d.ts +28 -0
  21. package/dist/addons/bird-nabirds-classifier/index.js +700 -0
  22. package/dist/addons/bird-nabirds-classifier/index.js.map +1 -0
  23. package/dist/addons/bird-nabirds-classifier/index.mjs +10 -0
  24. package/dist/addons/bird-nabirds-classifier/index.mjs.map +1 -0
  25. package/dist/addons/camera-native-detection/index.d.mts +32 -0
  26. package/dist/addons/camera-native-detection/index.d.ts +32 -0
  27. package/dist/addons/camera-native-detection/index.js +99 -0
  28. package/dist/addons/camera-native-detection/index.js.map +1 -0
  29. package/dist/addons/camera-native-detection/index.mjs +7 -0
  30. package/dist/addons/camera-native-detection/index.mjs.map +1 -0
  31. package/dist/addons/face-detection/index.d.mts +24 -0
  32. package/dist/addons/face-detection/index.d.ts +24 -0
  33. package/dist/addons/face-detection/index.js +720 -0
  34. package/dist/addons/face-detection/index.js.map +1 -0
  35. package/dist/addons/face-detection/index.mjs +10 -0
  36. package/dist/addons/face-detection/index.mjs.map +1 -0
  37. package/dist/addons/face-recognition/index.d.mts +24 -0
  38. package/dist/addons/face-recognition/index.d.ts +24 -0
  39. package/dist/addons/face-recognition/index.js +603 -0
  40. package/dist/addons/face-recognition/index.js.map +1 -0
  41. package/dist/addons/face-recognition/index.mjs +9 -0
  42. package/dist/addons/face-recognition/index.mjs.map +1 -0
  43. package/dist/addons/motion-detection/index.d.mts +26 -0
  44. package/dist/addons/motion-detection/index.d.ts +26 -0
  45. package/dist/addons/motion-detection/index.js +273 -0
  46. package/dist/addons/motion-detection/index.js.map +1 -0
  47. package/dist/addons/motion-detection/index.mjs +8 -0
  48. package/dist/addons/motion-detection/index.mjs.map +1 -0
  49. package/dist/addons/object-detection/index.d.mts +26 -0
  50. package/dist/addons/object-detection/index.d.ts +26 -0
  51. package/dist/addons/object-detection/index.js +1214 -0
  52. package/dist/addons/object-detection/index.js.map +1 -0
  53. package/dist/addons/object-detection/index.mjs +10 -0
  54. package/dist/addons/object-detection/index.mjs.map +1 -0
  55. package/dist/addons/plate-detection/index.d.mts +25 -0
  56. package/dist/addons/plate-detection/index.d.ts +25 -0
  57. package/dist/addons/plate-detection/index.js +646 -0
  58. package/dist/addons/plate-detection/index.js.map +1 -0
  59. package/dist/addons/plate-detection/index.mjs +10 -0
  60. package/dist/addons/plate-detection/index.mjs.map +1 -0
  61. package/dist/addons/plate-recognition/index.d.mts +25 -0
  62. package/dist/addons/plate-recognition/index.d.ts +25 -0
  63. package/dist/addons/plate-recognition/index.js +648 -0
  64. package/dist/addons/plate-recognition/index.js.map +1 -0
  65. package/dist/addons/plate-recognition/index.mjs +9 -0
  66. package/dist/addons/plate-recognition/index.mjs.map +1 -0
  67. package/dist/chunk-3MQFUDRU.mjs +260 -0
  68. package/dist/chunk-3MQFUDRU.mjs.map +1 -0
  69. package/dist/chunk-5AIQSN32.mjs +227 -0
  70. package/dist/chunk-5AIQSN32.mjs.map +1 -0
  71. package/dist/chunk-5JJZGKL7.mjs +186 -0
  72. package/dist/chunk-5JJZGKL7.mjs.map +1 -0
  73. package/dist/chunk-6OR5TE7A.mjs +101 -0
  74. package/dist/chunk-6OR5TE7A.mjs.map +1 -0
  75. package/dist/chunk-AYBFB7ID.mjs +763 -0
  76. package/dist/chunk-AYBFB7ID.mjs.map +1 -0
  77. package/dist/chunk-B3R66MPF.mjs +219 -0
  78. package/dist/chunk-B3R66MPF.mjs.map +1 -0
  79. package/dist/chunk-DTOAB2CE.mjs +79 -0
  80. package/dist/chunk-DTOAB2CE.mjs.map +1 -0
  81. package/dist/chunk-ISOIDU4U.mjs +54 -0
  82. package/dist/chunk-ISOIDU4U.mjs.map +1 -0
  83. package/dist/chunk-J4WRYHHY.mjs +212 -0
  84. package/dist/chunk-J4WRYHHY.mjs.map +1 -0
  85. package/dist/chunk-KUO2BVFY.mjs +90 -0
  86. package/dist/chunk-KUO2BVFY.mjs.map +1 -0
  87. package/dist/chunk-LPI42WL6.mjs +324 -0
  88. package/dist/chunk-LPI42WL6.mjs.map +1 -0
  89. package/dist/chunk-MEVASN3P.mjs +305 -0
  90. package/dist/chunk-MEVASN3P.mjs.map +1 -0
  91. package/dist/chunk-PDSHDDPV.mjs +255 -0
  92. package/dist/chunk-PDSHDDPV.mjs.map +1 -0
  93. package/dist/chunk-Q3SQOYG6.mjs +218 -0
  94. package/dist/chunk-Q3SQOYG6.mjs.map +1 -0
  95. package/dist/chunk-QIMDG34B.mjs +229 -0
  96. package/dist/chunk-QIMDG34B.mjs.map +1 -0
  97. package/dist/index.d.mts +171 -0
  98. package/dist/index.d.ts +171 -0
  99. package/dist/index.js +3463 -0
  100. package/dist/index.js.map +1 -0
  101. package/dist/index.mjs +111 -0
  102. package/dist/index.mjs.map +1 -0
  103. package/package.json +49 -0
  104. package/python/__pycache__/coreml_inference.cpython-313.pyc +0 -0
  105. package/python/__pycache__/openvino_inference.cpython-313.pyc +0 -0
  106. package/python/__pycache__/pytorch_inference.cpython-313.pyc +0 -0
  107. package/python/coreml_inference.py +319 -0
  108. package/python/openvino_inference.py +247 -0
  109. package/python/pytorch_inference.py +255 -0
@@ -0,0 +1,25 @@
1
+ import { IClassifierProvider, IDetectionAddon, AddonManifest, AddonContext, CropInput, ClassifierOutput, ConfigUISchema, ClassMapDefinition, ModelCatalogEntry, DetectionModel, LabelDefinition, ProbeResult } from '@camstack/types';
2
+
3
+ declare class AnimalClassifierAddon implements IClassifierProvider, IDetectionAddon {
4
+ readonly id = "animal-classifier";
5
+ readonly slot: "classifier";
6
+ readonly inputClasses: readonly ["animal"];
7
+ readonly outputClasses: readonly ["animal-type:*"];
8
+ readonly slotPriority = 0;
9
+ readonly requiredSteps: readonly [];
10
+ readonly manifest: AddonManifest;
11
+ private engine;
12
+ private modelEntry;
13
+ private minConfidence;
14
+ initialize(ctx: AddonContext): Promise<void>;
15
+ classify(input: CropInput): Promise<ClassifierOutput>;
16
+ shutdown(): Promise<void>;
17
+ getConfigSchema(): ConfigUISchema;
18
+ getClassMap(): ClassMapDefinition;
19
+ getModelCatalog(): ModelCatalogEntry[];
20
+ getAvailableModels(): DetectionModel[];
21
+ getActiveLabels(): readonly LabelDefinition[];
22
+ probe(): Promise<ProbeResult>;
23
+ }
24
+
25
+ export { AnimalClassifierAddon as default };
@@ -0,0 +1,25 @@
1
+ import { IClassifierProvider, IDetectionAddon, AddonManifest, AddonContext, CropInput, ClassifierOutput, ConfigUISchema, ClassMapDefinition, ModelCatalogEntry, DetectionModel, LabelDefinition, ProbeResult } from '@camstack/types';
2
+
3
+ declare class AnimalClassifierAddon implements IClassifierProvider, IDetectionAddon {
4
+ readonly id = "animal-classifier";
5
+ readonly slot: "classifier";
6
+ readonly inputClasses: readonly ["animal"];
7
+ readonly outputClasses: readonly ["animal-type:*"];
8
+ readonly slotPriority = 0;
9
+ readonly requiredSteps: readonly [];
10
+ readonly manifest: AddonManifest;
11
+ private engine;
12
+ private modelEntry;
13
+ private minConfidence;
14
+ initialize(ctx: AddonContext): Promise<void>;
15
+ classify(input: CropInput): Promise<ClassifierOutput>;
16
+ shutdown(): Promise<void>;
17
+ getConfigSchema(): ConfigUISchema;
18
+ getClassMap(): ClassMapDefinition;
19
+ getModelCatalog(): ModelCatalogEntry[];
20
+ getAvailableModels(): DetectionModel[];
21
+ getActiveLabels(): readonly LabelDefinition[];
22
+ probe(): Promise<ProbeResult>;
23
+ }
24
+
25
+ export { AnimalClassifierAddon as default };
@@ -0,0 +1,652 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+
30
+ // src/addons/animal-classifier/index.ts
31
+ var animal_classifier_exports = {};
32
+ __export(animal_classifier_exports, {
33
+ default: () => AnimalClassifierAddon
34
+ });
35
+ module.exports = __toCommonJS(animal_classifier_exports);
36
+
37
+ // src/catalogs/animal-classification-models.ts
38
+ var import_types = require("@camstack/types");
39
+ var HF_REPO = "camstack/camstack-models";
40
+ var hf = (path3) => (0, import_types.hfModelUrl)(HF_REPO, path3);
41
+ var BIRD_LABEL = { id: "species", name: "Bird Species" };
42
+ var ANIMAL_TYPE_LABEL = { id: "animal-type", name: "Animal Type" };
43
+ var BIRD_SPECIES_MODELS = [
44
+ {
45
+ id: "bird-species-525",
46
+ name: "Bird Species (525)",
47
+ description: "EfficientNet bird species classifier \u2014 525 species, MIT license",
48
+ inputSize: { width: 224, height: 224 },
49
+ inputNormalization: "imagenet",
50
+ labels: [BIRD_LABEL],
51
+ formats: {
52
+ onnx: { url: hf("animalClassification/bird-species/onnx/camstack-bird-species-525.onnx"), sizeMB: 32 }
53
+ }
54
+ }
55
+ ];
56
+ var BIRD_NABIRDS_MODELS = [
57
+ {
58
+ id: "bird-nabirds-404",
59
+ name: "NABirds (404 species)",
60
+ description: "ResNet50 trained on NABirds \u2014 404 North American species with ONNX, CoreML, OpenVINO",
61
+ inputSize: { width: 224, height: 224 },
62
+ inputNormalization: "imagenet",
63
+ labels: [{ id: "species", name: "Bird Species" }],
64
+ formats: {
65
+ onnx: { url: hf("animalClassification/bird-nabirds/onnx/camstack-bird-nabirds-404.onnx"), sizeMB: 93 },
66
+ coreml: { url: hf("animalClassification/bird-nabirds/coreml/camstack-bird-nabirds-404.mlpackage"), sizeMB: 47 },
67
+ openvino: { url: hf("animalClassification/bird-nabirds/openvino/camstack-bird-nabirds-404.xml"), sizeMB: 47 }
68
+ }
69
+ }
70
+ ];
71
+ var ANIMAL_TYPE_MODELS = [
72
+ {
73
+ id: "animals-10",
74
+ name: "Animal Classifier (10)",
75
+ description: "ViT-based animal type classifier \u2014 cat, cow, dog, dolphin, eagle, panda, horse, monkey, sheep, spider",
76
+ inputSize: { width: 224, height: 224 },
77
+ inputNormalization: "imagenet",
78
+ labels: [ANIMAL_TYPE_LABEL],
79
+ formats: {
80
+ onnx: { url: hf("animalClassification/animals-10/onnx/camstack-animals-10.onnx"), sizeMB: 328 }
81
+ }
82
+ }
83
+ ];
84
+
85
+ // src/shared/image-utils.ts
86
+ var import_sharp = __toESM(require("sharp"));
87
+ async function cropRegion(jpeg, roi) {
88
+ return (0, import_sharp.default)(jpeg).extract({
89
+ left: Math.round(roi.x),
90
+ top: Math.round(roi.y),
91
+ width: Math.round(roi.w),
92
+ height: Math.round(roi.h)
93
+ }).jpeg().toBuffer();
94
+ }
95
+ async function resizeAndNormalize(jpeg, targetWidth, targetHeight, normalization, layout) {
96
+ const { data } = await (0, import_sharp.default)(jpeg).resize(targetWidth, targetHeight).removeAlpha().raw().toBuffer({ resolveWithObject: true });
97
+ const numPixels = targetWidth * targetHeight;
98
+ const float32 = new Float32Array(3 * numPixels);
99
+ const mean = [0.485, 0.456, 0.406];
100
+ const std = [0.229, 0.224, 0.225];
101
+ if (layout === "nchw") {
102
+ for (let i = 0; i < numPixels; i++) {
103
+ const srcBase = i * 3;
104
+ for (let c = 0; c < 3; c++) {
105
+ const raw = data[srcBase + c] / 255;
106
+ let val;
107
+ if (normalization === "zero-one") {
108
+ val = raw;
109
+ } else if (normalization === "imagenet") {
110
+ val = (raw - mean[c]) / std[c];
111
+ } else {
112
+ val = data[srcBase + c];
113
+ }
114
+ float32[c * numPixels + i] = val;
115
+ }
116
+ }
117
+ } else {
118
+ for (let i = 0; i < numPixels; i++) {
119
+ const srcBase = i * 3;
120
+ for (let c = 0; c < 3; c++) {
121
+ const raw = data[srcBase + c] / 255;
122
+ let val;
123
+ if (normalization === "zero-one") {
124
+ val = raw;
125
+ } else if (normalization === "imagenet") {
126
+ val = (raw - mean[c]) / std[c];
127
+ } else {
128
+ val = data[srcBase + c];
129
+ }
130
+ float32[i * 3 + c] = val;
131
+ }
132
+ }
133
+ }
134
+ return float32;
135
+ }
136
+
137
+ // src/shared/engine-resolver.ts
138
+ var fs = __toESM(require("fs"));
139
+ var path2 = __toESM(require("path"));
140
+
141
+ // src/shared/node-engine.ts
142
+ var path = __toESM(require("path"));
143
+ var BACKEND_TO_PROVIDER = {
144
+ cpu: "cpu",
145
+ coreml: "coreml",
146
+ cuda: "cuda",
147
+ tensorrt: "tensorrt",
148
+ dml: "dml"
149
+ };
150
+ var BACKEND_TO_DEVICE = {
151
+ cpu: "cpu",
152
+ coreml: "gpu-mps",
153
+ cuda: "gpu-cuda",
154
+ tensorrt: "tensorrt"
155
+ };
156
+ var NodeInferenceEngine = class {
157
+ constructor(modelPath, backend) {
158
+ this.modelPath = modelPath;
159
+ this.backend = backend;
160
+ this.device = BACKEND_TO_DEVICE[backend] ?? "cpu";
161
+ }
162
+ runtime = "onnx";
163
+ device;
164
+ session = null;
165
+ async initialize() {
166
+ const ort = await import("onnxruntime-node");
167
+ const provider = BACKEND_TO_PROVIDER[this.backend] ?? "cpu";
168
+ const absModelPath = path.isAbsolute(this.modelPath) ? this.modelPath : path.resolve(process.cwd(), this.modelPath);
169
+ const sessionOptions = {
170
+ executionProviders: [provider]
171
+ };
172
+ this.session = await ort.InferenceSession.create(absModelPath, sessionOptions);
173
+ }
174
+ async run(input, inputShape) {
175
+ if (!this.session) {
176
+ throw new Error("NodeInferenceEngine: not initialized \u2014 call initialize() first");
177
+ }
178
+ const ort = await import("onnxruntime-node");
179
+ const sess = this.session;
180
+ const inputName = sess.inputNames[0];
181
+ const tensor = new ort.Tensor("float32", input, [...inputShape]);
182
+ const feeds = { [inputName]: tensor };
183
+ const results = await sess.run(feeds);
184
+ const outputName = sess.outputNames[0];
185
+ const outputTensor = results[outputName];
186
+ return outputTensor.data;
187
+ }
188
+ async runMultiOutput(input, inputShape) {
189
+ if (!this.session) {
190
+ throw new Error("NodeInferenceEngine: not initialized \u2014 call initialize() first");
191
+ }
192
+ const ort = await import("onnxruntime-node");
193
+ const sess = this.session;
194
+ const inputName = sess.inputNames[0];
195
+ const tensor = new ort.Tensor("float32", input, [...inputShape]);
196
+ const feeds = { [inputName]: tensor };
197
+ const results = await sess.run(feeds);
198
+ const out = {};
199
+ for (const name of sess.outputNames) {
200
+ out[name] = results[name].data;
201
+ }
202
+ return out;
203
+ }
204
+ async dispose() {
205
+ this.session = null;
206
+ }
207
+ };
208
+
209
+ // src/shared/python-engine.ts
210
+ var import_node_child_process = require("child_process");
211
+ var PythonInferenceEngine = class {
212
+ constructor(pythonPath, scriptPath, runtime, modelPath, extraArgs = []) {
213
+ this.pythonPath = pythonPath;
214
+ this.scriptPath = scriptPath;
215
+ this.modelPath = modelPath;
216
+ this.extraArgs = extraArgs;
217
+ this.runtime = runtime;
218
+ const runtimeDeviceMap = {
219
+ onnx: "cpu",
220
+ coreml: "gpu-mps",
221
+ pytorch: "cpu",
222
+ openvino: "cpu",
223
+ tflite: "cpu"
224
+ };
225
+ this.device = runtimeDeviceMap[runtime];
226
+ }
227
+ runtime;
228
+ device;
229
+ process = null;
230
+ receiveBuffer = Buffer.alloc(0);
231
+ pendingResolve = null;
232
+ pendingReject = null;
233
+ async initialize() {
234
+ const args = [this.scriptPath, this.modelPath, ...this.extraArgs];
235
+ this.process = (0, import_node_child_process.spawn)(this.pythonPath, args, {
236
+ stdio: ["pipe", "pipe", "pipe"]
237
+ });
238
+ if (!this.process.stdout || !this.process.stdin) {
239
+ throw new Error("PythonInferenceEngine: failed to create process pipes");
240
+ }
241
+ this.process.stderr?.on("data", (chunk) => {
242
+ process.stderr.write(`[python-engine] ${chunk.toString()}`);
243
+ });
244
+ this.process.on("error", (err) => {
245
+ this.pendingReject?.(err);
246
+ this.pendingReject = null;
247
+ this.pendingResolve = null;
248
+ });
249
+ this.process.on("exit", (code) => {
250
+ if (code !== 0) {
251
+ const err = new Error(`PythonInferenceEngine: process exited with code ${code}`);
252
+ this.pendingReject?.(err);
253
+ this.pendingReject = null;
254
+ this.pendingResolve = null;
255
+ }
256
+ });
257
+ this.process.stdout.on("data", (chunk) => {
258
+ this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
259
+ this._tryReceive();
260
+ });
261
+ await new Promise((resolve2, reject) => {
262
+ const timeout = setTimeout(() => resolve2(), 2e3);
263
+ this.process?.on("error", (err) => {
264
+ clearTimeout(timeout);
265
+ reject(err);
266
+ });
267
+ this.process?.on("exit", (code) => {
268
+ clearTimeout(timeout);
269
+ if (code !== 0) {
270
+ reject(new Error(`PythonInferenceEngine: process exited early with code ${code}`));
271
+ }
272
+ });
273
+ });
274
+ }
275
+ _tryReceive() {
276
+ if (this.receiveBuffer.length < 4) return;
277
+ const length = this.receiveBuffer.readUInt32LE(0);
278
+ if (this.receiveBuffer.length < 4 + length) return;
279
+ const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
280
+ this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
281
+ const resolve2 = this.pendingResolve;
282
+ const reject = this.pendingReject;
283
+ this.pendingResolve = null;
284
+ this.pendingReject = null;
285
+ if (!resolve2) return;
286
+ try {
287
+ const parsed = JSON.parse(jsonBytes.toString("utf8"));
288
+ resolve2(parsed);
289
+ } catch (err) {
290
+ reject?.(err instanceof Error ? err : new Error(String(err)));
291
+ }
292
+ }
293
+ /** Send JPEG buffer, receive JSON detection results */
294
+ async runJpeg(jpeg) {
295
+ if (!this.process?.stdin) {
296
+ throw new Error("PythonInferenceEngine: process not initialized");
297
+ }
298
+ return new Promise((resolve2, reject) => {
299
+ this.pendingResolve = resolve2;
300
+ this.pendingReject = reject;
301
+ const lengthBuf = Buffer.allocUnsafe(4);
302
+ lengthBuf.writeUInt32LE(jpeg.length, 0);
303
+ this.process.stdin.write(Buffer.concat([lengthBuf, jpeg]));
304
+ });
305
+ }
306
+ /** IInferenceEngine.run — wraps runJpeg for compatibility */
307
+ async run(_input, _inputShape) {
308
+ throw new Error(
309
+ "PythonInferenceEngine: use runJpeg() directly \u2014 this engine operates on JPEG input"
310
+ );
311
+ }
312
+ /** IInferenceEngine.runMultiOutput — not supported by Python engine (operates on JPEG input) */
313
+ async runMultiOutput(_input, _inputShape) {
314
+ throw new Error(
315
+ "PythonInferenceEngine: runMultiOutput() is not supported \u2014 this engine operates on JPEG input"
316
+ );
317
+ }
318
+ async dispose() {
319
+ if (this.process) {
320
+ this.process.stdin?.end();
321
+ this.process.kill("SIGTERM");
322
+ this.process = null;
323
+ }
324
+ }
325
+ };
326
+
327
+ // src/shared/engine-resolver.ts
328
+ var AUTO_BACKEND_PRIORITY = ["coreml", "cuda", "tensorrt", "cpu"];
329
+ var BACKEND_TO_FORMAT = {
330
+ cpu: "onnx",
331
+ coreml: "coreml",
332
+ cuda: "onnx",
333
+ tensorrt: "onnx"
334
+ };
335
+ var RUNTIME_TO_FORMAT = {
336
+ onnx: "onnx",
337
+ coreml: "coreml",
338
+ openvino: "openvino",
339
+ tflite: "tflite",
340
+ pytorch: "pt"
341
+ };
342
+ function modelFilePath(modelsDir, modelEntry, format) {
343
+ const formatEntry = modelEntry.formats[format];
344
+ if (!formatEntry) {
345
+ throw new Error(`Model ${modelEntry.id} has no ${format} format`);
346
+ }
347
+ const urlParts = formatEntry.url.split("/");
348
+ const filename = urlParts[urlParts.length - 1] ?? `${modelEntry.id}.${format}`;
349
+ return path2.join(modelsDir, filename);
350
+ }
351
+ function modelExists(filePath) {
352
+ try {
353
+ return fs.existsSync(filePath);
354
+ } catch {
355
+ return false;
356
+ }
357
+ }
358
+ async function resolveEngine(options) {
359
+ const { runtime, backend, modelEntry, modelsDir, downloadModel } = options;
360
+ let selectedFormat;
361
+ let selectedBackend;
362
+ if (runtime === "auto") {
363
+ const available = await probeOnnxBackends();
364
+ let chosen = null;
365
+ for (const b of AUTO_BACKEND_PRIORITY) {
366
+ if (!available.includes(b)) continue;
367
+ const fmt = BACKEND_TO_FORMAT[b];
368
+ if (!fmt) continue;
369
+ if (!modelEntry.formats[fmt]) continue;
370
+ chosen = { backend: b, format: fmt };
371
+ break;
372
+ }
373
+ if (!chosen) {
374
+ throw new Error(
375
+ `resolveEngine: no compatible backend found for model ${modelEntry.id}. Available backends: ${available.join(", ")}`
376
+ );
377
+ }
378
+ selectedFormat = chosen.format;
379
+ selectedBackend = chosen.backend;
380
+ } else {
381
+ const fmt = RUNTIME_TO_FORMAT[runtime];
382
+ if (!fmt) {
383
+ throw new Error(`resolveEngine: unsupported runtime "${runtime}"`);
384
+ }
385
+ if (!modelEntry.formats[fmt]) {
386
+ throw new Error(
387
+ `resolveEngine: model ${modelEntry.id} has no ${fmt} format for runtime ${runtime}`
388
+ );
389
+ }
390
+ selectedFormat = fmt;
391
+ selectedBackend = runtime === "onnx" ? backend || "cpu" : runtime;
392
+ }
393
+ let modelPath = modelFilePath(modelsDir, modelEntry, selectedFormat);
394
+ if (!modelExists(modelPath)) {
395
+ if (downloadModel) {
396
+ const formatEntry = modelEntry.formats[selectedFormat];
397
+ modelPath = await downloadModel(formatEntry.url, modelsDir);
398
+ } else {
399
+ throw new Error(
400
+ `resolveEngine: model file not found at ${modelPath} and no downloadModel function provided`
401
+ );
402
+ }
403
+ }
404
+ if (selectedFormat === "onnx" || selectedFormat === "coreml") {
405
+ const engine = new NodeInferenceEngine(modelPath, selectedBackend);
406
+ await engine.initialize();
407
+ return { engine, format: selectedFormat, modelPath };
408
+ }
409
+ const { pythonPath } = options;
410
+ const PYTHON_SCRIPT_MAP = {
411
+ coreml: "coreml_inference.py",
412
+ pytorch: "pytorch_inference.py",
413
+ openvino: "openvino_inference.py"
414
+ };
415
+ const effectiveRuntime = runtime === "auto" ? selectedBackend : runtime;
416
+ const scriptName = PYTHON_SCRIPT_MAP[effectiveRuntime];
417
+ if (scriptName && pythonPath) {
418
+ const scriptPath = path2.join(__dirname, "../../python", scriptName);
419
+ const inputSize = Math.max(modelEntry.inputSize.width, modelEntry.inputSize.height);
420
+ const engine = new PythonInferenceEngine(pythonPath, scriptPath, effectiveRuntime, modelPath, [
421
+ `--input-size=${inputSize}`,
422
+ `--confidence=0.25`
423
+ ]);
424
+ await engine.initialize();
425
+ return { engine, format: selectedFormat, modelPath };
426
+ }
427
+ const fallbackPath = modelFilePath(modelsDir, modelEntry, "onnx");
428
+ if (modelEntry.formats["onnx"] && modelExists(fallbackPath)) {
429
+ const engine = new NodeInferenceEngine(fallbackPath, "cpu");
430
+ await engine.initialize();
431
+ return { engine, format: "onnx", modelPath: fallbackPath };
432
+ }
433
+ throw new Error(
434
+ `resolveEngine: format ${selectedFormat} is not yet supported by NodeInferenceEngine, no Python runtime is available, and no ONNX fallback exists`
435
+ );
436
+ }
437
+ async function probeOnnxBackends() {
438
+ const available = ["cpu"];
439
+ try {
440
+ const ort = await import("onnxruntime-node");
441
+ const providers = ort.env?.webgl?.disabled !== void 0 ? ort.InferenceSession?.getAvailableProviders?.() ?? [] : [];
442
+ for (const p of providers) {
443
+ const normalized = p.toLowerCase().replace("executionprovider", "");
444
+ if (normalized === "coreml") available.push("coreml");
445
+ else if (normalized === "cuda") available.push("cuda");
446
+ else if (normalized === "tensorrt") available.push("tensorrt");
447
+ }
448
+ } catch {
449
+ }
450
+ if (process.platform === "darwin" && !available.includes("coreml")) {
451
+ available.push("coreml");
452
+ }
453
+ return [...new Set(available)];
454
+ }
455
+
456
+ // src/addons/animal-classifier/index.ts
457
+ var ANIMAL_TYPE_LABEL2 = { id: "animal-type", name: "Animal Type" };
458
+ var ANIMAL_TYPE_LABELS = [ANIMAL_TYPE_LABEL2];
459
+ var ANIMAL_CLASS_MAP = { mapping: {}, preserveOriginal: true };
460
+ var ANIMAL_10_CLASSES = [
461
+ "cat",
462
+ "cow",
463
+ "dog",
464
+ "dolphin",
465
+ "eagle",
466
+ "giant panda",
467
+ "horse",
468
+ "monkey",
469
+ "sheep",
470
+ "spider"
471
+ ];
472
+ function softmax(logits) {
473
+ const max = logits.reduce((a, b) => Math.max(a, b), -Infinity);
474
+ const exps = logits.map((v) => Math.exp(v - max));
475
+ const sum = exps.reduce((a, b) => a + b, 0);
476
+ return exps.map((v) => v / sum);
477
+ }
478
+ var AnimalClassifierAddon = class {
479
+ id = "animal-classifier";
480
+ slot = "classifier";
481
+ inputClasses = ["animal"];
482
+ outputClasses = ["animal-type:*"];
483
+ slotPriority = 0;
484
+ requiredSteps = [];
485
+ manifest = {
486
+ id: "animal-classifier",
487
+ name: "Animal Classifier",
488
+ version: "0.1.0",
489
+ description: "ViT-based animal type classifier \u2014 10 common species",
490
+ packageName: "@camstack/addon-vision",
491
+ slot: "classifier",
492
+ inputClasses: ["animal"],
493
+ outputClasses: ["animal-type:*"],
494
+ supportsCustomModels: false,
495
+ mayRequirePython: false,
496
+ defaultConfig: {
497
+ modelId: "animals-10",
498
+ runtime: "auto",
499
+ backend: "cpu",
500
+ minConfidence: 0.3
501
+ }
502
+ };
503
+ engine;
504
+ modelEntry;
505
+ minConfidence = 0.3;
506
+ async initialize(ctx) {
507
+ const cfg = ctx.addonConfig;
508
+ const modelId = cfg["modelId"] ?? "animals-10";
509
+ const runtime = cfg["runtime"] ?? "auto";
510
+ const backend = cfg["backend"] ?? "cpu";
511
+ this.minConfidence = cfg["minConfidence"] ?? 0.3;
512
+ const entry = ANIMAL_TYPE_MODELS.find((m) => m.id === modelId);
513
+ if (!entry) {
514
+ throw new Error(`AnimalClassifierAddon: unknown modelId "${modelId}"`);
515
+ }
516
+ this.modelEntry = entry;
517
+ const resolved = await resolveEngine({
518
+ runtime,
519
+ backend,
520
+ modelEntry: entry,
521
+ modelsDir: ctx.locationPaths.models
522
+ });
523
+ this.engine = resolved.engine;
524
+ }
525
+ async classify(input) {
526
+ const start = Date.now();
527
+ const { width: inputW, height: inputH } = this.modelEntry.inputSize;
528
+ const animalCrop = await cropRegion(input.frame.data, input.roi);
529
+ const normalized = await resizeAndNormalize(animalCrop, inputW, inputH, "imagenet", "nchw");
530
+ const rawOutput = await this.engine.run(normalized, [1, 3, inputH, inputW]);
531
+ const probs = softmax(rawOutput);
532
+ let maxIdx = 0;
533
+ let maxScore = probs[0] ?? 0;
534
+ for (let i = 1; i < probs.length; i++) {
535
+ const score = probs[i] ?? 0;
536
+ if (score > maxScore) {
537
+ maxScore = score;
538
+ maxIdx = i;
539
+ }
540
+ }
541
+ if (maxScore < this.minConfidence) {
542
+ return {
543
+ classifications: [],
544
+ inferenceMs: Date.now() - start,
545
+ modelId: this.modelEntry.id
546
+ };
547
+ }
548
+ const label = ANIMAL_10_CLASSES[maxIdx] ?? `animal_${maxIdx}`;
549
+ return {
550
+ classifications: [
551
+ {
552
+ class: label,
553
+ score: maxScore
554
+ }
555
+ ],
556
+ inferenceMs: Date.now() - start,
557
+ modelId: this.modelEntry.id
558
+ };
559
+ }
560
+ async shutdown() {
561
+ await this.engine?.dispose();
562
+ }
563
+ getConfigSchema() {
564
+ return {
565
+ sections: [
566
+ {
567
+ id: "model",
568
+ title: "Model",
569
+ columns: 1,
570
+ fields: [
571
+ {
572
+ key: "modelId",
573
+ label: "Model",
574
+ type: "model-selector",
575
+ catalog: [...ANIMAL_TYPE_MODELS],
576
+ allowCustom: false,
577
+ allowConversion: false,
578
+ acceptFormats: ["onnx", "coreml", "openvino"],
579
+ requiredMetadata: ["inputSize", "labels"],
580
+ outputFormatHint: "classification"
581
+ }
582
+ ]
583
+ },
584
+ {
585
+ id: "thresholds",
586
+ title: "Classification Settings",
587
+ columns: 1,
588
+ fields: [
589
+ {
590
+ key: "minConfidence",
591
+ label: "Minimum Confidence",
592
+ type: "slider",
593
+ min: 0.05,
594
+ max: 1,
595
+ step: 0.05,
596
+ default: 0.3
597
+ }
598
+ ]
599
+ },
600
+ {
601
+ id: "runtime",
602
+ title: "Runtime",
603
+ columns: 2,
604
+ fields: [
605
+ {
606
+ key: "runtime",
607
+ label: "Runtime",
608
+ type: "select",
609
+ options: [
610
+ { value: "auto", label: "Auto (recommended)" },
611
+ { value: "onnx", label: "ONNX Runtime" },
612
+ { value: "coreml", label: "CoreML (Apple)" }
613
+ ]
614
+ },
615
+ {
616
+ key: "backend",
617
+ label: "Backend",
618
+ type: "select",
619
+ dependsOn: { runtime: "onnx" },
620
+ options: [
621
+ { value: "cpu", label: "CPU" },
622
+ { value: "coreml", label: "CoreML" },
623
+ { value: "cuda", label: "CUDA (NVIDIA)" }
624
+ ]
625
+ }
626
+ ]
627
+ }
628
+ ]
629
+ };
630
+ }
631
+ getClassMap() {
632
+ return ANIMAL_CLASS_MAP;
633
+ }
634
+ getModelCatalog() {
635
+ return [...ANIMAL_TYPE_MODELS];
636
+ }
637
+ getAvailableModels() {
638
+ return [];
639
+ }
640
+ getActiveLabels() {
641
+ return ANIMAL_TYPE_LABELS;
642
+ }
643
+ async probe() {
644
+ return {
645
+ available: true,
646
+ runtime: this.engine?.runtime ?? "onnx",
647
+ device: this.engine?.device ?? "cpu",
648
+ capabilities: ["fp32"]
649
+ };
650
+ }
651
+ };
652
+ //# sourceMappingURL=index.js.map