@camstack/addon-post-analysis 0.1.20 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist-4mTLJ7BJ.mjs +20750 -0
- package/dist/dist-CS2K80so.js +20933 -0
- package/dist/embedding-encoder/index.js +977 -902
- package/dist/embedding-encoder/index.mjs +967 -860
- package/dist/enrichment-engine/index.js +834 -833
- package/dist/enrichment-engine/index.mjs +828 -832
- package/dist/pipeline-analytics/_stub.js +1680 -1396
- package/dist/pipeline-analytics/_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-DOSUJ-U0.mjs +156 -0
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DJvmVCso.mjs +26 -0
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_tanstack_mf_1_react_mf_2_query__loadShare__.js-B3Wx5J80.mjs +26 -0
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare__react__loadShare__.js-C0AuF9av.mjs +26 -0
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare__react_mf_1_jsx_mf_2_runtime__loadShare__.js-Bm-iyjmq.mjs +26 -0
- package/dist/pipeline-analytics/dist-CYZr2fwk.mjs +2726 -0
- package/dist/pipeline-analytics/hostInit-BazRS2O7.mjs +129 -0
- package/dist/pipeline-analytics/index.js +7112 -3100
- package/dist/pipeline-analytics/index.mjs +7105 -3100
- package/dist/pipeline-analytics/remoteEntry.js +134 -2973
- package/dist/pipeline-analytics/remoteEntry.ssr.js +33 -0
- package/dist/pipeline-analytics/virtualExposes-BgYzpJZG.mjs +27 -0
- package/dist/pipeline-analytics/virtual_mf-exposes-ssr___mfe_internal__addon_pipeline_analytics_widgets__remoteEntry_js-D7qgWCKX.mjs +10 -0
- package/dist/resolve-frame-5lMxmeI1.js +57 -0
- package/dist/resolve-frame-CT1T1tWy.mjs +44 -0
- package/package.json +15 -6
- package/dist/embedding-encoder/index.js.map +0 -1
- package/dist/embedding-encoder/index.mjs.map +0 -1
- package/dist/enrichment-engine/index.js.map +0 -1
- package/dist/enrichment-engine/index.mjs.map +0 -1
- package/dist/index-B0RhVv1c.js +0 -17107
- package/dist/index-B0RhVv1c.js.map +0 -1
- package/dist/index-ot5PeFg_.mjs +0 -17108
- package/dist/index-ot5PeFg_.mjs.map +0 -1
- package/dist/pipeline-analytics/@mf-types/compiled-types/pipeline-analytics/widgets/AudioHistoryChart.d.ts +0 -4
- package/dist/pipeline-analytics/@mf-types/compiled-types/pipeline-analytics/widgets/AudioMetricsPanel.d.ts +0 -10
- package/dist/pipeline-analytics/@mf-types/compiled-types/pipeline-analytics/widgets/DetectionHistoryChart.d.ts +0 -4
- package/dist/pipeline-analytics/@mf-types/compiled-types/pipeline-analytics/widgets/LiveStatsTab.d.ts +0 -5
- package/dist/pipeline-analytics/@mf-types/compiled-types/pipeline-analytics/widgets/MotionHistoryChart.d.ts +0 -4
- package/dist/pipeline-analytics/@mf-types/compiled-types/pipeline-analytics/widgets/OccupancyHistoryChart.d.ts +0 -4
- package/dist/pipeline-analytics/@mf-types/compiled-types/pipeline-analytics/widgets/OccupancyPanel.d.ts +0 -10
- package/dist/pipeline-analytics/@mf-types/compiled-types/pipeline-analytics/widgets/chart-utils.d.ts +0 -97
- package/dist/pipeline-analytics/@mf-types/compiled-types/pipeline-analytics/widgets/index.d.ts +0 -29
- package/dist/pipeline-analytics/@mf-types/widgets.d.ts +0 -2
- package/dist/pipeline-analytics/@mf-types.d.ts +0 -3
- package/dist/pipeline-analytics/@mf-types.zip +0 -0
- package/dist/pipeline-analytics/__mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_sdk__loadShare__.mjs-lantnv8e.mjs +0 -12
- package/dist/pipeline-analytics/__mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.mjs-BD3oMNGB.mjs +0 -29
- package/dist/pipeline-analytics/__mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.mjs-BgOHCakr.mjs +0 -18
- package/dist/pipeline-analytics/__mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_tanstack_mf_1_react_mf_2_query__loadShare__.mjs-DoWbefqS.mjs +0 -104
- package/dist/pipeline-analytics/__mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_trpc_mf_1_client__loadShare__.mjs-52bfkwC8.mjs +0 -85
- package/dist/pipeline-analytics/__mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_trpc_mf_1_react_mf_2_query__loadShare__.mjs-CVrnrGED.mjs +0 -62
- package/dist/pipeline-analytics/__mfe_internal__addon_pipeline_analytics_widgets__loadShare__react__loadShare__.mjs-D1qPKjvR.mjs +0 -89
- package/dist/pipeline-analytics/__mfe_internal__addon_pipeline_analytics_widgets__loadShare__react__loadShare__.mjs_commonjs-proxy-B5X50Xa4.mjs +0 -29
- package/dist/pipeline-analytics/__mfe_internal__addon_pipeline_analytics_widgets__loadShare__react_mf_1_jsx_mf_2_runtime__loadShare__.mjs-BsyrX6NO.mjs +0 -36
- package/dist/pipeline-analytics/__mfe_internal__addon_pipeline_analytics_widgets__loadShare__react_mf_2_dom__loadShare__.mjs-Dp8hqYOB.mjs +0 -45
- package/dist/pipeline-analytics/__mfe_internal__addon_pipeline_analytics_widgets__loadShare__react_mf_2_dom__loadShare__.mjs_commonjs-proxy-B10b5k5J.mjs +0 -6
- package/dist/pipeline-analytics/__mfe_internal__addon_pipeline_analytics_widgets__loadShare__react_mf_2_dom_mf_1_client__loadShare__.mjs-BZjEt71l.mjs +0 -34
- package/dist/pipeline-analytics/_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-DWB3apaJ.mjs +0 -156
- package/dist/pipeline-analytics/client-C6xdgLZU.mjs +0 -9836
- package/dist/pipeline-analytics/getErrorShape-BPSzUA7W-TlK8ipWe.mjs +0 -211
- package/dist/pipeline-analytics/hostInit-3cyL9eyG.mjs +0 -168
- package/dist/pipeline-analytics/index-BCTHeI2m.mjs +0 -1641
- package/dist/pipeline-analytics/index-BuWLz0GG.mjs +0 -2603
- package/dist/pipeline-analytics/index-CIwq-tQL.mjs +0 -725
- package/dist/pipeline-analytics/index-CWBMDbou.mjs +0 -435
- package/dist/pipeline-analytics/index-CWkKuNLr.mjs +0 -232
- package/dist/pipeline-analytics/index-CZhagnlH.mjs +0 -67784
- package/dist/pipeline-analytics/index-D883Q5B8.mjs +0 -185
- package/dist/pipeline-analytics/index-DtOI1aTU.mjs +0 -18504
- package/dist/pipeline-analytics/index-xncRG7-x.mjs +0 -2713
- package/dist/pipeline-analytics/index.js.map +0 -1
- package/dist/pipeline-analytics/index.mjs.map +0 -1
- package/dist/pipeline-analytics/jsx-runtime-DdLhuHmJ.mjs +0 -55
- package/dist/pipeline-analytics/schemas-B7L0qZtq.mjs +0 -3599
- package/dist/pipeline-analytics/virtualExposes-8FzWTdq3.mjs +0 -42
|
@@ -1,934 +1,1009 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
},
|
|
65
|
-
{
|
|
66
|
-
id: "clip-vit-b16",
|
|
67
|
-
name: "CLIP ViT-B/16",
|
|
68
|
-
description: "OpenAI CLIP ViT-B/16 — higher accuracy, 512-dim, int8 quantized (83 MB)",
|
|
69
|
-
inputSize: { width: 224, height: 224 },
|
|
70
|
-
labels: [],
|
|
71
|
-
inputLayout: "nchw",
|
|
72
|
-
inputNormalization: "none",
|
|
73
|
-
formats: {
|
|
74
|
-
onnx: {
|
|
75
|
-
url: "https://huggingface.co/Xenova/clip-vit-base-patch16/resolve/main/onnx/vision_model_quantized.onnx",
|
|
76
|
-
sizeMB: 83
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
},
|
|
80
|
-
{
|
|
81
|
-
id: "siglip2-b16-256",
|
|
82
|
-
name: "SigLIP2 Base/16 256",
|
|
83
|
-
description: "Google SigLIP2 — superior scene understanding, 768-dim, int8 quantized (90 MB)",
|
|
84
|
-
inputSize: { width: 256, height: 256 },
|
|
85
|
-
labels: [],
|
|
86
|
-
inputLayout: "nchw",
|
|
87
|
-
inputNormalization: "none",
|
|
88
|
-
formats: {
|
|
89
|
-
onnx: {
|
|
90
|
-
url: "https://huggingface.co/onnx-community/siglip2-base-patch16-256-ONNX/resolve/main/onnx/vision_model_quantized.onnx",
|
|
91
|
-
sizeMB: 90
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
}
|
|
1
|
+
Object.defineProperties(exports, {
|
|
2
|
+
__esModule: { value: true },
|
|
3
|
+
[Symbol.toStringTag]: { value: "Module" }
|
|
4
|
+
});
|
|
5
|
+
const require_dist = require("../dist-CS2K80so.js");
|
|
6
|
+
let sharp = require("sharp");
|
|
7
|
+
sharp = require_dist.__toESM(sharp);
|
|
8
|
+
let _camstack_core = require("@camstack/core");
|
|
9
|
+
let node_path = require("node:path");
|
|
10
|
+
node_path = require_dist.__toESM(node_path);
|
|
11
|
+
let node_fs = require("node:fs");
|
|
12
|
+
node_fs = require_dist.__toESM(node_fs);
|
|
13
|
+
let node_child_process = require("node:child_process");
|
|
14
|
+
//#region src/embedding-encoder/catalogs/embedding-models.ts
|
|
15
|
+
var CLIP_IMAGE_MODELS = [
|
|
16
|
+
{
|
|
17
|
+
id: "clip-vit-b32",
|
|
18
|
+
name: "CLIP ViT-B/32",
|
|
19
|
+
description: "OpenAI CLIP ViT-B/32 — fast, 512-dim, int8 quantized (85 MB)",
|
|
20
|
+
inputSize: {
|
|
21
|
+
width: 224,
|
|
22
|
+
height: 224
|
|
23
|
+
},
|
|
24
|
+
labels: [],
|
|
25
|
+
inputLayout: "nchw",
|
|
26
|
+
inputNormalization: "none",
|
|
27
|
+
formats: { onnx: {
|
|
28
|
+
url: "https://huggingface.co/Xenova/clip-vit-base-patch32/resolve/main/onnx/vision_model_quantized.onnx",
|
|
29
|
+
sizeMB: 85
|
|
30
|
+
} }
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
id: "clip-vit-b16",
|
|
34
|
+
name: "CLIP ViT-B/16",
|
|
35
|
+
description: "OpenAI CLIP ViT-B/16 — higher accuracy, 512-dim, int8 quantized (83 MB)",
|
|
36
|
+
inputSize: {
|
|
37
|
+
width: 224,
|
|
38
|
+
height: 224
|
|
39
|
+
},
|
|
40
|
+
labels: [],
|
|
41
|
+
inputLayout: "nchw",
|
|
42
|
+
inputNormalization: "none",
|
|
43
|
+
formats: { onnx: {
|
|
44
|
+
url: "https://huggingface.co/Xenova/clip-vit-base-patch16/resolve/main/onnx/vision_model_quantized.onnx",
|
|
45
|
+
sizeMB: 83
|
|
46
|
+
} }
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
id: "siglip2-b16-256",
|
|
50
|
+
name: "SigLIP2 Base/16 256",
|
|
51
|
+
description: "Google SigLIP2 — superior scene understanding, 768-dim, int8 quantized (90 MB)",
|
|
52
|
+
inputSize: {
|
|
53
|
+
width: 256,
|
|
54
|
+
height: 256
|
|
55
|
+
},
|
|
56
|
+
labels: [],
|
|
57
|
+
inputLayout: "nchw",
|
|
58
|
+
inputNormalization: "none",
|
|
59
|
+
formats: { onnx: {
|
|
60
|
+
url: "https://huggingface.co/onnx-community/siglip2-base-patch16-256-ONNX/resolve/main/onnx/vision_model_quantized.onnx",
|
|
61
|
+
sizeMB: 90
|
|
62
|
+
} }
|
|
63
|
+
}
|
|
95
64
|
];
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
65
|
+
var CLIP_TEXT_MODELS = [
|
|
66
|
+
{
|
|
67
|
+
id: "clip-vit-b32-text",
|
|
68
|
+
name: "CLIP ViT-B/32 Text Encoder",
|
|
69
|
+
description: "Text encoder for CLIP ViT-B/32, int8 quantized (62 MB)",
|
|
70
|
+
inputSize: {
|
|
71
|
+
width: 0,
|
|
72
|
+
height: 0
|
|
73
|
+
},
|
|
74
|
+
labels: [],
|
|
75
|
+
formats: { onnx: {
|
|
76
|
+
url: "https://huggingface.co/Xenova/clip-vit-base-patch32/resolve/main/onnx/text_model_quantized.onnx",
|
|
77
|
+
sizeMB: 62
|
|
78
|
+
} }
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
id: "clip-vit-b16-text",
|
|
82
|
+
name: "CLIP ViT-B/16 Text Encoder",
|
|
83
|
+
description: "Text encoder for CLIP ViT-B/16, int8 quantized (62 MB)",
|
|
84
|
+
inputSize: {
|
|
85
|
+
width: 0,
|
|
86
|
+
height: 0
|
|
87
|
+
},
|
|
88
|
+
labels: [],
|
|
89
|
+
formats: { onnx: {
|
|
90
|
+
url: "https://huggingface.co/Xenova/clip-vit-base-patch16/resolve/main/onnx/text_model_quantized.onnx",
|
|
91
|
+
sizeMB: 62
|
|
92
|
+
} }
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
id: "siglip2-b16-256-text",
|
|
96
|
+
name: "SigLIP2 Base/16 256 Text Encoder",
|
|
97
|
+
description: "Text encoder for SigLIP2, int8 quantized (270 MB)",
|
|
98
|
+
inputSize: {
|
|
99
|
+
width: 0,
|
|
100
|
+
height: 0
|
|
101
|
+
},
|
|
102
|
+
labels: [],
|
|
103
|
+
formats: { onnx: {
|
|
104
|
+
url: "https://huggingface.co/onnx-community/siglip2-base-patch16-256-ONNX/resolve/main/onnx/text_model_quantized.onnx",
|
|
105
|
+
sizeMB: 270
|
|
106
|
+
} }
|
|
107
|
+
}
|
|
136
108
|
];
|
|
137
|
-
|
|
138
|
-
|
|
109
|
+
//#endregion
|
|
110
|
+
//#region src/embedding-encoder/shared/noop-logger.ts
|
|
111
|
+
var noop = () => {};
|
|
139
112
|
function createNoopLogger() {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
113
|
+
const logger = {
|
|
114
|
+
debug: noop,
|
|
115
|
+
info: noop,
|
|
116
|
+
warn: noop,
|
|
117
|
+
error: noop,
|
|
118
|
+
child: () => logger,
|
|
119
|
+
withTags: (_tags) => logger
|
|
120
|
+
};
|
|
121
|
+
return logger;
|
|
149
122
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
123
|
+
//#endregion
|
|
124
|
+
//#region src/embedding-encoder/shared/node-raw-tensor-engine.ts
|
|
125
|
+
var BACKEND_TO_DEVICE$1 = {
|
|
126
|
+
cpu: "cpu",
|
|
127
|
+
coreml: "gpu-mps",
|
|
128
|
+
cuda: "gpu-cuda",
|
|
129
|
+
tensorrt: "tensorrt"
|
|
155
130
|
};
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
131
|
+
/**
|
|
132
|
+
* Raw tensor engine — runs ONNX inference on pre-processed Float32Array input.
|
|
133
|
+
* Used by addons that handle their own preprocessing (e.g. CLIP embedding encoder).
|
|
134
|
+
*/
|
|
135
|
+
var NodeRawTensorEngine = class {
|
|
136
|
+
modelPath;
|
|
137
|
+
backend;
|
|
138
|
+
runtime = "onnx";
|
|
139
|
+
device;
|
|
140
|
+
session = null;
|
|
141
|
+
log;
|
|
142
|
+
constructor(modelPath, backend, logger) {
|
|
143
|
+
this.modelPath = modelPath;
|
|
144
|
+
this.backend = backend;
|
|
145
|
+
this.device = BACKEND_TO_DEVICE$1[backend] ?? "cpu";
|
|
146
|
+
this.log = logger ?? createNoopLogger();
|
|
147
|
+
}
|
|
148
|
+
async initialize() {
|
|
149
|
+
const ort = await import("onnxruntime-node");
|
|
150
|
+
const provider = this.backend === "coreml" ? "coreml" : this.backend === "cuda" ? "cuda" : "cpu";
|
|
151
|
+
const absModelPath = node_path.isAbsolute(this.modelPath) ? this.modelPath : node_path.resolve(process.cwd(), this.modelPath);
|
|
152
|
+
this.session = await ort.InferenceSession.create(absModelPath, { executionProviders: [provider] });
|
|
153
|
+
this.log.info("ONNX session loaded", { meta: {
|
|
154
|
+
modelPath: absModelPath,
|
|
155
|
+
backend: this.backend,
|
|
156
|
+
provider
|
|
157
|
+
} });
|
|
158
|
+
}
|
|
159
|
+
async run(input, inputShape) {
|
|
160
|
+
if (!this.session) throw new Error("NodeRawTensorEngine: not initialized — call initialize() first");
|
|
161
|
+
const ort = await import("onnxruntime-node");
|
|
162
|
+
const sess = this.session;
|
|
163
|
+
const inputName = sess.inputNames[0];
|
|
164
|
+
const tensor = new ort.Tensor("float32", input, [...inputShape]);
|
|
165
|
+
const feeds = { [inputName]: tensor };
|
|
166
|
+
const start = Date.now();
|
|
167
|
+
let results;
|
|
168
|
+
try {
|
|
169
|
+
results = await sess.run(feeds);
|
|
170
|
+
} catch (err) {
|
|
171
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
172
|
+
this.log.error("Inference failed", { meta: { error: error.message } });
|
|
173
|
+
throw error;
|
|
174
|
+
}
|
|
175
|
+
const outputName = sess.outputNames[0];
|
|
176
|
+
this.log.debug("Inference complete", { meta: {
|
|
177
|
+
durationMs: Date.now() - start,
|
|
178
|
+
outputKeys: [outputName],
|
|
179
|
+
preprocessMode: "raw-tensor"
|
|
180
|
+
} });
|
|
181
|
+
return results[outputName].data;
|
|
182
|
+
}
|
|
183
|
+
async dispose() {
|
|
184
|
+
this.session = null;
|
|
185
|
+
this.log.debug("Session disposed");
|
|
186
|
+
}
|
|
187
|
+
};
|
|
188
|
+
//#endregion
|
|
189
|
+
//#region src/embedding-encoder/shared/image-utils.ts
|
|
190
|
+
/** Letterbox resize for YOLO: resize preserving aspect ratio, pad to square */
|
|
203
191
|
async function letterbox(jpeg, targetSize) {
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
192
|
+
const meta = await (0, sharp.default)(jpeg).metadata();
|
|
193
|
+
const originalWidth = meta.width ?? 0;
|
|
194
|
+
const originalHeight = meta.height ?? 0;
|
|
195
|
+
const scale = Math.min(targetSize / originalWidth, targetSize / originalHeight);
|
|
196
|
+
const scaledWidth = Math.round(originalWidth * scale);
|
|
197
|
+
const scaledHeight = Math.round(originalHeight * scale);
|
|
198
|
+
const padX = Math.floor((targetSize - scaledWidth) / 2);
|
|
199
|
+
const padY = Math.floor((targetSize - scaledHeight) / 2);
|
|
200
|
+
const { data } = await (0, sharp.default)(jpeg).resize(scaledWidth, scaledHeight).extend({
|
|
201
|
+
top: padY,
|
|
202
|
+
bottom: targetSize - scaledHeight - padY,
|
|
203
|
+
left: padX,
|
|
204
|
+
right: targetSize - scaledWidth - padX,
|
|
205
|
+
background: {
|
|
206
|
+
r: 114,
|
|
207
|
+
g: 114,
|
|
208
|
+
b: 114
|
|
209
|
+
}
|
|
210
|
+
}).removeAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
211
|
+
const numPixels = targetSize * targetSize;
|
|
212
|
+
const float32 = new Float32Array(3 * numPixels);
|
|
213
|
+
for (let i = 0; i < numPixels; i++) {
|
|
214
|
+
const srcBase = i * 3;
|
|
215
|
+
float32[0 * numPixels + i] = data[srcBase] / 255;
|
|
216
|
+
float32[1 * numPixels + i] = data[srcBase + 1] / 255;
|
|
217
|
+
float32[2 * numPixels + i] = data[srcBase + 2] / 255;
|
|
218
|
+
}
|
|
219
|
+
return {
|
|
220
|
+
data: float32,
|
|
221
|
+
scale,
|
|
222
|
+
padX,
|
|
223
|
+
padY,
|
|
224
|
+
originalWidth,
|
|
225
|
+
originalHeight
|
|
226
|
+
};
|
|
228
227
|
}
|
|
228
|
+
/** Resize and normalize to Float32Array */
|
|
229
229
|
async function resizeAndNormalize(jpeg, targetWidth, targetHeight, normalization, layout) {
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
return float32;
|
|
230
|
+
const { data } = await (0, sharp.default)(jpeg).resize(targetWidth, targetHeight, { fit: "fill" }).removeAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
231
|
+
const numPixels = targetWidth * targetHeight;
|
|
232
|
+
const float32 = new Float32Array(3 * numPixels);
|
|
233
|
+
const mean = [
|
|
234
|
+
.485,
|
|
235
|
+
.456,
|
|
236
|
+
.406
|
|
237
|
+
];
|
|
238
|
+
const std = [
|
|
239
|
+
.229,
|
|
240
|
+
.224,
|
|
241
|
+
.225
|
|
242
|
+
];
|
|
243
|
+
if (layout === "nchw") for (let i = 0; i < numPixels; i++) {
|
|
244
|
+
const srcBase = i * 3;
|
|
245
|
+
for (let c = 0; c < 3; c++) {
|
|
246
|
+
const raw = data[srcBase + c] / 255;
|
|
247
|
+
let val;
|
|
248
|
+
if (normalization === "zero-one") val = raw;
|
|
249
|
+
else if (normalization === "imagenet") val = (raw - mean[c]) / std[c];
|
|
250
|
+
else val = data[srcBase + c];
|
|
251
|
+
float32[c * numPixels + i] = val;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
else for (let i = 0; i < numPixels; i++) {
|
|
255
|
+
const srcBase = i * 3;
|
|
256
|
+
for (let c = 0; c < 3; c++) {
|
|
257
|
+
const raw = data[srcBase + c] / 255;
|
|
258
|
+
let val;
|
|
259
|
+
if (normalization === "zero-one") val = raw;
|
|
260
|
+
else if (normalization === "imagenet") val = (raw - mean[c]) / std[c];
|
|
261
|
+
else val = data[srcBase + c];
|
|
262
|
+
float32[i * 3 + c] = val;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
return float32;
|
|
269
266
|
}
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
267
|
+
//#endregion
|
|
268
|
+
//#region src/embedding-encoder/shared/node-engine.ts
|
|
269
|
+
var BACKEND_TO_PROVIDER = {
|
|
270
|
+
cpu: "cpu",
|
|
271
|
+
coreml: "coreml",
|
|
272
|
+
cuda: "cuda",
|
|
273
|
+
tensorrt: "tensorrt",
|
|
274
|
+
dml: "dml"
|
|
276
275
|
};
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
276
|
+
var BACKEND_TO_DEVICE = {
|
|
277
|
+
cpu: "cpu",
|
|
278
|
+
coreml: "gpu-mps",
|
|
279
|
+
cuda: "gpu-cuda",
|
|
280
|
+
tensorrt: "tensorrt"
|
|
282
281
|
};
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
282
|
+
var NodeInferenceEngine = class {
|
|
283
|
+
modelPath;
|
|
284
|
+
backend;
|
|
285
|
+
modelMeta;
|
|
286
|
+
runtime = "onnx";
|
|
287
|
+
device;
|
|
288
|
+
session = null;
|
|
289
|
+
log;
|
|
290
|
+
constructor(modelPath, backend, modelMeta, logger) {
|
|
291
|
+
this.modelPath = modelPath;
|
|
292
|
+
this.backend = backend;
|
|
293
|
+
this.modelMeta = modelMeta;
|
|
294
|
+
this.device = BACKEND_TO_DEVICE[backend] ?? "cpu";
|
|
295
|
+
this.log = logger ?? createNoopLogger();
|
|
296
|
+
}
|
|
297
|
+
async initialize() {
|
|
298
|
+
const ort = await import("onnxruntime-node");
|
|
299
|
+
const provider = BACKEND_TO_PROVIDER[this.backend] ?? "cpu";
|
|
300
|
+
const absModelPath = node_path.isAbsolute(this.modelPath) ? this.modelPath : node_path.resolve(process.cwd(), this.modelPath);
|
|
301
|
+
const sessionOptions = { executionProviders: [provider] };
|
|
302
|
+
this.session = await ort.InferenceSession.create(absModelPath, sessionOptions);
|
|
303
|
+
this.log.info("ONNX session loaded", { meta: {
|
|
304
|
+
modelPath: absModelPath,
|
|
305
|
+
backend: this.backend,
|
|
306
|
+
provider
|
|
307
|
+
} });
|
|
308
|
+
}
|
|
309
|
+
async infer(input) {
|
|
310
|
+
const jpeg = input.kind === "jpeg" ? input.data : await this.encodeRawAsJpeg(input.data, input.width, input.height, input.format);
|
|
311
|
+
const { data, letterboxMeta } = await this.preprocess(jpeg);
|
|
312
|
+
const { inputSize } = this.modelMeta;
|
|
313
|
+
const inputShape = this.modelMeta.preprocessMode === "letterbox" ? [
|
|
314
|
+
1,
|
|
315
|
+
3,
|
|
316
|
+
inputSize.height,
|
|
317
|
+
inputSize.width
|
|
318
|
+
] : [
|
|
319
|
+
1,
|
|
320
|
+
3,
|
|
321
|
+
inputSize.height,
|
|
322
|
+
inputSize.width
|
|
323
|
+
];
|
|
324
|
+
const start = Date.now();
|
|
325
|
+
let result;
|
|
326
|
+
try {
|
|
327
|
+
result = await this.runSession(data, inputShape);
|
|
328
|
+
} catch (err) {
|
|
329
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
330
|
+
this.log.error("Inference failed", { meta: { error: error.message } });
|
|
331
|
+
throw error;
|
|
332
|
+
}
|
|
333
|
+
const durationMs = Date.now() - start;
|
|
334
|
+
if ("tensor" in result) {
|
|
335
|
+
this.log.debug("Inference complete", { meta: {
|
|
336
|
+
durationMs,
|
|
337
|
+
outputKeys: ["tensor"],
|
|
338
|
+
preprocessMode: this.modelMeta.preprocessMode
|
|
339
|
+
} });
|
|
340
|
+
return {
|
|
341
|
+
tensor: result.tensor,
|
|
342
|
+
letterbox: letterboxMeta,
|
|
343
|
+
inferenceMs: durationMs
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
this.log.debug("Inference complete", { meta: {
|
|
347
|
+
durationMs,
|
|
348
|
+
outputKeys: Object.keys(result.tensors),
|
|
349
|
+
preprocessMode: this.modelMeta.preprocessMode
|
|
350
|
+
} });
|
|
351
|
+
return {
|
|
352
|
+
tensors: result.tensors,
|
|
353
|
+
letterbox: letterboxMeta,
|
|
354
|
+
inferenceMs: durationMs
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
/** Preprocess JPEG to Float32Array using the configured mode */
|
|
358
|
+
async preprocess(jpeg) {
|
|
359
|
+
const { inputSize, inputNormalization, inputLayout, preprocessMode } = this.modelMeta;
|
|
360
|
+
if (preprocessMode === "letterbox") {
|
|
361
|
+
const result = await letterbox(jpeg, Math.max(inputSize.width, inputSize.height));
|
|
362
|
+
const letterboxMeta = {
|
|
363
|
+
scale: result.scale,
|
|
364
|
+
padX: result.padX,
|
|
365
|
+
padY: result.padY,
|
|
366
|
+
originalWidth: result.originalWidth,
|
|
367
|
+
originalHeight: result.originalHeight
|
|
368
|
+
};
|
|
369
|
+
return {
|
|
370
|
+
data: result.data,
|
|
371
|
+
letterboxMeta
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
return { data: await resizeAndNormalize(jpeg, inputSize.width, inputSize.height, inputNormalization, inputLayout) };
|
|
375
|
+
}
|
|
376
|
+
async encodeRawAsJpeg(raw, width, height, format) {
|
|
377
|
+
const sharp$3 = (await import("sharp")).default;
|
|
378
|
+
return sharp$3(raw, { raw: {
|
|
379
|
+
width,
|
|
380
|
+
height,
|
|
381
|
+
channels: format === "gray" ? 1 : 3
|
|
382
|
+
} }).jpeg({
|
|
383
|
+
quality: 80,
|
|
384
|
+
mozjpeg: false
|
|
385
|
+
}).toBuffer();
|
|
386
|
+
}
|
|
387
|
+
/** Run an ONNX session with a single input, handling both single and multi-output models */
|
|
388
|
+
async runSession(input, inputShape) {
|
|
389
|
+
if (!this.session) throw new Error("NodeInferenceEngine: not initialized — call initialize() first");
|
|
390
|
+
const ort = await import("onnxruntime-node");
|
|
391
|
+
const sess = this.session;
|
|
392
|
+
const inputName = sess.inputNames[0];
|
|
393
|
+
const tensor = new ort.Tensor("float32", input, [...inputShape]);
|
|
394
|
+
const feeds = { [inputName]: tensor };
|
|
395
|
+
const results = await sess.run(feeds);
|
|
396
|
+
const outputNames = sess.outputNames;
|
|
397
|
+
if (outputNames.length === 1) return { tensor: results[outputNames[0]].data };
|
|
398
|
+
const tensors = {};
|
|
399
|
+
for (const name of outputNames) tensors[name] = results[name].data;
|
|
400
|
+
return { tensors };
|
|
401
|
+
}
|
|
402
|
+
async run(input, inputShape) {
|
|
403
|
+
const result = await this.runSession(input, inputShape);
|
|
404
|
+
if ("tensor" in result) return result.tensor;
|
|
405
|
+
const firstKey = Object.keys(result.tensors)[0];
|
|
406
|
+
return result.tensors[firstKey];
|
|
407
|
+
}
|
|
408
|
+
async dispose() {
|
|
409
|
+
this.session = null;
|
|
410
|
+
this.log.debug("Session disposed");
|
|
411
|
+
}
|
|
412
|
+
};
|
|
413
|
+
//#endregion
|
|
414
|
+
//#region src/embedding-encoder/shared/python-engine.ts
|
|
415
|
+
var PythonInferenceEngine = class {
|
|
416
|
+
pythonPath;
|
|
417
|
+
scriptPath;
|
|
418
|
+
modelPath;
|
|
419
|
+
extraArgs;
|
|
420
|
+
runtime;
|
|
421
|
+
device;
|
|
422
|
+
process = null;
|
|
423
|
+
receiveBuffer = Buffer.alloc(0);
|
|
424
|
+
pendingResolve = null;
|
|
425
|
+
pendingReject = null;
|
|
426
|
+
log;
|
|
427
|
+
constructor(pythonPath, scriptPath, runtime, modelPath, extraArgs = [], logger) {
|
|
428
|
+
this.pythonPath = pythonPath;
|
|
429
|
+
this.scriptPath = scriptPath;
|
|
430
|
+
this.modelPath = modelPath;
|
|
431
|
+
this.extraArgs = extraArgs;
|
|
432
|
+
this.runtime = runtime;
|
|
433
|
+
const runtimeDeviceMap = {
|
|
434
|
+
onnx: "cpu",
|
|
435
|
+
coreml: "gpu-mps",
|
|
436
|
+
pytorch: "cpu",
|
|
437
|
+
openvino: "cpu",
|
|
438
|
+
tflite: "cpu"
|
|
439
|
+
};
|
|
440
|
+
this.device = runtimeDeviceMap[runtime];
|
|
441
|
+
this.log = logger ?? createNoopLogger();
|
|
442
|
+
}
|
|
443
|
+
async initialize() {
|
|
444
|
+
const args = [
|
|
445
|
+
this.scriptPath,
|
|
446
|
+
this.modelPath,
|
|
447
|
+
...this.extraArgs
|
|
448
|
+
];
|
|
449
|
+
this.process = (0, node_child_process.spawn)(this.pythonPath, args, { stdio: [
|
|
450
|
+
"pipe",
|
|
451
|
+
"pipe",
|
|
452
|
+
"pipe"
|
|
453
|
+
] });
|
|
454
|
+
if (!this.process.stdout || !this.process.stdin) throw new Error("PythonInferenceEngine: failed to create process pipes");
|
|
455
|
+
this.log.info("Python process started", { meta: {
|
|
456
|
+
pythonPath: this.pythonPath,
|
|
457
|
+
scriptPath: this.scriptPath,
|
|
458
|
+
modelPath: this.modelPath
|
|
459
|
+
} });
|
|
460
|
+
this.process.stderr?.on("data", (chunk) => {
|
|
461
|
+
const lines = chunk.toString().split("\n");
|
|
462
|
+
for (const line of lines) {
|
|
463
|
+
const trimmed = line.trim();
|
|
464
|
+
if (trimmed) this.log.warn(trimmed);
|
|
465
|
+
}
|
|
466
|
+
});
|
|
467
|
+
this.process.on("error", (err) => {
|
|
468
|
+
this.log.error("Process error", { meta: { error: err.message } });
|
|
469
|
+
this.pendingReject?.(err);
|
|
470
|
+
this.pendingReject = null;
|
|
471
|
+
this.pendingResolve = null;
|
|
472
|
+
});
|
|
473
|
+
this.process.on("exit", (code) => {
|
|
474
|
+
if (code !== 0) {
|
|
475
|
+
this.log.error("Process exited", { meta: { code } });
|
|
476
|
+
const err = /* @__PURE__ */ new Error(`PythonInferenceEngine: process exited with code ${code}`);
|
|
477
|
+
this.pendingReject?.(err);
|
|
478
|
+
this.pendingReject = null;
|
|
479
|
+
this.pendingResolve = null;
|
|
480
|
+
}
|
|
481
|
+
});
|
|
482
|
+
this.process.stdout.on("data", (chunk) => {
|
|
483
|
+
this.receiveBuffer = Buffer.concat([this.receiveBuffer, chunk]);
|
|
484
|
+
this._tryReceive();
|
|
485
|
+
});
|
|
486
|
+
await new Promise((resolve, reject) => {
|
|
487
|
+
const timeout = setTimeout(() => resolve(), 2e3);
|
|
488
|
+
this.process?.on("error", (err) => {
|
|
489
|
+
clearTimeout(timeout);
|
|
490
|
+
reject(err);
|
|
491
|
+
});
|
|
492
|
+
this.process?.on("exit", (code) => {
|
|
493
|
+
clearTimeout(timeout);
|
|
494
|
+
if (code !== 0) reject(/* @__PURE__ */ new Error(`PythonInferenceEngine: process exited early with code ${code}`));
|
|
495
|
+
});
|
|
496
|
+
});
|
|
497
|
+
}
|
|
498
|
+
_tryReceive() {
|
|
499
|
+
if (this.receiveBuffer.length < 4) return;
|
|
500
|
+
const length = this.receiveBuffer.readUInt32LE(0);
|
|
501
|
+
if (this.receiveBuffer.length < 4 + length) return;
|
|
502
|
+
const jsonBytes = this.receiveBuffer.subarray(4, 4 + length);
|
|
503
|
+
this.receiveBuffer = this.receiveBuffer.subarray(4 + length);
|
|
504
|
+
const resolve = this.pendingResolve;
|
|
505
|
+
const reject = this.pendingReject;
|
|
506
|
+
this.pendingResolve = null;
|
|
507
|
+
this.pendingReject = null;
|
|
508
|
+
if (!resolve) return;
|
|
509
|
+
try {
|
|
510
|
+
resolve(JSON.parse(jsonBytes.toString("utf8")));
|
|
511
|
+
} catch (err) {
|
|
512
|
+
reject?.(err instanceof Error ? err : new Error(String(err)));
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
/** Run inference, returning structured detection results. Encodes raw input to JPEG when needed. */
|
|
516
|
+
async infer(input) {
|
|
517
|
+
const start = Date.now();
|
|
518
|
+
const jpeg = input.kind === "jpeg" ? input.data : await this.encodeRawAsJpeg(input.data, input.width, input.height, input.format);
|
|
519
|
+
const result = await this.sendJpeg(jpeg);
|
|
520
|
+
const durationMs = Date.now() - start;
|
|
521
|
+
this.log.debug("Inference complete", { meta: { durationMs } });
|
|
522
|
+
return {
|
|
523
|
+
structured: result,
|
|
524
|
+
inferenceMs: durationMs
|
|
525
|
+
};
|
|
526
|
+
}
|
|
527
|
+
async encodeRawAsJpeg(raw, width, height, format) {
|
|
528
|
+
const sharp$2 = (await import("sharp")).default;
|
|
529
|
+
return sharp$2(raw, { raw: {
|
|
530
|
+
width,
|
|
531
|
+
height,
|
|
532
|
+
channels: format === "gray" ? 1 : 3
|
|
533
|
+
} }).jpeg({
|
|
534
|
+
quality: 80,
|
|
535
|
+
mozjpeg: false
|
|
536
|
+
}).toBuffer();
|
|
537
|
+
}
|
|
538
|
+
/** Send JPEG buffer via binary IPC, receive JSON detection results */
|
|
539
|
+
async sendJpeg(jpeg) {
|
|
540
|
+
if (!this.process?.stdin) throw new Error("PythonInferenceEngine: process not initialized");
|
|
541
|
+
return new Promise((resolve, reject) => {
|
|
542
|
+
this.pendingResolve = resolve;
|
|
543
|
+
this.pendingReject = reject;
|
|
544
|
+
const lengthBuf = Buffer.allocUnsafe(4);
|
|
545
|
+
lengthBuf.writeUInt32LE(jpeg.length, 0);
|
|
546
|
+
this.process.stdin.write(Buffer.concat([lengthBuf, jpeg]));
|
|
547
|
+
});
|
|
548
|
+
}
|
|
549
|
+
async dispose() {
|
|
550
|
+
const proc = this.process;
|
|
551
|
+
if (!proc) return;
|
|
552
|
+
this.process = null;
|
|
553
|
+
proc.stdin?.end();
|
|
554
|
+
proc.kill("SIGTERM");
|
|
555
|
+
if (!await new Promise((resolve) => {
|
|
556
|
+
const timer = setTimeout(() => {
|
|
557
|
+
resolve(false);
|
|
558
|
+
}, 5e3);
|
|
559
|
+
proc.once("exit", () => {
|
|
560
|
+
clearTimeout(timer);
|
|
561
|
+
resolve(true);
|
|
562
|
+
});
|
|
563
|
+
})) {
|
|
564
|
+
try {
|
|
565
|
+
proc.kill("SIGKILL");
|
|
566
|
+
} catch {}
|
|
567
|
+
this.log.warn("Python process did not exit gracefully — sent SIGKILL");
|
|
568
|
+
} else this.log.debug("Python process terminated");
|
|
569
|
+
}
|
|
570
|
+
};
|
|
571
|
+
//#endregion
|
|
572
|
+
//#region src/embedding-encoder/shared/engine-resolver.ts
|
|
573
|
+
/** Priority order for auto-selection of ONNX backends */
|
|
574
|
+
var AUTO_BACKEND_PRIORITY = [
|
|
575
|
+
"coreml",
|
|
576
|
+
"cuda",
|
|
577
|
+
"tensorrt",
|
|
578
|
+
"cpu"
|
|
579
|
+
];
|
|
580
|
+
var BACKEND_TO_FORMAT = require_dist.BACKEND_TO_FORMAT;
|
|
581
|
+
var RUNTIME_TO_FORMAT = require_dist.RUNTIME_TO_FORMAT;
|
|
538
582
|
function extractModelMeta(entry) {
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
583
|
+
return {
|
|
584
|
+
inputSize: entry.inputSize,
|
|
585
|
+
inputNormalization: entry.inputNormalization ?? "zero-one",
|
|
586
|
+
inputLayout: entry.inputLayout ?? "nchw",
|
|
587
|
+
preprocessMode: entry.preprocessMode ?? "letterbox"
|
|
588
|
+
};
|
|
545
589
|
}
|
|
546
590
|
function modelFilePath(modelsDir, modelEntry, format) {
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
const filename = urlParts[urlParts.length - 1] ?? `${modelEntry.id}.${format}`;
|
|
553
|
-
return path__namespace.join(modelsDir, filename);
|
|
591
|
+
const formatEntry = modelEntry.formats[format];
|
|
592
|
+
if (!formatEntry) throw new Error(`Model ${modelEntry.id} has no ${format} format`);
|
|
593
|
+
const urlParts = formatEntry.url.split("/");
|
|
594
|
+
const filename = urlParts[urlParts.length - 1] ?? `${modelEntry.id}.${format}`;
|
|
595
|
+
return node_path.join(modelsDir, filename);
|
|
554
596
|
}
|
|
555
597
|
function modelExists(filePath) {
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
598
|
+
try {
|
|
599
|
+
return node_fs.existsSync(filePath);
|
|
600
|
+
} catch {
|
|
601
|
+
return false;
|
|
602
|
+
}
|
|
561
603
|
}
|
|
562
604
|
async function resolveEngine(options) {
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
const engine = new NodeInferenceEngine(fallbackPath, "cpu", extractModelMeta(modelEntry), options.logger);
|
|
659
|
-
await engine.initialize();
|
|
660
|
-
return { engine, format: "onnx", modelPath: fallbackPath };
|
|
661
|
-
}
|
|
662
|
-
throw new Error(
|
|
663
|
-
`resolveEngine: format ${selectedFormat} is not yet supported by NodeInferenceEngine, no Python runtime is available, and no ONNX fallback exists`
|
|
664
|
-
);
|
|
605
|
+
const { runtime, backend, modelEntry, modelsDir, models } = options;
|
|
606
|
+
const log = options.logger ?? createNoopLogger();
|
|
607
|
+
let selectedFormat;
|
|
608
|
+
let selectedBackend;
|
|
609
|
+
if (runtime === "auto") {
|
|
610
|
+
const available = await probeOnnxBackends();
|
|
611
|
+
let chosen = null;
|
|
612
|
+
for (const b of AUTO_BACKEND_PRIORITY) {
|
|
613
|
+
if (!available.includes(b)) continue;
|
|
614
|
+
const fmt = BACKEND_TO_FORMAT[b];
|
|
615
|
+
if (!fmt) continue;
|
|
616
|
+
if (!modelEntry.formats[fmt]) continue;
|
|
617
|
+
chosen = {
|
|
618
|
+
backend: b,
|
|
619
|
+
format: fmt
|
|
620
|
+
};
|
|
621
|
+
break;
|
|
622
|
+
}
|
|
623
|
+
if (!chosen) throw new Error(`resolveEngine: no compatible backend found for model ${modelEntry.id}. Available backends: ${available.join(", ")}`);
|
|
624
|
+
selectedFormat = chosen.format;
|
|
625
|
+
selectedBackend = chosen.backend;
|
|
626
|
+
} else {
|
|
627
|
+
const fmt = RUNTIME_TO_FORMAT[runtime];
|
|
628
|
+
if (!fmt) throw new Error(`resolveEngine: unsupported runtime "${runtime}"`);
|
|
629
|
+
if (!modelEntry.formats[fmt]) if (fmt !== "onnx" && modelEntry.formats["onnx"]) {
|
|
630
|
+
selectedFormat = "onnx";
|
|
631
|
+
selectedBackend = backend || "cpu";
|
|
632
|
+
} else throw new Error(`resolveEngine: model ${modelEntry.id} has no ${fmt} format for runtime ${runtime}`);
|
|
633
|
+
else {
|
|
634
|
+
selectedFormat = fmt;
|
|
635
|
+
selectedBackend = runtime === "onnx" ? backend || "cpu" : runtime;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
let modelPath;
|
|
639
|
+
if (models) modelPath = await models.ensure(modelEntry.id, selectedFormat);
|
|
640
|
+
else {
|
|
641
|
+
modelPath = modelFilePath(modelsDir, modelEntry, selectedFormat);
|
|
642
|
+
if (!modelExists(modelPath)) throw new Error(`resolveEngine: model file not found at ${modelPath} and no model service provided`);
|
|
643
|
+
}
|
|
644
|
+
log.info("Engine resolved", { meta: {
|
|
645
|
+
format: selectedFormat,
|
|
646
|
+
backend: selectedBackend,
|
|
647
|
+
modelId: modelEntry.id
|
|
648
|
+
} });
|
|
649
|
+
if (selectedFormat === "onnx") {
|
|
650
|
+
const engine = new NodeInferenceEngine(modelPath, selectedBackend, extractModelMeta(modelEntry), options.logger);
|
|
651
|
+
await engine.initialize();
|
|
652
|
+
return {
|
|
653
|
+
engine,
|
|
654
|
+
format: selectedFormat,
|
|
655
|
+
modelPath
|
|
656
|
+
};
|
|
657
|
+
}
|
|
658
|
+
const effectiveRuntime = runtime === "auto" ? selectedBackend : runtime;
|
|
659
|
+
let { pythonPath } = options;
|
|
660
|
+
if (!pythonPath) {
|
|
661
|
+
const { execFileSync: efs } = await import("node:child_process");
|
|
662
|
+
for (const cmd of ["python3", "python"]) try {
|
|
663
|
+
efs(cmd, ["--version"], {
|
|
664
|
+
timeout: 3e3,
|
|
665
|
+
stdio: "ignore"
|
|
666
|
+
});
|
|
667
|
+
pythonPath = cmd;
|
|
668
|
+
break;
|
|
669
|
+
} catch {}
|
|
670
|
+
}
|
|
671
|
+
const scriptName = require_dist.PYTHON_SCRIPT[effectiveRuntime];
|
|
672
|
+
if (scriptName && pythonPath) {
|
|
673
|
+
const candidates = [
|
|
674
|
+
node_path.join(__dirname, "../../python", scriptName),
|
|
675
|
+
node_path.join(__dirname, "../python", scriptName),
|
|
676
|
+
node_path.join(__dirname, "../../../python", scriptName)
|
|
677
|
+
];
|
|
678
|
+
const scriptPath = candidates.find((p) => node_fs.existsSync(p));
|
|
679
|
+
if (!scriptPath) throw new Error(`resolveEngine: Python script "${scriptName}" not found. Searched:\n${candidates.join("\n")}`);
|
|
680
|
+
const inputSize = Math.max(modelEntry.inputSize.width, modelEntry.inputSize.height);
|
|
681
|
+
const engine = new PythonInferenceEngine(pythonPath, scriptPath, effectiveRuntime, modelPath, [`--input-size=${inputSize}`, `--confidence=0.25`], options.logger);
|
|
682
|
+
await engine.initialize();
|
|
683
|
+
return {
|
|
684
|
+
engine,
|
|
685
|
+
format: selectedFormat,
|
|
686
|
+
modelPath
|
|
687
|
+
};
|
|
688
|
+
}
|
|
689
|
+
const fallbackPath = modelFilePath(modelsDir, modelEntry, "onnx");
|
|
690
|
+
if (modelEntry.formats["onnx"] && modelExists(fallbackPath)) {
|
|
691
|
+
const engine = new NodeInferenceEngine(fallbackPath, "cpu", extractModelMeta(modelEntry), options.logger);
|
|
692
|
+
await engine.initialize();
|
|
693
|
+
return {
|
|
694
|
+
engine,
|
|
695
|
+
format: "onnx",
|
|
696
|
+
modelPath: fallbackPath
|
|
697
|
+
};
|
|
698
|
+
}
|
|
699
|
+
throw new Error(`resolveEngine: format ${selectedFormat} is not yet supported by NodeInferenceEngine, no Python runtime is available, and no ONNX fallback exists`);
|
|
665
700
|
}
|
|
701
|
+
/** Probe which ONNX execution providers are available on this system */
|
|
666
702
|
async function probeOnnxBackends() {
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
available.push("coreml");
|
|
681
|
-
}
|
|
682
|
-
return [...new Set(available)];
|
|
703
|
+
const available = ["cpu"];
|
|
704
|
+
try {
|
|
705
|
+
const ort = await import("onnxruntime-node");
|
|
706
|
+
const providers = ort.env?.webgl?.disabled !== void 0 ? ort.InferenceSession.getAvailableProviders?.() ?? [] : [];
|
|
707
|
+
for (const p of providers) {
|
|
708
|
+
const normalized = p.toLowerCase().replace("executionprovider", "");
|
|
709
|
+
if (normalized === "coreml") available.push("coreml");
|
|
710
|
+
else if (normalized === "cuda") available.push("cuda");
|
|
711
|
+
else if (normalized === "tensorrt") available.push("tensorrt");
|
|
712
|
+
}
|
|
713
|
+
} catch {}
|
|
714
|
+
if (process.platform === "darwin" && !available.includes("coreml")) available.push("coreml");
|
|
715
|
+
return [...new Set(available)];
|
|
683
716
|
}
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
717
|
+
//#endregion
|
|
718
|
+
//#region src/embedding-encoder/addon/clip-models.ts
|
|
719
|
+
var CLIP_MODEL_META = {
|
|
720
|
+
"clip-vit-b32": {
|
|
721
|
+
imageModelId: "clip-vit-b32",
|
|
722
|
+
textModelId: "clip-vit-b32-text",
|
|
723
|
+
embeddingDim: 512,
|
|
724
|
+
inputSize: 224,
|
|
725
|
+
tokenizerType: "clip"
|
|
726
|
+
},
|
|
727
|
+
"clip-vit-b16": {
|
|
728
|
+
imageModelId: "clip-vit-b16",
|
|
729
|
+
textModelId: "clip-vit-b16-text",
|
|
730
|
+
embeddingDim: 512,
|
|
731
|
+
inputSize: 224,
|
|
732
|
+
tokenizerType: "clip"
|
|
733
|
+
},
|
|
734
|
+
"siglip2-b16-256": {
|
|
735
|
+
imageModelId: "siglip2-b16-256",
|
|
736
|
+
textModelId: "siglip2-b16-256-text",
|
|
737
|
+
embeddingDim: 768,
|
|
738
|
+
inputSize: 256,
|
|
739
|
+
tokenizerType: "siglip"
|
|
740
|
+
}
|
|
706
741
|
};
|
|
707
|
-
|
|
742
|
+
var DEFAULT_CLIP_MODEL = "clip-vit-b32";
|
|
708
743
|
function getModelMeta(modelId) {
|
|
709
|
-
|
|
744
|
+
return CLIP_MODEL_META[modelId] ?? CLIP_MODEL_META["clip-vit-b32"];
|
|
710
745
|
}
|
|
711
|
-
|
|
712
|
-
|
|
746
|
+
//#endregion
|
|
747
|
+
//#region src/embedding-encoder/addon/clip-preprocessing.ts
|
|
748
|
+
var CLIP_MEAN = [
|
|
749
|
+
.48145466,
|
|
750
|
+
.4578275,
|
|
751
|
+
.40821073
|
|
752
|
+
];
|
|
753
|
+
var CLIP_STD = [
|
|
754
|
+
.26862954,
|
|
755
|
+
.26130258,
|
|
756
|
+
.27577711
|
|
757
|
+
];
|
|
758
|
+
/**
|
|
759
|
+
* Preprocess raw RGB buffer for CLIP inference.
|
|
760
|
+
* Resizes (nearest-neighbor for speed), normalizes with CLIP mean/std, outputs NCHW Float32Array.
|
|
761
|
+
* For production use, the caller should use sharp to resize the JPEG to targetW×targetH
|
|
762
|
+
* before calling this with the raw RGB. This function handles normalization + layout.
|
|
763
|
+
*/
|
|
713
764
|
function preprocessForClip(rgb, srcWidth, srcHeight, targetWidth, targetHeight) {
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
return result;
|
|
765
|
+
const pixels = targetWidth * targetHeight;
|
|
766
|
+
const result = new Float32Array(3 * pixels);
|
|
767
|
+
for (let y = 0; y < targetHeight; y++) for (let x = 0; x < targetWidth; x++) {
|
|
768
|
+
const srcX = Math.min(Math.floor(x / targetWidth * srcWidth), srcWidth - 1);
|
|
769
|
+
const srcIdx = (Math.min(Math.floor(y / targetHeight * srcHeight), srcHeight - 1) * srcWidth + srcX) * 3;
|
|
770
|
+
const dstIdx = y * targetWidth + x;
|
|
771
|
+
for (let c = 0; c < 3; c++) {
|
|
772
|
+
const val = (rgb[srcIdx + c] ?? 0) / 255;
|
|
773
|
+
result[c * pixels + dstIdx] = (val - CLIP_MEAN[c]) / CLIP_STD[c];
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
return result;
|
|
729
777
|
}
|
|
778
|
+
/**
|
|
779
|
+
* L2-normalize a vector in-place and return it.
|
|
780
|
+
*/
|
|
730
781
|
function l2Normalize(vec) {
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
}
|
|
737
|
-
return vec;
|
|
738
|
-
}
|
|
739
|
-
class EmbeddingEncoderAddon extends index.BaseAddon {
|
|
740
|
-
imageRawEngine = null;
|
|
741
|
-
textRawEngine = null;
|
|
742
|
-
imagePythonEngine = null;
|
|
743
|
-
textPythonEngine = null;
|
|
744
|
-
models = null;
|
|
745
|
-
isPython = false;
|
|
746
|
-
constructor() {
|
|
747
|
-
super({ modelId: DEFAULT_CLIP_MODEL, runtime: "auto", backend: "cpu" });
|
|
748
|
-
}
|
|
749
|
-
async onInitialize() {
|
|
750
|
-
const modelsDir = await this.ctx.api.storage.resolve.query({ location: "models", relativePath: "" }).catch(() => "camstack-data/models");
|
|
751
|
-
this.models = new core.ModelDownloadService(modelsDir, []);
|
|
752
|
-
return [{ capability: index.embeddingEncoderCapability, provider: this }];
|
|
753
|
-
}
|
|
754
|
-
async encode(input) {
|
|
755
|
-
const { crop, width, height } = input;
|
|
756
|
-
await this.ensureImageEngine();
|
|
757
|
-
const meta = getModelMeta(this.config.modelId);
|
|
758
|
-
const start = Date.now();
|
|
759
|
-
if (this.isPython && this.imagePythonEngine) {
|
|
760
|
-
const jpegBuffer = Buffer.isBuffer(crop) ? crop : Buffer.from(crop);
|
|
761
|
-
const result = await this.imagePythonEngine.infer({ kind: "jpeg", data: jpegBuffer });
|
|
762
|
-
const rawEmbedding = result.structured?.["embedding"];
|
|
763
|
-
const normalized2 = l2Normalize(new Float32Array(rawEmbedding));
|
|
764
|
-
return {
|
|
765
|
-
embedding: Array.from(normalized2),
|
|
766
|
-
inferenceMs: result.inferenceMs ?? Date.now() - start
|
|
767
|
-
};
|
|
768
|
-
}
|
|
769
|
-
const cropBuffer = Buffer.isBuffer(crop) ? crop : Buffer.from(crop);
|
|
770
|
-
const preprocessed = preprocessForClip(cropBuffer, width, height, meta.inputSize, meta.inputSize);
|
|
771
|
-
const output = await this.imageRawEngine.run(preprocessed, [1, 3, meta.inputSize, meta.inputSize]);
|
|
772
|
-
const sliced = output.length > meta.embeddingDim ? output.slice(0, meta.embeddingDim) : output;
|
|
773
|
-
const normalized = l2Normalize(new Float32Array(sliced));
|
|
774
|
-
return {
|
|
775
|
-
embedding: Array.from(normalized),
|
|
776
|
-
inferenceMs: Date.now() - start
|
|
777
|
-
};
|
|
778
|
-
}
|
|
779
|
-
async encodeText(input) {
|
|
780
|
-
const { text } = input;
|
|
781
|
-
await this.ensureTextEngine();
|
|
782
|
-
const meta = getModelMeta(this.config.modelId);
|
|
783
|
-
const start = Date.now();
|
|
784
|
-
if (this.isPython && this.textPythonEngine) {
|
|
785
|
-
const textBuffer = Buffer.from(JSON.stringify({ text }), "utf-8");
|
|
786
|
-
const result = await this.textPythonEngine.infer({ kind: "jpeg", data: textBuffer });
|
|
787
|
-
const rawEmbedding = result.structured?.["embedding"];
|
|
788
|
-
const normalized2 = l2Normalize(new Float32Array(rawEmbedding));
|
|
789
|
-
return {
|
|
790
|
-
embedding: Array.from(normalized2),
|
|
791
|
-
inferenceMs: result.inferenceMs ?? Date.now() - start
|
|
792
|
-
};
|
|
793
|
-
}
|
|
794
|
-
const tokenIds = clipTokenize(text);
|
|
795
|
-
const inputTensor = new Float32Array(tokenIds);
|
|
796
|
-
const output = await this.textRawEngine.run(inputTensor, [1, tokenIds.length]);
|
|
797
|
-
const sliced = output.length > meta.embeddingDim ? output.slice(0, meta.embeddingDim) : output;
|
|
798
|
-
const normalized = l2Normalize(new Float32Array(sliced));
|
|
799
|
-
return {
|
|
800
|
-
embedding: Array.from(normalized),
|
|
801
|
-
inferenceMs: Date.now() - start
|
|
802
|
-
};
|
|
803
|
-
}
|
|
804
|
-
async getInfo() {
|
|
805
|
-
const meta = getModelMeta(this.config.modelId);
|
|
806
|
-
return {
|
|
807
|
-
modelId: this.config.modelId,
|
|
808
|
-
embeddingDim: meta.embeddingDim,
|
|
809
|
-
ready: this.imageRawEngine !== null || this.imagePythonEngine !== null
|
|
810
|
-
};
|
|
811
|
-
}
|
|
812
|
-
async ensureImageEngine() {
|
|
813
|
-
if (this.imageRawEngine || this.imagePythonEngine) return;
|
|
814
|
-
const meta = getModelMeta(this.config.modelId);
|
|
815
|
-
const imageEntry = CLIP_IMAGE_MODELS.find((m) => m.id === meta.imageModelId);
|
|
816
|
-
if (!imageEntry) {
|
|
817
|
-
throw new Error(`EmbeddingEncoderAddon: unknown image model "${meta.imageModelId}"`);
|
|
818
|
-
}
|
|
819
|
-
await this.resolveForEntry(imageEntry, "image");
|
|
820
|
-
}
|
|
821
|
-
async ensureTextEngine() {
|
|
822
|
-
if (this.textRawEngine || this.textPythonEngine) return;
|
|
823
|
-
const meta = getModelMeta(this.config.modelId);
|
|
824
|
-
const textEntry = CLIP_TEXT_MODELS.find((m) => m.id === meta.textModelId);
|
|
825
|
-
if (!textEntry) {
|
|
826
|
-
throw new Error(`EmbeddingEncoderAddon: unknown text model "${meta.textModelId}"`);
|
|
827
|
-
}
|
|
828
|
-
await this.resolveForEntry(textEntry, "text");
|
|
829
|
-
}
|
|
830
|
-
async resolveForEntry(entry, target) {
|
|
831
|
-
const runtime = this.config.runtime === "auto" ? "auto" : this.config.runtime === "node" ? "onnx" : this.config.runtime;
|
|
832
|
-
const modelsDir = this.models.getModelsDir();
|
|
833
|
-
const engineLogger = this.ctx.logger.withTags({
|
|
834
|
-
modelId: entry.id,
|
|
835
|
-
runtime: this.config.runtime,
|
|
836
|
-
backend: this.config.backend
|
|
837
|
-
});
|
|
838
|
-
await this.models.ensure(entry.id, "onnx");
|
|
839
|
-
const resolved = await resolveEngine({
|
|
840
|
-
runtime,
|
|
841
|
-
backend: this.config.backend,
|
|
842
|
-
modelEntry: entry,
|
|
843
|
-
modelsDir,
|
|
844
|
-
models: this.models ?? void 0,
|
|
845
|
-
logger: engineLogger
|
|
846
|
-
});
|
|
847
|
-
if (resolved.format !== "onnx") {
|
|
848
|
-
this.isPython = true;
|
|
849
|
-
if (target === "image") {
|
|
850
|
-
this.imagePythonEngine = resolved.engine;
|
|
851
|
-
} else {
|
|
852
|
-
this.textPythonEngine = resolved.engine;
|
|
853
|
-
}
|
|
854
|
-
} else {
|
|
855
|
-
const rawEngine = new NodeRawTensorEngine(resolved.modelPath, this.config.backend, engineLogger);
|
|
856
|
-
await rawEngine.initialize();
|
|
857
|
-
await resolved.engine.dispose();
|
|
858
|
-
if (target === "image") {
|
|
859
|
-
this.imageRawEngine = rawEngine;
|
|
860
|
-
} else {
|
|
861
|
-
this.textRawEngine = rawEngine;
|
|
862
|
-
}
|
|
863
|
-
}
|
|
864
|
-
}
|
|
865
|
-
async onShutdown() {
|
|
866
|
-
await this.imageRawEngine?.dispose();
|
|
867
|
-
await this.textRawEngine?.dispose();
|
|
868
|
-
await this.imagePythonEngine?.dispose();
|
|
869
|
-
await this.textPythonEngine?.dispose();
|
|
870
|
-
}
|
|
871
|
-
// ── Three-level settings API (Phase 3) ──────────────────────────────
|
|
872
|
-
globalSettingsSchema() {
|
|
873
|
-
return this.schema({
|
|
874
|
-
sections: [
|
|
875
|
-
{
|
|
876
|
-
id: "embedding-encoder-settings",
|
|
877
|
-
title: "Embedding Encoder",
|
|
878
|
-
columns: 2,
|
|
879
|
-
fields: [
|
|
880
|
-
{
|
|
881
|
-
type: "text",
|
|
882
|
-
key: "modelId",
|
|
883
|
-
label: "Model ID",
|
|
884
|
-
description: "CLIP model identifier to use for image/text embedding",
|
|
885
|
-
default: DEFAULT_CLIP_MODEL
|
|
886
|
-
},
|
|
887
|
-
{
|
|
888
|
-
type: "select",
|
|
889
|
-
key: "runtime",
|
|
890
|
-
label: "Runtime",
|
|
891
|
-
description: "Inference runtime (auto selects the best available)",
|
|
892
|
-
default: "auto",
|
|
893
|
-
options: [
|
|
894
|
-
{ label: "Auto", value: "auto" },
|
|
895
|
-
{ label: "Node (ONNX)", value: "node" },
|
|
896
|
-
{ label: "Python", value: "python" }
|
|
897
|
-
]
|
|
898
|
-
},
|
|
899
|
-
{
|
|
900
|
-
type: "select",
|
|
901
|
-
key: "backend",
|
|
902
|
-
label: "Backend",
|
|
903
|
-
description: "Hardware backend for inference acceleration",
|
|
904
|
-
default: "cpu",
|
|
905
|
-
options: [
|
|
906
|
-
{ label: "CPU", value: "cpu" },
|
|
907
|
-
{ label: "CUDA", value: "cuda" },
|
|
908
|
-
{ label: "CoreML", value: "coreml" }
|
|
909
|
-
]
|
|
910
|
-
}
|
|
911
|
-
]
|
|
912
|
-
}
|
|
913
|
-
]
|
|
914
|
-
});
|
|
915
|
-
}
|
|
916
|
-
async onConfigChanged() {
|
|
917
|
-
}
|
|
782
|
+
let norm = 0;
|
|
783
|
+
for (let i = 0; i < vec.length; i++) norm += vec[i] * vec[i];
|
|
784
|
+
norm = Math.sqrt(norm);
|
|
785
|
+
if (norm > 0) for (let i = 0; i < vec.length; i++) vec[i] /= norm;
|
|
786
|
+
return vec;
|
|
918
787
|
}
|
|
788
|
+
//#endregion
|
|
789
|
+
//#region src/embedding-encoder/addon/index.ts
|
|
790
|
+
var EmbeddingEncoderAddon = class extends require_dist.BaseAddon {
|
|
791
|
+
imageRawEngine = null;
|
|
792
|
+
textRawEngine = null;
|
|
793
|
+
imagePythonEngine = null;
|
|
794
|
+
textPythonEngine = null;
|
|
795
|
+
models = null;
|
|
796
|
+
isPython = false;
|
|
797
|
+
constructor() {
|
|
798
|
+
super({
|
|
799
|
+
modelId: DEFAULT_CLIP_MODEL,
|
|
800
|
+
runtime: "auto",
|
|
801
|
+
backend: "cpu"
|
|
802
|
+
});
|
|
803
|
+
}
|
|
804
|
+
async onInitialize() {
|
|
805
|
+
const modelsDir = await this.ctx.api.storage.resolve.query({
|
|
806
|
+
location: "models",
|
|
807
|
+
relativePath: ""
|
|
808
|
+
}).catch(() => "camstack-data/models");
|
|
809
|
+
this.models = new _camstack_core.ModelDownloadService(modelsDir, []);
|
|
810
|
+
return [{
|
|
811
|
+
capability: require_dist.embeddingEncoderCapability,
|
|
812
|
+
provider: this
|
|
813
|
+
}];
|
|
814
|
+
}
|
|
815
|
+
async encode(input) {
|
|
816
|
+
const { crop, width, height } = input;
|
|
817
|
+
await this.ensureImageEngine();
|
|
818
|
+
const meta = getModelMeta(this.config.modelId);
|
|
819
|
+
const start = Date.now();
|
|
820
|
+
if (this.isPython && this.imagePythonEngine) {
|
|
821
|
+
const jpegBuffer = Buffer.isBuffer(crop) ? crop : Buffer.from(crop);
|
|
822
|
+
const result = await this.imagePythonEngine.infer({
|
|
823
|
+
kind: "jpeg",
|
|
824
|
+
data: jpegBuffer
|
|
825
|
+
});
|
|
826
|
+
const rawEmbedding = result.structured?.["embedding"];
|
|
827
|
+
const normalized = l2Normalize(new Float32Array(rawEmbedding));
|
|
828
|
+
return {
|
|
829
|
+
embedding: Array.from(normalized),
|
|
830
|
+
inferenceMs: result.inferenceMs ?? Date.now() - start
|
|
831
|
+
};
|
|
832
|
+
}
|
|
833
|
+
const preprocessed = preprocessForClip(Buffer.isBuffer(crop) ? crop : Buffer.from(crop), width, height, meta.inputSize, meta.inputSize);
|
|
834
|
+
const output = await this.imageRawEngine.run(preprocessed, [
|
|
835
|
+
1,
|
|
836
|
+
3,
|
|
837
|
+
meta.inputSize,
|
|
838
|
+
meta.inputSize
|
|
839
|
+
]);
|
|
840
|
+
const sliced = output.length > meta.embeddingDim ? output.slice(0, meta.embeddingDim) : output;
|
|
841
|
+
const normalized = l2Normalize(new Float32Array(sliced));
|
|
842
|
+
return {
|
|
843
|
+
embedding: Array.from(normalized),
|
|
844
|
+
inferenceMs: Date.now() - start
|
|
845
|
+
};
|
|
846
|
+
}
|
|
847
|
+
async encodeText(input) {
|
|
848
|
+
const { text } = input;
|
|
849
|
+
await this.ensureTextEngine();
|
|
850
|
+
const meta = getModelMeta(this.config.modelId);
|
|
851
|
+
const start = Date.now();
|
|
852
|
+
if (this.isPython && this.textPythonEngine) {
|
|
853
|
+
const textBuffer = Buffer.from(JSON.stringify({ text }), "utf-8");
|
|
854
|
+
const result = await this.textPythonEngine.infer({
|
|
855
|
+
kind: "jpeg",
|
|
856
|
+
data: textBuffer
|
|
857
|
+
});
|
|
858
|
+
const rawEmbedding = result.structured?.["embedding"];
|
|
859
|
+
const normalized = l2Normalize(new Float32Array(rawEmbedding));
|
|
860
|
+
return {
|
|
861
|
+
embedding: Array.from(normalized),
|
|
862
|
+
inferenceMs: result.inferenceMs ?? Date.now() - start
|
|
863
|
+
};
|
|
864
|
+
}
|
|
865
|
+
const tokenIds = clipTokenize(text);
|
|
866
|
+
const inputTensor = new Float32Array(tokenIds);
|
|
867
|
+
const output = await this.textRawEngine.run(inputTensor, [1, tokenIds.length]);
|
|
868
|
+
const sliced = output.length > meta.embeddingDim ? output.slice(0, meta.embeddingDim) : output;
|
|
869
|
+
const normalized = l2Normalize(new Float32Array(sliced));
|
|
870
|
+
return {
|
|
871
|
+
embedding: Array.from(normalized),
|
|
872
|
+
inferenceMs: Date.now() - start
|
|
873
|
+
};
|
|
874
|
+
}
|
|
875
|
+
async getInfo() {
|
|
876
|
+
const meta = getModelMeta(this.config.modelId);
|
|
877
|
+
return {
|
|
878
|
+
modelId: this.config.modelId,
|
|
879
|
+
embeddingDim: meta.embeddingDim,
|
|
880
|
+
ready: this.imageRawEngine !== null || this.imagePythonEngine !== null
|
|
881
|
+
};
|
|
882
|
+
}
|
|
883
|
+
async ensureImageEngine() {
|
|
884
|
+
if (this.imageRawEngine || this.imagePythonEngine) return;
|
|
885
|
+
const meta = getModelMeta(this.config.modelId);
|
|
886
|
+
const imageEntry = CLIP_IMAGE_MODELS.find((m) => m.id === meta.imageModelId);
|
|
887
|
+
if (!imageEntry) throw new Error(`EmbeddingEncoderAddon: unknown image model "${meta.imageModelId}"`);
|
|
888
|
+
await this.resolveForEntry(imageEntry, "image");
|
|
889
|
+
}
|
|
890
|
+
async ensureTextEngine() {
|
|
891
|
+
if (this.textRawEngine || this.textPythonEngine) return;
|
|
892
|
+
const meta = getModelMeta(this.config.modelId);
|
|
893
|
+
const textEntry = CLIP_TEXT_MODELS.find((m) => m.id === meta.textModelId);
|
|
894
|
+
if (!textEntry) throw new Error(`EmbeddingEncoderAddon: unknown text model "${meta.textModelId}"`);
|
|
895
|
+
await this.resolveForEntry(textEntry, "text");
|
|
896
|
+
}
|
|
897
|
+
async resolveForEntry(entry, target) {
|
|
898
|
+
const runtime = this.config.runtime === "auto" ? "auto" : this.config.runtime === "node" ? "onnx" : this.config.runtime;
|
|
899
|
+
const modelsDir = this.models.getModelsDir();
|
|
900
|
+
const engineLogger = this.ctx.logger.withTags({
|
|
901
|
+
modelId: entry.id,
|
|
902
|
+
runtime: this.config.runtime,
|
|
903
|
+
backend: this.config.backend
|
|
904
|
+
});
|
|
905
|
+
await this.models.ensure(entry.id, "onnx");
|
|
906
|
+
const resolved = await resolveEngine({
|
|
907
|
+
runtime,
|
|
908
|
+
backend: this.config.backend,
|
|
909
|
+
modelEntry: entry,
|
|
910
|
+
modelsDir,
|
|
911
|
+
models: this.models ?? void 0,
|
|
912
|
+
logger: engineLogger
|
|
913
|
+
});
|
|
914
|
+
if (resolved.format !== "onnx") {
|
|
915
|
+
this.isPython = true;
|
|
916
|
+
if (target === "image") this.imagePythonEngine = resolved.engine;
|
|
917
|
+
else this.textPythonEngine = resolved.engine;
|
|
918
|
+
} else {
|
|
919
|
+
const rawEngine = new NodeRawTensorEngine(resolved.modelPath, this.config.backend, engineLogger);
|
|
920
|
+
await rawEngine.initialize();
|
|
921
|
+
await resolved.engine.dispose();
|
|
922
|
+
if (target === "image") this.imageRawEngine = rawEngine;
|
|
923
|
+
else this.textRawEngine = rawEngine;
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
async onShutdown() {
|
|
927
|
+
await this.imageRawEngine?.dispose();
|
|
928
|
+
await this.textRawEngine?.dispose();
|
|
929
|
+
await this.imagePythonEngine?.dispose();
|
|
930
|
+
await this.textPythonEngine?.dispose();
|
|
931
|
+
}
|
|
932
|
+
globalSettingsSchema() {
|
|
933
|
+
return this.schema({ sections: [{
|
|
934
|
+
id: "embedding-encoder-settings",
|
|
935
|
+
title: "Embedding Encoder",
|
|
936
|
+
columns: 2,
|
|
937
|
+
fields: [
|
|
938
|
+
{
|
|
939
|
+
type: "text",
|
|
940
|
+
key: "modelId",
|
|
941
|
+
label: "Model ID",
|
|
942
|
+
description: "CLIP model identifier to use for image/text embedding",
|
|
943
|
+
default: DEFAULT_CLIP_MODEL
|
|
944
|
+
},
|
|
945
|
+
{
|
|
946
|
+
type: "select",
|
|
947
|
+
key: "runtime",
|
|
948
|
+
label: "Runtime",
|
|
949
|
+
description: "Inference runtime (auto selects the best available)",
|
|
950
|
+
default: "auto",
|
|
951
|
+
options: [
|
|
952
|
+
{
|
|
953
|
+
label: "Auto",
|
|
954
|
+
value: "auto"
|
|
955
|
+
},
|
|
956
|
+
{
|
|
957
|
+
label: "Node (ONNX)",
|
|
958
|
+
value: "node"
|
|
959
|
+
},
|
|
960
|
+
{
|
|
961
|
+
label: "Python",
|
|
962
|
+
value: "python"
|
|
963
|
+
}
|
|
964
|
+
]
|
|
965
|
+
},
|
|
966
|
+
{
|
|
967
|
+
type: "select",
|
|
968
|
+
key: "backend",
|
|
969
|
+
label: "Backend",
|
|
970
|
+
description: "Hardware backend for inference acceleration",
|
|
971
|
+
default: "cpu",
|
|
972
|
+
options: [
|
|
973
|
+
{
|
|
974
|
+
label: "CPU",
|
|
975
|
+
value: "cpu"
|
|
976
|
+
},
|
|
977
|
+
{
|
|
978
|
+
label: "CUDA",
|
|
979
|
+
value: "cuda"
|
|
980
|
+
},
|
|
981
|
+
{
|
|
982
|
+
label: "CoreML",
|
|
983
|
+
value: "coreml"
|
|
984
|
+
}
|
|
985
|
+
]
|
|
986
|
+
}
|
|
987
|
+
]
|
|
988
|
+
}] });
|
|
989
|
+
}
|
|
990
|
+
async onConfigChanged() {}
|
|
991
|
+
};
|
|
992
|
+
/**
|
|
993
|
+
* Minimal CLIP tokenizer — encodes ASCII text to token IDs.
|
|
994
|
+
* Production implementations should use a proper BPE tokenizer;
|
|
995
|
+
* this is a simplified placeholder that maps characters to IDs
|
|
996
|
+
* with SOT/EOT tokens for basic functionality.
|
|
997
|
+
*/
|
|
919
998
|
function clipTokenize(text, maxLength = 77) {
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
while (tokens.length < maxLength) {
|
|
928
|
-
tokens.push(0);
|
|
929
|
-
}
|
|
930
|
-
return tokens;
|
|
999
|
+
const SOT_TOKEN = 49406;
|
|
1000
|
+
const EOT_TOKEN = 49407;
|
|
1001
|
+
const tokens = [SOT_TOKEN];
|
|
1002
|
+
for (let i = 0; i < text.length && tokens.length < maxLength - 1; i++) tokens.push(text.charCodeAt(i) + 256);
|
|
1003
|
+
tokens.push(EOT_TOKEN);
|
|
1004
|
+
while (tokens.length < maxLength) tokens.push(0);
|
|
1005
|
+
return tokens;
|
|
931
1006
|
}
|
|
1007
|
+
//#endregion
|
|
932
1008
|
exports.EmbeddingEncoderAddon = EmbeddingEncoderAddon;
|
|
933
1009
|
exports.default = EmbeddingEncoderAddon;
|
|
934
|
-
//# sourceMappingURL=index.js.map
|