@camstack/addon-benchmark 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-benchmark.css +1 -0
- package/dist/index.js +6 -743
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +5 -723
- package/dist/index.mjs.map +1 -1
- package/dist/page.mjs +34094 -0
- package/package.json +43 -12
- package/dist/index.d.mts +0 -222
- package/dist/index.d.ts +0 -222
package/dist/index.mjs
CHANGED
|
@@ -1,288 +1,3 @@
|
|
|
1
|
-
// src/runner/stats.ts
|
|
2
|
-
function percentile(sorted, p) {
|
|
3
|
-
if (sorted.length === 0) return 0;
|
|
4
|
-
const idx = p * (sorted.length - 1);
|
|
5
|
-
const lo = Math.floor(idx);
|
|
6
|
-
const hi = Math.ceil(idx);
|
|
7
|
-
const loVal = sorted[lo] ?? 0;
|
|
8
|
-
const hiVal = sorted[hi] ?? 0;
|
|
9
|
-
return loVal + (hiVal - loVal) * (idx - lo);
|
|
10
|
-
}
|
|
11
|
-
function computeLatencyStats(timingsMs) {
|
|
12
|
-
if (timingsMs.length === 0) {
|
|
13
|
-
return { mean: 0, median: 0, p95: 0, p99: 0, min: 0, max: 0 };
|
|
14
|
-
}
|
|
15
|
-
const sorted = [...timingsMs].sort((a, b) => a - b);
|
|
16
|
-
const sum = sorted.reduce((acc, v) => acc + v, 0);
|
|
17
|
-
const mean = sum / sorted.length;
|
|
18
|
-
return {
|
|
19
|
-
mean,
|
|
20
|
-
median: percentile(sorted, 0.5),
|
|
21
|
-
p95: percentile(sorted, 0.95),
|
|
22
|
-
p99: percentile(sorted, 0.99),
|
|
23
|
-
min: sorted[0] ?? 0,
|
|
24
|
-
max: sorted[sorted.length - 1] ?? 0
|
|
25
|
-
};
|
|
26
|
-
}
|
|
27
|
-
function computeFps(timingsMs) {
|
|
28
|
-
if (timingsMs.length === 0) return 0;
|
|
29
|
-
const stats = computeLatencyStats(timingsMs);
|
|
30
|
-
if (stats.mean === 0) return 0;
|
|
31
|
-
return 1e3 / stats.mean;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
// src/runner/accuracy.ts
|
|
35
|
-
function iou(pred, gt) {
|
|
36
|
-
const predX2 = pred.x + pred.w;
|
|
37
|
-
const predY2 = pred.y + pred.h;
|
|
38
|
-
const gtX2 = gt.x + gt.w;
|
|
39
|
-
const gtY2 = gt.y + gt.h;
|
|
40
|
-
const interX1 = Math.max(pred.x, gt.x);
|
|
41
|
-
const interY1 = Math.max(pred.y, gt.y);
|
|
42
|
-
const interX2 = Math.min(predX2, gtX2);
|
|
43
|
-
const interY2 = Math.min(predY2, gtY2);
|
|
44
|
-
const interW = Math.max(0, interX2 - interX1);
|
|
45
|
-
const interH = Math.max(0, interY2 - interY1);
|
|
46
|
-
const interArea = interW * interH;
|
|
47
|
-
if (interArea === 0) return 0;
|
|
48
|
-
const predArea = pred.w * pred.h;
|
|
49
|
-
const gtArea = gt.w * gt.h;
|
|
50
|
-
const unionArea = predArea + gtArea - interArea;
|
|
51
|
-
if (unionArea <= 0) return 0;
|
|
52
|
-
return interArea / unionArea;
|
|
53
|
-
}
|
|
54
|
-
function areaUnderPRCurve(precision, recall) {
|
|
55
|
-
if (precision.length < 2) return precision[0] ?? 0;
|
|
56
|
-
let area = 0;
|
|
57
|
-
for (let i = 1; i < recall.length; i++) {
|
|
58
|
-
const dr = (recall[i] ?? 0) - (recall[i - 1] ?? 0);
|
|
59
|
-
const avgP = ((precision[i] ?? 0) + (precision[i - 1] ?? 0)) / 2;
|
|
60
|
-
area += dr * avgP;
|
|
61
|
-
}
|
|
62
|
-
return Math.abs(area);
|
|
63
|
-
}
|
|
64
|
-
function computeClassAP(predictions, groundTruths, iouThreshold) {
|
|
65
|
-
const nGt = groundTruths.length;
|
|
66
|
-
if (nGt === 0) return 0;
|
|
67
|
-
if (predictions.length === 0) return 0;
|
|
68
|
-
const matched = /* @__PURE__ */ new Set();
|
|
69
|
-
const tp = [];
|
|
70
|
-
const fp = [];
|
|
71
|
-
for (const pred of predictions) {
|
|
72
|
-
let bestIou = -1;
|
|
73
|
-
let bestGtIdx = -1;
|
|
74
|
-
for (let i = 0; i < groundTruths.length; i++) {
|
|
75
|
-
if (matched.has(i)) continue;
|
|
76
|
-
const gt = groundTruths[i];
|
|
77
|
-
if (!gt) continue;
|
|
78
|
-
const score = iou(pred.bbox, gt.bbox);
|
|
79
|
-
if (score > bestIou) {
|
|
80
|
-
bestIou = score;
|
|
81
|
-
bestGtIdx = i;
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
if (bestIou >= iouThreshold && bestGtIdx >= 0) {
|
|
85
|
-
matched.add(bestGtIdx);
|
|
86
|
-
tp.push(1);
|
|
87
|
-
fp.push(0);
|
|
88
|
-
} else {
|
|
89
|
-
tp.push(0);
|
|
90
|
-
fp.push(1);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
const cumTp = [];
|
|
94
|
-
const cumFp = [];
|
|
95
|
-
let sumTp = 0;
|
|
96
|
-
let sumFp = 0;
|
|
97
|
-
for (let i = 0; i < tp.length; i++) {
|
|
98
|
-
sumTp += tp[i] ?? 0;
|
|
99
|
-
sumFp += fp[i] ?? 0;
|
|
100
|
-
cumTp.push(sumTp);
|
|
101
|
-
cumFp.push(sumFp);
|
|
102
|
-
}
|
|
103
|
-
const precisions = [];
|
|
104
|
-
const recalls = [];
|
|
105
|
-
precisions.push(1);
|
|
106
|
-
recalls.push(0);
|
|
107
|
-
for (let i = 0; i < cumTp.length; i++) {
|
|
108
|
-
const cTp = cumTp[i] ?? 0;
|
|
109
|
-
const cFp = cumFp[i] ?? 0;
|
|
110
|
-
const denom = cTp + cFp;
|
|
111
|
-
precisions.push(denom > 0 ? cTp / denom : 0);
|
|
112
|
-
recalls.push(nGt > 0 ? cTp / nGt : 0);
|
|
113
|
-
}
|
|
114
|
-
return areaUnderPRCurve(precisions, recalls);
|
|
115
|
-
}
|
|
116
|
-
function computeAccuracy(predictions, groundTruth, iouThreshold = 0.5) {
|
|
117
|
-
if (groundTruth.length === 0) {
|
|
118
|
-
return { mAP50: void 0, precision: void 0, recall: void 0 };
|
|
119
|
-
}
|
|
120
|
-
const classes = [...new Set(groundTruth.map((gt) => gt.class))];
|
|
121
|
-
const aps = [];
|
|
122
|
-
let totalTp = 0;
|
|
123
|
-
let totalFp = 0;
|
|
124
|
-
let totalGt = 0;
|
|
125
|
-
for (const cls of classes) {
|
|
126
|
-
const classPredictions = predictions.filter((p) => p.class === cls).sort((a, b) => b.score - a.score);
|
|
127
|
-
const classGt = groundTruth.filter((gt) => gt.class === cls);
|
|
128
|
-
totalGt += classGt.length;
|
|
129
|
-
const ap = computeClassAP(classPredictions, classGt, iouThreshold);
|
|
130
|
-
aps.push(ap);
|
|
131
|
-
const matched = /* @__PURE__ */ new Set();
|
|
132
|
-
for (const pred of classPredictions) {
|
|
133
|
-
let bestIou = -1;
|
|
134
|
-
let bestGtIdx = -1;
|
|
135
|
-
for (let i = 0; i < classGt.length; i++) {
|
|
136
|
-
if (matched.has(i)) continue;
|
|
137
|
-
const gt = classGt[i];
|
|
138
|
-
if (!gt) continue;
|
|
139
|
-
const score = iou(pred.bbox, gt.bbox);
|
|
140
|
-
if (score > bestIou) {
|
|
141
|
-
bestIou = score;
|
|
142
|
-
bestGtIdx = i;
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
if (bestIou >= iouThreshold && bestGtIdx >= 0) {
|
|
146
|
-
matched.add(bestGtIdx);
|
|
147
|
-
totalTp++;
|
|
148
|
-
} else {
|
|
149
|
-
totalFp++;
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
const mAP50 = aps.length > 0 ? aps.reduce((sum, ap) => sum + ap, 0) / aps.length : void 0;
|
|
154
|
-
const precision = totalTp + totalFp > 0 ? totalTp / (totalTp + totalFp) : void 0;
|
|
155
|
-
const recall = totalGt > 0 ? totalTp / totalGt : void 0;
|
|
156
|
-
return { mAP50, precision, recall };
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
// src/runner/benchmark-runner.ts
|
|
160
|
-
function measureResources() {
|
|
161
|
-
const memUsage = process.memoryUsage();
|
|
162
|
-
return {
|
|
163
|
-
peakMemoryMB: Math.round(memUsage.heapUsed / (1024 * 1024)),
|
|
164
|
-
avgCpuPercent: 0
|
|
165
|
-
// CPU profiling requires native tooling; best-effort zero
|
|
166
|
-
};
|
|
167
|
-
}
|
|
168
|
-
var BenchmarkRunner = class {
|
|
169
|
-
constructor(options) {
|
|
170
|
-
this.options = options;
|
|
171
|
-
}
|
|
172
|
-
/** Run inference on a single frame, picking a reference image round-robin if needed */
|
|
173
|
-
async runOnce(target, frameIdx = 0) {
|
|
174
|
-
const images = this.options.referenceImages;
|
|
175
|
-
if (images.length === 0) {
|
|
176
|
-
const syntheticFrame = {
|
|
177
|
-
data: Buffer.alloc(320 * 240 * 3),
|
|
178
|
-
format: "rgb",
|
|
179
|
-
width: 320,
|
|
180
|
-
height: 240,
|
|
181
|
-
timestamp: Date.now()
|
|
182
|
-
};
|
|
183
|
-
return this.options.runInference(syntheticFrame, target.config);
|
|
184
|
-
}
|
|
185
|
-
const idx = frameIdx % images.length;
|
|
186
|
-
const entry = images[idx];
|
|
187
|
-
if (!entry) {
|
|
188
|
-
throw new Error(`Reference image at index ${idx} is undefined`);
|
|
189
|
-
}
|
|
190
|
-
return this.options.runInference(entry.frame, target.config);
|
|
191
|
-
}
|
|
192
|
-
/** Measure accuracy across all reference images */
|
|
193
|
-
async measureAccuracy(target) {
|
|
194
|
-
const images = this.options.referenceImages;
|
|
195
|
-
if (images.length === 0) return void 0;
|
|
196
|
-
const allPredictions = [];
|
|
197
|
-
const allGroundTruth = [];
|
|
198
|
-
for (const entry of images) {
|
|
199
|
-
const output = await this.options.runInference(entry.frame, target.config);
|
|
200
|
-
allPredictions.push(...output.detections);
|
|
201
|
-
allGroundTruth.push(...entry.groundTruth.annotations);
|
|
202
|
-
}
|
|
203
|
-
return computeAccuracy(allPredictions, allGroundTruth);
|
|
204
|
-
}
|
|
205
|
-
/** Run benchmark for a single target configuration */
|
|
206
|
-
async runTarget(target, config) {
|
|
207
|
-
const warmup = config.warmup ?? 10;
|
|
208
|
-
const iterations = config.iterations ?? 100;
|
|
209
|
-
const timings = [];
|
|
210
|
-
for (let i = 0; i < warmup; i++) {
|
|
211
|
-
await this.runOnce(target, i);
|
|
212
|
-
}
|
|
213
|
-
for (let i = 0; i < iterations; i++) {
|
|
214
|
-
const start = performance.now();
|
|
215
|
-
await this.runOnce(target, i);
|
|
216
|
-
timings.push(performance.now() - start);
|
|
217
|
-
this.options.onProgress?.(i + 1, iterations, target.label);
|
|
218
|
-
}
|
|
219
|
-
const latency = computeLatencyStats(timings);
|
|
220
|
-
const fps = computeFps(timings);
|
|
221
|
-
const accuracy = await this.measureAccuracy(target);
|
|
222
|
-
const resources = measureResources();
|
|
223
|
-
return {
|
|
224
|
-
label: target.label,
|
|
225
|
-
addonId: target.addonId,
|
|
226
|
-
config: target.config,
|
|
227
|
-
latency,
|
|
228
|
-
fps,
|
|
229
|
-
accuracy,
|
|
230
|
-
resources
|
|
231
|
-
};
|
|
232
|
-
}
|
|
233
|
-
/** Run benchmark for all targets defined in the config */
|
|
234
|
-
async runAll(config) {
|
|
235
|
-
const results = [];
|
|
236
|
-
for (const target of config.targets) {
|
|
237
|
-
results.push(await this.runTarget(target, config));
|
|
238
|
-
}
|
|
239
|
-
return results;
|
|
240
|
-
}
|
|
241
|
-
};
|
|
242
|
-
|
|
243
|
-
// src/runner/system-info.ts
|
|
244
|
-
import os from "os";
|
|
245
|
-
import { execSync } from "child_process";
|
|
246
|
-
function detectGpuModel() {
|
|
247
|
-
try {
|
|
248
|
-
const output = execSync("nvidia-smi --query-gpu=name --format=csv,noheader", {
|
|
249
|
-
timeout: 3e3,
|
|
250
|
-
stdio: ["ignore", "pipe", "ignore"]
|
|
251
|
-
}).toString().trim();
|
|
252
|
-
if (output.length > 0) {
|
|
253
|
-
return output.split("\n")[0]?.trim();
|
|
254
|
-
}
|
|
255
|
-
} catch {
|
|
256
|
-
}
|
|
257
|
-
if (process.platform === "darwin") {
|
|
258
|
-
try {
|
|
259
|
-
const output = execSync(
|
|
260
|
-
"system_profiler SPDisplaysDataType 2>/dev/null | grep 'Chipset Model:' | head -1",
|
|
261
|
-
{ timeout: 5e3, stdio: ["ignore", "pipe", "ignore"] }
|
|
262
|
-
).toString().trim();
|
|
263
|
-
const match = output.match(/Chipset Model:\s*(.+)/);
|
|
264
|
-
if (match?.[1]) {
|
|
265
|
-
return match[1].trim();
|
|
266
|
-
}
|
|
267
|
-
} catch {
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
return void 0;
|
|
271
|
-
}
|
|
272
|
-
function collectSystemInfo() {
|
|
273
|
-
const cpus = os.cpus();
|
|
274
|
-
const firstCpu = cpus[0];
|
|
275
|
-
return {
|
|
276
|
-
os: `${os.type()} ${os.release()}`,
|
|
277
|
-
arch: os.arch(),
|
|
278
|
-
cpuModel: firstCpu?.model ?? "Unknown",
|
|
279
|
-
cpuCores: cpus.length,
|
|
280
|
-
totalMemoryMB: Math.round(os.totalmem() / (1024 * 1024)),
|
|
281
|
-
gpuModel: detectGpuModel(),
|
|
282
|
-
nodeVersion: process.version
|
|
283
|
-
};
|
|
284
|
-
}
|
|
285
|
-
|
|
286
1
|
// src/benchmark-addon.ts
|
|
287
2
|
var BenchmarkAddon = class {
|
|
288
3
|
id = "benchmark";
|
|
@@ -290,8 +5,8 @@ var BenchmarkAddon = class {
|
|
|
290
5
|
id: "benchmark",
|
|
291
6
|
name: "Benchmark",
|
|
292
7
|
version: "0.1.0",
|
|
293
|
-
|
|
294
|
-
|
|
8
|
+
packageName: "@camstack/addon-benchmark",
|
|
9
|
+
description: "Detection benchmark, image tester, and pipeline runner"
|
|
295
10
|
};
|
|
296
11
|
pages = [
|
|
297
12
|
{
|
|
@@ -299,18 +14,12 @@ var BenchmarkAddon = class {
|
|
|
299
14
|
label: "Benchmark",
|
|
300
15
|
icon: "gauge",
|
|
301
16
|
path: "/addon/benchmark",
|
|
302
|
-
bundle: "
|
|
303
|
-
element: "camstack-benchmark"
|
|
17
|
+
bundle: "page.mjs"
|
|
304
18
|
}
|
|
305
19
|
];
|
|
306
|
-
|
|
307
|
-
options = {};
|
|
308
|
-
async initialize(ctx, opts) {
|
|
309
|
-
this.context = ctx;
|
|
310
|
-
this.options = opts ?? {};
|
|
20
|
+
async initialize(_ctx) {
|
|
311
21
|
}
|
|
312
22
|
async shutdown() {
|
|
313
|
-
this.context = null;
|
|
314
23
|
}
|
|
315
24
|
getCapabilityProvider(name) {
|
|
316
25
|
if (name === "addon-pages") {
|
|
@@ -322,435 +31,8 @@ var BenchmarkAddon = class {
|
|
|
322
31
|
}
|
|
323
32
|
return null;
|
|
324
33
|
}
|
|
325
|
-
/**
|
|
326
|
-
* Run a benchmark locally and return a MultiBenchmarkReport.
|
|
327
|
-
*/
|
|
328
|
-
async run(config, onProgress) {
|
|
329
|
-
const systemInfo = collectSystemInfo();
|
|
330
|
-
const results = await this.runLocally(config, onProgress);
|
|
331
|
-
const leaderboard = results.sort((a, b) => a.latency.mean - b.latency.mean).map((r, idx) => ({
|
|
332
|
-
rank: idx + 1,
|
|
333
|
-
agentId: "local",
|
|
334
|
-
label: r.label,
|
|
335
|
-
meanMs: r.latency.mean,
|
|
336
|
-
fps: r.fps,
|
|
337
|
-
peakMemoryMB: r.resources.peakMemoryMB,
|
|
338
|
-
accuracy: r.accuracy
|
|
339
|
-
}));
|
|
340
|
-
return {
|
|
341
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
342
|
-
source: config.source,
|
|
343
|
-
agents: [
|
|
344
|
-
{
|
|
345
|
-
agentId: "local",
|
|
346
|
-
status: "completed",
|
|
347
|
-
systemInfo,
|
|
348
|
-
results
|
|
349
|
-
}
|
|
350
|
-
],
|
|
351
|
-
leaderboard
|
|
352
|
-
};
|
|
353
|
-
}
|
|
354
|
-
/**
|
|
355
|
-
* Run a distributed benchmark across agents.
|
|
356
|
-
*
|
|
357
|
-
* Stub: distributed execution requires a BenchmarkHub instance bound to
|
|
358
|
-
* live WebSocket connections. Call BenchmarkHub.dispatch() directly from
|
|
359
|
-
* the server transport layer. This method runs locally as a fallback.
|
|
360
|
-
*/
|
|
361
|
-
async runDistributed(config, onProgress) {
|
|
362
|
-
return this.run(config, onProgress);
|
|
363
|
-
}
|
|
364
|
-
async runLocally(config, onProgress) {
|
|
365
|
-
const { inferenceFactory, referenceImages = [] } = this.options;
|
|
366
|
-
const firstTarget = config.targets[0];
|
|
367
|
-
if (!firstTarget) {
|
|
368
|
-
throw new Error("BenchmarkConfig must have at least one target");
|
|
369
|
-
}
|
|
370
|
-
if (!inferenceFactory) {
|
|
371
|
-
return config.targets.map((target) => ({
|
|
372
|
-
label: target.label,
|
|
373
|
-
addonId: target.addonId,
|
|
374
|
-
config: target.config,
|
|
375
|
-
latency: { mean: 0, median: 0, p95: 0, p99: 0, min: 0, max: 0 },
|
|
376
|
-
fps: 0,
|
|
377
|
-
resources: { peakMemoryMB: 0, avgCpuPercent: 0 }
|
|
378
|
-
}));
|
|
379
|
-
}
|
|
380
|
-
const runInference = inferenceFactory(firstTarget.addonId, firstTarget.config);
|
|
381
|
-
const runner = new BenchmarkRunner({
|
|
382
|
-
runInference,
|
|
383
|
-
referenceImages,
|
|
384
|
-
onProgress
|
|
385
|
-
});
|
|
386
|
-
return runner.runAll(config);
|
|
387
|
-
}
|
|
388
|
-
};
|
|
389
|
-
|
|
390
|
-
// src/annotator/result-annotator.ts
|
|
391
|
-
import sharp from "sharp";
|
|
392
|
-
var COLOR_PALETTE = [
|
|
393
|
-
"#FF4136",
|
|
394
|
-
"#2ECC40",
|
|
395
|
-
"#0074D9",
|
|
396
|
-
"#FF851B",
|
|
397
|
-
"#B10DC9",
|
|
398
|
-
"#FFDC00",
|
|
399
|
-
"#7FDBFF",
|
|
400
|
-
"#01FF70",
|
|
401
|
-
"#F012BE",
|
|
402
|
-
"#3D9970"
|
|
403
|
-
];
|
|
404
|
-
function classColor(cls) {
|
|
405
|
-
let hash = 0;
|
|
406
|
-
for (let i = 0; i < cls.length; i++) {
|
|
407
|
-
hash = hash * 31 + (cls.codePointAt(i) ?? 0) >>> 0;
|
|
408
|
-
}
|
|
409
|
-
return COLOR_PALETTE[hash % COLOR_PALETTE.length] ?? "#FF4136";
|
|
410
|
-
}
|
|
411
|
-
function escSvg(str) {
|
|
412
|
-
return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
413
|
-
}
|
|
414
|
-
function buildSvgOverlay(detections, width, height) {
|
|
415
|
-
const rects = detections.map((det) => {
|
|
416
|
-
const color = classColor(det.class);
|
|
417
|
-
const label = escSvg(`${det.class} ${(det.score * 100).toFixed(0)}%`);
|
|
418
|
-
const { x, y, w, h } = det.bbox;
|
|
419
|
-
const clampX = Math.max(0, Math.min(x, width - 1));
|
|
420
|
-
const clampY = Math.max(0, Math.min(y, height - 1));
|
|
421
|
-
const clampW = Math.max(1, Math.min(w, width - clampX));
|
|
422
|
-
const clampH = Math.max(1, Math.min(h, height - clampY));
|
|
423
|
-
const labelY = clampY > 14 ? clampY - 2 : clampY + clampH + 12;
|
|
424
|
-
return `
|
|
425
|
-
<rect x="${clampX}" y="${clampY}" width="${clampW}" height="${clampH}"
|
|
426
|
-
stroke="${color}" stroke-width="2" fill="none"/>
|
|
427
|
-
<rect x="${clampX}" y="${labelY - 12}" width="${label.length * 7}" height="14"
|
|
428
|
-
fill="${color}" fill-opacity="0.75"/>
|
|
429
|
-
<text x="${clampX + 2}" y="${labelY}" font-family="monospace" font-size="11"
|
|
430
|
-
fill="white">${label}</text>`;
|
|
431
|
-
});
|
|
432
|
-
return `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}">${rects.join("")}</svg>`;
|
|
433
|
-
}
|
|
434
|
-
async function buildMaskOverlays(detections, width, height) {
|
|
435
|
-
const overlays = [];
|
|
436
|
-
for (const det of detections) {
|
|
437
|
-
if (!det.mask || !det.maskWidth || !det.maskHeight) continue;
|
|
438
|
-
const color = classColor(det.class);
|
|
439
|
-
const r = parseInt(color.slice(1, 3), 16);
|
|
440
|
-
const g = parseInt(color.slice(3, 5), 16);
|
|
441
|
-
const b = parseInt(color.slice(5, 7), 16);
|
|
442
|
-
const rgba = Buffer.alloc(det.maskWidth * det.maskHeight * 4);
|
|
443
|
-
for (let i = 0; i < det.mask.length; i++) {
|
|
444
|
-
const alpha = det.mask[i] ?? 0;
|
|
445
|
-
if (alpha > 0) {
|
|
446
|
-
const base = i * 4;
|
|
447
|
-
rgba[base] = r;
|
|
448
|
-
rgba[base + 1] = g;
|
|
449
|
-
rgba[base + 2] = b;
|
|
450
|
-
rgba[base + 3] = Math.round(alpha * 0.5);
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
const maskBuffer = await sharp(rgba, {
|
|
454
|
-
raw: { width: det.maskWidth, height: det.maskHeight, channels: 4 }
|
|
455
|
-
}).resize(width, height).png().toBuffer();
|
|
456
|
-
overlays.push(maskBuffer);
|
|
457
|
-
}
|
|
458
|
-
return overlays;
|
|
459
|
-
}
|
|
460
|
-
async function annotateImage(frame, detections, outputPath) {
|
|
461
|
-
let image;
|
|
462
|
-
if (frame.format === "jpeg") {
|
|
463
|
-
image = sharp(frame.data);
|
|
464
|
-
} else {
|
|
465
|
-
image = sharp(frame.data, {
|
|
466
|
-
raw: { width: frame.width, height: frame.height, channels: 3 }
|
|
467
|
-
});
|
|
468
|
-
}
|
|
469
|
-
const composites = [];
|
|
470
|
-
const maskBuffers = await buildMaskOverlays(detections, frame.width, frame.height);
|
|
471
|
-
for (const maskBuf of maskBuffers) {
|
|
472
|
-
composites.push({ input: maskBuf, blend: "over" });
|
|
473
|
-
}
|
|
474
|
-
const svgBuf = Buffer.from(buildSvgOverlay(detections, frame.width, frame.height));
|
|
475
|
-
composites.push({ input: svgBuf, blend: "over" });
|
|
476
|
-
await image.composite(composites).jpeg({ quality: 90 }).toFile(outputPath);
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
// src/distributed/benchmark-hub.ts
|
|
480
|
-
var BenchmarkHub = class {
|
|
481
|
-
constructor(send) {
|
|
482
|
-
this.send = send;
|
|
483
|
-
}
|
|
484
|
-
agents = /* @__PURE__ */ new Map();
|
|
485
|
-
activeTaskId = null;
|
|
486
|
-
resolveTask = null;
|
|
487
|
-
rejectTask = null;
|
|
488
|
-
pendingConfig = null;
|
|
489
|
-
/** Register a connected agent (typically on WS connect + hello message) */
|
|
490
|
-
registerAgent(agentId, systemInfo) {
|
|
491
|
-
this.agents.set(agentId, {
|
|
492
|
-
agentId,
|
|
493
|
-
systemInfo,
|
|
494
|
-
status: "idle"
|
|
495
|
-
});
|
|
496
|
-
}
|
|
497
|
-
/** Mark an agent as disconnected */
|
|
498
|
-
disconnectAgent(agentId) {
|
|
499
|
-
const entry = this.agents.get(agentId);
|
|
500
|
-
if (entry) {
|
|
501
|
-
this.agents.set(agentId, { ...entry, status: "disconnected" });
|
|
502
|
-
}
|
|
503
|
-
this.checkCompletion();
|
|
504
|
-
}
|
|
505
|
-
/** Handle incoming message from any agent */
|
|
506
|
-
handleMessage(message) {
|
|
507
|
-
switch (message.type) {
|
|
508
|
-
case "agent.hello":
|
|
509
|
-
this.onAgentHello(message);
|
|
510
|
-
break;
|
|
511
|
-
case "benchmark.progress":
|
|
512
|
-
this.onProgress(message);
|
|
513
|
-
break;
|
|
514
|
-
case "benchmark.result":
|
|
515
|
-
this.onResult(message);
|
|
516
|
-
break;
|
|
517
|
-
case "benchmark.error":
|
|
518
|
-
this.onError(message);
|
|
519
|
-
break;
|
|
520
|
-
default:
|
|
521
|
-
break;
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
|
-
/**
|
|
525
|
-
* Dispatch a benchmark to a subset of agents.
|
|
526
|
-
* Returns a promise that resolves when all targeted agents have finished.
|
|
527
|
-
*
|
|
528
|
-
* @param config - Benchmark configuration
|
|
529
|
-
* @param agentIds - If empty, dispatches to all connected/idle agents
|
|
530
|
-
* @param referenceImages - Reference image buffers to send to agents
|
|
531
|
-
*/
|
|
532
|
-
async dispatch(config, agentIds, referenceImages = []) {
|
|
533
|
-
const taskId = `task-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
534
|
-
this.activeTaskId = taskId;
|
|
535
|
-
this.pendingConfig = config;
|
|
536
|
-
const targets = agentIds.length > 0 ? agentIds : [...this.agents.values()].filter((a) => a.status === "idle").map((a) => a.agentId);
|
|
537
|
-
if (targets.length === 0) {
|
|
538
|
-
throw new Error("No idle agents available for benchmark dispatch");
|
|
539
|
-
}
|
|
540
|
-
for (const agentId of targets) {
|
|
541
|
-
const entry = this.agents.get(agentId);
|
|
542
|
-
if (entry) {
|
|
543
|
-
this.agents.set(agentId, { ...entry, status: "running", results: void 0, error: void 0 });
|
|
544
|
-
}
|
|
545
|
-
}
|
|
546
|
-
for (const agentId of targets) {
|
|
547
|
-
this.send(agentId, {
|
|
548
|
-
type: "benchmark.start",
|
|
549
|
-
taskId,
|
|
550
|
-
config,
|
|
551
|
-
referenceImages
|
|
552
|
-
});
|
|
553
|
-
}
|
|
554
|
-
return new Promise((resolve, reject) => {
|
|
555
|
-
this.resolveTask = resolve;
|
|
556
|
-
this.rejectTask = reject;
|
|
557
|
-
});
|
|
558
|
-
}
|
|
559
|
-
onAgentHello(message) {
|
|
560
|
-
this.registerAgent(message.agentId, message.systemInfo);
|
|
561
|
-
}
|
|
562
|
-
onProgress(message) {
|
|
563
|
-
const entry = this.agents.get(message.agentId);
|
|
564
|
-
if (entry) {
|
|
565
|
-
this.agents.set(message.agentId, {
|
|
566
|
-
...entry,
|
|
567
|
-
completedIterations: message.completed
|
|
568
|
-
});
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
onResult(message) {
|
|
572
|
-
const entry = this.agents.get(message.agentId);
|
|
573
|
-
if (entry) {
|
|
574
|
-
this.agents.set(message.agentId, {
|
|
575
|
-
...entry,
|
|
576
|
-
status: "done",
|
|
577
|
-
systemInfo: message.systemInfo,
|
|
578
|
-
results: message.results
|
|
579
|
-
});
|
|
580
|
-
}
|
|
581
|
-
this.checkCompletion();
|
|
582
|
-
}
|
|
583
|
-
onError(message) {
|
|
584
|
-
const entry = this.agents.get(message.agentId);
|
|
585
|
-
if (entry) {
|
|
586
|
-
this.agents.set(message.agentId, {
|
|
587
|
-
...entry,
|
|
588
|
-
status: "error",
|
|
589
|
-
error: message.error
|
|
590
|
-
});
|
|
591
|
-
}
|
|
592
|
-
this.checkCompletion();
|
|
593
|
-
}
|
|
594
|
-
checkCompletion() {
|
|
595
|
-
if (!this.resolveTask || !this.activeTaskId) return;
|
|
596
|
-
const runningAgents = [...this.agents.values()].filter(
|
|
597
|
-
(a) => a.status === "running"
|
|
598
|
-
);
|
|
599
|
-
if (runningAgents.length > 0) return;
|
|
600
|
-
const report = this.buildReport();
|
|
601
|
-
this.resolveTask(report);
|
|
602
|
-
this.resolveTask = null;
|
|
603
|
-
this.rejectTask = null;
|
|
604
|
-
this.activeTaskId = null;
|
|
605
|
-
}
|
|
606
|
-
buildReport() {
|
|
607
|
-
const source = this.pendingConfig?.source ?? { type: "reference", images: "all" };
|
|
608
|
-
const agentEntries = [...this.agents.values()];
|
|
609
|
-
const allResults = [];
|
|
610
|
-
for (const agent of agentEntries) {
|
|
611
|
-
if (agent.results) {
|
|
612
|
-
for (const result of agent.results) {
|
|
613
|
-
allResults.push({ agentId: agent.agentId, result });
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
const leaderboard = allResults.sort((a, b) => a.result.latency.mean - b.result.latency.mean).map((entry, idx) => ({
|
|
618
|
-
rank: idx + 1,
|
|
619
|
-
agentId: entry.agentId,
|
|
620
|
-
label: entry.result.label,
|
|
621
|
-
meanMs: entry.result.latency.mean,
|
|
622
|
-
fps: entry.result.fps,
|
|
623
|
-
peakMemoryMB: entry.result.resources.peakMemoryMB,
|
|
624
|
-
accuracy: entry.result.accuracy
|
|
625
|
-
}));
|
|
626
|
-
return {
|
|
627
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
628
|
-
source,
|
|
629
|
-
agents: agentEntries.map((a) => ({
|
|
630
|
-
agentId: a.agentId,
|
|
631
|
-
status: a.status === "done" ? "completed" : a.status === "error" ? "failed" : a.status === "disconnected" ? "disconnected" : "partial",
|
|
632
|
-
systemInfo: a.systemInfo ?? {
|
|
633
|
-
os: "unknown",
|
|
634
|
-
arch: "unknown",
|
|
635
|
-
cpuModel: "unknown",
|
|
636
|
-
cpuCores: 0,
|
|
637
|
-
totalMemoryMB: 0,
|
|
638
|
-
nodeVersion: "unknown"
|
|
639
|
-
},
|
|
640
|
-
results: a.results ?? [],
|
|
641
|
-
error: a.error,
|
|
642
|
-
completedIterations: a.completedIterations
|
|
643
|
-
})),
|
|
644
|
-
leaderboard
|
|
645
|
-
};
|
|
646
|
-
}
|
|
647
|
-
};
|
|
648
|
-
|
|
649
|
-
// src/distributed/benchmark-agent.ts
|
|
650
|
-
var BenchmarkAgent = class {
|
|
651
|
-
constructor(agentId, send, inferenceFactory) {
|
|
652
|
-
this.send = send;
|
|
653
|
-
this.inferenceFactory = inferenceFactory;
|
|
654
|
-
this.agentId = agentId;
|
|
655
|
-
this.systemInfo = collectSystemInfo();
|
|
656
|
-
}
|
|
657
|
-
agentId;
|
|
658
|
-
systemInfo;
|
|
659
|
-
cancelRequested = false;
|
|
660
|
-
/** Send hello to the hub — call this on WS connect */
|
|
661
|
-
hello() {
|
|
662
|
-
this.send({
|
|
663
|
-
type: "agent.hello",
|
|
664
|
-
agentId: this.agentId,
|
|
665
|
-
systemInfo: this.systemInfo
|
|
666
|
-
});
|
|
667
|
-
}
|
|
668
|
-
/** Handle incoming message from the hub */
|
|
669
|
-
handleMessage(message) {
|
|
670
|
-
switch (message.type) {
|
|
671
|
-
case "benchmark.start":
|
|
672
|
-
void this.onStart(message);
|
|
673
|
-
break;
|
|
674
|
-
case "benchmark.cancel":
|
|
675
|
-
this.onCancel(message);
|
|
676
|
-
break;
|
|
677
|
-
default:
|
|
678
|
-
break;
|
|
679
|
-
}
|
|
680
|
-
}
|
|
681
|
-
async onStart(message) {
|
|
682
|
-
this.cancelRequested = false;
|
|
683
|
-
const { taskId, config } = message;
|
|
684
|
-
try {
|
|
685
|
-
const referenceImages = message.referenceImages.map((img) => ({
|
|
686
|
-
frame: {
|
|
687
|
-
data: img.data,
|
|
688
|
-
format: "jpeg",
|
|
689
|
-
width: img.width,
|
|
690
|
-
height: img.height,
|
|
691
|
-
timestamp: Date.now()
|
|
692
|
-
},
|
|
693
|
-
groundTruth: { image: img.name, annotations: [] }
|
|
694
|
-
}));
|
|
695
|
-
const firstTarget = config.targets[0];
|
|
696
|
-
if (!firstTarget) {
|
|
697
|
-
throw new Error("No benchmark targets specified");
|
|
698
|
-
}
|
|
699
|
-
const runInference = this.inferenceFactory(firstTarget.addonId, firstTarget.config);
|
|
700
|
-
const runner = new BenchmarkRunner({
|
|
701
|
-
runInference,
|
|
702
|
-
referenceImages,
|
|
703
|
-
onProgress: (completed, total, currentTarget) => {
|
|
704
|
-
if (this.cancelRequested) return;
|
|
705
|
-
this.send({
|
|
706
|
-
type: "benchmark.progress",
|
|
707
|
-
taskId,
|
|
708
|
-
agentId: this.agentId,
|
|
709
|
-
completed,
|
|
710
|
-
total,
|
|
711
|
-
currentTarget
|
|
712
|
-
});
|
|
713
|
-
}
|
|
714
|
-
});
|
|
715
|
-
const results = await runner.runAll(config);
|
|
716
|
-
if (!this.cancelRequested) {
|
|
717
|
-
this.send({
|
|
718
|
-
type: "benchmark.result",
|
|
719
|
-
taskId,
|
|
720
|
-
agentId: this.agentId,
|
|
721
|
-
systemInfo: this.systemInfo,
|
|
722
|
-
results
|
|
723
|
-
});
|
|
724
|
-
}
|
|
725
|
-
} catch (err) {
|
|
726
|
-
this.send({
|
|
727
|
-
type: "benchmark.error",
|
|
728
|
-
taskId,
|
|
729
|
-
agentId: this.agentId,
|
|
730
|
-
error: err instanceof Error ? err.message : String(err)
|
|
731
|
-
});
|
|
732
|
-
}
|
|
733
|
-
}
|
|
734
|
-
onCancel(message) {
|
|
735
|
-
if (message.taskId === this.activeTaskId) {
|
|
736
|
-
this.cancelRequested = true;
|
|
737
|
-
}
|
|
738
|
-
}
|
|
739
|
-
/** Returns the current task ID (last started), or null if idle */
|
|
740
|
-
get activeTaskId() {
|
|
741
|
-
return null;
|
|
742
|
-
}
|
|
743
34
|
};
|
|
744
35
|
export {
|
|
745
|
-
BenchmarkAddon
|
|
746
|
-
BenchmarkAgent,
|
|
747
|
-
BenchmarkHub,
|
|
748
|
-
BenchmarkRunner,
|
|
749
|
-
annotateImage,
|
|
750
|
-
collectSystemInfo,
|
|
751
|
-
computeAccuracy,
|
|
752
|
-
computeFps,
|
|
753
|
-
computeLatencyStats,
|
|
754
|
-
iou
|
|
36
|
+
BenchmarkAddon
|
|
755
37
|
};
|
|
756
38
|
//# sourceMappingURL=index.mjs.map
|