@camstack/addon-benchmark 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,288 +1,3 @@
1
- // src/runner/stats.ts
2
- function percentile(sorted, p) {
3
- if (sorted.length === 0) return 0;
4
- const idx = p * (sorted.length - 1);
5
- const lo = Math.floor(idx);
6
- const hi = Math.ceil(idx);
7
- const loVal = sorted[lo] ?? 0;
8
- const hiVal = sorted[hi] ?? 0;
9
- return loVal + (hiVal - loVal) * (idx - lo);
10
- }
11
- function computeLatencyStats(timingsMs) {
12
- if (timingsMs.length === 0) {
13
- return { mean: 0, median: 0, p95: 0, p99: 0, min: 0, max: 0 };
14
- }
15
- const sorted = [...timingsMs].sort((a, b) => a - b);
16
- const sum = sorted.reduce((acc, v) => acc + v, 0);
17
- const mean = sum / sorted.length;
18
- return {
19
- mean,
20
- median: percentile(sorted, 0.5),
21
- p95: percentile(sorted, 0.95),
22
- p99: percentile(sorted, 0.99),
23
- min: sorted[0] ?? 0,
24
- max: sorted[sorted.length - 1] ?? 0
25
- };
26
- }
27
- function computeFps(timingsMs) {
28
- if (timingsMs.length === 0) return 0;
29
- const stats = computeLatencyStats(timingsMs);
30
- if (stats.mean === 0) return 0;
31
- return 1e3 / stats.mean;
32
- }
33
-
34
- // src/runner/accuracy.ts
35
- function iou(pred, gt) {
36
- const predX2 = pred.x + pred.w;
37
- const predY2 = pred.y + pred.h;
38
- const gtX2 = gt.x + gt.w;
39
- const gtY2 = gt.y + gt.h;
40
- const interX1 = Math.max(pred.x, gt.x);
41
- const interY1 = Math.max(pred.y, gt.y);
42
- const interX2 = Math.min(predX2, gtX2);
43
- const interY2 = Math.min(predY2, gtY2);
44
- const interW = Math.max(0, interX2 - interX1);
45
- const interH = Math.max(0, interY2 - interY1);
46
- const interArea = interW * interH;
47
- if (interArea === 0) return 0;
48
- const predArea = pred.w * pred.h;
49
- const gtArea = gt.w * gt.h;
50
- const unionArea = predArea + gtArea - interArea;
51
- if (unionArea <= 0) return 0;
52
- return interArea / unionArea;
53
- }
54
- function areaUnderPRCurve(precision, recall) {
55
- if (precision.length < 2) return precision[0] ?? 0;
56
- let area = 0;
57
- for (let i = 1; i < recall.length; i++) {
58
- const dr = (recall[i] ?? 0) - (recall[i - 1] ?? 0);
59
- const avgP = ((precision[i] ?? 0) + (precision[i - 1] ?? 0)) / 2;
60
- area += dr * avgP;
61
- }
62
- return Math.abs(area);
63
- }
64
- function computeClassAP(predictions, groundTruths, iouThreshold) {
65
- const nGt = groundTruths.length;
66
- if (nGt === 0) return 0;
67
- if (predictions.length === 0) return 0;
68
- const matched = /* @__PURE__ */ new Set();
69
- const tp = [];
70
- const fp = [];
71
- for (const pred of predictions) {
72
- let bestIou = -1;
73
- let bestGtIdx = -1;
74
- for (let i = 0; i < groundTruths.length; i++) {
75
- if (matched.has(i)) continue;
76
- const gt = groundTruths[i];
77
- if (!gt) continue;
78
- const score = iou(pred.bbox, gt.bbox);
79
- if (score > bestIou) {
80
- bestIou = score;
81
- bestGtIdx = i;
82
- }
83
- }
84
- if (bestIou >= iouThreshold && bestGtIdx >= 0) {
85
- matched.add(bestGtIdx);
86
- tp.push(1);
87
- fp.push(0);
88
- } else {
89
- tp.push(0);
90
- fp.push(1);
91
- }
92
- }
93
- const cumTp = [];
94
- const cumFp = [];
95
- let sumTp = 0;
96
- let sumFp = 0;
97
- for (let i = 0; i < tp.length; i++) {
98
- sumTp += tp[i] ?? 0;
99
- sumFp += fp[i] ?? 0;
100
- cumTp.push(sumTp);
101
- cumFp.push(sumFp);
102
- }
103
- const precisions = [];
104
- const recalls = [];
105
- precisions.push(1);
106
- recalls.push(0);
107
- for (let i = 0; i < cumTp.length; i++) {
108
- const cTp = cumTp[i] ?? 0;
109
- const cFp = cumFp[i] ?? 0;
110
- const denom = cTp + cFp;
111
- precisions.push(denom > 0 ? cTp / denom : 0);
112
- recalls.push(nGt > 0 ? cTp / nGt : 0);
113
- }
114
- return areaUnderPRCurve(precisions, recalls);
115
- }
116
- function computeAccuracy(predictions, groundTruth, iouThreshold = 0.5) {
117
- if (groundTruth.length === 0) {
118
- return { mAP50: void 0, precision: void 0, recall: void 0 };
119
- }
120
- const classes = [...new Set(groundTruth.map((gt) => gt.class))];
121
- const aps = [];
122
- let totalTp = 0;
123
- let totalFp = 0;
124
- let totalGt = 0;
125
- for (const cls of classes) {
126
- const classPredictions = predictions.filter((p) => p.class === cls).sort((a, b) => b.score - a.score);
127
- const classGt = groundTruth.filter((gt) => gt.class === cls);
128
- totalGt += classGt.length;
129
- const ap = computeClassAP(classPredictions, classGt, iouThreshold);
130
- aps.push(ap);
131
- const matched = /* @__PURE__ */ new Set();
132
- for (const pred of classPredictions) {
133
- let bestIou = -1;
134
- let bestGtIdx = -1;
135
- for (let i = 0; i < classGt.length; i++) {
136
- if (matched.has(i)) continue;
137
- const gt = classGt[i];
138
- if (!gt) continue;
139
- const score = iou(pred.bbox, gt.bbox);
140
- if (score > bestIou) {
141
- bestIou = score;
142
- bestGtIdx = i;
143
- }
144
- }
145
- if (bestIou >= iouThreshold && bestGtIdx >= 0) {
146
- matched.add(bestGtIdx);
147
- totalTp++;
148
- } else {
149
- totalFp++;
150
- }
151
- }
152
- }
153
- const mAP50 = aps.length > 0 ? aps.reduce((sum, ap) => sum + ap, 0) / aps.length : void 0;
154
- const precision = totalTp + totalFp > 0 ? totalTp / (totalTp + totalFp) : void 0;
155
- const recall = totalGt > 0 ? totalTp / totalGt : void 0;
156
- return { mAP50, precision, recall };
157
- }
158
-
159
- // src/runner/benchmark-runner.ts
160
- function measureResources() {
161
- const memUsage = process.memoryUsage();
162
- return {
163
- peakMemoryMB: Math.round(memUsage.heapUsed / (1024 * 1024)),
164
- avgCpuPercent: 0
165
- // CPU profiling requires native tooling; best-effort zero
166
- };
167
- }
168
- var BenchmarkRunner = class {
169
- constructor(options) {
170
- this.options = options;
171
- }
172
- /** Run inference on a single frame, picking a reference image round-robin if needed */
173
- async runOnce(target, frameIdx = 0) {
174
- const images = this.options.referenceImages;
175
- if (images.length === 0) {
176
- const syntheticFrame = {
177
- data: Buffer.alloc(320 * 240 * 3),
178
- format: "rgb",
179
- width: 320,
180
- height: 240,
181
- timestamp: Date.now()
182
- };
183
- return this.options.runInference(syntheticFrame, target.config);
184
- }
185
- const idx = frameIdx % images.length;
186
- const entry = images[idx];
187
- if (!entry) {
188
- throw new Error(`Reference image at index ${idx} is undefined`);
189
- }
190
- return this.options.runInference(entry.frame, target.config);
191
- }
192
- /** Measure accuracy across all reference images */
193
- async measureAccuracy(target) {
194
- const images = this.options.referenceImages;
195
- if (images.length === 0) return void 0;
196
- const allPredictions = [];
197
- const allGroundTruth = [];
198
- for (const entry of images) {
199
- const output = await this.options.runInference(entry.frame, target.config);
200
- allPredictions.push(...output.detections);
201
- allGroundTruth.push(...entry.groundTruth.annotations);
202
- }
203
- return computeAccuracy(allPredictions, allGroundTruth);
204
- }
205
- /** Run benchmark for a single target configuration */
206
- async runTarget(target, config) {
207
- const warmup = config.warmup ?? 10;
208
- const iterations = config.iterations ?? 100;
209
- const timings = [];
210
- for (let i = 0; i < warmup; i++) {
211
- await this.runOnce(target, i);
212
- }
213
- for (let i = 0; i < iterations; i++) {
214
- const start = performance.now();
215
- await this.runOnce(target, i);
216
- timings.push(performance.now() - start);
217
- this.options.onProgress?.(i + 1, iterations, target.label);
218
- }
219
- const latency = computeLatencyStats(timings);
220
- const fps = computeFps(timings);
221
- const accuracy = await this.measureAccuracy(target);
222
- const resources = measureResources();
223
- return {
224
- label: target.label,
225
- addonId: target.addonId,
226
- config: target.config,
227
- latency,
228
- fps,
229
- accuracy,
230
- resources
231
- };
232
- }
233
- /** Run benchmark for all targets defined in the config */
234
- async runAll(config) {
235
- const results = [];
236
- for (const target of config.targets) {
237
- results.push(await this.runTarget(target, config));
238
- }
239
- return results;
240
- }
241
- };
242
-
243
- // src/runner/system-info.ts
244
- import os from "os";
245
- import { execSync } from "child_process";
246
- function detectGpuModel() {
247
- try {
248
- const output = execSync("nvidia-smi --query-gpu=name --format=csv,noheader", {
249
- timeout: 3e3,
250
- stdio: ["ignore", "pipe", "ignore"]
251
- }).toString().trim();
252
- if (output.length > 0) {
253
- return output.split("\n")[0]?.trim();
254
- }
255
- } catch {
256
- }
257
- if (process.platform === "darwin") {
258
- try {
259
- const output = execSync(
260
- "system_profiler SPDisplaysDataType 2>/dev/null | grep 'Chipset Model:' | head -1",
261
- { timeout: 5e3, stdio: ["ignore", "pipe", "ignore"] }
262
- ).toString().trim();
263
- const match = output.match(/Chipset Model:\s*(.+)/);
264
- if (match?.[1]) {
265
- return match[1].trim();
266
- }
267
- } catch {
268
- }
269
- }
270
- return void 0;
271
- }
272
- function collectSystemInfo() {
273
- const cpus = os.cpus();
274
- const firstCpu = cpus[0];
275
- return {
276
- os: `${os.type()} ${os.release()}`,
277
- arch: os.arch(),
278
- cpuModel: firstCpu?.model ?? "Unknown",
279
- cpuCores: cpus.length,
280
- totalMemoryMB: Math.round(os.totalmem() / (1024 * 1024)),
281
- gpuModel: detectGpuModel(),
282
- nodeVersion: process.version
283
- };
284
- }
285
-
286
1
  // src/benchmark-addon.ts
287
2
  var BenchmarkAddon = class {
288
3
  id = "benchmark";
@@ -290,8 +5,8 @@ var BenchmarkAddon = class {
290
5
  id: "benchmark",
291
6
  name: "Benchmark",
292
7
  version: "0.1.0",
293
- description: "Detection addon benchmarking with reference images and distributed execution",
294
- packageName: "@camstack/addon-benchmark"
8
+ packageName: "@camstack/addon-benchmark",
9
+ description: "Detection benchmark, image tester, and pipeline runner"
295
10
  };
296
11
  pages = [
297
12
  {
@@ -299,18 +14,12 @@ var BenchmarkAddon = class {
299
14
  label: "Benchmark",
300
15
  icon: "gauge",
301
16
  path: "/addon/benchmark",
302
- bundle: "benchmark.js",
303
- element: "camstack-benchmark"
17
+ bundle: "page.mjs"
304
18
  }
305
19
  ];
306
- context = null;
307
- options = {};
308
- async initialize(ctx, opts) {
309
- this.context = ctx;
310
- this.options = opts ?? {};
20
+ async initialize(_ctx) {
311
21
  }
312
22
  async shutdown() {
313
- this.context = null;
314
23
  }
315
24
  getCapabilityProvider(name) {
316
25
  if (name === "addon-pages") {
@@ -322,435 +31,8 @@ var BenchmarkAddon = class {
322
31
  }
323
32
  return null;
324
33
  }
325
- /**
326
- * Run a benchmark locally and return a MultiBenchmarkReport.
327
- */
328
- async run(config, onProgress) {
329
- const systemInfo = collectSystemInfo();
330
- const results = await this.runLocally(config, onProgress);
331
- const leaderboard = results.sort((a, b) => a.latency.mean - b.latency.mean).map((r, idx) => ({
332
- rank: idx + 1,
333
- agentId: "local",
334
- label: r.label,
335
- meanMs: r.latency.mean,
336
- fps: r.fps,
337
- peakMemoryMB: r.resources.peakMemoryMB,
338
- accuracy: r.accuracy
339
- }));
340
- return {
341
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
342
- source: config.source,
343
- agents: [
344
- {
345
- agentId: "local",
346
- status: "completed",
347
- systemInfo,
348
- results
349
- }
350
- ],
351
- leaderboard
352
- };
353
- }
354
- /**
355
- * Run a distributed benchmark across agents.
356
- *
357
- * Stub: distributed execution requires a BenchmarkHub instance bound to
358
- * live WebSocket connections. Call BenchmarkHub.dispatch() directly from
359
- * the server transport layer. This method runs locally as a fallback.
360
- */
361
- async runDistributed(config, onProgress) {
362
- return this.run(config, onProgress);
363
- }
364
- async runLocally(config, onProgress) {
365
- const { inferenceFactory, referenceImages = [] } = this.options;
366
- const firstTarget = config.targets[0];
367
- if (!firstTarget) {
368
- throw new Error("BenchmarkConfig must have at least one target");
369
- }
370
- if (!inferenceFactory) {
371
- return config.targets.map((target) => ({
372
- label: target.label,
373
- addonId: target.addonId,
374
- config: target.config,
375
- latency: { mean: 0, median: 0, p95: 0, p99: 0, min: 0, max: 0 },
376
- fps: 0,
377
- resources: { peakMemoryMB: 0, avgCpuPercent: 0 }
378
- }));
379
- }
380
- const runInference = inferenceFactory(firstTarget.addonId, firstTarget.config);
381
- const runner = new BenchmarkRunner({
382
- runInference,
383
- referenceImages,
384
- onProgress
385
- });
386
- return runner.runAll(config);
387
- }
388
- };
389
-
390
- // src/annotator/result-annotator.ts
391
- import sharp from "sharp";
392
- var COLOR_PALETTE = [
393
- "#FF4136",
394
- "#2ECC40",
395
- "#0074D9",
396
- "#FF851B",
397
- "#B10DC9",
398
- "#FFDC00",
399
- "#7FDBFF",
400
- "#01FF70",
401
- "#F012BE",
402
- "#3D9970"
403
- ];
404
- function classColor(cls) {
405
- let hash = 0;
406
- for (let i = 0; i < cls.length; i++) {
407
- hash = hash * 31 + (cls.codePointAt(i) ?? 0) >>> 0;
408
- }
409
- return COLOR_PALETTE[hash % COLOR_PALETTE.length] ?? "#FF4136";
410
- }
411
- function escSvg(str) {
412
- return str.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
413
- }
414
- function buildSvgOverlay(detections, width, height) {
415
- const rects = detections.map((det) => {
416
- const color = classColor(det.class);
417
- const label = escSvg(`${det.class} ${(det.score * 100).toFixed(0)}%`);
418
- const { x, y, w, h } = det.bbox;
419
- const clampX = Math.max(0, Math.min(x, width - 1));
420
- const clampY = Math.max(0, Math.min(y, height - 1));
421
- const clampW = Math.max(1, Math.min(w, width - clampX));
422
- const clampH = Math.max(1, Math.min(h, height - clampY));
423
- const labelY = clampY > 14 ? clampY - 2 : clampY + clampH + 12;
424
- return `
425
- <rect x="${clampX}" y="${clampY}" width="${clampW}" height="${clampH}"
426
- stroke="${color}" stroke-width="2" fill="none"/>
427
- <rect x="${clampX}" y="${labelY - 12}" width="${label.length * 7}" height="14"
428
- fill="${color}" fill-opacity="0.75"/>
429
- <text x="${clampX + 2}" y="${labelY}" font-family="monospace" font-size="11"
430
- fill="white">${label}</text>`;
431
- });
432
- return `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}">${rects.join("")}</svg>`;
433
- }
434
- async function buildMaskOverlays(detections, width, height) {
435
- const overlays = [];
436
- for (const det of detections) {
437
- if (!det.mask || !det.maskWidth || !det.maskHeight) continue;
438
- const color = classColor(det.class);
439
- const r = parseInt(color.slice(1, 3), 16);
440
- const g = parseInt(color.slice(3, 5), 16);
441
- const b = parseInt(color.slice(5, 7), 16);
442
- const rgba = Buffer.alloc(det.maskWidth * det.maskHeight * 4);
443
- for (let i = 0; i < det.mask.length; i++) {
444
- const alpha = det.mask[i] ?? 0;
445
- if (alpha > 0) {
446
- const base = i * 4;
447
- rgba[base] = r;
448
- rgba[base + 1] = g;
449
- rgba[base + 2] = b;
450
- rgba[base + 3] = Math.round(alpha * 0.5);
451
- }
452
- }
453
- const maskBuffer = await sharp(rgba, {
454
- raw: { width: det.maskWidth, height: det.maskHeight, channels: 4 }
455
- }).resize(width, height).png().toBuffer();
456
- overlays.push(maskBuffer);
457
- }
458
- return overlays;
459
- }
460
- async function annotateImage(frame, detections, outputPath) {
461
- let image;
462
- if (frame.format === "jpeg") {
463
- image = sharp(frame.data);
464
- } else {
465
- image = sharp(frame.data, {
466
- raw: { width: frame.width, height: frame.height, channels: 3 }
467
- });
468
- }
469
- const composites = [];
470
- const maskBuffers = await buildMaskOverlays(detections, frame.width, frame.height);
471
- for (const maskBuf of maskBuffers) {
472
- composites.push({ input: maskBuf, blend: "over" });
473
- }
474
- const svgBuf = Buffer.from(buildSvgOverlay(detections, frame.width, frame.height));
475
- composites.push({ input: svgBuf, blend: "over" });
476
- await image.composite(composites).jpeg({ quality: 90 }).toFile(outputPath);
477
- }
478
-
479
- // src/distributed/benchmark-hub.ts
480
- var BenchmarkHub = class {
481
- constructor(send) {
482
- this.send = send;
483
- }
484
- agents = /* @__PURE__ */ new Map();
485
- activeTaskId = null;
486
- resolveTask = null;
487
- rejectTask = null;
488
- pendingConfig = null;
489
- /** Register a connected agent (typically on WS connect + hello message) */
490
- registerAgent(agentId, systemInfo) {
491
- this.agents.set(agentId, {
492
- agentId,
493
- systemInfo,
494
- status: "idle"
495
- });
496
- }
497
- /** Mark an agent as disconnected */
498
- disconnectAgent(agentId) {
499
- const entry = this.agents.get(agentId);
500
- if (entry) {
501
- this.agents.set(agentId, { ...entry, status: "disconnected" });
502
- }
503
- this.checkCompletion();
504
- }
505
- /** Handle incoming message from any agent */
506
- handleMessage(message) {
507
- switch (message.type) {
508
- case "agent.hello":
509
- this.onAgentHello(message);
510
- break;
511
- case "benchmark.progress":
512
- this.onProgress(message);
513
- break;
514
- case "benchmark.result":
515
- this.onResult(message);
516
- break;
517
- case "benchmark.error":
518
- this.onError(message);
519
- break;
520
- default:
521
- break;
522
- }
523
- }
524
- /**
525
- * Dispatch a benchmark to a subset of agents.
526
- * Returns a promise that resolves when all targeted agents have finished.
527
- *
528
- * @param config - Benchmark configuration
529
- * @param agentIds - If empty, dispatches to all connected/idle agents
530
- * @param referenceImages - Reference image buffers to send to agents
531
- */
532
- async dispatch(config, agentIds, referenceImages = []) {
533
- const taskId = `task-${Date.now()}-${Math.random().toString(36).slice(2)}`;
534
- this.activeTaskId = taskId;
535
- this.pendingConfig = config;
536
- const targets = agentIds.length > 0 ? agentIds : [...this.agents.values()].filter((a) => a.status === "idle").map((a) => a.agentId);
537
- if (targets.length === 0) {
538
- throw new Error("No idle agents available for benchmark dispatch");
539
- }
540
- for (const agentId of targets) {
541
- const entry = this.agents.get(agentId);
542
- if (entry) {
543
- this.agents.set(agentId, { ...entry, status: "running", results: void 0, error: void 0 });
544
- }
545
- }
546
- for (const agentId of targets) {
547
- this.send(agentId, {
548
- type: "benchmark.start",
549
- taskId,
550
- config,
551
- referenceImages
552
- });
553
- }
554
- return new Promise((resolve, reject) => {
555
- this.resolveTask = resolve;
556
- this.rejectTask = reject;
557
- });
558
- }
559
- onAgentHello(message) {
560
- this.registerAgent(message.agentId, message.systemInfo);
561
- }
562
- onProgress(message) {
563
- const entry = this.agents.get(message.agentId);
564
- if (entry) {
565
- this.agents.set(message.agentId, {
566
- ...entry,
567
- completedIterations: message.completed
568
- });
569
- }
570
- }
571
- onResult(message) {
572
- const entry = this.agents.get(message.agentId);
573
- if (entry) {
574
- this.agents.set(message.agentId, {
575
- ...entry,
576
- status: "done",
577
- systemInfo: message.systemInfo,
578
- results: message.results
579
- });
580
- }
581
- this.checkCompletion();
582
- }
583
- onError(message) {
584
- const entry = this.agents.get(message.agentId);
585
- if (entry) {
586
- this.agents.set(message.agentId, {
587
- ...entry,
588
- status: "error",
589
- error: message.error
590
- });
591
- }
592
- this.checkCompletion();
593
- }
594
- checkCompletion() {
595
- if (!this.resolveTask || !this.activeTaskId) return;
596
- const runningAgents = [...this.agents.values()].filter(
597
- (a) => a.status === "running"
598
- );
599
- if (runningAgents.length > 0) return;
600
- const report = this.buildReport();
601
- this.resolveTask(report);
602
- this.resolveTask = null;
603
- this.rejectTask = null;
604
- this.activeTaskId = null;
605
- }
606
- buildReport() {
607
- const source = this.pendingConfig?.source ?? { type: "reference", images: "all" };
608
- const agentEntries = [...this.agents.values()];
609
- const allResults = [];
610
- for (const agent of agentEntries) {
611
- if (agent.results) {
612
- for (const result of agent.results) {
613
- allResults.push({ agentId: agent.agentId, result });
614
- }
615
- }
616
- }
617
- const leaderboard = allResults.sort((a, b) => a.result.latency.mean - b.result.latency.mean).map((entry, idx) => ({
618
- rank: idx + 1,
619
- agentId: entry.agentId,
620
- label: entry.result.label,
621
- meanMs: entry.result.latency.mean,
622
- fps: entry.result.fps,
623
- peakMemoryMB: entry.result.resources.peakMemoryMB,
624
- accuracy: entry.result.accuracy
625
- }));
626
- return {
627
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
628
- source,
629
- agents: agentEntries.map((a) => ({
630
- agentId: a.agentId,
631
- status: a.status === "done" ? "completed" : a.status === "error" ? "failed" : a.status === "disconnected" ? "disconnected" : "partial",
632
- systemInfo: a.systemInfo ?? {
633
- os: "unknown",
634
- arch: "unknown",
635
- cpuModel: "unknown",
636
- cpuCores: 0,
637
- totalMemoryMB: 0,
638
- nodeVersion: "unknown"
639
- },
640
- results: a.results ?? [],
641
- error: a.error,
642
- completedIterations: a.completedIterations
643
- })),
644
- leaderboard
645
- };
646
- }
647
- };
648
-
649
- // src/distributed/benchmark-agent.ts
650
- var BenchmarkAgent = class {
651
- constructor(agentId, send, inferenceFactory) {
652
- this.send = send;
653
- this.inferenceFactory = inferenceFactory;
654
- this.agentId = agentId;
655
- this.systemInfo = collectSystemInfo();
656
- }
657
- agentId;
658
- systemInfo;
659
- cancelRequested = false;
660
- /** Send hello to the hub — call this on WS connect */
661
- hello() {
662
- this.send({
663
- type: "agent.hello",
664
- agentId: this.agentId,
665
- systemInfo: this.systemInfo
666
- });
667
- }
668
- /** Handle incoming message from the hub */
669
- handleMessage(message) {
670
- switch (message.type) {
671
- case "benchmark.start":
672
- void this.onStart(message);
673
- break;
674
- case "benchmark.cancel":
675
- this.onCancel(message);
676
- break;
677
- default:
678
- break;
679
- }
680
- }
681
- async onStart(message) {
682
- this.cancelRequested = false;
683
- const { taskId, config } = message;
684
- try {
685
- const referenceImages = message.referenceImages.map((img) => ({
686
- frame: {
687
- data: img.data,
688
- format: "jpeg",
689
- width: img.width,
690
- height: img.height,
691
- timestamp: Date.now()
692
- },
693
- groundTruth: { image: img.name, annotations: [] }
694
- }));
695
- const firstTarget = config.targets[0];
696
- if (!firstTarget) {
697
- throw new Error("No benchmark targets specified");
698
- }
699
- const runInference = this.inferenceFactory(firstTarget.addonId, firstTarget.config);
700
- const runner = new BenchmarkRunner({
701
- runInference,
702
- referenceImages,
703
- onProgress: (completed, total, currentTarget) => {
704
- if (this.cancelRequested) return;
705
- this.send({
706
- type: "benchmark.progress",
707
- taskId,
708
- agentId: this.agentId,
709
- completed,
710
- total,
711
- currentTarget
712
- });
713
- }
714
- });
715
- const results = await runner.runAll(config);
716
- if (!this.cancelRequested) {
717
- this.send({
718
- type: "benchmark.result",
719
- taskId,
720
- agentId: this.agentId,
721
- systemInfo: this.systemInfo,
722
- results
723
- });
724
- }
725
- } catch (err) {
726
- this.send({
727
- type: "benchmark.error",
728
- taskId,
729
- agentId: this.agentId,
730
- error: err instanceof Error ? err.message : String(err)
731
- });
732
- }
733
- }
734
- onCancel(message) {
735
- if (message.taskId === this.activeTaskId) {
736
- this.cancelRequested = true;
737
- }
738
- }
739
- /** Returns the current task ID (last started), or null if idle */
740
- get activeTaskId() {
741
- return null;
742
- }
743
34
  };
744
35
  export {
745
- BenchmarkAddon,
746
- BenchmarkAgent,
747
- BenchmarkHub,
748
- BenchmarkRunner,
749
- annotateImage,
750
- collectSystemInfo,
751
- computeAccuracy,
752
- computeFps,
753
- computeLatencyStats,
754
- iou
36
+ BenchmarkAddon
755
37
  };
756
38
  //# sourceMappingURL=index.mjs.map