@camstack/addon-benchmark 0.1.0 → 0.1.2

package/package.json

@@ -1,8 +1,15 @@
 {
   "name": "@camstack/addon-benchmark",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Detection addon benchmarking with reference images, accuracy metrics, and distributed execution",
-  "keywords": ["camstack", "addon", "camstack-addon", "benchmark", "detection", "accuracy"],
+  "keywords": [
+    "camstack",
+    "addon",
+    "camstack-addon",
+    "benchmark",
+    "detection",
+    "accuracy"
+  ],
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -12,16 +19,25 @@
   "module": "./dist/index.mjs",
   "types": "./dist/index.d.ts",
   "exports": {
-    ".": { "import": "./dist/index.mjs", "require": "./dist/index.js", "types": "./dist/index.d.ts" }
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
   },
   "camstack": {
+    "displayName": "Benchmark",
     "addons": [
       {
         "id": "benchmark",
         "entry": "./dist/index.js",
         "slot": null,
         "capabilities": [
-          { "name": "addon-pages", "mode": "collection" }
+          {
+            "name": "addon-pages",
+            "mode": "collection"
+          }
         ],
         "pages": [
           {
@@ -29,31 +45,47 @@
             "label": "Benchmark",
             "icon": "gauge",
             "path": "/addon/benchmark",
-            "bundle": "benchmark.js",
-            "element": "camstack-benchmark"
+            "bundle": "page.mjs"
           }
         ]
       }
     ]
   },
-  "files": ["dist", "reference-images"],
+  "files": [
+    "dist",
+    "reference-images"
+  ],
   "scripts": {
-    "build": "tsup && vite build",
+    "build": "tsup --no-dts && vite build",
     "build:server": "tsup",
-    "build:pages": "vite build",
+    "build:page": "vite build",
     "dev": "tsup --watch",
-    "dev:pages": "vite build --watch",
+    "dev:page": "vite",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",
     "test:watch": "vitest"
   },
-  "peerDependencies": { "@camstack/types": "^0.1.0" },
-  "dependencies": { "sharp": "^0.34.0" },
+  "peerDependencies": {
+    "@camstack/types": "^0.1.0",
+    "react": ">=18",
+    "react-dom": ">=18"
+  },
+  "dependencies": {
+    "sharp": "^0.34.0"
+  },
   "devDependencies": {
     "@camstack/types": "*",
     "@camstack/addon-vision": "*",
+    "@camstack/ui-library": "*",
+    "@tailwindcss/vite": "^4.2.0",
+    "@trpc/client": "^11.0.0",
+    "@vitejs/plugin-react": "^4.0.0",
+    "react": "^19.0.0",
+    "react-dom": "^19.0.0",
+    "superjson": "^2.0.0",
     "tsup": "^8.0.0",
     "typescript": "~5.9.0",
+    "tailwindcss": "^4.2.0",
     "vite": "^6.3.0",
     "vitest": "^3.0.0"
   }

package/dist/index.d.mts

@@ -1,222 +0,0 @@
-import { ICamstackAddon, AddonManifest, AddonPageDeclaration, AddonContext, FrameInput, DetectorOutput, GroundTruth, CapabilityProviderMap, BenchmarkConfig, MultiBenchmarkReport, BenchmarkTarget, BenchmarkTargetResult, LatencyStats, SpatialDetection, GroundTruthAnnotation, AccuracyStats, BoundingBox, SystemInfo } from '@camstack/types';
-
-type BenchmarkInferenceFactory = (addonId: string, config: Readonly<Record<string, unknown>>) => (frame: FrameInput, cfg: Readonly<Record<string, unknown>>) => Promise<DetectorOutput>;
-interface BenchmarkAddonOptions {
-    /** Factory to create an inference function for a given addonId + config */
-    readonly inferenceFactory?: BenchmarkInferenceFactory;
-    /** Reference images with ground truth for accuracy evaluation */
-    readonly referenceImages?: ReadonlyArray<{
-        readonly frame: FrameInput;
-        readonly groundTruth: GroundTruth;
-    }>;
-}
-declare class BenchmarkAddon implements ICamstackAddon {
-    readonly id = "benchmark";
-    readonly manifest: AddonManifest;
-    readonly pages: readonly AddonPageDeclaration[];
-    private context;
-    private options;
-    initialize(ctx: AddonContext, opts?: BenchmarkAddonOptions): Promise<void>;
-    shutdown(): Promise<void>;
-    getCapabilityProvider<K extends keyof CapabilityProviderMap>(name: K): CapabilityProviderMap[K] | null;
-    /**
-     * Run a benchmark locally and return a MultiBenchmarkReport.
-     */
-    run(config: BenchmarkConfig, onProgress?: (completed: number, total: number, currentTarget: string) => void): Promise<MultiBenchmarkReport>;
-    /**
-     * Run a distributed benchmark across agents.
-     *
-     * Stub: distributed execution requires a BenchmarkHub instance bound to
-     * live WebSocket connections. Call BenchmarkHub.dispatch() directly from
-     * the server transport layer. This method runs locally as a fallback.
-     */
-    runDistributed(config: BenchmarkConfig, onProgress?: (completed: number, total: number, currentTarget: string) => void): Promise<MultiBenchmarkReport>;
-    private runLocally;
-}
-
-interface BenchmarkRunnerOptions {
-    /** Function to run inference — provided by the addon that's being benchmarked */
-    readonly runInference: (frame: FrameInput, config: Readonly<Record<string, unknown>>) => Promise<DetectorOutput>;
-    /** Reference images with ground truth */
-    readonly referenceImages: readonly {
-        readonly frame: FrameInput;
-        readonly groundTruth: GroundTruth;
-    }[];
-    /** Progress callback */
-    readonly onProgress?: (completed: number, total: number, currentTarget: string) => void;
-}
-declare class BenchmarkRunner {
-    private readonly options;
-    constructor(options: BenchmarkRunnerOptions);
-    /** Run inference on a single frame, picking a reference image round-robin if needed */
-    private runOnce;
-    /** Measure accuracy across all reference images */
-    private measureAccuracy;
-    /** Run benchmark for a single target configuration */
-    runTarget(target: BenchmarkTarget, config: BenchmarkConfig): Promise<BenchmarkTargetResult>;
-    /** Run benchmark for all targets defined in the config */
-    runAll(config: BenchmarkConfig): Promise<BenchmarkTargetResult[]>;
-}
-
-/**
- * Compute latency statistics from an array of timing measurements in milliseconds.
- */
-declare function computeLatencyStats(timingsMs: readonly number[]): LatencyStats;
-/**
- * Compute frames per second from timing measurements.
- */
-declare function computeFps(timingsMs: readonly number[]): number;
-
-/**
- * Compute Intersection over Union (IoU) between two bounding boxes.
- * Boxes are in {x, y, w, h} format where x, y are top-left corner.
- */
-declare function iou(pred: BoundingBox, gt: BoundingBox): number;
-/**
- * Compute mAP@50 and per-class precision/recall for a set of predictions vs ground truth.
- *
- * Uses greedy matching (highest-score prediction matched first).
- * Each ground truth can be matched at most once.
- */
-declare function computeAccuracy(predictions: readonly SpatialDetection[], groundTruth: readonly GroundTruthAnnotation[], iouThreshold?: number): AccuracyStats;
-
-/**
- * Collect system hardware and runtime information.
- */
-declare function collectSystemInfo(): SystemInfo;
-
-/**
- * Draw bounding boxes and optional masks on a frame, write result as JPEG.
- *
- * @param frame - Input image frame
- * @param detections - Detections to annotate
- * @param outputPath - Where to save the annotated JPEG
- */
-declare function annotateImage(frame: FrameInput, detections: readonly SpatialDetection[], outputPath: string): Promise<void>;
-
-interface BenchmarkStartMessage {
-    readonly type: 'benchmark.start';
-    readonly taskId: string;
-    readonly config: BenchmarkConfig;
-    /** Serialised reference images sent to the agent for deterministic results */
-    readonly referenceImages: ReadonlyArray<{
-        readonly name: string;
-        readonly data: Buffer;
-        readonly width: number;
-        readonly height: number;
-    }>;
-}
-interface BenchmarkCancelMessage {
-    readonly type: 'benchmark.cancel';
-    readonly taskId: string;
-}
-interface BenchmarkProgressMessage {
-    readonly type: 'benchmark.progress';
-    readonly taskId: string;
-    readonly agentId: string;
-    readonly completed: number;
-    readonly total: number;
-    readonly currentTarget: string;
-}
-interface BenchmarkResultMessage {
-    readonly type: 'benchmark.result';
-    readonly taskId: string;
-    readonly agentId: string;
-    readonly systemInfo: SystemInfo;
-    readonly results: readonly BenchmarkTargetResult[];
-}
-interface BenchmarkErrorMessage {
-    readonly type: 'benchmark.error';
-    readonly taskId: string;
-    readonly agentId: string;
-    readonly error: string;
-}
-interface AgentHelloMessage {
-    readonly type: 'agent.hello';
-    readonly agentId: string;
-    readonly systemInfo: SystemInfo;
-}
-interface HubAckMessage {
-    readonly type: 'hub.ack';
-    readonly taskId: string;
-    readonly agentId: string;
-}
-type BenchmarkMessage = BenchmarkStartMessage | BenchmarkCancelMessage | BenchmarkProgressMessage | BenchmarkResultMessage | BenchmarkErrorMessage | AgentHelloMessage | HubAckMessage;
-
-type HubSendFn = (agentId: string, message: BenchmarkMessage) => void;
-interface AgentEntry {
-    readonly agentId: string;
-    systemInfo?: SystemInfo;
-    status: 'idle' | 'running' | 'done' | 'error' | 'disconnected';
-    results?: readonly BenchmarkTargetResult[];
-    error?: string;
-    completedIterations?: number;
-}
-/**
- * BenchmarkHub orchestrates benchmark task distribution to connected agents.
- *
- * Transport (WebSocket) is provided externally via the `send` callback.
- * Call `handleMessage()` for every inbound message from agents.
- */
-declare class BenchmarkHub {
-    private readonly send;
-    private readonly agents;
-    private activeTaskId;
-    private resolveTask;
-    private rejectTask;
-    private pendingConfig;
-    constructor(send: HubSendFn);
-    /** Register a connected agent (typically on WS connect + hello message) */
-    registerAgent(agentId: string, systemInfo?: SystemInfo): void;
-    /** Mark an agent as disconnected */
-    disconnectAgent(agentId: string): void;
-    /** Handle incoming message from any agent */
-    handleMessage(message: BenchmarkMessage): void;
-    /**
-     * Dispatch a benchmark to a subset of agents.
-     * Returns a promise that resolves when all targeted agents have finished.
-     *
-     * @param config - Benchmark configuration
-     * @param agentIds - If empty, dispatches to all connected/idle agents
-     * @param referenceImages - Reference image buffers to send to agents
-     */
-    dispatch(config: BenchmarkConfig, agentIds: readonly string[], referenceImages?: ReadonlyArray<{
-        readonly name: string;
-        readonly data: Buffer;
-        readonly width: number;
-        readonly height: number;
-    }>): Promise<MultiBenchmarkReport>;
-    private onAgentHello;
-    private onProgress;
-    private onResult;
-    private onError;
-    private checkCompletion;
-    private buildReport;
-}
-
-type AgentSendFn = (message: BenchmarkMessage) => void;
-type InferenceFactory = (addonId: string, config: Readonly<Record<string, unknown>>) => (frame: FrameInput, cfg: Readonly<Record<string, unknown>>) => Promise<DetectorOutput>;
-/**
- * BenchmarkAgent executes benchmark tasks dispatched by the hub.
- *
- * Transport (WebSocket) is provided externally via the `send` callback.
- * Call `handleMessage()` for every inbound message from the hub.
- */
-declare class BenchmarkAgent {
-    private readonly send;
-    private readonly inferenceFactory;
-    private readonly agentId;
-    private readonly systemInfo;
-    private cancelRequested;
-    constructor(agentId: string, send: AgentSendFn, inferenceFactory: InferenceFactory);
-    /** Send hello to the hub — call this on WS connect */
-    hello(): void;
-    /** Handle incoming message from the hub */
-    handleMessage(message: BenchmarkMessage): void;
-    private onStart;
-    private onCancel;
-    /** Returns the current task ID (last started), or null if idle */
-    private get activeTaskId();
-}
-
-export { type AgentEntry, type AgentHelloMessage, type AgentSendFn, BenchmarkAddon, type BenchmarkAddonOptions, BenchmarkAgent, type BenchmarkCancelMessage, type BenchmarkErrorMessage, BenchmarkHub, type BenchmarkInferenceFactory, type BenchmarkMessage, type BenchmarkProgressMessage, type BenchmarkResultMessage, BenchmarkRunner, type BenchmarkRunnerOptions, type BenchmarkStartMessage, type HubAckMessage, type HubSendFn, type InferenceFactory, annotateImage, collectSystemInfo, computeAccuracy, computeFps, computeLatencyStats, iou };

package/dist/index.d.ts

@@ -1,222 +0,0 @@
-import { ICamstackAddon, AddonManifest, AddonPageDeclaration, AddonContext, FrameInput, DetectorOutput, GroundTruth, CapabilityProviderMap, BenchmarkConfig, MultiBenchmarkReport, BenchmarkTarget, BenchmarkTargetResult, LatencyStats, SpatialDetection, GroundTruthAnnotation, AccuracyStats, BoundingBox, SystemInfo } from '@camstack/types';
-
-type BenchmarkInferenceFactory = (addonId: string, config: Readonly<Record<string, unknown>>) => (frame: FrameInput, cfg: Readonly<Record<string, unknown>>) => Promise<DetectorOutput>;
-interface BenchmarkAddonOptions {
-    /** Factory to create an inference function for a given addonId + config */
-    readonly inferenceFactory?: BenchmarkInferenceFactory;
-    /** Reference images with ground truth for accuracy evaluation */
-    readonly referenceImages?: ReadonlyArray<{
-        readonly frame: FrameInput;
-        readonly groundTruth: GroundTruth;
-    }>;
-}
-declare class BenchmarkAddon implements ICamstackAddon {
-    readonly id = "benchmark";
-    readonly manifest: AddonManifest;
-    readonly pages: readonly AddonPageDeclaration[];
-    private context;
-    private options;
-    initialize(ctx: AddonContext, opts?: BenchmarkAddonOptions): Promise<void>;
-    shutdown(): Promise<void>;
-    getCapabilityProvider<K extends keyof CapabilityProviderMap>(name: K): CapabilityProviderMap[K] | null;
-    /**
-     * Run a benchmark locally and return a MultiBenchmarkReport.
-     */
-    run(config: BenchmarkConfig, onProgress?: (completed: number, total: number, currentTarget: string) => void): Promise<MultiBenchmarkReport>;
-    /**
-     * Run a distributed benchmark across agents.
-     *
-     * Stub: distributed execution requires a BenchmarkHub instance bound to
-     * live WebSocket connections. Call BenchmarkHub.dispatch() directly from
-     * the server transport layer. This method runs locally as a fallback.
-     */
-    runDistributed(config: BenchmarkConfig, onProgress?: (completed: number, total: number, currentTarget: string) => void): Promise<MultiBenchmarkReport>;
-    private runLocally;
-}
-
-interface BenchmarkRunnerOptions {
-    /** Function to run inference — provided by the addon that's being benchmarked */
-    readonly runInference: (frame: FrameInput, config: Readonly<Record<string, unknown>>) => Promise<DetectorOutput>;
-    /** Reference images with ground truth */
-    readonly referenceImages: readonly {
-        readonly frame: FrameInput;
-        readonly groundTruth: GroundTruth;
-    }[];
-    /** Progress callback */
-    readonly onProgress?: (completed: number, total: number, currentTarget: string) => void;
-}
-declare class BenchmarkRunner {
-    private readonly options;
-    constructor(options: BenchmarkRunnerOptions);
-    /** Run inference on a single frame, picking a reference image round-robin if needed */
-    private runOnce;
-    /** Measure accuracy across all reference images */
-    private measureAccuracy;
-    /** Run benchmark for a single target configuration */
-    runTarget(target: BenchmarkTarget, config: BenchmarkConfig): Promise<BenchmarkTargetResult>;
-    /** Run benchmark for all targets defined in the config */
-    runAll(config: BenchmarkConfig): Promise<BenchmarkTargetResult[]>;
-}
-
-/**
- * Compute latency statistics from an array of timing measurements in milliseconds.
- */
-declare function computeLatencyStats(timingsMs: readonly number[]): LatencyStats;
-/**
- * Compute frames per second from timing measurements.
- */
-declare function computeFps(timingsMs: readonly number[]): number;
-
-/**
- * Compute Intersection over Union (IoU) between two bounding boxes.
- * Boxes are in {x, y, w, h} format where x, y are top-left corner.
- */
-declare function iou(pred: BoundingBox, gt: BoundingBox): number;
-/**
- * Compute mAP@50 and per-class precision/recall for a set of predictions vs ground truth.
- *
- * Uses greedy matching (highest-score prediction matched first).
- * Each ground truth can be matched at most once.
- */
-declare function computeAccuracy(predictions: readonly SpatialDetection[], groundTruth: readonly GroundTruthAnnotation[], iouThreshold?: number): AccuracyStats;
-
-/**
- * Collect system hardware and runtime information.
- */
-declare function collectSystemInfo(): SystemInfo;
-
-/**
- * Draw bounding boxes and optional masks on a frame, write result as JPEG.
- *
- * @param frame - Input image frame
- * @param detections - Detections to annotate
- * @param outputPath - Where to save the annotated JPEG
- */
-declare function annotateImage(frame: FrameInput, detections: readonly SpatialDetection[], outputPath: string): Promise<void>;
-
-interface BenchmarkStartMessage {
-    readonly type: 'benchmark.start';
-    readonly taskId: string;
-    readonly config: BenchmarkConfig;
-    /** Serialised reference images sent to the agent for deterministic results */
-    readonly referenceImages: ReadonlyArray<{
-        readonly name: string;
-        readonly data: Buffer;
-        readonly width: number;
-        readonly height: number;
-    }>;
-}
-interface BenchmarkCancelMessage {
-    readonly type: 'benchmark.cancel';
-    readonly taskId: string;
-}
-interface BenchmarkProgressMessage {
-    readonly type: 'benchmark.progress';
-    readonly taskId: string;
-    readonly agentId: string;
-    readonly completed: number;
-    readonly total: number;
-    readonly currentTarget: string;
-}
-interface BenchmarkResultMessage {
-    readonly type: 'benchmark.result';
-    readonly taskId: string;
-    readonly agentId: string;
-    readonly systemInfo: SystemInfo;
-    readonly results: readonly BenchmarkTargetResult[];
-}
-interface BenchmarkErrorMessage {
-    readonly type: 'benchmark.error';
-    readonly taskId: string;
-    readonly agentId: string;
-    readonly error: string;
-}
-interface AgentHelloMessage {
-    readonly type: 'agent.hello';
-    readonly agentId: string;
-    readonly systemInfo: SystemInfo;
-}
-interface HubAckMessage {
-    readonly type: 'hub.ack';
-    readonly taskId: string;
-    readonly agentId: string;
-}
-type BenchmarkMessage = BenchmarkStartMessage | BenchmarkCancelMessage | BenchmarkProgressMessage | BenchmarkResultMessage | BenchmarkErrorMessage | AgentHelloMessage | HubAckMessage;
-
-type HubSendFn = (agentId: string, message: BenchmarkMessage) => void;
-interface AgentEntry {
-    readonly agentId: string;
-    systemInfo?: SystemInfo;
-    status: 'idle' | 'running' | 'done' | 'error' | 'disconnected';
-    results?: readonly BenchmarkTargetResult[];
-    error?: string;
-    completedIterations?: number;
-}
-/**
- * BenchmarkHub orchestrates benchmark task distribution to connected agents.
- *
- * Transport (WebSocket) is provided externally via the `send` callback.
- * Call `handleMessage()` for every inbound message from agents.
- */
-declare class BenchmarkHub {
-    private readonly send;
-    private readonly agents;
-    private activeTaskId;
-    private resolveTask;
-    private rejectTask;
-    private pendingConfig;
-    constructor(send: HubSendFn);
-    /** Register a connected agent (typically on WS connect + hello message) */
-    registerAgent(agentId: string, systemInfo?: SystemInfo): void;
-    /** Mark an agent as disconnected */
-    disconnectAgent(agentId: string): void;
-    /** Handle incoming message from any agent */
-    handleMessage(message: BenchmarkMessage): void;
-    /**
-     * Dispatch a benchmark to a subset of agents.
-     * Returns a promise that resolves when all targeted agents have finished.
-     *
-     * @param config - Benchmark configuration
-     * @param agentIds - If empty, dispatches to all connected/idle agents
-     * @param referenceImages - Reference image buffers to send to agents
-     */
-    dispatch(config: BenchmarkConfig, agentIds: readonly string[], referenceImages?: ReadonlyArray<{
-        readonly name: string;
-        readonly data: Buffer;
-        readonly width: number;
-        readonly height: number;
-    }>): Promise<MultiBenchmarkReport>;
-    private onAgentHello;
-    private onProgress;
-    private onResult;
-    private onError;
-    private checkCompletion;
-    private buildReport;
-}
-
-type AgentSendFn = (message: BenchmarkMessage) => void;
-type InferenceFactory = (addonId: string, config: Readonly<Record<string, unknown>>) => (frame: FrameInput, cfg: Readonly<Record<string, unknown>>) => Promise<DetectorOutput>;
-/**
- * BenchmarkAgent executes benchmark tasks dispatched by the hub.
- *
- * Transport (WebSocket) is provided externally via the `send` callback.
- * Call `handleMessage()` for every inbound message from the hub.
- */
-declare class BenchmarkAgent {
-    private readonly send;
-    private readonly inferenceFactory;
-    private readonly agentId;
-    private readonly systemInfo;
-    private cancelRequested;
-    constructor(agentId: string, send: AgentSendFn, inferenceFactory: InferenceFactory);
-    /** Send hello to the hub — call this on WS connect */
-    hello(): void;
-    /** Handle incoming message from the hub */
-    handleMessage(message: BenchmarkMessage): void;
-    private onStart;
-    private onCancel;
-    /** Returns the current task ID (last started), or null if idle */
-    private get activeTaskId();
-}
-
-export { type AgentEntry, type AgentHelloMessage, type AgentSendFn, BenchmarkAddon, type BenchmarkAddonOptions, BenchmarkAgent, type BenchmarkCancelMessage, type BenchmarkErrorMessage, BenchmarkHub, type BenchmarkInferenceFactory, type BenchmarkMessage, type BenchmarkProgressMessage, type BenchmarkResultMessage, BenchmarkRunner, type BenchmarkRunnerOptions, type BenchmarkStartMessage, type HubAckMessage, type HubSendFn, type InferenceFactory, annotateImage, collectSystemInfo, computeAccuracy, computeFps, computeLatencyStats, iou };