agent-relay 2.0.28 → 2.0.32

package/packages/benchmark/dist/index.d.ts

@@ -0,0 +1,48 @@
+/**
+ * @agent-relay/benchmark
+ *
+ * Performance benchmarking for agent swarms, sub-agents, and single agents.
+ *
+ * ## Quick Start
+ *
+ * ```typescript
+ * import { ComparisonBenchmark, type Task } from '@agent-relay/benchmark';
+ *
+ * const task: Task = {
+ *   id: 'refactor-auth',
+ *   description: 'Refactor authentication to use JWT',
+ *   files: ['src/auth/session.ts', 'src/auth/middleware.ts'],
+ *   expectedOutcome: 'All tests pass, JWT tokens used',
+ *   complexity: 'medium',
+ * };
+ *
+ * const benchmark = new ComparisonBenchmark();
+ * const comparison = await benchmark.runComparison(task);
+ *
+ * console.log(`Winner: ${comparison.winner}`);
+ * benchmark.printComparison(comparison);
+ * ```
+ *
+ * ## With Harbor
+ *
+ * ```bash
+ * harbor run \
+ *   --dataset tasks.yaml \
+ *   --agent @agent-relay/benchmark/harbor \
+ *   --parallel 10
+ * ```
+ *
+ * ## CLI Usage
+ *
+ * ```bash
+ * relay-benchmark run --dataset tasks.yaml --config all
+ * relay-benchmark run --dataset tasks.yaml --config swarm
+ * relay-benchmark list tasks.yaml
+ * ```
+ */
+export type { ConfigurationType, TaskComplexity, Task, RunResult, ComparisonResult, ScoreBreakdown, BenchmarkConfig, RunMetrics, TaskDataset, HarborTaskInput, HarborEvaluationOutput, } from './types.js';
+export { DEFAULT_BENCHMARK_CONFIG } from './types.js';
+export { ComparisonBenchmark, runComparison } from './benchmark.js';
+export { ConfigurationRunner, SingleAgentRunner, SubAgentRunner, SwarmRunner, } from './runners/index.js';
+export { evaluate, evaluateSingle, evaluateCustom } from './harbor.js';
+//# sourceMappingURL=index.d.ts.map

package/packages/benchmark/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAGH,YAAY,EACV,iBAAiB,EACjB,cAAc,EACd,IAAI,EACJ,SAAS,EACT,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,UAAU,EACV,WAAW,EACX,eAAe,EACf,sBAAsB,GACvB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAGtD,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpE,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC"}

package/packages/benchmark/dist/index.js

@@ -0,0 +1,50 @@
+/**
+ * @agent-relay/benchmark
+ *
+ * Performance benchmarking for agent swarms, sub-agents, and single agents.
+ *
+ * ## Quick Start
+ *
+ * ```typescript
+ * import { ComparisonBenchmark, type Task } from '@agent-relay/benchmark';
+ *
+ * const task: Task = {
+ *   id: 'refactor-auth',
+ *   description: 'Refactor authentication to use JWT',
+ *   files: ['src/auth/session.ts', 'src/auth/middleware.ts'],
+ *   expectedOutcome: 'All tests pass, JWT tokens used',
+ *   complexity: 'medium',
+ * };
+ *
+ * const benchmark = new ComparisonBenchmark();
+ * const comparison = await benchmark.runComparison(task);
+ *
+ * console.log(`Winner: ${comparison.winner}`);
+ * benchmark.printComparison(comparison);
+ * ```
+ *
+ * ## With Harbor
+ *
+ * ```bash
+ * harbor run \
+ *   --dataset tasks.yaml \
+ *   --agent @agent-relay/benchmark/harbor \
+ *   --parallel 10
+ * ```
+ *
+ * ## CLI Usage
+ *
+ * ```bash
+ * relay-benchmark run --dataset tasks.yaml --config all
+ * relay-benchmark run --dataset tasks.yaml --config swarm
+ * relay-benchmark list tasks.yaml
+ * ```
+ */
+export { DEFAULT_BENCHMARK_CONFIG } from './types.js';
+// Main benchmark class
+export { ComparisonBenchmark, runComparison } from './benchmark.js';
+// Runners
+export { ConfigurationRunner, SingleAgentRunner, SubAgentRunner, SwarmRunner, } from './runners/index.js';
+// Harbor integration
+export { evaluate, evaluateSingle, evaluateCustom } from './harbor.js';
+//# sourceMappingURL=index.js.map

package/packages/benchmark/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAiBH,OAAO,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAEtD,uBAAuB;AACvB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpE,UAAU;AACV,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAE5B,qBAAqB;AACrB,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC"}

package/packages/benchmark/dist/runners/base.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Base Configuration Runner
+ *
+ * Abstract base class for benchmark configuration runners.
+ */
+import { RelayClient, type MetricsResponsePayload, type Relay } from '@agent-relay/sdk';
+import type { ConfigurationType, Task, RunResult, RunMetrics, BenchmarkConfig } from '../types.js';
+/**
+ * Abstract base class for configuration runners
+ */
+export declare abstract class ConfigurationRunner {
+    protected relay: Relay;
+    protected orchestrator: RelayClient;
+    protected config: BenchmarkConfig;
+    protected metrics: RunMetrics;
+    constructor(config?: Partial<BenchmarkConfig>);
+    /**
+     * Get the configuration type this runner handles
+     */
+    abstract get configurationType(): ConfigurationType;
+    /**
+     * Set up the relay and orchestrator client
+     */
+    setup(): Promise<void>;
+    /**
+     * Run a task and return the result
+     */
+    abstract run(task: Task): Promise<RunResult>;
+    /**
+     * Clean up resources
+     */
+    teardown(): Promise<void>;
+    /**
+     * Reset metrics for a new run
+     */
+    protected resetMetrics(): void;
+    /**
+     * Calculate percentile from an array of values
+     */
+    protected percentile(arr: number[], p: number): number;
+    /**
+     * Extract total tokens from metrics response
+     */
+    protected extractTokens(metrics: MetricsResponsePayload): number;
+    /**
+     * Extract peak memory from metrics response
+     */
+    protected extractMemory(metrics: MetricsResponsePayload): number;
+    /**
+     * Build a failed result when setup fails
+     */
+    protected buildFailedResult(task: Task, startTime: number, errors: string[]): RunResult;
+    /**
+     * Wait for an agent to complete their task
+     */
+    protected waitForCompletion(agentName: string, task: Task, donePrefix?: string): Promise<boolean>;
+    /**
+     * Log a message if not in quiet mode
+     */
+    protected log(message: string): void;
+}
+export { DEFAULT_BENCHMARK_CONFIG } from '../types.js';
+//# sourceMappingURL=base.d.ts.map

package/packages/benchmark/dist/runners/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../src/runners/base.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAEL,WAAW,EACX,KAAK,sBAAsB,EAC3B,KAAK,KAAK,EACX,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EACV,iBAAiB,EACjB,IAAI,EACJ,SAAS,EACT,UAAU,EACV,eAAe,EAChB,MAAM,aAAa,CAAC;AAQrB;;GAEG;AACH,8BAAsB,mBAAmB;IACvC,SAAS,CAAC,KAAK,EAAG,KAAK,CAAC;IACxB,SAAS,CAAC,YAAY,EAAG,WAAW,CAAC;IACrC,SAAS,CAAC,MAAM,EAAE,eAAe,CAAC;IAClC,SAAS,CAAC,OAAO,EAAE,UAAU,CAM3B;gBAEU,MAAM,GAAE,OAAO,CAAC,eAAe,CAAM;IAIjD;;OAEG;IACH,QAAQ,KAAK,iBAAiB,IAAI,iBAAiB,CAAC;IAEpD;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5B;;OAEG;IACH,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC;IAE5C;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAa/B;;OAEG;IACH,SAAS,CAAC,YAAY,IAAI,IAAI;IAU9B;;OAEG;IACH,SAAS,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM;IAOtD;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM;IAOhE;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM;IAUhE;;OAEG;IACH,SAAS,CAAC,iBAAiB,CACzB,IAAI,EAAE,IAAI,EACV,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EAAE,GACf,SAAS;IAuBZ;;OAEG;IACH,SAAS,CAAC,iBAAiB,CACzB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,IAAI,EACV,UAAU,SAAU,GACnB,OAAO,CAAC,OAAO,CAAC;IAwBnB;;OAEG;IACH,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;CAKrC;AAGD,OAAO,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC"}

package/packages/benchmark/dist/runners/base.js

@@ -0,0 +1,155 @@
+/**
+ * Base Configuration Runner
+ *
+ * Abstract base class for benchmark configuration runners.
+ */
+import { createRelay, } from '@agent-relay/sdk';
+import { DEFAULT_BENCHMARK_CONFIG } from '../types.js';
+/**
+ * Abstract base class for configuration runners
+ */
+export class ConfigurationRunner {
+    relay;
+    orchestrator;
+    config;
+    metrics = {
+        messages: 0,
+        latencies: [],
+        startTime: 0,
+        spawnedAgents: [],
+        errors: [],
+    };
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_BENCHMARK_CONFIG, ...config };
+    }
+    /**
+     * Set up the relay and orchestrator client
+     */
+    async setup() {
+        this.relay = await createRelay({
+            socketPath: this.config.socketPath,
+            quiet: this.config.quiet,
+        });
+        this.orchestrator = await this.relay.client('Orchestrator', {
+            quiet: this.config.quiet,
+        });
+        this.resetMetrics();
+    }
+    /**
+     * Clean up resources
+     */
+    async teardown() {
+        // Release any remaining agents
+        for (const agent of this.metrics.spawnedAgents) {
+            try {
+                await this.orchestrator.release(agent);
+            }
+            catch {
+                // Ignore release errors during cleanup
+            }
+        }
+        await this.relay.stop();
+    }
+    /**
+     * Reset metrics for a new run
+     */
+    resetMetrics() {
+        this.metrics = {
+            messages: 0,
+            latencies: [],
+            startTime: Date.now(),
+            spawnedAgents: [],
+            errors: [],
+        };
+    }
+    /**
+     * Calculate percentile from an array of values
+     */
+    percentile(arr, p) {
+        if (arr.length === 0)
+            return 0;
+        const sorted = [...arr].sort((a, b) => a - b);
+        const idx = Math.ceil((p / 100) * sorted.length) - 1;
+        return sorted[Math.max(0, idx)];
+    }
+    /**
+     * Extract total tokens from metrics response
+     */
+    extractTokens(metrics) {
+        const agents = metrics.agents;
+        return (agents?.reduce((sum, agent) => sum + (agent.tokens || 0), 0) || 0);
+    }
+    /**
+     * Extract peak memory from metrics response
+     */
+    extractMemory(metrics) {
+        const agents = metrics.agents;
+        const memoryValues = agents?.map((agent) => {
+            if (agent.memoryMb != null)
+                return agent.memoryMb;
+            if (agent.rssBytes != null)
+                return agent.rssBytes / 1024 / 1024;
+            return 0;
+        });
+        return Math.max(...(memoryValues || [0]));
+    }
+    /**
+     * Build a failed result when setup fails
+     */
+    buildFailedResult(task, startTime, errors) {
+        const now = Date.now();
+        return {
+            taskId: task.id,
+            configuration: this.configurationType,
+            totalTimeMs: now - startTime,
+            timeToFirstActionMs: 0,
+            messageCount: 0,
+            avgLatencyMs: 0,
+            latencyP50Ms: 0,
+            latencyP99Ms: 0,
+            coordinationRounds: 0,
+            agentCount: 0,
+            totalTokensUsed: 0,
+            peakMemoryMb: 0,
+            success: false,
+            completionRate: 0,
+            errors,
+            startedAt: startTime,
+            completedAt: now,
+        };
+    }
+    /**
+     * Wait for an agent to complete their task
+     */
+    waitForCompletion(agentName, task, donePrefix = 'DONE:') {
+        const timeoutMs = task.timeoutMs || 300000;
+        return new Promise((resolve) => {
+            const timeout = setTimeout(() => {
+                this.metrics.errors.push(`Timeout waiting for ${agentName}`);
+                resolve(false);
+            }, timeoutMs);
+            const originalHandler = this.orchestrator.onMessage;
+            this.orchestrator.onMessage = (from, payload, id, meta, originalTo) => {
+                // Call original handler if exists
+                if (originalHandler) {
+                    originalHandler(from, payload, id, meta, originalTo);
+                }
+                if (from === agentName && payload.body.startsWith(donePrefix)) {
+                    clearTimeout(timeout);
+                    resolve(true);
+                }
+            };
+        });
+    }
+    /**
+     * Log a message if not in quiet mode
+     */
+    log(message) {
+        if (!this.config.quiet) {
+            console.log(`[${this.configurationType}] ${message}`);
+        }
+    }
+}
+// Re-export the DEFAULT_BENCHMARK_CONFIG
+export { DEFAULT_BENCHMARK_CONFIG } from '../types.js';
+//# sourceMappingURL=base.js.map

package/packages/benchmark/dist/runners/base.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/runners/base.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,WAAW,GAIZ,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAOvD;;GAEG;AACH,MAAM,OAAgB,mBAAmB;IAC7B,KAAK,CAAS;IACd,YAAY,CAAe;IAC3B,MAAM,CAAkB;IACxB,OAAO,GAAe;QAC9B,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,CAAC;QACZ,aAAa,EAAE,EAAE;QACjB,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,YAAY,SAAmC,EAAE;QAC/C,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,wBAAwB,EAAE,GAAG,MAAM,EAAE,CAAC;IAC3D,CAAC;IAOD;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,KAAK,GAAG,MAAM,WAAW,CAAC;YAC7B,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;YAClC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;SACzB,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE;YAC1D,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;SACzB,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,EAAE,CAAC;IACtB,CAAC;IAOD;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,+BAA+B;QAC/B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;YAC/C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACzC,CAAC;YAAC,MAAM,CAAC;gBACP,uCAAuC;YACzC,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,YAAY;QACpB,IAAI,CAAC,OAAO,GAAG;YACb,QAAQ,EAAE,CAAC;YACX,SAAS,EAAE,EAAE;YACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,aAAa,EAAE,EAAE;YACjB,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;IAED;;OAEG;IACO,UAAU,CAAC,GAAa,EAAE,CAAS;QAC3C,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,OAA+B;QACrD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAoC,CAAC;QAC5D,OAAO,CACL,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAClE,CAAC;IACJ,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,OAA+B;QACrD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAoC,CAAC;QAC5D,MAAM,YAAY,GAAG,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACzC,IAAI,KAAK,CAAC,QAAQ,IAAI,IAAI;gBAAE,OAAO,KAAK,CAAC,QAAQ,CAAC;YAClD,IAAI,KAAK,CAAC,QAAQ,IAAI,IAAI;gBAAE,OAAO,KAAK,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC;YAChE,OAAO,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,iBAAiB,CACzB,IAAU,EACV,SAAiB,EACjB,MAAgB;QAEhB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,aAAa,EAAE,IAAI,CAAC,iBAAiB;YACrC,WAAW,EAAE,GAAG,GAAG,SAAS;YAC5B,mBAAmB,EAAE,CAAC;YACtB,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,CAAC;YACf,kBAAkB,EAAE,CAAC;YACrB,UAAU,EAAE,CAAC;YACb,eAAe,EAAE,CAAC;YAClB,YAAY,EAAE,CAAC;YACf,OAAO,EAAE,KAAK;YACd,cAAc,EAAE,CAAC;YACjB,MAAM;YACN,SAAS,EAAE,SAAS;YACpB,WAAW,EAAE,GAAG;SACjB,CAAC;IACJ,CAAC;IAED;;OAEG;IACO,iBAAiB,CACzB,SAAiB,EACjB,IAAU,EACV,UAAU,GAAG,OAAO;QAEpB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC;QAE3C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC9B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,SAAS,EAAE,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,CAAC;YACjB,CAAC,EAAE,SAAS,CAAC,CAAC;YAEd,MAAM,eAAe,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC;YACpD,IAAI,CAAC,YAAY,CAAC,SAAS,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE;gBACpE,kCAAkC;gBAClC,IAAI,eAAe,EAAE,CAAC;oBACpB,eAAe,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;gBACvD,CAAC;gBAED,IAAI,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC9D,YAAY,CAAC,OAAO,CAAC,CAAC;oBACtB,OAAO,CAAC,IAAI,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACO,GAAG,CAAC,OAAe;QAC3B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,iBAAiB,KAAK,OAAO,EAAE,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;CACF;AAED,yCAAyC;AACzC,OAAO,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC"}

package/packages/benchmark/dist/runners/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Benchmark Runners
+ *
+ * Export all configuration runners.
+ */
+export { ConfigurationRunner, DEFAULT_BENCHMARK_CONFIG } from './base.js';
+export { SingleAgentRunner } from './single.js';
+export { SubAgentRunner } from './subagent.js';
+export { SwarmRunner } from './swarm.js';
+//# sourceMappingURL=index.d.ts.map

package/packages/benchmark/dist/runners/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/runners/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,WAAW,CAAC;AAC1E,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC"}

package/packages/benchmark/dist/runners/index.js

@@ -0,0 +1,10 @@
+/**
+ * Benchmark Runners
+ *
+ * Export all configuration runners.
+ */
+export { ConfigurationRunner, DEFAULT_BENCHMARK_CONFIG } from './base.js';
+export { SingleAgentRunner } from './single.js';
+export { SubAgentRunner } from './subagent.js';
+export { SwarmRunner } from './swarm.js';
+//# sourceMappingURL=index.js.map

package/packages/benchmark/dist/runners/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/runners/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,WAAW,CAAC;AAC1E,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC"}

package/packages/benchmark/dist/runners/single.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Single Agent Runner
+ *
+ * Runs tasks using a single agent with no delegation.
+ */
+import type { Task, RunResult, ConfigurationType } from '../types.js';
+import { ConfigurationRunner } from './base.js';
+/**
+ * Runner for single-agent configuration
+ */
+export declare class SingleAgentRunner extends ConfigurationRunner {
+    get configurationType(): ConfigurationType;
+    run(task: Task): Promise<RunResult>;
+    /**
+     * Build the task prompt for a single agent
+     */
+    private buildTaskPrompt;
+}
+//# sourceMappingURL=single.d.ts.map

package/packages/benchmark/dist/runners/single.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"single.d.ts","sourceRoot":"","sources":["../../src/runners/single.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AAEhD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,mBAAmB;IACxD,IAAI,iBAAiB,IAAI,iBAAiB,CAEzC;IAEK,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC;IAwEzC;;OAEG;IACH,OAAO,CAAC,eAAe;CA4BxB"}

package/packages/benchmark/dist/runners/single.js

@@ -0,0 +1,111 @@
+/**
+ * Single Agent Runner
+ *
+ * Runs tasks using a single agent with no delegation.
+ */
+import { ConfigurationRunner } from './base.js';
+/**
+ * Runner for single-agent configuration
+ */
+export class SingleAgentRunner extends ConfigurationRunner {
+    get configurationType() {
+        return 'single';
+    }
+    async run(task) {
+        const startTime = Date.now();
+        let firstActionTime = 0;
+        let success = false;
+        this.resetMetrics();
+        this.log(`Starting task: ${task.id}`);
+        try {
+            // Spawn single agent with full task
+            const result = await this.orchestrator.spawn({
+                name: 'SoloAgent',
+                cli: this.config.cli,
+                task: this.buildTaskPrompt(task),
+                cwd: this.config.cwd,
+            });
+            if (!result.success) {
+                this.metrics.errors.push(result.error || 'Spawn failed');
+                return this.buildFailedResult(task, startTime, this.metrics.errors);
+            }
+            this.metrics.spawnedAgents.push('SoloAgent');
+            firstActionTime = Date.now() - startTime;
+            this.log(`Agent spawned in ${firstActionTime}ms`);
+            // Wait for completion signal
+            success = await this.waitForCompletion('SoloAgent', task);
+            if (success) {
+                this.log('Task completed successfully');
+            }
+            else {
+                this.log('Task failed or timed out');
+            }
+        }
+        catch (err) {
+            this.metrics.errors.push(err.message);
+        }
+        // Collect final metrics
+        let totalTokens = 0;
+        let peakMemory = 0;
+        try {
+            const metrics = await this.orchestrator.getMetrics();
+            totalTokens = this.extractTokens(metrics);
+            peakMemory = this.extractMemory(metrics);
+        }
+        catch {
+            // Metrics collection failed, use defaults
+        }
+        const completedAt = Date.now();
+        return {
+            taskId: task.id,
+            configuration: 'single',
+            totalTimeMs: completedAt - startTime,
+            timeToFirstActionMs: firstActionTime,
+            messageCount: 0, // No inter-agent communication
+            avgLatencyMs: 0,
+            latencyP50Ms: 0,
+            latencyP99Ms: 0,
+            coordinationRounds: 0,
+            agentCount: 1,
+            totalTokensUsed: totalTokens,
+            peakMemoryMb: peakMemory,
+            success,
+            completionRate: success ? 1.0 : 0.0,
+            errors: this.metrics.errors,
+            startedAt: startTime,
+            completedAt,
+        };
+    }
+    /**
+     * Build the task prompt for a single agent
+     */
+    buildTaskPrompt(task) {
+        return `Complete this task entirely on your own:
+
+## Task
+${task.description}
+
+## Files to Work On
+${task.files.map((f) => `- ${f}`).join('\n')}
+
+## Success Criteria
+${task.expectedOutcome}
+
+## Instructions
+1. Analyze the task requirements
+2. Plan your approach
+3. Implement the solution
+4. Verify your work meets the success criteria
+
+When complete, send a message to Orchestrator:
+\`\`\`
+DONE: <brief summary of what you accomplished>
+\`\`\`
+
+If you encounter an error you cannot resolve:
+\`\`\`
+ERROR: <description of the problem>
+\`\`\``;
+    }
+}
+//# sourceMappingURL=single.js.map

package/packages/benchmark/dist/runners/single.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"single.js","sourceRoot":"","sources":["../../src/runners/single.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AAEhD;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,mBAAmB;IACxD,IAAI,iBAAiB;QACnB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QAClB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QAEtC,IAAI,CAAC;YACH,oCAAoC;YACpC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;gBAC3C,IAAI,EAAE,WAAW;gBACjB,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;gBACpB,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;gBAChC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,cAAc,CAAC,CAAC;gBACzD,OAAO,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACtE,CAAC;YAED,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC7C,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,oBAAoB,eAAe,IAAI,CAAC,CAAC;YAElD,6BAA6B;YAC7B,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YAE1D,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAE,GAAa,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,wBAAwB;QACxB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC;YACrD,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YAC1C,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,0CAA0C;QAC5C,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE/B,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,aAAa,EAAE,QAAQ;YACvB,WAAW,EAAE,WAAW,GAAG,SAAS;YACpC,mBAAmB,EAAE,eAAe;YACpC,YAAY,EAAE,CAAC,EAAE,+BAA+B;YAChD,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,CAAC;YACf,kBAAkB,EAAE,CAAC;YACrB,UAAU,EAAE,CAAC;YACb,eAAe,EAAE,WAAW;YAC5B,YAAY,EAAE,UAAU;YACxB,OAAO;YACP,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;YACnC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;YAC3B,SAAS,EAAE,SAAS;YACpB,WAAW;SACZ,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,IAAU;QAChC,OAAO;;;EAGT,IAAI,CAAC,WAAW;;;EAGhB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;EAG1C,IAAI,CAAC,eAAe;;;;;;;;;;;;;;;;OAgBf,CAAC;IACN,CAAC;CACF"}

package/packages/benchmark/dist/runners/subagent.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Sub-Agent Runner
+ *
+ * Runs tasks using a lead agent that spawns and coordinates workers hierarchically.
+ */
+import type { Task, RunResult, ConfigurationType } from '../types.js';
+import { ConfigurationRunner } from './base.js';
+/**
+ * Runner for sub-agent (hierarchical) configuration
+ */
+export declare class SubAgentRunner extends ConfigurationRunner {
+    private workerCount;
+    get configurationType(): ConfigurationType;
+    run(task: Task): Promise<RunResult>;
+    /**
+     * Set up message monitoring to track worker spawns and coordination
+     */
+    private setupMessageMonitoring;
+    /**
+     * Wait for the lead agent to signal completion
+     */
+    private waitForLeadCompletion;
+    /**
+     * Build the task prompt for the lead agent
+     */
+    private buildLeadPrompt;
+    /**
+     * Suggest number of workers based on task complexity
+     */
+    private suggestWorkerCount;
+}
+//# sourceMappingURL=subagent.d.ts.map

package/packages/benchmark/dist/runners/subagent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"subagent.d.ts","sourceRoot":"","sources":["../../src/runners/subagent.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AAEhD;;GAEG;AACH,qBAAa,cAAe,SAAQ,mBAAmB;IACrD,OAAO,CAAC,WAAW,CAAK;IAExB,IAAI,iBAAiB,IAAI,iBAAiB,CAEzC;IAEK,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC;IAmFzC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IA0B9B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAgC7B;;OAEG;IACH,OAAO,CAAC,eAAe;IAuDvB;;OAEG;IACH,OAAO,CAAC,kBAAkB;CAY3B"}