agent-relay 2.0.28 → 2.0.32

package/packages/benchmark/dist/benchmark.js

@@ -0,0 +1,224 @@
+/**
+ * Comparison Benchmark
+ *
+ * Main orchestrator for running comparison benchmarks across configurations.
+ */
+import { DEFAULT_BENCHMARK_CONFIG } from './types.js';
+import { SingleAgentRunner, SubAgentRunner, SwarmRunner, } from './runners/index.js';
+/**
+ * Main benchmark orchestrator
+ */
+export class ComparisonBenchmark {
+    config;
+    runners;
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_BENCHMARK_CONFIG, ...config };
+        // Initialize runners for configured configurations
+        this.runners = new Map();
+        for (const configType of this.config.configurations) {
+            this.runners.set(configType, this.createRunner(configType));
+        }
+    }
+    /**
+     * Create a runner for a configuration type
+     */
+    createRunner(type) {
+        switch (type) {
+            case 'single':
+                return new SingleAgentRunner(this.config);
+            case 'subagent':
+                return new SubAgentRunner(this.config);
+            case 'swarm':
+                return new SwarmRunner(this.config);
+            default:
+                throw new Error(`Unknown configuration type: ${type}`);
+        }
+    }
+    /**
+     * Run a comparison across all configured configurations
+     */
+    async runComparison(task) {
+        const results = new Map();
+        const scores = new Map();
+        for (const [configType, runner] of this.runners) {
+            if (!this.config.quiet) {
+                console.log(`\n=== Running ${configType} configuration ===`);
+                console.log(`Task: ${task.id}`);
+            }
+            try {
+                await runner.setup();
+                const result = await runner.run(task);
+                await runner.teardown();
+                results.set(configType, result);
+                scores.set(configType, this.calculateScore(result));
+                if (!this.config.quiet) {
+                    this.printRunResult(result);
+                }
+            }
+            catch (err) {
+                console.error(`Error running ${configType}:`, err.message);
+                // Create failed result
+                const failedResult = {
+                    taskId: task.id,
+                    configuration: configType,
+                    totalTimeMs: 0,
+                    timeToFirstActionMs: 0,
+                    messageCount: 0,
+                    avgLatencyMs: 0,
+                    latencyP50Ms: 0,
+                    latencyP99Ms: 0,
+                    coordinationRounds: 0,
+                    agentCount: 0,
+                    totalTokensUsed: 0,
+                    peakMemoryMb: 0,
+                    success: false,
+                    completionRate: 0,
+                    errors: [err.message],
+                    startedAt: Date.now(),
+                    completedAt: Date.now(),
+                };
+                results.set(configType, failedResult);
+                scores.set(configType, { total: 0, successScore: 0, timeScore: 0, efficiencyScore: 0 });
+            }
+            // Cool-down between runs
+            if (this.config.cooldownMs > 0) {
+                await new Promise((r) => setTimeout(r, this.config.cooldownMs));
+            }
+        }
+        const winner = this.determineWinner(results, scores);
+        return {
+            taskId: task.id,
+            results,
+            winner,
+            scores,
+        };
+    }
+    /**
+     * Run a single configuration
+     */
+    async runSingle(task, configType) {
+        const runner = this.runners.get(configType);
+        if (!runner) {
+            throw new Error(`Configuration ${configType} not enabled`);
+        }
+        await runner.setup();
+        const result = await runner.run(task);
+        await runner.teardown();
+        return result;
+    }
+    /**
+     * Calculate score breakdown for a result
+     */
+    calculateScore(result) {
+        const maxTimeMs = 300000; // 5 minutes baseline
+        // Success component (0-50 points)
+        const successScore = result.success ? 50 : result.completionRate * 25;
+        // Time component (0-30 points) - faster is better
+        const timeScore = result.success
+            ? 30 * Math.max(0, 1 - result.totalTimeMs / maxTimeMs)
+            : 0;
+        // Efficiency component (0-20 points) - fewer agents is better for same result
+        const efficiencyScore = result.success
+            ? 20 / Math.max(1, result.agentCount)
+            : 0;
+        return {
+            total: successScore + timeScore + efficiencyScore,
+            successScore,
+            timeScore,
+            efficiencyScore,
+        };
+    }
+    /**
+     * Determine the winning configuration
+     */
+    determineWinner(results, scores) {
+        let best = 'single';
+        let bestScore = -1;
+        for (const [configType, score] of scores) {
+            if (score.total > bestScore) {
+                bestScore = score.total;
+                best = configType;
+            }
+        }
+        return best;
+    }
+    /**
+     * Print a single run result
+     */
+    printRunResult(result) {
+        console.log(`\nResult for ${result.configuration}:`);
+        console.log(`  Success: ${result.success ? '✓' : '✗'}`);
+        console.log(`  Time: ${(result.totalTimeMs / 1000).toFixed(1)}s`);
+        console.log(`  Agents: ${result.agentCount}`);
+        console.log(`  Messages: ${result.messageCount}`);
+        if (result.errors.length > 0) {
+            console.log(`  Errors: ${result.errors.join(', ')}`);
+        }
+    }
+    /**
+     * Print comparison table
+     */
+    printComparison(comparison) {
+        console.log('\n' + '='.repeat(60));
+        console.log('COMPARISON RESULTS');
+        console.log('='.repeat(60));
+        console.log(`Task: ${comparison.taskId}`);
+        console.log(`Winner: ${comparison.winner.toUpperCase()}`);
+        console.log('');
+        // Build table data
+        const configs = Array.from(comparison.results.keys());
+        const headers = ['Metric', ...configs.map((c) => c.charAt(0).toUpperCase() + c.slice(1))];
+        const rows = [
+            [
+                'Success',
+                ...configs.map((c) => comparison.results.get(c)?.success ? '✓' : '✗'),
+            ],
+            [
+                'Time (s)',
+                ...configs.map((c) => ((comparison.results.get(c)?.totalTimeMs || 0) / 1000).toFixed(1)),
+            ],
+            [
+                'Agents',
+                ...configs.map((c) => String(comparison.results.get(c)?.agentCount || 0)),
+            ],
+            [
+                'Messages',
+                ...configs.map((c) => String(comparison.results.get(c)?.messageCount || 0)),
+            ],
+            [
+                'Avg Latency (ms)',
+                ...configs.map((c) => (comparison.results.get(c)?.avgLatencyMs || 0).toFixed(0)),
+            ],
+            [
+                'Completion %',
+                ...configs.map((c) => ((comparison.results.get(c)?.completionRate || 0) * 100).toFixed(0) + '%'),
+            ],
+            [
+                'Score',
+                ...configs.map((c) => (comparison.scores.get(c)?.total || 0).toFixed(1)),
+            ],
+        ];
+        // Print table
+        const colWidths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => String(r[i]).length)));
+        const separator = colWidths.map((w) => '-'.repeat(w + 2)).join('+');
+        console.log(separator);
+        console.log('|' +
+            headers.map((h, i) => ` ${h.padEnd(colWidths[i])} `).join('|') +
+            '|');
+        console.log(separator);
+        for (const row of rows) {
+            console.log('|' +
+                row.map((cell, i) => ` ${String(cell).padEnd(colWidths[i])} `).join('|') +
+                '|');
+        }
+        console.log(separator);
+    }
+}
+/**
+ * Quick helper to run a comparison benchmark
+ */
+export async function runComparison(task, config) {
+    const benchmark = new ComparisonBenchmark(config);
+    return benchmark.runComparison(task);
+}
+//# sourceMappingURL=benchmark.js.map

package/packages/benchmark/dist/benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../src/benchmark.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAUH,OAAO,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,EAEL,iBAAiB,EACjB,cAAc,EACd,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAE5B;;GAEG;AACH,MAAM,OAAO,mBAAmB;IACtB,MAAM,CAAkB;IACxB,OAAO,CAA8C;IAE7D,YAAY,SAAmC,EAAE;QAC/C,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,wBAAwB,EAAE,GAAG,MAAM,EAAE,CAAC;QAEzD,mDAAmD;QACnD,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;QACzB,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACpD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,IAAuB;QAC1C,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,QAAQ;gBACX,OAAO,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5C,KAAK,UAAU;gBACb,OAAO,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzC,KAAK,OAAO;gBACV,OAAO,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC;gBACE,MAAM,IAAI,KAAK,CAAC,+BAA+B,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,IAAU;QAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAgC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqC,CAAC;QAE5D,KAAK,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAChD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,iBAAiB,UAAU,oBAAoB,CAAC,CAAC;gBAC7D,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YAClC,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;gBACrB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACtC,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAExB,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAChC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;gBAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;oBACvB,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,iBAAiB,UAAU,GAAG,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;gBAEtE,uBAAuB;gBACvB,MAAM,YAAY,GAAc;oBAC9B,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,aAAa,EAAE,UAAU;oBACzB,WAAW,EAAE,CAAC;oBACd,mBAAmB,EAAE,CAAC;oBACtB,YAAY,EAAE,CAAC;oBACf,YAAY,EAAE,CAAC;oBACf,YAAY,EAAE,CAAC;oBACf,YAAY,EAAE,CAAC;oBACf,kBAAkB,EAAE,CAAC;oBACrB,UAAU,EAAE,CAAC;oBACb,eAAe,EAAE,CAAC;oBAClB,YAAY,EAAE,CAAC;oBACf,OAAO,EAAE,KAAK;oBACd,cAAc,EAAE,CAAC;oBACjB,MAAM,EAAE,CAAE,GAAa,CAAC,OAAO,CAAC;oBAChC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;oBACrB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;iBACxB,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;gBACtC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1F,CAAC;YAED,yBAAyB;YACzB,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAErD,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,OAAO;YACP,MAAM;YACN,MAAM;SACP,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CACb,IAAU,EACV,UAA6B;QAE7B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,iBAAiB,UAAU,cAAc,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;QAExB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,MAAiB;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,qBAAqB;QAE/C,kCAAkC;QAClC,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,GAAG,EAAE,CAAC;QAEtE,kDAAkD;QAClD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO;YAC9B,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,WAAW,GAAG,SAAS,CAAC;YACtD,CAAC,CAAC,CAAC,CAAC;QAEN,8EAA8E;QAC9E,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO;YACpC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;YACrC,CAAC,CAAC,CAAC,CAAC;QAEN,OAAO;YACL,KAAK,EAAE,YAAY,GAAG,SAAS,GAAG,eAAe;YACjD,YAAY;YACZ,SAAS;YACT,eAAe;SAChB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,eAAe,CACrB,OAA0C,EAC1C,MAA8C;QAE9C,IAAI,IAAI,GAAsB,QAAQ,CAAC;QACvC,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;QAEnB,KAAK,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YACzC,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;gBAC5B,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;gBACxB,IAAI,GAAG,UAAU,CAAC;YACpB,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,MAAiB;QACtC,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QAClD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,UAA4B;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,SAAS,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEhB,mBAAmB;QACnB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE1F,MAAM,IAAI,GAAG;YACX;gBACE,SAAS;gBACT,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAC/C;aACF;YACD;gBACE,UAAU;gBACV,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAClE;aACF;YACD;gBACE,QAAQ;gBACR,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC,CACnD;aACF;YACD;gBACE,UAAU;gBACV,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,YAAY,IAAI,CAAC,CAAC,CACrD;aACF;YACD;gBACE,kBAAkB;gBAClB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAC1D;aACF;YACD;gBACE,cAAc;gBACd,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,cAAc,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAC1E;aACF;YACD;gBACE,OAAO;gBACP,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAClD;aACF;SACF,CAAC;QAEF,cAAc;QACd,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACrC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAC5D,CAAC;QAEF,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEpE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvB,OAAO,CAAC,GAAG,CACT,GAAG;YACD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAC9D,GAAG,CACN,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAEvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CACT,GAAG;gBACD,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;gBACxE,GAAG,CACN,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACzB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAAU,EACV,MAAiC;IAEjC,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAClD,OAAO,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;AACvC,CAAC"}

package/packages/benchmark/dist/cli.d.ts

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+/**
+ * Benchmark CLI
+ *
+ * Command-line interface for running agent swarm benchmarks.
+ */
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/packages/benchmark/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;GAIG"}

package/packages/benchmark/dist/cli.js

@@ -0,0 +1,185 @@
+#!/usr/bin/env node
+/**
+ * Benchmark CLI
+ *
+ * Command-line interface for running agent swarm benchmarks.
+ */
+import { Command } from 'commander';
+import { readFileSync } from 'node:fs';
+import { parse as parseYaml } from 'yaml';
+import { ComparisonBenchmark } from './benchmark.js';
+const program = new Command();
+program
+    .name('relay-benchmark')
+    .description('Benchmark agent swarms, sub-agents, and single agents')
+    .version('1.0.0');
+program
+    .command('run')
+    .description('Run a benchmark comparison')
+    .option('-d, --dataset <path>', 'Path to task dataset (YAML or JSON)')
+    .option('-t, --task <id>', 'Run only a specific task by ID')
+    .option('-c, --config <types>', 'Configurations to run (single,subagent,swarm,all)', 'all')
+    .option('--cli <name>', 'CLI to use for agents', 'claude')
+    .option('--cwd <path>', 'Working directory for tasks')
+    .option('-q, --quiet', 'Suppress output', false)
+    .option('--cooldown <ms>', 'Cooldown between runs in ms', '5000')
+    .option('--max-swarm <n>', 'Maximum swarm size', '10')
+    .option('-o, --output <path>', 'Output results to JSON file')
+    .action(async (options) => {
+    try {
+        await runBenchmark(options);
+    }
+    catch (err) {
+        console.error('Error:', err.message);
+        process.exit(1);
+    }
+});
+program
+    .command('list')
+    .description('List tasks in a dataset')
+    .argument('<dataset>', 'Path to task dataset')
+    .action((datasetPath) => {
+    const dataset = loadDataset(datasetPath);
+    console.log(`\nDataset: ${dataset.name || 'Unnamed'}`);
+    if (dataset.description) {
+        console.log(`Description: ${dataset.description}`);
+    }
+    console.log(`\nTasks (${dataset.tasks.length}):\n`);
+    for (const task of dataset.tasks) {
+        console.log(`  ${task.id}`);
+        console.log(`    Complexity: ${task.complexity}`);
+        console.log(`    Files: ${task.files.length}`);
+        console.log(`    ${task.description.substring(0, 60)}...`);
+        console.log('');
+    }
+});
+async function runBenchmark(options) {
+    // Parse configurations
+    const configurations = parseConfigurations(options.config);
+    // Build benchmark config
+    const benchmarkConfig = {
+        configurations,
+        cli: options.cli,
+        cwd: options.cwd,
+        quiet: options.quiet,
+        cooldownMs: parseInt(options.cooldown, 10),
+        maxSwarmSize: parseInt(options.maxSwarm, 10),
+    };
+    const benchmark = new ComparisonBenchmark(benchmarkConfig);
+    // Load tasks
+    let tasks;
+    if (options.dataset) {
+        const dataset = loadDataset(options.dataset);
+        tasks = dataset.tasks;
+        if (options.task) {
+            tasks = tasks.filter((t) => t.id === options.task);
+            if (tasks.length === 0) {
+                throw new Error(`Task not found: ${options.task}`);
+            }
+        }
+    }
+    else if (options.task) {
+        // Create a simple task from command line
+        tasks = [
+            {
+                id: options.task,
+                description: options.task,
+                files: [],
+                expectedOutcome: 'Task completed',
+                complexity: 'medium',
+            },
+        ];
+    }
+    else {
+        throw new Error('Either --dataset or --task is required');
+    }
+    // Run benchmarks
+    const results = [];
+    for (const task of tasks) {
+        if (!options.quiet) {
+            console.log(`\n${'='.repeat(60)}`);
+            console.log(`Running task: ${task.id}`);
+            console.log('='.repeat(60));
+        }
+        const comparison = await benchmark.runComparison(task);
+        if (!options.quiet) {
+            benchmark.printComparison(comparison);
+        }
+        results.push({
+            taskId: task.id,
+            winner: comparison.winner,
+            results: Object.fromEntries(comparison.results),
+            scores: Object.fromEntries(comparison.scores),
+        });
+    }
+    // Output results
+    if (options.output) {
+        const { writeFileSync } = await import('node:fs');
+        writeFileSync(options.output, JSON.stringify(results, null, 2));
+        console.log(`\nResults written to: ${options.output}`);
+    }
+    // Print summary
+    if (!options.quiet && results.length > 1) {
+        printSummary(results);
+    }
+}
+function parseConfigurations(config) {
+    if (config === 'all') {
+        return ['single', 'subagent', 'swarm'];
+    }
+    const configs = config.split(',').map((c) => c.trim());
+    const valid = ['single', 'subagent', 'swarm'];
+    for (const c of configs) {
+        if (!valid.includes(c)) {
+            throw new Error(`Invalid configuration: ${c}. Valid: ${valid.join(', ')}`);
+        }
+    }
+    return configs;
+}
+function loadDataset(path) {
+    const content = readFileSync(path, 'utf-8');
+    let data;
+    if (path.endsWith('.yaml') || path.endsWith('.yml')) {
+        data = parseYaml(content);
+    }
+    else {
+        data = JSON.parse(content);
+    }
+    // Validate and normalize tasks
+    if (!data.tasks || !Array.isArray(data.tasks)) {
+        throw new Error('Dataset must have a "tasks" array');
+    }
+    data.tasks = data.tasks.map((t, i) => ({
+        id: t.id || `task-${i}`,
+        description: t.description || '',
+        files: t.files || [],
+        expectedOutcome: t.expectedOutcome || t.success_criteria || 'Completed',
+        complexity: t.complexity || 'medium',
+        timeoutMs: t.timeoutMs || 300000,
+        tags: t.tags || [],
+    }));
+    return data;
+}
+function printSummary(results) {
+    console.log('\n' + '='.repeat(60));
+    console.log('BENCHMARK SUMMARY');
+    console.log('='.repeat(60));
+    const wins = {
+        single: 0,
+        subagent: 0,
+        swarm: 0,
+    };
+    for (const result of results) {
+        wins[result.winner]++;
+    }
+    console.log('\nWins by configuration:');
+    for (const [config, count] of Object.entries(wins)) {
+        const bar = '█'.repeat(count) + '░'.repeat(results.length - count);
+        console.log(`  ${config.padEnd(10)} ${bar} ${count}/${results.length}`);
+    }
+    const overallWinner = Object.entries(wins)
+        .sort((a, b) => b[1] - a[1])[0][0];
+    console.log(`\nOverall winner: ${overallWinner.toUpperCase()}`);
+}
+program.parse();
+//# sourceMappingURL=cli.js.map

package/packages/benchmark/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAQrD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,iBAAiB,CAAC;KACvB,WAAW,CAAC,uDAAuD,CAAC;KACpE,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,4BAA4B,CAAC;KACzC,MAAM,CAAC,sBAAsB,EAAE,qCAAqC,CAAC;KACrE,MAAM,CAAC,iBAAiB,EAAE,gCAAgC,CAAC;KAC3D,MAAM,CACL,sBAAsB,EACtB,mDAAmD,EACnD,KAAK,CACN;KACA,MAAM,CAAC,cAAc,EAAE,uBAAuB,EAAE,QAAQ,CAAC;KACzD,MAAM,CAAC,cAAc,EAAE,6BAA6B,CAAC;KACrD,MAAM,CAAC,aAAa,EAAE,iBAAiB,EAAE,KAAK,CAAC;KAC/C,MAAM,CAAC,iBAAiB,EAAE,6BAA6B,EAAE,MAAM,CAAC;KAChE,MAAM,CAAC,iBAAiB,EAAE,oBAAoB,EAAE,IAAI,CAAC;KACrD,MAAM,CAAC,qBAAqB,EAAE,6BAA6B,CAAC;KAC5D,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,yBAAyB,CAAC;KACtC,QAAQ,CAAC,WAAW,EAAE,sBAAsB,CAAC;KAC7C,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE;IACtB,MAAM,OAAO,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,IAAI,IAAI,SAAS,EAAE,CAAC,CAAC;IACvD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,gBAAgB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,KAAK,CAAC,MAAM,MAAM,CAAC,CAAC;IAEpD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,KAAK,UAAU,YAAY,CAAC,OAU3B;IACC,uBAAuB;IACvB,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAE3D,yBAAyB;IACzB,MAAM,eAAe,GAA6B;QAChD,cAAc;QACd,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC1C,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;KAC7C,CAAC;IAEF,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC,eAAe,CAAC,CAAC;IAE3D,aAAa;IACb,IAAI,KAAa,CAAC;IAClB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC7C,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAEtB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;IACH,CAAC;SAAM,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACxB,yCAAyC;QACzC,KAAK,GAAG;YACN;gBACE,EAAE,EAAE,OAAO,CAAC,IAAI;gBAChB,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,KAAK,EAAE,EAAE;gBACT,eAAe,EAAE,gBAAgB;gBACjC,UAAU,EAAE,QAAQ;aACrB;SACF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IAED,iBAAiB;IACjB,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAEvD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,SAAS,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACxC,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC;YAC/C,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;SAC9C,CAAC,CAAC;IACL,CAAC;IAED,iBAAiB;IACjB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAClD,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,yBAAyB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,gBAAgB;IAChB,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,YAAY,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAc;IACzC,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QACrB,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAwB,CAAC;IAC9E,MAAM,KAAK,GAAwB,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IAEnE,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE5C,IAAI,IAAiB,CAAC;IACtB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,IAAI,GAAG,SAAS,CAAC,OAAO,CAAgB,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAgB,CAAC;IAC5C,CAAC;IAED,+BAA+B;IAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACrC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,QAAQ,CAAC,EAAE;QACvB,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,EAAE;QAChC,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;QACpB,eAAe,EAAE,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,gBAAgB,IAAI,WAAW;QACvE,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,QAAQ;QACpC,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,MAAM;QAChC,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE;KACnB,CAAC,CAAC,CAAC;IAEJ,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CACnB,OAKE;IAEF,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,MAAM,IAAI,GAAsC;QAC9C,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,CAAC;QACX,KAAK,EAAE,CAAC;KACT,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;QACnE,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,IAAI,KAAK,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,aAAa,GAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAmC;SAC1E,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAErC,OAAO,CAAC,GAAG,CAAC,qBAAqB,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/packages/benchmark/dist/harbor.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Harbor Integration
+ *
+ * Entry points for Harbor benchmark framework integration.
+ * https://github.com/laude-institute/harbor
+ */
+import type { ConfigurationType, HarborTaskInput, HarborEvaluationOutput, BenchmarkConfig } from './types.js';
+/**
+ * Main Harbor evaluation entry point
+ *
+ * This function is called by Harbor to evaluate a task across all configurations.
+ *
+ * @example Harbor dataset format:
+ * ```yaml
+ * tasks:
+ *   - id: refactor-auth
+ *     description: "Refactor authentication to use JWT"
+ *     files:
+ *       - src/auth/session.ts
+ *       - src/auth/middleware.ts
+ *     success_criteria: "All tests pass, JWT tokens used"
+ *     complexity: medium
+ * ```
+ *
+ * @example Running with Harbor:
+ * ```bash
+ * harbor run \
+ *   --dataset tasks.yaml \
+ *   --agent @agent-relay/benchmark/harbor \
+ *   --parallel 10
+ * ```
+ */
+export declare function evaluate(input: HarborTaskInput): Promise<HarborEvaluationOutput>;
+/**
+ * Run a single configuration (for targeted Harbor evaluations)
+ *
+ * @example Running single config with Harbor:
+ * ```bash
+ * harbor run \
+ *   --dataset tasks.yaml \
+ *   --agent "@agent-relay/benchmark/harbor:evaluateSingle" \
+ *   --env-var CONFIG=swarm
+ * ```
+ */
+export declare function evaluateSingle(input: HarborTaskInput & {
+    config?: ConfigurationType;
+}): Promise<Record<string, unknown>>;
+/**
+ * Evaluate with custom configuration
+ */
+export declare function evaluateCustom(input: HarborTaskInput, config: Partial<BenchmarkConfig>): Promise<HarborEvaluationOutput>;
+export default evaluate;
+//# sourceMappingURL=harbor.d.ts.map

package/packages/benchmark/dist/harbor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harbor.d.ts","sourceRoot":"","sources":["../src/harbor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAGV,iBAAiB,EACjB,eAAe,EACf,sBAAsB,EACtB,eAAe,EAChB,MAAM,YAAY,CAAC;AAoBpB;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,sBAAsB,CAAC,CAiCjC;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,eAAe,GAAG;IAAE,MAAM,CAAC,EAAE,iBAAiB,CAAA;CAAE,GACtD,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAmBlC;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,eAAe,EACtB,MAAM,EAAE,OAAO,CAAC,eAAe,CAAC,GAC/B,OAAO,CAAC,sBAAsB,CAAC,CA8BjC;AAGD,eAAe,QAAQ,CAAC"}

package/packages/benchmark/dist/harbor.js

@@ -0,0 +1,127 @@
+/**
+ * Harbor Integration
+ *
+ * Entry points for Harbor benchmark framework integration.
+ * https://github.com/laude-institute/harbor
+ */
+import { ComparisonBenchmark } from './benchmark.js';
+const BENCHMARK_VERSION = '1.0.0';
+/**
+ * Convert Harbor task input to internal Task format
+ */
+function convertHarborTask(input) {
+    return {
+        id: input.id,
+        description: input.description,
+        files: input.files || [],
+        expectedOutcome: input.success_criteria || 'Task completed successfully',
+        complexity: input.complexity || 'medium',
+        timeoutMs: 300000, // 5 minute default
+        tags: [],
+    };
+}
+/**
+ * Main Harbor evaluation entry point
+ *
+ * This function is called by Harbor to evaluate a task across all configurations.
+ *
+ * @example Harbor dataset format:
+ * ```yaml
+ * tasks:
+ *   - id: refactor-auth
+ *     description: "Refactor authentication to use JWT"
+ *     files:
+ *       - src/auth/session.ts
+ *       - src/auth/middleware.ts
+ *     success_criteria: "All tests pass, JWT tokens used"
+ *     complexity: medium
+ * ```
+ *
+ * @example Running with Harbor:
+ * ```bash
+ * harbor run \
+ *   --dataset tasks.yaml \
+ *   --agent @agent-relay/benchmark/harbor \
+ *   --parallel 10
+ * ```
+ */
+export async function evaluate(input) {
+    const startedAt = Date.now();
+    const task = convertHarborTask(input);
+    const benchmark = new ComparisonBenchmark({
+        configurations: ['single', 'subagent', 'swarm'],
+        cli: 'claude',
+        quiet: true, // Suppress output in Harbor runs
+        cooldownMs: 2000,
+    });
+    const comparison = await benchmark.runComparison(task);
+    const completedAt = Date.now();
+    return {
+        task_id: task.id,
+        configurations: Object.fromEntries(comparison.results),
+        winner: comparison.winner,
+        scores: Object.fromEntries(comparison.scores),
+        metadata: {
+            benchmark_version: BENCHMARK_VERSION,
+            started_at: startedAt,
+            completed_at: completedAt,
+            total_duration_ms: completedAt - startedAt,
+        },
+    };
+}
+/**
+ * Run a single configuration (for targeted Harbor evaluations)
+ *
+ * @example Running single config with Harbor:
+ * ```bash
+ * harbor run \
+ *   --dataset tasks.yaml \
+ *   --agent "@agent-relay/benchmark/harbor:evaluateSingle" \
+ *   --env-var CONFIG=swarm
+ * ```
+ */
+export async function evaluateSingle(input) {
+    const config = input.config || 'single';
+    const task = convertHarborTask(input);
+    const benchmark = new ComparisonBenchmark({
+        configurations: [config],
+        cli: 'claude',
+        quiet: true,
+        cooldownMs: 0,
+    });
+    const result = await benchmark.runSingle(task, config);
+    return {
+        task_id: task.id,
+        configuration: config,
+        result,
+        success: result.success,
+    };
+}
+/**
+ * Evaluate with custom configuration
+ */
+export async function evaluateCustom(input, config) {
+    const startedAt = Date.now();
+    const task = convertHarborTask(input);
+    const benchmark = new ComparisonBenchmark({
+        ...config,
+        quiet: true,
+    });
+    const comparison = await benchmark.runComparison(task);
+    const completedAt = Date.now();
+    return {
+        task_id: task.id,
+        configurations: Object.fromEntries(comparison.results),
+        winner: comparison.winner,
+        scores: Object.fromEntries(comparison.scores),
+        metadata: {
+            benchmark_version: BENCHMARK_VERSION,
+            started_at: startedAt,
+            completed_at: completedAt,
+            total_duration_ms: completedAt - startedAt,
+        },
+    };
+}
+// Default export for Harbor
+export default evaluate;
+//# sourceMappingURL=harbor.js.map

package/packages/benchmark/dist/harbor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harbor.js","sourceRoot":"","sources":["../src/harbor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAUH,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAErD,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAElC;;GAEG;AACH,SAAS,iBAAiB,CAAC,KAAsB;IAC/C,OAAO;QACL,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE;QACxB,eAAe,EAAE,KAAK,CAAC,gBAAgB,IAAI,6BAA6B;QACxE,UAAU,EAAG,KAAK,CAAC,UAA6B,IAAI,QAAQ;QAC5D,SAAS,EAAE,MAAM,EAAE,mBAAmB;QACtC,IAAI,EAAE,EAAE;KACT,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAEtC,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC;QACxC,cAAc,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC;QAC/C,GAAG,EAAE,QAAQ;QACb,KAAK,EAAE,IAAI,EAAE,iCAAiC;QAC9C,UAAU,EAAE,IAAI;KACjB,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAEvD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE/B,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,EAAE;QAChB,cAAc,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAGpD;QACD,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,CAG3C;QACD,QAAQ,EAAE;YACR,iBAAiB,EAAE,iBAAiB;YACpC,UAAU,EAAE,SAAS;YACrB,YAAY,EAAE,WAAW;YACzB,iBAAiB,EAAE,WAAW,GAAG,SAAS;SAC3C;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAuD;IAEvD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC;IACxC,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAEtC,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC;QACxC,cAAc,EAAE,CAAC,MAAM,CAAC;QACxB,GAAG,EAAE,QAAQ;QACb,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,CAAC;KACd,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAEvD,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,EAAE;QAChB,aAAa,EAAE,MAAM;QACrB,MAAM;QACN,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAsB,EACtB,MAAgC;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAEtC,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC;QACxC,GAAG,MAAM;QACT,KAAK,EAAE,IAAI;KACZ,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IACvD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE/B,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,EAAE;QAChB,cAAc,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAGpD;QACD,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,CAG3C;QACD,QAAQ,EAAE;YACR,iBAAiB,EAAE,iBAAiB;YACpC,UAAU,EAAE,SAAS;YACrB,YAAY,EAAE,WAAW;YACzB,iBAAiB,EAAE,WAAW,GAAG,SAAS;SAC3C;KACF,CAAC;AACJ,CAAC;AAED,4BAA4B;AAC5B,eAAe,QAAQ,CAAC"}