agent-relay 2.3.4 → 2.3.5

package/packages/benchmark/dist/benchmark.js

@@ -1,224 +0,0 @@
-/**
- * Comparison Benchmark
- *
- * Main orchestrator for running comparison benchmarks across configurations.
- */
-import { DEFAULT_BENCHMARK_CONFIG } from './types.js';
-import { SingleAgentRunner, SubAgentRunner, SwarmRunner, } from './runners/index.js';
-/**
- * Main benchmark orchestrator
- */
-export class ComparisonBenchmark {
-    config;
-    runners;
-    constructor(config = {}) {
-        this.config = { ...DEFAULT_BENCHMARK_CONFIG, ...config };
-        // Initialize runners for configured configurations
-        this.runners = new Map();
-        for (const configType of this.config.configurations) {
-            this.runners.set(configType, this.createRunner(configType));
-        }
-    }
-    /**
-     * Create a runner for a configuration type
-     */
-    createRunner(type) {
-        switch (type) {
-            case 'single':
-                return new SingleAgentRunner(this.config);
-            case 'subagent':
-                return new SubAgentRunner(this.config);
-            case 'swarm':
-                return new SwarmRunner(this.config);
-            default:
-                throw new Error(`Unknown configuration type: ${type}`);
-        }
-    }
-    /**
-     * Run a comparison across all configured configurations
-     */
-    async runComparison(task) {
-        const results = new Map();
-        const scores = new Map();
-        for (const [configType, runner] of this.runners) {
-            if (!this.config.quiet) {
-                console.log(`\n=== Running ${configType} configuration ===`);
-                console.log(`Task: ${task.id}`);
-            }
-            try {
-                await runner.setup();
-                const result = await runner.run(task);
-                await runner.teardown();
-                results.set(configType, result);
-                scores.set(configType, this.calculateScore(result));
-                if (!this.config.quiet) {
-                    this.printRunResult(result);
-                }
-            }
-            catch (err) {
-                console.error(`Error running ${configType}:`, err.message);
-                // Create failed result
-                const failedResult = {
-                    taskId: task.id,
-                    configuration: configType,
-                    totalTimeMs: 0,
-                    timeToFirstActionMs: 0,
-                    messageCount: 0,
-                    avgLatencyMs: 0,
-                    latencyP50Ms: 0,
-                    latencyP99Ms: 0,
-                    coordinationRounds: 0,
-                    agentCount: 0,
-                    totalTokensUsed: 0,
-                    peakMemoryMb: 0,
-                    success: false,
-                    completionRate: 0,
-                    errors: [err.message],
-                    startedAt: Date.now(),
-                    completedAt: Date.now(),
-                };
-                results.set(configType, failedResult);
-                scores.set(configType, { total: 0, successScore: 0, timeScore: 0, efficiencyScore: 0 });
-            }
-            // Cool-down between runs
-            if (this.config.cooldownMs > 0) {
-                await new Promise((r) => setTimeout(r, this.config.cooldownMs));
-            }
-        }
-        const winner = this.determineWinner(results, scores);
-        return {
-            taskId: task.id,
-            results,
-            winner,
-            scores,
-        };
-    }
-    /**
-     * Run a single configuration
-     */
-    async runSingle(task, configType) {
-        const runner = this.runners.get(configType);
-        if (!runner) {
-            throw new Error(`Configuration ${configType} not enabled`);
-        }
-        await runner.setup();
-        const result = await runner.run(task);
-        await runner.teardown();
-        return result;
-    }
-    /**
-     * Calculate score breakdown for a result
-     */
-    calculateScore(result) {
-        const maxTimeMs = 300000; // 5 minutes baseline
-        // Success component (0-50 points)
-        const successScore = result.success ? 50 : result.completionRate * 25;
-        // Time component (0-30 points) - faster is better
-        const timeScore = result.success
-            ? 30 * Math.max(0, 1 - result.totalTimeMs / maxTimeMs)
-            : 0;
-        // Efficiency component (0-20 points) - fewer agents is better for same result
-        const efficiencyScore = result.success
-            ? 20 / Math.max(1, result.agentCount)
-            : 0;
-        return {
-            total: successScore + timeScore + efficiencyScore,
-            successScore,
-            timeScore,
-            efficiencyScore,
-        };
-    }
-    /**
-     * Determine the winning configuration
-     */
-    determineWinner(results, scores) {
-        let best = 'single';
-        let bestScore = -1;
-        for (const [configType, score] of scores) {
-            if (score.total > bestScore) {
-                bestScore = score.total;
-                best = configType;
-            }
-        }
-        return best;
-    }
-    /**
-     * Print a single run result
-     */
-    printRunResult(result) {
-        console.log(`\nResult for ${result.configuration}:`);
-        console.log(`  Success: ${result.success ? '✓' : '✗'}`);
-        console.log(`  Time: ${(result.totalTimeMs / 1000).toFixed(1)}s`);
-        console.log(`  Agents: ${result.agentCount}`);
-        console.log(`  Messages: ${result.messageCount}`);
-        if (result.errors.length > 0) {
-            console.log(`  Errors: ${result.errors.join(', ')}`);
-        }
-    }
-    /**
-     * Print comparison table
-     */
-    printComparison(comparison) {
-        console.log('\n' + '='.repeat(60));
-        console.log('COMPARISON RESULTS');
-        console.log('='.repeat(60));
-        console.log(`Task: ${comparison.taskId}`);
-        console.log(`Winner: ${comparison.winner.toUpperCase()}`);
-        console.log('');
-        // Build table data
-        const configs = Array.from(comparison.results.keys());
-        const headers = ['Metric', ...configs.map((c) => c.charAt(0).toUpperCase() + c.slice(1))];
-        const rows = [
-            [
-                'Success',
-                ...configs.map((c) => comparison.results.get(c)?.success ? '✓' : '✗'),
-            ],
-            [
-                'Time (s)',
-                ...configs.map((c) => ((comparison.results.get(c)?.totalTimeMs || 0) / 1000).toFixed(1)),
-            ],
-            [
-                'Agents',
-                ...configs.map((c) => String(comparison.results.get(c)?.agentCount || 0)),
-            ],
-            [
-                'Messages',
-                ...configs.map((c) => String(comparison.results.get(c)?.messageCount || 0)),
-            ],
-            [
-                'Avg Latency (ms)',
-                ...configs.map((c) => (comparison.results.get(c)?.avgLatencyMs || 0).toFixed(0)),
-            ],
-            [
-                'Completion %',
-                ...configs.map((c) => ((comparison.results.get(c)?.completionRate || 0) * 100).toFixed(0) + '%'),
-            ],
-            [
-                'Score',
-                ...configs.map((c) => (comparison.scores.get(c)?.total || 0).toFixed(1)),
-            ],
-        ];
-        // Print table
-        const colWidths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => String(r[i]).length)));
-        const separator = colWidths.map((w) => '-'.repeat(w + 2)).join('+');
-        console.log(separator);
-        console.log('|' +
-            headers.map((h, i) => ` ${h.padEnd(colWidths[i])} `).join('|') +
-            '|');
-        console.log(separator);
-        for (const row of rows) {
-            console.log('|' +
-                row.map((cell, i) => ` ${String(cell).padEnd(colWidths[i])} `).join('|') +
-                '|');
-        }
-        console.log(separator);
-    }
-}
-/**
- * Quick helper to run a comparison benchmark
- */
-export async function runComparison(task, config) {
-    const benchmark = new ComparisonBenchmark(config);
-    return benchmark.runComparison(task);
-}
-//# sourceMappingURL=benchmark.js.map

package/packages/benchmark/dist/benchmark.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../src/benchmark.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAUH,OAAO,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,EAEL,iBAAiB,EACjB,cAAc,EACd,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAE5B;;GAEG;AACH,MAAM,OAAO,mBAAmB;IACtB,MAAM,CAAkB;IACxB,OAAO,CAA8C;IAE7D,YAAY,SAAmC,EAAE;QAC/C,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,wBAAwB,EAAE,GAAG,MAAM,EAAE,CAAC;QAEzD,mDAAmD;QACnD,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;QACzB,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACpD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,IAAuB;QAC1C,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,QAAQ;gBACX,OAAO,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5C,KAAK,UAAU;gBACb,OAAO,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzC,KAAK,OAAO;gBACV,OAAO,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC;gBACE,MAAM,IAAI,KAAK,CAAC,+BAA+B,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,IAAU;QAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAgC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqC,CAAC;QAE5D,KAAK,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAChD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,iBAAiB,UAAU,oBAAoB,CAAC,CAAC;gBAC7D,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YAClC,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;gBACrB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACtC,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAExB,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAChC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;gBAEpD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;oBACvB,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,iBAAiB,UAAU,GAAG,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;gBAEtE,uBAAuB;gBACvB,MAAM,YAAY,GAAc;oBAC9B,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,aAAa,EAAE,UAAU;oBACzB,WAAW,EAAE,CAAC;oBACd,mBAAmB,EAAE,CAAC;oBACtB,YAAY,EAAE,CAAC;oBACf,YAAY,EAAE,CAAC;oBACf,YAAY,EAAE,CAAC;oBACf,YAAY,EAAE,CAAC;oBACf,kBAAkB,EAAE,CAAC;oBACrB,UAAU,EAAE,CAAC;oBACb,eAAe,EAAE,CAAC;oBAClB,YAAY,EAAE,CAAC;oBACf,OAAO,EAAE,KAAK;oBACd,cAAc,EAAE,CAAC;oBACjB,MAAM,EAAE,CAAE,GAAa,CAAC,OAAO,CAAC;oBAChC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;oBACrB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;iBACxB,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;gBACtC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;YAC1F,CAAC;YAED,yBAAyB;YACzB,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAErD,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,OAAO;YACP,MAAM;YACN,MAAM;SACP,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CACb,IAAU,EACV,UAA6B;QAE7B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,iBAAiB,UAAU,cAAc,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;QAExB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,MAAiB;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,qBAAqB;QAE/C,kCAAkC;QAClC,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,GAAG,EAAE,CAAC;QAEtE,kDAAkD;QAClD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO;YAC9B,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,WAAW,GAAG,SAAS,CAAC;YACtD,CAAC,CAAC,CAAC,CAAC;QAEN,8EAA8E;QAC9E,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO;YACpC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;YACrC,CAAC,CAAC,CAAC,CAAC;QAEN,OAAO;YACL,KAAK,EAAE,YAAY,GAAG,SAAS,GAAG,eAAe;YACjD,YAAY;YACZ,SAAS;YACT,eAAe;SAChB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,eAAe,CACrB,OAA0C,EAC1C,MAA8C;QAE9C,IAAI,IAAI,GAAsB,QAAQ,CAAC;QACvC,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;QAEnB,KAAK,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YACzC,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;gBAC5B,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC;gBACxB,IAAI,GAAG,UAAU,CAAC;YACpB,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,MAAiB;QACtC,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QAClD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,UAA4B;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,SAAS,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEhB,mBAAmB;QACnB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE1F,MAAM,IAAI,GAAG;YACX;gBACE,SAAS;gBACT,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAC/C;aACF;YACD;gBACE,UAAU;gBACV,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAClE;aACF;YACD;gBACE,QAAQ;gBACR,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC,CACnD;aACF;YACD;gBACE,UAAU;gBACV,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,YAAY,IAAI,CAAC,CAAC,CACrD;aACF;YACD;gBACE,kBAAkB;gBAClB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAC1D;aACF;YACD;gBACE,cAAc;gBACd,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,cAAc,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAC1E;aACF;YACD;gBACE,OAAO;gBACP,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAClD;aACF;SACF,CAAC;QAEF,cAAc;QACd,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACrC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAC5D,CAAC;QAEF,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEpE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvB,OAAO,CAAC,GAAG,CACT,GAAG;YACD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAC9D,GAAG,CACN,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAEvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CACT,GAAG;gBACD,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;gBACxE,GAAG,CACN,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACzB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAAU,EACV,MAAiC;IAEjC,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAClD,OAAO,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;AACvC,CAAC"}

package/packages/benchmark/dist/cli.d.ts

@@ -1,8 +0,0 @@
-#!/usr/bin/env node
-/**
- * Benchmark CLI
- *
- * Command-line interface for running agent swarm benchmarks.
- */
-export {};
-//# sourceMappingURL=cli.d.ts.map

package/packages/benchmark/dist/cli.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;GAIG"}

package/packages/benchmark/dist/cli.js

@@ -1,185 +0,0 @@
-#!/usr/bin/env node
-/**
- * Benchmark CLI
- *
- * Command-line interface for running agent swarm benchmarks.
- */
-import { Command } from 'commander';
-import { readFileSync } from 'node:fs';
-import { parse as parseYaml } from 'yaml';
-import { ComparisonBenchmark } from './benchmark.js';
-const program = new Command();
-program
-    .name('relay-benchmark')
-    .description('Benchmark agent swarms, sub-agents, and single agents')
-    .version('1.0.0');
-program
-    .command('run')
-    .description('Run a benchmark comparison')
-    .option('-d, --dataset <path>', 'Path to task dataset (YAML or JSON)')
-    .option('-t, --task <id>', 'Run only a specific task by ID')
-    .option('-c, --config <types>', 'Configurations to run (single,subagent,swarm,all)', 'all')
-    .option('--cli <name>', 'CLI to use for agents', 'claude')
-    .option('--cwd <path>', 'Working directory for tasks')
-    .option('-q, --quiet', 'Suppress output', false)
-    .option('--cooldown <ms>', 'Cooldown between runs in ms', '5000')
-    .option('--max-swarm <n>', 'Maximum swarm size', '10')
-    .option('-o, --output <path>', 'Output results to JSON file')
-    .action(async (options) => {
-    try {
-        await runBenchmark(options);
-    }
-    catch (err) {
-        console.error('Error:', err.message);
-        process.exit(1);
-    }
-});
-program
-    .command('list')
-    .description('List tasks in a dataset')
-    .argument('<dataset>', 'Path to task dataset')
-    .action((datasetPath) => {
-    const dataset = loadDataset(datasetPath);
-    console.log(`\nDataset: ${dataset.name || 'Unnamed'}`);
-    if (dataset.description) {
-        console.log(`Description: ${dataset.description}`);
-    }
-    console.log(`\nTasks (${dataset.tasks.length}):\n`);
-    for (const task of dataset.tasks) {
-        console.log(`  ${task.id}`);
-        console.log(`    Complexity: ${task.complexity}`);
-        console.log(`    Files: ${task.files.length}`);
-        console.log(`    ${task.description.substring(0, 60)}...`);
-        console.log('');
-    }
-});
-async function runBenchmark(options) {
-    // Parse configurations
-    const configurations = parseConfigurations(options.config);
-    // Build benchmark config
-    const benchmarkConfig = {
-        configurations,
-        cli: options.cli,
-        cwd: options.cwd,
-        quiet: options.quiet,
-        cooldownMs: parseInt(options.cooldown, 10),
-        maxSwarmSize: parseInt(options.maxSwarm, 10),
-    };
-    const benchmark = new ComparisonBenchmark(benchmarkConfig);
-    // Load tasks
-    let tasks;
-    if (options.dataset) {
-        const dataset = loadDataset(options.dataset);
-        tasks = dataset.tasks;
-        if (options.task) {
-            tasks = tasks.filter((t) => t.id === options.task);
-            if (tasks.length === 0) {
-                throw new Error(`Task not found: ${options.task}`);
-            }
-        }
-    }
-    else if (options.task) {
-        // Create a simple task from command line
-        tasks = [
-            {
-                id: options.task,
-                description: options.task,
-                files: [],
-                expectedOutcome: 'Task completed',
-                complexity: 'medium',
-            },
-        ];
-    }
-    else {
-        throw new Error('Either --dataset or --task is required');
-    }
-    // Run benchmarks
-    const results = [];
-    for (const task of tasks) {
-        if (!options.quiet) {
-            console.log(`\n${'='.repeat(60)}`);
-            console.log(`Running task: ${task.id}`);
-            console.log('='.repeat(60));
-        }
-        const comparison = await benchmark.runComparison(task);
-        if (!options.quiet) {
-            benchmark.printComparison(comparison);
-        }
-        results.push({
-            taskId: task.id,
-            winner: comparison.winner,
-            results: Object.fromEntries(comparison.results),
-            scores: Object.fromEntries(comparison.scores),
-        });
-    }
-    // Output results
-    if (options.output) {
-        const { writeFileSync } = await import('node:fs');
-        writeFileSync(options.output, JSON.stringify(results, null, 2));
-        console.log(`\nResults written to: ${options.output}`);
-    }
-    // Print summary
-    if (!options.quiet && results.length > 1) {
-        printSummary(results);
-    }
-}
-function parseConfigurations(config) {
-    if (config === 'all') {
-        return ['single', 'subagent', 'swarm'];
-    }
-    const configs = config.split(',').map((c) => c.trim());
-    const valid = ['single', 'subagent', 'swarm'];
-    for (const c of configs) {
-        if (!valid.includes(c)) {
-            throw new Error(`Invalid configuration: ${c}. Valid: ${valid.join(', ')}`);
-        }
-    }
-    return configs;
-}
-function loadDataset(path) {
-    const content = readFileSync(path, 'utf-8');
-    let data;
-    if (path.endsWith('.yaml') || path.endsWith('.yml')) {
-        data = parseYaml(content);
-    }
-    else {
-        data = JSON.parse(content);
-    }
-    // Validate and normalize tasks
-    if (!data.tasks || !Array.isArray(data.tasks)) {
-        throw new Error('Dataset must have a "tasks" array');
-    }
-    data.tasks = data.tasks.map((t, i) => ({
-        id: t.id || `task-${i}`,
-        description: t.description || '',
-        files: t.files || [],
-        expectedOutcome: t.expectedOutcome || t.success_criteria || 'Completed',
-        complexity: t.complexity || 'medium',
-        timeoutMs: t.timeoutMs || 300000,
-        tags: t.tags || [],
-    }));
-    return data;
-}
-function printSummary(results) {
-    console.log('\n' + '='.repeat(60));
-    console.log('BENCHMARK SUMMARY');
-    console.log('='.repeat(60));
-    const wins = {
-        single: 0,
-        subagent: 0,
-        swarm: 0,
-    };
-    for (const result of results) {
-        wins[result.winner]++;
-    }
-    console.log('\nWins by configuration:');
-    for (const [config, count] of Object.entries(wins)) {
-        const bar = '█'.repeat(count) + '░'.repeat(results.length - count);
-        console.log(`  ${config.padEnd(10)} ${bar} ${count}/${results.length}`);
-    }
-    const overallWinner = Object.entries(wins)
-        .sort((a, b) => b[1] - a[1])[0][0];
-    console.log(`\nOverall winner: ${overallWinner.toUpperCase()}`);
-}
-program.parse();
-//# sourceMappingURL=cli.js.map

package/packages/benchmark/dist/cli.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAQrD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,iBAAiB,CAAC;KACvB,WAAW,CAAC,uDAAuD,CAAC;KACpE,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,4BAA4B,CAAC;KACzC,MAAM,CAAC,sBAAsB,EAAE,qCAAqC,CAAC;KACrE,MAAM,CAAC,iBAAiB,EAAE,gCAAgC,CAAC;KAC3D,MAAM,CACL,sBAAsB,EACtB,mDAAmD,EACnD,KAAK,CACN;KACA,MAAM,CAAC,cAAc,EAAE,uBAAuB,EAAE,QAAQ,CAAC;KACzD,MAAM,CAAC,cAAc,EAAE,6BAA6B,CAAC;KACrD,MAAM,CAAC,aAAa,EAAE,iBAAiB,EAAE,KAAK,CAAC;KAC/C,MAAM,CAAC,iBAAiB,EAAE,6BAA6B,EAAE,MAAM,CAAC;KAChE,MAAM,CAAC,iBAAiB,EAAE,oBAAoB,EAAE,IAAI,CAAC;KACrD,MAAM,CAAC,qBAAqB,EAAE,6BAA6B,CAAC;KAC5D,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,IAAI,CAAC;QACH,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,yBAAyB,CAAC;KACtC,QAAQ,CAAC,WAAW,EAAE,sBAAsB,CAAC;KAC7C,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE;IACtB,MAAM,OAAO,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,IAAI,IAAI,SAAS,EAAE,CAAC,CAAC;IACvD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,gBAAgB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,KAAK,CAAC,MAAM,MAAM,CAAC,CAAC;IAEpD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,KAAK,UAAU,YAAY,CAAC,OAU3B;IACC,uBAAuB;IACvB,MAAM,cAAc,GAAG,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAE3D,yBAAyB;IACzB,MAAM,eAAe,GAA6B;QAChD,cAAc;QACd,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC1C,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;KAC7C,CAAC;IAEF,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC,eAAe,CAAC,CAAC;IAE3D,aAAa;IACb,IAAI,KAAa,CAAC;IAClB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC7C,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAEtB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;IACH,CAAC;SAAM,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACxB,yCAAyC;QACzC,KAAK,GAAG;YACN;gBACE,EAAE,EAAE,OAAO,CAAC,IAAI;gBAChB,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,KAAK,EAAE,EAAE;gBACT,eAAe,EAAE,gBAAgB;gBACjC,UAAU,EAAE,QAAQ;aACrB;SACF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IAED,iBAAiB;IACjB,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAEvD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,SAAS,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QACxC,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC;YAC/C,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;SAC9C,CAAC,CAAC;IACL,CAAC;IAED,iBAAiB;IACjB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAClD,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,yBAAyB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,gBAAgB;IAChB,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,YAAY,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAc;IACzC,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QACrB,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAwB,CAAC;IAC9E,MAAM,KAAK,GAAwB,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IAEnE,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE5C,IAAI,IAAiB,CAAC;IACtB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,IAAI,GAAG,SAAS,CAAC,OAAO,CAAgB,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAgB,CAAC;IAC5C,CAAC;IAED,+BAA+B;IAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACrC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,QAAQ,CAAC,EAAE;QACvB,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,EAAE;QAChC,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;QACpB,eAAe,EAAE,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,gBAAgB,IAAI,WAAW;QACvE,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,QAAQ;QACpC,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,MAAM;QAChC,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE;KACnB,CAAC,CAAC,CAAC;IAEJ,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CACnB,OAKE;IAEF,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,MAAM,IAAI,GAAsC;QAC9C,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,CAAC;QACX,KAAK,EAAE,CAAC;KACT,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;QACnE,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,IAAI,KAAK,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,aAAa,GAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAmC;SAC1E,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAErC,OAAO,CAAC,GAAG,CAAC,qBAAqB,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/packages/benchmark/dist/harbor.d.ts

@@ -1,53 +0,0 @@
-/**
- * Harbor Integration
- *
- * Entry points for Harbor benchmark framework integration.
- * https://github.com/laude-institute/harbor
- */
-import type { ConfigurationType, HarborTaskInput, HarborEvaluationOutput, BenchmarkConfig } from './types.js';
-/**
- * Main Harbor evaluation entry point
- *
- * This function is called by Harbor to evaluate a task across all configurations.
- *
- * @example Harbor dataset format:
- * ```yaml
- * tasks:
- *   - id: refactor-auth
- *     description: "Refactor authentication to use JWT"
- *     files:
- *       - src/auth/session.ts
- *       - src/auth/middleware.ts
- *     success_criteria: "All tests pass, JWT tokens used"
- *     complexity: medium
- * ```
- *
- * @example Running with Harbor:
- * ```bash
- * harbor run \
- *   --dataset tasks.yaml \
- *   --agent @agent-relay/benchmark/harbor \
- *   --parallel 10
- * ```
- */
-export declare function evaluate(input: HarborTaskInput): Promise<HarborEvaluationOutput>;
-/**
- * Run a single configuration (for targeted Harbor evaluations)
- *
- * @example Running single config with Harbor:
- * ```bash
- * harbor run \
- *   --dataset tasks.yaml \
- *   --agent "@agent-relay/benchmark/harbor:evaluateSingle" \
- *   --env-var CONFIG=swarm
- * ```
- */
-export declare function evaluateSingle(input: HarborTaskInput & {
-    config?: ConfigurationType;
-}): Promise<Record<string, unknown>>;
-/**
- * Evaluate with custom configuration
- */
-export declare function evaluateCustom(input: HarborTaskInput, config: Partial<BenchmarkConfig>): Promise<HarborEvaluationOutput>;
-export default evaluate;
-//# sourceMappingURL=harbor.d.ts.map

package/packages/benchmark/dist/harbor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"harbor.d.ts","sourceRoot":"","sources":["../src/harbor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAGV,iBAAiB,EACjB,eAAe,EACf,sBAAsB,EACtB,eAAe,EAChB,MAAM,YAAY,CAAC;AAoBpB;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,sBAAsB,CAAC,CAiCjC;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,eAAe,GAAG;IAAE,MAAM,CAAC,EAAE,iBAAiB,CAAA;CAAE,GACtD,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAmBlC;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,eAAe,EACtB,MAAM,EAAE,OAAO,CAAC,eAAe,CAAC,GAC/B,OAAO,CAAC,sBAAsB,CAAC,CA8BjC;AAGD,eAAe,QAAQ,CAAC"}

package/packages/benchmark/dist/harbor.js

@@ -1,127 +0,0 @@
-/**
- * Harbor Integration
- *
- * Entry points for Harbor benchmark framework integration.
- * https://github.com/laude-institute/harbor
- */
-import { ComparisonBenchmark } from './benchmark.js';
-const BENCHMARK_VERSION = '1.0.0';
-/**
- * Convert Harbor task input to internal Task format
- */
-function convertHarborTask(input) {
-    return {
-        id: input.id,
-        description: input.description,
-        files: input.files || [],
-        expectedOutcome: input.success_criteria || 'Task completed successfully',
-        complexity: input.complexity || 'medium',
-        timeoutMs: 300000, // 5 minute default
-        tags: [],
-    };
-}
-/**
- * Main Harbor evaluation entry point
- *
- * This function is called by Harbor to evaluate a task across all configurations.
- *
- * @example Harbor dataset format:
- * ```yaml
- * tasks:
- *   - id: refactor-auth
- *     description: "Refactor authentication to use JWT"
- *     files:
- *       - src/auth/session.ts
- *       - src/auth/middleware.ts
- *     success_criteria: "All tests pass, JWT tokens used"
- *     complexity: medium
- * ```
- *
- * @example Running with Harbor:
- * ```bash
- * harbor run \
- *   --dataset tasks.yaml \
- *   --agent @agent-relay/benchmark/harbor \
- *   --parallel 10
- * ```
- */
-export async function evaluate(input) {
-    const startedAt = Date.now();
-    const task = convertHarborTask(input);
-    const benchmark = new ComparisonBenchmark({
-        configurations: ['single', 'subagent', 'swarm'],
-        cli: 'claude',
-        quiet: true, // Suppress output in Harbor runs
-        cooldownMs: 2000,
-    });
-    const comparison = await benchmark.runComparison(task);
-    const completedAt = Date.now();
-    return {
-        task_id: task.id,
-        configurations: Object.fromEntries(comparison.results),
-        winner: comparison.winner,
-        scores: Object.fromEntries(comparison.scores),
-        metadata: {
-            benchmark_version: BENCHMARK_VERSION,
-            started_at: startedAt,
-            completed_at: completedAt,
-            total_duration_ms: completedAt - startedAt,
-        },
-    };
-}
-/**
- * Run a single configuration (for targeted Harbor evaluations)
- *
- * @example Running single config with Harbor:
- * ```bash
- * harbor run \
- *   --dataset tasks.yaml \
- *   --agent "@agent-relay/benchmark/harbor:evaluateSingle" \
- *   --env-var CONFIG=swarm
- * ```
- */
-export async function evaluateSingle(input) {
-    const config = input.config || 'single';
-    const task = convertHarborTask(input);
-    const benchmark = new ComparisonBenchmark({
-        configurations: [config],
-        cli: 'claude',
-        quiet: true,
-        cooldownMs: 0,
-    });
-    const result = await benchmark.runSingle(task, config);
-    return {
-        task_id: task.id,
-        configuration: config,
-        result,
-        success: result.success,
-    };
-}
-/**
- * Evaluate with custom configuration
- */
-export async function evaluateCustom(input, config) {
-    const startedAt = Date.now();
-    const task = convertHarborTask(input);
-    const benchmark = new ComparisonBenchmark({
-        ...config,
-        quiet: true,
-    });
-    const comparison = await benchmark.runComparison(task);
-    const completedAt = Date.now();
-    return {
-        task_id: task.id,
-        configurations: Object.fromEntries(comparison.results),
-        winner: comparison.winner,
-        scores: Object.fromEntries(comparison.scores),
-        metadata: {
-            benchmark_version: BENCHMARK_VERSION,
-            started_at: startedAt,
-            completed_at: completedAt,
-            total_duration_ms: completedAt - startedAt,
-        },
-    };
-}
-// Default export for Harbor
-export default evaluate;
-//# sourceMappingURL=harbor.js.map

package/packages/benchmark/dist/harbor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"harbor.js","sourceRoot":"","sources":["../src/harbor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAUH,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAErD,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAElC;;GAEG;AACH,SAAS,iBAAiB,CAAC,KAAsB;IAC/C,OAAO;QACL,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE;QACxB,eAAe,EAAE,KAAK,CAAC,gBAAgB,IAAI,6BAA6B;QACxE,UAAU,EAAG,KAAK,CAAC,UAA6B,IAAI,QAAQ;QAC5D,SAAS,EAAE,MAAM,EAAE,mBAAmB;QACtC,IAAI,EAAE,EAAE;KACT,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAEtC,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC;QACxC,cAAc,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC;QAC/C,GAAG,EAAE,QAAQ;QACb,KAAK,EAAE,IAAI,EAAE,iCAAiC;QAC9C,UAAU,EAAE,IAAI;KACjB,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAEvD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE/B,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,EAAE;QAChB,cAAc,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAGpD;QACD,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,CAG3C;QACD,QAAQ,EAAE;YACR,iBAAiB,EAAE,iBAAiB;YACpC,UAAU,EAAE,SAAS;YACrB,YAAY,EAAE,WAAW;YACzB,iBAAiB,EAAE,WAAW,GAAG,SAAS;SAC3C;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAuD;IAEvD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC;IACxC,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAEtC,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC;QACxC,cAAc,EAAE,CAAC,MAAM,CAAC;QACxB,GAAG,EAAE,QAAQ;QACb,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,CAAC;KACd,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAEvD,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,EAAE;QAChB,aAAa,EAAE,MAAM;QACrB,MAAM;QACN,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAsB,EACtB,MAAgC;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAEtC,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC;QACxC,GAAG,MAAM;QACT,KAAK,EAAE,IAAI;KACZ,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IACvD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE/B,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,EAAE;QAChB,cAAc,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAGpD;QACD,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,CAG3C;QACD,QAAQ,EAAE;YACR,iBAAiB,EAAE,iBAAiB;YACpC,UAAU,EAAE,SAAS;YACrB,YAAY,EAAE,WAAW;YACzB,iBAAiB,EAAE,WAAW,GAAG,SAAS;SAC3C;KACF,CAAC;AACJ,CAAC;AAED,4BAA4B;AAC5B,eAAe,QAAQ,CAAC"}