agent-relay 2.3.2 → 2.3.5

package/packages/benchmark/src/cli.ts

@@ -1,240 +0,0 @@
-#!/usr/bin/env node
-/**
- * Benchmark CLI
- *
- * Command-line interface for running agent swarm benchmarks.
- */
-
-import { Command } from 'commander';
-import { readFileSync } from 'node:fs';
-import { parse as parseYaml } from 'yaml';
-import { ComparisonBenchmark } from './benchmark.js';
-import type {
-  Task,
-  TaskDataset,
-  ConfigurationType,
-  BenchmarkConfig,
-} from './types.js';
-
-const program = new Command();
-
-program
-  .name('relay-benchmark')
-  .description('Benchmark agent swarms, sub-agents, and single agents')
-  .version('1.0.0');
-
-program
-  .command('run')
-  .description('Run a benchmark comparison')
-  .option('-d, --dataset <path>', 'Path to task dataset (YAML or JSON)')
-  .option('-t, --task <id>', 'Run only a specific task by ID')
-  .option(
-    '-c, --config <types>',
-    'Configurations to run (single,subagent,swarm,all)',
-    'all'
-  )
-  .option('--cli <name>', 'CLI to use for agents', 'claude')
-  .option('--cwd <path>', 'Working directory for tasks')
-  .option('-q, --quiet', 'Suppress output', false)
-  .option('--cooldown <ms>', 'Cooldown between runs in ms', '5000')
-  .option('--max-swarm <n>', 'Maximum swarm size', '10')
-  .option('-o, --output <path>', 'Output results to JSON file')
-  .action(async (options) => {
-    try {
-      await runBenchmark(options);
-    } catch (err) {
-      console.error('Error:', (err as Error).message);
-      process.exit(1);
-    }
-  });
-
-program
-  .command('list')
-  .description('List tasks in a dataset')
-  .argument('<dataset>', 'Path to task dataset')
-  .action((datasetPath) => {
-    const dataset = loadDataset(datasetPath);
-    console.log(`\nDataset: ${dataset.name || 'Unnamed'}`);
-    if (dataset.description) {
-      console.log(`Description: ${dataset.description}`);
-    }
-    console.log(`\nTasks (${dataset.tasks.length}):\n`);
-
-    for (const task of dataset.tasks) {
-      console.log(`  ${task.id}`);
-      console.log(`    Complexity: ${task.complexity}`);
-      console.log(`    Files: ${task.files.length}`);
-      console.log(`    ${task.description.substring(0, 60)}...`);
-      console.log('');
-    }
-  });
-
-async function runBenchmark(options: {
-  dataset?: string;
-  task?: string;
-  config: string;
-  cli: string;
-  cwd?: string;
-  quiet: boolean;
-  cooldown: string;
-  maxSwarm: string;
-  output?: string;
-}): Promise<void> {
-  // Parse configurations
-  const configurations = parseConfigurations(options.config);
-
-  // Build benchmark config
-  const benchmarkConfig: Partial<BenchmarkConfig> = {
-    configurations,
-    cli: options.cli,
-    cwd: options.cwd,
-    quiet: options.quiet,
-    cooldownMs: parseInt(options.cooldown, 10),
-    maxSwarmSize: parseInt(options.maxSwarm, 10),
-  };
-
-  const benchmark = new ComparisonBenchmark(benchmarkConfig);
-
-  // Load tasks
-  let tasks: Task[];
-  if (options.dataset) {
-    const dataset = loadDataset(options.dataset);
-    tasks = dataset.tasks;
-
-    if (options.task) {
-      tasks = tasks.filter((t) => t.id === options.task);
-      if (tasks.length === 0) {
-        throw new Error(`Task not found: ${options.task}`);
-      }
-    }
-  } else if (options.task) {
-    // Create a simple task from command line
-    tasks = [
-      {
-        id: options.task,
-        description: options.task,
-        files: [],
-        expectedOutcome: 'Task completed',
-        complexity: 'medium',
-      },
-    ];
-  } else {
-    throw new Error('Either --dataset or --task is required');
-  }
-
-  // Run benchmarks
-  const results = [];
-  for (const task of tasks) {
-    if (!options.quiet) {
-      console.log(`\n${'='.repeat(60)}`);
-      console.log(`Running task: ${task.id}`);
-      console.log('='.repeat(60));
-    }
-
-    const comparison = await benchmark.runComparison(task);
-
-    if (!options.quiet) {
-      benchmark.printComparison(comparison);
-    }
-
-    results.push({
-      taskId: task.id,
-      winner: comparison.winner,
-      results: Object.fromEntries(comparison.results),
-      scores: Object.fromEntries(comparison.scores),
-    });
-  }
-
-  // Output results
-  if (options.output) {
-    const { writeFileSync } = await import('node:fs');
-    writeFileSync(options.output, JSON.stringify(results, null, 2));
-    console.log(`\nResults written to: ${options.output}`);
-  }
-
-  // Print summary
-  if (!options.quiet && results.length > 1) {
-    printSummary(results);
-  }
-}
-
-function parseConfigurations(config: string): ConfigurationType[] {
-  if (config === 'all') {
-    return ['single', 'subagent', 'swarm'];
-  }
-
-  const configs = config.split(',').map((c) => c.trim()) as ConfigurationType[];
-  const valid: ConfigurationType[] = ['single', 'subagent', 'swarm'];
-
-  for (const c of configs) {
-    if (!valid.includes(c)) {
-      throw new Error(`Invalid configuration: ${c}. Valid: ${valid.join(', ')}`);
-    }
-  }
-
-  return configs;
-}
-
-function loadDataset(path: string): TaskDataset {
-  const content = readFileSync(path, 'utf-8');
-
-  let data: TaskDataset;
-  if (path.endsWith('.yaml') || path.endsWith('.yml')) {
-    data = parseYaml(content) as TaskDataset;
-  } else {
-    data = JSON.parse(content) as TaskDataset;
-  }
-
-  // Validate and normalize tasks
-  if (!data.tasks || !Array.isArray(data.tasks)) {
-    throw new Error('Dataset must have a "tasks" array');
-  }
-
-  data.tasks = data.tasks.map((t, i) => ({
-    id: t.id || `task-${i}`,
-    description: t.description || '',
-    files: t.files || [],
-    expectedOutcome: t.expectedOutcome || t.success_criteria || 'Completed',
-    complexity: t.complexity || 'medium',
-    timeoutMs: t.timeoutMs || 300000,
-    tags: t.tags || [],
-  }));
-
-  return data;
-}
-
-function printSummary(
-  results: Array<{
-    taskId: string;
-    winner: ConfigurationType;
-    results: Record<string, unknown>;
-    scores: Record<string, unknown>;
-  }>
-): void {
-  console.log('\n' + '='.repeat(60));
-  console.log('BENCHMARK SUMMARY');
-  console.log('='.repeat(60));
-
-  const wins: Record<ConfigurationType, number> = {
-    single: 0,
-    subagent: 0,
-    swarm: 0,
-  };
-
-  for (const result of results) {
-    wins[result.winner]++;
-  }
-
-  console.log('\nWins by configuration:');
-  for (const [config, count] of Object.entries(wins)) {
-    const bar = '█'.repeat(count) + '░'.repeat(results.length - count);
-    console.log(`  ${config.padEnd(10)} ${bar} ${count}/${results.length}`);
-  }
-
-  const overallWinner = (Object.entries(wins) as [ConfigurationType, number][])
-    .sort((a, b) => b[1] - a[1])[0][0];
-
-  console.log(`\nOverall winner: ${overallWinner.toUpperCase()}`);
-}
-
-program.parse();

package/packages/benchmark/src/harbor.ts

@@ -1,170 +0,0 @@
-/**
- * Harbor Integration
- *
- * Entry points for Harbor benchmark framework integration.
- * https://github.com/laude-institute/harbor
- */
-
-import type {
-  Task,
-  TaskComplexity,
-  ConfigurationType,
-  HarborTaskInput,
-  HarborEvaluationOutput,
-  BenchmarkConfig,
-} from './types.js';
-import { ComparisonBenchmark } from './benchmark.js';
-
-const BENCHMARK_VERSION = '1.0.0';
-
-/**
- * Convert Harbor task input to internal Task format
- */
-function convertHarborTask(input: HarborTaskInput): Task {
-  return {
-    id: input.id,
-    description: input.description,
-    files: input.files || [],
-    expectedOutcome: input.success_criteria || 'Task completed successfully',
-    complexity: (input.complexity as TaskComplexity) || 'medium',
-    timeoutMs: 300000, // 5 minute default
-    tags: [],
-  };
-}
-
-/**
- * Main Harbor evaluation entry point
- *
- * This function is called by Harbor to evaluate a task across all configurations.
- *
- * @example Harbor dataset format:
- * ```yaml
- * tasks:
- *   - id: refactor-auth
- *     description: "Refactor authentication to use JWT"
- *     files:
- *       - src/auth/session.ts
- *       - src/auth/middleware.ts
- *     success_criteria: "All tests pass, JWT tokens used"
- *     complexity: medium
- * ```
- *
- * @example Running with Harbor:
- * ```bash
- * harbor run \
- *   --dataset tasks.yaml \
- *   --agent @agent-relay/benchmark/harbor \
- *   --parallel 10
- * ```
- */
-export async function evaluate(
-  input: HarborTaskInput
-): Promise<HarborEvaluationOutput> {
-  const startedAt = Date.now();
-  const task = convertHarborTask(input);
-
-  const benchmark = new ComparisonBenchmark({
-    configurations: ['single', 'subagent', 'swarm'],
-    cli: 'claude',
-    quiet: true, // Suppress output in Harbor runs
-    cooldownMs: 2000,
-  });
-
-  const comparison = await benchmark.runComparison(task);
-
-  const completedAt = Date.now();
-
-  return {
-    task_id: task.id,
-    configurations: Object.fromEntries(comparison.results) as Record<
-      ConfigurationType,
-      any
-    >,
-    winner: comparison.winner,
-    scores: Object.fromEntries(comparison.scores) as Record<
-      ConfigurationType,
-      any
-    >,
-    metadata: {
-      benchmark_version: BENCHMARK_VERSION,
-      started_at: startedAt,
-      completed_at: completedAt,
-      total_duration_ms: completedAt - startedAt,
-    },
-  };
-}
-
-/**
- * Run a single configuration (for targeted Harbor evaluations)
- *
- * @example Running single config with Harbor:
- * ```bash
- * harbor run \
- *   --dataset tasks.yaml \
- *   --agent "@agent-relay/benchmark/harbor:evaluateSingle" \
- *   --env-var CONFIG=swarm
- * ```
- */
-export async function evaluateSingle(
-  input: HarborTaskInput & { config?: ConfigurationType }
-): Promise<Record<string, unknown>> {
-  const config = input.config || 'single';
-  const task = convertHarborTask(input);
-
-  const benchmark = new ComparisonBenchmark({
-    configurations: [config],
-    cli: 'claude',
-    quiet: true,
-    cooldownMs: 0,
-  });
-
-  const result = await benchmark.runSingle(task, config);
-
-  return {
-    task_id: task.id,
-    configuration: config,
-    result,
-    success: result.success,
-  };
-}
-
-/**
- * Evaluate with custom configuration
- */
-export async function evaluateCustom(
-  input: HarborTaskInput,
-  config: Partial<BenchmarkConfig>
-): Promise<HarborEvaluationOutput> {
-  const startedAt = Date.now();
-  const task = convertHarborTask(input);
-
-  const benchmark = new ComparisonBenchmark({
-    ...config,
-    quiet: true,
-  });
-
-  const comparison = await benchmark.runComparison(task);
-  const completedAt = Date.now();
-
-  return {
-    task_id: task.id,
-    configurations: Object.fromEntries(comparison.results) as Record<
-      ConfigurationType,
-      any
-    >,
-    winner: comparison.winner,
-    scores: Object.fromEntries(comparison.scores) as Record<
-      ConfigurationType,
-      any
-    >,
-    metadata: {
-      benchmark_version: BENCHMARK_VERSION,
-      started_at: startedAt,
-      completed_at: completedAt,
-      total_duration_ms: completedAt - startedAt,
-    },
-  };
-}
-
-// Default export for Harbor
-export default evaluate;

package/packages/benchmark/src/index.ts

@@ -1,73 +0,0 @@
-/**
- * @agent-relay/benchmark
- *
- * Performance benchmarking for agent swarms, sub-agents, and single agents.
- *
- * ## Quick Start
- *
- * ```typescript
- * import { ComparisonBenchmark, type Task } from '@agent-relay/benchmark';
- *
- * const task: Task = {
- *   id: 'refactor-auth',
- *   description: 'Refactor authentication to use JWT',
- *   files: ['src/auth/session.ts', 'src/auth/middleware.ts'],
- *   expectedOutcome: 'All tests pass, JWT tokens used',
- *   complexity: 'medium',
- * };
- *
- * const benchmark = new ComparisonBenchmark();
- * const comparison = await benchmark.runComparison(task);
- *
- * console.log(`Winner: ${comparison.winner}`);
- * benchmark.printComparison(comparison);
- * ```
- *
- * ## With Harbor
- *
- * ```bash
- * harbor run \
- *   --dataset tasks.yaml \
- *   --agent @agent-relay/benchmark/harbor \
- *   --parallel 10
- * ```
- *
- * ## CLI Usage
- *
- * ```bash
- * relay-benchmark run --dataset tasks.yaml --config all
- * relay-benchmark run --dataset tasks.yaml --config swarm
- * relay-benchmark list tasks.yaml
- * ```
- */
-
-// Types
-export type {
-  ConfigurationType,
-  TaskComplexity,
-  Task,
-  RunResult,
-  ComparisonResult,
-  ScoreBreakdown,
-  BenchmarkConfig,
-  RunMetrics,
-  TaskDataset,
-  HarborTaskInput,
-  HarborEvaluationOutput,
-} from './types.js';
-
-export { DEFAULT_BENCHMARK_CONFIG } from './types.js';
-
-// Main benchmark class
-export { ComparisonBenchmark, runComparison } from './benchmark.js';
-
-// Runners
-export {
-  ConfigurationRunner,
-  SingleAgentRunner,
-  SubAgentRunner,
-  SwarmRunner,
-} from './runners/index.js';
-
-// Harbor integration
-export { evaluate, evaluateSingle, evaluateCustom } from './harbor.js';

package/packages/benchmark/src/runners/base.ts

@@ -1,205 +0,0 @@
-/**
- * Base Configuration Runner
- *
- * Abstract base class for benchmark configuration runners.
- */
-
-import {
-  createRelay,
-  RelayClient,
-  type MetricsResponsePayload,
-  type Relay,
-} from '@agent-relay/sdk';
-import type {
-  ConfigurationType,
-  Task,
-  RunResult,
-  RunMetrics,
-  BenchmarkConfig,
-} from '../types.js';
-import { DEFAULT_BENCHMARK_CONFIG } from '../types.js';
-
-type AgentMetrics = MetricsResponsePayload['agents'][number] & {
-  tokens?: number;
-  memoryMb?: number;
-};
-
-/**
- * Abstract base class for configuration runners
- */
-export abstract class ConfigurationRunner {
-  protected relay!: Relay;
-  protected orchestrator!: RelayClient;
-  protected config: BenchmarkConfig;
-  protected metrics: RunMetrics = {
-    messages: 0,
-    latencies: [],
-    startTime: 0,
-    spawnedAgents: [],
-    errors: [],
-  };
-
-  constructor(config: Partial<BenchmarkConfig> = {}) {
-    this.config = { ...DEFAULT_BENCHMARK_CONFIG, ...config };
-  }
-
-  /**
-   * Get the configuration type this runner handles
-   */
-  abstract get configurationType(): ConfigurationType;
-
-  /**
-   * Set up the relay and orchestrator client
-   */
-  async setup(): Promise<void> {
-    this.relay = await createRelay({
-      socketPath: this.config.socketPath,
-      quiet: this.config.quiet,
-      spawnManager: true,
-    });
-    this.orchestrator = await this.relay.client('Orchestrator', {
-      quiet: this.config.quiet,
-    });
-    this.resetMetrics();
-  }
-
-  /**
-   * Run a task and return the result
-   */
-  abstract run(task: Task): Promise<RunResult>;
-
-  /**
-   * Clean up resources
-   */
-  async teardown(): Promise<void> {
-    // Release any remaining agents
-    for (const agent of this.metrics.spawnedAgents) {
-      try {
-        await this.orchestrator.release(agent);
-      } catch {
-        // Ignore release errors during cleanup
-      }
-    }
-
-    await this.relay.stop();
-  }
-
-  /**
-   * Reset metrics for a new run
-   */
-  protected resetMetrics(): void {
-    this.metrics = {
-      messages: 0,
-      latencies: [],
-      startTime: Date.now(),
-      spawnedAgents: [],
-      errors: [],
-    };
-  }
-
-  /**
-   * Calculate percentile from an array of values
-   */
-  protected percentile(arr: number[], p: number): number {
-    if (arr.length === 0) return 0;
-    const sorted = [...arr].sort((a, b) => a - b);
-    const idx = Math.ceil((p / 100) * sorted.length) - 1;
-    return sorted[Math.max(0, idx)];
-  }
-
-  /**
-   * Extract total tokens from metrics response
-   */
-  protected extractTokens(metrics: MetricsResponsePayload): number {
-    const agents = metrics.agents as AgentMetrics[] | undefined;
-    return (
-      agents?.reduce((sum, agent) => sum + (agent.tokens || 0), 0) || 0
-    );
-  }
-
-  /**
-   * Extract peak memory from metrics response
-   */
-  protected extractMemory(metrics: MetricsResponsePayload): number {
-    const agents = metrics.agents as AgentMetrics[] | undefined;
-    const memoryValues = agents?.map((agent) => {
-      if (agent.memoryMb != null) return agent.memoryMb;
-      if (agent.rssBytes != null) return agent.rssBytes / 1024 / 1024;
-      return 0;
-    });
-    return Math.max(...(memoryValues || [0]));
-  }
-
-  /**
-   * Build a failed result when setup fails
-   */
-  protected buildFailedResult(
-    task: Task,
-    startTime: number,
-    errors: string[]
-  ): RunResult {
-    const now = Date.now();
-    return {
-      taskId: task.id,
-      configuration: this.configurationType,
-      totalTimeMs: now - startTime,
-      timeToFirstActionMs: 0,
-      messageCount: 0,
-      avgLatencyMs: 0,
-      latencyP50Ms: 0,
-      latencyP99Ms: 0,
-      coordinationRounds: 0,
-      agentCount: 0,
-      totalTokensUsed: 0,
-      peakMemoryMb: 0,
-      success: false,
-      completionRate: 0,
-      errors,
-      startedAt: startTime,
-      completedAt: now,
-    };
-  }
-
-  /**
-   * Wait for an agent to complete their task
-   */
-  protected waitForCompletion(
-    agentName: string,
-    task: Task,
-    donePrefix = 'DONE:'
-  ): Promise<boolean> {
-    const timeoutMs = task.timeoutMs || 300000;
-
-    return new Promise((resolve) => {
-      const timeout = setTimeout(() => {
-        this.metrics.errors.push(`Timeout waiting for ${agentName}`);
-        resolve(false);
-      }, timeoutMs);
-
-      const originalHandler = this.orchestrator.onMessage;
-      this.orchestrator.onMessage = (from, payload, id, meta, originalTo) => {
-        // Call original handler if exists
-        if (originalHandler) {
-          originalHandler(from, payload, id, meta, originalTo);
-        }
-
-        if (from === agentName && payload.body.startsWith(donePrefix)) {
-          clearTimeout(timeout);
-          resolve(true);
-        }
-      };
-    });
-  }
-
-  /**
-   * Log a message if not in quiet mode
-   */
-  protected log(message: string): void {
-    if (!this.config.quiet) {
-      console.log(`[${this.configurationType}] ${message}`);
-    }
-  }
-}
-
-// Re-export the DEFAULT_BENCHMARK_CONFIG
-export { DEFAULT_BENCHMARK_CONFIG } from '../types.js';

package/packages/benchmark/src/runners/index.ts

@@ -1,10 +0,0 @@
-/**
- * Benchmark Runners
- *
- * Export all configuration runners.
- */
-
-export { ConfigurationRunner, DEFAULT_BENCHMARK_CONFIG } from './base.js';
-export { SingleAgentRunner } from './single.js';
-export { SubAgentRunner } from './subagent.js';
-export { SwarmRunner } from './swarm.js';