agent-relay 2.0.29 → 2.0.33

package/packages/benchmark/src/cli.ts

@@ -0,0 +1,240 @@
+#!/usr/bin/env node
+/**
+ * Benchmark CLI
+ *
+ * Command-line interface for running agent swarm benchmarks.
+ */
+
+import { Command } from 'commander';
+import { readFileSync } from 'node:fs';
+import { parse as parseYaml } from 'yaml';
+import { ComparisonBenchmark } from './benchmark.js';
+import type {
+  Task,
+  TaskDataset,
+  ConfigurationType,
+  BenchmarkConfig,
+} from './types.js';
+
+const program = new Command();
+
+program
+  .name('relay-benchmark')
+  .description('Benchmark agent swarms, sub-agents, and single agents')
+  .version('1.0.0');
+
+program
+  .command('run')
+  .description('Run a benchmark comparison')
+  .option('-d, --dataset <path>', 'Path to task dataset (YAML or JSON)')
+  .option('-t, --task <id>', 'Run only a specific task by ID')
+  .option(
+    '-c, --config <types>',
+    'Configurations to run (single,subagent,swarm,all)',
+    'all'
+  )
+  .option('--cli <name>', 'CLI to use for agents', 'claude')
+  .option('--cwd <path>', 'Working directory for tasks')
+  .option('-q, --quiet', 'Suppress output', false)
+  .option('--cooldown <ms>', 'Cooldown between runs in ms', '5000')
+  .option('--max-swarm <n>', 'Maximum swarm size', '10')
+  .option('-o, --output <path>', 'Output results to JSON file')
+  .action(async (options) => {
+    try {
+      await runBenchmark(options);
+    } catch (err) {
+      console.error('Error:', (err as Error).message);
+      process.exit(1);
+    }
+  });
+
+program
+  .command('list')
+  .description('List tasks in a dataset')
+  .argument('<dataset>', 'Path to task dataset')
+  .action((datasetPath) => {
+    const dataset = loadDataset(datasetPath);
+    console.log(`\nDataset: ${dataset.name || 'Unnamed'}`);
+    if (dataset.description) {
+      console.log(`Description: ${dataset.description}`);
+    }
+    console.log(`\nTasks (${dataset.tasks.length}):\n`);
+
+    for (const task of dataset.tasks) {
+      console.log(`  ${task.id}`);
+      console.log(`    Complexity: ${task.complexity}`);
+      console.log(`    Files: ${task.files.length}`);
+      console.log(`    ${task.description.substring(0, 60)}...`);
+      console.log('');
+    }
+  });
+
+async function runBenchmark(options: {
+  dataset?: string;
+  task?: string;
+  config: string;
+  cli: string;
+  cwd?: string;
+  quiet: boolean;
+  cooldown: string;
+  maxSwarm: string;
+  output?: string;
+}): Promise<void> {
+  // Parse configurations
+  const configurations = parseConfigurations(options.config);
+
+  // Build benchmark config
+  const benchmarkConfig: Partial<BenchmarkConfig> = {
+    configurations,
+    cli: options.cli,
+    cwd: options.cwd,
+    quiet: options.quiet,
+    cooldownMs: parseInt(options.cooldown, 10),
+    maxSwarmSize: parseInt(options.maxSwarm, 10),
+  };
+
+  const benchmark = new ComparisonBenchmark(benchmarkConfig);
+
+  // Load tasks
+  let tasks: Task[];
+  if (options.dataset) {
+    const dataset = loadDataset(options.dataset);
+    tasks = dataset.tasks;
+
+    if (options.task) {
+      tasks = tasks.filter((t) => t.id === options.task);
+      if (tasks.length === 0) {
+        throw new Error(`Task not found: ${options.task}`);
+      }
+    }
+  } else if (options.task) {
+    // Create a simple task from command line
+    tasks = [
+      {
+        id: options.task,
+        description: options.task,
+        files: [],
+        expectedOutcome: 'Task completed',
+        complexity: 'medium',
+      },
+    ];
+  } else {
+    throw new Error('Either --dataset or --task is required');
+  }
+
+  // Run benchmarks
+  const results = [];
+  for (const task of tasks) {
+    if (!options.quiet) {
+      console.log(`\n${'='.repeat(60)}`);
+      console.log(`Running task: ${task.id}`);
+      console.log('='.repeat(60));
+    }
+
+    const comparison = await benchmark.runComparison(task);
+
+    if (!options.quiet) {
+      benchmark.printComparison(comparison);
+    }
+
+    results.push({
+      taskId: task.id,
+      winner: comparison.winner,
+      results: Object.fromEntries(comparison.results),
+      scores: Object.fromEntries(comparison.scores),
+    });
+  }
+
+  // Output results
+  if (options.output) {
+    const { writeFileSync } = await import('node:fs');
+    writeFileSync(options.output, JSON.stringify(results, null, 2));
+    console.log(`\nResults written to: ${options.output}`);
+  }
+
+  // Print summary
+  if (!options.quiet && results.length > 1) {
+    printSummary(results);
+  }
+}
+
+function parseConfigurations(config: string): ConfigurationType[] {
+  if (config === 'all') {
+    return ['single', 'subagent', 'swarm'];
+  }
+
+  const configs = config.split(',').map((c) => c.trim()) as ConfigurationType[];
+  const valid: ConfigurationType[] = ['single', 'subagent', 'swarm'];
+
+  for (const c of configs) {
+    if (!valid.includes(c)) {
+      throw new Error(`Invalid configuration: ${c}. Valid: ${valid.join(', ')}`);
+    }
+  }
+
+  return configs;
+}
+
+function loadDataset(path: string): TaskDataset {
+  const content = readFileSync(path, 'utf-8');
+
+  let data: TaskDataset;
+  if (path.endsWith('.yaml') || path.endsWith('.yml')) {
+    data = parseYaml(content) as TaskDataset;
+  } else {
+    data = JSON.parse(content) as TaskDataset;
+  }
+
+  // Validate and normalize tasks
+  if (!data.tasks || !Array.isArray(data.tasks)) {
+    throw new Error('Dataset must have a "tasks" array');
+  }
+
+  data.tasks = data.tasks.map((t, i) => ({
+    id: t.id || `task-${i}`,
+    description: t.description || '',
+    files: t.files || [],
+    expectedOutcome: t.expectedOutcome || t.success_criteria || 'Completed',
+    complexity: t.complexity || 'medium',
+    timeoutMs: t.timeoutMs || 300000,
+    tags: t.tags || [],
+  }));
+
+  return data;
+}
+
+function printSummary(
+  results: Array<{
+    taskId: string;
+    winner: ConfigurationType;
+    results: Record<string, unknown>;
+    scores: Record<string, unknown>;
+  }>
+): void {
+  console.log('\n' + '='.repeat(60));
+  console.log('BENCHMARK SUMMARY');
+  console.log('='.repeat(60));
+
+  const wins: Record<ConfigurationType, number> = {
+    single: 0,
+    subagent: 0,
+    swarm: 0,
+  };
+
+  for (const result of results) {
+    wins[result.winner]++;
+  }
+
+  console.log('\nWins by configuration:');
+  for (const [config, count] of Object.entries(wins)) {
+    const bar = '█'.repeat(count) + '░'.repeat(results.length - count);
+    console.log(`  ${config.padEnd(10)} ${bar} ${count}/${results.length}`);
+  }
+
+  const overallWinner = (Object.entries(wins) as [ConfigurationType, number][])
+    .sort((a, b) => b[1] - a[1])[0][0];
+
+  console.log(`\nOverall winner: ${overallWinner.toUpperCase()}`);
+}
+
+program.parse();

package/packages/benchmark/src/harbor.ts

@@ -0,0 +1,170 @@
+/**
+ * Harbor Integration
+ *
+ * Entry points for Harbor benchmark framework integration.
+ * https://github.com/laude-institute/harbor
+ */
+
+import type {
+  Task,
+  TaskComplexity,
+  ConfigurationType,
+  HarborTaskInput,
+  HarborEvaluationOutput,
+  BenchmarkConfig,
+} from './types.js';
+import { ComparisonBenchmark } from './benchmark.js';
+
+const BENCHMARK_VERSION = '1.0.0';
+
+/**
+ * Convert Harbor task input to internal Task format
+ */
+function convertHarborTask(input: HarborTaskInput): Task {
+  return {
+    id: input.id,
+    description: input.description,
+    files: input.files || [],
+    expectedOutcome: input.success_criteria || 'Task completed successfully',
+    complexity: (input.complexity as TaskComplexity) || 'medium',
+    timeoutMs: 300000, // 5 minute default
+    tags: [],
+  };
+}
+
+/**
+ * Main Harbor evaluation entry point
+ *
+ * This function is called by Harbor to evaluate a task across all configurations.
+ *
+ * @example Harbor dataset format:
+ * ```yaml
+ * tasks:
+ *   - id: refactor-auth
+ *     description: "Refactor authentication to use JWT"
+ *     files:
+ *       - src/auth/session.ts
+ *       - src/auth/middleware.ts
+ *     success_criteria: "All tests pass, JWT tokens used"
+ *     complexity: medium
+ * ```
+ *
+ * @example Running with Harbor:
+ * ```bash
+ * harbor run \
+ *   --dataset tasks.yaml \
+ *   --agent @agent-relay/benchmark/harbor \
+ *   --parallel 10
+ * ```
+ */
+export async function evaluate(
+  input: HarborTaskInput
+): Promise<HarborEvaluationOutput> {
+  const startedAt = Date.now();
+  const task = convertHarborTask(input);
+
+  const benchmark = new ComparisonBenchmark({
+    configurations: ['single', 'subagent', 'swarm'],
+    cli: 'claude',
+    quiet: true, // Suppress output in Harbor runs
+    cooldownMs: 2000,
+  });
+
+  const comparison = await benchmark.runComparison(task);
+
+  const completedAt = Date.now();
+
+  return {
+    task_id: task.id,
+    configurations: Object.fromEntries(comparison.results) as Record<
+      ConfigurationType,
+      any
+    >,
+    winner: comparison.winner,
+    scores: Object.fromEntries(comparison.scores) as Record<
+      ConfigurationType,
+      any
+    >,
+    metadata: {
+      benchmark_version: BENCHMARK_VERSION,
+      started_at: startedAt,
+      completed_at: completedAt,
+      total_duration_ms: completedAt - startedAt,
+    },
+  };
+}
+
+/**
+ * Run a single configuration (for targeted Harbor evaluations)
+ *
+ * @example Running single config with Harbor:
+ * ```bash
+ * harbor run \
+ *   --dataset tasks.yaml \
+ *   --agent "@agent-relay/benchmark/harbor:evaluateSingle" \
+ *   --env-var CONFIG=swarm
+ * ```
+ */
+export async function evaluateSingle(
+  input: HarborTaskInput & { config?: ConfigurationType }
+): Promise<Record<string, unknown>> {
+  const config = input.config || 'single';
+  const task = convertHarborTask(input);
+
+  const benchmark = new ComparisonBenchmark({
+    configurations: [config],
+    cli: 'claude',
+    quiet: true,
+    cooldownMs: 0,
+  });
+
+  const result = await benchmark.runSingle(task, config);
+
+  return {
+    task_id: task.id,
+    configuration: config,
+    result,
+    success: result.success,
+  };
+}
+
+/**
+ * Evaluate with custom configuration
+ */
+export async function evaluateCustom(
+  input: HarborTaskInput,
+  config: Partial<BenchmarkConfig>
+): Promise<HarborEvaluationOutput> {
+  const startedAt = Date.now();
+  const task = convertHarborTask(input);
+
+  const benchmark = new ComparisonBenchmark({
+    ...config,
+    quiet: true,
+  });
+
+  const comparison = await benchmark.runComparison(task);
+  const completedAt = Date.now();
+
+  return {
+    task_id: task.id,
+    configurations: Object.fromEntries(comparison.results) as Record<
+      ConfigurationType,
+      any
+    >,
+    winner: comparison.winner,
+    scores: Object.fromEntries(comparison.scores) as Record<
+      ConfigurationType,
+      any
+    >,
+    metadata: {
+      benchmark_version: BENCHMARK_VERSION,
+      started_at: startedAt,
+      completed_at: completedAt,
+      total_duration_ms: completedAt - startedAt,
+    },
+  };
+}
+
+// Default export for Harbor
+export default evaluate;

package/packages/benchmark/src/index.ts

@@ -0,0 +1,73 @@
+/**
+ * @agent-relay/benchmark
+ *
+ * Performance benchmarking for agent swarms, sub-agents, and single agents.
+ *
+ * ## Quick Start
+ *
+ * ```typescript
+ * import { ComparisonBenchmark, type Task } from '@agent-relay/benchmark';
+ *
+ * const task: Task = {
+ *   id: 'refactor-auth',
+ *   description: 'Refactor authentication to use JWT',
+ *   files: ['src/auth/session.ts', 'src/auth/middleware.ts'],
+ *   expectedOutcome: 'All tests pass, JWT tokens used',
+ *   complexity: 'medium',
+ * };
+ *
+ * const benchmark = new ComparisonBenchmark();
+ * const comparison = await benchmark.runComparison(task);
+ *
+ * console.log(`Winner: ${comparison.winner}`);
+ * benchmark.printComparison(comparison);
+ * ```
+ *
+ * ## With Harbor
+ *
+ * ```bash
+ * harbor run \
+ *   --dataset tasks.yaml \
+ *   --agent @agent-relay/benchmark/harbor \
+ *   --parallel 10
+ * ```
+ *
+ * ## CLI Usage
+ *
+ * ```bash
+ * relay-benchmark run --dataset tasks.yaml --config all
+ * relay-benchmark run --dataset tasks.yaml --config swarm
+ * relay-benchmark list tasks.yaml
+ * ```
+ */
+
+// Types
+export type {
+  ConfigurationType,
+  TaskComplexity,
+  Task,
+  RunResult,
+  ComparisonResult,
+  ScoreBreakdown,
+  BenchmarkConfig,
+  RunMetrics,
+  TaskDataset,
+  HarborTaskInput,
+  HarborEvaluationOutput,
+} from './types.js';
+
+export { DEFAULT_BENCHMARK_CONFIG } from './types.js';
+
+// Main benchmark class
+export { ComparisonBenchmark, runComparison } from './benchmark.js';
+
+// Runners
+export {
+  ConfigurationRunner,
+  SingleAgentRunner,
+  SubAgentRunner,
+  SwarmRunner,
+} from './runners/index.js';
+
+// Harbor integration
+export { evaluate, evaluateSingle, evaluateCustom } from './harbor.js';

package/packages/benchmark/src/runners/base.ts

@@ -0,0 +1,204 @@
+/**
+ * Base Configuration Runner
+ *
+ * Abstract base class for benchmark configuration runners.
+ */
+
+import {
+  createRelay,
+  RelayClient,
+  type MetricsResponsePayload,
+  type Relay,
+} from '@agent-relay/sdk';
+import type {
+  ConfigurationType,
+  Task,
+  RunResult,
+  RunMetrics,
+  BenchmarkConfig,
+} from '../types.js';
+import { DEFAULT_BENCHMARK_CONFIG } from '../types.js';
+
+type AgentMetrics = MetricsResponsePayload['agents'][number] & {
+  tokens?: number;
+  memoryMb?: number;
+};
+
+/**
+ * Abstract base class for configuration runners
+ */
+export abstract class ConfigurationRunner {
+  protected relay!: Relay;
+  protected orchestrator!: RelayClient;
+  protected config: BenchmarkConfig;
+  protected metrics: RunMetrics = {
+    messages: 0,
+    latencies: [],
+    startTime: 0,
+    spawnedAgents: [],
+    errors: [],
+  };
+
+  constructor(config: Partial<BenchmarkConfig> = {}) {
+    this.config = { ...DEFAULT_BENCHMARK_CONFIG, ...config };
+  }
+
+  /**
+   * Get the configuration type this runner handles
+   */
+  abstract get configurationType(): ConfigurationType;
+
+  /**
+   * Set up the relay and orchestrator client
+   */
+  async setup(): Promise<void> {
+    this.relay = await createRelay({
+      socketPath: this.config.socketPath,
+      quiet: this.config.quiet,
+    });
+    this.orchestrator = await this.relay.client('Orchestrator', {
+      quiet: this.config.quiet,
+    });
+    this.resetMetrics();
+  }
+
+  /**
+   * Run a task and return the result
+   */
+  abstract run(task: Task): Promise<RunResult>;
+
+  /**
+   * Clean up resources
+   */
+  async teardown(): Promise<void> {
+    // Release any remaining agents
+    for (const agent of this.metrics.spawnedAgents) {
+      try {
+        await this.orchestrator.release(agent);
+      } catch {
+        // Ignore release errors during cleanup
+      }
+    }
+
+    await this.relay.stop();
+  }
+
+  /**
+   * Reset metrics for a new run
+   */
+  protected resetMetrics(): void {
+    this.metrics = {
+      messages: 0,
+      latencies: [],
+      startTime: Date.now(),
+      spawnedAgents: [],
+      errors: [],
+    };
+  }
+
+  /**
+   * Calculate percentile from an array of values
+   */
+  protected percentile(arr: number[], p: number): number {
+    if (arr.length === 0) return 0;
+    const sorted = [...arr].sort((a, b) => a - b);
+    const idx = Math.ceil((p / 100) * sorted.length) - 1;
+    return sorted[Math.max(0, idx)];
+  }
+
+  /**
+   * Extract total tokens from metrics response
+   */
+  protected extractTokens(metrics: MetricsResponsePayload): number {
+    const agents = metrics.agents as AgentMetrics[] | undefined;
+    return (
+      agents?.reduce((sum, agent) => sum + (agent.tokens || 0), 0) || 0
+    );
+  }
+
+  /**
+   * Extract peak memory from metrics response
+   */
+  protected extractMemory(metrics: MetricsResponsePayload): number {
+    const agents = metrics.agents as AgentMetrics[] | undefined;
+    const memoryValues = agents?.map((agent) => {
+      if (agent.memoryMb != null) return agent.memoryMb;
+      if (agent.rssBytes != null) return agent.rssBytes / 1024 / 1024;
+      return 0;
+    });
+    return Math.max(...(memoryValues || [0]));
+  }
+
+  /**
+   * Build a failed result when setup fails
+   */
+  protected buildFailedResult(
+    task: Task,
+    startTime: number,
+    errors: string[]
+  ): RunResult {
+    const now = Date.now();
+    return {
+      taskId: task.id,
+      configuration: this.configurationType,
+      totalTimeMs: now - startTime,
+      timeToFirstActionMs: 0,
+      messageCount: 0,
+      avgLatencyMs: 0,
+      latencyP50Ms: 0,
+      latencyP99Ms: 0,
+      coordinationRounds: 0,
+      agentCount: 0,
+      totalTokensUsed: 0,
+      peakMemoryMb: 0,
+      success: false,
+      completionRate: 0,
+      errors,
+      startedAt: startTime,
+      completedAt: now,
+    };
+  }
+
+  /**
+   * Wait for an agent to complete their task
+   */
+  protected waitForCompletion(
+    agentName: string,
+    task: Task,
+    donePrefix = 'DONE:'
+  ): Promise<boolean> {
+    const timeoutMs = task.timeoutMs || 300000;
+
+    return new Promise((resolve) => {
+      const timeout = setTimeout(() => {
+        this.metrics.errors.push(`Timeout waiting for ${agentName}`);
+        resolve(false);
+      }, timeoutMs);
+
+      const originalHandler = this.orchestrator.onMessage;
+      this.orchestrator.onMessage = (from, payload, id, meta, originalTo) => {
+        // Call original handler if exists
+        if (originalHandler) {
+          originalHandler(from, payload, id, meta, originalTo);
+        }
+
+        if (from === agentName && payload.body.startsWith(donePrefix)) {
+          clearTimeout(timeout);
+          resolve(true);
+        }
+      };
+    });
+  }
+
+  /**
+   * Log a message if not in quiet mode
+   */
+  protected log(message: string): void {
+    if (!this.config.quiet) {
+      console.log(`[${this.configurationType}] ${message}`);
+    }
+  }
+}
+
+// Re-export the DEFAULT_BENCHMARK_CONFIG
+export { DEFAULT_BENCHMARK_CONFIG } from '../types.js';

package/packages/benchmark/src/runners/index.ts

@@ -0,0 +1,10 @@
+/**
+ * Benchmark Runners
+ *
+ * Export all configuration runners.
+ */
+
+export { ConfigurationRunner, DEFAULT_BENCHMARK_CONFIG } from './base.js';
+export { SingleAgentRunner } from './single.js';
+export { SubAgentRunner } from './subagent.js';
+export { SwarmRunner } from './swarm.js';