agent-relay 2.3.2 → 2.3.5

package/packages/benchmark/src/runners/single.ts

@@ -1,121 +0,0 @@
-/**
- * Single Agent Runner
- *
- * Runs tasks using a single agent with no delegation.
- */
-
-import type { Task, RunResult, ConfigurationType } from '../types.js';
-import { ConfigurationRunner } from './base.js';
-
-/**
- * Runner for single-agent configuration
- */
-export class SingleAgentRunner extends ConfigurationRunner {
-  get configurationType(): ConfigurationType {
-    return 'single';
-  }
-
-  async run(task: Task): Promise<RunResult> {
-    const startTime = Date.now();
-    let firstActionTime = 0;
-    let success = false;
-
-    this.resetMetrics();
-    this.log(`Starting task: ${task.id}`);
-
-    try {
-      // Spawn single agent with full task
-      const result = await this.orchestrator.spawn({
-        name: 'SoloAgent',
-        cli: this.config.cli,
-        task: this.buildTaskPrompt(task),
-        cwd: this.config.cwd,
-      });
-
-      if (!result.success) {
-        this.metrics.errors.push(result.error || 'Spawn failed');
-        return this.buildFailedResult(task, startTime, this.metrics.errors);
-      }
-
-      this.metrics.spawnedAgents.push('SoloAgent');
-      firstActionTime = Date.now() - startTime;
-      this.log(`Agent spawned in ${firstActionTime}ms`);
-
-      // Wait for completion signal
-      success = await this.waitForCompletion('SoloAgent', task);
-
-      if (success) {
-        this.log('Task completed successfully');
-      } else {
-        this.log('Task failed or timed out');
-      }
-    } catch (err) {
-      this.metrics.errors.push((err as Error).message);
-    }
-
-    // Collect final metrics
-    let totalTokens = 0;
-    let peakMemory = 0;
-    try {
-      const metrics = await this.orchestrator.getMetrics();
-      totalTokens = this.extractTokens(metrics);
-      peakMemory = this.extractMemory(metrics);
-    } catch {
-      // Metrics collection failed, use defaults
-    }
-
-    const completedAt = Date.now();
-
-    return {
-      taskId: task.id,
-      configuration: 'single',
-      totalTimeMs: completedAt - startTime,
-      timeToFirstActionMs: firstActionTime,
-      messageCount: 0, // No inter-agent communication
-      avgLatencyMs: 0,
-      latencyP50Ms: 0,
-      latencyP99Ms: 0,
-      coordinationRounds: 0,
-      agentCount: 1,
-      totalTokensUsed: totalTokens,
-      peakMemoryMb: peakMemory,
-      success,
-      completionRate: success ? 1.0 : 0.0,
-      errors: this.metrics.errors,
-      startedAt: startTime,
-      completedAt,
-    };
-  }
-
-  /**
-   * Build the task prompt for a single agent
-   */
-  private buildTaskPrompt(task: Task): string {
-    return `Complete this task entirely on your own:
-
-## Task
-${task.description}
-
-## Files to Work On
-${task.files.map((f) => `- ${f}`).join('\n')}
-
-## Success Criteria
-${task.expectedOutcome}
-
-## Instructions
-1. Analyze the task requirements
-2. Plan your approach
-3. Implement the solution
-4. Verify your work meets the success criteria
-
-When complete, send a message to Orchestrator:
-\`\`\`
-DONE: <brief summary of what you accomplished>
-\`\`\`
-
-If you encounter an error you cannot resolve:
-\`\`\`
-ERROR: <description of the problem>
-\`\`\``;
-  }
-}

package/packages/benchmark/src/runners/subagent.ts

@@ -1,240 +0,0 @@
-/**
- * Sub-Agent Runner
- *
- * Runs tasks using a lead agent that spawns and coordinates workers hierarchically.
- */
-
-import type { Task, RunResult, ConfigurationType } from '../types.js';
-import { ConfigurationRunner } from './base.js';
-
-/**
- * Runner for sub-agent (hierarchical) configuration
- */
-export class SubAgentRunner extends ConfigurationRunner {
-  private workerCount = 0;
-
-  get configurationType(): ConfigurationType {
-    return 'subagent';
-  }
-
-  async run(task: Task): Promise<RunResult> {
-    const startTime = Date.now();
-    let firstActionTime = 0;
-    let success = false;
-
-    this.resetMetrics();
-    this.workerCount = 0;
-    this.log(`Starting task: ${task.id}`);
-
-    try {
-      // Set up message monitoring before spawning
-      this.setupMessageMonitoring();
-
-      // Spawn lead agent that will delegate to workers
-      const leadResult = await this.orchestrator.spawn({
-        name: 'Lead',
-        cli: this.config.cli,
-        task: this.buildLeadPrompt(task),
-        cwd: this.config.cwd,
-      });
-
-      if (!leadResult.success) {
-        this.metrics.errors.push(leadResult.error || 'Lead spawn failed');
-        return this.buildFailedResult(task, startTime, this.metrics.errors);
-      }
-
-      this.metrics.spawnedAgents.push('Lead');
-      firstActionTime = Date.now() - startTime;
-      this.log(`Lead agent spawned in ${firstActionTime}ms`);
-
-      // Wait for lead to complete (including all worker coordination)
-      success = await this.waitForLeadCompletion(task);
-
-      if (success) {
-        this.log(`Task completed with ${this.workerCount} workers`);
-      } else {
-        this.log('Task failed or timed out');
-      }
-    } catch (err) {
-      this.metrics.errors.push((err as Error).message);
-    }
-
-    // Collect final metrics
-    let totalTokens = 0;
-    let peakMemory = 0;
-    try {
-      const metrics = await this.orchestrator.getMetrics();
-      totalTokens = this.extractTokens(metrics);
-      peakMemory = this.extractMemory(metrics);
-    } catch {
-      // Metrics collection failed
-    }
-
-    const completedAt = Date.now();
-    const agentCount = 1 + this.workerCount;
-
-    return {
-      taskId: task.id,
-      configuration: 'subagent',
-      totalTimeMs: completedAt - startTime,
-      timeToFirstActionMs: firstActionTime,
-      messageCount: this.metrics.messages,
-      avgLatencyMs:
-        this.metrics.latencies.length > 0
-          ? this.metrics.latencies.reduce((a, b) => a + b, 0) /
-            this.metrics.latencies.length
-          : 0,
-      latencyP50Ms: this.percentile(this.metrics.latencies, 50),
-      latencyP99Ms: this.percentile(this.metrics.latencies, 99),
-      coordinationRounds: Math.ceil(
-        this.metrics.messages / Math.max(1, this.workerCount)
-      ),
-      agentCount,
-      totalTokensUsed: totalTokens,
-      peakMemoryMb: peakMemory,
-      success,
-      completionRate: success ? 1.0 : this.workerCount > 0 ? 0.5 : 0.0,
-      errors: this.metrics.errors,
-      startedAt: startTime,
-      completedAt,
-    };
-  }
-
-  /**
-   * Set up message monitoring to track worker spawns and coordination
-   */
-  private setupMessageMonitoring(): void {
-    this.orchestrator.onMessage = (from, payload, id, meta) => {
-      this.metrics.messages++;
-
-      // Track when lead spawns workers
-      if (from === 'Lead') {
-        // Check for spawn patterns in messages
-        if (
-          payload.body.includes('->relay-file:spawn') ||
-          typeof payload.data?.name === 'string'
-        ) {
-          this.workerCount++;
-          const workerName =
-            (payload.data?.name as string) || `Worker${this.workerCount}`;
-          this.metrics.spawnedAgents.push(workerName);
-          this.log(`Worker spawned: ${workerName}`);
-        }
-      }
-
-      // Track latencies if we have sync metadata
-      if (meta?.sync?.correlationId) {
-        // This would require tracking send times, simplified here
-      }
-    };
-  }
-
-  /**
-   * Wait for the lead agent to signal completion
-   */
-  private waitForLeadCompletion(task: Task): Promise<boolean> {
-    const timeoutMs = task.timeoutMs || 300000;
-
-    return new Promise((resolve) => {
-      const timeout = setTimeout(() => {
-        this.metrics.errors.push('Timeout waiting for Lead completion');
-        resolve(false);
-      }, timeoutMs);
-
-      const originalHandler = this.orchestrator.onMessage;
-      this.orchestrator.onMessage = (from, payload, id, meta, originalTo) => {
-        // Call existing handler for tracking
-        if (originalHandler) {
-          originalHandler(from, payload, id, meta, originalTo);
-        }
-
-        // Check for completion from Lead
-        if (from === 'Lead' && payload.body.startsWith('DONE:')) {
-          clearTimeout(timeout);
-          resolve(true);
-        }
-
-        // Check for fatal errors
-        if (from === 'Lead' && payload.body.startsWith('ERROR:')) {
-          this.metrics.errors.push(payload.body);
-          clearTimeout(timeout);
-          resolve(false);
-        }
-      };
-    });
-  }
-
-  /**
-   * Build the task prompt for the lead agent
-   */
-  private buildLeadPrompt(task: Task): string {
-    const suggestedWorkers = this.suggestWorkerCount(task);
-
-    return `You are the Lead agent. Your job is to delegate subtasks to workers and coordinate their efforts.
-
-## Task
-${task.description}
-
-## Files to Work On
-${task.files.map((f) => `- ${f}`).join('\n')}
-
-## Success Criteria
-${task.expectedOutcome}
-
-## Your Responsibilities
-
-1. **Analyze the task** and break it into subtasks
-2. **Spawn workers** for each subtask (suggested: ${suggestedWorkers} workers)
-3. **Coordinate** their work and handle dependencies
-4. **Aggregate results** and verify success criteria are met
-
-## Spawning Workers
-
-Use the relay protocol to spawn workers:
-
-\`\`\`bash
-cat > $AGENT_RELAY_OUTBOX/spawn << 'EOF'
-KIND: spawn
-NAME: Worker1
-CLI: ${this.config.cli}
-
-<Task description for this worker>
-EOF
-\`\`\`
-Then output: ->relay-file:spawn
-
-## Communication Protocol
-
-- Workers will send you status updates
-- Monitor for "DONE:" messages from workers
-- Handle "BLOCKED:" or "ERROR:" messages by reassigning or helping
-
-## Completion
-
-When ALL subtasks are complete and verified:
-\`\`\`
-DONE: <summary of what was accomplished, including worker contributions>
-\`\`\`
-
-If you encounter an unrecoverable error:
-\`\`\`
-ERROR: <description of the problem>
-\`\`\``;
-  }
-
-  /**
-   * Suggest number of workers based on task complexity
-   */
-  private suggestWorkerCount(task: Task): number {
-    switch (task.complexity) {
-      case 'low':
-        return Math.min(2, task.files.length);
-      case 'medium':
-        return Math.min(3, task.files.length);
-      case 'high':
-        return Math.min(5, task.files.length, this.config.maxSwarmSize);
-      default:
-        return 2;
-    }
-  }
-}

package/packages/benchmark/src/runners/swarm.ts

@@ -1,326 +0,0 @@
-/**
- * Swarm Runner
- *
- * Runs tasks using peer agents that coordinate as equals via messaging.
- */
-
-import type { Task, RunResult, ConfigurationType } from '../types.js';
-import { ConfigurationRunner } from './base.js';
-
-/**
- * Runner for swarm (peer-to-peer) configuration
- */
-export class SwarmRunner extends ConfigurationRunner {
-  private completedPeers = new Set<string>();
-
-  get configurationType(): ConfigurationType {
-    return 'swarm';
-  }
-
-  async run(task: Task): Promise<RunResult> {
-    const startTime = Date.now();
-    let firstActionTime = 0;
-    let success = false;
-
-    this.resetMetrics();
-    this.completedPeers.clear();
-    this.log(`Starting task: ${task.id}`);
-
-    // Determine swarm size based on task complexity
-    const agentCount = this.determineSwarmSize(task);
-    const peerNames: string[] = [];
-
-    try {
-      // Set up channel monitoring
-      this.setupChannelMonitoring();
-
-      // Partition task among peers
-      const subtasks = this.partitionTask(task, agentCount);
-
-      // Spawn peer agents simultaneously
-      this.log(`Spawning ${agentCount} peer agents...`);
-      const spawnPromises = subtasks.map((subtask, i) =>
-        this.orchestrator.spawn({
-          name: `Peer${i}`,
-          cli: this.config.cli,
-          task: this.buildPeerPrompt(subtask, i, agentCount, task),
-          cwd: this.config.cwd,
-          team: 'swarm',
-        })
-      );
-
-      const results = await Promise.all(spawnPromises);
-      firstActionTime = Date.now() - startTime;
-      this.log(`Peers spawned in ${firstActionTime}ms`);
-
-      // Track successful spawns
-      for (let i = 0; i < results.length; i++) {
-        if (results[i].success) {
-          peerNames.push(`Peer${i}`);
-          this.metrics.spawnedAgents.push(`Peer${i}`);
-        } else {
-          this.metrics.errors.push(
-            `Peer${i} spawn failed: ${results[i].error}`
-          );
-        }
-      }
-
-      if (peerNames.length === 0) {
-        return this.buildFailedResult(task, startTime, this.metrics.errors);
-      }
-
-      // Have orchestrator join #swarm channel to monitor
-      this.orchestrator.joinChannel('#swarm');
-
-      // Wait for swarm consensus on completion
-      success = await this.waitForSwarmCompletion(peerNames, task);
-
-      if (success) {
-        this.log(
-          `Task completed by swarm of ${peerNames.length} peers`
-        );
-      } else {
-        this.log(
-          `Swarm completed ${this.completedPeers.size}/${peerNames.length} subtasks`
-        );
-      }
-    } catch (err) {
-      this.metrics.errors.push((err as Error).message);
-    }
-
-    // Collect final metrics
-    let totalTokens = 0;
-    let peakMemory = 0;
-    try {
-      const metrics = await this.orchestrator.getMetrics();
-      totalTokens = this.extractTokens(metrics);
-      peakMemory = this.extractMemory(metrics);
-    } catch {
-      // Metrics collection failed
-    }
-
-    const completedAt = Date.now();
-
-    return {
-      taskId: task.id,
-      configuration: 'swarm',
-      totalTimeMs: completedAt - startTime,
-      timeToFirstActionMs: firstActionTime,
-      messageCount: this.metrics.messages,
-      avgLatencyMs:
-        this.metrics.latencies.length > 0
-          ? this.metrics.latencies.reduce((a, b) => a + b, 0) /
-            this.metrics.latencies.length
-          : 0,
-      latencyP50Ms: this.percentile(this.metrics.latencies, 50),
-      latencyP99Ms: this.percentile(this.metrics.latencies, 99),
-      coordinationRounds: Math.ceil(
-        this.metrics.messages / Math.max(1, peerNames.length)
-      ),
-      agentCount: peerNames.length,
-      totalTokensUsed: totalTokens,
-      peakMemoryMb: peakMemory,
-      success,
-      completionRate: this.completedPeers.size / Math.max(1, peerNames.length),
-      errors: this.metrics.errors,
-      startedAt: startTime,
-      completedAt,
-    };
-  }
-
-  /**
-   * Set up channel message monitoring
-   */
-  private setupChannelMonitoring(): void {
-    this.orchestrator.onChannelMessage = (from, channel, body, envelope) => {
-      this.metrics.messages++;
-
-      // Track completion announcements
-      if (channel === '#swarm' && body.startsWith('DONE:')) {
-        this.completedPeers.add(from);
-        this.log(`${from} completed: ${body.substring(5).trim()}`);
-      }
-
-      // Track errors
-      if (body.startsWith('ERROR:') || body.startsWith('BLOCKED:')) {
-        this.metrics.errors.push(`${from}: ${body}`);
-      }
-    };
-
-    this.orchestrator.onMessage = (from, payload, id, meta, originalTo) => {
-      this.metrics.messages++;
-    };
-  }
-
-  /**
-   * Determine swarm size based on task complexity and files
-   */
-  private determineSwarmSize(task: Task): number {
-    let baseSize: number;
-    switch (task.complexity) {
-      case 'low':
-        baseSize = 2;
-        break;
-      case 'medium':
-        baseSize = 3;
-        break;
-      case 'high':
-        baseSize = 5;
-        break;
-      default:
-        baseSize = 3;
-    }
-
-    // Don't have more agents than files
-    const fileCount = task.files.length;
-    const size = Math.min(baseSize, fileCount, this.config.maxSwarmSize);
-
-    return Math.max(2, size); // At least 2 for swarm
-  }
-
-  /**
-   * Partition task files among agents
-   */
-  private partitionTask(task: Task, count: number): Task[] {
-    const filesPerAgent = Math.ceil(task.files.length / count);
-    const subtasks: Task[] = [];
-
-    for (let i = 0; i < count; i++) {
-      const start = i * filesPerAgent;
-      const files = task.files.slice(start, start + filesPerAgent);
-
-      // Handle case where we have more agents than files
-      if (files.length === 0) continue;
-
-      subtasks.push({
-        ...task,
-        id: `${task.id}-part${i}`,
-        files,
-      });
-    }
-
-    return subtasks;
-  }
-
-  /**
-   * Build prompt for a peer agent
-   */
-  private buildPeerPrompt(
-    subtask: Task,
-    index: number,
-    totalPeers: number,
-    fullTask: Task
-  ): string {
-    const otherPeers = Array.from({ length: totalPeers }, (_, i) => `Peer${i}`)
-      .filter((_, i) => i !== index)
-      .join(', ');
-
-    return `You are Peer${index} in a swarm of ${totalPeers} peer agents working together.
-
-## Your Subtask
-Work on these files: ${subtask.files.map((f) => `\`${f}\``).join(', ')}
-
-## Full Task Context
-${fullTask.description}
-
-## Success Criteria (for full task)
-${fullTask.expectedOutcome}
-
-## Peer Coordination
-
-You are working alongside: ${otherPeers}
-
-**Communication via #swarm channel:**
-
-Share your progress:
-\`\`\`bash
-cat > $AGENT_RELAY_OUTBOX/msg << 'EOF'
-TO: #swarm
-
-STATUS: Working on <what you're doing>
-EOF
-\`\`\`
-Then: ->relay-file:msg
-
-Ask questions to the swarm:
-\`\`\`bash
-cat > $AGENT_RELAY_OUTBOX/msg << 'EOF'
-TO: #swarm
-
-QUESTION: <your question>
-EOF
-\`\`\`
-Then: ->relay-file:msg
-
-Report blockers:
-\`\`\`bash
-cat > $AGENT_RELAY_OUTBOX/msg << 'EOF'
-TO: #swarm
-
-BLOCKED: <what's blocking you>
-EOF
-\`\`\`
-Then: ->relay-file:msg
-
-## Completion
-
-When YOUR part is complete:
-\`\`\`bash
-cat > $AGENT_RELAY_OUTBOX/msg << 'EOF'
-TO: #swarm
-
-DONE: Peer${index} completed <brief summary>
-EOF
-\`\`\`
-Then: ->relay-file:msg
-
-## Guidelines
-
-1. Focus on your assigned files
-2. Coordinate with peers on shared interfaces/dependencies
-3. Don't duplicate work - check the channel for updates
-4. Help peers if they're blocked and you can assist`;
-  }
-
-  /**
-   * Wait for all peers to complete
-   */
-  private waitForSwarmCompletion(
-    peers: string[],
-    task: Task
-  ): Promise<boolean> {
-    const timeoutMs = task.timeoutMs || 300000;
-
-    return new Promise((resolve) => {
-      const timeout = setTimeout(() => {
-        this.metrics.errors.push(
-          `Swarm timeout: ${this.completedPeers.size}/${peers.length} completed`
-        );
-        resolve(false);
-      }, timeoutMs);
-
-      // Check periodically if all peers are done
-      const checkInterval = setInterval(() => {
-        if (this.completedPeers.size >= peers.length) {
-          clearInterval(checkInterval);
-          clearTimeout(timeout);
-          resolve(true);
-        }
-      }, 1000);
-
-      // Also check on each channel message
-      const originalHandler = this.orchestrator.onChannelMessage;
-      this.orchestrator.onChannelMessage = (from, channel, body, envelope) => {
-        if (originalHandler) {
-          originalHandler(from, channel, body, envelope);
-        }
-
-        if (this.completedPeers.size >= peers.length) {
-          clearInterval(checkInterval);
-          clearTimeout(timeout);
-          resolve(true);
-        }
-      };
-    });
-  }
-}