agent-relay 2.3.13 → 2.4.0

package/packages/resiliency/src/gossip-health.ts

@@ -1,333 +0,0 @@
-/**
- * Gossip-Based Health Broadcast
- *
- * Implements P4: Agents broadcast heartbeats via relay.
- * Collective failure detection without central monitor.
- *
- * Each agent:
- * - Periodically broadcasts HEARTBEAT message to all agents
- * - Tracks health of all known peers
- * - Detects failures when peer heartbeats go stale
- * - Emits events for peer health changes
- */
-
-import { EventEmitter } from 'events';
-
-/**
- * Peer health state
- */
-export interface PeerHealth {
-  name: string;
-  lastHeartbeat: number;
-  load: number;
-  healthy: boolean;
-  isLeader: boolean;
-  taskCount: number;
-}
-
-/**
- * Heartbeat payload (broadcast via relay)
- */
-export interface GossipHeartbeat {
-  type: 'HEARTBEAT';
-  agent: string;
-  agentId: string;
-  timestamp: number;
-  load: number;
-  healthy: boolean;
-  isLeader: boolean;
-  taskCount: number;
-}
-
-/**
- * Gossip health configuration
- */
-export interface GossipHealthConfig {
-  /** This agent's name */
-  agentName: string;
-  /** This agent's unique ID */
-  agentId: string;
-  /** How often to broadcast heartbeat (ms) */
-  broadcastIntervalMs: number;
-  /** Peer considered stale after this duration (ms) */
-  staleThresholdMs: number;
-  /** How often to check for stale peers (ms) */
-  checkIntervalMs: number;
-  /** Callback to broadcast message to all agents */
-  broadcast: (message: string) => Promise<void>;
-  /** Callback to get current load (0-1) */
-  getLoad?: () => number;
-  /** Callback to get current task count */
-  getTaskCount?: () => number;
-  /** Callback to check if this agent is leader */
-  isLeader?: () => boolean;
-}
-
-const DEFAULT_CONFIG: Partial<GossipHealthConfig> = {
-  broadcastIntervalMs: 10000,
-  staleThresholdMs: 30000,
-  checkIntervalMs: 5000,
-};
-
-/**
- * Gossip Health Monitor
- *
- * Broadcasts heartbeats and tracks peer health via gossip protocol.
- */
-export class GossipHealthMonitor extends EventEmitter {
-  private config: GossipHealthConfig;
-  private peers = new Map<string, PeerHealth>();
-  private broadcastInterval?: ReturnType<typeof setInterval>;
-  private checkInterval?: ReturnType<typeof setInterval>;
-  private isRunning = false;
-  private healthy = true;
-
-  constructor(config: GossipHealthConfig) {
-    super();
-    this.config = { ...DEFAULT_CONFIG, ...config } as GossipHealthConfig;
-  }
-
-  /**
-   * Start gossip health monitoring
-   */
-  start(): void {
-    if (this.isRunning) return;
-    this.isRunning = true;
-
-    console.log(`[gossip] Started health broadcast for ${this.config.agentName}`);
-
-    // Start broadcasting heartbeats
-    this.broadcastInterval = setInterval(async () => {
-      try {
-        await this.broadcastHeartbeat();
-      } catch (err) {
-        console.error('[gossip] Broadcast error:', err);
-      }
-    }, this.config.broadcastIntervalMs);
-
-    // Start checking for stale peers
-    this.checkInterval = setInterval(() => {
-      this.checkStalePeers();
-    }, this.config.checkIntervalMs);
-
-    // Initial broadcast
-    this.broadcastHeartbeat().catch((err) => {
-      console.error('[gossip] Initial broadcast error:', err);
-    });
-
-    this.emit('started');
-  }
-
-  /**
-   * Stop gossip health monitoring
-   */
-  stop(): void {
-    if (!this.isRunning) return;
-    this.isRunning = false;
-
-    if (this.broadcastInterval) {
-      clearInterval(this.broadcastInterval);
-      this.broadcastInterval = undefined;
-    }
-
-    if (this.checkInterval) {
-      clearInterval(this.checkInterval);
-      this.checkInterval = undefined;
-    }
-
-    console.log(`[gossip] Stopped health broadcast for ${this.config.agentName}`);
-    this.emit('stopped');
-  }
-
-  /**
-   * Broadcast heartbeat to all agents
-   */
-  private async broadcastHeartbeat(): Promise<void> {
-    const heartbeat: GossipHeartbeat = {
-      type: 'HEARTBEAT',
-      agent: this.config.agentName,
-      agentId: this.config.agentId,
-      timestamp: Date.now(),
-      load: this.config.getLoad?.() ?? 0,
-      healthy: this.healthy,
-      isLeader: this.config.isLeader?.() ?? false,
-      taskCount: this.config.getTaskCount?.() ?? 0,
-    };
-
-    const message = `HEARTBEAT: ${JSON.stringify(heartbeat)}`;
-    await this.config.broadcast(message);
-  }
-
-  /**
-   * Process incoming heartbeat from another agent
-   */
-  processHeartbeat(heartbeat: GossipHeartbeat): void {
-    // Ignore our own heartbeats
-    if (heartbeat.agentId === this.config.agentId) return;
-
-    const existing = this.peers.get(heartbeat.agent);
-    const wasHealthy = existing?.healthy ?? true;
-    const wasLeader = existing?.isLeader ?? false;
-
-    // Update peer state
-    const peer: PeerHealth = {
-      name: heartbeat.agent,
-      lastHeartbeat: heartbeat.timestamp,
-      load: heartbeat.load,
-      healthy: heartbeat.healthy,
-      isLeader: heartbeat.isLeader,
-      taskCount: heartbeat.taskCount,
-    };
-
-    this.peers.set(heartbeat.agent, peer);
-
-    // Emit events for state changes
-    if (!existing) {
-      this.emit('peerDiscovered', peer);
-    } else {
-      if (!wasHealthy && heartbeat.healthy) {
-        this.emit('peerRecovered', peer);
-      }
-      if (wasHealthy && !heartbeat.healthy) {
-        this.emit('peerUnhealthy', peer);
-      }
-      if (!wasLeader && heartbeat.isLeader) {
-        this.emit('newLeader', peer);
-      }
-    }
-  }
-
-  /**
-   * Parse heartbeat from relay message
-   */
-  static parseHeartbeat(message: string): GossipHeartbeat | null {
-    const match = message.match(/^HEARTBEAT:\s*(.+)$/);
-    if (!match) return null;
-
-    try {
-      const data = JSON.parse(match[1]);
-      if (data.type === 'HEARTBEAT') {
-        return data as GossipHeartbeat;
-      }
-    } catch {
-      // Invalid JSON
-    }
-
-    return null;
-  }
-
-  /**
-   * Check for stale peers
-   */
-  private checkStalePeers(): void {
-    const now = Date.now();
-
-    for (const [name, peer] of this.peers) {
-      const age = now - peer.lastHeartbeat;
-
-      if (age > this.config.staleThresholdMs) {
-        if (peer.healthy) {
-          // Mark as unhealthy
-          peer.healthy = false;
-          console.log(`[gossip] Peer ${name} is stale (${Math.round(age / 1000)}s since last heartbeat)`);
-          this.emit('peerStale', { peer, age });
-
-          // If the stale peer was leader, emit leader lost
-          if (peer.isLeader) {
-            this.emit('leaderLost', peer);
-          }
-        }
-      }
-    }
-  }
-
-  /**
-   * Set this agent's health status
-   */
-  setHealthy(healthy: boolean): void {
-    if (this.healthy !== healthy) {
-      this.healthy = healthy;
-      // Broadcast immediately on health change
-      this.broadcastHeartbeat().catch((err) => {
-        console.error('[gossip] Immediate broadcast error:', err);
-      });
-    }
-  }
-
-  /**
-   * Get all known peers
-   */
-  getPeers(): PeerHealth[] {
-    return Array.from(this.peers.values());
-  }
-
-  /**
-   * Get healthy peers
-   */
-  getHealthyPeers(): PeerHealth[] {
-    return this.getPeers().filter((p) => p.healthy);
-  }
-
-  /**
-   * Get current leader from gossip
-   */
-  getLeader(): PeerHealth | null {
-    for (const peer of this.peers.values()) {
-      if (peer.isLeader && peer.healthy) {
-        return peer;
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Get peer by name
-   */
-  getPeer(name: string): PeerHealth | undefined {
-    return this.peers.get(name);
-  }
-
-  /**
-   * Get status
-   */
-  getStatus(): {
-    isRunning: boolean;
-    agentName: string;
-    peerCount: number;
-    healthyPeerCount: number;
-    leader: string | null;
-  } {
-    const leader = this.getLeader();
-    return {
-      isRunning: this.isRunning,
-      agentName: this.config.agentName,
-      peerCount: this.peers.size,
-      healthyPeerCount: this.getHealthyPeers().length,
-      leader: leader?.name ?? null,
-    };
-  }
-}
-
-/**
- * Create gossip health monitor with defaults
- */
-export function createGossipHealth(
-  agentName: string,
-  agentId: string,
-  broadcast: (message: string) => Promise<void>,
-  options?: {
-    getLoad?: () => number;
-    getTaskCount?: () => number;
-    isLeader?: () => boolean;
-  }
-): GossipHealthMonitor {
-  return new GossipHealthMonitor({
-    agentName,
-    agentId,
-    broadcast,
-    broadcastIntervalMs: 10000,
-    staleThresholdMs: 30000,
-    checkIntervalMs: 5000,
-    ...options,
-  });
-}

package/packages/resiliency/src/health-monitor.ts

@@ -1,371 +0,0 @@
-/**
- * Agent Health Monitor
- *
- * Monitors spawned agent processes and ensures they stay alive.
- * - Periodic health checks (process liveness)
- * - Auto-restart on crash
- * - Death detection and logging
- * - Metrics collection
- */
-
-import { EventEmitter } from 'events';
-
-export interface AgentHealth {
-  name: string;
-  pid: number;
-  status: 'healthy' | 'unresponsive' | 'dead' | 'restarting';
-  lastHealthCheck: Date;
-  lastResponse: Date;
-  restartCount: number;
-  consecutiveFailures: number;
-  uptime: number; // ms
-  startedAt: Date;
-  memoryUsage?: number;
-  cpuUsage?: number;
-  lastError?: string;
-}
-
-export interface HealthMonitorConfig {
-  checkIntervalMs: number; // How often to check health (default: 5000)
-  responseTimeoutMs: number; // Max time to wait for response (default: 10000)
-  maxRestarts: number; // Max restarts before giving up (default: 5)
-  restartCooldownMs: number; // Time between restarts (default: 2000)
-  maxConsecutiveFailures: number; // Failures before marking dead (default: 3)
-}
-
-export interface AgentProcess {
-  name: string;
-  pid: number;
-  isAlive: () => boolean;
-  kill: (signal?: string) => void;
-  restart: () => Promise<void>;
-  sendHealthCheck?: () => Promise<boolean>;
-}
-
-const DEFAULT_CONFIG: HealthMonitorConfig = {
-  checkIntervalMs: 5000,
-  responseTimeoutMs: 10000,
-  maxRestarts: 5,
-  restartCooldownMs: 2000,
-  maxConsecutiveFailures: 3,
-};
-
-export class AgentHealthMonitor extends EventEmitter {
-  private agents = new Map<string, AgentProcess>();
-  private health = new Map<string, AgentHealth>();
-  private intervalId?: ReturnType<typeof setInterval>;
-  private config: HealthMonitorConfig;
-  private isRunning = false;
-
-  constructor(config: Partial<HealthMonitorConfig> = {}) {
-    super();
-    this.config = { ...DEFAULT_CONFIG, ...config };
-  }
-
-  /**
-   * Register an agent for health monitoring
-   */
-  register(agent: AgentProcess): void {
-    this.agents.set(agent.name, agent);
-    this.health.set(agent.name, {
-      name: agent.name,
-      pid: agent.pid,
-      status: 'healthy',
-      lastHealthCheck: new Date(),
-      lastResponse: new Date(),
-      restartCount: 0,
-      consecutiveFailures: 0,
-      uptime: 0,
-      startedAt: new Date(),
-    });
-
-    this.emit('registered', { name: agent.name, pid: agent.pid });
-    this.log('info', `Registered agent for health monitoring: ${agent.name} (PID: ${agent.pid})`);
-  }
-
-  /**
-   * Unregister an agent from health monitoring
-   */
-  unregister(name: string): void {
-    this.agents.delete(name);
-    this.health.delete(name);
-    this.emit('unregistered', { name });
-    this.log('info', `Unregistered agent: ${name}`);
-  }
-
-  /**
-   * Start the health monitoring loop
-   */
-  start(): void {
-    if (this.isRunning) return;
-    this.isRunning = true;
-
-    this.log('info', 'Health monitor started', {
-      checkInterval: this.config.checkIntervalMs,
-      maxRestarts: this.config.maxRestarts,
-    });
-
-    this.intervalId = setInterval(() => {
-      this.checkAll();
-    }, this.config.checkIntervalMs);
-
-    // Initial check
-    this.checkAll();
-  }
-
-  /**
-   * Stop the health monitoring loop
-   */
-  stop(): void {
-    if (this.intervalId) {
-      clearInterval(this.intervalId);
-      this.intervalId = undefined;
-    }
-    this.isRunning = false;
-    this.log('info', 'Health monitor stopped');
-  }
-
-  /**
-   * Get health status for all agents
-   */
-  getAll(): AgentHealth[] {
-    return Array.from(this.health.values()).map((h) => ({
-      ...h,
-      uptime: Date.now() - h.startedAt.getTime(),
-    }));
-  }
-
-  /**
-   * Get health status for a specific agent
-   */
-  get(name: string): AgentHealth | undefined {
-    const health = this.health.get(name);
-    if (health) {
-      return { ...health, uptime: Date.now() - health.startedAt.getTime() };
-    }
-    return undefined;
-  }
-
-  /**
-   * Check health of all registered agents
-   */
-  private async checkAll(): Promise<void> {
-    const checks = Array.from(this.agents.entries()).map(([name, agent]) =>
-      this.checkAgent(name, agent)
-    );
-    await Promise.all(checks);
-  }
-
-  /**
-   * Check health of a single agent
-   */
-  private async checkAgent(name: string, agent: AgentProcess): Promise<void> {
-    const health = this.health.get(name);
-    if (!health) return;
-
-    health.lastHealthCheck = new Date();
-
-    try {
-      // First check: Is the process alive?
-      const isAlive = this.isProcessAlive(agent.pid);
-
-      if (!isAlive) {
-        await this.handleDeath(name, agent, health, 'Process not found');
-        return;
-      }
-
-      // Second check: Does it respond to health check?
-      if (agent.sendHealthCheck) {
-        const responded = await Promise.race([
-          agent.sendHealthCheck(),
-          new Promise<false>((resolve) =>
-            setTimeout(() => resolve(false), this.config.responseTimeoutMs)
-          ),
-        ]);
-
-        if (!responded) {
-          health.consecutiveFailures++;
-          this.log('warn', `Agent unresponsive: ${name}`, {
-            failures: health.consecutiveFailures,
-            max: this.config.maxConsecutiveFailures,
-          });
-
-          if (health.consecutiveFailures >= this.config.maxConsecutiveFailures) {
-            health.status = 'unresponsive';
-            await this.handleDeath(name, agent, health, 'Unresponsive after multiple health checks');
-          } else {
-            health.status = 'unresponsive';
-            this.emit('unhealthy', { name, health });
-          }
-          return;
-        }
-      }
-
-      // Get memory/CPU usage if available
-      try {
-        const usage = await this.getProcessUsage(agent.pid);
-        health.memoryUsage = usage.memory;
-        health.cpuUsage = usage.cpu;
-      } catch {
-        // Ignore usage errors
-      }
-
-      // All good
-      health.status = 'healthy';
-      health.lastResponse = new Date();
-      health.consecutiveFailures = 0;
-
-      this.emit('healthy', { name, health });
-    } catch (error) {
-      health.consecutiveFailures++;
-      health.lastError = error instanceof Error ? error.message : String(error);
-      this.log('error', `Health check error for ${name}`, { error: health.lastError });
-
-      if (health.consecutiveFailures >= this.config.maxConsecutiveFailures) {
-        await this.handleDeath(name, agent, health, health.lastError);
-      }
-    }
-  }
-
-  /**
-   * Handle agent death - attempt restart or mark as dead
-   */
-  private async handleDeath(
-    name: string,
-    agent: AgentProcess,
-    health: AgentHealth,
-    reason: string
-  ): Promise<void> {
-    this.log('error', `Agent died: ${name}`, {
-      reason,
-      restartCount: health.restartCount,
-      maxRestarts: this.config.maxRestarts,
-    });
-
-    this.emit('died', { name, reason, restartCount: health.restartCount });
-
-    // Check if we should attempt restart
-    if (health.restartCount >= this.config.maxRestarts) {
-      health.status = 'dead';
-      health.lastError = `Exceeded max restarts (${this.config.maxRestarts}): ${reason}`;
-      this.log('error', `Agent permanently dead: ${name}`, { reason: health.lastError });
-      this.emit('permanentlyDead', { name, health });
-      return;
-    }
-
-    // Attempt restart
-    health.status = 'restarting';
-    health.restartCount++;
-
-    this.log('info', `Attempting restart ${health.restartCount}/${this.config.maxRestarts}: ${name}`);
-    this.emit('restarting', { name, attempt: health.restartCount });
-
-    // Wait cooldown
-    await new Promise((resolve) => setTimeout(resolve, this.config.restartCooldownMs));
-
-    try {
-      await agent.restart();
-
-      // Update health after successful restart
-      health.status = 'healthy';
-      health.consecutiveFailures = 0;
-      health.startedAt = new Date();
-      health.lastResponse = new Date();
-      health.pid = agent.pid;
-
-      this.log('info', `Agent restarted successfully: ${name}`, {
-        newPid: agent.pid,
-        attempt: health.restartCount,
-      });
-
-      this.emit('restarted', { name, pid: agent.pid, attempt: health.restartCount });
-    } catch (error) {
-      health.lastError = error instanceof Error ? error.message : String(error);
-      this.log('error', `Restart failed: ${name}`, { error: health.lastError });
-      this.emit('restartFailed', { name, error: health.lastError });
-
-      // Recursively try again if under limit
-      if (health.restartCount < this.config.maxRestarts) {
-        await this.handleDeath(name, agent, health, health.lastError);
-      } else {
-        health.status = 'dead';
-        this.emit('permanentlyDead', { name, health });
-      }
-    }
-  }
-
-  /**
-   * Check if a process is alive by PID
-   */
-  private isProcessAlive(pid: number): boolean {
-    try {
-      // Sending signal 0 checks if process exists without killing it
-      process.kill(pid, 0);
-      return true;
-    } catch {
-      return false;
-    }
-  }
-
-  /**
-   * Get memory and CPU usage for a process
-   */
-  private async getProcessUsage(pid: number): Promise<{ memory: number; cpu: number }> {
-    const { execSync } = await import('child_process');
-
-    try {
-      // This works on Linux/Mac
-      const output = execSync(`ps -o rss=,pcpu= -p ${pid}`, { encoding: 'utf8' }).trim();
-      const [rss, cpu] = output.split(/\s+/);
-      return {
-        memory: parseInt(rss, 10) * 1024, // RSS in bytes
-        cpu: parseFloat(cpu),
-      };
-    } catch {
-      return { memory: 0, cpu: 0 };
-    }
-  }
-
-  /**
-   * Structured logging
-   */
-  private log(
-    level: 'info' | 'warn' | 'error',
-    message: string,
-    context?: Record<string, unknown>
-  ): void {
-    const entry = {
-      timestamp: new Date().toISOString(),
-      level,
-      component: 'health-monitor',
-      message,
-      ...context,
-    };
-
-    this.emit('log', entry);
-
-    // Also log to console with structure
-    const prefix = `[health-monitor]`;
-    switch (level) {
-      case 'info':
-        console.log(prefix, message, context ? JSON.stringify(context) : '');
-        break;
-      case 'warn':
-        console.warn(prefix, message, context ? JSON.stringify(context) : '');
-        break;
-      case 'error':
-        console.error(prefix, message, context ? JSON.stringify(context) : '');
-        break;
-    }
-  }
-}
-
-// Singleton instance
-let _monitor: AgentHealthMonitor | null = null;
-
-export function getHealthMonitor(config?: Partial<HealthMonitorConfig>): AgentHealthMonitor {
-  if (!_monitor) {
-    _monitor = new AgentHealthMonitor(config);
-  }
-  return _monitor;
-}