agent-relay 2.0.29 → 2.0.32

package/packages/resiliency/src/gossip-health.ts

@@ -0,0 +1,333 @@
+/**
+ * Gossip-Based Health Broadcast
+ *
+ * Implements P4: Agents broadcast heartbeats via relay.
+ * Collective failure detection without central monitor.
+ *
+ * Each agent:
+ * - Periodically broadcasts HEARTBEAT message to all agents
+ * - Tracks health of all known peers
+ * - Detects failures when peer heartbeats go stale
+ * - Emits events for peer health changes
+ */
+
+import { EventEmitter } from 'events';
+
+/**
+ * Peer health state
+ */
+export interface PeerHealth {
+  name: string;
+  lastHeartbeat: number;
+  load: number;
+  healthy: boolean;
+  isLeader: boolean;
+  taskCount: number;
+}
+
+/**
+ * Heartbeat payload (broadcast via relay)
+ */
+export interface GossipHeartbeat {
+  type: 'HEARTBEAT';
+  agent: string;
+  agentId: string;
+  timestamp: number;
+  load: number;
+  healthy: boolean;
+  isLeader: boolean;
+  taskCount: number;
+}
+
+/**
+ * Gossip health configuration
+ */
+export interface GossipHealthConfig {
+  /** This agent's name */
+  agentName: string;
+  /** This agent's unique ID */
+  agentId: string;
+  /** How often to broadcast heartbeat (ms) */
+  broadcastIntervalMs: number;
+  /** Peer considered stale after this duration (ms) */
+  staleThresholdMs: number;
+  /** How often to check for stale peers (ms) */
+  checkIntervalMs: number;
+  /** Callback to broadcast message to all agents */
+  broadcast: (message: string) => Promise<void>;
+  /** Callback to get current load (0-1) */
+  getLoad?: () => number;
+  /** Callback to get current task count */
+  getTaskCount?: () => number;
+  /** Callback to check if this agent is leader */
+  isLeader?: () => boolean;
+}
+
+const DEFAULT_CONFIG: Partial<GossipHealthConfig> = {
+  broadcastIntervalMs: 10000,
+  staleThresholdMs: 30000,
+  checkIntervalMs: 5000,
+};
+
+/**
+ * Gossip Health Monitor
+ *
+ * Broadcasts heartbeats and tracks peer health via gossip protocol.
+ */
+export class GossipHealthMonitor extends EventEmitter {
+  private config: GossipHealthConfig;
+  private peers = new Map<string, PeerHealth>();
+  private broadcastInterval?: ReturnType<typeof setInterval>;
+  private checkInterval?: ReturnType<typeof setInterval>;
+  private isRunning = false;
+  private healthy = true;
+
+  constructor(config: GossipHealthConfig) {
+    super();
+    this.config = { ...DEFAULT_CONFIG, ...config } as GossipHealthConfig;
+  }
+
+  /**
+   * Start gossip health monitoring
+   */
+  start(): void {
+    if (this.isRunning) return;
+    this.isRunning = true;
+
+    console.log(`[gossip] Started health broadcast for ${this.config.agentName}`);
+
+    // Start broadcasting heartbeats
+    this.broadcastInterval = setInterval(async () => {
+      try {
+        await this.broadcastHeartbeat();
+      } catch (err) {
+        console.error('[gossip] Broadcast error:', err);
+      }
+    }, this.config.broadcastIntervalMs);
+
+    // Start checking for stale peers
+    this.checkInterval = setInterval(() => {
+      this.checkStalePeers();
+    }, this.config.checkIntervalMs);
+
+    // Initial broadcast
+    this.broadcastHeartbeat().catch((err) => {
+      console.error('[gossip] Initial broadcast error:', err);
+    });
+
+    this.emit('started');
+  }
+
+  /**
+   * Stop gossip health monitoring
+   */
+  stop(): void {
+    if (!this.isRunning) return;
+    this.isRunning = false;
+
+    if (this.broadcastInterval) {
+      clearInterval(this.broadcastInterval);
+      this.broadcastInterval = undefined;
+    }
+
+    if (this.checkInterval) {
+      clearInterval(this.checkInterval);
+      this.checkInterval = undefined;
+    }
+
+    console.log(`[gossip] Stopped health broadcast for ${this.config.agentName}`);
+    this.emit('stopped');
+  }
+
+  /**
+   * Broadcast heartbeat to all agents
+   */
+  private async broadcastHeartbeat(): Promise<void> {
+    const heartbeat: GossipHeartbeat = {
+      type: 'HEARTBEAT',
+      agent: this.config.agentName,
+      agentId: this.config.agentId,
+      timestamp: Date.now(),
+      load: this.config.getLoad?.() ?? 0,
+      healthy: this.healthy,
+      isLeader: this.config.isLeader?.() ?? false,
+      taskCount: this.config.getTaskCount?.() ?? 0,
+    };
+
+    const message = `HEARTBEAT: ${JSON.stringify(heartbeat)}`;
+    await this.config.broadcast(message);
+  }
+
+  /**
+   * Process incoming heartbeat from another agent
+   */
+  processHeartbeat(heartbeat: GossipHeartbeat): void {
+    // Ignore our own heartbeats
+    if (heartbeat.agentId === this.config.agentId) return;
+
+    const existing = this.peers.get(heartbeat.agent);
+    const wasHealthy = existing?.healthy ?? true;
+    const wasLeader = existing?.isLeader ?? false;
+
+    // Update peer state
+    const peer: PeerHealth = {
+      name: heartbeat.agent,
+      lastHeartbeat: heartbeat.timestamp,
+      load: heartbeat.load,
+      healthy: heartbeat.healthy,
+      isLeader: heartbeat.isLeader,
+      taskCount: heartbeat.taskCount,
+    };
+
+    this.peers.set(heartbeat.agent, peer);
+
+    // Emit events for state changes
+    if (!existing) {
+      this.emit('peerDiscovered', peer);
+    } else {
+      if (!wasHealthy && heartbeat.healthy) {
+        this.emit('peerRecovered', peer);
+      }
+      if (wasHealthy && !heartbeat.healthy) {
+        this.emit('peerUnhealthy', peer);
+      }
+      if (!wasLeader && heartbeat.isLeader) {
+        this.emit('newLeader', peer);
+      }
+    }
+  }
+
+  /**
+   * Parse heartbeat from relay message
+   */
+  static parseHeartbeat(message: string): GossipHeartbeat | null {
+    const match = message.match(/^HEARTBEAT:\s*(.+)$/);
+    if (!match) return null;
+
+    try {
+      const data = JSON.parse(match[1]);
+      if (data.type === 'HEARTBEAT') {
+        return data as GossipHeartbeat;
+      }
+    } catch {
+      // Invalid JSON
+    }
+
+    return null;
+  }
+
+  /**
+   * Check for stale peers
+   */
+  private checkStalePeers(): void {
+    const now = Date.now();
+
+    for (const [name, peer] of this.peers) {
+      const age = now - peer.lastHeartbeat;
+
+      if (age > this.config.staleThresholdMs) {
+        if (peer.healthy) {
+          // Mark as unhealthy
+          peer.healthy = false;
+          console.log(`[gossip] Peer ${name} is stale (${Math.round(age / 1000)}s since last heartbeat)`);
+          this.emit('peerStale', { peer, age });
+
+          // If the stale peer was leader, emit leader lost
+          if (peer.isLeader) {
+            this.emit('leaderLost', peer);
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Set this agent's health status
+   */
+  setHealthy(healthy: boolean): void {
+    if (this.healthy !== healthy) {
+      this.healthy = healthy;
+      // Broadcast immediately on health change
+      this.broadcastHeartbeat().catch((err) => {
+        console.error('[gossip] Immediate broadcast error:', err);
+      });
+    }
+  }
+
+  /**
+   * Get all known peers
+   */
+  getPeers(): PeerHealth[] {
+    return Array.from(this.peers.values());
+  }
+
+  /**
+   * Get healthy peers
+   */
+  getHealthyPeers(): PeerHealth[] {
+    return this.getPeers().filter((p) => p.healthy);
+  }
+
+  /**
+   * Get current leader from gossip
+   */
+  getLeader(): PeerHealth | null {
+    for (const peer of this.peers.values()) {
+      if (peer.isLeader && peer.healthy) {
+        return peer;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Get peer by name
+   */
+  getPeer(name: string): PeerHealth | undefined {
+    return this.peers.get(name);
+  }
+
+  /**
+   * Get status
+   */
+  getStatus(): {
+    isRunning: boolean;
+    agentName: string;
+    peerCount: number;
+    healthyPeerCount: number;
+    leader: string | null;
+  } {
+    const leader = this.getLeader();
+    return {
+      isRunning: this.isRunning,
+      agentName: this.config.agentName,
+      peerCount: this.peers.size,
+      healthyPeerCount: this.getHealthyPeers().length,
+      leader: leader?.name ?? null,
+    };
+  }
+}
+
+/**
+ * Create gossip health monitor with defaults
+ */
+export function createGossipHealth(
+  agentName: string,
+  agentId: string,
+  broadcast: (message: string) => Promise<void>,
+  options?: {
+    getLoad?: () => number;
+    getTaskCount?: () => number;
+    isLeader?: () => boolean;
+  }
+): GossipHealthMonitor {
+  return new GossipHealthMonitor({
+    agentName,
+    agentId,
+    broadcast,
+    broadcastIntervalMs: 10000,
+    staleThresholdMs: 30000,
+    checkIntervalMs: 5000,
+    ...options,
+  });
+}

package/packages/resiliency/src/health-monitor.ts

@@ -0,0 +1,371 @@
+/**
+ * Agent Health Monitor
+ *
+ * Monitors spawned agent processes and ensures they stay alive.
+ * - Periodic health checks (process liveness)
+ * - Auto-restart on crash
+ * - Death detection and logging
+ * - Metrics collection
+ */
+
+import { EventEmitter } from 'events';
+
+export interface AgentHealth {
+  name: string;
+  pid: number;
+  status: 'healthy' | 'unresponsive' | 'dead' | 'restarting';
+  lastHealthCheck: Date;
+  lastResponse: Date;
+  restartCount: number;
+  consecutiveFailures: number;
+  uptime: number; // ms
+  startedAt: Date;
+  memoryUsage?: number;
+  cpuUsage?: number;
+  lastError?: string;
+}
+
+export interface HealthMonitorConfig {
+  checkIntervalMs: number; // How often to check health (default: 5000)
+  responseTimeoutMs: number; // Max time to wait for response (default: 10000)
+  maxRestarts: number; // Max restarts before giving up (default: 5)
+  restartCooldownMs: number; // Time between restarts (default: 2000)
+  maxConsecutiveFailures: number; // Failures before marking dead (default: 3)
+}
+
+export interface AgentProcess {
+  name: string;
+  pid: number;
+  isAlive: () => boolean;
+  kill: (signal?: string) => void;
+  restart: () => Promise<void>;
+  sendHealthCheck?: () => Promise<boolean>;
+}
+
+const DEFAULT_CONFIG: HealthMonitorConfig = {
+  checkIntervalMs: 5000,
+  responseTimeoutMs: 10000,
+  maxRestarts: 5,
+  restartCooldownMs: 2000,
+  maxConsecutiveFailures: 3,
+};
+
+export class AgentHealthMonitor extends EventEmitter {
+  private agents = new Map<string, AgentProcess>();
+  private health = new Map<string, AgentHealth>();
+  private intervalId?: ReturnType<typeof setInterval>;
+  private config: HealthMonitorConfig;
+  private isRunning = false;
+
+  constructor(config: Partial<HealthMonitorConfig> = {}) {
+    super();
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Register an agent for health monitoring
+   */
+  register(agent: AgentProcess): void {
+    this.agents.set(agent.name, agent);
+    this.health.set(agent.name, {
+      name: agent.name,
+      pid: agent.pid,
+      status: 'healthy',
+      lastHealthCheck: new Date(),
+      lastResponse: new Date(),
+      restartCount: 0,
+      consecutiveFailures: 0,
+      uptime: 0,
+      startedAt: new Date(),
+    });
+
+    this.emit('registered', { name: agent.name, pid: agent.pid });
+    this.log('info', `Registered agent for health monitoring: ${agent.name} (PID: ${agent.pid})`);
+  }
+
+  /**
+   * Unregister an agent from health monitoring
+   */
+  unregister(name: string): void {
+    this.agents.delete(name);
+    this.health.delete(name);
+    this.emit('unregistered', { name });
+    this.log('info', `Unregistered agent: ${name}`);
+  }
+
+  /**
+   * Start the health monitoring loop
+   */
+  start(): void {
+    if (this.isRunning) return;
+    this.isRunning = true;
+
+    this.log('info', 'Health monitor started', {
+      checkInterval: this.config.checkIntervalMs,
+      maxRestarts: this.config.maxRestarts,
+    });
+
+    this.intervalId = setInterval(() => {
+      this.checkAll();
+    }, this.config.checkIntervalMs);
+
+    // Initial check
+    this.checkAll();
+  }
+
+  /**
+   * Stop the health monitoring loop
+   */
+  stop(): void {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = undefined;
+    }
+    this.isRunning = false;
+    this.log('info', 'Health monitor stopped');
+  }
+
+  /**
+   * Get health status for all agents
+   */
+  getAll(): AgentHealth[] {
+    return Array.from(this.health.values()).map((h) => ({
+      ...h,
+      uptime: Date.now() - h.startedAt.getTime(),
+    }));
+  }
+
+  /**
+   * Get health status for a specific agent
+   */
+  get(name: string): AgentHealth | undefined {
+    const health = this.health.get(name);
+    if (health) {
+      return { ...health, uptime: Date.now() - health.startedAt.getTime() };
+    }
+    return undefined;
+  }
+
+  /**
+   * Check health of all registered agents
+   */
+  private async checkAll(): Promise<void> {
+    const checks = Array.from(this.agents.entries()).map(([name, agent]) =>
+      this.checkAgent(name, agent)
+    );
+    await Promise.all(checks);
+  }
+
+  /**
+   * Check health of a single agent
+   */
+  private async checkAgent(name: string, agent: AgentProcess): Promise<void> {
+    const health = this.health.get(name);
+    if (!health) return;
+
+    health.lastHealthCheck = new Date();
+
+    try {
+      // First check: Is the process alive?
+      const isAlive = this.isProcessAlive(agent.pid);
+
+      if (!isAlive) {
+        await this.handleDeath(name, agent, health, 'Process not found');
+        return;
+      }
+
+      // Second check: Does it respond to health check?
+      if (agent.sendHealthCheck) {
+        const responded = await Promise.race([
+          agent.sendHealthCheck(),
+          new Promise<false>((resolve) =>
+            setTimeout(() => resolve(false), this.config.responseTimeoutMs)
+          ),
+        ]);
+
+        if (!responded) {
+          health.consecutiveFailures++;
+          this.log('warn', `Agent unresponsive: ${name}`, {
+            failures: health.consecutiveFailures,
+            max: this.config.maxConsecutiveFailures,
+          });
+
+          if (health.consecutiveFailures >= this.config.maxConsecutiveFailures) {
+            health.status = 'unresponsive';
+            await this.handleDeath(name, agent, health, 'Unresponsive after multiple health checks');
+          } else {
+            health.status = 'unresponsive';
+            this.emit('unhealthy', { name, health });
+          }
+          return;
+        }
+      }
+
+      // Get memory/CPU usage if available
+      try {
+        const usage = await this.getProcessUsage(agent.pid);
+        health.memoryUsage = usage.memory;
+        health.cpuUsage = usage.cpu;
+      } catch {
+        // Ignore usage errors
+      }
+
+      // All good
+      health.status = 'healthy';
+      health.lastResponse = new Date();
+      health.consecutiveFailures = 0;
+
+      this.emit('healthy', { name, health });
+    } catch (error) {
+      health.consecutiveFailures++;
+      health.lastError = error instanceof Error ? error.message : String(error);
+      this.log('error', `Health check error for ${name}`, { error: health.lastError });
+
+      if (health.consecutiveFailures >= this.config.maxConsecutiveFailures) {
+        await this.handleDeath(name, agent, health, health.lastError);
+      }
+    }
+  }
+
+  /**
+   * Handle agent death - attempt restart or mark as dead
+   */
+  private async handleDeath(
+    name: string,
+    agent: AgentProcess,
+    health: AgentHealth,
+    reason: string
+  ): Promise<void> {
+    this.log('error', `Agent died: ${name}`, {
+      reason,
+      restartCount: health.restartCount,
+      maxRestarts: this.config.maxRestarts,
+    });
+
+    this.emit('died', { name, reason, restartCount: health.restartCount });
+
+    // Check if we should attempt restart
+    if (health.restartCount >= this.config.maxRestarts) {
+      health.status = 'dead';
+      health.lastError = `Exceeded max restarts (${this.config.maxRestarts}): ${reason}`;
+      this.log('error', `Agent permanently dead: ${name}`, { reason: health.lastError });
+      this.emit('permanentlyDead', { name, health });
+      return;
+    }
+
+    // Attempt restart
+    health.status = 'restarting';
+    health.restartCount++;
+
+    this.log('info', `Attempting restart ${health.restartCount}/${this.config.maxRestarts}: ${name}`);
+    this.emit('restarting', { name, attempt: health.restartCount });
+
+    // Wait cooldown
+    await new Promise((resolve) => setTimeout(resolve, this.config.restartCooldownMs));
+
+    try {
+      await agent.restart();
+
+      // Update health after successful restart
+      health.status = 'healthy';
+      health.consecutiveFailures = 0;
+      health.startedAt = new Date();
+      health.lastResponse = new Date();
+      health.pid = agent.pid;
+
+      this.log('info', `Agent restarted successfully: ${name}`, {
+        newPid: agent.pid,
+        attempt: health.restartCount,
+      });
+
+      this.emit('restarted', { name, pid: agent.pid, attempt: health.restartCount });
+    } catch (error) {
+      health.lastError = error instanceof Error ? error.message : String(error);
+      this.log('error', `Restart failed: ${name}`, { error: health.lastError });
+      this.emit('restartFailed', { name, error: health.lastError });
+
+      // Recursively try again if under limit
+      if (health.restartCount < this.config.maxRestarts) {
+        await this.handleDeath(name, agent, health, health.lastError);
+      } else {
+        health.status = 'dead';
+        this.emit('permanentlyDead', { name, health });
+      }
+    }
+  }
+
+  /**
+   * Check if a process is alive by PID
+   */
+  private isProcessAlive(pid: number): boolean {
+    try {
+      // Sending signal 0 checks if process exists without killing it
+      process.kill(pid, 0);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Get memory and CPU usage for a process
+   */
+  private async getProcessUsage(pid: number): Promise<{ memory: number; cpu: number }> {
+    const { execSync } = await import('child_process');
+
+    try {
+      // This works on Linux/Mac
+      const output = execSync(`ps -o rss=,pcpu= -p ${pid}`, { encoding: 'utf8' }).trim();
+      const [rss, cpu] = output.split(/\s+/);
+      return {
+        memory: parseInt(rss, 10) * 1024, // RSS in bytes
+        cpu: parseFloat(cpu),
+      };
+    } catch {
+      return { memory: 0, cpu: 0 };
+    }
+  }
+
+  /**
+   * Structured logging
+   */
+  private log(
+    level: 'info' | 'warn' | 'error',
+    message: string,
+    context?: Record<string, unknown>
+  ): void {
+    const entry = {
+      timestamp: new Date().toISOString(),
+      level,
+      component: 'health-monitor',
+      message,
+      ...context,
+    };
+
+    this.emit('log', entry);
+
+    // Also log to console with structure
+    const prefix = `[health-monitor]`;
+    switch (level) {
+      case 'info':
+        console.log(prefix, message, context ? JSON.stringify(context) : '');
+        break;
+      case 'warn':
+        console.warn(prefix, message, context ? JSON.stringify(context) : '');
+        break;
+      case 'error':
+        console.error(prefix, message, context ? JSON.stringify(context) : '');
+        break;
+    }
+  }
+}
+
+// Singleton instance
+let _monitor: AgentHealthMonitor | null = null;
+
+export function getHealthMonitor(config?: Partial<HealthMonitorConfig>): AgentHealthMonitor {
+  if (!_monitor) {
+    _monitor = new AgentHealthMonitor(config);
+  }
+  return _monitor;
+}