agent-relay 2.0.28 → 2.0.32

package/packages/resiliency/src/index.ts

@@ -0,0 +1,157 @@
+/**
+ * Agent Resiliency Module
+ *
+ * Provides comprehensive health monitoring, auto-restart, logging,
+ * metrics, and context persistence for agent-relay agents.
+ *
+ * Features:
+ * - Health monitoring with process liveness checks
+ * - Auto-restart on crash with configurable limits
+ * - Prometheus-compatible metrics export
+ * - Structured JSON logging with rotation
+ * - Context persistence across restarts (inspired by Continuous-Claude-v2)
+ * - Provider-specific context injection (Claude hooks, Codex config, Gemini instructions)
+ *
+ * Usage:
+ *
+ * ```ts
+ * import { getSupervisor, metrics, createLogger } from './resiliency';
+ *
+ * // Start the supervisor with context persistence
+ * const supervisor = getSupervisor({
+ *   autoRestart: true,
+ *   maxRestarts: 5,
+ *   contextPersistence: {
+ *     enabled: true,
+ *     autoInjectOnRestart: true,
+ *   },
+ * });
+ * supervisor.start();
+ *
+ * // Add an agent to supervision
+ * supervisor.supervise(
+ *   {
+ *     name: 'worker-1',
+ *     cli: 'claude',
+ *     pid: 12345,
+ *     spawnedAt: new Date(),
+ *     workingDir: '/path/to/repo',
+ *     provider: 'claude', // or 'codex', 'gemini'
+ *   },
+ *   {
+ *     isAlive: () => process.kill(12345, 0),
+ *     kill: (sig) => process.kill(12345, sig),
+ *     restart: async () => { ... },
+ *   }
+ * );
+ *
+ * // Get metrics
+ * console.log(metrics.toPrometheus());
+ * ```
+ *
+ * Context persistence works differently per provider:
+ * - Claude: Uses hooks to inject context into CLAUDE.md
+ * - Codex: Uses config for periodic context refresh via system prompt
+ * - Gemini: Updates system instruction file
+ */
+
+export {
+  AgentHealthMonitor,
+  getHealthMonitor,
+  type AgentHealth,
+  type AgentProcess,
+  type HealthMonitorConfig,
+} from './health-monitor.js';
+
+export {
+  Logger,
+  createLogger,
+  configure as configureLogging,
+  loggers,
+  type LogLevel,
+  type LogEntry,
+  type LoggerConfig,
+} from './logger.js';
+
+export { metrics, type AgentMetrics, type SystemMetrics, type MetricPoint } from './metrics.js';
+
+export {
+  AgentSupervisor,
+  getSupervisor,
+  type SupervisedAgent,
+  type SupervisorConfig,
+} from './supervisor.js';
+
+export {
+  ContextPersistence,
+  getContextPersistence,
+  type AgentState,
+  type Decision,
+  type Artifact,
+  type Handoff,
+  type LedgerEntry,
+} from './context-persistence.js';
+
+export {
+  createContextHandler,
+  detectProvider,
+  ClaudeContextHandler,
+  CodexContextHandler,
+  GeminiContextHandler,
+  type ProviderType,
+  type ProviderContextConfig,
+  type ClaudeHooksConfig,
+  type CodexContextConfig,
+} from './provider-context.js';
+
+export {
+  AgentMemoryMonitor,
+  getMemoryMonitor,
+  formatBytes,
+  type MemorySnapshot,
+  type AgentMemoryMetrics,
+  type MemoryThresholds,
+  type MemoryMonitorConfig,
+  type MemoryAlert,
+  type CrashMemoryContext,
+} from './memory-monitor.js';
+
+export {
+  CrashInsightsService,
+  getCrashInsights,
+  type CrashRecord,
+  type CrashAnalysis,
+  type CrashPattern,
+  type CrashStats,
+} from './crash-insights.js';
+
+export {
+  StatelessLeadCoordinator,
+  createStatelessLead,
+  type BeadsTask,
+  type LeadHeartbeat,
+  type StatelessLeadConfig,
+} from './stateless-lead.js';
+
+export {
+  LeaderWatchdog,
+  createLeaderWatchdog,
+  type LeaderWatchdogConfig,
+  type ElectionResult,
+} from './leader-watchdog.js';
+
+export {
+  GossipHealthMonitor,
+  createGossipHealth,
+  type GossipHeartbeat,
+  type PeerHealth,
+  type GossipHealthConfig,
+} from './gossip-health.js';
+
+export {
+  CgroupManager,
+  getCgroupManager,
+  formatCpuTime,
+  type CpuLimitConfig,
+  type AgentCgroupInfo,
+} from './cgroup-manager.js';

package/packages/resiliency/src/leader-watchdog.ts

@@ -0,0 +1,260 @@
+/**
+ * Leader Watchdog
+ *
+ * Implements P3: Monitor lead health, trigger promotion if lead dies.
+ * Integrates with AgentSupervisor and heartbeat system.
+ *
+ * Features:
+ * - Monitors leader heartbeat file
+ * - Detects stale/missing leader
+ * - Triggers leader election or self-promotion
+ * - Integrates with supervisor events
+ */
+
+import * as path from 'path';
+import { EventEmitter } from 'events';
+import { StatelessLeadCoordinator, LeadHeartbeat } from './stateless-lead.js';
+
+/**
+ * Watchdog configuration
+ */
+export interface LeaderWatchdogConfig {
+  /** Path to .beads directory */
+  beadsDir: string;
+  /** This agent's name */
+  agentName: string;
+  /** This agent's unique ID */
+  agentId: string;
+  /** How often to check leader health (ms) */
+  checkIntervalMs: number;
+  /** Leader considered stale after this duration (ms) */
+  staleThresholdMs: number;
+  /** Callback when this agent should become leader */
+  onBecomeLeader: () => Promise<void>;
+  /** Callback to get all healthy agents for election */
+  getHealthyAgents: () => Promise<Array<{ name: string; id: string; spawnedAt: Date }>>;
+}
+
+const DEFAULT_CONFIG: Partial<LeaderWatchdogConfig> = {
+  checkIntervalMs: 5000,
+  staleThresholdMs: 30000,
+};
+
+/**
+ * Election result
+ */
+export interface ElectionResult {
+  winner: string;
+  winnerId: string;
+  candidates: string[];
+  method: 'oldest' | 'self' | 'none';
+}
+
+/**
+ * Leader Watchdog
+ *
+ * Runs on each agent, monitors leader health, triggers election if needed.
+ */
+export class LeaderWatchdog extends EventEmitter {
+  private config: LeaderWatchdogConfig;
+  private heartbeatPath: string;
+  private checkInterval?: ReturnType<typeof setInterval>;
+  private isRunning = false;
+  private currentLeader: LeadHeartbeat | null = null;
+  private isLeader = false;
+
+  constructor(config: LeaderWatchdogConfig) {
+    super();
+    this.config = { ...DEFAULT_CONFIG, ...config } as LeaderWatchdogConfig;
+    this.heartbeatPath = path.join(this.config.beadsDir, 'leader-heartbeat.json');
+  }
+
+  /**
+   * Start watching for leader health
+   */
+  start(): void {
+    if (this.isRunning) return;
+    this.isRunning = true;
+
+    console.log(`[watchdog] Started monitoring leader health (${this.config.agentName})`);
+
+    this.checkInterval = setInterval(async () => {
+      try {
+        await this.checkLeaderHealth();
+      } catch (err) {
+        console.error('[watchdog] Check error:', err);
+        this.emit('error', err);
+      }
+    }, this.config.checkIntervalMs);
+
+    // Initial check
+    this.checkLeaderHealth().catch((err) => {
+      console.error('[watchdog] Initial check error:', err);
+    });
+
+    this.emit('started');
+  }
+
+  /**
+   * Stop watching
+   */
+  stop(): void {
+    if (!this.isRunning) return;
+    this.isRunning = false;
+
+    if (this.checkInterval) {
+      clearInterval(this.checkInterval);
+      this.checkInterval = undefined;
+    }
+
+    console.log(`[watchdog] Stopped monitoring (${this.config.agentName})`);
+    this.emit('stopped');
+  }
+
+  /**
+   * Check if current leader is healthy
+   */
+  private async checkLeaderHealth(): Promise<void> {
+    const heartbeat = await StatelessLeadCoordinator.readHeartbeat(this.config.beadsDir);
+
+    // No leader - trigger election
+    if (!heartbeat) {
+      console.log('[watchdog] No leader detected, triggering election');
+      await this.triggerElection('no_leader');
+      return;
+    }
+
+    // Check if this is us
+    if (heartbeat.leadId === this.config.agentId) {
+      this.isLeader = true;
+      this.currentLeader = heartbeat;
+      return;
+    }
+
+    this.isLeader = false;
+
+    // Check if stale
+    const age = Date.now() - heartbeat.timestamp;
+    if (age > this.config.staleThresholdMs) {
+      console.log(`[watchdog] Leader ${heartbeat.leadName} is stale (${Math.round(age / 1000)}s old)`);
+      this.emit('leaderStale', { leader: heartbeat, age });
+      await this.triggerElection('stale_leader');
+      return;
+    }
+
+    // Leader is healthy
+    if (!this.currentLeader || this.currentLeader.leadId !== heartbeat.leadId) {
+      console.log(`[watchdog] Leader detected: ${heartbeat.leadName}`);
+      this.emit('leaderDetected', heartbeat);
+    }
+    this.currentLeader = heartbeat;
+  }
+
+  /**
+   * Trigger leader election
+   */
+  private async triggerElection(reason: string): Promise<void> {
+    console.log(`[watchdog] Triggering election (reason: ${reason})`);
+    this.emit('electionStarted', { reason });
+
+    const result = await this.electLeader();
+
+    if (result.method === 'none') {
+      console.log('[watchdog] No candidates for election');
+      this.emit('electionFailed', { reason: 'no_candidates' });
+      return;
+    }
+
+    console.log(`[watchdog] Election result: ${result.winner} (method: ${result.method})`);
+    this.emit('electionComplete', result);
+
+    // If we won, become leader
+    if (result.winnerId === this.config.agentId) {
+      console.log(`[watchdog] This agent (${this.config.agentName}) won the election, becoming leader`);
+      this.isLeader = true;
+      await this.config.onBecomeLeader();
+      this.emit('becameLeader');
+    }
+  }
+
+  /**
+   * Simple leader election: oldest healthy agent wins
+   */
+  private async electLeader(): Promise<ElectionResult> {
+    const candidates = await this.config.getHealthyAgents();
+
+    if (candidates.length === 0) {
+      return {
+        winner: '',
+        winnerId: '',
+        candidates: [],
+        method: 'none',
+      };
+    }
+
+    // Sort by spawn time (oldest first)
+    candidates.sort((a, b) => a.spawnedAt.getTime() - b.spawnedAt.getTime());
+
+    const winner = candidates[0];
+
+    return {
+      winner: winner.name,
+      winnerId: winner.id,
+      candidates: candidates.map((c) => c.name),
+      method: 'oldest',
+    };
+  }
+
+  /**
+   * Check if this agent is currently the leader
+   */
+  isCurrentLeader(): boolean {
+    return this.isLeader;
+  }
+
+  /**
+   * Get current leader info
+   */
+  getCurrentLeader(): LeadHeartbeat | null {
+    return this.currentLeader;
+  }
+
+  /**
+   * Get watchdog status
+   */
+  getStatus(): {
+    isRunning: boolean;
+    isLeader: boolean;
+    currentLeader: LeadHeartbeat | null;
+    agentName: string;
+  } {
+    return {
+      isRunning: this.isRunning,
+      isLeader: this.isLeader,
+      currentLeader: this.currentLeader,
+      agentName: this.config.agentName,
+    };
+  }
+}
+
+/**
+ * Create a leader watchdog with defaults
+ */
+export function createLeaderWatchdog(
+  beadsDir: string,
+  agentName: string,
+  agentId: string,
+  callbacks: {
+    onBecomeLeader: () => Promise<void>;
+    getHealthyAgents: () => Promise<Array<{ name: string; id: string; spawnedAt: Date }>>;
+  }
+): LeaderWatchdog {
+  return new LeaderWatchdog({
+    beadsDir,
+    agentName,
+    agentId,
+    ...callbacks,
+    checkIntervalMs: 5000,
+    staleThresholdMs: 30000,
+  });
+}