agent-relay 2.3.13 → 2.4.0

package/packages/resiliency/src/index.ts

@@ -1,157 +0,0 @@
-/**
- * Agent Resiliency Module
- *
- * Provides comprehensive health monitoring, auto-restart, logging,
- * metrics, and context persistence for agent-relay agents.
- *
- * Features:
- * - Health monitoring with process liveness checks
- * - Auto-restart on crash with configurable limits
- * - Prometheus-compatible metrics export
- * - Structured JSON logging with rotation
- * - Context persistence across restarts (inspired by Continuous-Claude-v2)
- * - Provider-specific context injection (Claude hooks, Codex config, Gemini instructions)
- *
- * Usage:
- *
- * ```ts
- * import { getSupervisor, metrics, createLogger } from './resiliency';
- *
- * // Start the supervisor with context persistence
- * const supervisor = getSupervisor({
- *   autoRestart: true,
- *   maxRestarts: 5,
- *   contextPersistence: {
- *     enabled: true,
- *     autoInjectOnRestart: true,
- *   },
- * });
- * supervisor.start();
- *
- * // Add an agent to supervision
- * supervisor.supervise(
- *   {
- *     name: 'worker-1',
- *     cli: 'claude',
- *     pid: 12345,
- *     spawnedAt: new Date(),
- *     workingDir: '/path/to/repo',
- *     provider: 'claude', // or 'codex', 'gemini'
- *   },
- *   {
- *     isAlive: () => process.kill(12345, 0),
- *     kill: (sig) => process.kill(12345, sig),
- *     restart: async () => { ... },
- *   }
- * );
- *
- * // Get metrics
- * console.log(metrics.toPrometheus());
- * ```
- *
- * Context persistence works differently per provider:
- * - Claude: Uses hooks to inject context into CLAUDE.md
- * - Codex: Uses config for periodic context refresh via system prompt
- * - Gemini: Updates system instruction file
- */
-
-export {
-  AgentHealthMonitor,
-  getHealthMonitor,
-  type AgentHealth,
-  type AgentProcess,
-  type HealthMonitorConfig,
-} from './health-monitor.js';
-
-export {
-  Logger,
-  createLogger,
-  configure as configureLogging,
-  loggers,
-  type LogLevel,
-  type LogEntry,
-  type LoggerConfig,
-} from './logger.js';
-
-export { metrics, type AgentMetrics, type SystemMetrics, type MetricPoint } from './metrics.js';
-
-export {
-  AgentSupervisor,
-  getSupervisor,
-  type SupervisedAgent,
-  type SupervisorConfig,
-} from './supervisor.js';
-
-export {
-  ContextPersistence,
-  getContextPersistence,
-  type AgentState,
-  type Decision,
-  type Artifact,
-  type Handoff,
-  type LedgerEntry,
-} from './context-persistence.js';
-
-export {
-  createContextHandler,
-  detectProvider,
-  ClaudeContextHandler,
-  CodexContextHandler,
-  GeminiContextHandler,
-  type ProviderType,
-  type ProviderContextConfig,
-  type ClaudeHooksConfig,
-  type CodexContextConfig,
-} from './provider-context.js';
-
-export {
-  AgentMemoryMonitor,
-  getMemoryMonitor,
-  formatBytes,
-  type MemorySnapshot,
-  type AgentMemoryMetrics,
-  type MemoryThresholds,
-  type MemoryMonitorConfig,
-  type MemoryAlert,
-  type CrashMemoryContext,
-} from './memory-monitor.js';
-
-export {
-  CrashInsightsService,
-  getCrashInsights,
-  type CrashRecord,
-  type CrashAnalysis,
-  type CrashPattern,
-  type CrashStats,
-} from './crash-insights.js';
-
-export {
-  StatelessLeadCoordinator,
-  createStatelessLead,
-  type BeadsTask,
-  type LeadHeartbeat,
-  type StatelessLeadConfig,
-} from './stateless-lead.js';
-
-export {
-  LeaderWatchdog,
-  createLeaderWatchdog,
-  type LeaderWatchdogConfig,
-  type ElectionResult,
-} from './leader-watchdog.js';
-
-export {
-  GossipHealthMonitor,
-  createGossipHealth,
-  type GossipHeartbeat,
-  type PeerHealth,
-  type GossipHealthConfig,
-} from './gossip-health.js';
-
-export {
-  CgroupManager,
-  getCgroupManager,
-  formatCpuTime,
-  type CpuLimitConfig,
-  type AgentCgroupInfo,
-} from './cgroup-manager.js';

package/packages/resiliency/src/leader-watchdog.ts

@@ -1,260 +0,0 @@
-/**
- * Leader Watchdog
- *
- * Implements P3: Monitor lead health, trigger promotion if lead dies.
- * Integrates with AgentSupervisor and heartbeat system.
- *
- * Features:
- * - Monitors leader heartbeat file
- * - Detects stale/missing leader
- * - Triggers leader election or self-promotion
- * - Integrates with supervisor events
- */
-
-import * as path from 'path';
-import { EventEmitter } from 'events';
-import { StatelessLeadCoordinator, LeadHeartbeat } from './stateless-lead.js';
-
-/**
- * Watchdog configuration
- */
-export interface LeaderWatchdogConfig {
-  /** Path to .beads directory */
-  beadsDir: string;
-  /** This agent's name */
-  agentName: string;
-  /** This agent's unique ID */
-  agentId: string;
-  /** How often to check leader health (ms) */
-  checkIntervalMs: number;
-  /** Leader considered stale after this duration (ms) */
-  staleThresholdMs: number;
-  /** Callback when this agent should become leader */
-  onBecomeLeader: () => Promise<void>;
-  /** Callback to get all healthy agents for election */
-  getHealthyAgents: () => Promise<Array<{ name: string; id: string; spawnedAt: Date }>>;
-}
-
-const DEFAULT_CONFIG: Partial<LeaderWatchdogConfig> = {
-  checkIntervalMs: 5000,
-  staleThresholdMs: 30000,
-};
-
-/**
- * Election result
- */
-export interface ElectionResult {
-  winner: string;
-  winnerId: string;
-  candidates: string[];
-  method: 'oldest' | 'self' | 'none';
-}
-
-/**
- * Leader Watchdog
- *
- * Runs on each agent, monitors leader health, triggers election if needed.
- */
-export class LeaderWatchdog extends EventEmitter {
-  private config: LeaderWatchdogConfig;
-  private heartbeatPath: string;
-  private checkInterval?: ReturnType<typeof setInterval>;
-  private isRunning = false;
-  private currentLeader: LeadHeartbeat | null = null;
-  private isLeader = false;
-
-  constructor(config: LeaderWatchdogConfig) {
-    super();
-    this.config = { ...DEFAULT_CONFIG, ...config } as LeaderWatchdogConfig;
-    this.heartbeatPath = path.join(this.config.beadsDir, 'leader-heartbeat.json');
-  }
-
-  /**
-   * Start watching for leader health
-   */
-  start(): void {
-    if (this.isRunning) return;
-    this.isRunning = true;
-
-    console.log(`[watchdog] Started monitoring leader health (${this.config.agentName})`);
-
-    this.checkInterval = setInterval(async () => {
-      try {
-        await this.checkLeaderHealth();
-      } catch (err) {
-        console.error('[watchdog] Check error:', err);
-        this.emit('error', err);
-      }
-    }, this.config.checkIntervalMs);
-
-    // Initial check
-    this.checkLeaderHealth().catch((err) => {
-      console.error('[watchdog] Initial check error:', err);
-    });
-
-    this.emit('started');
-  }
-
-  /**
-   * Stop watching
-   */
-  stop(): void {
-    if (!this.isRunning) return;
-    this.isRunning = false;
-
-    if (this.checkInterval) {
-      clearInterval(this.checkInterval);
-      this.checkInterval = undefined;
-    }
-
-    console.log(`[watchdog] Stopped monitoring (${this.config.agentName})`);
-    this.emit('stopped');
-  }
-
-  /**
-   * Check if current leader is healthy
-   */
-  private async checkLeaderHealth(): Promise<void> {
-    const heartbeat = await StatelessLeadCoordinator.readHeartbeat(this.config.beadsDir);
-
-    // No leader - trigger election
-    if (!heartbeat) {
-      console.log('[watchdog] No leader detected, triggering election');
-      await this.triggerElection('no_leader');
-      return;
-    }
-
-    // Check if this is us
-    if (heartbeat.leadId === this.config.agentId) {
-      this.isLeader = true;
-      this.currentLeader = heartbeat;
-      return;
-    }
-
-    this.isLeader = false;
-
-    // Check if stale
-    const age = Date.now() - heartbeat.timestamp;
-    if (age > this.config.staleThresholdMs) {
-      console.log(`[watchdog] Leader ${heartbeat.leadName} is stale (${Math.round(age / 1000)}s old)`);
-      this.emit('leaderStale', { leader: heartbeat, age });
-      await this.triggerElection('stale_leader');
-      return;
-    }
-
-    // Leader is healthy
-    if (!this.currentLeader || this.currentLeader.leadId !== heartbeat.leadId) {
-      console.log(`[watchdog] Leader detected: ${heartbeat.leadName}`);
-      this.emit('leaderDetected', heartbeat);
-    }
-    this.currentLeader = heartbeat;
-  }
-
-  /**
-   * Trigger leader election
-   */
-  private async triggerElection(reason: string): Promise<void> {
-    console.log(`[watchdog] Triggering election (reason: ${reason})`);
-    this.emit('electionStarted', { reason });
-
-    const result = await this.electLeader();
-
-    if (result.method === 'none') {
-      console.log('[watchdog] No candidates for election');
-      this.emit('electionFailed', { reason: 'no_candidates' });
-      return;
-    }
-
-    console.log(`[watchdog] Election result: ${result.winner} (method: ${result.method})`);
-    this.emit('electionComplete', result);
-
-    // If we won, become leader
-    if (result.winnerId === this.config.agentId) {
-      console.log(`[watchdog] This agent (${this.config.agentName}) won the election, becoming leader`);
-      this.isLeader = true;
-      await this.config.onBecomeLeader();
-      this.emit('becameLeader');
-    }
-  }
-
-  /**
-   * Simple leader election: oldest healthy agent wins
-   */
-  private async electLeader(): Promise<ElectionResult> {
-    const candidates = await this.config.getHealthyAgents();
-
-    if (candidates.length === 0) {
-      return {
-        winner: '',
-        winnerId: '',
-        candidates: [],
-        method: 'none',
-      };
-    }
-
-    // Sort by spawn time (oldest first)
-    candidates.sort((a, b) => a.spawnedAt.getTime() - b.spawnedAt.getTime());
-
-    const winner = candidates[0];
-
-    return {
-      winner: winner.name,
-      winnerId: winner.id,
-      candidates: candidates.map((c) => c.name),
-      method: 'oldest',
-    };
-  }
-
-  /**
-   * Check if this agent is currently the leader
-   */
-  isCurrentLeader(): boolean {
-    return this.isLeader;
-  }
-
-  /**
-   * Get current leader info
-   */
-  getCurrentLeader(): LeadHeartbeat | null {
-    return this.currentLeader;
-  }
-
-  /**
-   * Get watchdog status
-   */
-  getStatus(): {
-    isRunning: boolean;
-    isLeader: boolean;
-    currentLeader: LeadHeartbeat | null;
-    agentName: string;
-  } {
-    return {
-      isRunning: this.isRunning,
-      isLeader: this.isLeader,
-      currentLeader: this.currentLeader,
-      agentName: this.config.agentName,
-    };
-  }
-}
-
-/**
- * Create a leader watchdog with defaults
- */
-export function createLeaderWatchdog(
-  beadsDir: string,
-  agentName: string,
-  agentId: string,
-  callbacks: {
-    onBecomeLeader: () => Promise<void>;
-    getHealthyAgents: () => Promise<Array<{ name: string; id: string; spawnedAt: Date }>>;
-  }
-): LeaderWatchdog {
-  return new LeaderWatchdog({
-    beadsDir,
-    agentName,
-    agentId,
-    ...callbacks,
-    checkIntervalMs: 5000,
-    staleThresholdMs: 30000,
-  });
-}