agent-relay 1.1.0 → 1.2.3

package/dist/resiliency/gossip-health.js

@@ -0,0 +1,241 @@
+/**
+ * Gossip-Based Health Broadcast
+ *
+ * Implements P4: Agents broadcast heartbeats via relay.
+ * Collective failure detection without central monitor.
+ *
+ * Each agent:
+ * - Periodically broadcasts HEARTBEAT message to all agents
+ * - Tracks health of all known peers
+ * - Detects failures when peer heartbeats go stale
+ * - Emits events for peer health changes
+ */
+import { EventEmitter } from 'events';
+const DEFAULT_CONFIG = {
+    broadcastIntervalMs: 10000,
+    staleThresholdMs: 30000,
+    checkIntervalMs: 5000,
+};
+/**
+ * Gossip Health Monitor
+ *
+ * Broadcasts heartbeats and tracks peer health via gossip protocol.
+ */
+export class GossipHealthMonitor extends EventEmitter {
+    config;
+    peers = new Map();
+    broadcastInterval;
+    checkInterval;
+    isRunning = false;
+    healthy = true;
+    constructor(config) {
+        super();
+        this.config = { ...DEFAULT_CONFIG, ...config };
+    }
+    /**
+     * Start gossip health monitoring
+     */
+    start() {
+        if (this.isRunning)
+            return;
+        this.isRunning = true;
+        console.log(`[gossip] Started health broadcast for ${this.config.agentName}`);
+        // Start broadcasting heartbeats
+        this.broadcastInterval = setInterval(async () => {
+            try {
+                await this.broadcastHeartbeat();
+            }
+            catch (err) {
+                console.error('[gossip] Broadcast error:', err);
+            }
+        }, this.config.broadcastIntervalMs);
+        // Start checking for stale peers
+        this.checkInterval = setInterval(() => {
+            this.checkStalePeers();
+        }, this.config.checkIntervalMs);
+        // Initial broadcast
+        this.broadcastHeartbeat().catch((err) => {
+            console.error('[gossip] Initial broadcast error:', err);
+        });
+        this.emit('started');
+    }
+    /**
+     * Stop gossip health monitoring
+     */
+    stop() {
+        if (!this.isRunning)
+            return;
+        this.isRunning = false;
+        if (this.broadcastInterval) {
+            clearInterval(this.broadcastInterval);
+            this.broadcastInterval = undefined;
+        }
+        if (this.checkInterval) {
+            clearInterval(this.checkInterval);
+            this.checkInterval = undefined;
+        }
+        console.log(`[gossip] Stopped health broadcast for ${this.config.agentName}`);
+        this.emit('stopped');
+    }
+    /**
+     * Broadcast heartbeat to all agents
+     */
+    async broadcastHeartbeat() {
+        const heartbeat = {
+            type: 'HEARTBEAT',
+            agent: this.config.agentName,
+            agentId: this.config.agentId,
+            timestamp: Date.now(),
+            load: this.config.getLoad?.() ?? 0,
+            healthy: this.healthy,
+            isLeader: this.config.isLeader?.() ?? false,
+            taskCount: this.config.getTaskCount?.() ?? 0,
+        };
+        const message = `HEARTBEAT: ${JSON.stringify(heartbeat)}`;
+        await this.config.broadcast(message);
+    }
+    /**
+     * Process incoming heartbeat from another agent
+     */
+    processHeartbeat(heartbeat) {
+        // Ignore our own heartbeats
+        if (heartbeat.agentId === this.config.agentId)
+            return;
+        const existing = this.peers.get(heartbeat.agent);
+        const wasHealthy = existing?.healthy ?? true;
+        const wasLeader = existing?.isLeader ?? false;
+        // Update peer state
+        const peer = {
+            name: heartbeat.agent,
+            lastHeartbeat: heartbeat.timestamp,
+            load: heartbeat.load,
+            healthy: heartbeat.healthy,
+            isLeader: heartbeat.isLeader,
+            taskCount: heartbeat.taskCount,
+        };
+        this.peers.set(heartbeat.agent, peer);
+        // Emit events for state changes
+        if (!existing) {
+            this.emit('peerDiscovered', peer);
+        }
+        else {
+            if (!wasHealthy && heartbeat.healthy) {
+                this.emit('peerRecovered', peer);
+            }
+            if (wasHealthy && !heartbeat.healthy) {
+                this.emit('peerUnhealthy', peer);
+            }
+            if (!wasLeader && heartbeat.isLeader) {
+                this.emit('newLeader', peer);
+            }
+        }
+    }
+    /**
+     * Parse heartbeat from relay message
+     */
+    static parseHeartbeat(message) {
+        const match = message.match(/^HEARTBEAT:\s*(.+)$/);
+        if (!match)
+            return null;
+        try {
+            const data = JSON.parse(match[1]);
+            if (data.type === 'HEARTBEAT') {
+                return data;
+            }
+        }
+        catch {
+            // Invalid JSON
+        }
+        return null;
+    }
+    /**
+     * Check for stale peers
+     */
+    checkStalePeers() {
+        const now = Date.now();
+        for (const [name, peer] of this.peers) {
+            const age = now - peer.lastHeartbeat;
+            if (age > this.config.staleThresholdMs) {
+                if (peer.healthy) {
+                    // Mark as unhealthy
+                    peer.healthy = false;
+                    console.log(`[gossip] Peer ${name} is stale (${Math.round(age / 1000)}s since last heartbeat)`);
+                    this.emit('peerStale', { peer, age });
+                    // If the stale peer was leader, emit leader lost
+                    if (peer.isLeader) {
+                        this.emit('leaderLost', peer);
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Set this agent's health status
+     */
+    setHealthy(healthy) {
+        if (this.healthy !== healthy) {
+            this.healthy = healthy;
+            // Broadcast immediately on health change
+            this.broadcastHeartbeat().catch((err) => {
+                console.error('[gossip] Immediate broadcast error:', err);
+            });
+        }
+    }
+    /**
+     * Get all known peers
+     */
+    getPeers() {
+        return Array.from(this.peers.values());
+    }
+    /**
+     * Get healthy peers
+     */
+    getHealthyPeers() {
+        return this.getPeers().filter((p) => p.healthy);
+    }
+    /**
+     * Get current leader from gossip
+     */
+    getLeader() {
+        for (const peer of this.peers.values()) {
+            if (peer.isLeader && peer.healthy) {
+                return peer;
+            }
+        }
+        return null;
+    }
+    /**
+     * Get peer by name
+     */
+    getPeer(name) {
+        return this.peers.get(name);
+    }
+    /**
+     * Get status
+     */
+    getStatus() {
+        const leader = this.getLeader();
+        return {
+            isRunning: this.isRunning,
+            agentName: this.config.agentName,
+            peerCount: this.peers.size,
+            healthyPeerCount: this.getHealthyPeers().length,
+            leader: leader?.name ?? null,
+        };
+    }
+}
+/**
+ * Create gossip health monitor with defaults
+ */
+export function createGossipHealth(agentName, agentId, broadcast, options) {
+    return new GossipHealthMonitor({
+        agentName,
+        agentId,
+        broadcast,
+        broadcastIntervalMs: 10000,
+        staleThresholdMs: 30000,
+        checkIntervalMs: 5000,
+        ...options,
+    });
+}
+//# sourceMappingURL=gossip-health.js.map

package/dist/resiliency/index.d.ts

@@ -60,4 +60,9 @@ export { metrics, type AgentMetrics, type SystemMetrics, type MetricPoint } from
 export { AgentSupervisor, getSupervisor, type SupervisedAgent, type SupervisorConfig, } from './supervisor.js';
 export { ContextPersistence, getContextPersistence, type AgentState, type Decision, type Artifact, type Handoff, type LedgerEntry, } from './context-persistence.js';
 export { createContextHandler, detectProvider, ClaudeContextHandler, CodexContextHandler, GeminiContextHandler, type ProviderType, type ProviderContextConfig, type ClaudeHooksConfig, type CodexContextConfig, } from './provider-context.js';
+export { AgentMemoryMonitor, getMemoryMonitor, formatBytes, type MemorySnapshot, type AgentMemoryMetrics, type MemoryThresholds, type MemoryMonitorConfig, type MemoryAlert, type CrashMemoryContext, } from './memory-monitor.js';
+export { CrashInsightsService, getCrashInsights, type CrashRecord, type CrashAnalysis, type CrashPattern, type CrashStats, } from './crash-insights.js';
+export { StatelessLeadCoordinator, createStatelessLead, type BeadsTask, type LeadHeartbeat, type StatelessLeadConfig, } from './stateless-lead.js';
+export { LeaderWatchdog, createLeaderWatchdog, type LeaderWatchdogConfig, type ElectionResult, } from './leader-watchdog.js';
+export { GossipHealthMonitor, createGossipHealth, type GossipHeartbeat, type PeerHealth, type GossipHealthConfig, } from './gossip-health.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/resiliency/index.js

@@ -60,4 +60,9 @@ export { metrics } from './metrics.js';
 export { AgentSupervisor, getSupervisor, } from './supervisor.js';
 export { ContextPersistence, getContextPersistence, } from './context-persistence.js';
 export { createContextHandler, detectProvider, ClaudeContextHandler, CodexContextHandler, GeminiContextHandler, } from './provider-context.js';
+export { AgentMemoryMonitor, getMemoryMonitor, formatBytes, } from './memory-monitor.js';
+export { CrashInsightsService, getCrashInsights, } from './crash-insights.js';
+export { StatelessLeadCoordinator, createStatelessLead, } from './stateless-lead.js';
+export { LeaderWatchdog, createLeaderWatchdog, } from './leader-watchdog.js';
+export { GossipHealthMonitor, createGossipHealth, } from './gossip-health.js';
 //# sourceMappingURL=index.js.map

package/dist/resiliency/leader-watchdog.d.ts

@@ -0,0 +1,109 @@
+/**
+ * Leader Watchdog
+ *
+ * Implements P3: Monitor lead health, trigger promotion if lead dies.
+ * Integrates with AgentSupervisor and heartbeat system.
+ *
+ * Features:
+ * - Monitors leader heartbeat file
+ * - Detects stale/missing leader
+ * - Triggers leader election or self-promotion
+ * - Integrates with supervisor events
+ */
+import { EventEmitter } from 'events';
+import { LeadHeartbeat } from './stateless-lead.js';
+/**
+ * Watchdog configuration
+ */
+export interface LeaderWatchdogConfig {
+    /** Path to .beads directory */
+    beadsDir: string;
+    /** This agent's name */
+    agentName: string;
+    /** This agent's unique ID */
+    agentId: string;
+    /** How often to check leader health (ms) */
+    checkIntervalMs: number;
+    /** Leader considered stale after this duration (ms) */
+    staleThresholdMs: number;
+    /** Callback when this agent should become leader */
+    onBecomeLeader: () => Promise<void>;
+    /** Callback to get all healthy agents for election */
+    getHealthyAgents: () => Promise<Array<{
+        name: string;
+        id: string;
+        spawnedAt: Date;
+    }>>;
+}
+/**
+ * Election result
+ */
+export interface ElectionResult {
+    winner: string;
+    winnerId: string;
+    candidates: string[];
+    method: 'oldest' | 'self' | 'none';
+}
+/**
+ * Leader Watchdog
+ *
+ * Runs on each agent, monitors leader health, triggers election if needed.
+ */
+export declare class LeaderWatchdog extends EventEmitter {
+    private config;
+    private heartbeatPath;
+    private checkInterval?;
+    private isRunning;
+    private currentLeader;
+    private isLeader;
+    constructor(config: LeaderWatchdogConfig);
+    /**
+     * Start watching for leader health
+     */
+    start(): void;
+    /**
+     * Stop watching
+     */
+    stop(): void;
+    /**
+     * Check if current leader is healthy
+     */
+    private checkLeaderHealth;
+    /**
+     * Trigger leader election
+     */
+    private triggerElection;
+    /**
+     * Simple leader election: oldest healthy agent wins
+     */
+    private electLeader;
+    /**
+     * Check if this agent is currently the leader
+     */
+    isCurrentLeader(): boolean;
+    /**
+     * Get current leader info
+     */
+    getCurrentLeader(): LeadHeartbeat | null;
+    /**
+     * Get watchdog status
+     */
+    getStatus(): {
+        isRunning: boolean;
+        isLeader: boolean;
+        currentLeader: LeadHeartbeat | null;
+        agentName: string;
+    };
+}
+/**
+ * Create a leader watchdog with defaults
+ */
+export declare function createLeaderWatchdog(beadsDir: string, agentName: string, agentId: string, callbacks: {
+    onBecomeLeader: () => Promise<void>;
+    getHealthyAgents: () => Promise<Array<{
+        name: string;
+        id: string;
+        spawnedAt: Date;
+    }>>;
+}): LeaderWatchdog;
+//# sourceMappingURL=leader-watchdog.d.ts.map

package/dist/resiliency/leader-watchdog.js

@@ -0,0 +1,189 @@
+/**
+ * Leader Watchdog
+ *
+ * Implements P3: Monitor lead health, trigger promotion if lead dies.
+ * Integrates with AgentSupervisor and heartbeat system.
+ *
+ * Features:
+ * - Monitors leader heartbeat file
+ * - Detects stale/missing leader
+ * - Triggers leader election or self-promotion
+ * - Integrates with supervisor events
+ */
+import * as path from 'path';
+import { EventEmitter } from 'events';
+import { StatelessLeadCoordinator } from './stateless-lead.js';
+const DEFAULT_CONFIG = {
+    checkIntervalMs: 5000,
+    staleThresholdMs: 30000,
+};
+/**
+ * Leader Watchdog
+ *
+ * Runs on each agent, monitors leader health, triggers election if needed.
+ */
+export class LeaderWatchdog extends EventEmitter {
+    config;
+    heartbeatPath;
+    checkInterval;
+    isRunning = false;
+    currentLeader = null;
+    isLeader = false;
+    constructor(config) {
+        super();
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        this.heartbeatPath = path.join(this.config.beadsDir, 'leader-heartbeat.json');
+    }
+    /**
+     * Start watching for leader health
+     */
+    start() {
+        if (this.isRunning)
+            return;
+        this.isRunning = true;
+        console.log(`[watchdog] Started monitoring leader health (${this.config.agentName})`);
+        this.checkInterval = setInterval(async () => {
+            try {
+                await this.checkLeaderHealth();
+            }
+            catch (err) {
+                console.error('[watchdog] Check error:', err);
+                this.emit('error', err);
+            }
+        }, this.config.checkIntervalMs);
+        // Initial check
+        this.checkLeaderHealth().catch((err) => {
+            console.error('[watchdog] Initial check error:', err);
+        });
+        this.emit('started');
+    }
+    /**
+     * Stop watching
+     */
+    stop() {
+        if (!this.isRunning)
+            return;
+        this.isRunning = false;
+        if (this.checkInterval) {
+            clearInterval(this.checkInterval);
+            this.checkInterval = undefined;
+        }
+        console.log(`[watchdog] Stopped monitoring (${this.config.agentName})`);
+        this.emit('stopped');
+    }
+    /**
+     * Check if current leader is healthy
+     */
+    async checkLeaderHealth() {
+        const heartbeat = await StatelessLeadCoordinator.readHeartbeat(this.config.beadsDir);
+        // No leader - trigger election
+        if (!heartbeat) {
+            console.log('[watchdog] No leader detected, triggering election');
+            await this.triggerElection('no_leader');
+            return;
+        }
+        // Check if this is us
+        if (heartbeat.leadId === this.config.agentId) {
+            this.isLeader = true;
+            this.currentLeader = heartbeat;
+            return;
+        }
+        this.isLeader = false;
+        // Check if stale
+        const age = Date.now() - heartbeat.timestamp;
+        if (age > this.config.staleThresholdMs) {
+            console.log(`[watchdog] Leader ${heartbeat.leadName} is stale (${Math.round(age / 1000)}s old)`);
+            this.emit('leaderStale', { leader: heartbeat, age });
+            await this.triggerElection('stale_leader');
+            return;
+        }
+        // Leader is healthy
+        if (!this.currentLeader || this.currentLeader.leadId !== heartbeat.leadId) {
+            console.log(`[watchdog] Leader detected: ${heartbeat.leadName}`);
+            this.emit('leaderDetected', heartbeat);
+        }
+        this.currentLeader = heartbeat;
+    }
+    /**
+     * Trigger leader election
+     */
+    async triggerElection(reason) {
+        console.log(`[watchdog] Triggering election (reason: ${reason})`);
+        this.emit('electionStarted', { reason });
+        const result = await this.electLeader();
+        if (result.method === 'none') {
+            console.log('[watchdog] No candidates for election');
+            this.emit('electionFailed', { reason: 'no_candidates' });
+            return;
+        }
+        console.log(`[watchdog] Election result: ${result.winner} (method: ${result.method})`);
+        this.emit('electionComplete', result);
+        // If we won, become leader
+        if (result.winnerId === this.config.agentId) {
+            console.log(`[watchdog] This agent (${this.config.agentName}) won the election, becoming leader`);
+            this.isLeader = true;
+            await this.config.onBecomeLeader();
+            this.emit('becameLeader');
+        }
+    }
+    /**
+     * Simple leader election: oldest healthy agent wins
+     */
+    async electLeader() {
+        const candidates = await this.config.getHealthyAgents();
+        if (candidates.length === 0) {
+            return {
+                winner: '',
+                winnerId: '',
+                candidates: [],
+                method: 'none',
+            };
+        }
+        // Sort by spawn time (oldest first)
+        candidates.sort((a, b) => a.spawnedAt.getTime() - b.spawnedAt.getTime());
+        const winner = candidates[0];
+        return {
+            winner: winner.name,
+            winnerId: winner.id,
+            candidates: candidates.map((c) => c.name),
+            method: 'oldest',
+        };
+    }
+    /**
+     * Check if this agent is currently the leader
+     */
+    isCurrentLeader() {
+        return this.isLeader;
+    }
+    /**
+     * Get current leader info
+     */
+    getCurrentLeader() {
+        return this.currentLeader;
+    }
+    /**
+     * Get watchdog status
+     */
+    getStatus() {
+        return {
+            isRunning: this.isRunning,
+            isLeader: this.isLeader,
+            currentLeader: this.currentLeader,
+            agentName: this.config.agentName,
+        };
+    }
+}
+/**
+ * Create a leader watchdog with defaults
+ */
+export function createLeaderWatchdog(beadsDir, agentName, agentId, callbacks) {
+    return new LeaderWatchdog({
+        beadsDir,
+        agentName,
+        agentId,
+        ...callbacks,
+        checkIntervalMs: 5000,
+        staleThresholdMs: 30000,
+    });
+}
+//# sourceMappingURL=leader-watchdog.js.map