agent-relay 1.1.0 → 1.2.3

package/dist/resiliency/crash-insights.js

@@ -0,0 +1,492 @@
+/**
+ * Crash Insights Service
+ *
+ * Captures and analyzes agent crashes to provide actionable insights:
+ * - Memory state at crash time
+ * - Crash history and patterns
+ * - Root cause analysis
+ * - Recommendations for prevention
+ */
+import { EventEmitter } from 'events';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { formatBytes, } from './memory-monitor.js';
+export class CrashInsightsService extends EventEmitter {
+    crashes = [];
+    memoryMonitor = null;
+    persistPath;
+    maxCrashHistory = 1000;
+    constructor(memoryMonitor) {
+        super();
+        this.memoryMonitor = memoryMonitor || null;
+        // Set up persistence path
+        const dataDir = process.env.AGENT_RELAY_DATA_DIR ||
+            path.join(os.homedir(), '.local', 'share', 'agent-relay');
+        this.persistPath = path.join(dataDir, 'crash-insights.json');
+        // Load existing crash history
+        this.loadCrashes();
+    }
+    /**
+     * Set the memory monitor instance
+     */
+    setMemoryMonitor(monitor) {
+        this.memoryMonitor = monitor;
+    }
+    /**
+     * Record a crash event
+     */
+    recordCrash(params) {
+        const id = `crash-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+        const crashTime = new Date();
+        // Get memory context from memory monitor
+        const memoryContext = this.memoryMonitor
+            ? this.memoryMonitor.getCrashContext(params.agentName)
+            : this.createEmptyMemoryContext(params.agentName, params.pid, crashTime);
+        // Analyze the crash
+        const analysis = this.analyzeCrash({
+            ...params,
+            memoryContext,
+        });
+        const record = {
+            id,
+            agentName: params.agentName,
+            pid: params.pid,
+            crashTime,
+            exitCode: params.exitCode,
+            signal: params.signal,
+            reason: params.reason,
+            memoryContext,
+            stackTrace: params.stackTrace,
+            lastOutput: params.lastOutput?.slice(-2000), // Keep last 2KB
+            environment: {
+                nodeVersion: process.version,
+                platform: process.platform,
+                arch: process.arch,
+                systemMemory: {
+                    total: os.totalmem(),
+                    free: os.freemem(),
+                },
+                uptime: process.uptime(),
+            },
+            analysis,
+        };
+        // Add to history
+        this.crashes.unshift(record);
+        // Trim history
+        if (this.crashes.length > this.maxCrashHistory) {
+            this.crashes = this.crashes.slice(0, this.maxCrashHistory);
+        }
+        // Persist
+        this.saveCrashes();
+        // Emit event
+        this.emit('crash', record);
+        this.log('error', `Crash recorded for ${params.agentName}`, {
+            id,
+            cause: analysis.likelyCause,
+            confidence: analysis.confidence,
+        });
+        return record;
+    }
+    /**
+     * Get crash history for an agent
+     */
+    getCrashHistory(agentName, limit = 50) {
+        let history = this.crashes;
+        if (agentName) {
+            history = history.filter((c) => c.agentName === agentName);
+        }
+        return history.slice(0, limit);
+    }
+    /**
+     * Get a specific crash record
+     */
+    getCrash(id) {
+        return this.crashes.find((c) => c.id === id);
+    }
+    /**
+     * Get crash statistics
+     */
+    getStats() {
+        const crashesByAgent = {};
+        const crashesByCause = {};
+        const agentCrashTimes = {};
+        for (const crash of this.crashes) {
+            crashesByAgent[crash.agentName] = (crashesByAgent[crash.agentName] || 0) + 1;
+            crashesByCause[crash.analysis.likelyCause] =
+                (crashesByCause[crash.analysis.likelyCause] || 0) + 1;
+            if (!agentCrashTimes[crash.agentName]) {
+                agentCrashTimes[crash.agentName] = [];
+            }
+            agentCrashTimes[crash.agentName].push(crash.crashTime.getTime());
+        }
+        // Find most crash-prone agent
+        let mostCrashProne = null;
+        for (const [agent, count] of Object.entries(crashesByAgent)) {
+            if (!mostCrashProne || count > mostCrashProne.count) {
+                mostCrashProne = { agent, count };
+            }
+        }
+        // Calculate average time between crashes
+        let totalIntervals = 0;
+        let intervalCount = 0;
+        for (const times of Object.values(agentCrashTimes)) {
+            if (times.length > 1) {
+                const sorted = times.sort((a, b) => a - b);
+                for (let i = 1; i < sorted.length; i++) {
+                    totalIntervals += sorted[i] - sorted[i - 1];
+                    intervalCount++;
+                }
+            }
+        }
+        const avgTimeBetweenCrashes = intervalCount > 0 ? totalIntervals / intervalCount : 0;
+        // Detect patterns
+        const patterns = this.detectPatterns();
+        return {
+            totalCrashes: this.crashes.length,
+            crashesByAgent,
+            crashesByCause,
+            avgTimeBetweenCrashes,
+            mostCrashProne,
+            recentCrashes: this.crashes.slice(0, 10),
+            patterns,
+        };
+    }
+    /**
+     * Get insights and recommendations
+     */
+    getInsights() {
+        const stats = this.getStats();
+        const issues = [];
+        const trends = [];
+        // Analyze OOM crashes
+        const oomCrashes = stats.crashesByCause['oom'] || 0;
+        if (oomCrashes > 0) {
+            issues.push({
+                issue: `${oomCrashes} crash${oomCrashes > 1 ? 'es' : ''} caused by out of memory`,
+                severity: 'high',
+                recommendation: 'Increase memory limits or optimize agent memory usage',
+            });
+        }
+        // Analyze memory leaks
+        const leakCrashes = stats.crashesByCause['memory_leak'] || 0;
+        if (leakCrashes > 0) {
+            issues.push({
+                issue: `${leakCrashes} crash${leakCrashes > 1 ? 'es' : ''} likely caused by memory leaks`,
+                severity: 'high',
+                recommendation: 'Investigate agent code for memory leaks, consider periodic restarts',
+            });
+        }
+        // Check crash frequency
+        const recentCrashes = this.crashes.filter((c) => Date.now() - c.crashTime.getTime() < 24 * 60 * 60 * 1000).length;
+        if (recentCrashes > 5) {
+            issues.push({
+                issue: `${recentCrashes} crashes in the last 24 hours`,
+                severity: recentCrashes > 10 ? 'high' : 'medium',
+                recommendation: 'Investigate root cause, consider rolling back recent changes',
+            });
+        }
+        // Check repeat offenders
+        if (stats.mostCrashProne && stats.mostCrashProne.count > 5) {
+            issues.push({
+                issue: `Agent "${stats.mostCrashProne.agent}" has crashed ${stats.mostCrashProne.count} times`,
+                severity: 'medium',
+                recommendation: 'Investigate why this agent is unstable',
+            });
+        }
+        // Calculate health score (0-100)
+        let healthScore = 100;
+        healthScore -= oomCrashes * 10;
+        healthScore -= leakCrashes * 8;
+        healthScore -= recentCrashes * 3;
+        healthScore = Math.max(0, Math.min(100, healthScore));
+        // Analyze trends
+        const last24h = this.crashes.filter((c) => Date.now() - c.crashTime.getTime() < 24 * 60 * 60 * 1000).length;
+        const prev24h = this.crashes.filter((c) => Date.now() - c.crashTime.getTime() >= 24 * 60 * 60 * 1000 &&
+            Date.now() - c.crashTime.getTime() < 48 * 60 * 60 * 1000).length;
+        let crashTrend = 'stable';
+        if (last24h < prev24h * 0.7)
+            crashTrend = 'improving';
+        else if (last24h > prev24h * 1.3)
+            crashTrend = 'degrading';
+        trends.push({
+            metric: 'Crash frequency',
+            trend: crashTrend,
+            details: `${last24h} crashes in last 24h vs ${prev24h} in previous 24h`,
+        });
+        return {
+            summary: this.generateSummary(stats),
+            topIssues: issues.sort((a, b) => {
+                const severityOrder = { high: 0, medium: 1, low: 2 };
+                return severityOrder[a.severity] - severityOrder[b.severity];
+            }),
+            healthScore,
+            trends,
+        };
+    }
+    /**
+     * Analyze a crash and determine likely cause
+     */
+    analyzeCrash(params) {
+        const details = [];
+        const recommendations = [];
+        let likelyCause = 'unknown';
+        let confidence = 'low';
+        // Check memory-based causes first
+        if (params.memoryContext.likelyCause !== 'unknown') {
+            likelyCause = params.memoryContext.likelyCause;
+            confidence = 'high';
+            details.push(...params.memoryContext.analysisNotes);
+        }
+        // Check signal
+        if (params.signal) {
+            details.push(`Process received signal: ${params.signal}`);
+            if (params.signal === 'SIGKILL') {
+                if (likelyCause === 'unknown') {
+                    likelyCause = 'oom';
+                    confidence = 'medium';
+                }
+                details.push('SIGKILL often indicates OOM killer intervention');
+                recommendations.push('Check system logs for OOM killer activity');
+            }
+            else if (params.signal === 'SIGSEGV') {
+                likelyCause = 'error';
+                confidence = 'high';
+                details.push('Segmentation fault - memory access violation');
+                recommendations.push('Check for native module issues or memory corruption');
+            }
+        }
+        // Check exit code
+        if (params.exitCode !== null) {
+            details.push(`Exit code: ${params.exitCode}`);
+            if (params.exitCode === 137) {
+                // 128 + 9 (SIGKILL)
+                if (likelyCause === 'unknown') {
+                    likelyCause = 'oom';
+                    confidence = 'high';
+                }
+                details.push('Exit code 137 typically indicates OOM kill');
+            }
+        }
+        // Check stack trace for clues
+        if (params.stackTrace) {
+            if (params.stackTrace.includes('FATAL ERROR: CALL_AND_RETRY_LAST')) {
+                likelyCause = 'oom';
+                confidence = 'high';
+                details.push('V8 heap allocation failure detected');
+                recommendations.push('Increase Node.js memory limit with --max-old-space-size');
+            }
+            if (params.stackTrace.includes('RangeError: Invalid array length')) {
+                likelyCause = 'memory_leak';
+                confidence = 'medium';
+                details.push('Array grew too large - possible unbounded growth');
+                recommendations.push('Review array handling code for unbounded growth');
+            }
+        }
+        // Add memory-specific recommendations
+        if (likelyCause === 'oom' || likelyCause === 'memory_leak') {
+            recommendations.push('Review agent memory usage patterns');
+            recommendations.push('Consider implementing memory limits or checkpoints');
+            if (params.memoryContext.peakMemory > 1024 * 1024 * 1024) {
+                recommendations.push(`Peak memory was ${formatBytes(params.memoryContext.peakMemory)} - consider memory profiling`);
+            }
+        }
+        // Find related crashes
+        const relatedCrashes = this.findRelatedCrashes(params.agentName, likelyCause);
+        // Generate summary
+        const summary = this.generateCrashSummary(likelyCause, confidence, params);
+        return {
+            likelyCause,
+            confidence,
+            summary,
+            details,
+            recommendations: recommendations.length > 0
+                ? recommendations
+                : ['Monitor agent for recurrence', 'Check logs for additional context'],
+            relatedCrashes,
+        };
+    }
+    /**
+     * Find related crashes
+     */
+    findRelatedCrashes(agentName, cause) {
+        return this.crashes
+            .filter((c) => (c.agentName === agentName || c.analysis.likelyCause === cause) &&
+            Date.now() - c.crashTime.getTime() < 7 * 24 * 60 * 60 * 1000 // Last 7 days
+        )
+            .slice(0, 5)
+            .map((c) => c.id);
+    }
+    /**
+     * Detect crash patterns
+     */
+    detectPatterns() {
+        const patterns = [];
+        const causeGroups = {};
+        // Group by cause
+        for (const crash of this.crashes) {
+            const cause = crash.analysis.likelyCause;
+            if (!causeGroups[cause]) {
+                causeGroups[cause] = [];
+            }
+            causeGroups[cause].push(crash);
+        }
+        // Create patterns for significant groups
+        for (const [cause, crashes] of Object.entries(causeGroups)) {
+            if (crashes.length >= 3) {
+                const agents = [...new Set(crashes.map((c) => c.agentName))];
+                const avgMemory = crashes.reduce((sum, c) => sum + (c.memoryContext.peakMemory || 0), 0) /
+                    crashes.length;
+                patterns.push({
+                    pattern: `${cause}_pattern`,
+                    occurrences: crashes.length,
+                    lastSeen: crashes[0].crashTime,
+                    affectedAgents: agents,
+                    avgMemoryAtCrash: avgMemory,
+                    commonCause: cause,
+                });
+            }
+        }
+        return patterns;
+    }
+    /**
+     * Generate crash summary
+     */
+    generateCrashSummary(cause, confidence, params) {
+        const causeDescriptions = {
+            oom: 'ran out of memory',
+            memory_leak: 'experienced a memory leak',
+            sudden_spike: 'had a sudden memory spike',
+            signal: 'was terminated by a signal',
+            error: 'encountered an error',
+            unknown: 'crashed for unknown reasons',
+        };
+        return `Agent "${params.agentName}" ${causeDescriptions[cause] || 'crashed'} (${confidence} confidence). ${params.reason}`;
+    }
+    /**
+     * Generate overall summary
+     */
+    generateSummary(stats) {
+        if (stats.totalCrashes === 0) {
+            return 'No crashes recorded. System is stable.';
+        }
+        const parts = [];
+        parts.push(`${stats.totalCrashes} total crash${stats.totalCrashes > 1 ? 'es' : ''} recorded.`);
+        if (stats.mostCrashProne) {
+            parts.push(`Most unstable: "${stats.mostCrashProne.agent}" (${stats.mostCrashProne.count} crashes).`);
+        }
+        const topCause = Object.entries(stats.crashesByCause).sort((a, b) => b[1] - a[1])[0];
+        if (topCause) {
+            parts.push(`Primary cause: ${topCause[0]} (${topCause[1]} occurrences).`);
+        }
+        return parts.join(' ');
+    }
+    /**
+     * Create empty memory context when no monitor available
+     */
+    createEmptyMemoryContext(agentName, pid, crashTime) {
+        return {
+            agentName,
+            pid,
+            crashTime,
+            lastKnownMemory: null,
+            peakMemory: 0,
+            averageMemory: 0,
+            memoryTrend: 'unknown',
+            recentHistory: [],
+            likelyCause: 'unknown',
+            analysisNotes: ['Memory monitoring was not enabled'],
+        };
+    }
+    /**
+     * Load crashes from disk
+     */
+    loadCrashes() {
+        try {
+            if (fs.existsSync(this.persistPath)) {
+                const data = fs.readFileSync(this.persistPath, 'utf-8');
+                const parsed = JSON.parse(data);
+                this.crashes = parsed.crashes.map((c) => ({
+                    ...c,
+                    crashTime: new Date(c.crashTime),
+                    memoryContext: {
+                        ...c.memoryContext,
+                        crashTime: new Date(c.memoryContext.crashTime),
+                        lastKnownMemory: c.memoryContext.lastKnownMemory
+                            ? {
+                                ...c.memoryContext.lastKnownMemory,
+                                timestamp: new Date(c.memoryContext.lastKnownMemory.timestamp),
+                            }
+                            : null,
+                        recentHistory: c.memoryContext.recentHistory.map((h) => ({
+                            ...h,
+                            timestamp: new Date(h.timestamp),
+                        })),
+                    },
+                }));
+                this.log('info', `Loaded ${this.crashes.length} crash records`);
+            }
+        }
+        catch (error) {
+            this.log('warn', 'Failed to load crash history', { error: String(error) });
+            this.crashes = [];
+        }
+    }
+    /**
+     * Save crashes to disk
+     */
+    saveCrashes() {
+        try {
+            const dir = path.dirname(this.persistPath);
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir, { recursive: true });
+            }
+            fs.writeFileSync(this.persistPath, JSON.stringify({ crashes: this.crashes }, null, 2));
+        }
+        catch (error) {
+            this.log('error', 'Failed to save crash history', { error: String(error) });
+        }
+    }
+    /**
+     * Clear all crash history
+     */
+    clear() {
+        this.crashes = [];
+        this.saveCrashes();
+        this.emit('cleared');
+    }
+    /**
+     * Structured logging
+     */
+    log(level, message, context) {
+        const entry = {
+            timestamp: new Date().toISOString(),
+            level,
+            component: 'crash-insights',
+            message,
+            ...context,
+        };
+        this.emit('log', entry);
+        const prefix = `[crash-insights]`;
+        switch (level) {
+            case 'info':
+                console.log(prefix, message, context ? JSON.stringify(context) : '');
+                break;
+            case 'warn':
+                console.warn(prefix, message, context ? JSON.stringify(context) : '');
+                break;
+            case 'error':
+                console.error(prefix, message, context ? JSON.stringify(context) : '');
+                break;
+        }
+    }
+}
+// Singleton instance
+let _crashInsights = null;
+export function getCrashInsights(memoryMonitor) {
+    if (!_crashInsights) {
+        _crashInsights = new CrashInsightsService(memoryMonitor);
+    }
+    return _crashInsights;
+}
+//# sourceMappingURL=crash-insights.js.map

package/dist/resiliency/gossip-health.d.ts

@@ -0,0 +1,137 @@
+/**
+ * Gossip-Based Health Broadcast
+ *
+ * Implements P4: Agents broadcast heartbeats via relay.
+ * Collective failure detection without central monitor.
+ *
+ * Each agent:
+ * - Periodically broadcasts HEARTBEAT message to all agents
+ * - Tracks health of all known peers
+ * - Detects failures when peer heartbeats go stale
+ * - Emits events for peer health changes
+ */
+import { EventEmitter } from 'events';
+/**
+ * Peer health state
+ */
+export interface PeerHealth {
+    name: string;
+    lastHeartbeat: number;
+    load: number;
+    healthy: boolean;
+    isLeader: boolean;
+    taskCount: number;
+}
+/**
+ * Heartbeat payload (broadcast via relay)
+ */
+export interface GossipHeartbeat {
+    type: 'HEARTBEAT';
+    agent: string;
+    agentId: string;
+    timestamp: number;
+    load: number;
+    healthy: boolean;
+    isLeader: boolean;
+    taskCount: number;
+}
+/**
+ * Gossip health configuration
+ */
+export interface GossipHealthConfig {
+    /** This agent's name */
+    agentName: string;
+    /** This agent's unique ID */
+    agentId: string;
+    /** How often to broadcast heartbeat (ms) */
+    broadcastIntervalMs: number;
+    /** Peer considered stale after this duration (ms) */
+    staleThresholdMs: number;
+    /** How often to check for stale peers (ms) */
+    checkIntervalMs: number;
+    /** Callback to broadcast message to all agents */
+    broadcast: (message: string) => Promise<void>;
+    /** Callback to get current load (0-1) */
+    getLoad?: () => number;
+    /** Callback to get current task count */
+    getTaskCount?: () => number;
+    /** Callback to check if this agent is leader */
+    isLeader?: () => boolean;
+}
+/**
+ * Gossip Health Monitor
+ *
+ * Broadcasts heartbeats and tracks peer health via gossip protocol.
+ */
+export declare class GossipHealthMonitor extends EventEmitter {
+    private config;
+    private peers;
+    private broadcastInterval?;
+    private checkInterval?;
+    private isRunning;
+    private healthy;
+    constructor(config: GossipHealthConfig);
+    /**
+     * Start gossip health monitoring
+     */
+    start(): void;
+    /**
+     * Stop gossip health monitoring
+     */
+    stop(): void;
+    /**
+     * Broadcast heartbeat to all agents
+     */
+    private broadcastHeartbeat;
+    /**
+     * Process incoming heartbeat from another agent
+     */
+    processHeartbeat(heartbeat: GossipHeartbeat): void;
+    /**
+     * Parse heartbeat from relay message
+     */
+    static parseHeartbeat(message: string): GossipHeartbeat | null;
+    /**
+     * Check for stale peers
+     */
+    private checkStalePeers;
+    /**
+     * Set this agent's health status
+     */
+    setHealthy(healthy: boolean): void;
+    /**
+     * Get all known peers
+     */
+    getPeers(): PeerHealth[];
+    /**
+     * Get healthy peers
+     */
+    getHealthyPeers(): PeerHealth[];
+    /**
+     * Get current leader from gossip
+     */
+    getLeader(): PeerHealth | null;
+    /**
+     * Get peer by name
+     */
+    getPeer(name: string): PeerHealth | undefined;
+    /**
+     * Get status
+     */
+    getStatus(): {
+        isRunning: boolean;
+        agentName: string;
+        peerCount: number;
+        healthyPeerCount: number;
+        leader: string | null;
+    };
+}
+/**
+ * Create gossip health monitor with defaults
+ */
+export declare function createGossipHealth(agentName: string, agentId: string, broadcast: (message: string) => Promise<void>, options?: {
+    getLoad?: () => number;
+    getTaskCount?: () => number;
+    isLeader?: () => boolean;
+}): GossipHealthMonitor;
+//# sourceMappingURL=gossip-health.d.ts.map