agent-relay 1.0.22 → 1.2.0

package/dist/resiliency/gossip-health.js

@@ -0,0 +1,241 @@
+/**
+ * Gossip-Based Health Broadcast
+ *
+ * Implements P4: Agents broadcast heartbeats via relay.
+ * Collective failure detection without central monitor.
+ *
+ * Each agent:
+ * - Periodically broadcasts HEARTBEAT message to all agents
+ * - Tracks health of all known peers
+ * - Detects failures when peer heartbeats go stale
+ * - Emits events for peer health changes
+ */
+import { EventEmitter } from 'events';
+const DEFAULT_CONFIG = {
+    broadcastIntervalMs: 10000,
+    staleThresholdMs: 30000,
+    checkIntervalMs: 5000,
+};
+/**
+ * Gossip Health Monitor
+ *
+ * Broadcasts heartbeats and tracks peer health via gossip protocol.
+ */
+export class GossipHealthMonitor extends EventEmitter {
+    config;
+    peers = new Map();
+    broadcastInterval;
+    checkInterval;
+    isRunning = false;
+    healthy = true;
+    constructor(config) {
+        super();
+        this.config = { ...DEFAULT_CONFIG, ...config };
+    }
+    /**
+     * Start gossip health monitoring
+     */
+    start() {
+        if (this.isRunning)
+            return;
+        this.isRunning = true;
+        console.log(`[gossip] Started health broadcast for ${this.config.agentName}`);
+        // Start broadcasting heartbeats
+        this.broadcastInterval = setInterval(async () => {
+            try {
+                await this.broadcastHeartbeat();
+            }
+            catch (err) {
+                console.error('[gossip] Broadcast error:', err);
+            }
+        }, this.config.broadcastIntervalMs);
+        // Start checking for stale peers
+        this.checkInterval = setInterval(() => {
+            this.checkStalePeers();
+        }, this.config.checkIntervalMs);
+        // Initial broadcast
+        this.broadcastHeartbeat().catch((err) => {
+            console.error('[gossip] Initial broadcast error:', err);
+        });
+        this.emit('started');
+    }
+    /**
+     * Stop gossip health monitoring
+     */
+    stop() {
+        if (!this.isRunning)
+            return;
+        this.isRunning = false;
+        if (this.broadcastInterval) {
+            clearInterval(this.broadcastInterval);
+            this.broadcastInterval = undefined;
+        }
+        if (this.checkInterval) {
+            clearInterval(this.checkInterval);
+            this.checkInterval = undefined;
+        }
+        console.log(`[gossip] Stopped health broadcast for ${this.config.agentName}`);
+        this.emit('stopped');
+    }
+    /**
+     * Broadcast heartbeat to all agents
+     */
+    async broadcastHeartbeat() {
+        const heartbeat = {
+            type: 'HEARTBEAT',
+            agent: this.config.agentName,
+            agentId: this.config.agentId,
+            timestamp: Date.now(),
+            load: this.config.getLoad?.() ?? 0,
+            healthy: this.healthy,
+            isLeader: this.config.isLeader?.() ?? false,
+            taskCount: this.config.getTaskCount?.() ?? 0,
+        };
+        const message = `HEARTBEAT: ${JSON.stringify(heartbeat)}`;
+        await this.config.broadcast(message);
+    }
+    /**
+     * Process incoming heartbeat from another agent
+     */
+    processHeartbeat(heartbeat) {
+        // Ignore our own heartbeats
+        if (heartbeat.agentId === this.config.agentId)
+            return;
+        const existing = this.peers.get(heartbeat.agent);
+        const wasHealthy = existing?.healthy ?? true;
+        const wasLeader = existing?.isLeader ?? false;
+        // Update peer state
+        const peer = {
+            name: heartbeat.agent,
+            lastHeartbeat: heartbeat.timestamp,
+            load: heartbeat.load,
+            healthy: heartbeat.healthy,
+            isLeader: heartbeat.isLeader,
+            taskCount: heartbeat.taskCount,
+        };
+        this.peers.set(heartbeat.agent, peer);
+        // Emit events for state changes
+        if (!existing) {
+            this.emit('peerDiscovered', peer);
+        }
+        else {
+            if (!wasHealthy && heartbeat.healthy) {
+                this.emit('peerRecovered', peer);
+            }
+            if (wasHealthy && !heartbeat.healthy) {
+                this.emit('peerUnhealthy', peer);
+            }
+            if (!wasLeader && heartbeat.isLeader) {
+                this.emit('newLeader', peer);
+            }
+        }
+    }
+    /**
+     * Parse heartbeat from relay message
+     */
+    static parseHeartbeat(message) {
+        const match = message.match(/^HEARTBEAT:\s*(.+)$/);
+        if (!match)
+            return null;
+        try {
+            const data = JSON.parse(match[1]);
+            if (data.type === 'HEARTBEAT') {
+                return data;
+            }
+        }
+        catch {
+            // Invalid JSON
+        }
+        return null;
+    }
+    /**
+     * Check for stale peers
+     */
+    checkStalePeers() {
+        const now = Date.now();
+        for (const [name, peer] of this.peers) {
+            const age = now - peer.lastHeartbeat;
+            if (age > this.config.staleThresholdMs) {
+                if (peer.healthy) {
+                    // Mark as unhealthy
+                    peer.healthy = false;
+                    console.log(`[gossip] Peer ${name} is stale (${Math.round(age / 1000)}s since last heartbeat)`);
+                    this.emit('peerStale', { peer, age });
+                    // If the stale peer was leader, emit leader lost
+                    if (peer.isLeader) {
+                        this.emit('leaderLost', peer);
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Set this agent's health status
+     */
+    setHealthy(healthy) {
+        if (this.healthy !== healthy) {
+            this.healthy = healthy;
+            // Broadcast immediately on health change
+            this.broadcastHeartbeat().catch((err) => {
+                console.error('[gossip] Immediate broadcast error:', err);
+            });
+        }
+    }
+    /**
+     * Get all known peers
+     */
+    getPeers() {
+        return Array.from(this.peers.values());
+    }
+    /**
+     * Get healthy peers
+     */
+    getHealthyPeers() {
+        return this.getPeers().filter((p) => p.healthy);
+    }
+    /**
+     * Get current leader from gossip
+     */
+    getLeader() {
+        for (const peer of this.peers.values()) {
+            if (peer.isLeader && peer.healthy) {
+                return peer;
+            }
+        }
+        return null;
+    }
+    /**
+     * Get peer by name
+     */
+    getPeer(name) {
+        return this.peers.get(name);
+    }
+    /**
+     * Get status
+     */
+    getStatus() {
+        const leader = this.getLeader();
+        return {
+            isRunning: this.isRunning,
+            agentName: this.config.agentName,
+            peerCount: this.peers.size,
+            healthyPeerCount: this.getHealthyPeers().length,
+            leader: leader?.name ?? null,
+        };
+    }
+}
+/**
+ * Create gossip health monitor with defaults
+ */
+export function createGossipHealth(agentName, agentId, broadcast, options) {
+    return new GossipHealthMonitor({
+        agentName,
+        agentId,
+        broadcast,
+        broadcastIntervalMs: 10000,
+        staleThresholdMs: 30000,
+        checkIntervalMs: 5000,
+        ...options,
+    });
+}
+//# sourceMappingURL=gossip-health.js.map

package/dist/resiliency/gossip-health.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gossip-health.js","sourceRoot":"","sources":["../../src/resiliency/gossip-health.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAoDtC,MAAM,cAAc,GAAgC;IAClD,mBAAmB,EAAE,KAAK;IAC1B,gBAAgB,EAAE,KAAK;IACvB,eAAe,EAAE,IAAI;CACtB,CAAC;AAEF;;;;GAIG;AACH,MAAM,OAAO,mBAAoB,SAAQ,YAAY;IAC3C,MAAM,CAAqB;IAC3B,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IACtC,iBAAiB,CAAkC;IACnD,aAAa,CAAkC;IAC/C,SAAS,GAAG,KAAK,CAAC;IAClB,OAAO,GAAG,IAAI,CAAC;IAEvB,YAAY,MAA0B;QACpC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAwB,CAAC;IACvE,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,OAAO,CAAC,GAAG,CAAC,yCAAyC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QAE9E,gCAAgC;QAChC,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YAC9C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAClC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC;YAClD,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;QAEpC,iCAAiC;QACjC,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;YACpC,IAAI,CAAC,eAAe,EAAE,CAAC;QACzB,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAEhC,oBAAoB;QACpB,IAAI,CAAC,kBAAkB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACtC,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,IAAI;QACF,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QAEvB,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACtC,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAClC,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;QACjC,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,yCAAyC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QAC9E,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACvB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB;QAC9B,MAAM,SAAS,GAAoB;YACjC,IAAI,EAAE,WAAW;YACjB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YAC5B,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC;YAClC,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,KAAK;YAC3C,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,EAAE,IAAI,CAAC;SAC7C,CAAC;QAEF,MAAM,OAAO,GAAG,cAAc,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,SAA0B;QACzC,4BAA4B;QAC5B,IAAI,SAAS,CAAC,OAAO,KAAK,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO;QAEtD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,QAAQ,EAAE,OAAO,IAAI,IAAI,CAAC;QAC7C,MAAM,SAAS,GAAG,QAAQ,EAAE,QAAQ,IAAI,KAAK,CAAC;QAE9C,oBAAoB;QACpB,MAAM,IAAI,GAAe;YACvB,IAAI,EAAE,SAAS,CAAC,KAAK;YACrB,aAAa,EAAE,SAAS,CAAC,SAAS;YAClC,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,QAAQ,EAAE,SAAS,CAAC,QAAQ;YAC5B,SAAS,EAAE,SAAS,CAAC,SAAS;SAC/B,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAEtC,gCAAgC;QAChC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,UAAU,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;gBACrC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;YACnC,CAAC;YACD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;gBACrC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;YACnC,CAAC;YACD,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;gBACrC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,cAAc,CAAC,OAAe;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAExB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBAC9B,OAAO,IAAuB,CAAC;YACjC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,eAAe;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC;YAErC,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBACvC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBACjB,oBAAoB;oBACpB,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;oBACrB,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,cAAc,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,yBAAyB,CAAC,CAAC;oBAChG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;oBAEtC,iDAAiD;oBACjD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;wBAClB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;oBAChC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,OAAgB;QACzB,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;YACvB,yCAAyC;YACzC,IAAI,CAAC,kBAAkB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACtC,OAAO,CAAC,KAAK,CAAC,qCAAqC,EAAE,GAAG,CAAC,CAAC;YAC5D,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,SAAS;QACP,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACvC,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClC,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,IAAY;QAClB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,SAAS;QAOP,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAChC,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YAChC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YAC1B,gBAAgB,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,MAAM;YAC/C,MAAM,EAAE,MAAM,EAAE,IAAI,IAAI,IAAI;SAC7B,CAAC;IACJ,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,SAAiB,EACjB,OAAe,EACf,SAA6C,EAC7C,OAIC;IAED,OAAO,IAAI,mBAAmB,CAAC;QAC7B,SAAS;QACT,OAAO;QACP,SAAS;QACT,mBAAmB,EAAE,KAAK;QAC1B,gBAAgB,EAAE,KAAK;QACvB,eAAe,EAAE,IAAI;QACrB,GAAG,OAAO;KACX,CAAC,CAAC;AACL,CAAC"}

package/dist/resiliency/health-monitor.d.ts

@@ -0,0 +1,97 @@
+/**
+ * Agent Health Monitor
+ *
+ * Monitors spawned agent processes and ensures they stay alive.
+ * - Periodic health checks (process liveness)
+ * - Auto-restart on crash
+ * - Death detection and logging
+ * - Metrics collection
+ */
+import { EventEmitter } from 'events';
+export interface AgentHealth {
+    name: string;
+    pid: number;
+    status: 'healthy' | 'unresponsive' | 'dead' | 'restarting';
+    lastHealthCheck: Date;
+    lastResponse: Date;
+    restartCount: number;
+    consecutiveFailures: number;
+    uptime: number;
+    startedAt: Date;
+    memoryUsage?: number;
+    cpuUsage?: number;
+    lastError?: string;
+}
+export interface HealthMonitorConfig {
+    checkIntervalMs: number;
+    responseTimeoutMs: number;
+    maxRestarts: number;
+    restartCooldownMs: number;
+    maxConsecutiveFailures: number;
+}
+export interface AgentProcess {
+    name: string;
+    pid: number;
+    isAlive: () => boolean;
+    kill: (signal?: string) => void;
+    restart: () => Promise<void>;
+    sendHealthCheck?: () => Promise<boolean>;
+}
+export declare class AgentHealthMonitor extends EventEmitter {
+    private agents;
+    private health;
+    private intervalId?;
+    private config;
+    private isRunning;
+    constructor(config?: Partial<HealthMonitorConfig>);
+    /**
+     * Register an agent for health monitoring
+     */
+    register(agent: AgentProcess): void;
+    /**
+     * Unregister an agent from health monitoring
+     */
+    unregister(name: string): void;
+    /**
+     * Start the health monitoring loop
+     */
+    start(): void;
+    /**
+     * Stop the health monitoring loop
+     */
+    stop(): void;
+    /**
+     * Get health status for all agents
+     */
+    getAll(): AgentHealth[];
+    /**
+     * Get health status for a specific agent
+     */
+    get(name: string): AgentHealth | undefined;
+    /**
+     * Check health of all registered agents
+     */
+    private checkAll;
+    /**
+     * Check health of a single agent
+     */
+    private checkAgent;
+    /**
+     * Handle agent death - attempt restart or mark as dead
+     */
+    private handleDeath;
+    /**
+     * Check if a process is alive by PID
+     */
+    private isProcessAlive;
+    /**
+     * Get memory and CPU usage for a process
+     */
+    private getProcessUsage;
+    /**
+     * Structured logging
+     */
+    private log;
+}
+export declare function getHealthMonitor(config?: Partial<HealthMonitorConfig>): AgentHealthMonitor;
+//# sourceMappingURL=health-monitor.d.ts.map

package/dist/resiliency/health-monitor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"health-monitor.d.ts","sourceRoot":"","sources":["../../src/resiliency/health-monitor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,SAAS,GAAG,cAAc,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3D,eAAe,EAAE,IAAI,CAAC;IACtB,YAAY,EAAE,IAAI,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,IAAI,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,OAAO,CAAC;IACvB,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,eAAe,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;CAC1C;AAUD,qBAAa,kBAAmB,SAAQ,YAAY;IAClD,OAAO,CAAC,MAAM,CAAmC;IACjD,OAAO,CAAC,MAAM,CAAkC;IAChD,OAAO,CAAC,UAAU,CAAC,CAAiC;IACpD,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,SAAS,CAAS;gBAEd,MAAM,GAAE,OAAO,CAAC,mBAAmB,CAAM;IAKrD;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IAkBnC;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAO9B;;OAEG;IACH,KAAK,IAAI,IAAI;IAiBb;;OAEG;IACH,IAAI,IAAI,IAAI;IASZ;;OAEG;IACH,MAAM,IAAI,WAAW,EAAE;IAOvB;;OAEG;IACH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAQ1C;;OAEG;YACW,QAAQ;IAOtB;;OAEG;YACW,UAAU;IAoExB;;OAEG;YACW,WAAW;IAgEzB;;OAEG;IACH,OAAO,CAAC,cAAc;IAUtB;;OAEG;YACW,eAAe;IAgB7B;;OAEG;IACH,OAAO,CAAC,GAAG;CA6BZ;AAKD,wBAAgB,gBAAgB,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,kBAAkB,CAK1F"}

package/dist/resiliency/health-monitor.js

@@ -0,0 +1,291 @@
+/**
+ * Agent Health Monitor
+ *
+ * Monitors spawned agent processes and ensures they stay alive.
+ * - Periodic health checks (process liveness)
+ * - Auto-restart on crash
+ * - Death detection and logging
+ * - Metrics collection
+ */
+import { EventEmitter } from 'events';
+const DEFAULT_CONFIG = {
+    checkIntervalMs: 5000,
+    responseTimeoutMs: 10000,
+    maxRestarts: 5,
+    restartCooldownMs: 2000,
+    maxConsecutiveFailures: 3,
+};
+export class AgentHealthMonitor extends EventEmitter {
+    agents = new Map();
+    health = new Map();
+    intervalId;
+    config;
+    isRunning = false;
+    constructor(config = {}) {
+        super();
+        this.config = { ...DEFAULT_CONFIG, ...config };
+    }
+    /**
+     * Register an agent for health monitoring
+     */
+    register(agent) {
+        this.agents.set(agent.name, agent);
+        this.health.set(agent.name, {
+            name: agent.name,
+            pid: agent.pid,
+            status: 'healthy',
+            lastHealthCheck: new Date(),
+            lastResponse: new Date(),
+            restartCount: 0,
+            consecutiveFailures: 0,
+            uptime: 0,
+            startedAt: new Date(),
+        });
+        this.emit('registered', { name: agent.name, pid: agent.pid });
+        this.log('info', `Registered agent for health monitoring: ${agent.name} (PID: ${agent.pid})`);
+    }
+    /**
+     * Unregister an agent from health monitoring
+     */
+    unregister(name) {
+        this.agents.delete(name);
+        this.health.delete(name);
+        this.emit('unregistered', { name });
+        this.log('info', `Unregistered agent: ${name}`);
+    }
+    /**
+     * Start the health monitoring loop
+     */
+    start() {
+        if (this.isRunning)
+            return;
+        this.isRunning = true;
+        this.log('info', 'Health monitor started', {
+            checkInterval: this.config.checkIntervalMs,
+            maxRestarts: this.config.maxRestarts,
+        });
+        this.intervalId = setInterval(() => {
+            this.checkAll();
+        }, this.config.checkIntervalMs);
+        // Initial check
+        this.checkAll();
+    }
+    /**
+     * Stop the health monitoring loop
+     */
+    stop() {
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+            this.intervalId = undefined;
+        }
+        this.isRunning = false;
+        this.log('info', 'Health monitor stopped');
+    }
+    /**
+     * Get health status for all agents
+     */
+    getAll() {
+        return Array.from(this.health.values()).map((h) => ({
+            ...h,
+            uptime: Date.now() - h.startedAt.getTime(),
+        }));
+    }
+    /**
+     * Get health status for a specific agent
+     */
+    get(name) {
+        const health = this.health.get(name);
+        if (health) {
+            return { ...health, uptime: Date.now() - health.startedAt.getTime() };
+        }
+        return undefined;
+    }
+    /**
+     * Check health of all registered agents
+     */
+    async checkAll() {
+        const checks = Array.from(this.agents.entries()).map(([name, agent]) => this.checkAgent(name, agent));
+        await Promise.all(checks);
+    }
+    /**
+     * Check health of a single agent
+     */
+    async checkAgent(name, agent) {
+        const health = this.health.get(name);
+        if (!health)
+            return;
+        health.lastHealthCheck = new Date();
+        try {
+            // First check: Is the process alive?
+            const isAlive = this.isProcessAlive(agent.pid);
+            if (!isAlive) {
+                await this.handleDeath(name, agent, health, 'Process not found');
+                return;
+            }
+            // Second check: Does it respond to health check?
+            if (agent.sendHealthCheck) {
+                const responded = await Promise.race([
+                    agent.sendHealthCheck(),
+                    new Promise((resolve) => setTimeout(() => resolve(false), this.config.responseTimeoutMs)),
+                ]);
+                if (!responded) {
+                    health.consecutiveFailures++;
+                    this.log('warn', `Agent unresponsive: ${name}`, {
+                        failures: health.consecutiveFailures,
+                        max: this.config.maxConsecutiveFailures,
+                    });
+                    if (health.consecutiveFailures >= this.config.maxConsecutiveFailures) {
+                        health.status = 'unresponsive';
+                        await this.handleDeath(name, agent, health, 'Unresponsive after multiple health checks');
+                    }
+                    else {
+                        health.status = 'unresponsive';
+                        this.emit('unhealthy', { name, health });
+                    }
+                    return;
+                }
+            }
+            // Get memory/CPU usage if available
+            try {
+                const usage = await this.getProcessUsage(agent.pid);
+                health.memoryUsage = usage.memory;
+                health.cpuUsage = usage.cpu;
+            }
+            catch {
+                // Ignore usage errors
+            }
+            // All good
+            health.status = 'healthy';
+            health.lastResponse = new Date();
+            health.consecutiveFailures = 0;
+            this.emit('healthy', { name, health });
+        }
+        catch (error) {
+            health.consecutiveFailures++;
+            health.lastError = error instanceof Error ? error.message : String(error);
+            this.log('error', `Health check error for ${name}`, { error: health.lastError });
+            if (health.consecutiveFailures >= this.config.maxConsecutiveFailures) {
+                await this.handleDeath(name, agent, health, health.lastError);
+            }
+        }
+    }
+    /**
+     * Handle agent death - attempt restart or mark as dead
+     */
+    async handleDeath(name, agent, health, reason) {
+        this.log('error', `Agent died: ${name}`, {
+            reason,
+            restartCount: health.restartCount,
+            maxRestarts: this.config.maxRestarts,
+        });
+        this.emit('died', { name, reason, restartCount: health.restartCount });
+        // Check if we should attempt restart
+        if (health.restartCount >= this.config.maxRestarts) {
+            health.status = 'dead';
+            health.lastError = `Exceeded max restarts (${this.config.maxRestarts}): ${reason}`;
+            this.log('error', `Agent permanently dead: ${name}`, { reason: health.lastError });
+            this.emit('permanentlyDead', { name, health });
+            return;
+        }
+        // Attempt restart
+        health.status = 'restarting';
+        health.restartCount++;
+        this.log('info', `Attempting restart ${health.restartCount}/${this.config.maxRestarts}: ${name}`);
+        this.emit('restarting', { name, attempt: health.restartCount });
+        // Wait cooldown
+        await new Promise((resolve) => setTimeout(resolve, this.config.restartCooldownMs));
+        try {
+            await agent.restart();
+            // Update health after successful restart
+            health.status = 'healthy';
+            health.consecutiveFailures = 0;
+            health.startedAt = new Date();
+            health.lastResponse = new Date();
+            health.pid = agent.pid;
+            this.log('info', `Agent restarted successfully: ${name}`, {
+                newPid: agent.pid,
+                attempt: health.restartCount,
+            });
+            this.emit('restarted', { name, pid: agent.pid, attempt: health.restartCount });
+        }
+        catch (error) {
+            health.lastError = error instanceof Error ? error.message : String(error);
+            this.log('error', `Restart failed: ${name}`, { error: health.lastError });
+            this.emit('restartFailed', { name, error: health.lastError });
+            // Recursively try again if under limit
+            if (health.restartCount < this.config.maxRestarts) {
+                await this.handleDeath(name, agent, health, health.lastError);
+            }
+            else {
+                health.status = 'dead';
+                this.emit('permanentlyDead', { name, health });
+            }
+        }
+    }
+    /**
+     * Check if a process is alive by PID
+     */
+    isProcessAlive(pid) {
+        try {
+            // Sending signal 0 checks if process exists without killing it
+            process.kill(pid, 0);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Get memory and CPU usage for a process
+     */
+    async getProcessUsage(pid) {
+        const { execSync } = await import('child_process');
+        try {
+            // This works on Linux/Mac
+            const output = execSync(`ps -o rss=,pcpu= -p ${pid}`, { encoding: 'utf8' }).trim();
+            const [rss, cpu] = output.split(/\s+/);
+            return {
+                memory: parseInt(rss, 10) * 1024, // RSS in bytes
+                cpu: parseFloat(cpu),
+            };
+        }
+        catch {
+            return { memory: 0, cpu: 0 };
+        }
+    }
+    /**
+     * Structured logging
+     */
+    log(level, message, context) {
+        const entry = {
+            timestamp: new Date().toISOString(),
+            level,
+            component: 'health-monitor',
+            message,
+            ...context,
+        };
+        this.emit('log', entry);
+        // Also log to console with structure
+        const prefix = `[health-monitor]`;
+        switch (level) {
+            case 'info':
+                console.log(prefix, message, context ? JSON.stringify(context) : '');
+                break;
+            case 'warn':
+                console.warn(prefix, message, context ? JSON.stringify(context) : '');
+                break;
+            case 'error':
+                console.error(prefix, message, context ? JSON.stringify(context) : '');
+                break;
+        }
+    }
+}
+// Singleton instance
+let _monitor = null;
+export function getHealthMonitor(config) {
+    if (!_monitor) {
+        _monitor = new AgentHealthMonitor(config);
+    }
+    return _monitor;
+}
+//# sourceMappingURL=health-monitor.js.map

package/dist/resiliency/health-monitor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"health-monitor.js","sourceRoot":"","sources":["../../src/resiliency/health-monitor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAkCtC,MAAM,cAAc,GAAwB;IAC1C,eAAe,EAAE,IAAI;IACrB,iBAAiB,EAAE,KAAK;IACxB,WAAW,EAAE,CAAC;IACd,iBAAiB,EAAE,IAAI;IACvB,sBAAsB,EAAE,CAAC;CAC1B,CAAC;AAEF,MAAM,OAAO,kBAAmB,SAAQ,YAAY;IAC1C,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IACzC,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;IACxC,UAAU,CAAkC;IAC5C,MAAM,CAAsB;IAC5B,SAAS,GAAG,KAAK,CAAC;IAE1B,YAAY,SAAuC,EAAE;QACnD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,KAAmB;QAC1B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE;YAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,MAAM,EAAE,SAAS;YACjB,eAAe,EAAE,IAAI,IAAI,EAAE;YAC3B,YAAY,EAAE,IAAI,IAAI,EAAE;YACxB,YAAY,EAAE,CAAC;YACf,mBAAmB,EAAE,CAAC;YACtB,MAAM,EAAE,CAAC;YACT,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;QAC9D,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,2CAA2C,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;IAChG,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,IAAY;QACrB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,uBAAuB,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,wBAAwB,EAAE;YACzC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe;YAC1C,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;SACrC,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE;YACjC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAEhC,gBAAgB;QAChB,IAAI,CAAC,QAAQ,EAAE,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,IAAI;QACF,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAClD,GAAG,CAAC;YACJ,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE;SAC3C,CAAC,CAAC,CAAC;IACN,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,IAAY;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;QACxE,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,QAAQ;QACpB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CACrE,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,CAC7B,CAAC;QACF,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,IAAY,EAAE,KAAmB;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,MAAM,CAAC,eAAe,GAAG,IAAI,IAAI,EAAE,CAAC;QAEpC,IAAI,CAAC;YACH,qCAAqC;YACrC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAE/C,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,CAAC,CAAC;gBACjE,OAAO;YACT,CAAC;YAED,iDAAiD;YACjD,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;gBAC1B,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;oBACnC,KAAK,CAAC,eAAe,EAAE;oBACvB,IAAI,OAAO,CAAQ,CAAC,OAAO,EAAE,EAAE,CAC7B,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAChE;iBACF,CAAC,CAAC;gBAEH,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,MAAM,CAAC,mBAAmB,EAAE,CAAC;oBAC7B,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,uBAAuB,IAAI,EAAE,EAAE;wBAC9C,QAAQ,EAAE,MAAM,CAAC,mBAAmB;wBACpC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,sBAAsB;qBACxC,CAAC,CAAC;oBAEH,IAAI,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;wBACrE,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC;wBAC/B,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,2CAA2C,CAAC,CAAC;oBAC3F,CAAC;yBAAM,CAAC;wBACN,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC;wBAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;oBAC3C,CAAC;oBACD,OAAO;gBACT,CAAC;YACH,CAAC;YAED,oCAAoC;YACpC,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACpD,MAAM,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC;gBAClC,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;YAED,WAAW;YACX,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;YAC1B,MAAM,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;YACjC,MAAM,CAAC,mBAAmB,GAAG,CAAC,CAAC;YAE/B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,mBAAmB,EAAE,CAAC;YAC7B,MAAM,CAAC,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC1E,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,0BAA0B,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;YAEjF,IAAI,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC;gBACrE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CACvB,IAAY,EACZ,KAAmB,EACnB,MAAmB,EACnB,MAAc;QAEd,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,IAAI,EAAE,EAAE;YACvC,MAAM;YACN,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;SACrC,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QAEvE,qCAAqC;QACrC,IAAI,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YACnD,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;YACvB,MAAM,CAAC,SAAS,GAAG,0BAA0B,IAAI,CAAC,MAAM,CAAC,WAAW,MAAM,MAAM,EAAE,CAAC;YACnF,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,2BAA2B,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;YACnF,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YAC/C,OAAO;QACT,CAAC;QAED,kBAAkB;QAClB,MAAM,CAAC,MAAM,GAAG,YAAY,CAAC;QAC7B,MAAM,CAAC,YAAY,EAAE,CAAC;QAEtB,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,sBAAsB,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC,CAAC;QAClG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QAEhE,gBAAgB;QAChB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAEnF,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YAEtB,yCAAyC;YACzC,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;YAC1B,MAAM,CAAC,mBAAmB,GAAG,CAAC,CAAC;YAC/B,MAAM,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,MAAM,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;YAEvB,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,iCAAiC,IAAI,EAAE,EAAE;gBACxD,MAAM,EAAE,KAAK,CAAC,GAAG;gBACjB,OAAO,EAAE,MAAM,CAAC,YAAY;aAC7B,CAAC,CAAC;YAEH,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QACjF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC1E,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,mBAAmB,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;YAC1E,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;YAE9D,uCAAuC;YACvC,IAAI,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBAClD,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAChE,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;gBACvB,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,GAAW;QAChC,IAAI,CAAC;YACH,+DAA+D;YAC/D,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAAC,GAAW;QACvC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;QAEnD,IAAI,CAAC;YACH,0BAA0B;YAC1B,MAAM,MAAM,GAAG,QAAQ,CAAC,uBAAuB,GAAG,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACnF,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,eAAe;gBACjD,GAAG,EAAE,UAAU,CAAC,GAAG,CAAC;aACrB,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;OAEG;IACK,GAAG,CACT,KAAgC,EAChC,OAAe,EACf,OAAiC;QAEjC,MAAM,KAAK,GAAG;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,KAAK;YACL,SAAS,EAAE,gBAAgB;YAC3B,OAAO;YACP,GAAG,OAAO;SACX,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAExB,qCAAqC;QACrC,MAAM,MAAM,GAAG,kBAAkB,CAAC;QAClC,QAAQ,KAAK,EAAE,CAAC;YACd,KAAK,MAAM;gBACT,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrE,MAAM;YACR,KAAK,MAAM;gBACT,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACtE,MAAM;YACR,KAAK,OAAO;gBACV,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;QACV,CAAC;IACH,CAAC;CACF;AAED,qBAAqB;AACrB,IAAI,QAAQ,GAA8B,IAAI,CAAC;AAE/C,MAAM,UAAU,gBAAgB,CAAC,MAAqC;IACpE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}