@sequence0/blockclaw 2.0.0

package/dist/core/healer.js

@@ -0,0 +1,355 @@
+import fs from "node:fs";
+import { isAgentRunning, stopAgent, spawnAgent, findAvailablePort, isPortAvailable, findAgentProcess } from "../agent/process.js";
+import { resolveBinary, downloadBinary } from "../agent/binary.js";
+import { mergeConfigs, readBlockclawConfig, setConfigValue } from "../agent/config.js";
+import { logger } from "../utils/logger.js";
+import { AGENT_LOG } from "../utils/paths.js";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const DEFAULT_MAX_RESTARTS = 5;
+const DEFAULT_WINDOW_MS = 300_000; // 5 minutes
+const BACKOFF_BASE_MS = 1_000;
+const BACKOFF_MAX_EXPONENT = 4; // cap at 2^4 = 16s
+// ---------------------------------------------------------------------------
+// SelfHealer
+// ---------------------------------------------------------------------------
+export class SelfHealer {
+    client;
+    memory;
+    maxRestarts;
+    windowMs;
+    /** Timestamps of recent restarts within the rolling window */
+    restartTimestamps = [];
+    /** Consecutive restart count for exponential backoff */
+    consecutiveRestarts = 0;
+    constructor(client, memory, options) {
+        this.client = client;
+        this.memory = memory;
+        this.maxRestarts = options?.maxRestarts ?? DEFAULT_MAX_RESTARTS;
+        this.windowMs = options?.windowMs ?? DEFAULT_WINDOW_MS;
+    }
+    // -------------------------------------------------------------------------
+    // Public API
+    // -------------------------------------------------------------------------
+    /**
+     * Evaluate a heartbeat event and decide on a healing action.
+     * Executes the action and returns which action was taken.
+     */
+    async handleEvent(event) {
+        // Check restart budget first
+        this._pruneRestartWindow();
+        if (this.restartTimestamps.length >= this.maxRestarts) {
+            // Too many restarts in window — stop healing, alert operator
+            return this._alertOperator(`Too many restarts (${this.restartTimestamps.length}/${this.maxRestarts}) within ${this.windowMs / 1000}s window. Self-healing paused.`);
+        }
+        switch (event.type) {
+            case "health_ok":
+                // Agent is healthy — reset backoff counter so future restarts start fresh
+                if (this.consecutiveRestarts > 0) {
+                    logger.debug(`Agent healthy — resetting restart counter (was ${this.consecutiveRestarts})`);
+                    this.consecutiveRestarts = 0;
+                }
+                return "none";
+            case "agent_down":
+                return this._handleAgentDown();
+            case "peers_lost":
+                return this._handlePeersLost();
+            case "disk_warning":
+                return this._handleDiskWarning();
+            case "memory_warning":
+                return this._handleMemoryWarning();
+            case "update_available":
+                return this._handleUpdateAvailable(event);
+            case "health_degraded":
+                return this._handleHealthDegraded(event);
+            case "health_critical":
+                return this._handleHealthCritical(event);
+            case "network_unreachable":
+                logger.warn("Internet connectivity lost — waiting for network to recover (no restart)");
+                return "none";
+            case "rpc_unreachable":
+                logger.warn("Sequence0 RPC is unreachable — waiting for RPC to recover (no restart)");
+                return "none";
+            case "rpc_stale":
+                return this._alertOperator(`Chain RPC is stale — block number has not changed for ${event.data?.staleChecks ?? "multiple"} consecutive checks. ` +
+                    `RPC: ${event.data?.rpcUrl ?? "unknown"}, block: ${event.data?.blockNumber ?? "unknown"}`);
+            default:
+                return "none";
+        }
+    }
+    /** Reset restart counter and backoff state. */
+    reset() {
+        this.restartTimestamps = [];
+        this.consecutiveRestarts = 0;
+        logger.debug("SelfHealer counters reset");
+    }
+    // -------------------------------------------------------------------------
+    // Event handlers
+    // -------------------------------------------------------------------------
+    /** Agent is unreachable — decide between restart and port conflict fix. */
+    async _handleAgentDown() {
+        const { running } = isAgentRunning();
+        if (!running) {
+            // Process is dead — straightforward restart
+            logger.warn("Node process is not running, attempting restart...");
+            return this._restartAgent();
+        }
+        // Process is alive but not responding — could be a port conflict
+        const config = mergeConfigs();
+        const portFree = await isPortAvailable(config.apiPort);
+        if (!portFree) {
+            // Something else is on the port, or agent is stuck
+            const agentPid = await findAgentProcess();
+            if (agentPid) {
+                // Our agent is running but not responding on expected port — port conflict
+                logger.warn(`Port ${config.apiPort} occupied, fixing port conflict...`);
+                return this._fixPortConflict();
+            }
+        }
+        // Agent process exists but unresponsive — restart
+        logger.warn("Node unresponsive, restarting...");
+        return this._restartAgent();
+    }
+    /** Zero peers — restart agent to re-bootstrap. */
+    async _handlePeersLost() {
+        logger.warn("No connected peers, restarting node to reconnect...");
+        return this._reconnectPeers();
+    }
+    /** Disk usage above 95% — truncate agent logs. */
+    async _handleDiskWarning() {
+        logger.warn("Disk usage critical, rotating logs...");
+        return this._rotateLogs();
+    }
+    /** Memory usage above 90% — restart agent to reclaim memory. */
+    async _handleMemoryWarning() {
+        logger.warn("Memory usage critical, restarting node...");
+        return this._restartAgent();
+    }
+    /** Update available — download and restart if auto-update is enabled. */
+    async _handleUpdateAvailable(event) {
+        const bcConfig = readBlockclawConfig();
+        if (!bcConfig.auto_update) {
+            logger.info(`Update available (${event.data?.version}), but auto_update is disabled`);
+            return "none";
+        }
+        const version = event.data?.version;
+        if (!version)
+            return "none";
+        logger.info(`Auto-updating node to ${version}...`);
+        try {
+            await stopAgent(mergeConfigs().apiPort);
+            await downloadBinary(version);
+            await this._spawnWithBackoff();
+            this._recordRestart();
+            logger.success(`Node updated to ${version} and restarted`);
+            return "update_binary";
+        }
+        catch (err) {
+            logger.error(`Auto-update failed: ${err instanceof Error ? err.message : String(err)}`);
+            return this._alertOperator(`Auto-update to ${version} failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    // -------------------------------------------------------------------------
+    // Healing actions
+    // -------------------------------------------------------------------------
+    /** Stop the agent (if running), resolve binary, and spawn with merged config. */
+    async _restartAgent() {
+        try {
+            // Apply exponential backoff
+            await this._backoff();
+            const config = mergeConfigs();
+            // Stop existing agent if running
+            const { running } = isAgentRunning();
+            if (running) {
+                await stopAgent(config.apiPort);
+            }
+            await this._spawnWithBackoff();
+            this._recordRestart();
+            this.consecutiveRestarts++;
+            logger.success("Node restarted successfully");
+            return "restart_agent";
+        }
+        catch (err) {
+            logger.error(`Restart failed: ${err instanceof Error ? err.message : String(err)}`);
+            return this._alertOperator(`Restart failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    /** Find an available port, update config, and restart. */
+    async _fixPortConflict() {
+        try {
+            const config = mergeConfigs();
+            // Stop agent first
+            const { running } = isAgentRunning();
+            if (running) {
+                await stopAgent(config.apiPort);
+            }
+            // Find a new available port
+            const newPort = await findAvailablePort(config.apiPort + 1);
+            logger.info(`Switching API port from ${config.apiPort} to ${newPort}`);
+            // Resolve binary and spawn with new port
+            const binPath = await resolveBinary();
+            spawnAgent({
+                binPath,
+                network: config.network,
+                port: config.port,
+                apiPort: newPort,
+                dataDir: config.dataDir,
+                rpcUrl: config.rpcUrl,
+                registry: config.registry,
+                walletFactory: config.walletFactory,
+                feeCollector: config.feeCollector,
+                programRegistry: config.programRegistry,
+                delegationRegistry: config.delegationRegistry,
+                daemon: config.daemon,
+                bootstrap: config.bootstrap,
+                logLevel: config.logLevel,
+                externalIp: config.externalIp,
+                noRegistry: config.noRegistry,
+            });
+            // Persist the new port to config so future restarts use it
+            setConfigValue("api_port", String(newPort));
+            this._recordRestart();
+            logger.success(`Node restarted on port ${newPort}`);
+            return "fix_port_conflict";
+        }
+        catch (err) {
+            logger.error(`Port conflict fix failed: ${err instanceof Error ? err.message : String(err)}`);
+            return this._alertOperator(`Port conflict fix failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    /** Restart the agent to re-bootstrap peer connections. */
+    async _reconnectPeers() {
+        try {
+            await this._backoff();
+            const config = mergeConfigs();
+            const { running } = isAgentRunning();
+            if (running) {
+                await stopAgent(config.apiPort);
+            }
+            await this._spawnWithBackoff();
+            this._recordRestart();
+            this.consecutiveRestarts++;
+            logger.success("Node restarted to reconnect peers");
+            return "reconnect_peers";
+        }
+        catch (err) {
+            logger.error(`Peer reconnect failed: ${err instanceof Error ? err.message : String(err)}`);
+            return this._alertOperator(`Peer reconnect failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    /** Truncate the agent log file to reclaim disk space. */
+    async _rotateLogs() {
+        try {
+            if (fs.existsSync(AGENT_LOG)) {
+                // Truncate the log file to zero bytes
+                fs.truncateSync(AGENT_LOG, 0);
+                logger.success(`Rotated (truncated) agent log: ${AGENT_LOG}`);
+            }
+            else {
+                logger.debug("No agent log file found to rotate");
+            }
+            return "rotate_logs";
+        }
+        catch (err) {
+            logger.error(`Log rotation failed: ${err instanceof Error ? err.message : String(err)}`);
+            return "rotate_logs"; // still report the action
+        }
+    }
+    /** Committee is degraded — alert operator about at-risk wallets. */
+    async _handleHealthDegraded(event) {
+        const walletId = event.data?.walletId ?? "unknown";
+        const alive = event.data?.alive ?? 0;
+        const total = event.data?.total ?? 0;
+        const threshold = event.data?.threshold ?? 0;
+        const deadPeers = event.data?.deadPeers ?? [];
+        logger.warn(`Wallet "${walletId}" security group is degraded: ${alive}/${total} members online ` +
+            `(need ${threshold} minimum signers). ${deadPeers.length} member(s) offline.`);
+        if (alive <= threshold) {
+            logger.warn(`Wallet "${walletId}" is at risk — approaching minimum signers needed. ` +
+                `Consider running: blockclaw wallet reshare ${walletId}`);
+        }
+        // Record the issue for tracking
+        try {
+            this.memory.recordIssue(`Committee degraded: ${walletId} (${alive}/${total} alive, threshold=${threshold})`, { walletId, alive, total, threshold, deadPeers });
+        }
+        catch {
+            // Non-fatal
+        }
+        return "alert_operator";
+    }
+    /** Committee is critical/dead — urgent alert, suggest reshare. */
+    async _handleHealthCritical(event) {
+        const walletId = event.data?.walletId ?? "unknown";
+        const alive = event.data?.alive ?? 0;
+        const total = event.data?.total ?? 0;
+        const threshold = event.data?.threshold ?? 0;
+        return this._alertOperator(`CRITICAL: Wallet "${walletId}" cannot sign — only ${alive}/${total} members online ` +
+            `(need ${threshold} minimum). Run: blockclaw wallet reshare ${walletId}`);
+    }
+    /** Emit a critical alert and stop self-healing. */
+    _alertOperator(message) {
+        logger.error("===================================================");
+        logger.error("  OPERATOR ALERT — Self-healing has been paused");
+        logger.error(`  ${message}`);
+        logger.error("===================================================");
+        logger.error("Manual intervention required. Run: blockclaw doctor");
+        // Record the alert in memory for persistence
+        try {
+            this.memory.recordIssue(message, {
+                type: "operator_alert",
+                timestamp: Date.now(),
+            });
+        }
+        catch {
+            // Memory may not be initialized — non-fatal
+        }
+        return "alert_operator";
+    }
+    // -------------------------------------------------------------------------
+    // Helpers
+    // -------------------------------------------------------------------------
+    /** Resolve binary, merge config, and spawn the agent process. */
+    async _spawnWithBackoff() {
+        const binPath = await resolveBinary();
+        const config = mergeConfigs();
+        spawnAgent({
+            binPath,
+            network: config.network,
+            port: config.port,
+            apiPort: config.apiPort,
+            dataDir: config.dataDir,
+            rpcUrl: config.rpcUrl,
+            registry: config.registry,
+            walletFactory: config.walletFactory,
+            feeCollector: config.feeCollector,
+            programRegistry: config.programRegistry,
+            delegationRegistry: config.delegationRegistry,
+            daemon: config.daemon,
+            bootstrap: config.bootstrap,
+            logLevel: config.logLevel,
+            externalIp: config.externalIp,
+            noRegistry: config.noRegistry,
+        });
+    }
+    /** Apply exponential backoff before a restart: 1s, 2s, 4s, 8s, 16s. */
+    async _backoff() {
+        const exponent = Math.min(this.consecutiveRestarts, BACKOFF_MAX_EXPONENT);
+        const delayMs = BACKOFF_BASE_MS * Math.pow(2, exponent);
+        if (delayMs > BACKOFF_BASE_MS) {
+            logger.info(`Backoff: waiting ${delayMs}ms before restart (attempt ${this.consecutiveRestarts + 1})`);
+            await new Promise((resolve) => setTimeout(resolve, delayMs));
+        }
+    }
+    /** Record a restart timestamp in the rolling window. */
+    _recordRestart() {
+        this.restartTimestamps.push(Date.now());
+    }
+    /** Remove restart timestamps older than the rolling window. */
+    _pruneRestartWindow() {
+        const cutoff = Date.now() - this.windowMs;
+        this.restartTimestamps = this.restartTimestamps.filter((ts) => ts > cutoff);
+    }
+}
+export default SelfHealer;
+//# sourceMappingURL=healer.js.map

package/dist/core/healer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"healer.js","sourceRoot":"","sources":["../../src/core/healer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AAGzB,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,UAAU,EAAE,iBAAiB,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAClI,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAyB9C,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAC/B,MAAM,iBAAiB,GAAG,OAAO,CAAC,CAAC,YAAY;AAC/C,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,oBAAoB,GAAG,CAAC,CAAC,CAAC,mBAAmB;AAEnD,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,MAAM,OAAO,UAAU;IACJ,MAAM,CAAc;IACpB,MAAM,CAAS;IACf,WAAW,CAAS;IACpB,QAAQ,CAAS;IAElC,8DAA8D;IACtD,iBAAiB,GAAa,EAAE,CAAC;IAEzC,wDAAwD;IAChD,mBAAmB,GAAG,CAAC,CAAC;IAEhC,YAAY,MAAmB,EAAE,MAAc,EAAE,OAA2B;QAC1E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,OAAO,EAAE,WAAW,IAAI,oBAAoB,CAAC;QAChE,IAAI,CAAC,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,iBAAiB,CAAC;IACzD,CAAC;IAED,4EAA4E;IAC5E,aAAa;IACb,4EAA4E;IAE5E;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,KAAqB;QACrC,6BAA6B;QAC7B,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC3B,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtD,6DAA6D;YAC7D,OAAO,IAAI,CAAC,cAAc,CACxB,sBAAsB,IAAI,CAAC,iBAAiB,CAAC,MAAM,IAAI,IAAI,CAAC,WAAW,YAAY,IAAI,CAAC,QAAQ,GAAG,IAAI,gCAAgC,CACxI,CAAC;QACJ,CAAC;QAED,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,WAAW;gBACd,0EAA0E;gBAC1E,IAAI,IAAI,CAAC,mBAAmB,GAAG,CAAC,EAAE,CAAC;oBACjC,MAAM,CAAC,KAAK,CAAC,kDAAkD,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;oBAC5F,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;gBAC/B,CAAC;gBACD,OAAO,MAAM,CAAC;YAEhB,KAAK,YAAY;gBACf,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAEjC,KAAK,YAAY;gBACf,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAEjC,KAAK,cAAc;gBACjB,OAAO,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAEnC,KAAK,gBAAgB;gBACnB,OAAO,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAErC,KAAK,kBAAkB;gBACrB,OAAO,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;YAE5C,KAAK,iBAAiB;gBACpB,OAAO,IAAI,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC;YAE3C,KAAK,iBAAiB;gBACpB,OAAO,IAAI,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC;YAE3C,KAAK,qBAAqB;gBACxB,MAAM,CAAC,IAAI,CAAC,0EAA0E,CAAC,CAAC;gBACxF,OAAO,MAAM,CAAC;YAEhB,KAAK,iBAAiB;gBACpB,MAAM,CAAC,IAAI,CAAC,wEAAwE,CAAC,CAAC;gBACtF,OAAO,MAAM,CAAC;YAEhB,KAAK,WAAW;gBACd,OAAO,IAAI,CAAC,cAAc,CACxB,yDAA0D,KAAK,CAAC,IAAI,EAAE,WAAsB,IAAI,UAAU,uBAAuB;oBACjI,QAAS,KAAK,CAAC,IAAI,EAAE,MAAiB,IAAI,SAAS,YAAa,KAAK,CAAC,IAAI,EAAE,WAAsB,IAAI,SAAS,EAAE,CAClH,CAAC;YAEJ;gBACE,OAAO,MAAM,CAAC;QAClB,CAAC;IACH,CAAC;IAED,+CAA+C;IAC/C,KAAK;QACH,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC5C,CAAC;IAED,4EAA4E;IAC5E,iBAAiB;IACjB,4EAA4E;IAE5E,2EAA2E;IACnE,KAAK,CAAC,gBAAgB;QAC5B,MAAM,EAAE,OAAO,EAAE,GAAG,cAAc,EAAE,CAAC;QAErC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,4CAA4C;YAC5C,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;YAClE,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC;QAC9B,CAAC;QAED,iEAAiE;QACjE,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,mDAAmD;YACnD,MAAM,QAAQ,GAAG,MAAM,gBAAgB,EAAE,CAAC;YAC1C,IAAI,QAAQ,EAAE,CAAC;gBACb,2EAA2E;gBAC3E,MAAM,CAAC,IAAI,CAAC,QAAQ,MAAM,CAAC,OAAO,oCAAoC,CAAC,CAAC;gBACxE,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACjC,CAAC;QACH,CAAC;QAED,kDAAkD;QAClD,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC;IAC9B,CAAC;IAED,kDAAkD;IAC1C,KAAK,CAAC,gBAAgB;QAC5B,MAAM,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC;IAChC,CAAC;IAED,kDAAkD;IAC1C,KAAK,CAAC,kBAAkB;QAC9B,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;IAED,gEAAgE;IACxD,KAAK,CAAC,oBAAoB;QAChC,MAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;QACzD,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC;IAC9B,CAAC;IAED,yEAAyE;IACjE,KAAK,CAAC,sBAAsB,CAAC,KAAqB;QACxD,MAAM,QAAQ,GAAG,mBAAmB,EAAE,CAAC;QACvC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,qBAAqB,KAAK,CAAC,IAAI,EAAE,OAAO,gCAAgC,CAAC,CAAC;YACtF,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,OAA6B,CAAC;QAC1D,IAAI,CAAC,OAAO;YAAE,OAAO,MAAM,CAAC;QAE5B,MAAM,CAAC,IAAI,CAAC,yBAAyB,OAAO,KAAK,CAAC,CAAC;QACnD,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,YAAY,EAAE,CAAC,OAAO,CAAC,CAAC;YACxC,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;YAC9B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,CAAC,OAAO,CAAC,mBAAmB,OAAO,gBAAgB,CAAC,CAAC;YAC3D,OAAO,eAAe,CAAC;QACzB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,uBAAuB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxF,OAAO,IAAI,CAAC,cAAc,CAAC,kBAAkB,OAAO,YAAY,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtH,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E,iFAAiF;IACzE,KAAK,CAAC,aAAa;QACzB,IAAI,CAAC;YACH,4BAA4B;YAC5B,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEtB,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;YAE9B,iCAAiC;YACjC,MAAM,EAAE,OAAO,EAAE,GAAG,cAAc,EAAE,CAAC;YACrC,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;YAED,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAE3B,MAAM,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC;YAC9C,OAAO,eAAe,CAAC;QACzB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACpF,OAAO,IAAI,CAAC,cAAc,CAAC,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpG,CAAC;IACH,CAAC;IAED,0DAA0D;IAClD,KAAK,CAAC,gBAAgB;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;YAE9B,mBAAmB;YACnB,MAAM,EAAE,OAAO,EAAE,GAAG,cAAc,EAAE,CAAC;YACrC,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;YAED,4BAA4B;YAC5B,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YAC5D,MAAM,CAAC,IAAI,CAAC,2BAA2B,MAAM,CAAC,OAAO,OAAO,OAAO,EAAE,CAAC,CAAC;YAEvE,yCAAyC;YACzC,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;YACtC,UAAU,CAAC;gBACT,OAAO;gBACP,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,OAAO,EAAE,OAAO;gBAChB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;gBAC7C,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC,CAAC;YAEH,2DAA2D;YAC3D,cAAc,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YAE5C,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,CAAC,OAAO,CAAC,0BAA0B,OAAO,EAAE,CAAC,CAAC;YACpD,OAAO,mBAAmB,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,6BAA6B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9F,OAAO,IAAI,CAAC,cAAc,CAAC,6BAA6B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9G,CAAC;IACH,CAAC;IAED,0DAA0D;IAClD,KAAK,CAAC,eAAe;QAC3B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEtB,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;YAC9B,MAAM,EAAE,OAAO,EAAE,GAAG,cAAc,EAAE,CAAC;YACrC,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;YAED,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAE3B,MAAM,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC;YACpD,OAAO,iBAAiB,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC3F,OAAO,IAAI,CAAC,cAAc,CAAC,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3G,CAAC;IACH,CAAC;IAED,yDAAyD;IACjD,KAAK,CAAC,WAAW;QACvB,IAAI,CAAC;YACH,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7B,sCAAsC;gBACtC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;gBAC9B,MAAM,CAAC,OAAO,CAAC,kCAAkC,SAAS,EAAE,CAAC,CAAC;YAChE,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;YACpD,CAAC;YACD,OAAO,aAAa,CAAC;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,wBAAwB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzF,OAAO,aAAa,CAAC,CAAC,0BAA0B;QAClD,CAAC;IACH,CAAC;IAED,oEAAoE;IAC5D,KAAK,CAAC,qBAAqB,CAAC,KAAqB;QACvD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,EAAE,QAAkB,IAAI,SAAS,CAAC;QAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,KAAe,IAAI,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,KAAe,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,SAAmB,IAAI,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,SAAqB,IAAI,EAAE,CAAC;QAE1D,MAAM,CAAC,IAAI,CACT,WAAW,QAAQ,iCAAiC,KAAK,IAAI,KAAK,kBAAkB;YACpF,SAAS,SAAS,sBAAsB,SAAS,CAAC,MAAM,qBAAqB,CAC9E,CAAC;QAEF,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CACT,WAAW,QAAQ,qDAAqD;gBACxE,8CAA8C,QAAQ,EAAE,CACzD,CAAC;QACJ,CAAC;QAED,gCAAgC;QAChC,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,WAAW,CACrB,uBAAuB,QAAQ,KAAK,KAAK,IAAI,KAAK,qBAAqB,SAAS,GAAG,EACnF,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CACjD,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,kEAAkE;IAC1D,KAAK,CAAC,qBAAqB,CAAC,KAAqB;QACvD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,EAAE,QAAkB,IAAI,SAAS,CAAC;QAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,KAAe,IAAI,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,KAAe,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,SAAmB,IAAI,CAAC,CAAC;QAEvD,OAAO,IAAI,CAAC,cAAc,CACxB,qBAAqB,QAAQ,wBAAwB,KAAK,IAAI,KAAK,kBAAkB;YACrF,SAAS,SAAS,4CAA4C,QAAQ,EAAE,CACzE,CAAC;IACJ,CAAC;IAED,mDAAmD;IAC3C,cAAc,CAAC,OAAe;QACpC,MAAM,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACpE,MAAM,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;QAChE,MAAM,CAAC,KAAK,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACpE,MAAM,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;QAEpE,6CAA6C;QAC7C,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE;gBAC/B,IAAI,EAAE,gBAAgB;gBACtB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,4CAA4C;QAC9C,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,4EAA4E;IAC5E,UAAU;IACV,4EAA4E;IAE5E,iEAAiE;IACzD,KAAK,CAAC,iBAAiB;QAC7B,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;QAE9B,UAAU,CAAC;YACT,OAAO;YACP,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;YAC7C,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,uEAAuE;IAC/D,KAAK,CAAC,QAAQ;QACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,oBAAoB,CAAC,CAAC;QAC1E,MAAM,OAAO,GAAG,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACxD,IAAI,OAAO,GAAG,eAAe,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,oBAAoB,OAAO,8BAA8B,IAAI,CAAC,mBAAmB,GAAG,CAAC,GAAG,CAAC,CAAC;YACtG,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,wDAAwD;IAChD,cAAc;QACpB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,+DAA+D;IACvD,mBAAmB;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC1C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC;IAC9E,CAAC;CACF;AAED,eAAe,UAAU,CAAC"}

package/dist/core/heartbeat.d.ts

@@ -0,0 +1,67 @@
+import { AgentClient } from "../agent/client.js";
+import { SelfHealer } from "./healer.js";
+export interface HeartbeatEvent {
+    timestamp: number;
+    type: "health_ok" | "health_degraded" | "health_critical" | "agent_down" | "peers_lost" | "disk_warning" | "memory_warning" | "update_available" | "network_unreachable" | "rpc_unreachable" | "rpc_stale";
+    message: string;
+    data?: Record<string, unknown>;
+}
+export interface HeartbeatOptions {
+    /** Interval between checks in milliseconds (default: 30000 = 30s) */
+    intervalMs?: number;
+    /** Callback invoked for every heartbeat event */
+    onEvent?: (event: HeartbeatEvent) => void;
+}
+export declare class Heartbeat {
+    private readonly client;
+    private readonly intervalMs;
+    private readonly onEvent?;
+    private timer;
+    private consecutiveFailures;
+    /** RPC health tracking */
+    private lastBlockNumber;
+    private staleBlockCount;
+    /** Cached RPC URL resolved from config */
+    private rpcUrl;
+    /** Optional healer — set externally to enable self-healing */
+    healer: SelfHealer | null;
+    constructor(client: AgentClient, options?: HeartbeatOptions);
+    /** Start the periodic heartbeat loop. */
+    start(): void;
+    /** Stop the heartbeat loop. */
+    stop(): void;
+    /** Whether the heartbeat loop is currently running. */
+    isRunning(): boolean;
+    /** Run a single heartbeat check cycle. Useful for testing. */
+    runOnce(): Promise<HeartbeatEvent[]>;
+    /** Resolve the RPC URL from config, caching the result. */
+    private _getRpcUrl;
+    /**
+     * Check internet connectivity. Tries RPC eth_blockNumber first, falls back
+     * to DNS resolution for sequence0.network. Returns true if internet is up.
+     */
+    private _checkInternet;
+    /**
+     * Check RPC health by calling eth_blockNumber. Tracks block progression
+     * and emits rpc_unreachable or rpc_stale events.
+     */
+    private _checkRpc;
+    /** GET /health — track latency and consecutive failures. */
+    private _checkHealth;
+    /** GET /status — check peers, sessions, wallets. */
+    private _checkStatus;
+    /** GET /health/committees — detect degraded/critical committees. */
+    private _checkCommittees;
+    /** Check system memory usage via os module. */
+    private _checkMemory;
+    /** Check disk usage via platform utility. */
+    private _checkDisk;
+    /** Maximum events.log size in bytes before rotation (10 MB). */
+    private static readonly MAX_LOG_BYTES;
+    /** Rotate events.log if it exceeds the size limit. */
+    private _rotateLogIfNeeded;
+    /** Append a heartbeat event as a JSON line to the events log file. */
+    private _persistEvent;
+}
+export default Heartbeat;
+//# sourceMappingURL=heartbeat.d.ts.map

package/dist/core/heartbeat.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"heartbeat.d.ts","sourceRoot":"","sources":["../../src/core/heartbeat.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,WAAW,EAAmD,MAAM,oBAAoB,CAAC;AAKlG,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAOzC,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EACA,WAAW,GACX,iBAAiB,GACjB,iBAAiB,GACjB,YAAY,GACZ,YAAY,GACZ,cAAc,GACd,gBAAgB,GAChB,kBAAkB,GAClB,qBAAqB,GACrB,iBAAiB,GACjB,WAAW,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,gBAAgB;IAC/B,qEAAqE;IACrE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iDAAiD;IACjD,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;CAC3C;AAqBD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAkC;IAE3D,OAAO,CAAC,KAAK,CAA+C;IAC5D,OAAO,CAAC,mBAAmB,CAAK;IAEhC,0BAA0B;IAC1B,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,eAAe,CAAK;IAE5B,0CAA0C;IAC1C,OAAO,CAAC,MAAM,CAAuB;IAErC,8DAA8D;IACvD,MAAM,EAAE,UAAU,GAAG,IAAI,CAAQ;gBAE5B,MAAM,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,gBAAgB;IAU3D,yCAAyC;IACzC,KAAK,IAAI,IAAI;IAQb,+BAA+B;IAC/B,IAAI,IAAI,IAAI;IAQZ,uDAAuD;IACvD,SAAS,IAAI,OAAO;IAQpB,8DAA8D;IACxD,OAAO,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAgD1C,2DAA2D;IAC3D,OAAO,CAAC,UAAU;IAelB;;;OAGG;YACW,cAAc;IAoC5B;;;OAGG;YACW,SAAS;IAqEvB,4DAA4D;YAC9C,YAAY;IAoD1B,oDAAoD;YACtC,YAAY;IAkC1B,oEAAoE;YACtD,gBAAgB;IAyC9B,+CAA+C;IAC/C,OAAO,CAAC,YAAY;IAmBpB,6CAA6C;YAC/B,UAAU;IAyBxB,gEAAgE;IAChE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAoB;IAEzD,sDAAsD;IACtD,OAAO,CAAC,kBAAkB;IAW1B,sEAAsE;IACtE,OAAO,CAAC,aAAa;CAStB;AAED,eAAe,SAAS,CAAC"}