@omnitron-dev/omnitron 0.2.0 → 0.3.1

package/dist/services/node-manager.service.js

@@ -1,7 +1,11 @@
 /**
  * NodeManagerService — Manages infrastructure nodes (machines)
  *
- * File-based storage at ~/.omnitron/nodes.json (no PG dependency).
+ * Registry lives in the SQLite-backed `nodes` table of DaemonStateStore.
+ * `~/.omnitron/nodes.json` is read once, on a boot that finds the table
+ * empty, and then unlinked — the header claimed the JSON file was still the
+ * storage long after T-7 moved it, which is the kind of thing a reader
+ * believes because there is nothing to contradict it.
  * Always includes a "local" node for the local machine.
  * Remote nodes are accessed via SSH for provisioning and management.
  */
@@ -10,53 +14,213 @@ import path from 'node:path';
 import os from 'node:os';
 import { EventEmitter } from 'node:events';
 import { randomUUID } from 'node:crypto';
-import { RemoteOpsService, DEFAULT_CHECK_CONFIG } from './remote-ops.service.js';
+import { RemoteOpsService, DEFAULT_CHECK_CONFIG, normalizeCheckConfig, assertNodeHost, } from './remote-ops.service.js';
+import { CLI_VERSION } from '../config/defaults.js';
 // =============================================================================
 // Constants
 // =============================================================================
 const OMNITRON_HOME = path.join(os.homedir(), '.omnitron');
 const NODES_FILE = path.join(OMNITRON_HOME, 'nodes.json');
 const LOCAL_NODE_ID = 'local';
+/** DaemonStateStore key holding the operator-set check configuration. */
+const CHECK_CONFIG_KEY = 'nodes:check-config';
 // =============================================================================
 // Service
 // =============================================================================
 export class NodeManagerService extends EventEmitter {
     logger;
+    store;
     secrets;
     nodes = new Map();
     statusCache = new Map();
+    /**
+     * Aggregated health per node — the shape the worker reports and the console
+     * reads. Kept here so a daemon whose worker is down can still answer
+     * `triggerNodeCheck`, and so status TRANSITIONS have something to compare
+     * against: `node.went_offline` and its siblings are declared event channels
+     * that nothing ever emitted, because no one held the previous value.
+     */
+    summaryCache = new Map();
     remoteOps;
     checkConfig = DEFAULT_CHECK_CONFIG;
-    constructor(logger, secrets) {
+    /** Set by the daemon from its own config; see `setHistoryConfig`. */
+    historyConfig = { uptimeIntervalMs: 86_400_000, retentionDays: 90 };
+    /**
+     * Secrets read out of a node row and not yet written to the vault.
+     *
+     * The migration in `healLeakedSecrets` is async and starts at construction;
+     * without this, a check running in that window would find no passphrase and
+     * report a perfectly reachable node as unreachable.
+     */
+    pendingSecrets = new Map();
+    /** The master's mesh — see `setSlaveConnector`. */
+    mesh = null;
+    constructor(logger, 
+    /**
+     * T-7 — fleet node registry persistence moved off
+     * ~/.omnitron/nodes.json onto the SQLite-backed `nodes` table
+     * in DaemonStateStore.
+     * Canonical name/host/port are denormalised into typed columns
+     * for queries; the full INode payload (SSH config, tags, etc.)
+     * lives in the `metadata` column as JSON.
+     */
+    store, secrets) {
         super();
         this.logger = logger;
+        this.store = store;
         this.secrets = secrets;
         this.remoteOps = new RemoteOpsService(logger);
         this.load();
         this.ensureLocalNode();
-        this.initLocalStatus();
     }
-    /** Update check configuration (ping/SSH timeouts, ping enabled/disabled) */
+    /**
+     * Update check configuration (ping/SSH timeouts, ping enabled/disabled).
+     *
+     * Persisted and announced. Both were missing: the setting lived in this
+     * field alone, so it was lost on restart, and the health-monitor worker —
+     * which performs every check the console displays — kept running with the
+     * values it was handed at boot. An operator turning ping off in the console
+     * changed a value that only the daemon's own fallback path ever read.
+     */
     setCheckConfig(config) {
-        this.checkConfig = { ...this.checkConfig, ...config };
+        const next = normalizeCheckConfig({ ...this.checkConfig, ...config });
+        const changed = Object.keys(next)
+            .some((k) => next[k] !== this.checkConfig[k]);
+        this.checkConfig = next;
+        if (!changed)
+            return;
+        try {
+            this.store.kvSetSync(CHECK_CONFIG_KEY, next);
+        }
+        catch (err) {
+            this.logger.warn({ error: err.message }, 'Failed to persist node check config');
+        }
+        this.logger.info({ config: next }, 'Node check config updated');
+        this.emit('checkConfig:changed', next);
     }
     getCheckConfig() {
         return { ...this.checkConfig };
     }
+    /**
+     * Tell the service how history is kept, so the console can be told too.
+     *
+     * `uptimeIntervalMs` was declared in the daemon config, documented, given a
+     * default — and read by nothing. The console used a constant of its own,
+     * and no caller anywhere could have discovered the retention window that
+     * bounds what the bars can show.
+     */
+    setHistoryConfig(config) {
+        if (Number.isFinite(config.uptimeIntervalMs)) {
+            this.historyConfig.uptimeIntervalMs = config.uptimeIntervalMs;
+        }
+        if (Number.isFinite(config.retentionDays)) {
+            this.historyConfig.retentionDays = config.retentionDays;
+        }
+    }
+    getHistoryConfig() {
+        return { ...this.historyConfig };
+    }
+    /**
+     * The master's mesh, so a check asks a node's daemon over the connection
+     * the master already holds to it.
+     *
+     * Without it this service could only dial `tcp://<host>:<daemonPort>`, and
+     * a hardened node's daemon port is closed to the master — that is why the
+     * mesh tunnels over SSH. Measured 2026-09-23 with the health worker
+     * detached: `omnitron node check` on sample-test printed `Omnitron: ○
+     * offline` / `Connection timeout to tcp://37.27.130.185:9700` while the
+     * mesh held that node `connected via ssh-tunnel, authenticated` and
+     * `fleet status` read six apps from it through that connection. The
+     * comment above the probe said the SlaveConnector was asked first; the
+     * code never asked it.
+     */
+    setSlaveConnector(connector) {
+        this.mesh = connector;
+    }
     // ===========================================================================
     // CRUD
     // ===========================================================================
     listNodes() {
         return Array.from(this.nodes.values()).map((node) => ({
-            ...node,
-            status: this.statusCache.get(node.id) ?? null,
+            ...toWireNode(node),
+            status: this.statusFor(node.id),
         }));
     }
     getNode(id) {
         const node = this.nodes.get(id);
         if (!node)
             return null;
-        return { ...node, status: this.statusCache.get(id) ?? null };
+        return { ...toWireNode(node), status: this.statusFor(id) };
+    }
+    /**
+     * Status for one node.
+     *
+     * For the local node the facts about this daemon — version, pid, uptime,
+     * role — are read from `process` on every call. They used to be written
+     * once into the cache in the constructor, so `omnitronUptime` was whatever
+     * `process.uptime()` returned a few milliseconds into boot, and the
+     * console renders that as "Uptime": a handful of seconds, however long the
+     * daemon had been up.
+     *
+     * The health-monitor worker also checks the local node, and what it
+     * reports is not worthless — whether sshd answers on loopback is a real
+     * check, and it is the only source for `checkedAt`, which dates the
+     * reachability reading rather than this call. So the worker's status is
+     * kept and only the fields the daemon knows first-hand are overwritten.
+     * Neither source is discarded, because neither one is redundant.
+     */
+    statusFor(id) {
+        if (id !== LOCAL_NODE_ID)
+            return this.statusCache.get(id) ?? null;
+        const live = this.localStatus();
+        const checked = this.statusCache.get(LOCAL_NODE_ID);
+        if (!checked)
+            return live;
+        const merged = { ...checked, omnitronConnected: true };
+        // Assigned rather than spread: the fields are optional under
+        // `exactOptionalPropertyTypes`, so writing `undefined` into them is a
+        // different thing from leaving them out.
+        if (live.omnitronVersion !== undefined)
+            merged.omnitronVersion = live.omnitronVersion;
+        if (live.omnitronPid !== undefined)
+            merged.omnitronPid = live.omnitronPid;
+        if (live.omnitronUptime !== undefined)
+            merged.omnitronUptime = live.omnitronUptime;
+        if (live.omnitronRole !== undefined)
+            merged.omnitronRole = live.omnitronRole;
+        return merged;
+    }
+    /**
+     * One row per daemon.
+     *
+     * `host` plus `daemonPort` names a DAEMON, and two registry rows naming
+     * the same one are always a mistake — there is no second thing there to
+     * manage. Two daemons on one machine are two ports, and those are still
+     * allowed.
+     *
+     * It was worth every cost it caused, all measured on one machine that had
+     * been registered twice:
+     *
+     *   - the health monitor checked it on both rows, 120 connections an hour
+     *     to one socket;
+     *   - `fleet upgrade` transferred and installed the bundle to it twice;
+     *   - the mesh holds one connection per ADDRESS, so exactly one of the two
+     *     rows receives the node's data and the other reads "not joined"
+     *     forever — and which one is arbitrary.
+     *
+     * Refused rather than merged: the existing row may carry credentials,
+     * tags and history, and picking which of two to keep is the operator's
+     * call, not this function's.
+     */
+    assertAddressIsFree(host, daemonPort, exceptId) {
+        for (const existing of this.nodes.values()) {
+            if (existing.id === exceptId)
+                continue;
+            if (existing.host !== host || existing.daemonPort !== daemonPort)
+                continue;
+            throw new Error(`${host}:${daemonPort} is already registered as "${existing.name}". ` +
+                'One row per daemon — edit that node, or give this one a different daemon port.');
+        }
     }
     async addNode(input) {
         const id = randomUUID();
@@ -64,19 +228,25 @@ export class NodeManagerService extends EventEmitter {
         const node = {
             id,
             name: input.name,
-            host: input.host,
-            sshPort: input.sshPort ?? 22,
+            // A host is not free text: it is spelled into an SSH target and into
+            // the argument list of `ping`. Rejected at the WRITE so a stored value
+            // cannot surprise a reader downstream.
+            host: assertNodeHost(input.host),
+            sshPort: assertPort('SSH port', input.sshPort ?? 22),
             sshUser: input.sshUser ?? 'root',
             sshAuthMethod: input.sshAuthMethod ?? 'key',
             ...(input.sshPrivateKey && { sshPrivateKey: input.sshPrivateKey }),
             runtime: input.runtime ?? 'node',
-            daemonPort: input.daemonPort ?? 9700,
+            daemonPort: assertPort('daemon port', input.daemonPort ?? 9700),
             tags: input.tags ?? [],
             isLocal: false,
             createdAt: now,
             updatedAt: now,
             ...(input.offlineTimeout != null && { offlineTimeout: input.offlineTimeout }),
         };
+        if (!node.name.trim())
+            throw new Error('Node name is required');
+        this.assertAddressIsFree(node.host, node.daemonPort);
         // Store secrets encrypted (passphrase, password) — only boolean markers in nodes.json.
         // Must await before connectivity check, otherwise getSecret() reads stale file.
         if (input.sshPassphrase) {
@@ -91,8 +261,14 @@ export class NodeManagerService extends EventEmitter {
         this.save();
         this.emit('node:added', node);
         this.logger.info({ nodeId: id, name: node.name, host: node.host }, 'Node added');
-        // Check connectivity in background — secrets are already persisted
-        void this.checkNodeStatus(id).catch(() => { });
+        // Check connectivity in background — secrets are already persisted.
+        // `checkNodeStatus` records an unreachable host in the status it returns
+        // rather than throwing, so anything arriving here is a fault in the probe
+        // itself, and swallowing it leaves the node "unchecked" for ever with
+        // nothing to explain why.
+        void this.checkNodeStatus(id).catch((err) => {
+            this.logger.warn({ nodeId: id, err }, 'Connectivity check failed to run');
+        });
         return node;
     }
     async updateNode(id, input) {
@@ -103,6 +279,12 @@ export class NodeManagerService extends EventEmitter {
             throw new Error('Cannot change host of local node');
         // Extract secrets before spreading into node (they must not be saved to nodes.json)
         const { sshPassphrase, sshPassword, ...safeInput } = input;
+        if (safeInput.host !== undefined)
+            safeInput.host = assertNodeHost(safeInput.host);
+        if (safeInput.sshPort !== undefined)
+            safeInput.sshPort = assertPort('SSH port', safeInput.sshPort);
+        if (safeInput.daemonPort !== undefined)
+            safeInput.daemonPort = assertPort('daemon port', safeInput.daemonPort);
         const updated = {
             ...node,
             ...safeInput,
@@ -111,6 +293,25 @@ export class NodeManagerService extends EventEmitter {
             createdAt: node.createdAt,
             updatedAt: new Date().toISOString(),
         };
+        // An edit can collide as readily as an add — correcting one node's host
+        // to the address another already holds is exactly how a duplicate gets
+        // made. Checked before any secret is written, so a refused edit leaves
+        // nothing behind.
+        //
+        // Only when the edit MOVES the row. A registry that already holds two
+        // rows for one daemon — made before this rule existed — refused every
+        // edit to either of them, including the one that would have repaired it:
+        //
+        //     omnitron node update <id> --ssh-auth password
+        //     37.27.130.185:9700 is already registered as "acme-deploy-test"
+        //
+        // The guard exists to stop a duplicate being CREATED. Refusing an edit
+        // that changes neither the host nor the port cannot prevent one, and it
+        // leaves an operator with a registry they can neither use nor mend.
+        const addressChanged = updated.host !== node.host || updated.daemonPort !== node.daemonPort;
+        if (addressChanged) {
+            this.assertAddressIsFree(updated.host, updated.daemonPort, id);
+        }
         // Update encrypted secrets — await to ensure they're persisted before connectivity check
         if (sshPassphrase !== undefined) {
             if (sshPassphrase) {
@@ -132,26 +333,66 @@ export class NodeManagerService extends EventEmitter {
                 await this.deleteSecret(id, 'password');
             }
         }
+        // A credential that belongs to the other auth method is dead weight that
+        // still authenticates: leaving a stored password on a node switched to
+        // key auth means the console shows "SSH Key" while the daemon may log in
+        // with a password the operator believes they stopped using.
+        if (updated.sshAuthMethod !== node.sshAuthMethod) {
+            if (updated.sshAuthMethod === 'key' && updated.hasPassword) {
+                updated.hasPassword = false;
+                await this.deleteSecret(id, 'password');
+            }
+            if (updated.sshAuthMethod === 'password') {
+                delete updated.sshPrivateKey;
+                if (updated.hasPassphrase) {
+                    updated.hasPassphrase = false;
+                    await this.deleteSecret(id, 'passphrase');
+                }
+            }
+        }
         this.nodes.set(id, updated);
         this.save();
         this.emit('node:updated', updated);
         // Re-check connectivity after update
         if (!updated.isLocal) {
-            void this.checkNodeStatus(id).catch(() => { });
+            void this.checkNodeStatus(id).catch((err) => {
+                this.logger.warn({ nodeId: id, err }, 'Connectivity check failed to run');
+            });
         }
         return updated;
     }
-    removeNode(id) {
+    /**
+     * Remove a node from the registry.
+     *
+     * Deletes the ROW, not just the map entry. `save()` rewrites every node
+     * still held in memory, which is how the file-backed registry used to
+     * express a deletion — but an UPSERT-per-row store expresses it by
+     * deleting, and the row for a removed node stayed behind. The console
+     * reported the node gone, and the next daemon start read it back in.
+     *
+     * Secret deletion is awaited for the same reason: fired and forgotten, a
+     * failure left an SSH password in the vault under the id of a node nobody
+     * can see any more.
+     */
+    async removeNode(id) {
         const node = this.nodes.get(id);
         if (!node)
             throw new Error(`Node not found: ${id}`);
         if (node.isLocal)
             throw new Error('Cannot remove local node');
         // Clean up encrypted secrets
-        void this.deleteSecret(id, 'passphrase');
-        void this.deleteSecret(id, 'password');
+        await this.deleteSecret(id, 'passphrase');
+        await this.deleteSecret(id, 'password');
         this.nodes.delete(id);
         this.statusCache.delete(id);
+        this.summaryCache.delete(id);
+        try {
+            this.store.deleteNodeSync(id);
+        }
+        catch (err) {
+            this.logger.warn({ nodeId: id, error: err.message }, 'Failed to delete node row');
+            throw err;
+        }
         this.save();
         this.emit('node:removed', id);
         this.logger.info({ nodeId: id, name: node.name }, 'Node removed');
@@ -185,25 +426,29 @@ export class NodeManagerService extends EventEmitter {
                 entry === 'authorized_keys') {
                 continue;
             }
-            // Check if it looks like a private key (starts with -----BEGIN)
+            // Check if it looks like a private key (starts with -----BEGIN).
+            // Read the HEADER, not the file: this walks a directory of private
+            // keys, and `readFileSync(...).slice(0, 100)` pulled every byte of
+            // every one of them into the daemon's heap to look at the first line.
+            let head;
             try {
-                const head = fs.readFileSync(fullPath, 'utf-8').slice(0, 100);
-                if (head.includes('-----BEGIN') && head.includes('KEY')) {
-                    // Detect key type from header
-                    let type = 'unknown';
-                    if (head.includes('RSA'))
-                        type = 'rsa';
-                    else if (head.includes('EC'))
-                        type = 'ecdsa';
-                    else if (head.includes('OPENSSH'))
-                        type = 'ed25519';
-                    else if (head.includes('DSA'))
-                        type = 'dsa';
-                    keys.push({ name: entry, path: fullPath, type });
-                }
+                head = readFileHead(fullPath, 100);
             }
             catch {
-                // Can't read — skip
+                continue; // Can't read — skip
+            }
+            if (head.includes('-----BEGIN') && head.includes('KEY')) {
+                // Detect key type from header
+                let type = 'unknown';
+                if (head.includes('RSA'))
+                    type = 'rsa';
+                else if (head.includes('EC'))
+                    type = 'ecdsa';
+                else if (head.includes('OPENSSH'))
+                    type = 'ed25519';
+                else if (head.includes('DSA'))
+                    type = 'dsa';
+                keys.push({ name: entry, path: fullPath, type });
             }
         }
         return keys;
@@ -219,7 +464,9 @@ export class NodeManagerService extends EventEmitter {
             nodeId: id,
             pingReachable: false,
             pingLatencyMs: null,
-            sshConnected: false,
+            // Not `false`: this check does not open an SSH session at all, and
+            // `false` reads as "SSH was refused" everywhere it is consumed.
+            sshConnected: null,
             sshLatencyMs: null,
             omnitronConnected: false,
             checkedAt: new Date().toISOString(),
@@ -234,22 +481,18 @@ export class NodeManagerService extends EventEmitter {
             status.omnitronPid = process.pid;
             status.omnitronUptime = process.uptime() * 1000;
             status.omnitronRole = 'master';
+            status.omnitronVersion = CLI_VERSION;
             status.os = {
                 platform: os.platform(),
                 arch: os.arch(),
                 hostname: os.hostname(),
                 release: os.release(),
             };
-            try {
-                const { CLI_VERSION } = await import('../config/defaults.js');
-                status.omnitronVersion = CLI_VERSION;
-            }
-            catch {
-                status.omnitronVersion = '0.1.0';
-            }
         }
         else {
-            // Remote node — ping + Netron TCP (no SSH — SSH is manual-only via UI)
+            // Remote node — ping, then the daemon itself (no SSH — SSH is
+            // manual-only via UI, and the health worker's round is the one that
+            // opens a session).
             // 1. Ping (ICMP) — lightweight reachability check
             if (this.checkConfig.pingEnabled) {
                 const ping = await this.remoteOps.ping(node.host, this.checkConfig.pingTimeout);
@@ -262,47 +505,92 @@ export class NodeManagerService extends EventEmitter {
                 status.pingReachable = true;
                 status.pingLatencyMs = null;
             }
-            // 2. Netron TCP ping — check if omnitron daemon is running and responsive
-            //    Master connects to slave via SlaveConnector; if already connected,
-            //    the heartbeat confirms status. Otherwise try a quick TCP connect + ping.
-            const port = node.daemonPort ?? 9700;
-            try {
-                const { Netron } = await import('@omnitron-dev/titan/netron');
-                const { TcpTransport } = await import('@omnitron-dev/titan/netron/transport/tcp');
-                const { createNullLogger } = await import('@omnitron-dev/titan/module/logger');
-                const probeNetron = new Netron(createNullLogger(), { id: `probe-${node.host}` });
-                probeNetron.registerTransport('tcp', () => new TcpTransport());
-                const connectStart = Date.now();
-                const peer = await Promise.race([
-                    probeNetron.connect(`tcp://${node.host}:${port}`, false),
-                    new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), this.checkConfig.omnitronCheckTimeout)),
-                ]);
-                // Ping via OmnitronDaemon service
-                const daemon = await peer.queryInterface('OmnitronDaemon');
-                const info = await daemon.ping();
-                const latency = Date.now() - connectStart;
+            // 2. The daemon: asked over the mesh first, dialled directly second.
+            const answer = await this.askDaemon(node);
+            if (answer.ok) {
                 status.omnitronConnected = true;
-                status.sshConnected = false; // SSH not checked
-                status.sshLatencyMs = null;
-                if (info?.version)
-                    status.omnitronVersion = info.version;
-                if (info?.pid)
-                    status.omnitronPid = info.pid;
-                if (info?.uptime)
-                    status.omnitronUptime = info.uptime;
-                this.logger.debug({ node: node.name, host: node.host, latencyMs: latency, version: info?.version }, 'Node check: Netron TCP ping OK');
-                // Disconnect probe — SlaveConnector manages persistent connections
-                await probeNetron.stop();
+                if (answer.info?.version)
+                    status.omnitronVersion = answer.info.version;
+                if (answer.info?.pid)
+                    status.omnitronPid = answer.info.pid;
+                if (answer.info?.uptime)
+                    status.omnitronUptime = answer.info.uptime;
             }
-            catch (err) {
-                status.omnitronConnected = false;
-                this.logger.debug({ node: node.name, host: node.host, port, error: err.message }, 'Node check: Netron TCP ping failed');
+            else {
+                // `null`, not `false`. Neither path reached the daemon, and that says
+                // nothing about whether it runs: a closed daemon port is the normal
+                // state of a hardened node, and a mesh link can be down while every
+                // app on the node serves. `false` reads as "the daemon is down" to
+                // every consumer — `node list`, `node check`, the console's dot —
+                // and that is the claim this check had no evidence for. The reasons
+                // travel in `omnitronError`, each naming its path.
+                status.omnitronConnected = null;
+                status.omnitronError = `Neither path reached the daemon — ${answer.reasons.join('; ')}`;
             }
         }
-        this.statusCache.set(id, status);
-        this.emit('node:status', id, status);
+        this.recordStatus(node, status);
         return status;
     }
+    /**
+     * Ask a node's daemon to answer `ping`, the way the master can reach it.
+     *
+     * The mesh first: it is the connection the master already holds, and for
+     * a node whose daemon port is firewalled it is the only one that works —
+     * over an SSH tunnel. The direct dial second, for a node outside the mesh
+     * (just added, or on a master whose mesh did not start) whose port IS open.
+     * Each failure is kept with the name of its path, because «connection
+     * timeout» alone does not say which door was shut.
+     */
+    async askDaemon(node) {
+        const port = node.daemonPort ?? 9700;
+        const reasons = [];
+        if (this.mesh) {
+            try {
+                const info = (await withTimeout(this.mesh.invokeOnSlave(node.host, port, 'OmnitronDaemon', 'ping', []), this.checkConfig.omnitronCheckTimeout));
+                this.logger.debug({ node: node.name, host: node.host, version: info?.version }, 'Node check: daemon answered over the mesh');
+                return { ok: true, via: 'mesh', info };
+            }
+            catch (err) {
+                reasons.push(`mesh: ${err.message}`);
+            }
+        }
+        else {
+            reasons.push('mesh: this daemon has no mesh connector');
+        }
+        let probeNetron = null;
+        try {
+            const { Netron } = await import('@omnitron-dev/titan/netron');
+            const { TcpTransport } = await import('@omnitron-dev/titan/netron/transport/tcp');
+            const { createNullLogger } = await import('@omnitron-dev/titan/module/logger');
+            const probe = new Netron(createNullLogger(), { id: `probe-${node.host}` });
+            probeNetron = probe;
+            probe.registerTransport('tcp', () => new TcpTransport());
+            const connectStart = Date.now();
+            const peer = await withTimeout(probe.connect(`tcp://${node.host}:${port}`, false), this.checkConfig.omnitronCheckTimeout);
+            // Ping via OmnitronDaemon service
+            const daemon = await peer.queryInterface('OmnitronDaemon');
+            const info = await daemon.ping();
+            this.logger.debug({ node: node.name, host: node.host, latencyMs: Date.now() - connectStart, version: info?.version }, 'Node check: daemon answered a direct dial');
+            return { ok: true, via: 'direct', info };
+        }
+        catch (err) {
+            reasons.push(`direct dial to ${node.host}:${port}: ${err.message}`);
+            this.logger.debug({ node: node.name, host: node.host, port, reasons }, 'Node check: no path reached the daemon');
+            return { ok: false, reasons };
+        }
+        finally {
+            // Stop the probe on EVERY path. It only ran on success, so each
+            // failed check — the common case for an offline node, once a minute,
+            // for ever — left a Netron instance with a registered TCP transport
+            // behind in the daemon.
+            if (probeNetron) {
+                try {
+                    await probeNetron.stop();
+                }
+                catch { /* the probe is being discarded either way */ }
+            }
+        }
+    }
     /** Convert node to SSH target — used for manual SSH checks from UI */
     async nodeToSshTarget(node) {
         const target = {
@@ -325,16 +613,73 @@ export class NodeManagerService extends EventEmitter {
         }
         return target;
     }
+    /**
+     * Convert a registered node into something the deployer can reach.
+     *
+     * The bridge between the two registries. An operator registers machines
+     * here, in the console; deployment reads `stacks.nodes` out of a project
+     * config. Nothing joined them, which is half of why a machine added in the
+     * console could not be deployed to — the other half being that the deployer
+     * could not have used this node's credentials even if it had been handed
+     * one: it shelled out to `ssh -o BatchMode=yes`, and what this method
+     * returns is a password.
+     *
+     * The credentials come from `nodeToSshTarget`, which the manual SSH check
+     * already uses, so there is one place that knows how to unlock a node and
+     * both callers use it.
+     */
+    async nodeToDeployTarget(id) {
+        const node = this.nodes.get(id);
+        if (!node)
+            throw new Error(`Node not found: ${id}`);
+        if (node.isLocal) {
+            // Not a refusal on principle — a local daemon is deployed to by running
+            // omnitron, not by SSHing to yourself — but a refusal that says so,
+            // rather than a loopback SSH that fails for a reason the operator then
+            // has to work out.
+            throw new Error('The local node is this daemon; it is not deployed to over SSH.');
+        }
+        const ssh = await this.nodeToSshTarget(node);
+        const target = {
+            host: ssh.host,
+            label: node.name,
+            daemonPort: node.daemonPort ?? 9700,
+        };
+        if (ssh.port != null)
+            target.sshPort = ssh.port;
+        if (ssh.username)
+            target.username = ssh.username;
+        if (ssh.privateKey)
+            target.privateKey = ssh.privateKey;
+        if (ssh.passphrase)
+            target.passphrase = ssh.passphrase;
+        if (ssh.password)
+            target.password = ssh.password;
+        return target;
+    }
+    /**
+     * Check every node.
+     *
+     * Concurrent, bounded by the same `concurrency` the worker uses. It was a
+     * `for` loop with an `await` in it, so a fleet of twenty nodes with one
+     * unreachable host took the ping timeout plus the TCP timeout SERIALLY —
+     * twenty seconds of a thirty-second check interval spent on one node.
+     */
     async checkAllNodes() {
+        const ids = Array.from(this.nodes.keys());
         const results = [];
-        for (const node of this.nodes.values()) {
-            try {
-                const status = await this.checkNodeStatus(node.id);
-                results.push(status);
-            }
-            catch (err) {
-                this.logger.warn({ nodeId: node.id, error: err.message }, 'Failed to check node');
-            }
+        const width = Math.max(1, this.checkConfig.concurrency);
+        for (let i = 0; i < ids.length; i += width) {
+            const batch = ids.slice(i, i + width);
+            const settled = await Promise.allSettled(batch.map((id) => this.checkNodeStatus(id)));
+            settled.forEach((r, idx) => {
+                if (r.status === 'fulfilled') {
+                    results.push(r.value);
+                }
+                else {
+                    this.logger.warn({ nodeId: batch[idx], error: r.reason?.message }, 'Failed to check node');
+                }
+            });
         }
         return results;
     }
@@ -344,11 +689,19 @@ export class NodeManagerService extends EventEmitter {
     /**
      * Update the in-memory status cache from health-monitor worker summaries.
      * Called by daemon when worker sends IPC status batch.
+     *
+     * Only for nodes the registry holds. The worker's batch is its own memory,
+     * and it can name a node this registry removed — it did, every minute, for
+     * a node removed 82 minutes earlier — and writing that back put a status
+     * with no node behind it into `statusCache`, `summaryCache` and the
+     * heartbeat column of a row that no longer exists.
      */
     updateStatusCacheFromWorker(summaries) {
         for (const summary of summaries) {
             if (!summary.lastCheck)
                 continue;
+            if (!this.nodes.has(summary.nodeId))
+                continue;
             const check = summary.lastCheck;
             const status = {
                 nodeId: summary.nodeId,
@@ -377,7 +730,38 @@ export class NodeManagerService extends EventEmitter {
                 status.omnitronError = check.omnitronError;
             this.statusCache.set(summary.nodeId, status);
             this.emit('node:status', summary.nodeId, status);
+            this.setSummary(summary);
+        }
+    }
+    /**
+     * Aggregated health, for a daemon serving checks without the worker.
+     *
+     * `triggerNodeCheck` used to return `[]` when the worker was absent, and an
+     * empty array on a page whose subject is the fleet reads as "no nodes".
+     */
+    getHealthSummaries(nodeId) {
+        if (nodeId) {
+            const one = this.summaryCache.get(nodeId);
+            return one ? [one] : [];
         }
+        return Array.from(this.summaryCache.values());
+    }
+    /**
+     * Note that one call to the health worker failed.
+     *
+     * Called by the RPC layer when a worker call throws. The call is served
+     * from the daemon instead; the worker keeps its proxy — a process that has
+     * gone is noticed by its exit handler, not guessed from one error. This
+     * exists so the reason appears in the daemon's log rather than only in the
+     * error text of whichever console button was pressed.
+     */
+    reportWorkerUnavailable(method, err) {
+        this.logger.warn({ method, error: err.message }, 'Health monitor worker did not answer this call — serving it from the daemon');
+        this.emit('worker:unavailable', method, err);
+    }
+    /** Report a non-fatal failure in node bookkeeping, with the reason. */
+    reportProblem(operation, err) {
+        this.logger.warn({ operation, error: err.message }, 'Node maintenance step failed');
     }
     /**
      * Convert current node list to serializable check targets for the worker.
@@ -423,7 +807,8 @@ export class NodeManagerService extends EventEmitter {
     secretKey(nodeId, field) {
         return `node:${nodeId}:${field}`;
     }
-    async getSecret(nodeId, field) {
+    /** The vault's own answer, with no in-memory fallback. */
+    async readVault(nodeId, field) {
         if (!this.secrets)
             return null;
         try {
@@ -433,6 +818,18 @@ export class NodeManagerService extends EventEmitter {
             return null;
         }
     }
+    async getSecret(nodeId, field) {
+        const key = this.secretKey(nodeId, field);
+        if (this.secrets) {
+            try {
+                const stored = await this.secrets.get(key);
+                if (stored)
+                    return stored;
+            }
+            catch { /* fall through to the pending value, if any */ }
+        }
+        return this.pendingSecrets.get(key) ?? null;
+    }
     async setSecret(nodeId, field, value) {
         if (!this.secrets) {
             this.logger.warn({ nodeId, field }, 'SecretsService not available — secret not stored');
@@ -449,18 +846,26 @@ export class NodeManagerService extends EventEmitter {
         catch { /* non-critical */ }
     }
     /**
-     * Synchronously populate local node status at construction time.
-     * We ARE the daemon — no RPC/socket needed, just read process info.
+     * Status of the machine this daemon runs on.
+     *
+     * We ARE the daemon — no RPC or socket needed, just read process info, and
+     * read it now rather than at construction. The version came from a string
+     * literal `'0.1.0'` while the package was at 0.2.0, so the fleet view
+     * reported a version the daemon has never been; it is `CLI_VERSION` now,
+     * the same source `omnitron status` uses.
+     *
+     * The role is `master` unconditionally, and that is correct rather than
+     * lazy: this service is only constructed on a non-slave daemon.
      */
-    initLocalStatus() {
-        this.statusCache.set(LOCAL_NODE_ID, {
+    localStatus() {
+        return {
             nodeId: LOCAL_NODE_ID,
             pingReachable: true,
             pingLatencyMs: 0,
             sshConnected: true,
             sshLatencyMs: 0,
             omnitronConnected: true,
-            omnitronVersion: '0.1.0',
+            omnitronVersion: CLI_VERSION,
             omnitronPid: process.pid,
             omnitronUptime: process.uptime() * 1000,
             omnitronRole: 'master',
@@ -471,34 +876,243 @@ export class NodeManagerService extends EventEmitter {
                 release: os.release(),
             },
             checkedAt: new Date().toISOString(),
+        };
+    }
+    // ===========================================================================
+    // Status bookkeeping
+    // ===========================================================================
+    /** Record a check this service performed, and derive its health summary. */
+    recordStatus(node, status) {
+        this.statusCache.set(node.id, status);
+        this.emit('node:status', node.id, status);
+        const previous = this.summaryCache.get(node.id);
+        const reachable = status.sshConnected || status.pingReachable || status.omnitronConnected;
+        const offlineTimeout = node.offlineTimeout ?? DEFAULT_OFFLINE_TIMEOUT_MS;
+        let health;
+        if (status.omnitronConnected) {
+            health = 'online';
+        }
+        else if (reachable) {
+            health = 'degraded';
+        }
+        else {
+            const lastSeen = previous?.lastSeenOnline;
+            health = lastSeen && Date.now() - Date.parse(lastSeen) < offlineTimeout ? 'degraded' : 'offline';
+        }
+        this.setSummary({
+            nodeId: node.id,
+            status: health,
+            lastCheck: {
+                nodeId: node.id,
+                checkedAt: status.checkedAt,
+                checkDurationMs: 0,
+                pingReachable: status.pingReachable,
+                pingLatencyMs: status.pingLatencyMs,
+                pingError: status.pingError ?? null,
+                sshConnected: status.sshConnected,
+                sshLatencyMs: status.sshLatencyMs,
+                sshError: status.sshError ?? null,
+                omnitronConnected: status.omnitronConnected,
+                omnitronVersion: status.omnitronVersion ?? null,
+                omnitronPid: status.omnitronPid ?? null,
+                omnitronUptime: status.omnitronUptime ?? null,
+                omnitronRole: status.omnitronRole ?? null,
+                omnitronError: status.omnitronError ?? null,
+                os: status.os ?? null,
+            },
+            lastSeenOnline: reachable ? status.checkedAt : (previous?.lastSeenOnline ?? null),
+            consecutiveFailures: reachable ? 0 : (previous?.consecutiveFailures ?? 0) + 1,
         });
     }
+    /**
+     * Store a summary, write it to the row, and announce a change of state.
+     *
+     * `nodes.status` and `nodes.last_heartbeat` are columns the schema declares,
+     * `selectNodesSync` reads back, and nothing ever wrote: every row on a live
+     * daemon said `status = 'unknown'` with a null heartbeat, because the only
+     * writer — `touchNodeHeartbeatSync` — had no callers. The upsert filled
+     * them with its own `'unknown'` default on every save.
+     *
+     * What this buys beyond tidiness: `lastSeenOnline` survives a daemon
+     * restart, so the offline grace period is measured from when the node was
+     * actually last seen rather than from boot.
+     */
+    setSummary(summary) {
+        const previous = this.summaryCache.get(summary.nodeId) ?? null;
+        this.summaryCache.set(summary.nodeId, summary);
+        try {
+            this.store.touchNodeHeartbeatSync(summary.nodeId, summary.status, summary.lastCheck?.checkedAt ?? new Date().toISOString());
+        }
+        catch { /* the in-memory answer is still correct */ }
+        if (previous?.status === summary.status)
+            return;
+        const transition = {
+            nodeId: summary.nodeId,
+            status: summary.status,
+            previousStatus: previous?.status ?? null,
+        };
+        this.emit('node:health', transition);
+    }
     // ===========================================================================
     // Persistence
     // ===========================================================================
     load() {
+        /** Rows found carrying a plaintext secret; healed after the loop. */
+        const leaked = [];
         try {
-            if (fs.existsSync(NODES_FILE)) {
-                const data = JSON.parse(fs.readFileSync(NODES_FILE, 'utf-8'));
-                if (Array.isArray(data.nodes)) {
-                    for (const node of data.nodes) {
-                        this.nodes.set(node.id, node);
+            // First-pass: read from SQLite. Each row's `metadata` column
+            // holds the full INode JSON; the denormalised columns are for
+            // future queries (status filters, by-host lookups, etc.).
+            const rows = this.store.selectNodesSync();
+            for (const row of rows) {
+                if (!row.metadata)
+                    continue;
+                try {
+                    const raw = JSON.parse(row.metadata);
+                    const { node, strays } = splitStoredSecrets(raw);
+                    this.nodes.set(node.id, node);
+                    if (strays.length > 0) {
+                        for (const stray of strays)
+                            this.pendingSecrets.set(this.secretKey(node.id, stray.field), stray.value);
+                        leaked.push({ node, strays });
+                    }
+                    // Restore the last known health, but NOT a status reading: a
+                    // reachability answer from a previous daemon is not a current one,
+                    // and the console has no way to tell them apart on the dots. What
+                    // is worth keeping is when the node was last up, which is what the
+                    // offline grace period is measured against.
+                    if (row.status && row.status !== 'unknown') {
+                        this.summaryCache.set(node.id, {
+                            nodeId: node.id,
+                            status: row.status,
+                            lastCheck: null,
+                            lastSeenOnline: row.status === 'offline' ? null : row.last_heartbeat ?? null,
+                            consecutiveFailures: 0,
+                        });
                     }
                 }
+                catch {
+                    // Corrupt row — skip and let the next mutation overwrite.
+                }
+            }
+            // One-shot legacy import. If the SQLite table is empty but the
+            // pre-T-7 nodes.json file exists, migrate it in-place. The
+            // existing file is then unlinked so subsequent boots skip the
+            // branch.
+            if (this.nodes.size === 0 && fs.existsSync(NODES_FILE)) {
+                try {
+                    const data = JSON.parse(fs.readFileSync(NODES_FILE, 'utf-8'));
+                    if (Array.isArray(data.nodes)) {
+                        for (const node of data.nodes) {
+                            this.nodes.set(node.id, node);
+                            this.persistNode(node);
+                        }
+                    }
+                    try {
+                        fs.unlinkSync(NODES_FILE);
+                    }
+                    catch { /* best-effort */ }
+                }
+                catch (err) {
+                    this.logger.warn({ error: err.message }, 'Legacy nodes.json migration failed — leaving file in place');
+                }
             }
         }
         catch (err) {
-            this.logger.warn({ error: err.message }, 'Failed to load nodes file');
+            this.logger.warn({ error: err.message }, 'Failed to load nodes from store');
         }
+        if (leaked.length > 0)
+            void this.healLeakedSecrets(leaked);
+        this.loadCheckConfig();
     }
-    save() {
+    /**
+     * Move a plaintext secret out of a node row and into the vault.
+     *
+     * Observed on a live daemon: a node row whose `metadata` JSON contained
+     * `"sshPassphrase": "<the operator's actual passphrase>"` next to
+     * `"hasPassphrase": true`. An older `updateNode` spread its whole input
+     * into the node — the destructure that stops that was added afterwards, and
+     * it fixed new writes without touching rows already written. `listNodes`
+     * then spread the parsed row onto the wire, so the passphrase went to every
+     * caller with the VIEWER role, and to `omnitron node list`.
+     *
+     * Two things are needed and they are not the same: the wire shape is now
+     * built field by field (`toWireNode`), which stops the leak for any
+     * residue, present or future; and this moves the value to where it should
+     * have been, so the row itself stops holding it.
+     */
+    async healLeakedSecrets(leaked) {
+        for (const { node, strays } of leaked) {
+            for (const { field, value } of strays) {
+                this.logger.warn({ nodeId: node.id, field }, 'Node row held a plaintext SSH secret — moving it to the secret store');
+                // Read the VAULT, not `getSecret`: that one falls back to
+                // `pendingSecrets`, which is where this value already is — so asking
+                // it whether the secret is stored answers "yes" about the copy we are
+                // trying to store, and the write never happens.
+                const existing = await this.readVault(node.id, field);
+                if (!existing) {
+                    if (!this.secrets) {
+                        // Nothing to migrate into. The row keeps the value for now — the
+                        // node would otherwise stop authenticating — but it no longer
+                        // reaches the wire, which is the part that matters.
+                        this.logger.error({ nodeId: node.id, field }, 'No secret store available — plaintext secret left in the node row');
+                        continue;
+                    }
+                    await this.setSecret(node.id, field, value);
+                }
+            }
+            // Rewrite the row from the sanitised node.
+            this.persistNode(node);
+            for (const { field } of strays)
+                this.pendingSecrets.delete(this.secretKey(node.id, field));
+        }
+    }
+    /** Restore the operator's check configuration, if one was ever set. */
+    loadCheckConfig() {
         try {
-            fs.mkdirSync(path.dirname(NODES_FILE), { recursive: true });
-            const data = { nodes: Array.from(this.nodes.values()) };
-            fs.writeFileSync(NODES_FILE, JSON.stringify(data, null, 2));
+            const stored = this.store.kvGetSync(CHECK_CONFIG_KEY);
+            if (!stored)
+                return;
+            this.checkConfig = normalizeCheckConfig({ ...DEFAULT_CHECK_CONFIG, ...stored });
         }
         catch (err) {
-            this.logger.warn({ error: err.message }, 'Failed to save nodes file');
+            this.logger.warn({ error: err.message }, 'Failed to load node check config — using defaults');
+        }
+    }
+    /**
+     * Persist a single node to SQLite. Called on add / update /
+     * status-touch. Pre-T-7 every mutation rewrote the entire
+     * registry JSON file; now it's a single atomic UPSERT.
+     */
+    persistNode(node) {
+        try {
+            // Carry the status forward. `upsertNodeSync` defaults it to `'unknown'`,
+            // so a `save()` triggered by an unrelated edit used to erase what the
+            // last check had established.
+            const known = this.summaryCache.get(node.id);
+            this.store.upsertNodeSync({
+                id: node.id,
+                name: node.name,
+                host: node.host,
+                port: node.daemonPort,
+                role: node.isLocal ? 'master' : 'slave',
+                status: known?.status ?? 'unknown',
+                last_heartbeat: known?.lastCheck?.checkedAt ?? null,
+                metadata: node,
+            });
+        }
+        catch (err) {
+            this.logger.warn({ id: node.id, error: err.message }, 'Failed to persist node');
+        }
+    }
+    /**
+     * Persist every node currently held in memory. Used after a
+     * batch mutation that affected multiple rows (the previous
+     * fs.writeFileSync semantics).
+     */
+    save() {
+        for (const node of this.nodes.values()) {
+            this.persistNode(node);
         }
     }
     ensureLocalNode() {
@@ -528,6 +1142,94 @@ export class NodeManagerService extends EventEmitter {
 // =============================================================================
 // Helpers
 // =============================================================================
+/**
+ * The node as it may be sent to a caller.
+ *
+ * Built field by field on purpose. `listNodes` used to spread the object
+ * parsed out of the `metadata` column, which means the payload was whatever
+ * happened to have been written there — and on a live daemon that included
+ * an SSH passphrase in plaintext. A DTO is a promise about what is returned;
+ * spreading a JSON blob delegates that promise to whoever last wrote the row.
+ */
+function toWireNode(node) {
+    const wire = {
+        id: node.id,
+        name: node.name,
+        host: node.host,
+        sshPort: node.sshPort,
+        sshUser: node.sshUser,
+        sshAuthMethod: node.sshAuthMethod,
+        runtime: node.runtime,
+        daemonPort: node.daemonPort,
+        tags: node.tags ?? [],
+        isLocal: node.isLocal,
+        createdAt: node.createdAt,
+        updatedAt: node.updatedAt,
+    };
+    // The key is a PATH or key content chosen by the operator; the booleans say
+    // only whether a secret exists. Neither is a secret itself.
+    if (node.sshPrivateKey !== undefined)
+        wire.sshPrivateKey = node.sshPrivateKey;
+    if (node.hasPassphrase !== undefined)
+        wire.hasPassphrase = node.hasPassphrase;
+    if (node.hasPassword !== undefined)
+        wire.hasPassword = node.hasPassword;
+    if (node.offlineTimeout !== undefined)
+        wire.offlineTimeout = node.offlineTimeout;
+    return wire;
+}
+/** Separate a stored node from any plaintext secret its row still carries. */
+function splitStoredSecrets(raw) {
+    const { sshPassphrase, sshPassword, ...rest } = raw;
+    const node = rest;
+    const strays = [];
+    if (typeof sshPassphrase === 'string' && sshPassphrase) {
+        node.hasPassphrase = true;
+        strays.push({ field: 'passphrase', value: sshPassphrase });
+    }
+    if (typeof sshPassword === 'string' && sshPassword) {
+        node.hasPassword = true;
+        strays.push({ field: 'password', value: sshPassword });
+    }
+    return { node, strays };
+}
+/** Fallback offline timeout, matching the health monitor's own default. */
+const DEFAULT_OFFLINE_TIMEOUT_MS = 90_000;
+/** A TCP port the daemon will actually try to connect to. */
+function assertPort(kind, value) {
+    if (!Number.isInteger(value) || value < 1 || value > 65_535) {
+        throw new Error(`Invalid ${kind}: ${JSON.stringify(value)}. Expected an integer between 1 and 65535.`);
+    }
+    return value;
+}
+/** Reject a promise that has not settled within `ms`. */
+async function withTimeout(promise, ms) {
+    let timer;
+    try {
+        return await Promise.race([
+            promise,
+            new Promise((_, reject) => {
+                timer = setTimeout(() => reject(new Error(`Timed out after ${ms}ms`)), ms);
+            }),
+        ]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+/** Read the first `bytes` of a file without loading the whole thing. */
+function readFileHead(filePath, bytes) {
+    const fd = fs.openSync(filePath, 'r');
+    try {
+        const buf = Buffer.alloc(bytes);
+        const read = fs.readSync(fd, buf, 0, bytes, 0);
+        return buf.subarray(0, read).toString('utf-8');
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+}
 /**
  * Read SSH private key file contents.
  * If the value is already key content (starts with -----BEGIN), return as-is.