@omnitron-dev/omnitron 0.1.3 → 0.3.0

package/dist/services/node-manager.rpc-service.js

@@ -2,6 +2,21 @@
  * NodeManager RPC Service — Netron endpoints for node management
  *
  * Only available on master omnitron. Slave cannot manage nodes.
+ *
+ * The health-monitor worker is an OPTIMISATION, not a dependency. Every
+ * endpoint that prefers it degrades to the daemon's own checks when the call
+ * fails. The guards used to test `this.healthWorkerProxy` for null — which is
+ * only ever true before the worker is wired and after the daemon begins
+ * shutting down — so a worker that CRASHED left a proxy that looked present
+ * and answered every call with
+ * `TitanError: Service with id HealthMonitor@1.0.0 not found`. The console's
+ * Refresh and Check buttons returned that error verbatim, and the fleet view
+ * went on rendering a status frozen at the worker's last report as though it
+ * were current.
+ *
+ * A crash is now the daemon's to notice: the worker's `onExit` drops the
+ * proxy and schedules a respawn, which wires a new one. What a failed CALL
+ * costs is that call, and no more — see `callWorker`.
  */
 var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
     var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
@@ -13,18 +28,51 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 import { Service, Public } from '@omnitron-dev/titan/decorators';
-import { VIEWER_ROLES, OPERATOR_ROLES } from '../shared/roles.js';
+import { VIEWER_ROLES, OPERATOR_ROLES, CONTROL_PLANE_READ_ROLES } from '../shared/roles.js';
+import { Errors } from '@omnitron-dev/titan/errors';
+/**
+ * Caps on what one call may ask the database for.
+ *
+ * `limit` and `bucketCount` arrived from the caller and went straight into
+ * the query. A page size is not a suggestion the client makes — it is the
+ * cost of the request, and the server is the only side that knows what it can
+ * afford. The console asks for 200 buckets; these numbers are above what any
+ * caller in the tree needs and below what would hurt.
+ */
+const MAX_HISTORY_LIMIT = 500;
+const MAX_UPTIME_BUCKETS = 400;
+/** Clamp an untrusted count to `[1, max]`, falling back for NaN/absent. */
+function boundedCount(value, fallback, max) {
+    if (value == null || !Number.isFinite(value))
+        return fallback;
+    return Math.min(max, Math.max(1, Math.floor(value)));
+}
 let NodeManagerRpcService = class NodeManagerRpcService {
     nodeManager;
+    audit;
     healthWorkerProxy = null;
     healthRepo = null;
-    constructor(nodeManager) {
+    remoteDeployer = null;
+    upgrades = null;
+    slaveConnector = null;
+    titanHealth = null;
+    syncService = null;
+    telemetryRelay = null;
+    leaderElection = null;
+    constructor(nodeManager, 
+    /** The audit trail, when this daemon has one — see `ProjectRpcService`. */
+    audit) {
         this.nodeManager = nodeManager;
+        this.audit = audit;
     }
     /** Set the health worker proxy after the worker is spawned */
     setHealthWorkerProxy(proxy) {
         this.healthWorkerProxy = proxy;
     }
+    /** Whether fleet checks are currently being served by the worker. */
+    hasHealthWorker() {
+        return this.healthWorkerProxy !== null;
+    }
     /** Set the PG repository for direct history reads */
     setHealthRepository(repo) {
         this.healthRepo = repo;
@@ -35,36 +83,99 @@ let NodeManagerRpcService = class NodeManagerRpcService {
     getNode(data) {
         return this.nodeManager.getNode(data.id);
     }
-    addNode(data) {
-        return this.nodeManager.addNode(data);
+    async addNode(data) {
+        const node = await this.nodeManager.addNode(data);
+        // Identifiers, not the input: `data` carries `sshPassword` on this path.
+        await this.audit?.record({
+            action: 'node.add',
+            resourceType: 'node',
+            resourceId: node.id,
+            details: { name: node.name, host: node.host, daemonPort: node.daemonPort, sshAuthMethod: node.sshAuthMethod },
+        });
+        return node;
     }
-    updateNode(data) {
+    async updateNode(data) {
         const { id, ...input } = data;
-        return this.nodeManager.updateNode(id, input);
+        const node = await this.nodeManager.updateNode(id, input);
+        // WHICH fields were touched, never their values — `sshPassword` and
+        // `sshPassphrase` arrive through here.
+        await this.audit?.record({
+            action: 'node.update',
+            resourceType: 'node',
+            resourceId: id,
+            details: { name: node.name, fields: Object.keys(input).sort().join(',') },
+        });
+        return node;
     }
-    removeNode(data) {
-        return this.nodeManager.removeNode(data.id);
+    /**
+     * Remove a node, and everything keyed to it.
+     *
+     * The check history goes too. It is keyed by an id that will never be
+     * issued again, nothing can read it back, and the retention sweep only
+     * looks at age — so a fleet that churns nodes accumulates rows no query
+     * will ever name. Deletion that stops at the registry row is deletion of
+     * the part you can see.
+     */
+    async removeNode(data) {
+        const before = this.nodeManager.getNode(data.id);
+        await this.nodeManager.removeNode(data.id);
+        await this.audit?.record({
+            action: 'node.remove',
+            resourceType: 'node',
+            resourceId: data.id,
+            details: { name: before?.name ?? null, host: before?.host ?? null },
+        });
+        if (!this.healthRepo)
+            return;
+        try {
+            await this.healthRepo.deleteHistory(data.id);
+        }
+        catch (err) {
+            // The node IS gone; failing the call now would invite a retry that
+            // cannot succeed. Say so instead.
+            this.nodeManager.reportProblem('deleteHistory', err);
+        }
     }
     async checkNodeStatus(data) {
+        this.assertRegistered(data.id);
         // Delegate to worker if available — returns fresh result directly
-        if (this.healthWorkerProxy) {
-            const summaries = await this.healthWorkerProxy.triggerCheck(data.id);
-            const summary = summaries.find((s) => s.nodeId === data.id);
-            if (summary?.lastCheck) {
-                const status = summaryToNodeStatus(summary);
-                this.nodeManager.updateStatusCacheFromWorker([summary]);
-                return status;
-            }
+        const summaries = await this.callWorker((w) => w.triggerCheck(data.id), 'triggerCheck');
+        const summary = summaries?.find((s) => s.nodeId === data.id);
+        if (summary?.lastCheck) {
+            const status = summaryToNodeStatus(summary);
+            this.nodeManager.updateStatusCacheFromWorker([summary]);
+            return status;
         }
         return this.nodeManager.checkNodeStatus(data.id);
     }
+    /**
+     * Check every node, returning each one's STATUS.
+     *
+     * Two methods run the same round. This one answers `INodeStatus[]` — the
+     * three reachability layers with their errors — and `triggerNodeCheck({})`
+     * answers `INodeHealthSummary[]`, which adds `consecutiveFailures` and
+     * `lastSeenOnline` and drops the per-layer detail. Neither is redundant and
+     * neither had a production caller: the console now uses both, one for the
+     * diagnosis panel and one for "has it been down since Tuesday or did it
+     * blink once".
+     *
+     * Which to call is decided by which of those two answers you need, and
+     * saying so here is cheaper than the next reader discovering the difference
+     * after choosing the wrong one.
+     */
     async checkAllNodes() {
-        if (this.healthWorkerProxy) {
-            const summaries = await this.healthWorkerProxy.triggerCheck();
-            if (summaries.length > 0) {
-                this.nodeManager.updateStatusCacheFromWorker(summaries);
-                return summaries.filter((s) => s.lastCheck).map(summaryToNodeStatus);
-            }
+        const summaries = await this.callWorker((w) => w.triggerCheck(), 'triggerCheck');
+        if (summaries) {
+            this.nodeManager.updateStatusCacheFromWorker(summaries);
+            // The FILTERED list decides, not the raw one. A worker that answered
+            // with summaries none of which carries a `lastCheck` — one that has
+            // been spawned but not yet told which nodes exist, say — passed the
+            // `length > 0` test and then returned an empty array, and an empty
+            // array on the page whose subject is the fleet reads as "you have no
+            // nodes". Whatever empties it, the daemon can still answer.
+            const checked = this.registeredOnly(summaries).filter((s) => s.lastCheck).map(summaryToNodeStatus);
+            if (checked.length > 0)
+                return checked;
         }
         return this.nodeManager.checkAllNodes();
     }
@@ -72,7 +183,7 @@ let NodeManagerRpcService = class NodeManagerRpcService {
     async getCheckHistory(data) {
         if (!this.healthRepo)
             return [];
-        return this.healthRepo.getHistory(data.nodeId, data.limit ?? 50);
+        return this.healthRepo.getHistory(data.nodeId, boundedCount(data.limit, 50, MAX_HISTORY_LIMIT));
     }
     /**
      * Uptime bar data from PG — aggregated into fixed-interval buckets.
@@ -81,17 +192,445 @@ let NodeManagerRpcService = class NodeManagerRpcService {
     async getUptimeBar(data) {
         if (!this.healthRepo)
             return [];
-        return this.healthRepo.getUptimeBar(data.nodeId, data.bucketCount ?? 60, data.intervalMs ?? 300_000);
+        return this.healthRepo.getUptimeBar(data.nodeId, boundedCount(data.bucketCount, 60, MAX_UPTIME_BUCKETS), data.intervalMs ?? 300_000);
     }
+    /**
+     * Every node's aggregated health.
+     *
+     * From the daemon's own summaries when the worker cannot answer, like
+     * every other endpoint here. It answered `[]` — measured 2026-09-23: 3
+     * entries, then 0 the moment one unrelated call detached the worker — and
+     * an empty list on the page whose subject is the fleet reads as «no
+     * nodes», while the daemon held a summary for each of them.
+     */
     async getNodeHealthSummaries() {
-        if (!this.healthWorkerProxy)
-            return [];
-        return this.healthWorkerProxy.getStatusSummaries();
+        return this.registeredOnly((await this.callWorker((w) => w.getStatusSummaries(), 'getStatusSummaries')) ??
+            this.nodeManager.getHealthSummaries());
     }
     async triggerNodeCheck(data) {
-        if (!this.healthWorkerProxy)
-            return [];
-        return this.healthWorkerProxy.triggerCheck(data.nodeId);
+        if (data.nodeId)
+            this.assertRegistered(data.nodeId);
+        const fromWorker = await this.callWorker((w) => w.triggerCheck(data.nodeId), 'triggerCheck');
+        const summaries = fromWorker ? this.registeredOnly(fromWorker) : null;
+        if (summaries && summaries.length > 0) {
+            this.nodeManager.updateStatusCacheFromWorker(summaries);
+            return summaries;
+        }
+        // No worker: run the checks here so the caller gets an answer rather than
+        // an empty array that reads as "the fleet is empty".
+        if (data.nodeId) {
+            await this.nodeManager.checkNodeStatus(data.nodeId);
+        }
+        else {
+            await this.nodeManager.checkAllNodes();
+        }
+        return this.nodeManager.getHealthSummaries(data.nodeId);
+    }
+    /**
+     * Install a bundle on a node, beside what it is running.
+     *
+     * On the daemon rather than in the CLI because the node's credentials are
+     * in the daemon's vault. `nodeToDeployTarget` resolves them here and they
+     * go no further; the caller passes an id and gets back whether it worked.
+     *
+     * OPERATOR, not VIEWER: this writes several hundred megabytes to a remote
+     * machine.
+     */
+    async installBundleOnNode(data) {
+        const target = await this.nodeManager.nodeToDeployTarget(data.nodeId);
+        // Under the machine's deploy lease: one writer per node, whoever writes.
+        const installed = await this.deployer().underLease(target, `fleet upgrade: install ${data.version}`, () => this.deployer().installBundle(target, data.archivePath, data.version));
+        await this.audit?.record({
+            action: 'node.bundle.install',
+            resourceType: 'node',
+            resourceId: data.nodeId,
+            details: { version: data.version, installed },
+        });
+        return installed;
+    }
+    /**
+     * Make an installed version current, and restart the node into it.
+     *
+     * Separate from the install for the reason the versioned layout exists:
+     * this is the only step that changes what the node serves.
+     */
+    async activateBundleOnNode(data) {
+        const target = await this.nodeManager.nodeToDeployTarget(data.nodeId);
+        // The step that restarts the node's daemon — the last thing to do under
+        // somebody else's deployment.
+        const activated = await this.deployer().underLease(target, `fleet upgrade: activate ${data.version}`, () => 
+        // Held: activation stops every app on the node (`activateBundleHeld`).
+        this.deployer().activateBundleHeld(target, data.version, '/opt/omnitron', data.keepVersions ?? 3));
+        // The one step that changes what a node SERVES, and the one an operator
+        // will want to find afterwards: `fleet upgrade` reaches it through this
+        // method, so every node upgrade lands in the trail whether it was asked
+        // for from the CLI or the console.
+        await this.audit?.record({
+            action: 'node.bundle.activate',
+            resourceType: 'node',
+            resourceId: data.nodeId,
+            details: { version: data.version, activated, keepVersions: data.keepVersions ?? 3 },
+        });
+        return activated;
+    }
+    /**
+     * Upgrade a node's omnitron to the one this daemon is built from.
+     *
+     * `fleet upgrade` does the same thing from a terminal, and the console
+     * could not do it at all: the two methods above take an archive path on
+     * THIS filesystem, and only the CLI knew how to produce one. The service
+     * behind this builds it, then calls exactly those two.
+     *
+     * Started and polled, not awaited: a build is minutes, and an RPC that
+     * takes minutes is one that times out somewhere in the middle.
+     */
+    async upgradeNode(data) {
+        if (!this.upgrades) {
+            return { started: false, reason: 'Node upgrades are not configured on this daemon.' };
+        }
+        const outcome = await this.upgrades.start(data.nodeId);
+        await this.audit?.record({
+            action: outcome.started ? 'node.upgrade.start' : 'node.upgrade.refused',
+            resourceType: 'node',
+            resourceId: data.nodeId,
+            ...(outcome.reason ? { details: { reason: outcome.reason } } : {}),
+        });
+        return outcome;
+    }
+    /** Where each node's upgrade got to. Newest state per node. */
+    async getUpgradeProgress() {
+        return this.upgrades?.listProgress() ?? [];
+    }
+    // ---------------------------------------------------------------------------
+    // Fleet rollout — declared before it is built, so the console can be written
+    // against the real contract instead of an untyped `invoke`. Each refuses
+    // out loud: a stub that answered with an empty plan would be read as «this
+    // fleet needs no upgrades», which is the worst available lie here.
+    // ---------------------------------------------------------------------------
+    /**
+     * What a rollout would do to each node, before anything ships.
+     *
+     * A read, not an action — `CONTROL_PLANE_READ_ROLES` rather than
+     * `OPERATOR_ROLES` — so an operator without deploy rights can still see
+     * what the fleet is running and what it would take to move it.
+     */
+    async planUpgrade(data) {
+        if (!this.upgrades) {
+            return {
+                targetVersion: null,
+                compared: false,
+                rows: [],
+                refusal: 'Node upgrades are not configured on this daemon.',
+            };
+        }
+        const plan = await this.upgrades.plan(data?.nodeIds, { build: data?.build === true });
+        return {
+            targetVersion: plan.compared ? plan.targetVersion : null,
+            compared: plan.compared,
+            refusal: plan.refusal ?? null,
+            rows: plan.steps.map((step) => ({
+                nodeId: step.node.nodeId,
+                label: step.node.name,
+                // `null`, not `''`: an entry with no usable address is a different
+                // thing from one whose machine could not be reached, and an empty
+                // string in a column of addresses reads as the former being the
+                // latter.
+                host: step.node.address ?? null,
+                currentVersion: step.node.currentVersion,
+                targetVersion: plan.compared ? plan.targetVersion : null,
+                action: step.decision.action,
+                // An `upgrade` row has no reason to give: it is doing what was
+                // asked. `skip` and `refuse` carry theirs verbatim — they are
+                // written for an operator to read, and shortening them here would
+                // undo that.
+                // An `upgrade` row normally owes no explanation — it is doing what
+                // was asked. But in an unbuilt plan it owes one: without a target
+                // version nothing was compared, so «upgrade» means «not ruled out»
+                // rather than «out of date», and twelve such rows must not read as
+                // twelve nodes needing work.
+                because: step.decision.action !== 'upgrade'
+                    ? step.decision.because
+                    : plan.compared
+                        ? ''
+                        : 'not compared — no bundle has been built',
+            })),
+        };
+    }
+    /** Queue a rollout across several nodes. Returns once queued, not once done. */
+    async upgradeNodes(data) {
+        // An empty selection is refused before anything else: answering it with
+        // `{ accepted: [], refused: [] }` would read as «queued nothing, nothing wrong».
+        if (!Array.isArray(data?.nodeIds) || data.nodeIds.length === 0) {
+            throw Errors.badRequest('A rollout needs at least one node');
+        }
+        if (!this.upgrades) {
+            return {
+                accepted: [],
+                refused: data.nodeIds.map((nodeId) => ({ nodeId, because: 'Node upgrades are not configured on this daemon.' })),
+            };
+        }
+        const outcome = this.upgrades.rollout(data.nodeIds, data.concurrency ?? 1);
+        await this.audit?.record({
+            action: 'node.rollout',
+            resourceType: 'node',
+            resourceId: null,
+            details: {
+                accepted: outcome.accepted.join(',') || '(none)',
+                refused: outcome.refused.map((r) => r.nodeId).join(',') || '(none)',
+                concurrency: data.concurrency ?? 1,
+            },
+        });
+        return outcome;
+    }
+    /** Drop a node from a running rollout, if it has not started installing. */
+    async cancelUpgrade(data) {
+        if (!this.upgrades)
+            return { stopped: false, because: 'Node upgrades are not configured on this daemon.' };
+        const outcome = this.upgrades.cancel(data.nodeId);
+        if (outcome.stopped) {
+            await this.audit?.record({ action: 'node.upgrade.cancel', resourceType: 'node', resourceId: data.nodeId });
+        }
+        return outcome;
+    }
+    /** Wired by the daemon at startup, on a master. */
+    setUpgradeService(service) {
+        this.upgrades = service;
+    }
+    /**
+     * The deployer, made once and kept.
+     *
+     * Lazily, because a daemon that never upgrades a node should not pay for an
+     * execution engine at boot — and because `RemoteDeployer` is only reachable
+     * from these two methods, so there is nothing else to construct it for.
+     */
+    deployer() {
+        if (!this.remoteDeployer) {
+            throw new Error('Remote deployment is not configured on this daemon.');
+        }
+        return this.remoteDeployer;
+    }
+    /** Wired by the daemon at startup, on a master. */
+    setRemoteDeployer(deployer) {
+        this.remoteDeployer = deployer;
+    }
+    /**
+     * The daemon's mesh connector, so the console can be told what it sees —
+     * and so the daemon's own node check asks the daemon over it.
+     *
+     * Handed on to the node manager here rather than by a second line in the
+     * daemon: this is the one place the daemon gives the connector to the node
+     * services, and the check path that needs it is the one this service falls
+     * back to (`checkNodeStatus` without a worker), as is the daemon's own
+     * fallback loop.
+     */
+    setSlaveConnector(connector) {
+        this.slaveConnector = connector;
+        this.nodeManager.setSlaveConnector(connector);
+    }
+    /**
+     * This daemon's own titan-health, for the local node's indicators.
+     *
+     * A remote node is asked over the mesh; the local one cannot be, because a
+     * daemon has no mesh connection to itself.
+     */
+    setTitanHealth(service) {
+        this.titanHealth = service;
+    }
+    /** This daemon's own replication state, for the local node. */
+    setSyncService(service) {
+        this.syncService = service;
+    }
+    /** This daemon's own telemetry relay, for the local node. */
+    setTelemetryRelay(relay) {
+        this.telemetryRelay = relay;
+    }
+    /** This daemon's own election state, for the local node. */
+    setLeaderElection(election) {
+        this.leaderElection = election;
+    }
+    /**
+       * Ask a service ON the node, wherever the node is.
+       *
+       * The console's own clients are bound to the daemon it is connected to, so
+       * every question about a node used to be answerable only about the local
+       * one. `SlaveConnector.invokeOnSlave` has always been able to reach any
+       * service on any node over the mesh and had exactly one caller; this is the
+       * seam the node readers share, so adding the next one is a method name
+       * rather than another copy of the plumbing.
+       *
+       * `ok: false` carries the REASON and never a verdict. A node outside the
+       * mesh has not answered; saying "unhealthy" or "not replicating" about
+       * silence sends an operator to repair the wrong thing.
+       */
+    async askNode(nodeId, service, method, args, local) {
+        const node = this.nodeManager.getNode(nodeId);
+        if (!node)
+            return { ok: false, error: 'No such node' };
+        if (node.isLocal) {
+            if (!local)
+                return { ok: false, error: `${service} is not wired on this daemon` };
+            try {
+                return { ok: true, value: await local() };
+            }
+            catch (err) {
+                return { ok: false, error: err.message };
+            }
+        }
+        if (!this.slaveConnector)
+            return { ok: false, error: 'This daemon has no mesh connector' };
+        try {
+            const value = (await this.slaveConnector.invokeOnSlave(node.host, node.daemonPort, service, method, args));
+            return { ok: true, value };
+        }
+        catch (err) {
+            return { ok: false, error: err.message };
+        }
+    }
+    /**
+     * What the node's OWN titan-health says about it.
+     *
+     * Every omnitron daemon runs `TitanHealthModule` and answers `Health@1.0.0`,
+     * with titan's built-in indicators — memory, event loop, disk, database,
+     * redis — plus the two this daemon registers: docker and the apps it
+     * supervises. The remote nodes have had all of it since they were
+     * provisioned, and nothing ever asked them: the console's `health` client is
+     * bound to `daemonClient.daemon`, so it only ever spoke to the daemon it was
+     * connected to.
+     *
+     * There is no new mechanism here. `SlaveConnector.invokeOnSlave` already
+     * calls any service on any node over the mesh — it had exactly one caller,
+     * `OmnitronInfra.provisionStack` — and `Health@1.0.0` is already answering
+     * on the other side.
+     *
+     * `reachable: false` with a reason is NOT `unhealthy`. A node that cannot be
+     * asked has not reported anything, and rendering silence as a verdict is the
+     * same mistake as reading an unmeasured SSH layer as a refusal.
+     */
+    async getNodeIndicators(data) {
+        const r = await this.askNode(data.nodeId, 'Health@1.0.0', 'check', [], this.titanHealth ? () => this.titanHealth.check() : null);
+        return r.ok
+            ? { nodeId: data.nodeId, reachable: true, error: null, status: r.value?.status ?? null, indicators: r.value?.indicators ?? {} }
+            : { nodeId: data.nodeId, reachable: false, error: r.error, status: null, indicators: {} };
+    }
+    /**
+     * What the node's own daemon reports, asked over the mesh.
+     *
+     * `fleet status`, `fleet health` and `fleet metrics` each dial
+     * `host:9700` and ask `OmnitronDaemon` from the CLI. A node whose daemon
+     * port is not open to this master answers nothing — which is the normal
+     * state of a hardened server and the whole reason the mesh tunnels over
+     * SSH. Measured on the test node while it was running six apps with every
+     * port listening:
+     *
+     *     SERVER     HOST                 STATUS   APPS
+     *     daos-test  37.27.130.185:9700   offline  0
+     *
+     * Three methods rather than one relay taking a service name: a viewer may
+     * ask a node what it is running, and may not use this master as a proxy for
+     * arbitrary calls into it.
+     */
+    async getNodeDaemonStatus(data) {
+        return this.daemonAnswer(data.nodeId, 'status', []);
+    }
+    async getNodeDaemonHealth(data) {
+        return this.daemonAnswer(data.nodeId, 'getHealth', [{}]);
+    }
+    async getNodeDaemonMetrics(data) {
+        return this.daemonAnswer(data.nodeId, 'getMetrics', [{}]);
+    }
+    /** One of the three questions above, put through `askNode`. */
+    async daemonAnswer(nodeId, method, args) {
+        const r = await this.askNode(nodeId, 'OmnitronDaemon', method, args, null);
+        return r.ok
+            ? { nodeId, reachable: true, error: null, answer: r.value ?? null }
+            : { nodeId, reachable: false, error: r.error, answer: null };
+    }
+    /**
+     * Whether the node's data is actually MOVING.
+     *
+     * Every other reading on the node page answers "can we reach it". A node can
+     * be green on all of them and replicate nothing — and for every registered
+     * node that no stack had been deployed onto, that is exactly what happened:
+     * 47,407 entries buffered on one, none delivered, over eleven hours, while
+     * the page showed it healthy.
+     *
+     * `OmnitronSync.getSyncStatus` has answered this since it was written; its
+     * own docblock says "for webapp monitoring" and the console has never known
+     * the service exists. `pendingItems` climbing with `lastSyncAt` standing
+     * still is the whole diagnosis, and it was one call away the entire time.
+     */
+    async getNodeSyncStatus(data) {
+        const r = await this.askNode(data.nodeId, 'OmnitronSync', 'getSyncStatus', [], this.syncService ? () => this.syncService.getStatus() : null);
+        return r.ok
+            ? { nodeId: data.nodeId, reachable: true, error: null, sync: r.value }
+            : { nodeId: data.nodeId, reachable: false, error: r.error, sync: null };
+    }
+    /**
+       * The node's telemetry relay: what it buffered, sent, failed and DROPPED.
+       *
+       * `getNodeSyncStatus` answers whether the log/metric replication is moving.
+       * This answers the other pipe — the telemetry relay — and it carries the one
+       * counter in the fleet that reports LOSS. `totalDropped` is what the buffer
+       * threw away because it was full, and a number that only ever goes up while
+       * nobody looks is how a gap in the metrics is discovered months later from
+       * the chart rather than from the daemon that made it.
+       *
+       * `OmnitronTelemetry.getRelayStats` has answered this since it was written —
+       * the file's own header says "Webapp → Leader telemetry stats (relay
+       * health)" — and the console has never called it.
+       */
+    async getNodeRelayStats(data) {
+        const r = await this.askNode(data.nodeId, 'OmnitronTelemetry', 'getRelayStats', [], this.telemetryRelay ? () => Promise.resolve(this.telemetryRelay.stats()) : null);
+        return r.ok
+            ? { nodeId: data.nodeId, reachable: true, error: null, relay: r.value ?? null }
+            : { nodeId: data.nodeId, reachable: false, error: r.error, relay: null };
+    }
+    /**
+       * Which node each node thinks is the leader.
+       *
+       * The fourth reader through `askNode`, and the one that is only meaningful
+       * ACROSS nodes: a single node's answer is unremarkable, and two nodes naming
+       * different leaders, or sitting in different terms, is a split brain — the
+       * state in which every node is individually healthy and the fleet is not.
+       * Nothing in the console could see it, because `OmnitronCluster` is one of
+       * the services its client does not know, and `getClusterState`'s own
+       * docblock says it is "used by CLI and webapp dashboard".
+       */
+    async getNodeClusterState(data) {
+        const r = await this.askNode(data.nodeId, 'OmnitronCluster', 'getClusterState', [], this.leaderElection ? () => Promise.resolve(this.leaderElection.getClusterState()) : null);
+        return r.ok
+            ? { nodeId: data.nodeId, reachable: true, error: null, cluster: r.value ?? null }
+            : { nodeId: data.nodeId, reachable: false, error: r.error, cluster: null };
+    }
+    /**
+     * Whether each node is replicating, and how it is being reached.
+     *
+     * The node page could say a node was up — SSH answers, the daemon answers
+     * a ping — while nothing it collected ever arrived. Reachability and
+     * membership are different questions, and only the second one is about
+     * whether the data on the master is the whole of what the fleet knows.
+     *
+     * Three states are worth distinguishing, and the console renders all
+     * three: not in the mesh at all; connected but UNAUTHENTICATED, which
+     * answers pings and can replicate nothing; and connected over an SSH
+     * tunnel, which works but says the node's daemon port is closed to this
+     * master and is worth knowing before someone debugs the latency.
+     */
+    async getMeshStatus() {
+        const connections = this.slaveConnector?.getConnections() ?? [];
+        const byAddress = new Map(connections.map((c) => [`${c.host}:${c.port}`, c]));
+        return this.nodeManager.listNodes().map((node) => {
+            const conn = byAddress.get(`${node.host}:${node.daemonPort}`);
+            return {
+                nodeId: node.id,
+                inMesh: Boolean(conn),
+                status: conn?.status ?? 'disconnected',
+                via: conn?.via ?? null,
+                authenticated: conn?.authenticated ?? false,
+                lastHeartbeat: conn?.lastHeartbeat ?? null,
+                lastError: conn?.lastError ?? null,
+            };
+        });
     }
     listSshKeys() {
         return this.nodeManager.listSshKeys();
@@ -99,10 +638,71 @@ let NodeManagerRpcService = class NodeManagerRpcService {
     getCheckConfig() {
         return this.nodeManager.getCheckConfig();
     }
+    /** How much history exists, and how wide one uptime-bar segment is. */
+    getHistoryConfig() {
+        return this.nodeManager.getHistoryConfig();
+    }
     setCheckConfig(data) {
         this.nodeManager.setCheckConfig(data);
         return this.nodeManager.getCheckConfig();
     }
+    // ===========================================================================
+    // Private
+    // ===========================================================================
+    /**
+     * Call the health worker, or report that this call could not be served by it.
+     *
+     * `null` means "no answer from the worker for THIS call" — no proxy, or a
+     * call that threw. Either way the caller takes the in-process path.
+     *
+     * The proxy is kept. It used to be dropped on ANY error, and nothing but a
+     * respawn ever wired it again — so one call the worker refused, from a
+     * worker that was alive and checking, detached it for the life of the
+     * process. Measured 2026-09-23: `omnitron node check 16f3dd5a` (the id
+     * `node list` prints) reached the worker as an unknown id, the worker threw
+     * `Node not found`, `getNodeHealthSummaries` fell from 3 entries to 0, and
+     * every later check went through the daemon's own path. The same happened
+     * on 2026-09-22 from one `RPC request timed out after 5000ms` on a loaded
+     * host. A process that has actually gone is the daemon's to notice, and it
+     * does: the worker's `onExit` clears this proxy (daemon.ts,
+     * startHealthMonitorWorker) and a respawn wires the next one.
+     */
+    async callWorker(fn, method) {
+        const worker = this.healthWorkerProxy;
+        if (!worker)
+            return null;
+        try {
+            return await fn(worker);
+        }
+        catch (err) {
+            this.nodeManager.reportWorkerUnavailable(method, err);
+            return null;
+        }
+    }
+    /**
+     * Refuse an id the registry does not hold, before anything is asked about it.
+     *
+     * Checked here, not left to the worker: the worker throws for an id it
+     * does not know, and a throw from the worker is what used to detach it
+     * (see `callWorker`). An unknown id is the caller's mistake and is answered
+     * as one, in the registry's words.
+     */
+    assertRegistered(id) {
+        if (!this.nodeManager.getNode(id))
+            throw Errors.notFound('Node', id);
+    }
+    /**
+     * The worker's answers about nodes the registry still holds.
+     *
+     * The worker is told of a removal by an IPC call that can fail, and its
+     * answers are its own memory: measured 2026-09-23, a node removed at
+     * 07:28:00Z came back from `checkAllNodes` 82 minutes later as `online`,
+     * and the CLI printed it as a bare id with two green dots. A status with no
+     * node behind it is not a status of anything.
+     */
+    registeredOnly(items) {
+        return items.filter((item) => Boolean(this.nodeManager.getNode(item.nodeId)));
+    }
 };
 __decorate([
     Public({ auth: { roles: VIEWER_ROLES } }),
@@ -132,7 +732,7 @@ __decorate([
     Public({ auth: { roles: OPERATOR_ROLES } }),
     __metadata("design:type", Function),
     __metadata("design:paramtypes", [Object]),
-    __metadata("design:returntype", void 0)
+    __metadata("design:returntype", Promise)
 ], NodeManagerRpcService.prototype, "removeNode", null);
 __decorate([
     Public({ auth: { roles: VIEWER_ROLES } }),
@@ -170,6 +770,96 @@ __decorate([
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
 ], NodeManagerRpcService.prototype, "triggerNodeCheck", null);
+__decorate([
+    Public({ auth: { roles: OPERATOR_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "installBundleOnNode", null);
+__decorate([
+    Public({ auth: { roles: OPERATOR_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "activateBundleOnNode", null);
+__decorate([
+    Public({ auth: { roles: OPERATOR_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "upgradeNode", null);
+__decorate([
+    Public({ auth: { roles: VIEWER_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "getUpgradeProgress", null);
+__decorate([
+    Public({ auth: { roles: CONTROL_PLANE_READ_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "planUpgrade", null);
+__decorate([
+    Public({ auth: { roles: OPERATOR_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "upgradeNodes", null);
+__decorate([
+    Public({ auth: { roles: OPERATOR_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "cancelUpgrade", null);
+__decorate([
+    Public({ auth: { roles: VIEWER_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "getNodeIndicators", null);
+__decorate([
+    Public({ auth: { roles: VIEWER_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "getNodeDaemonStatus", null);
+__decorate([
+    Public({ auth: { roles: VIEWER_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "getNodeDaemonHealth", null);
+__decorate([
+    Public({ auth: { roles: VIEWER_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "getNodeDaemonMetrics", null);
+__decorate([
+    Public({ auth: { roles: VIEWER_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "getNodeSyncStatus", null);
+__decorate([
+    Public({ auth: { roles: VIEWER_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "getNodeRelayStats", null);
+__decorate([
+    Public({ auth: { roles: VIEWER_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "getNodeClusterState", null);
+__decorate([
+    Public({ auth: { roles: VIEWER_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], NodeManagerRpcService.prototype, "getMeshStatus", null);
 __decorate([
     Public({ auth: { roles: OPERATOR_ROLES } }),
     __metadata("design:type", Function),
@@ -180,17 +870,23 @@ __decorate([
     Public({ auth: { roles: VIEWER_ROLES } }),
     __metadata("design:type", Function),
     __metadata("design:paramtypes", []),
-    __metadata("design:returntype", void 0)
+    __metadata("design:returntype", Object)
 ], NodeManagerRpcService.prototype, "getCheckConfig", null);
+__decorate([
+    Public({ auth: { roles: VIEWER_ROLES } }),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Object)
+], NodeManagerRpcService.prototype, "getHistoryConfig", null);
 __decorate([
     Public({ auth: { roles: OPERATOR_ROLES } }),
     __metadata("design:type", Function),
     __metadata("design:paramtypes", [Object]),
-    __metadata("design:returntype", void 0)
+    __metadata("design:returntype", Object)
 ], NodeManagerRpcService.prototype, "setCheckConfig", null);
 NodeManagerRpcService = __decorate([
     Service({ name: 'OmnitronNodes' }),
-    __metadata("design:paramtypes", [Function])
+    __metadata("design:paramtypes", [Function, Object])
 ], NodeManagerRpcService);
 export { NodeManagerRpcService };
 // =============================================================================