@omnitron-dev/omnitron 0.1.3 → 0.3.0

package/dist/cluster/mesh-link.js

@@ -0,0 +1,212 @@
+/**
+ * How a master reaches a node's daemon, and what it presents when it gets
+ * there.
+ *
+ * Both halves were missing, and each hid the other.
+ *
+ * TRANSPORT. `SlaveConnector` dials `tcp://<host>:<port>` directly. A node on
+ * the public internet normally permits SSH and little else — the first one
+ * provisioned by this code allows 22/tcp and three ports from one private
+ * address — so the daemon port is unreachable and the connection times out.
+ * Nothing above reads that as "the mesh cannot form": the node answers SSH,
+ * deploys, reports healthy, and replicates nothing.
+ *
+ * CREDENTIAL. Past the transport, `OmnitronSync.drainBuffer` answers
+ * `Authentication required`. It is declared for `admin | operator |
+ * service_role`, and the master had no token any node would accept. The
+ * comment on that endpoint has said so since it was written: "`service_role`
+ * cannot be presented today … this is one end of an unfinished cross-daemon
+ * path". This is the other end.
+ *
+ * A daemon persists its own `auth.jwtSecret` in `~/.omnitron/config.json` and
+ * verifies HS256 tokens issued `omnitron` against it. The master already
+ * holds root SSH for every node it provisioned — that is how it put the
+ * daemon there — so it can read that secret and mint a short-lived
+ * `service_role` token. This grants the master nothing it did not already
+ * have: whoever can run `omnitron up` on a machine can read the file.
+ *
+ * Measured against the live node, in this order:
+ *
+ *     direct tcp://37.27.130.185:9700        → connection timeout
+ *     through the tunnel, OmnitronDaemon.ping → v0.2.0+local.de0ccff6b022…
+ *     OmnitronSync.drainBuffer, no token      → Authentication required
+ *     OmnitronSync.drainBuffer, service_role  → 2 entries
+ */
+import net from 'node:net';
+import { createHash } from 'node:crypto';
+/** The transport that needs nothing arranged. */
+export const directLink = async (target) => ({
+    url: `tcp://${target.host}:${target.port}`,
+    via: 'direct',
+});
+/**
+ * Can a TCP connection be made, within a budget?
+ *
+ * Its own probe rather than "try netron and see": a netron connect that
+ * fails has already built a Netron, registered a transport and started a
+ * handshake, and the failure surfaces as a protocol timeout that reads like
+ * a broken daemon. A refused or timed-out socket is the plain fact, and it
+ * is what decides between the two transports.
+ */
+export function probeTcp(host, port, timeoutMs) {
+    return new Promise((resolve) => {
+        const socket = new net.Socket();
+        let settled = false;
+        const done = (reachable) => {
+            if (settled)
+                return;
+            settled = true;
+            socket.destroy();
+            resolve(reachable);
+        };
+        socket.setTimeout(timeoutMs);
+        socket.once('connect', () => done(true));
+        socket.once('timeout', () => done(false));
+        socket.once('error', () => done(false));
+        socket.connect(port, host);
+    });
+}
+/**
+ * The command that reads a daemon's own signing secret off a node.
+ *
+ * Node's own JSON parse rather than a shell tool, because `jq` is not
+ * something a freshly provisioned machine has, and the runtime omnitron
+ * installed certainly is. Written to stdout with no newline so the caller
+ * can trim nothing and still be right.
+ */
+export const READ_DAEMON_SECRET = `node -e 'const c=require(process.env.HOME+"/.omnitron/config.json");` +
+    `process.stdout.write((c.auth&&c.auth.jwtSecret)||"")'`;
+/**
+ * Mint a token in the shape a daemon verifies.
+ *
+ * Deliberately short-lived and without a `sid`. A master daemon validates
+ * `sid` against its session table; a slave has no session store and skips
+ * that check — so a token carrying a session id that exists nowhere would
+ * work on a slave and be refused by a master, which is the wrong way for a
+ * credential to behave. Carrying none is true on both.
+ */
+export async function mintServiceToken(secret, subject, ttl = '5m') {
+    const { SignJWT } = await import('jose');
+    return new SignJWT({ role: 'service_role' })
+        .setProtectedHeader({ alg: 'HS256' })
+        .setSubject(subject)
+        .setIssuer('omnitron')
+        .setIssuedAt()
+        .setExpirationTime(ttl)
+        .sign(new TextEncoder().encode(secret));
+}
+/**
+ * Build the dialer the connector uses: direct where direct works, SSH where
+ * it does not.
+ *
+ * Direct is tried first on every connect rather than remembered, because
+ * which one works is a property of the network at this moment — a firewall
+ * rule changes, a node moves onto the cluster's private side — and a cached
+ * answer to that question is a node that stays on the slow path forever, or
+ * one that stops connecting at all.
+ *
+ * A node the master has no SSH credentials for still gets a direct link:
+ * that is the cluster case, where nodes share a private network and the
+ * console never held a password. Failing instead would be refusing to do
+ * the thing that had always worked.
+ */
+export function createMeshDialer(options) {
+    const { logger, execution, sshTargetFor, subject } = options;
+    const probeTimeout = options.directProbeTimeoutMs ?? 2_000;
+    /** Per node, so one unreadable node does not cost every other one a read. */
+    const secrets = new Map();
+    return async (target) => {
+        const key = `${target.host}:${target.port}`;
+        const ssh = await sshTargetFor(target).catch(() => null);
+        const token = ssh ? await tokenFor(key, ssh) : undefined;
+        const onRejected = (reason) => {
+            secrets.delete(key);
+            logger.warn(
+            // The node's own words for the refusal. Without them this line names
+            // a conclusion and withholds the evidence for it, and the next
+            // question — WHY did it refuse — has nowhere to go.
+            { host: target.host, reason: reason ?? 'no reason given' }, 'Node refused the master credential — its signing secret will be read again on the next attempt');
+        };
+        if (await probeTcp(target.host, target.port, probeTimeout)) {
+            return { url: `tcp://${target.host}:${target.port}`, token, via: 'direct', onRejected };
+        }
+        if (!ssh) {
+            // Say which door was tried. "Connection refused" against a node that
+            // is plainly up sends an operator to the daemon; the answer is the
+            // firewall, or a credential this master does not hold.
+            throw new Error(`${key} is not reachable on its daemon port, and this master holds no SSH credentials for it`);
+        }
+        const tunnel = await execution.tunnel(ssh, target.port);
+        logger.info({ host: target.host, daemonPort: target.port, via: 'ssh-tunnel' }, 'Node daemon reached over SSH — its daemon port is not open to this master');
+        return {
+            url: `tcp://${tunnel.host}:${tunnel.port}`,
+            token,
+            via: 'ssh-tunnel',
+            close: tunnel.close,
+            onRejected,
+        };
+    };
+    async function tokenFor(key, ssh) {
+        const cached = secrets.get(key);
+        if (cached)
+            return mintServiceToken(cached, subject);
+        const result = await execution.ssh(ssh, READ_DAEMON_SECRET, { timeout: 20_000, quiet: true });
+        const secret = result.stdout.trim();
+        if (result.exitCode !== 0 || secret === '') {
+            // Not fatal: a node may legitimately have no auth configured, and the
+            // connection is still worth having for `ping` and the fleet view. What
+            // it cannot do is pull data, and the log has to say which of the two
+            // states this is.
+            logger.warn({ host: ssh.host, exitCode: result.exitCode }, 'Could not read the node daemon signing secret — the mesh link will connect unauthenticated and cannot replicate');
+            return undefined;
+        }
+        secrets.set(key, secret);
+        return mintServiceToken(secret, subject);
+    }
+}
+/**
+ * A stable uuid for a node the master has no registry entry for.
+ *
+ * The master stores replicated rows against `logs.nodeId`, which is a `uuid`
+ * column, and a slave identifies itself as `${hostname}-${port}`. Every
+ * ingest of every entry therefore failed with
+ *
+ *     invalid input syntax for type uuid: "daos-cpp-9700"
+ *
+ * — measured, on a real pull, once the transport and the credential were
+ * working and there was finally something for this to be wrong about. The
+ * entries were left unacknowledged and stayed on the node, which is the one
+ * part of this that behaved.
+ *
+ * A node the console registered has a `randomUUID()` of its own, and that is
+ * what the master should label its data with: it is the id the node list,
+ * the charts and the log filters all join on. A node that arrived through a
+ * STACK has no such row — the stack config carries a host and a port — and
+ * for those this derives an id from the address instead. Deterministic, so
+ * the same node lands in the same series across restarts; RFC 4122 v5, so
+ * it is a real uuid rather than something shaped like one.
+ *
+ * It will not match a registry row, and that is honest: no such row exists.
+ */
+const NODE_NAMESPACE = 'b4d4a3c1-8f2e-4c77-9a1f-0e6d5b2c7a30';
+/**
+ * RFC 4122 name-based uuid, version 5.
+ *
+ * Exported with the namespace as a parameter so a test can run it against
+ * the published vector — `v5("www.example.com", DNS)` is
+ * `2ed6657d-e927-568b-95e1-2665a8aea6a2` — rather than against a second copy
+ * of the same arithmetic written in the test, which would agree with itself
+ * whatever either of them did.
+ */
+export function uuidV5(namespace, name) {
+    const ns = Buffer.from(namespace.replace(/-/g, ''), 'hex');
+    const bytes = Buffer.from(createHash('sha1').update(Buffer.concat([ns, Buffer.from(name, 'utf8')])).digest().subarray(0, 16));
+    bytes[6] = (bytes[6] & 0x0f) | 0x50; // version 5
+    bytes[8] = (bytes[8] & 0x3f) | 0x80; // RFC 4122 variant
+    const hex = bytes.toString('hex');
+    return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
+}
+export function stableNodeUuid(host, port) {
+    return uuidV5(NODE_NAMESPACE, `${host}:${port}`);
+}
+//# sourceMappingURL=mesh-link.js.map

package/dist/cluster/mesh-link.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mesh-link.js","sourceRoot":"","sources":["../../src/cluster/mesh-link.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,GAAG,MAAM,UAAU,CAAC;AAC3B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAsCzC,iDAAiD;AACjD,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,MAAkB,EAAqB,EAAE,CAAC,CAAC;IAC1E,GAAG,EAAE,SAAS,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE;IAC1C,GAAG,EAAE,QAAQ;CACd,CAAC,CAAC;AAEH;;;;;;;;GAQG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,IAAY,EAAE,SAAiB;IACpE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAChC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,IAAI,GAAG,CAAC,SAAkB,EAAE,EAAE;YAClC,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO,CAAC,SAAS,CAAC,CAAC;QACrB,CAAC,CAAC;QACF,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAC7B,sEAAsE;IACtE,uDAAuD,CAAC;AAE1D;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAc,EACd,OAAe,EACf,GAAG,GAAG,IAAI;IAEV,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;IACzC,OAAO,IAAI,OAAO,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC;SACzC,kBAAkB,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;SACpC,UAAU,CAAC,OAAO,CAAC;SACnB,SAAS,CAAC,UAAU,CAAC;SACrB,WAAW,EAAE;SACb,iBAAiB,CAAC,GAAG,CAAC;SACtB,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;AAC5C,CAAC;AAaD;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA0B;IACzD,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAC7D,MAAM,YAAY,GAAG,OAAO,CAAC,oBAAoB,IAAI,KAAK,CAAC;IAC3D,6EAA6E;IAC7E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE1C,OAAO,KAAK,EAAE,MAAkB,EAAqB,EAAE;QACrD,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAEzD,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACzD,MAAM,UAAU,GAAG,CAAC,MAAe,EAAE,EAAE;YACrC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACpB,MAAM,CAAC,IAAI;YACT,qEAAqE;YACrE,+DAA+D;YAC/D,oDAAoD;YACpD,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,IAAI,iBAAiB,EAAE,EAC1D,gGAAgG,CACjG,CAAC;QACJ,CAAC,CAAC;QAEF,IAAI,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,EAAE,CAAC;YAC3D,OAAO,EAAE,GAAG,EAAE,SAAS,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC;QAC1F,CAAC;QAED,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,qEAAqE;YACrE,mEAAmE;YACnE,uDAAuD;YACvD,MAAM,IAAI,KAAK,CACb,GAAG,GAAG,uFAAuF,CAC9F,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QACxD,MAAM,CAAC,IAAI,CACT,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,YAAY,EAAE,EACjE,2EAA2E,CAC5E,CAAC;QACF,OAAO;YACL,GAAG,EAAE,SAAS,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE;YAC1C,KAAK;YACL,GAAG,EAAE,YAAY;YACjB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,UAAU;SACX,CAAC;IACJ,CAAC,CAAC;IAEF,KAAK,UAAU,QAAQ,CAAC,GAAW,EAAE,GAAc;QACjD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,MAAM;YAAE,OAAO,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAErD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,kBAAkB,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9F,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAC3C,sEAAsE;YACtE,uEAAuE;YACvE,qEAAqE;YACrE,kBAAkB;YAClB,MAAM,CAAC,IAAI,CACT,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,EAC7C,iHAAiH,CAClH,CAAC;YACF,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACzB,OAAO,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAE9D;;;;;;;;GAQG;AACH,MAAM,UAAU,MAAM,CAAC,SAAiB,EAAE,IAAY;IACpD,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CACvB,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CACnG,CAAC;IACF,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,YAAY;IAClD,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,mBAAmB;IAEzD,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAClC,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;AAC7G,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,IAAY;IACvD,OAAO,MAAM,CAAC,cAAc,EAAE,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;AACnD,CAAC"}

package/dist/cluster/mesh.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Join every registered node to the mesh, and keep it joined.
+ *
+ * `SlaveConnector` maintains master→node connections and pulls each node's
+ * replication buffer on connect and on every heartbeat. All of it worked.
+ * The only thing that ever called `addSlave` was a remote or cluster STACK
+ * starting, so the set of nodes the master connected to was "the ones
+ * something was deployed onto", not "the ones this master has".
+ *
+ * A node added through the console, provisioned, running its own daemon and
+ * collecting its own metrics and logs, was therefore never dialled. It
+ * buffers locally by design when it has no master — that is the correct
+ * behaviour for a node that has lost one — and nothing distinguishes that
+ * from a master that was never going to call. Measured on the first such
+ * node: 47,407 entries in the write-ahead buffer, none delivered, over
+ * eleven hours, with the node reporting healthy throughout.
+ *
+ * Membership follows the registry, because the registry is what an operator
+ * edits. Added is joined, removed is dropped; whether a node is currently
+ * reachable is the connector's problem, and it already has backoff for it.
+ */
+import type { ILogger } from '@omnitron-dev/titan/module/logger';
+import type { SlaveConnector } from './slave-connector.js';
+/** A registered node, in the terms the mesh needs. */
+export interface MeshNode {
+    id: string;
+    name: string;
+    host: string;
+    daemonPort: number;
+    isLocal: boolean;
+}
+/** The part of the node registry this needs — an EventEmitter with a list. */
+export interface MeshRegistry {
+    listNodes(): MeshNode[];
+    on(event: 'node:added' | 'node:updated', listener: (node: MeshNode) => void): unknown;
+    on(event: 'node:removed', listener: (id: string) => void): unknown;
+}
+export interface MeshHandle {
+    /** Stop following the registry. Does not disconnect — the connector owns that. */
+    stop(): void;
+}
+/**
+ * The local machine is not a peer of itself.
+ *
+ * The registry contains one entry for the machine the master runs on, so
+ * that the console can show it beside the others. Dialling it would have the
+ * master connect to its own daemon and pull its own buffer into itself —
+ * every entry ingested with its own node id, which is how one machine's
+ * readings become two nodes on a chart.
+ */
+export declare function meshMembers<T extends {
+    isLocal: boolean;
+}>(nodes: T[]): T[];
+export declare function startMesh(options: {
+    registry: MeshRegistry;
+    connector: SlaveConnector;
+    logger: ILogger;
+}): MeshHandle;
+//# sourceMappingURL=mesh.d.ts.map

package/dist/cluster/mesh.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mesh.d.ts","sourceRoot":"","sources":["../../src/cluster/mesh.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE3D,sDAAsD;AACtD,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,8EAA8E;AAC9E,MAAM,WAAW,YAAY;IAC3B,SAAS,IAAI,QAAQ,EAAE,CAAC;IACxB,EAAE,CAAC,KAAK,EAAE,YAAY,GAAG,cAAc,EAAE,QAAQ,EAAE,CAAC,IAAI,EAAE,QAAQ,KAAK,IAAI,GAAG,OAAO,CAAC;IACtF,EAAE,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC;CACpE;AAED,MAAM,WAAW,UAAU;IACzB,kFAAkF;IAClF,IAAI,IAAI,IAAI,CAAC;CACd;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,CAAC,SAAS;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAE3E;AAED,wBAAgB,SAAS,CAAC,OAAO,EAAE;IACjC,QAAQ,EAAE,YAAY,CAAC;IACvB,SAAS,EAAE,cAAc,CAAC;IAC1B,MAAM,EAAE,OAAO,CAAC;CACjB,GAAG,UAAU,CA2Fb"}

package/dist/cluster/mesh.js

@@ -0,0 +1,117 @@
+/**
+ * Join every registered node to the mesh, and keep it joined.
+ *
+ * `SlaveConnector` maintains master→node connections and pulls each node's
+ * replication buffer on connect and on every heartbeat. All of it worked.
+ * The only thing that ever called `addSlave` was a remote or cluster STACK
+ * starting, so the set of nodes the master connected to was "the ones
+ * something was deployed onto", not "the ones this master has".
+ *
+ * A node added through the console, provisioned, running its own daemon and
+ * collecting its own metrics and logs, was therefore never dialled. It
+ * buffers locally by design when it has no master — that is the correct
+ * behaviour for a node that has lost one — and nothing distinguishes that
+ * from a master that was never going to call. Measured on the first such
+ * node: 47,407 entries in the write-ahead buffer, none delivered, over
+ * eleven hours, with the node reporting healthy throughout.
+ *
+ * Membership follows the registry, because the registry is what an operator
+ * edits. Added is joined, removed is dropped; whether a node is currently
+ * reachable is the connector's problem, and it already has backoff for it.
+ */
+/**
+ * The local machine is not a peer of itself.
+ *
+ * The registry contains one entry for the machine the master runs on, so
+ * that the console can show it beside the others. Dialling it would have the
+ * master connect to its own daemon and pull its own buffer into itself —
+ * every entry ingested with its own node id, which is how one machine's
+ * readings become two nodes on a chart.
+ */
+export function meshMembers(nodes) {
+    return nodes.filter((node) => !node.isLocal);
+}
+export function startMesh(options) {
+    const { registry, connector, logger } = options;
+    let stopped = false;
+    /**
+     * Node id → the address it was joined at.
+     *
+     * `node:removed` carries the id and fires AFTER the row is deleted, so
+     * asking the registry where that node lived answers nothing. The connector
+     * is keyed on host:port, so the address has to be remembered while it
+     * still exists or a removed node keeps its connection — and its SSH
+     * tunnel — for the life of the daemon.
+     */
+    const joined = new Map();
+    const join = (node) => {
+        if (stopped || node.isLocal)
+            return;
+        joined.set(node.id, { host: node.host, port: node.daemonPort });
+        void connector
+            .addSlave({ host: node.host, port: node.daemonPort, label: node.name, nodeId: node.id })
+            .catch((err) => logger.warn({ host: node.host, port: node.daemonPort, error: err.message }, 'Could not add a registered node to the mesh'));
+    };
+    const drop = (id) => {
+        const at = joined.get(id);
+        if (!at)
+            return;
+        joined.delete(id);
+        // One connection per ADDRESS, and the registry can hold two rows for one
+        // machine — it does here, which is how a single node once drew two
+        // health checks an hour apart on the same socket. The connector is keyed
+        // on host:port, so removing the second row would disconnect the
+        // connection the first row still needs, and nothing would re-add it:
+        // the remaining node would sit in the mesh's own map, joined, with no
+        // connection behind it.
+        const stillWanted = [...joined.values()].some((other) => other.host === at.host && other.port === at.port);
+        if (stillWanted) {
+            logger.debug({ host: at.host, port: at.port }, 'Node removed, but another registry entry still points at this address');
+            return;
+        }
+        void connector.removeSlave(at.host, at.port).catch(() => undefined);
+    };
+    const known = meshMembers(registry.listNodes());
+    for (const node of known)
+        join(node);
+    logger.info({ nodes: known.length }, 'Mesh following the node registry');
+    const onAdded = (node) => join(node);
+    /**
+     * An edited address is a different node to the connector.
+     *
+     * Editing a node's host or daemon port in the console is how an operator
+     * corrects a mistake or follows a machine that moved. Without this the
+     * master keeps dialling the old address forever and the corrected one is
+     * never tried — the edit appears to do nothing, which is the worst way for
+     * a correction to fail.
+     */
+    const onUpdated = (node) => {
+        if (stopped)
+            return;
+        const at = joined.get(node.id);
+        const moved = !at || at.host !== node.host || at.port !== node.daemonPort;
+        if (!moved)
+            return;
+        if (at)
+            drop(node.id);
+        join(node);
+    };
+    const onRemoved = (id) => {
+        if (stopped)
+            return;
+        drop(id);
+    };
+    registry.on('node:added', onAdded);
+    registry.on('node:updated', onUpdated);
+    registry.on('node:removed', onRemoved);
+    return {
+        stop() {
+            stopped = true;
+            const emitter = registry;
+            emitter.off?.('node:added', onAdded);
+            emitter.off?.('node:updated', onUpdated);
+            emitter.off?.('node:removed', onRemoved);
+        },
+    };
+}
+//# sourceMappingURL=mesh.js.map

package/dist/cluster/mesh.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mesh.js","sourceRoot":"","sources":["../../src/cluster/mesh.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AA0BH;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAiC,KAAU;IACpE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,OAIzB;IACC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAChD,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB;;;;;;;;OAQG;IACH,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0C,CAAC;IAEjE,MAAM,IAAI,GAAG,CAAC,IAAc,EAAE,EAAE;QAC9B,IAAI,OAAO,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACpC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAChE,KAAK,SAAS;aACX,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;aACvF,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE,CACtB,MAAM,CAAC,IAAI,CACT,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAG,GAAa,CAAC,OAAO,EAAE,EACzE,6CAA6C,CAC9C,CACF,CAAC;IACN,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,CAAC,EAAU,EAAE,EAAE;QAC1B,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,EAAE;YAAE,OAAO;QAChB,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAElB,yEAAyE;QACzE,mEAAmE;QACnE,yEAAyE;QACzE,gEAAgE;QAChE,qEAAqE;QACrE,sEAAsE;QACtE,wBAAwB;QACxB,MAAM,WAAW,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC;QAC3G,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,uEAAuE,CAAC,CAAC;YACxH,OAAO;QACT,CAAC;QAED,KAAK,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACtE,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;IAChD,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,kCAAkC,CAAC,CAAC;IAEzE,MAAM,OAAO,GAAG,CAAC,IAAc,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE/C;;;;;;;;OAQG;IACH,MAAM,SAAS,GAAG,CAAC,IAAc,EAAE,EAAE;QACnC,IAAI,OAAO;YAAE,OAAO;QACpB,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,UAAU,CAAC;QAC1E,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,IAAI,EAAE;YAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,EAAU,EAAE,EAAE;QAC/B,IAAI,OAAO;YAAE,OAAO;QACpB,IAAI,CAAC,EAAE,CAAC,CAAC;IACX,CAAC,CAAC;IAEF,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACnC,QAAQ,CAAC,EAAE,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;IACvC,QAAQ,CAAC,EAAE,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;IAEvC,OAAO;QACL,IAAI;YACF,OAAO,GAAG,IAAI,CAAC;YACf,MAAM,OAAO,GAAG,QAA2E,CAAC;YAC5F,OAAO,CAAC,GAAG,EAAE,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACrC,OAAO,CAAC,GAAG,EAAE,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YACzC,OAAO,CAAC,GAAG,EAAE,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;QAC3C,CAAC;KACF,CAAC;AACJ,CAAC"}