@omnitron-dev/omnitron 0.3.1 → 0.3.2

package/dist/cluster/slave-connector.js

@@ -7,7 +7,7 @@
  * - Auto-connect on stack start
  * - Exponential backoff reconnection on disconnect
  * - Heartbeat monitoring (detect slave failures)
- * - Sync wiring (push sync invoke to SyncService)
+ * - Authenticated replication pulls into SyncService
  * - Fleet node status updates
  *
  * Each slave runs its own omnitron daemon on the remote machine.
@@ -19,10 +19,12 @@
  *   on disconnect → backoff → reconnect()
  *   on stack stop → disconnect() cleanup
  */
-import { Netron } from '@omnitron-dev/titan/netron';
+import { Netron, NETRON_EVENT_PEER_DISCONNECT } from '@omnitron-dev/titan/netron';
 import { TcpTransport } from '@omnitron-dev/titan/netron/transport/tcp';
 import { createNullLogger } from '@omnitron-dev/titan/module/logger';
-import { directLink, stableNodeUuid } from './mesh-link.js';
+import { LONG_REQUEST_TIMEOUT } from '../shared/dto/services.js';
+import { sweepMadeProgress } from '../services/sync-policy.js';
+import { assertMeshCredentialEndpoint, directLink, stableNodeUuid } from './mesh-link.js';
 /**
  * Present the master's credential, and tell a refusal from a broken line.
  *
@@ -41,6 +43,9 @@ import { directLink, stableNodeUuid } from './mesh-link.js';
  * looking at authentication, which is the one thing that was working.
  */
 export async function authenticatePeer(peer, link) {
+    assertMeshCredentialEndpoint(link.url);
+    if (!link.token)
+        throw new Error('Missing mesh service credential');
     const runTask = peer.runTask;
     if (typeof runTask !== 'function') {
         throw new Error(`cannot present a credential over ${link.url} — this transport has no authenticate task`);
@@ -50,16 +55,34 @@ export async function authenticatePeer(peer, link) {
         link.onRejected?.(auth?.error);
         throw new Error(`node refused the master's credential: ${auth?.error ?? 'no reason given'}`);
     }
+    const context = auth.context;
+    if (!Array.isArray(context?.roles) || context.roles.length !== 1 || context.roles[0] !== 'service_role' ||
+        context.metadata?.['principalType'] !== 'service') {
+        throw new Error('Node did not confirm a service_role principal; verify the registered daemon and its version');
+    }
+    const expiresAt = context.metadata['expiresAt'];
+    if (typeof expiresAt !== 'number' || !Number.isFinite(expiresAt)) {
+        throw new Error('Node did not confirm a finite service credential expiry; verify the registered daemon and its version');
+    }
+    link.expiresAt = Math.min(link.expiresAt ?? expiresAt, expiresAt);
+    if (!Number.isFinite(link.expiresAt) || link.expiresAt <= Date.now()) {
+        throw new Error('Node accepted a service credential that is already expired');
+    }
+}
+/** A failed resource must not prevent the next connection from being opened. */
+async function bounded(work, ms, message) {
+    let timer;
+    try {
+        return await Promise.race([work, new Promise((_, reject) => {
+                timer = setTimeout(() => reject(new Error(message)), ms);
+                timer.unref();
+            })]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
 }
-/**
- * How long a call to a node may take.
- *
- * Ten minutes, because the operations a master asks of a node are
- * provisioning ones: pulling images, waiting on health checks, installing a
- * bundle. A caller that needs a shorter bound imposes it itself — the
- * heartbeat does.
- */
-const SLAVE_REQUEST_TIMEOUT = 10 * 60_000;
 /**
  * Whether an error means the connection is gone rather than the call failed.
  *
@@ -83,6 +106,7 @@ export function isConnectionGone(err) {
     const message = err?.message ?? String(err);
     return /socket closed|not connected|ECONNRESET|EPIPE|connection closed|socket is not open|Peer .* disconnected/i.test(message);
 }
+const LIVENESS_ONLY = 'Only OmnitronDaemon.ping is available without a verified service credential. Configure SSH access and daemon authentication for the registered node.';
 export class SlaveConnector {
     logger;
     syncService;
@@ -91,6 +115,8 @@ export class SlaveConnector {
     /** Said once per master, not once per node: it is a property of this daemon. */
     noSyncServiceReported = false;
     disposed = false;
+    heartbeatRunning = false;
+    cleanupTimeout;
     /** Heartbeat interval — how often we ping each slave (ms) */
     heartbeatInterval;
     /** Max reconnect backoff (ms) */
@@ -110,6 +136,7 @@ export class SlaveConnector {
         this.heartbeatInterval = options?.heartbeatInterval ?? 15_000;
         this.maxBackoff = options?.maxBackoff ?? 120_000;
         this.dial = options?.dial ?? directLink;
+        this.cleanupTimeout = options?.cleanupTimeout ?? 5_000;
     }
     // ===========================================================================
     // Connection Management
@@ -119,12 +146,20 @@ export class SlaveConnector {
      */
     async addSlave(config) {
         const key = `${config.host}:${config.port}`;
-        if (this.connections.has(key)) {
-            // Debug, not warn: the node registry re-asserts its members whenever a
-            // health check sees one, so "already registered" is the normal case
-            // and a warning per node per check is noise that hides the real ones.
-            this.logger.debug({ host: config.host, port: config.port }, 'Node already in the mesh');
-            return;
+        const previous = this.connections.get(key);
+        let replacing;
+        if (previous) {
+            if (!config.nodeId || config.nodeId === previous.config.nodeId) {
+                this.logger.debug({ host: config.host, port: config.port }, 'Node already in the mesh');
+                return;
+            }
+            if (previous.config.nodeId) {
+                throw new Error(`Conflicting registered node identities for ${key}. Resolve duplicate registrations before joining the mesh.`);
+            }
+            // A stack can discover this endpoint before the registry does. Revoke
+            // that transport before publishing a connection with a different identity.
+            replacing = this.removeSlave(config.host, config.port);
+            config = { ...previous.config, ...config };
         }
         const conn = {
             config,
@@ -143,8 +178,13 @@ export class SlaveConnector {
         };
         this.connections.set(key, conn);
         this.logger.info({ host: config.host, port: config.port, stack: config.stack }, 'Slave registered');
-        // Start connection attempt
-        this.connectSlave(key, conn);
+        // Publish the replacement before awaiting cleanup, so removal or disposal
+        // during a slow close cannot resurrect a revoked registration.
+        if (replacing)
+            await replacing;
+        if (this.disposed || this.connections.get(key) !== conn)
+            return;
+        void this.connectSlave(key, conn);
         // Start heartbeat loop if not running
         if (!this.heartbeatTimer) {
             this.startHeartbeatLoop();
@@ -204,8 +244,9 @@ export class SlaveConnector {
         const conn = this.connections.get(key);
         if (!conn)
             return;
-        await this.disconnectSlave(conn);
+        conn.removed = true;
         this.connections.delete(key);
+        await this.disconnectSlave(conn);
         this.logger.info({ host, port }, 'Slave removed');
     }
     /**
@@ -238,8 +279,19 @@ export class SlaveConnector {
             lastHeartbeat: conn.lastHeartbeat,
             lastError: conn.lastError,
             via: conn.link?.via ?? null,
-            authenticated: Boolean(conn.link?.token),
+            authenticated: conn.status === 'connected' && conn.authenticated === true &&
+                Boolean(conn.link?.token && conn.link.expiresAt && conn.link.expiresAt > Date.now()),
+            credentialExpiresAt: conn.link?.expiresAt ?? null,
+            credentialRefreshedAt: conn.credentialRefreshedAt ?? null,
+            lastCredentialError: conn.lastCredentialError ?? null,
             syncStatus: conn.syncStatus,
+            syncStatusAt: conn.syncStatusAt ?? null,
+            lastSyncStatusError: conn.lastSyncStatusError,
+            lastPullError: conn.lastPullError,
+            lastPullAt: conn.lastPullAt ?? null,
+            reconnectAt: conn.reconnectAt ?? null,
+            reconnectAttempt: conn.reconnectAttempt,
+            cleanupError: conn.cleanupError ?? null,
         }));
     }
     /**
@@ -247,14 +299,18 @@ export class SlaveConnector {
      */
     async invokeOnSlave(host, port, service, method, args, options) {
         try {
-            return await this.callOnSlave(host, port, service, method, args);
+            return await this.callOnSlave(host, port, service, method, args, options?.requireCredential);
         }
         catch (err) {
             if (!options?.retryOnDisconnect || !isConnectionGone(err))
                 throw err;
             this.logger.info({ host, port, service, method, error: err.message }, 'The mesh connection was already gone — reconnecting and trying once more');
-            await this.removeSlave(host, port);
-            await this.addSlave({ host, port });
+            const key = `${host}:${port}`;
+            const conn = this.connections.get(key);
+            if (conn)
+                await this.connectionLost(key, conn, err);
+            else
+                await this.addSlave({ host, port });
             if (!(await this.waitUntilConnected(host, port, 60_000))) {
                 // The symptom is «did not come back»; the diagnosis is in what
                 // dropped it, and without the cause the second failure hides the
@@ -263,19 +319,31 @@ export class SlaveConnector {
                     cause: err,
                 });
             }
-            return await this.callOnSlave(host, port, service, method, args);
+            return await this.callOnSlave(host, port, service, method, args, options?.requireCredential);
         }
     }
-    async callOnSlave(host, port, service, method, args) {
+    async callOnSlave(host, port, service, method, args, requireCredential = false) {
         const key = `${host}:${port}`;
         const conn = this.connections.get(key);
-        if (!conn?.peer)
-            throw new Error(`Slave ${key} not connected`);
-        const proxy = await conn.peer.queryInterface(service);
-        const fn = proxy[method];
-        if (typeof fn !== 'function')
-            throw new Error(`Method ${method} not found on service ${service}`);
-        return fn.call(proxy, ...args);
+        if (conn?.status !== 'connected' || !conn.peer) {
+            throw new Error(`Slave ${key} not connected${conn?.lastError ? `: ${conn.lastError}` : ''}`);
+        }
+        const peer = conn.peer;
+        try {
+            const publicPing = !requireCredential && service === 'OmnitronDaemon' && method === 'ping' && args.length === 0;
+            await this.ensureCredential(conn, peer, publicPing);
+            const proxy = await peer.queryInterface(service);
+            await this.ensureCredential(conn, peer, publicPing);
+            const fn = proxy[method];
+            if (typeof fn !== 'function')
+                throw new Error(`Method ${method} not found on service ${service}`);
+            return await fn.call(proxy, ...args);
+        }
+        catch (error) {
+            if (conn.peer === peer && isConnectionGone(error))
+                void this.connectionLost(key, conn, error);
+            throw error;
+        }
     }
     /**
      * Broadcast an RPC method to all connected slaves.
@@ -300,145 +368,159 @@ export class SlaveConnector {
     // Private — Connection Lifecycle
     // ===========================================================================
     async connectSlave(key, conn) {
-        if (this.disposed || conn.status === 'connected')
+        if (this.disposed || conn.removed || conn.status === 'connected' || conn.status === 'connecting')
             return;
         conn.status = 'connecting';
-        // Anything left from a previous attempt goes back before a new one is
-        // made. The disconnect path releases it too; this is the guarantee that
-        // does not depend on which path got here.
-        await this.releaseLink(conn);
+        conn.authenticated = false;
+        const generation = (conn.generation ?? 0) + 1;
+        conn.generation = generation;
+        conn.reconnectAt = null;
+        const current = () => !this.disposed && !conn.removed && conn.generation === generation;
+        let link = null;
+        let netron = null;
         try {
-            const link = await this.dial({ host: conn.config.host, port: conn.config.port });
+            if (conn.cleanup)
+                await conn.cleanup;
+            if (!current())
+                return;
+            link = await this.dial({ host: conn.config.host, port: conn.config.port, ...(conn.config.nodeId && { nodeId: conn.config.nodeId }) });
+            if (!current()) {
+                await this.closeResources(null, link);
+                return;
+            }
             conn.link = link;
-            const netron = new Netron(createNullLogger(), { id: `master-to-${key}` });
+            netron = new Netron(createNullLogger(), { id: `master-to-${key}` });
+            conn.netron = netron;
             netron.registerTransport('tcp', () => new TcpTransport());
-            // The calls this connection carries ARE the work: a container set
-            // coming up, an image being pulled, a bundle being installed. netron's
-            // default is five seconds, and the failure it produces names a
-            // timeout while the node goes on doing exactly what it was asked:
-            //
-            //     Could not bring up this node's infrastructure:
-            //       RPC request timed out after 5000ms
-            //
-            // measured against a node that provisioned it successfully.
-            //
-            // The one call that must fail fast is the heartbeat, and it is bounded
-            // by its own race rather than by this — which is why raising this does
-            // not make a dead node look alive.
-            netron.setTransportOptions('tcp', { requestTimeout: SLAVE_REQUEST_TIMEOUT });
+            netron.setTransportOptions('tcp', { requestTimeout: LONG_REQUEST_TIMEOUT });
+            const ownedNetron = netron;
+            netron.on(NETRON_EVENT_PEER_DISCONNECT, () => {
+                if (conn.netron !== ownedNetron || conn.status !== 'connected')
+                    return;
+                void this.connectionLost(key, conn, new Error('RPC transport disconnected'));
+            });
             const peer = await netron.connect(link.url, false);
-            // Present the credential before asking for anything that needs it.
-            //
-            // `ping` is reachable without one, which is why this connection looked
-            // healthy while every data call behind it answered `Authentication
-            // required` — the connector's only failure signal was a debug line in
-            // the pull, and a slave with nothing to say looks the same as one that
-            // is refusing.
-            if (link.token) {
-                const runTask = peer
-                    .runTask;
-                if (typeof runTask !== 'function') {
-                    throw new Error(`cannot present a credential over ${link.url} — this transport has no authenticate task`);
+            if (!current()) {
+                await this.closeResources(netron, link);
+                return;
+            }
+            const connectedLink = link;
+            const pingResult = await bounded((async () => {
+                if (connectedLink.token) {
+                    try {
+                        await authenticatePeer(peer, connectedLink);
+                    }
+                    catch (error) {
+                        if (current())
+                            conn.lastCredentialError = error instanceof Error ? error.message : String(error);
+                        throw error;
+                    }
                 }
-                await authenticatePeer(peer, link);
+                const daemon = await peer.queryInterface('OmnitronDaemon');
+                return daemon.ping();
+            })(), 10_000, 'RPC authentication or initial heartbeat timed out');
+            if (!current()) {
+                await this.closeResources(netron, link);
+                return;
             }
-            // Verify slave is alive
-            const daemon = await peer.queryInterface('OmnitronDaemon');
-            const pingResult = await daemon.ping();
-            conn.netron = netron;
             conn.peer = peer;
             conn.status = 'connected';
+            conn.authenticated = Boolean(connectedLink.token);
+            conn.credentialRefreshedAt = conn.authenticated ? Date.now() : null;
+            conn.lastCredentialError = conn.authenticated ? null : LIVENESS_ONLY;
             conn.lastHeartbeat = Date.now();
             conn.lastError = null;
+            conn.cleanupError = null;
             conn.reconnectAttempt = 0;
-            this.logger.info({
-                host: conn.config.host,
-                port: conn.config.port,
-                slaveVersion: pingResult?.version,
-                via: link.via,
-                authenticated: Boolean(link.token),
-            }, 'Node joined the mesh');
-            // Ask what it is holding, then start pulling it. Both in that order
-            // and neither waiting on the other: without the first call here a node
-            // that just joined shows an empty «Sync» column until the sweep fifteen
-            // seconds later, which is exactly when somebody is looking, and putting
-            // it after the drain would queue it behind up to 10 000 entries.
+            this.logger.info({ host: conn.config.host, port: conn.config.port,
+                slaveVersion: pingResult?.version, via: link.via, authenticated: conn.authenticated }, 'Node joined the mesh');
             void this.refreshSyncStatus(conn);
             void this.pullSyncData(key, conn);
-            // Monitor for disconnection
-            netron.on('peer:disconnected', () => {
-                if (this.disposed)
-                    return;
-                this.logger.warn({ host: conn.config.host, port: conn.config.port }, 'Slave disconnected');
-                // Give back the netron and the link BEFORE reconnecting.
-                //
-                // This dropped the peer and kept both, and `connectSlave` overwrites
-                // them with the new ones — so every reconnect leaked a Netron and,
-                // for a node reached over SSH, a tunnel holding an open SSH session.
-                // The nodes that reconnect are the ones on a poor link, so the leak
-                // is fastest exactly where it is worst, and it ends at sshd's
-                // session limit: the master locked out of the node entirely,
-                // including the path it deploys over.
-                void this.disconnectSlave(conn).finally(() => this.scheduleReconnect(key, conn));
-            });
         }
-        catch (err) {
-            conn.status = 'error';
-            conn.lastError = err.message;
-            conn.netron = null;
-            conn.peer = null;
-            // A tunnel outlives the netron that failed over it, and each one holds
-            // an SSH connection. Released here or they accumulate one per retry.
-            await this.releaseLink(conn);
-            this.logger.debug({ host: conn.config.host, port: conn.config.port, error: conn.lastError, attempt: conn.reconnectAttempt }, 'Slave connection failed');
-            this.scheduleReconnect(key, conn);
+        catch (error) {
+            if (current())
+                await this.connectionLost(key, conn, error);
+            else
+                await this.closeResources(netron, link);
+        }
+    }
+    async connectionLost(key, conn, error) {
+        if (this.disposed || conn.removed)
+            return;
+        conn.lastError = error instanceof Error ? error.message : String(error);
+        if (conn.cleanup)
+            await conn.cleanup;
+        else
+            await this.disconnectSlave(conn);
+        if (this.disposed || conn.removed || conn.status === 'connected' || conn.status === 'connecting')
+            return;
+        conn.status = 'error';
+        this.scheduleReconnect(key, conn);
+    }
+    async closeResources(netron, link) {
+        try {
+            // Release both independently: neither a stuck peer nor a half-open SSH
+            // socket may prevent closing the other, or scheduling the next attempt.
+            const results = await bounded(Promise.allSettled([
+                Promise.resolve().then(() => netron?.stop()),
+                Promise.resolve().then(() => link?.close?.()),
+            ]), this.cleanupTimeout, 'Connection cleanup timed out; retry will use a new connection');
+            return results.filter((r) => r.status === 'rejected')
+                .map(r => r.reason instanceof Error ? r.reason.message : String(r.reason)).join('; ') || null;
+        }
+        catch (error) {
+            return error instanceof Error ? error.message : String(error);
         }
     }
     async disconnectSlave(conn) {
-        if (conn.reconnectTimer) {
+        if (conn.reconnectTimer)
             clearTimeout(conn.reconnectTimer);
-            conn.reconnectTimer = null;
-        }
-        if (conn.netron) {
-            try {
-                await conn.netron.stop();
-            }
-            catch {
-                // Already stopped
-            }
-            conn.netron = null;
+        conn.reconnectTimer = null;
+        conn.reconnectAt = null;
+        if (conn.cleanup) {
+            await conn.cleanup;
+            return;
         }
-        await this.releaseLink(conn);
+        conn.generation = (conn.generation ?? 0) + 1;
+        const netron = conn.netron;
+        const link = conn.link;
+        // Revoke ownership before closing. A late callback belongs to these old
+        // resources and cannot disconnect or overwrite the replacement connection.
+        conn.netron = null;
+        conn.link = null;
         conn.peer = null;
+        conn.authenticated = false;
+        conn.credentialRefresh = null;
         conn.status = 'disconnected';
-        // What it last told us about its buffer is no longer something we know.
-        // Keeping the last figure would report a stale «0 pending» about a node
-        // that has been silent for hours, which is the exact reading this whole
-        // path exists to prevent.
         conn.syncStatus = null;
-    }
-    /** Give back whatever was opened to make this connection possible. */
-    async releaseLink(conn) {
-        const close = conn.link?.close;
-        conn.link = null;
-        if (!close)
-            return;
+        conn.syncStatusAt = null;
+        const cleanup = this.closeResources(netron, link).then(error => {
+            conn.cleanupError = error;
+            if (error)
+                this.logger.warn({ host: conn.config.host, port: conn.config.port, error }, 'Mesh resource cleanup failed; reconnect remains enabled');
+        });
+        conn.cleanup = cleanup;
         try {
-            await close();
+            await cleanup;
         }
-        catch (err) {
-            this.logger.debug({ host: conn.config.host, error: err.message }, 'Releasing the mesh link failed');
+        finally {
+            if (conn.cleanup === cleanup)
+                conn.cleanup = null;
         }
     }
     scheduleReconnect(key, conn) {
-        if (this.disposed)
+        if (this.disposed || conn.removed)
             return;
         if (conn.reconnectTimer)
             return;
         conn.reconnectAttempt++;
         const delay = Math.min(5000 * Math.pow(1.5, conn.reconnectAttempt - 1), this.maxBackoff);
+        conn.reconnectAt = Date.now() + delay;
+        this.logger.warn({ host: conn.config.host, port: conn.config.port, attempt: conn.reconnectAttempt,
+            retryInMs: delay, error: conn.lastError, cleanupError: conn.cleanupError ?? null }, 'Mesh disconnected; reconnect scheduled');
         conn.reconnectTimer = setTimeout(() => {
             conn.reconnectTimer = null;
+            conn.reconnectAt = null;
             if (!this.disposed) {
                 this.connectSlave(key, conn);
             }
@@ -449,6 +531,61 @@ export class SlaveConnector {
     // ===========================================================================
     // Private — Heartbeat
     // ===========================================================================
+    /** Coalesce renewal across calls and polling. Never replay a business invocation. */
+    async ensureCredential(conn, peer, publicPing = false) {
+        if (conn.peer !== peer || conn.status !== 'connected')
+            throw new Error('RPC connection changed before authentication');
+        const link = conn.link;
+        if (!link?.token || !conn.authenticated) {
+            if (publicPing)
+                return;
+            throw new Error(LIVENESS_ONLY);
+        }
+        const pending = conn.credentialRefresh;
+        if (pending && pending.peer === peer) {
+            await pending.promise;
+            return;
+        }
+        if (link.expiresAt === undefined || !Number.isFinite(link.expiresAt))
+            throw new Error('Missing service credential expiry');
+        if (link.expiresAt - Date.now() > 30_000)
+            return;
+        // A non-renewable caller may use its valid token until expiry; it must
+        // redial afterwards. Built-in mesh credentials always support renewal.
+        if (!link.refreshCredential && link.expiresAt > Date.now())
+            return;
+        const refresh = { peer, promise: undefined };
+        const current = () => conn.peer === peer && conn.link === link && conn.status === 'connected';
+        refresh.promise = bounded((async () => {
+            if (!link.refreshCredential)
+                throw new Error('Service credential expired; a new connection is required');
+            const credential = await link.refreshCredential();
+            if (!current())
+                throw new Error('RPC connection changed during credential renewal');
+            if (!credential.token || !Number.isFinite(credential.expiresAt) || credential.expiresAt <= Date.now()) {
+                throw new Error('Credential renewal returned an expired or invalid credential');
+            }
+            const renewed = { ...link, ...credential };
+            await authenticatePeer(peer, renewed);
+            if (!current())
+                throw new Error('RPC connection changed during authentication');
+            link.token = renewed.token;
+            link.expiresAt = renewed.expiresAt;
+            conn.credentialRefreshedAt = Date.now();
+            conn.lastCredentialError = null;
+        })(), 10_000, 'Service credential renewal timed out').catch(async (error) => {
+            if (current()) {
+                conn.lastCredentialError = error instanceof Error ? error.message : String(error);
+                await this.connectionLost(`${conn.config.host}:${conn.config.port}`, conn, error);
+            }
+            throw error;
+        }).finally(() => {
+            if (conn.credentialRefresh === refresh)
+                conn.credentialRefresh = null;
+        });
+        conn.credentialRefresh = refresh;
+        await refresh.promise;
+    }
     startHeartbeatLoop() {
         this.heartbeatTimer = setInterval(() => {
             this.heartbeatSweep().catch((err) => {
@@ -458,49 +595,34 @@ export class SlaveConnector {
         this.heartbeatTimer.unref();
     }
     async heartbeatSweep() {
-        for (const [key, conn] of this.connections) {
-            if (conn.status !== 'connected' || !conn.peer)
-                continue;
-            try {
-                const daemon = await conn.peer.queryInterface('OmnitronDaemon');
-                await Promise.race([
-                    daemon.ping(),
-                    new Promise((_, reject) => setTimeout(() => reject(new Error('Heartbeat timeout')), 10_000)),
-                ]);
-                // This connection's own record, and the only one this class keeps.
-                // It also wrote the node into the fleet table — the control plane's
-                // registry of daemons, where a managed machine has no row — so every
-                // one of those UPDATEs reached nothing. A machine's liveness has one
-                // writer: the health monitor, into the node registry.
-                conn.lastHeartbeat = Date.now();
-            }
-            catch (err) {
-                this.logger.warn({ host: conn.config.host, port: conn.config.port, error: err.message }, 'Heartbeat failed — marking slave as disconnected');
-                conn.status = 'disconnected';
-                conn.lastError = err.message;
-                // Force reconnect
-                await this.disconnectSlave(conn);
-                this.scheduleReconnect(key, conn);
-            }
+        if (this.heartbeatRunning || this.disposed)
+            return;
+        this.heartbeatRunning = true;
+        try {
+            await Promise.all([...this.connections].map(async ([key, conn]) => {
+                if (conn.status !== 'connected' || !conn.peer)
+                    return;
+                const peer = conn.peer;
+                try {
+                    await bounded((async () => {
+                        await this.ensureCredential(conn, peer, true);
+                        const daemon = await peer.queryInterface('OmnitronDaemon');
+                        await daemon.ping();
+                    })(), 10_000, 'Heartbeat timeout');
+                    if (conn.peer !== peer || conn.status !== 'connected')
+                        return;
+                    conn.lastHeartbeat = Date.now();
+                    void this.refreshSyncStatus(conn);
+                    void this.pullSyncData(key, conn);
+                }
+                catch (error) {
+                    if (conn.peer === peer)
+                        await this.connectionLost(key, conn, error);
+                }
+            }));
         }
-        // Pull sync data from all connected slaves during heartbeat
-        for (const [key, conn] of this.connections) {
-            if (conn.status === 'connected' && conn.peer) {
-                // The status is read BEFORE the drain, and independently of it.
-                //
-                // It used to be chained onto the drain's completion, on the reasoning
-                // that `pendingItems` should mean «what is still waiting» rather than
-                // «what was waiting before this sweep touched it». The stand refuted
-                // that within the hour: a node with a real backlog drains 10 000
-                // entries per sweep — the safety limit — and the first status did not
-                // arrive until 40 s after the master connected, because it was queued
-                // behind the very work it measures. A backlog large enough to matter
-                // is exactly a backlog large enough to delay the figure describing
-                // it, and a diagnostic that waits on the thing it diagnoses is not
-                // available when it is needed.
-                void this.refreshSyncStatus(conn);
-                void this.pullSyncData(key, conn);
-            }
+        finally {
+            this.heartbeatRunning = false;
         }
     }
     /**
@@ -520,15 +642,30 @@ export class SlaveConnector {
     async refreshSyncStatus(conn) {
         if (conn.status !== 'connected' || !conn.peer)
             return;
+        const peer = conn.peer;
         try {
-            const syncProxy = await conn.peer.queryInterface('OmnitronSync');
-            const status = (await syncProxy.getSyncStatus());
+            const status = await bounded((async () => {
+                await this.ensureCredential(conn, peer);
+                const syncProxy = await peer.queryInterface('OmnitronSync');
+                return await syncProxy.getSyncStatus();
+            })(), 10_000, 'Replication status read timed out');
+            if (conn.peer !== peer || conn.status !== 'connected')
+                return;
+            await this.ensureCredential(conn, peer);
+            if (conn.peer !== peer || conn.status !== 'connected')
+                return;
             conn.syncStatus = status ?? null;
+            conn.syncStatusAt = Date.now();
             // Cleared on success, so a refusal that comes BACK is reported again
             // rather than being swallowed as «already said that once».
             conn.lastSyncStatusError = null;
         }
         catch (err) {
+            if (conn.peer !== peer || conn.status !== 'connected')
+                return;
+            conn.syncStatusAt = null;
+            if (isConnectionGone(err))
+                void this.connectionLost(`${conn.config.host}:${conn.config.port}`, conn, err);
             // A node that cannot answer is a node we do not know about, not a node
             // with an empty buffer. The heartbeat above is what decides whether it
             // is still connected; this only decides what we claim to know.
@@ -573,7 +710,7 @@ export class SlaveConnector {
      * grew instead.
      *
      * The turn belongs to the PEER, not to the connection. A call to a node
-     * may wait out SLAVE_REQUEST_TIMEOUT — ten minutes — and a pull stuck on a
+     * may wait out the lifecycle timeout — ten minutes — and a pull stuck on a
      * peer that has died would otherwise hold the reconnected one off for that
      * long: the guard would become the outage. A reconnect replaces the peer,
      * and a pull on the new one may start while the old one is still stuck.
@@ -586,6 +723,7 @@ export class SlaveConnector {
         // not wired. Measured on the stand — 33 614 → 34 189 pending over 40 s
         // with not one line about it.
         if (!this.syncService) {
+            conn.lastPullError = 'Replication is unavailable: this master has no sync service';
             if (!this.noSyncServiceReported) {
                 this.noSyncServiceReported = true;
                 this.logger.warn({ host: conn.config.host, port: conn.config.port }, 'No sync service on this master — nothing will be pulled from any node');
@@ -605,6 +743,7 @@ export class SlaveConnector {
         const pull = { peer, startedAt: Date.now(), sweepsSkipped: 0 };
         conn.pull = pull;
         try {
+            await this.ensureCredential(conn, peer);
             const syncProxy = await peer.queryInterface('OmnitronSync');
             let totalPulled = 0;
             const seen = new Set();
@@ -616,12 +755,22 @@ export class SlaveConnector {
             // series; the master dialled this connection and knows what it dialled.
             const nodeId = conn.config.nodeId ?? stableNodeUuid(conn.config.host, conn.config.port);
             // Pull in batches until slave buffer is empty
-            while (true) {
+            while (conn.peer === peer && conn.status === 'connected') {
+                await this.ensureCredential(conn, peer);
                 const drained = await syncProxy.drainBuffer({ limit: 1000 });
+                await this.ensureCredential(conn, peer);
+                if (conn.peer !== peer || conn.status !== 'connected')
+                    return;
                 if (!drained || !drained.entries || drained.entries.length === 0)
                     break;
                 const batch = { ...drained, nodeId };
                 // Ingest into master, then release on the slave — in that order.
+                const ids = batch.entries.map((entry) => entry.id);
+                if (!sweepMadeProgress(seen, ids)) {
+                    throw new Error(`Replication stalled: ${ids.length} oldest entries remain pending after ingestion; inspect master sync errors`);
+                }
+                for (const id of ids)
+                    seen.add(id);
                 // `drainBuffer` used to mark entries synced before returning them,
                 // so anything that went wrong from here on lost the data on both
                 // sides while this loop logged the failure at debug level.
@@ -634,23 +783,23 @@ export class SlaveConnector {
                     ...(result.duplicateIds ?? []),
                     ...(result.discardedIds ?? []),
                 ];
-                if (delivered.length > 0)
-                    await syncProxy.ackDrained({ ids: delivered });
-                totalPulled += delivered.length;
-                // Entries the master rejected stay pending on the slave, so the next
-                // fetch returns them again. Without this the sweep would re-fetch and
-                // re-reject the same page until the safety limit.
-                const ids = batch.entries.map((e) => e.id);
-                if (!ids.some((id) => !seen.has(id))) {
-                    this.logger.warn({ host: conn.config.host, pending: ids.length }, 'Sync pull stalled — the master is rejecting the slave\'s oldest entries');
-                    break;
+                if (conn.peer !== peer || conn.status !== 'connected')
+                    return;
+                if (delivered.length > 0) {
+                    await this.ensureCredential(conn, peer);
+                    await syncProxy.ackDrained({ ids: delivered, ...(result.receiptVersion === 1 && { receipt: {
+                                version: 1, committedIds: result.committedIds ?? [], discardedIds: result.discardedIds ?? [],
+                            } }) });
+                    if (conn.peer === peer)
+                        conn.lastPullAt = Date.now();
                 }
-                for (const id of ids)
-                    seen.add(id);
+                totalPulled += delivered.length;
                 // Safety: don't pull more than 10k entries in one sweep
                 if (totalPulled >= 10_000)
                     break;
             }
+            if (conn.peer !== peer || conn.status !== 'connected')
+                return;
             conn.lastPullError = null;
             if (totalPulled > 0) {
                 // How long it took, and how many heartbeats it outlasted: a pull that
@@ -665,6 +814,10 @@ export class SlaveConnector {
             }
         }
         catch (err) {
+            if (conn.peer !== peer || conn.status !== 'connected')
+                return;
+            if (isConnectionGone(err))
+                void this.connectionLost(`${conn.config.host}:${conn.config.port}`, conn, err);
             // Same rule as the status read beside it: once at warn, quietly after,
             // cleared on success. A pull that keeps failing is a backlog that keeps
             // growing, and the column showing the backlog cannot say why.