kuzzle 2.56.0 → 2.57.0-beta.2

package/dist/lib/cluster/subscriber.js

@@ -1,3 +1,4 @@
+"use strict";
 /*
  * Kuzzle, a backend software, self-hostable and ready to use
  * to power modern apps
@@ -18,15 +19,72 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-"use strict";
-const { Subscriber } = require("zeromq");
-const protobuf = require("protobufjs");
-const Long = require("long");
-const debug = require("../util/debug")("kuzzle:cluster:sync");
-const DocumentNotification = require("../core/realtime/notification/document");
-const UserNotification = require("../core/realtime/notification/user");
-const { has } = require("../util/safeObject");
-const { fromKoncordeIndex } = require("../util/koncordeCompat");
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+const node_util_1 = require("node:util");
+const long_1 = __importDefault(require("long"));
+const protobuf = __importStar(require("protobufjs"));
+const zeromq_1 = require("zeromq");
+const document_1 = __importDefault(require("../core/realtime/notification/document"));
+const user_1 = __importDefault(require("../core/realtime/notification/user"));
+const debug_1 = __importDefault(require("../util/debug"));
+const koncordeCompat_1 = require("../util/koncordeCompat");
+const debug = (0, debug_1.default)("kuzzle:cluster:sync");
+/**
+ * `sync.proto` types `scope` and `user` as plain strings, while the
+ * notifications built from them take closed unions — and nothing between the
+ * wire and the notification checked that the string was a member. See TD-68
+ * (#2779); keep these lists in step with the unions they `satisfies`.
+ */
+const REALTIME_SCOPES = [
+    "all",
+    "in",
+    "out",
+];
+const REALTIME_USERS = [
+    "all",
+    "in",
+    "none",
+    "out",
+];
+/**
+ * A topic is a protobuf type name, and the handler table below covers exactly
+ * `sync.proto`'s message list — but `lookup()` also resolves nested types, so a
+ * decodable topic is not automatically one this node knows how to apply.
+ */
+function isSyncTopic(handlers, topic) {
+    return Object.hasOwn(handlers, topic);
+}
+function isRealtimeScope(value) {
+    return REALTIME_SCOPES.includes(value);
+}
+function isRealtimeUsers(value) {
+    return REALTIME_USERS.includes(value);
+}
 /* eslint-disable sort-keys */
 const stateEnum = Object.freeze({
     BUFFERING: 1,
@@ -34,14 +92,12 @@ const stateEnum = Object.freeze({
     MISSING_HEARTBEAT: 3,
     EVICTED: 4,
 });
-/* eslint-enable sort-keys */
 // Handles messages received from other nodes
 class ClusterSubscriber {
     /**
-     * @constructor
-     * @param {ClusterNode} localNode
-     * @param {string} remoteNodeId - Remote node unique ID
-     * @param {string} remoteNodeIP - address of the distant node
+     * @param localNode the cluster node this subscriber belongs to
+     * @param remoteNodeId - Remote node unique ID
+     * @param remoteNodeIP - address of the distant node
      */
     constructor(localNode, remoteNodeId, remoteNodeIP) {
         // not to be confused with remote nodes
@@ -54,12 +110,25 @@ class ClusterSubscriber {
         this.socket = null;
         this.protoroot = null;
         // keeps track of received message ids from the remote node
-        this.lastMessageId = new Long(0, 0, true);
+        this.lastMessageId = new long_1.default(0, 0, true);
+        // Receiving anything at all from the remote node is the only observable
+        // proof that this subscription has taken effect on its PUB socket: ZeroMQ
+        // propagates a subscription as a message sent TO the publisher, which does
+        // the filtering, and until it lands the publisher silently drops what it
+        // sends. See waitForSubscription() and TD-65 (#2773).
+        this.subscriptionConfirmed = false;
+        // The executor runs synchronously, so the no-op is replaced before anything
+        // can call it — it is there to state that this field is never unassigned.
+        this.confirmSubscription = () => { };
+        this.subscriptionProof = new Promise((resolve) => {
+            this.confirmSubscription = resolve;
+        });
         // A subscriber starts in the BUFFERING state, meaning it stores incoming
         // sync messages without processing them, until it enters the SANE state
         // This delays applying sync messages while the local node initializes
         this.state = stateEnum.BUFFERING;
         this.buffer = [];
+        this.recovering = false;
         // keeps track of the remote node heartbeats, and evicts it if no
         // heartbeats have been received after some time
         this.heartbeatTimer = null;
@@ -101,7 +170,7 @@ class ClusterSubscriber {
      */
     async init() {
         this.protoroot = await protobuf.load(`${__dirname}/protobuf/sync.proto`);
-        this.socket = new Subscriber();
+        this.socket = new zeromq_1.Subscriber();
         this.socket.connect(this.remoteNodeAddress);
         this.socket.subscribe();
         this.heartbeatTimer = setInterval(() => this.checkHeartbeat(), this.heartbeatDelay);
@@ -114,20 +183,20 @@ class ClusterSubscriber {
      */
     async listen() {
         while (this.state !== stateEnum.EVICTED) {
-            let topic;
-            let data;
-            try {
-                [topic, data] = await this.socket.receive();
-            }
-            catch (e) {
-                if (this.state !== stateEnum.EVICTED) {
-                    await this.evictNode({
-                        broadcast: true,
-                        reason: e.message,
-                    });
-                }
+            const frame = await this.receiveFrame();
+            if (frame === null) {
                 return;
             }
+            const [topic, data] = frame;
+            // A sync message is a topic and a payload. One that is neither is not
+            // something to dispatch on, and both frames were read unchecked.
+            if (topic === undefined || data === undefined) {
+                continue;
+            }
+            if (!this.subscriptionConfirmed) {
+                this.subscriptionConfirmed = true;
+                this.confirmSubscription();
+            }
             // do nothing is the node was evicted even if we already check
             // for this in processData()
             if (this.state === stateEnum.EVICTED) {
@@ -141,12 +210,71 @@ class ClusterSubscriber {
             }
         }
     }
+    /**
+     * One frame off the sync socket.
+     *
+     * @returns `null` when there is nothing more to read — the socket is gone,
+     *          or the read failed and this node has been evicted for it, which
+     *          is what ends `listen()`'s loop.
+     */
+    async receiveFrame() {
+        try {
+            // `dispose()` drops the socket, and the loop's own condition is checked
+            // before the state it sets is visible here.
+            if (this.socket === null) {
+                return null;
+            }
+            return await this.socket.receive();
+        }
+        catch (e) {
+            if (this.state !== stateEnum.EVICTED) {
+                await this.evictNode({
+                    broadcast: true,
+                    reason: e instanceof Error ? e.message : (0, node_util_1.inspect)(e),
+                });
+            }
+            return null;
+        }
+    }
+    /**
+     * Resolves once a message has been received from the remote node, which is
+     * the only observable proof that this subscription is live on its side.
+     *
+     * The handshake reads a remote node's `lastMessageId` over the *command*
+     * channel and then resumes this subscription from it. Nothing orders that
+     * snapshot after the subscription taking effect on the *sync* channel — they
+     * are different sockets — so without this wait, anything the remote publishes
+     * in between is dropped by its PUB socket while this node is told to resume
+     * from just before it. That gap is read as a desync. See TD-65 (#2773).
+     *
+     * Every node publishes a heartbeat every `cluster.heartbeat` ms (2s by
+     * default) from before it is discoverable, so this normally costs less than
+     * one round.
+     *
+     * @param timeout - ms to wait before giving up
+     * @return whether the subscription was proven live
+     */
+    async waitForSubscription(timeout) {
+        if (this.subscriptionConfirmed) {
+            return true;
+        }
+        let timer;
+        const expired = new Promise((resolve) => {
+            timer = setTimeout(() => resolve(false), timeout);
+        });
+        const confirmed = await Promise.race([
+            this.subscriptionProof.then(() => true),
+            expired,
+        ]);
+        clearTimeout(timer);
+        return confirmed;
+    }
     /**
      * Plays all buffered sync messages, and switches this subscriber state from
      * BUFFERING to SANE.
      *
-     * @param  {Long} lastMessageId
-     * @return {void}
+     * @param lastMessageId
+     * @return false if a gap was found while replaying, which evicts this node
      */
     async sync(lastMessageId) {
         this.lastMessageId = lastMessageId;
@@ -155,11 +283,18 @@ class ClusterSubscriber {
         while (this.buffer.length > 0) {
             const _buffer = this.buffer;
             this.buffer = [];
-            for (let i = 0; i < _buffer.length; i++) {
-                await this.processData(_buffer[i][0], _buffer[i][1]);
+            for (const [topic, data] of _buffer) {
+                await this.processData(topic, data);
+                // A gap found during the replay evicts this node. Without this check
+                // the loop would go on applying messages and then overwrite EVICTED
+                // with SANE on the way out.
+                if (this.state === stateEnum.EVICTED) {
+                    return false;
+                }
             }
         }
         this.state = stateEnum.SANE;
+        return true;
     }
     /**
      * Decodes an incoming message, and dispatches it.
@@ -170,16 +305,23 @@ class ClusterSubscriber {
      * and it also cannot afford to attach a rejection handler everytime a message
      * is received to prevent clogging the event loop with unnecessary promises
      *
-     * @param  {string} topic
-     * @param  {Buffer} data
-     * @return {void}
+     * @param topic
+     * @param data
      */
     async processData(topic, data) {
         if (this.state === stateEnum.EVICTED) {
             return;
         }
-        const decoder = this.protoroot.lookup(topic);
-        if (decoder === null) {
+        const decoder = this.protoroot?.lookup(topic) ?? null;
+        // `lookup` resolves any reflection object, not only message types, so a
+        // topic naming — say — a nested namespace used to reach `decoder.decode`
+        // with that method undefined: a TypeError thrown out of a method documented
+        // as never throwing, into `listen()`'s un-awaitable loop. The compiler asked
+        // for the narrowing; treating it as the unknown-topic case below is what the
+        // `null` branch already meant.
+        if (decoder === null ||
+            !(decoder instanceof protobuf.Type) ||
+            !isSyncTopic(this.handlers, topic)) {
             await this.evictNode({
                 broadcast: true,
                 reason: `received an invalid message from ${this.remoteNodeId} (unknown topic "${topic}")`,
@@ -194,19 +336,33 @@ class ClusterSubscriber {
             // If we are receiving messages from a node,
             // it means the node is alive so it should counts as an heartbeat
             this.handleHeartbeat();
-            await this.handlers[topic].call(this, message);
+            // `Reflect.apply`, not `handlers[topic].call`: the handler table is a
+            // union of per-topic signatures, and `call` asks each of them to accept
+            // `this` plus every other member's message. `isSyncTopic` above is what
+            // pairs the topic with its message.
+            await Reflect.apply(this.handlers[topic], this, [message]);
         }
         catch (e) {
-            this.localNode.evictSelf(`Unable to process sync message (topic: ${topic}, message: ${JSON.stringify(message)}`, e);
+            this.localNode.evictSelf(`Unable to process sync message (topic: ${topic}, message: ${JSON.stringify(message)}`, e instanceof Error ? e : new Error((0, node_util_1.inspect)(e)));
         }
     }
+    /**
+     * Set from two places: a `NodePreventEviction` message published by this
+     * peer, and `node.preventEviction()` on this node. The local caller used to
+     * reach the wire handler below with a hand-built `{ evictionPrevented }` —
+     * an object shaped like a sync message that never came off the wire and
+     * carries no `messageId`. The conversion could not type that, and it should
+     * not have to: the two callers want the same effect, not the same envelope.
+     */
+    setEvictionPrevented(evictionPrevented) {
+        this.remoteNodeEvictionPrevented = evictionPrevented;
+    }
     async handleNodePreventEviction(message) {
-        this.remoteNodeEvictionPrevented = message.evictionPrevented;
+        this.setEvictionPrevented(message.evictionPrevented);
     }
     /**
      * Handles a heartbeat from the remote node
      *
-     * @return {void}
      */
     handleHeartbeat() {
         this.lastHeartbeat = Date.now();
@@ -214,13 +370,13 @@ class ClusterSubscriber {
     /**
      * Handles a node eviction message.
      *
-     * @param  {Object} message - decoded NodeEvicted protobuf message
-     * @return {void}
+     * @param message - decoded NodeEvicted protobuf message
      */
     async handleNodeEviction(message) {
         if (message.nodeId === this.localNode.nodeId) {
             this.logger.error(`[CLUSTER] Node evicted by ${message.evictor}. Reason: ${message.reason}`);
-            global.kuzzle.shutdown();
+            // A failure, like `evictSelf`'s: exit non-zero (#2785).
+            global.kuzzle.shutdown(1);
             return;
         }
         await this.localNode.evictNode(message.nodeId, {
@@ -231,8 +387,7 @@ class ClusterSubscriber {
     /**
      * Handles a node shutdown.
      *
-     * @param  {Object} message - decoded NodeShutdown protobuf message
-     * @return {void}
+     * @param message - decoded NodeShutdown protobuf message
      */
     async handleNodeShutdown(message) {
         await this.localNode.evictNode(message.nodeId, {
@@ -243,12 +398,11 @@ class ClusterSubscriber {
     /**
      * Handles messages about realtime room creations
      *
-     * @param  {Object} message - decoded NewRealtimeRoom protobuf message
-     * @return {void}
+     * @param message - decoded NewRealtimeRoom protobuf message
      */
     handleNewRealtimeRoom(message) {
         const { id, index, filter, messageId } = message;
-        const icpair = fromKoncordeIndex(index);
+        const icpair = (0, koncordeCompat_1.fromKoncordeIndex)(index);
         debug("New realtime room created by node %s (message: %d, room: %s, index: %s, collection: %s)", this.remoteNodeId, messageId, id, icpair.index, icpair.collection);
         this.localNode.fullState.addRealtimeRoom(id, icpair.index, icpair.collection, JSON.parse(filter), {
             messageId,
@@ -259,8 +413,7 @@ class ClusterSubscriber {
     /**
      * Handles messages about realtime subscriptions
      *
-     * @param  {Object} message - decoded Subscription protobuf message
-     * @return {void}
+     * @param message - decoded Subscription protobuf message
      */
     handleSubscription(message) {
         debug("New realtime subscription received from node %s (message: %d, room: %s)", this.remoteNodeId, message.messageId, message.roomId);
@@ -269,8 +422,7 @@ class ClusterSubscriber {
     /**
      * Handles messages about realtime room removal
      *
-     * @param  {Object} message - decoded RemoveRealtimeRoom protobuf message
-     * @return {void}
+     * @param message - decoded RemoveRealtimeRoom protobuf message
      */
     handleRealtimeRoomRemoval(message) {
         debug("Realtime room removal received from node %s (message: %d, room: %s)", this.remoteNodeId, message.messageId, message.roomId);
@@ -279,8 +431,7 @@ class ClusterSubscriber {
     /**
      * Handles messages about user unsubscriptions
      *
-     * @param  {Object} message - decoded Unscription protobuf message
-     * @return {void}
+     * @param message - decoded Unsubscription protobuf message
      */
     handleUnsubscription(message) {
         debug("Realtime unsubscription received from node %s (message: %d, room: %s)", this.remoteNodeId, message.messageId, message.roomId);
@@ -289,8 +440,7 @@ class ClusterSubscriber {
     /**
      * Handles messages about cluster-wide events
      *
-     * @param {Object} message - decoded ClusterWideEvent protobuf message
-     * @return {void}
+     * @param message - decoded ClusterWideEvent protobuf message
      */
     handleClusterWideEvent(message) {
         const payload = JSON.parse(message.payload);
@@ -299,11 +449,21 @@ class ClusterSubscriber {
     /**
      * Handles messages about document notifications
      *
-     * @param {Object} message - decoded DocumentNotification protobuf message
-     * @return {void}
+     * @param message - decoded DocumentNotification protobuf message
      */
     async handleDocumentNotification(message) {
-        const notification = new DocumentNotification({
+        // The wire says `string`, `DocumentNotification` says "in" | "out" | "all",
+        // and until this conversion nothing checked. Treated as the malformed
+        // message it is — the same answer this file already gives to an unknown
+        // topic or a missing `messageId`. TD-68 (#2779).
+        if (!isRealtimeScope(message.scope)) {
+            await this.evictNode({
+                broadcast: true,
+                reason: `received an invalid message from ${this.remoteNodeId} (unknown document notification scope "${message.scope}")`,
+            });
+            return;
+        }
+        const notification = new document_1.default({
             action: message.action,
             collection: message.collection,
             controller: message.controller,
@@ -322,11 +482,19 @@ class ClusterSubscriber {
     /**
      * Handles messages about user notifications
      *
-     * @param {Object} message - decoded UserNotification protobuf message
-     * @return {void}
+     * @param message - decoded UserNotification protobuf message
      */
     async handleUserNotification(message) {
-        const notification = new UserNotification({
+        // Same as handleDocumentNotification above: "in" | "out" | "all" | "none"
+        // on this side, an unchecked string on the wire. TD-68 (#2779).
+        if (!isRealtimeUsers(message.user)) {
+            await this.evictNode({
+                broadcast: true,
+                reason: `received an invalid message from ${this.remoteNodeId} (unknown user notification scope "${message.user}")`,
+            });
+            return;
+        }
+        const notification = new user_1.default({
             action: message.action,
             collection: message.collection,
             controller: message.controller,
@@ -344,8 +512,7 @@ class ClusterSubscriber {
     /**
      * Handles messages about new authentication strategies
      *
-     * @param  {Object} message - decoded NewAuthStrategy protobuf message
-     * @return {void}
+     * @param message - decoded NewAuthStrategy protobuf message
      */
     handleNewAuthStrategy(message) {
         const { pluginName, strategy, strategyName } = message;
@@ -356,8 +523,7 @@ class ClusterSubscriber {
     /**
      * Handles messages about authentication strategy removals
      *
-     * @param  {Object} message - decoded RemoveAuthStrategy protobuf message
-     * @return {void}
+     * @param message - decoded RemoveAuthStrategy protobuf message
      */
     handleAuthStrategyRemoval(message) {
         const { pluginName, strategyName } = message;
@@ -368,7 +534,6 @@ class ClusterSubscriber {
     /**
      * Handles messages about security resets
      *
-     * @return {void}
      */
     async handleResetSecurity() {
         debug("Security reset received from node %s", this.remoteNodeId);
@@ -378,8 +543,7 @@ class ClusterSubscriber {
     /**
      * Handles a cross-nodes dump request
      *
-     * @param {Object} message - decoded DumpRequest protobuf message
-     * @return {void}
+     * @param message - decoded DumpRequest protobuf message
      */
     handleDumpRequest(message) {
         debug("Dump generation request received from node %s", this.remoteNodeId);
@@ -387,7 +551,6 @@ class ClusterSubscriber {
     }
     /**
      * Handles cluster-wide shutdown
-     * @return {void}
      */
     handleShutdown() {
         debug("Cluster-wide shutdown request received from node %s", this.remoteNodeId);
@@ -397,7 +560,6 @@ class ClusterSubscriber {
     /**
      * Handles changes on document validators
      *
-     * @return {void}
      */
     async handleRefreshValidators() {
         debug("Validators changed notification received from node %s", this.remoteNodeId);
@@ -415,8 +577,7 @@ class ClusterSubscriber {
     /**
      * Invalidates a profile to force reloading it from the storage space
      *
-     * @param {Object} message - decoded DumpRequest protobuf message
-     * @return {void}
+     * @param message - decoded InvalidateProfile protobuf message
      */
     async handleProfileInvalidation(message) {
         debug("Profile invalidation request received from node %s (profile: %s)", this.remoteNodeId, message.profileId);
@@ -425,8 +586,7 @@ class ClusterSubscriber {
     /**
      * Invalidates a role to force reloading it from the storage space
      *
-     * @param {Object} message - decoded DumpRequest protobuf message
-     * @return {void}
+     * @param message - decoded InvalidateRole protobuf message
      */
     async handleRoleInvalidation(message) {
         debug("Role invalidation request received from node %s (role: %s)", this.remoteNodeId, message.roleId);
@@ -435,8 +595,7 @@ class ClusterSubscriber {
     /**
      * Adds a new index to the index cache
      *
-     * @param {Object} message - decoded IndexCacheAdd protobuf message
-     * @return {void}
+     * @param message - decoded AddIndex protobuf message
      */
     async handleIndexAddition(message) {
         const { index, scope } = message;
@@ -446,8 +605,7 @@ class ClusterSubscriber {
     /**
      * Removes indexes from the index cache
      *
-     * @param {Object} message - decoded IndexCacheAdd protobuf message
-     * @return {void}
+     * @param message - decoded RemoveIndexes protobuf message
      */
     async handleIndexesRemoval(message) {
         const { indexes, scope } = message;
@@ -457,8 +615,7 @@ class ClusterSubscriber {
     /**
      * Adds a new collection to the index cache
      *
-     * @param {Object} message - decoded IndexCacheAdd protobuf message
-     * @return {void}
+     * @param message - decoded AddCollection protobuf message
      */
     async handleCollectionAddition(message) {
         const { collection, index, scope } = message;
@@ -468,8 +625,7 @@ class ClusterSubscriber {
     /**
      * Removes a collection from the index cache
      *
-     * @param {Object} message - decoded IndexCacheAdd protobuf message
-     * @return {void}
+     * @param message - decoded RemoveCollection protobuf message
      */
     async handleCollectionRemoval(message) {
         const { collection, index, scope } = message;
@@ -492,6 +648,29 @@ class ClusterSubscriber {
         if (this.state === stateEnum.EVICTED) {
             return;
         }
+        // A recovery holds `listen()` until the remote node has retransmitted
+        // (up to `cluster.syncTimeout`), so its heartbeats wait unread in the
+        // socket and `lastHeartbeat` goes stale while the node is alive and
+        // answering. Evicting it for that would broadcast the eviction of a
+        // healthy node (step 15, F-04). The recovery settles it either way: the
+        // retransmitted messages count as heartbeats, and a node that does not
+        // answer makes this node evict itself, as it did before #2785.
+        if (this.recovering) {
+            return;
+        }
+        // BUFFERING is left by `sync()` only, once the handshake has told this
+        // subscriber where to resume from. Setting SANE here used to end it early
+        // whenever the handshake outlasted one check period — which the waits of
+        // TD-65 (#2777, #2781) made routine: `listen()` then applied live messages
+        // against a counter still at 0, and `sync()` replayed a buffer opening on
+        // the very message it resumes after, outside the one state that drops
+        // those. The joining node read it as a loss and left the cluster (step 15,
+        // F-12). And there is no verdict to reach meanwhile: buffered frames are
+        // not applied, so they do not refresh `lastHeartbeat`; the handshake that
+        // owns this phase has its own timeouts.
+        if (this.state === stateEnum.BUFFERING) {
+            return;
+        }
         const now = Date.now();
         if (now - this.lastHeartbeat > this.heartbeatDelay) {
             if (this.state === stateEnum.MISSING_HEARTBEAT) {
@@ -516,17 +695,20 @@ class ClusterSubscriber {
             return;
         }
         this.state = stateEnum.EVICTED;
-        this.socket.close();
+        this.socket?.close();
         this.socket = null;
-        clearInterval(this.heartbeatTimer);
+        clearInterval(this.heartbeatTimer ?? undefined);
     }
     /**
      * Checks that the received message is the one we expect.
-     * @param  {Object} message - decoded protobuf message
-     * @return {boolean} false: the message must be discarded, true otherwise
+     * @param message - decoded protobuf message
+     * @return false: the message must be discarded, true otherwise
      */
     async validateMessage(message) {
-        if (!has(message, "messageId")) {
+        const messageId = message.messageId;
+        // Read once and checked for what it is: `has()` answers whether the key
+        // is there, which is not what the three `Long` operations below need.
+        if (messageId === undefined) {
             this.logger.warn(`Invalid message received from node ${this.remoteNodeId}. Evicting it.`);
             await this.evictNode({
                 broadcast: true,
@@ -534,18 +716,98 @@ class ClusterSubscriber {
             });
             return false;
         }
-        if (this.state === stateEnum.BUFFERING &&
-            this.lastMessageId.greaterThanOrEqual(message.messageId)) {
+        const received = long_1.default.fromValue(messageId);
+        // An id this subscriber has already reached is a message already applied
+        // — or, while buffering, one the full state it resumes from already
+        // includes. Nothing is missing, so there is nothing to recover and no
+        // reason to leave the cluster: it used to be dropped while buffering only,
+        // and anywhere else it was reported as a loss of "18446744073709551615"
+        // messages — the difference, negative, printed as an unsigned 64-bit
+        // number (step 15, F-12).
+        if (this.lastMessageId.greaterThanOrEqual(received)) {
+            debug("Ignoring message %s from node %s: already applied (last applied: %s)", received.toString(), this.remoteNodeId, this.lastMessageId.toString());
             return false;
         }
-        this.lastMessageId = this.lastMessageId.add(1);
-        if (this.lastMessageId.notEquals(message.messageId)) {
-            await this.localNode.evictSelf(`Node out-of-sync: ${message.messageId - this.lastMessageId - 1} messages lost from node ${this.remoteNodeId}`);
+        const expected = this.lastMessageId.add(1);
+        if (expected.equals(received)) {
+            this.lastMessageId = expected;
+            return true;
+        }
+        // Messages were lost in between: ask the remote node for them, and apply
+        // them before this one. `recover` leaves `lastMessageId` on the last one
+        // it applied, which is then this message's predecessor.
+        if (!this.recovering && (await this.recover(expected, received))) {
+            this.lastMessageId = received;
+            return true;
+        }
+        // `recover` may have evicted this node itself, applying a replayed message.
+        if (this.state === stateEnum.EVICTED) {
+            return false;
+        }
+        // Long arithmetic rather than `-`: message ids are 64-bit and
+        // `message.messageId` may arrive as a Long or as a number depending on how
+        // the protobuf reader was configured, so subtracting with `-` leans on
+        // `Long.prototype.valueOf` — which works, loses precision past 2^53, and is
+        // what SonarCloud's S3757 objects to. `fromValue` accepts either shape and
+        // `subtract` is exact. The count is against the id this node EXPECTED, so
+        // a single-message loss reads "1" — see TD-58 (#2762). `received` is past
+        // `expected` here, as the checks above leave nothing else, so the
+        // difference cannot wrap.
+        const lost = received.subtract(expected);
+        // Stop before evicting, for two reasons. `lastMessageId` is not
+        // resynchronised here, so leaving this subscriber running would re-trip
+        // this same check on every subsequent message: one drop was reported nine
+        // times in five seconds in CI, and occurrence counts read off those lines
+        // overcount. It also ends `listen()`'s loop, which is what "this node is
+        // out of the cluster" should mean locally. See TD-67 (#2776).
+        this.state = stateEnum.EVICTED;
+        await this.localNode.evictSelf(`Node out-of-sync: ${lost.toString()} messages lost from node ${this.remoteNodeId}`);
+        return false;
+    }
+    /**
+     * Fetches the messages with ids `from` to `upTo - 1` from the remote node,
+     * which keeps its last ones for that purpose, and applies them in order
+     * (#2785).
+     *
+     * Awaited where the gap is found, so nothing that came after it is applied
+     * before it: `listen()` stops reading meanwhile (ZeroMQ queues what arrives)
+     * and `sync()` stops replaying.
+     *
+     * @return whether every missing message was applied; `false` leaves the
+     *         decision — eviction — to the caller
+     */
+    async recover(from, upTo) {
+        const to = upTo.subtract(1);
+        // Set for the whole recovery, the wait included. Each frame goes through
+        // `validateMessage` again, which is what checks that the remote node
+        // answered with the ids that were asked for; a gap found *there* is not
+        // recovered from a second time: the remote node just answered with
+        // something other than what it was asked. And `checkHeartbeat` stands
+        // down meanwhile — see there.
+        this.recovering = true;
+        let frames;
+        try {
+            frames = await this.localNode.command.requestRetransmit(this.remoteNodeIP, from, to);
+            if (frames === null || this.state === stateEnum.EVICTED) {
+                return false;
+            }
+            for (const [topic, data] of frames) {
+                await this.processData(topic, data);
+                if (this.state === stateEnum.EVICTED) {
+                    return false;
+                }
+            }
+        }
+        finally {
+            this.recovering = false;
+        }
+        if (!this.lastMessageId.equals(to)) {
             return false;
         }
+        this.logger.warn(`[CLUSTER] Recovered ${frames.length} message(s) lost from node ${this.remoteNodeId}`);
         return true;
     }
-    async evictNode({ broadcast, reason }) {
+    async evictNode({ broadcast, reason, }) {
         this.state = stateEnum.EVICTED;
         await this.localNode.evictNode(this.remoteNodeId, { broadcast, reason });
     }