kuzzle 2.56.0 → 2.57.0-beta.2

package/dist/lib/cluster/node.js

@@ -1,3 +1,4 @@
+"use strict";
 /*
  * Kuzzle, a backend software, self-hostable and ready to use
  * to power modern apps
@@ -18,22 +19,51 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-"use strict";
-const os = require("os");
-const net = require("net");
-const assert = require("assert");
-const Bluebird = require("bluebird");
-const EventEmitter = require("eventemitter3");
-const _ = require("lodash");
-const debug = require("../util/debug")("kuzzle:cluster:sync");
-const { Mutex } = require("../util/mutex");
-const { ClusterIdCardHandler } = require("./idCardHandler");
-const ClusterPublisher = require("./publisher");
-const ClusterSubscriber = require("./subscriber");
-const ClusterState = require("./state");
-const ClusterCommand = require("./command");
-const kuzzleStateEnum = require("../kuzzle/kuzzleStateEnum");
-const { fromKoncordeIndex } = require("../util/koncordeCompat");
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+// Bare specifiers, not the `node:` prefix: `node.test.js` intercepts `os` with
+// `mock-require` to feed `getIP()` a fixed set of network interfaces, and a
+// `node:os` import compiles to `require("node:os")`, which that interception
+// does not see. A conversion must not change how a module is resolved — see
+// J3a, where a hard-coded `.js` next to a renamed file took the whole
+// functional matrix down.
+const assert_1 = __importDefault(require("assert")); // NOSONAR
+const util_1 = require("util"); // NOSONAR
+const net_1 = __importDefault(require("net")); // NOSONAR
+const os_1 = __importDefault(require("os")); // NOSONAR
+const bluebird_1 = __importDefault(require("bluebird"));
+const eventemitter3_1 = __importDefault(require("eventemitter3"));
+const intersection_1 = __importDefault(require("lodash/intersection"));
+const xor_1 = __importDefault(require("lodash/xor"));
+const long_1 = __importDefault(require("long"));
+const kuzzleStateEnum_1 = __importDefault(require("../kuzzle/kuzzleStateEnum"));
+const debug_1 = __importDefault(require("../util/debug"));
+const koncordeCompat_1 = require("../util/koncordeCompat");
+// NOSONAR: `Mutex` is deprecated in favour of `withLock`, but the two use
+// incompatible acquisition/TTL formats and must not contend on the same key —
+// every node takes "clusterHandshake" with `Mutex`. Deferred to #2894.
+const mutex_1 = require("../util/mutex"); // NOSONAR
+const command_1 = __importDefault(require("./command"));
+const idCardHandler_1 = require("./idCardHandler");
+const publisher_1 = __importDefault(require("./publisher"));
+const state_1 = __importDefault(require("./state"));
+const subscriber_1 = __importDefault(require("./subscriber"));
+const debug = (0, debug_1.default)("kuzzle:cluster:sync");
+/**
+ * How long `addNode()` may wait for a joining node's first sync message before
+ * it answers that node's handshake anyway.
+ *
+ * `addNode()` runs inside the handshake command handler, and the joining node
+ * stops waiting for the answer after 2000 ms — hard-coded in
+ * `ClusterCommand._sendSingleHandshake`, and in every release before it, so a
+ * node joining during a rolling upgrade has the same limit. Waiting up to
+ * `2 × cluster.heartbeat` (4 s by default) there made the joiner give up on a
+ * node that had already added it: a one-sided membership (step 15, F-03).
+ * Half the joiner's limit leaves the rest for the Redis round trip and the
+ * command socket.
+ */
+const HANDSHAKE_PROOF_BUDGET = 1000;
 /**
  * Test an IP address and determine if it's in the public or private range.
  *
@@ -41,15 +71,19 @@ const { fromKoncordeIndex } = require("../util/koncordeCompat");
  * @return {Boolean}
  */
 function isPrivateIP(ip) {
-    if (net.isIPv6(ip)) {
-        const prefix = ip.split(":")[0];
+    if (net_1.default.isIPv6(ip)) {
+        // `split` always yields a first element, empty string included.
+        const prefix = ip.split(":")[0] ?? "";
         return ((prefix.startsWith("fd") && prefix.length === 4) || prefix === "fe80");
     }
-    // IPv4
+    // IPv4. `?? Number.NaN` on the octets a malformed address does not have: every
+    // comparison below is then false, which is what comparing `undefined` did.
     const exploded = ip.split(".").map((s) => Number.parseInt(s));
-    return (exploded[0] === 10 ||
-        (exploded[0] === 172 && exploded[1] >= 16 && exploded[1] <= 31) ||
-        (exploded[0] === 192 && exploded[1] === 168));
+    const first = exploded[0] ?? Number.NaN;
+    const second = exploded[1] ?? Number.NaN;
+    return (first === 10 ||
+        (first === 172 && second >= 16 && second <= 31) ||
+        (first === 192 && second === 168));
 }
 /**
  * Some IPv4 addresses are reserved for internal uses only, and cannot be
@@ -60,18 +94,20 @@ function isPrivateIP(ip) {
 function isInternalIP(ip) {
     // To my knowledge, there aren't any reserved, non-loopback and non-routable
     // IPv6 addresses
-    if (net.isIPv6(ip)) {
+    if (net_1.default.isIPv6(ip)) {
         return false;
     }
     const exploded = ip.split(".").map((s) => Number.parseInt(s));
+    const first = exploded[0] ?? Number.NaN;
+    const second = exploded[1] ?? Number.NaN;
     // 127.x.x: loopback addresses are already flagged as "internal" by
     // os.networkInterfaces.
-    return (exploded[0] === 127 ||
+    return (first === 127 ||
         // 169.254.x.x addresses are APIPA addresses: temporary and non-routable.
         // We need to remove them from the accepted list of IP addresses
         // (this is a "just in case" scenario: APIPA addresses are obsolete and
         // should not be used anymore, but we never know...)
-        (exploded[0] === 169 && exploded[1] === 254));
+        (first === 169 && second === 254));
 }
 /**
  * Return the first IP address matching the provided configuration
@@ -81,11 +117,12 @@ function isInternalIP(ip) {
  * @param  {String} [options.ip] Used to target public or private addresses
  * @return {String|null}
  */
-function getIP({ family = "IPv4", interface: netInterface, ip } = {}) {
+function getIP({ family = "IPv4", interface: netInterface, ip, } = {}) {
     const mustBePrivate = ip === "private";
     let interfaces = [];
-    for (const [key, value] of Object.entries(os.networkInterfaces())) {
-        for (const _interface of value) {
+    for (const [key, value] of Object.entries(os_1.default.networkInterfaces())) {
+        // `os.networkInterfaces()` declares its values optional.
+        for (const _interface of value ?? []) {
             interfaces.push({
                 interface: key,
                 ..._interface,
@@ -100,12 +137,13 @@ function getIP({ family = "IPv4", interface: netInterface, ip } = {}) {
             (!ip || mustBePrivate === isPrivateIP(n.address)));
     });
     debug("Filtered interfaces %o", interfaces);
-    if (interfaces.length === 0) {
+    const [firstInterface] = interfaces;
+    if (firstInterface === undefined) {
         return null;
     }
     // take the first IP from the list if no interface has been defined
     if (!netInterface) {
-        return interfaces[0].address;
+        return firstInterface.address;
     }
     for (const i of interfaces) {
         if ([i.interface, i.address, i.mac].includes(netInterface)) {
@@ -114,9 +152,7 @@ function getIP({ family = "IPv4", interface: netInterface, ip } = {}) {
     }
     return null;
 }
-/**
- * @typedef {nodeActivityEnum}
- */
+/** What `trackActivity` records: a node joined, or a node was evicted. */
 const nodeActivityEnum = Object.freeze({
     ADDED: 1,
     EVICTED: 2,
@@ -124,34 +160,43 @@ const nodeActivityEnum = Object.freeze({
 // Handles the node logic: discovery, eviction, heartbeat, ...
 // Dependencies: core:cache module must be started
 class ClusterNode {
+    /**
+     * The id of the ID card this node reserved during `handshake()`.
+     *
+     * Null until the handshake gets that far, and read as a `string` at
+     * seventeen sites here plus `subscriber.ts` and `command.ts` — all of which
+     * run after it. The accessor says once where that can fail.
+     */
+    get nodeId() {
+        if (this._nodeId === null) {
+            throw new Error("[CLUSTER] No node id yet: the handshake has not run");
+        }
+        return this._nodeId;
+    }
+    set nodeId(id) {
+        this._nodeId = id;
+    }
     constructor() {
+        this._nodeId = null;
         this.config = global.kuzzle.config.cluster;
         this.logger = global.kuzzle.log.child("cluster:node");
         this.heartbeatDelay = this.config.heartbeat;
         const family = this.config.ipv6 ? "IPv6" : "IPv4";
-        this.ip = getIP({
+        const ip = getIP({
             family,
             interface: this.config.interface,
             ip: this.config.ip,
         });
-        debug("Found IP address: %s with config %o", this.ip, this.config);
-        assert(this.ip !== null, `[CLUSTER] No suitable IP address found with the provided configuration (family: ${family}, interface: ${this.config.interface}, ip: ${this.config.ip})`);
-        this.nodeId = null;
+        debug("Found IP address: %s with config %o", ip, this.config);
+        (0, assert_1.default)(ip !== null, `[CLUSTER] No suitable IP address found with the provided configuration (family: ${family}, interface: ${this.config.interface}, ip: ${this.config.ip})`);
+        this.ip = ip;
         this.heartbeatTimer = null;
-        this.idCardHandler = new ClusterIdCardHandler(this);
-        this.publisher = new ClusterPublisher(this);
-        this.fullState = new ClusterState();
-        this.command = new ClusterCommand(this);
-        this.eventEmitter = new EventEmitter();
-        /**
-         * Links remote node IDs with their subscriber counterpart
-         * @type {Map.<string, ClusterSubscriber>}
-         */
+        this.idCardHandler = new idCardHandler_1.ClusterIdCardHandler(this);
+        this.publisher = new publisher_1.default(this);
+        this.fullState = new state_1.default();
+        this.command = new command_1.default(this);
+        this.eventEmitter = new eventemitter3_1.default();
         this.remoteNodes = new Map();
-        /**
-         * Cluster nodes activity, used to keep track of nodes being added or
-         * removed, to give more insights to cluster statuses
-         */
         this.activityMaxLength = this.config.activityDepth;
         this.activity = [];
     }
@@ -174,7 +219,7 @@ class ClusterNode {
         if (this.countActiveNodes() < this.config.minimumNodes) {
             this.logger.info("[CLUSTER] Not enough nodes active. Waiting for other nodes to join the cluster...");
             while (this.countActiveNodes() < this.config.minimumNodes) {
-                await Bluebird.delay(100);
+                await bluebird_1.default.delay(100);
             }
         }
         return this.nodeId;
@@ -184,12 +229,18 @@ class ClusterNode {
      * nodes, and removes entries from the cache
      */
     async shutdown() {
-        clearInterval(this.heartbeatTimer);
+        clearInterval(this.heartbeatTimer ?? undefined);
         await this.idCardHandler.dispose();
         for (const subscriber of this.remoteNodes.values()) {
             subscriber.dispose();
         }
-        this.publisher.sendNodeShutdown(this.nodeId);
+        // `_nodeId`, not the accessor, which throws before the handshake: a SIGTERM
+        // during cluster init must still reach the two disposals below, as it did
+        // on v2.56.0. No peer knows this node before it has an id, so there is
+        // nobody to announce the shutdown to.
+        if (this._nodeId !== null) {
+            this.publisher.sendNodeShutdown(this._nodeId);
+        }
         await this.publisher.dispose();
         this.command.dispose();
     }
@@ -205,9 +256,7 @@ class ClusterNode {
         // when debugging.
         // Otherwise when recovering from a debug session, all the other nodes will be evicted.
         for (const subscriber of this.remoteNodes.values()) {
-            subscriber.handleNodePreventEviction({
-                evictionPrevented,
-            });
+            subscriber.setEvictionPrevented(evictionPrevented);
         }
     }
     /**
@@ -221,16 +270,37 @@ class ClusterNode {
         if (this.remoteNodes.has(id)) {
             return false;
         }
-        const subscriber = new ClusterSubscriber(this, id, ip);
+        const subscriber = new subscriber_1.default(this, id, ip);
         this.remoteNodes.set(id, subscriber);
         await subscriber.init();
+        // Same ordering `handshake()` needs, and for the same reason: `lastMessageId`
+        // reached this node over the *command* channel, in the joining node's
+        // handshake, while this subscription lives on the *sync* channel. Until the
+        // subscription takes effect the remote's PUB socket drops what it publishes,
+        // and `sync()` below would resume from just before the dropped message.
+        // TD-65 (#2773).
+        //
+        // What the wait protects is the *answer*: the joining node publishes
+        // nothing but heartbeats until every node has answered its handshake, so
+        // answering once the subscription is live keeps its first real messages
+        // from being dropped. It is bounded well below the joiner's handshake
+        // timeout (see HANDSHAKE_PROOF_BUDGET): the subscription itself is live
+        // within milliseconds, and what takes up to a heartbeat round is only the
+        // joiner's next heartbeat, the proof. Missing the proof is then routine
+        // rather than a warning, and a message lost anyway is a gap, which is
+        // retransmitted (#2785).
+        const proofTimeout = Math.min(this.heartbeatDelay * 2, HANDSHAKE_PROOF_BUDGET);
+        const live = await subscriber.waitForSubscription(proofTimeout);
+        if (!live) {
+            debug("[CLUSTER] No sync message received from node %s within %dms: answering its handshake with a subscription not proven live yet", id, proofTimeout);
+        }
         await subscriber.sync(lastMessageId);
         await this.idCardHandler.addNode(id);
         this.logger.info(`[CLUSTER] Node "${id}" joined the cluster`);
         this.trackActivity(id, ip, nodeActivityEnum.ADDED);
-        if (global.kuzzle.state === kuzzleStateEnum.NOT_ENOUGH_NODES &&
+        if (global.kuzzle.state === kuzzleStateEnum_1.default.NOT_ENOUGH_NODES &&
             this.countActiveNodes() >= this.config.minimumNodes) {
-            global.kuzzle.state = kuzzleStateEnum.RUNNING;
+            global.kuzzle.state = kuzzleStateEnum_1.default.RUNNING;
             this.logger.warn(`[CLUSTER] Minimum number of nodes reached (${this.countActiveNodes()}). This node is now accepting requests again.`);
         }
         return true;
@@ -247,7 +317,22 @@ class ClusterNode {
         if (error) {
             this.logger.error(error.stack);
         }
-        this.publisher.sendNodeEvicted(this.nodeId, this.nodeId, reason);
+        // Same guard as `shutdown()`: before the handshake there is no id to
+        // broadcast, and throwing here would skip the shutdown below.
+        if (this._nodeId !== null) {
+            this.publisher.sendNodeEvicted(this._nodeId, this._nodeId, reason);
+        }
+        // The broadcast above tells the *other* nodes to forget this one. It cannot
+        // tell this one: a ZeroMQ PUB socket does not deliver to its own process, so
+        // `ClusterSubscriber.handleNodeEviction`'s `message.nodeId === localNode
+        // .nodeId` branch — the one that shuts the node down — is unreachable by the
+        // only node that needs it. Without this call the node leaves the cluster and
+        // keeps answering requests behind the load balancer, from state that has
+        // stopped advancing. See TD-67 (#2776).
+        //
+        // Exit code 1: this is a failure, and a node that exits 0 is not restarted
+        // by an `on-failure` policy (#2785).
+        global.kuzzle.shutdown(1);
     }
     /**
      * Evicts a remote from the list
@@ -256,7 +341,7 @@ class ClusterNode {
      * @param {boolean} [options.broadcast] - broadcast the eviction to the cluster
      * @param {string}  [options.reason] - reason of eviction
      */
-    async evictNode(nodeId, { broadcast = false, reason = "" }) {
+    async evictNode(nodeId, { broadcast = false, reason = "", }) {
         const subscriber = this.remoteNodes.get(nodeId);
         if (!subscriber) {
             this.logger.warn(`[CLUSTER] Node "${nodeId}" with no subscriber evicted. Reason: ${reason}`);
@@ -272,7 +357,7 @@ class ClusterNode {
             this.publisher.sendNodeEvicted(this.nodeId, nodeId, reason);
         }
         if (this.countActiveNodes() < this.config.minimumNodes) {
-            global.kuzzle.state = kuzzleStateEnum.NOT_ENOUGH_NODES;
+            global.kuzzle.state = kuzzleStateEnum_1.default.NOT_ENOUGH_NODES;
             this.logger.warn(`[CLUSTER] Not enough nodes active (expected: ${this.config.minimumNodes}, active: ${this.countActiveNodes()}). Deactivating node until new ones are added.`);
         }
     }
@@ -286,80 +371,21 @@ class ClusterNode {
     async enforceClusterConsistency() {
         // Delay the check to 1 heartbeat round, to allow all nodes to update
         // their ID cards
-        await Bluebird.delay(this.heartbeatDelay);
+        await bluebird_1.default.delay(this.heartbeatDelay);
         try {
             const idCards = await this.idCardHandler.getRemoteIdCards();
-            idCards.push(this.idCardHandler.idCard);
-            let splits = [];
-            for (const idCard of idCards) {
-                let topology = Array.from(idCard.topology);
-                topology.push(idCard.id);
-                if (topology.length !== idCards.length) {
-                    topology = topology.sort();
-                    const found = splits.some((split) => {
-                        if (split.length !== topology.length) {
-                            return false;
-                        }
-                        return split.every((id, index) => id === topology[index]);
-                    });
-                    if (!found) {
-                        splits.push(topology);
-                    }
-                }
+            const ownIdCard = this.idCardHandler.idCard;
+            // A node with no ID card contributes no topology. Pushing `null` is
+            // what `detectSplits` then read `.topology` off, one line later.
+            if (ownIdCard !== null) {
+                idCards.push(ownIdCard);
             }
+            const splits = this.detectSplits(idCards);
             // No split detected, the cluster is consistent
             if (splits.length === 0) {
                 return;
             }
-            // There is at least 1 cluster split detected.
-            //
-            // First we elect the smallest split possible.
-            // If multiple splits are eligibles, we choose amongst them the split
-            // containing the youngest isolated node (isolated = the node is in this
-            // split alone, and in no other splits). If no split is eligible using
-            // this method, we fall back to the smallest split containing the youngest
-            // node (isolated or not).
-            //
-            // The goal of this process is to force at least 1 node to kill itself,
-            // with all nodes concluding on their own on the same list of nodes to
-            // shut down.
-            // First remove every non existing node from topologies
-            splits = splits.map((topology) => topology.filter((nodeId) => idCards.find((card) => card.id === nodeId)));
-            splits = splits.sort((a, b) => a.length - b.length);
-            const eligibleSplits = splits.filter((split) => split.length === splits[0].length);
-            let candidates;
-            if (eligibleSplits.length === 1) {
-                candidates = eligibleSplits[0];
-            }
-            else {
-                // Beware: search isolated nodes in ALL the splits, not only the
-                // smallest ones
-                let isolatedNodes = _.xor(...splits);
-                const eligibleNodes = _.uniq(_.flatten(eligibleSplits));
-                isolatedNodes = _.intersection(isolatedNodes, eligibleNodes);
-                const isIsolated = isolatedNodes.length > 0;
-                // safety measure: this should never happen
-                if (isolatedNodes.length === 0) {
-                    isolatedNodes = eligibleNodes;
-                }
-                let youngestNode;
-                for (let i = 0; i < isolatedNodes.length; i++) {
-                    const idCard = idCards.find((card) => card.id === isolatedNodes[i]);
-                    if (!youngestNode || idCard.birthdate > youngestNode.birthdate) {
-                        youngestNode = idCard;
-                    }
-                }
-                if (isIsolated) {
-                    for (let i = 0; !candidates && i < eligibleSplits.length; i++) {
-                        if (eligibleSplits[i].includes(youngestNode.id)) {
-                            candidates = _.intersection(eligibleSplits[i], isolatedNodes);
-                        }
-                    }
-                }
-                else {
-                    candidates = [youngestNode.id];
-                }
-            }
+            const candidates = this.electShutdownCandidates(splits, idCards);
             if (candidates.includes(this.nodeId)) {
                 this.logger.error("[CLUSTER] Network split detected. This node is outside the cluster: shutting down.");
                 global.kuzzle.shutdown();
@@ -368,10 +394,160 @@ class ClusterNode {
         }
         catch (err) {
             this.logger.error("[CLUSTER] Unexpected exception caught during a cluster consistency check. Shutting down...");
-            this.logger.error(err.stack);
+            this.logger.error(err instanceof Error ? err.stack : (0, util_1.inspect)(err));
             global.kuzzle.shutdown();
         }
     }
+    /**
+     * Builds the distinct topologies the cluster's ID cards disagree on. A
+     * topology shorter than the number of known cards is a split: that card sees
+     * fewer nodes than exist.
+     *
+     * Extracted from `enforceClusterConsistency()` unchanged — a pure function of
+     * the cards it is given, which is what makes moving it safe.
+     */
+    detectSplits(idCards) {
+        const splits = [];
+        for (const idCard of idCards) {
+            let topology = Array.from(idCard.topology);
+            topology.push(idCard.id);
+            if (topology.length !== idCards.length) {
+                // Explicit comparator rather than the default: identical for these
+                // ASCII node ids, and the split election compares the sorted arrays
+                // element by element, so the ordering is load-bearing.
+                topology = topology.sort((a, b) => {
+                    if (a === b) {
+                        return 0;
+                    }
+                    return a < b ? -1 : 1;
+                });
+                const found = splits.some((split) => {
+                    if (split.length !== topology.length) {
+                        return false;
+                    }
+                    return split.every((id, index) => id === topology[index]);
+                });
+                if (!found) {
+                    splits.push(topology);
+                }
+            }
+        }
+        return splits;
+    }
+    /**
+     * Decides which nodes must shut themselves down, given the detected splits.
+     *
+     * Extracted from `enforceClusterConsistency()` unchanged. Every node runs
+     * this on its own and must reach the same list, so it is a pure function of
+     * `splits` and `idCards` — no clock, no randomness, no state.
+     */
+    electShutdownCandidates(splits, idCards) {
+        // There is at least 1 cluster split detected.
+        //
+        // First we elect the smallest split possible.
+        // If multiple splits are eligibles, we choose amongst them the split
+        // containing the youngest isolated node (isolated = the node is in this
+        // split alone, and in no other splits). If no split is eligible using
+        // this method, we fall back to the smallest split containing the youngest
+        // node (isolated or not).
+        //
+        // The goal of this process is to force at least 1 node to kill itself,
+        // with all nodes concluding on their own on the same list of nodes to
+        // shut down.
+        // First remove every non existing node from topologies
+        splits = splits.map((topology) => topology.filter((nodeId) => idCards.find((card) => card.id === nodeId)));
+        splits = splits.sort((a, b) => a.length - b.length);
+        const [smallest] = splits;
+        // The caller only reaches here with at least one split — the empty case
+        // returns before — and the sort then makes `splits[0]` the smallest.
+        if (smallest === undefined) {
+            return [];
+        }
+        const eligibleSplits = splits.filter((split) => split.length === smallest.length);
+        const [onlyEligible] = eligibleSplits;
+        if (eligibleSplits.length === 1 && onlyEligible !== undefined) {
+            return onlyEligible;
+        }
+        return this.electFromTiedSplits(eligibleSplits, splits, idCards);
+    }
+    /**
+     * Breaks a tie between equally small splits: prefer the one holding the
+     * youngest *isolated* node (present in that split and no other), and fall
+     * back to the youngest node overall.
+     *
+     * Extracted from `electShutdownCandidates()` unchanged, and pure like it —
+     * every node must reach the same answer from the same inputs.
+     */
+    electFromTiedSplits(eligibleSplits, splits, idCards) {
+        // Beware: search isolated nodes in ALL the splits, not only the
+        // smallest ones
+        let isolatedNodes = (0, xor_1.default)(...splits);
+        const eligibleNodes = [...new Set(eligibleSplits.flat())];
+        isolatedNodes = (0, intersection_1.default)(isolatedNodes, eligibleNodes);
+        const isIsolated = isolatedNodes.length > 0;
+        // safety measure: this should never happen
+        if (isolatedNodes.length === 0) {
+            isolatedNodes = eligibleNodes;
+        }
+        let youngestNode;
+        for (const isolatedNode of isolatedNodes) {
+            const idCard = idCards.find((card) => card.id === isolatedNode);
+            // `find` answers undefined for a node whose ID card has expired since
+            // the topologies were read. It was dereferenced for its birthdate.
+            if (idCard === undefined) {
+                continue;
+            }
+            if (!youngestNode || idCard.birthdate > youngestNode.birthdate) {
+                youngestNode = idCard;
+            }
+        }
+        // Every isolated node's ID card is gone: there is nobody left to elect.
+        if (youngestNode === undefined) {
+            return [];
+        }
+        if (!isIsolated) {
+            return [youngestNode.id];
+        }
+        for (const split of eligibleSplits) {
+            if (split.includes(youngestNode.id)) {
+                return (0, intersection_1.default)(split, isolatedNodes);
+            }
+        }
+        return [];
+    }
+    /**
+     * Handles a full-state round that no node answered: gives up if this was
+     * already the second attempt, otherwise drops every subscriber and waits a
+     * redis heartbeat round so the discovery can be redone against ID cards that
+     * may since have expired.
+     *
+     * Extracted from `handshake()` unchanged.
+     *
+     * @returns false when the caller must stop — the cluster is split and this
+     *          node is shutting down
+     */
+    async retryFullState(retried) {
+        // Uh oh... no node was able to give us the full state.
+        // We must retry later, to check if the redis keys have expired. If they
+        // are still there and we still aren't able to fetch a full state, this
+        // means we're probably facing a network split, and we must then shut
+        // down.
+        if (retried) {
+            this.logger.error("[CLUSTER] Could not connect to discovered cluster nodes (network split detected). Shutting down.");
+            global.kuzzle.shutdown();
+            return false;
+        }
+        // Disposes all subscribers
+        for (const subscriber of this.remoteNodes.values()) {
+            subscriber.dispose();
+        }
+        this.remoteNodes.clear();
+        // Waits for a redis heartbeat round
+        const retryDelay = this.heartbeatDelay * 1.5;
+        this.logger.warn(`[CLUSTER] Unable to connect to discovered cluster nodes. Retrying in ${retryDelay}ms...`);
+        await bluebird_1.default.delay(retryDelay);
+        return true;
+    }
     /**
      * Discovers other active nodes from the cluster and, if other nodes exist,
      * starts a handshake procedure to sync this node and to make it able to
@@ -384,9 +560,11 @@ class ClusterNode {
             this.logger.error(`[CLUSTER] Failed to join the cluster: timed out (joinTimeout: ${this.config.joinTimeout}ms)`);
             global.kuzzle.shutdown();
         }, this.config.joinTimeout);
-        const mutex = new Mutex("clusterHandshake", {
-            timeout: this.config.joinTimeout,
-        });
+        // NOSONAR (S1874): same deferral as the import above — every node takes
+        // "clusterHandshake" with `Mutex`, so converting one site to `withLock`
+        // would remove the exclusion it exists for. #2894.
+        const lockOptions = { timeout: this.config.joinTimeout };
+        const mutex = new mutex_1.Mutex("clusterHandshake", lockOptions); // NOSONAR
         try {
             await mutex.lock();
             // Create the ID Key AFTER the handshake mutex is actually locked,
@@ -394,8 +572,10 @@ class ClusterNode {
             // node while it's still initializing)
             await this.idCardHandler.createIdCard();
             debug("[CLUSTER] ID Card created");
-            this.nodeId = this.idCardHandler.nodeId;
-            await this.startHeartbeat();
+            const nodeId = this.idCardHandler.nodeId;
+            (0, assert_1.default)(nodeId !== null, "[CLUSTER] createIdCard() returned without reserving a node id");
+            this.nodeId = nodeId;
+            await this.startHeartbeat(); // NOSONAR: TD-26
             debug("[CLUSTER] Start heartbeat");
             let retried = false;
             let fullState = null;
@@ -412,64 +592,35 @@ class ClusterNode {
                 // Verify that no other node share the same IP address as this one
                 const duplicate = nodes.filter((node) => node.ip === this.ip);
                 if (duplicate.length > 0) {
-                    this.logger.error(`[CLUSTER] Another node share the same IP address as this one (${this.ip}): ${duplicate[0].id}. Shutting down.`);
+                    this.logger.error(`[CLUSTER] Another node share the same IP address as this one (${this.ip}): ${duplicate[0]?.id}. Shutting down.`);
                     global.kuzzle.shutdown();
                     return;
                 }
-                // Subscribe to remote nodes and start buffering sync messages
-                await Bluebird.map(nodes, ({ id, ip }) => {
-                    const subscriber = new ClusterSubscriber(this, id, ip);
-                    this.remoteNodes.set(id, subscriber);
-                    return subscriber.init();
-                });
+                await this.subscribeToNodes(nodes);
                 debug("[CLUSTER] Successfully subscribed to nodes");
                 fullState = await this.command.getFullState(nodes);
-                // Uh oh... no node was able to give us the full state.
-                // We must retry later, to check if the redis keys have expired. If they
-                // are still there and we still aren't able to fetch a full state, this
-                // means we're probably facing a network split, and we must then shut
-                // down.
                 if (fullState === null) {
-                    if (retried) {
-                        this.logger.error("[CLUSTER] Could not connect to discovered cluster nodes (network split detected). Shutting down.");
-                        global.kuzzle.shutdown();
+                    if (!(await this.retryFullState(retried))) {
                         return;
                     }
-                    // Disposes all subscribers
-                    for (const subscriber of this.remoteNodes.values()) {
-                        subscriber.dispose();
-                    }
-                    this.remoteNodes.clear();
-                    // Waits for a redis heartbeat round
                     retried = true;
-                    const retryDelay = this.heartbeatDelay * 1.5;
-                    this.logger.warn(`[CLUSTER] Unable to connect to discovered cluster nodes. Retrying in ${retryDelay}ms...`);
-                    await Bluebird.delay(retryDelay);
                 }
             } while (fullState === null);
             debug("[CLUSTER] Fullstate retrieved, loading into node..");
-            await this.fullState.loadFullState(fullState);
+            await this.fullState.loadFullState(fullState); // NOSONAR: TD-26
             this.activity = fullState.activity ? fullState.activity : this.activity;
             debug("[CLUSTER] Fullstate loaded.");
             const handshakeResponses = await this.command.broadcastHandshake(nodes);
             debug("[CLUSTER] Successful handshakes with other nodes.");
-            // Update subscribers: start synchronizing, or unsubscribes from nodes who
-            // didn't respond
-            for (const [nodeId, handshakeData] of Object.entries(handshakeResponses)) {
-                const subscriber = this.remoteNodes.get(nodeId);
-                if (handshakeData === null) {
-                    subscriber.dispose();
-                    this.remoteNodes.delete(nodeId);
-                }
-                else {
-                    await this.idCardHandler.addNode(nodeId);
-                    const nodesStates = fullState.nodesState || [];
-                    const nodeStatus = nodesStates.find((node) => node.id === nodeId);
-                    subscriber.sync(nodeStatus ? nodeStatus.lastMessageId : handshakeData.lastMessageId);
-                    this.logger.info(`[CLUSTER] Successfully completed the handshake with node ${nodeId}`);
-                }
-            }
-            this.logger.info("[CLUSTER] Successfully joined the cluster.");
+            await this.syncWithPeers(handshakeResponses, fullState);
+            // `createIdCard()` adopts `global.nodeId`, so these are normally the same
+            // string and the line reads as one name (TD-59, #2764). They still differ
+            // in the two cases the adoption cannot cover — a stale IdCard holding the
+            // key, and a Kuzzle running without a Backend — and those are exactly the
+            // cases where a reader needs to be told, so the line says both.
+            this.logger.info(global.nodeId && global.nodeId !== this.nodeId
+                ? `[CLUSTER] Successfully joined the cluster as "${this.nodeId}" (process node id: ${global.nodeId}).`
+                : `[CLUSTER] Successfully joined the cluster as "${this.nodeId}".`);
             this.trackActivity(this.nodeId, this.ip, nodeActivityEnum.ADDED);
         }
         finally {
@@ -477,6 +628,85 @@ class ClusterNode {
             await mutex.unlock();
         }
     }
+    /**
+     * Subscribes to every discovered node and starts buffering their sync
+     * messages.
+     *
+     * Extracted from `handshake()` unchanged, including its awaits: the wait
+     * below is what makes the `getFullState` that follows it safe, so the
+     * ordering is the point of the method, not an implementation detail.
+     */
+    async subscribeToNodes(nodes) {
+        // Subscribe to remote nodes and start buffering sync messages.
+        //
+        // The wait is what makes `getFullState` below safe: it snapshots each
+        // node's `lastMessageId` on the *command* channel, and this
+        // subscription lives on the *sync* channel, so nothing orders the
+        // snapshot after the subscription taking effect. Until it does, the
+        // remote's PUB socket drops what it publishes — and this node is then
+        // told to resume from just before the dropped message. See TD-65
+        // (#2773).
+        await bluebird_1.default.map(nodes, async ({ id, ip }) => {
+            const subscriber = new subscriber_1.default(this, id, ip);
+            this.remoteNodes.set(id, subscriber);
+            await subscriber.init();
+            const live = await subscriber.waitForSubscription(this.heartbeatDelay * 2);
+            if (!live) {
+                // Not fatal: a node whose ID card outlived it is exactly what the
+                // `fullState === null` retry below exists for. Say so, and let that
+                // path decide.
+                this.logger.warn(`[CLUSTER] No sync message received from node ${id} within ${this.heartbeatDelay * 2}ms: proceeding with a subscription that is not proven live.`);
+            }
+        });
+    }
+    /**
+     * Starts synchronizing with each peer that answered the handshake, and drops
+     * the ones that did not.
+     *
+     * Extracted from `handshake()` unchanged. `sync()` is awaited here for the
+     * reason written inside: a gap found during its replay evicts this node, and
+     * the success line must not be printed before that is known.
+     */
+    async syncWithPeers(handshakeResponses, 
+    // `NonNullable`: the caller loops until the full state is not null, and
+    // this reads `nodesState` off it.
+    fullState) {
+        // Update subscribers: start synchronizing, or unsubscribes from nodes who
+        // didn't respond
+        for (const [nodeId, handshakeData] of Object.entries(handshakeResponses)) {
+            const subscriber = this.remoteNodes.get(nodeId);
+            // A response from a node this one never subscribed to: there is nothing
+            // to dispose of and nothing to sync. `subscribeToNodes` fills the map
+            // from the same list the handshake was broadcast to, so this is the
+            // gap between the two — and both branches below dereferenced it.
+            if (subscriber === undefined) {
+                this.remoteNodes.delete(nodeId);
+                continue;
+            }
+            if (handshakeData === null) {
+                subscriber.dispose();
+                this.remoteNodes.delete(nodeId);
+            }
+            else {
+                await this.idCardHandler.addNode(nodeId);
+                const nodesStates = fullState.nodesState || [];
+                const nodeStatus = nodesStates.find((node) => node.id === nodeId);
+                // Protobuf leaves an absent field undefined, and zero is where a
+                // subscriber that has seen nothing starts.
+                const lastMessageId = nodeStatus?.lastMessageId ??
+                    handshakeData.lastMessageId ??
+                    new long_1.default(0, 0, true);
+                // Awaited: `sync()` replays the buffered messages, and a gap found
+                // there evicts this node. Fired and forgotten, it used to print
+                // "Successfully completed the handshake" 2ms AFTER the eviction that
+                // contradicts it — a log describing a state the code is not in.
+                const synced = await subscriber.sync(lastMessageId);
+                if (synced) {
+                    this.logger.info(`[CLUSTER] Successfully completed the handshake with node ${nodeId}`);
+                }
+            }
+        }
+    }
     countActiveNodes() {
         return this.remoteNodes.size + 1;
     }
@@ -522,7 +752,10 @@ class ClusterNode {
             nodes: [],
         };
         const idCards = await this.idCardHandler.getRemoteIdCards();
-        idCards.push(this.idCardHandler.idCard);
+        const ownIdCard = this.idCardHandler.idCard;
+        if (ownIdCard !== null) {
+            idCards.push(ownIdCard);
+        }
         for (const idCard of idCards) {
             status.nodes.push({
                 address: idCard.ip,
@@ -665,7 +898,7 @@ class ClusterNode {
     onNewRealtimeRoom(payload) {
         const roomMessageId = this.publisher.sendNewRealtimeRoom(payload);
         debug("[%s] Broadcasting new realtime room %s (message: %d)", this.nodeId, payload.id, roomMessageId);
-        const icpair = fromKoncordeIndex(payload.index);
+        const icpair = (0, koncordeCompat_1.fromKoncordeIndex)(payload.index);
         this.fullState.addRealtimeRoom(payload.id, icpair.index, icpair.collection, payload.filter, {
             messageId: roomMessageId,
             nodeId: this.nodeId,