kuzzle 2.60.0-beta.1 → 2.60.0-beta.3

package/dist/lib/api/request/requestResponse.js

@@ -35,13 +35,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RequestResponse = exports.Headers = void 0;
 const assert = __importStar(require("../../util/assertType"));
@@ -72,11 +82,11 @@ class Headers {
         if (typeof name === "symbol") {
             return this.headers[name];
         }
-        assert.assertString("header name", name);
-        if (!name) {
+        const header = assert.assertString("header name", name);
+        if (!header) {
             return;
         }
-        const storedName = this.namesMap.get(name.toLowerCase());
+        const storedName = this.namesMap.get(header.toLowerCase());
         return storedName === undefined ? undefined : this.headers[storedName];
     }
     removeHeader(name) {
@@ -239,6 +249,7 @@ class RequestResponse {
     /**
      * Request result
      */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- kept for v2 compatibility, see docs/v3-breaking-changes.md
     get result() {
         return this[_request].result;
     }

package/dist/lib/cluster/command.d.ts

@@ -1,5 +1,3 @@
-/// <reference types="node" />
-/// <reference types="node" />
 import Long from "long";
 import type { SerializedState } from "./state";
 import type { Activity, Decoded, FullStateResponse, HandshakeResponse } from "./protobuf/commandMessages";

package/dist/lib/cluster/command.js

@@ -35,13 +35,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };

package/dist/lib/cluster/idCardHandler.d.ts

@@ -1,4 +1,5 @@
 import "../types";
+import type ClusterNode from "./node";
 export type SerializedIdCard = {
     id: string;
     ip: string;
@@ -41,7 +42,7 @@ export declare class ClusterIdCardHandler {
     /**
      * Node instance. Represents the local node.
      */
-    private node;
+    private readonly node;
     /**
      * Local node ID Card
      */
@@ -85,7 +86,11 @@ export declare class ClusterIdCardHandler {
      */
     private disposed;
     private readonly logger;
-    constructor(node: any);
+    constructor(node: ClusterNode);
+    /**
+     * How long an ID card outlives the last renewal of its node, in ms
+     */
+    get ttl(): number;
     /**
      * Generates and reserves a unique ID for this node instance.
      * Makes sure that the ID is not already taken by another node instance.

package/dist/lib/cluster/idCardHandler.js

@@ -94,6 +94,12 @@ class ClusterIdCardHandler {
         this.ip = node.ip;
         this.refreshDelay = node.heartbeatDelay;
     }
+    /**
+     * How long an ID card outlives the last renewal of its node, in ms
+     */
+    get ttl() {
+        return this.refreshDelay * this.refreshMultiplier;
+    }
     /**
      * Generates and reserves a unique ID for this node instance.
      * Makes sure that the ID is not already taken by another node instance.
@@ -139,7 +145,8 @@ class ClusterIdCardHandler {
         const refreshWorker = this.constructWorker(`${__dirname}/workers/IDCardRenewer${(0, node_path_1.extname)(__filename)}`);
         this.refreshWorker = refreshWorker;
         refreshWorker.on("message", async (message) => {
-            if (message.error) {
+            // A renewal racing the disposal finds the card deleted: not a failure
+            if (message.error && !this.disposed) {
                 await this.node.evictSelf(message.error);
             }
         });
@@ -207,7 +214,7 @@ class ClusterIdCardHandler {
         // this method runs on the line after it is filled.
         refreshWorker.on("message", ({ initialized }) => {
             if (initialized) {
-                clearInterval(this.refreshTimer);
+                clearInterval(this.refreshTimer ?? undefined);
                 this.refreshTimer = null;
             }
         });
@@ -228,6 +235,20 @@ class ClusterIdCardHandler {
                 // It could happens that the worker has been killed before the dispose causing send to fail
             }
         }
+        clearInterval(this.refreshTimer ?? undefined);
+        this.refreshTimer = null;
+        // The worker deletes the card too, but nothing waits for it: the process
+        // usually exits first, and the card then outlives it by a TTL, long
+        // enough for a node restarted on this IP to take it for a live duplicate
+        if (this.nodeIdKey !== null) {
+            try {
+                await global.kuzzle.ask("core:cache:internal:del", this.nodeIdKey);
+                await global.kuzzle.ask("core:cache:internal:execute", "srem", REDIS_ID_CARDS_INDEX, this.nodeIdKey);
+            }
+            catch (error) {
+                this.logger.warn(`Could not delete the ID card ${this.nodeIdKey}: it expires in ${this.ttl}ms (${error instanceof Error ? error.message : String(error)})`);
+            }
+        }
     }
     /**
      * Retrieves the ID cards from other nodes
@@ -309,7 +330,7 @@ class ClusterIdCardHandler {
         if (!this.idCard) {
             return false;
         }
-        return global.kuzzle.ask("core:cache:internal:store", this.nodeIdKey, JSON.stringify(this.idCard.serialize()), { onlyIfNew: creation, ttl: this.refreshDelay * this.refreshMultiplier });
+        return global.kuzzle.ask("core:cache:internal:store", this.nodeIdKey, JSON.stringify(this.idCard.serialize()), { onlyIfNew: creation, ttl: this.ttl });
     }
 }
 exports.ClusterIdCardHandler = ClusterIdCardHandler;

package/dist/lib/cluster/node.d.ts

@@ -143,16 +143,28 @@ declare class ClusterNode {
      *
      * Extracted from `handshake()` unchanged.
      *
-     * @returns false when the caller must stop — the cluster is split and this
-     *          node is shutting down
+     * @throws {cluster.join.full_state_unavailable} on the second failure: the
+     *         cluster is split
      */
     private retryFullState;
+    /**
+     * The ID cards of the other nodes, none of which has this node's IP.
+     *
+     * A card with this IP may be the stale one of a previous run of this node
+     * (a process killed, a container restarted at once): it expires within a
+     * TTL, so it is only a real duplicate if it is still there after that.
+     *
+     * @throws {cluster.join.duplicate_ip} if it is
+     */
+    private discoverNodes;
     /**
      * Discovers other active nodes from the cluster and, if other nodes exist,
      * starts a handshake procedure to sync this node and to make it able to
      * handle new client requests
      *
-     * @return {void}
+     * Rejects with a `cluster.join.*` error, and shuts the node down, when it
+     * cannot join: the node must not go on starting as if it had (a duplicate
+     * IP, a split cluster, `joinTimeout` reached).
      */
     handshake(): Promise<void>;
     /**

package/dist/lib/cluster/node.js

@@ -19,6 +19,39 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -37,19 +70,19 @@ const eventemitter3_1 = __importDefault(require("eventemitter3"));
 const intersection_1 = __importDefault(require("lodash/intersection"));
 const xor_1 = __importDefault(require("lodash/xor"));
 const long_1 = __importDefault(require("long"));
+const kerror = __importStar(require("../kerror"));
+const kuzzleError_1 = require("../kerror/errors/kuzzleError");
 const kuzzleStateEnum_1 = __importDefault(require("../kuzzle/kuzzleStateEnum"));
 const debug_1 = __importDefault(require("../util/debug"));
 const koncordeCompat_1 = require("../util/koncordeCompat");
-// NOSONAR: `Mutex` is deprecated in favour of `withLock`, but the two use
-// incompatible acquisition/TTL formats and must not contend on the same key —
-// every node takes "clusterHandshake" with `Mutex`. Deferred to #2894.
-const mutex_1 = require("../util/mutex"); // NOSONAR
+const distributedLock_1 = require("../util/distributedLock");
 const command_1 = __importDefault(require("./command"));
 const idCardHandler_1 = require("./idCardHandler");
 const publisher_1 = __importDefault(require("./publisher"));
 const state_1 = __importDefault(require("./state"));
 const subscriber_1 = __importDefault(require("./subscriber"));
 const debug = (0, debug_1.default)("kuzzle:cluster:sync");
+const joinError = kerror.wrap("cluster", "join");
 /** Cluster-wide events carrying the collection write locks (see registerEvents) */
 const COLLECTION_LOCK_EVENT = "core:storage:collection:lock";
 const COLLECTION_UNLOCK_EVENT = "core:storage:collection:unlock";
@@ -335,7 +368,7 @@ class ClusterNode {
         //
         // Exit code 1: this is a failure, and a node that exits 0 is not restarted
         // by an `on-failure` policy (#2785).
-        global.kuzzle.shutdown(1);
+        void global.kuzzle.shutdown(1);
     }
     /**
      * Evicts a remote from the list
@@ -391,14 +424,15 @@ class ClusterNode {
             const candidates = this.electShutdownCandidates(splits, idCards);
             if (candidates.includes(this.nodeId)) {
                 this.logger.error("[CLUSTER] Network split detected. This node is outside the cluster: shutting down.");
-                global.kuzzle.shutdown();
+                // A failure, like `evictSelf`'s: exit non-zero (#2785)
+                void global.kuzzle.shutdown(1);
                 return;
             }
         }
         catch (err) {
             this.logger.error("[CLUSTER] Unexpected exception caught during a cluster consistency check. Shutting down...");
             this.logger.error(err instanceof Error ? err.stack : (0, util_1.inspect)(err));
-            global.kuzzle.shutdown();
+            void global.kuzzle.shutdown(1);
         }
     }
     /**
@@ -526,8 +560,8 @@ class ClusterNode {
      *
      * Extracted from `handshake()` unchanged.
      *
-     * @returns false when the caller must stop — the cluster is split and this
-     *          node is shutting down
+     * @throws {cluster.join.full_state_unavailable} on the second failure: the
+     *         cluster is split
      */
     async retryFullState(retried) {
         // Uh oh... no node was able to give us the full state.
@@ -536,9 +570,7 @@ class ClusterNode {
         // means we're probably facing a network split, and we must then shut
         // down.
         if (retried) {
-            this.logger.error("[CLUSTER] Could not connect to discovered cluster nodes (network split detected). Shutting down.");
-            global.kuzzle.shutdown();
-            return false;
+            throw joinError.get("full_state_unavailable");
         }
         // Disposes all subscribers
         for (const subscriber of this.remoteNodes.values()) {
@@ -549,28 +581,59 @@ class ClusterNode {
         const retryDelay = this.heartbeatDelay * 1.5;
         this.logger.warn(`[CLUSTER] Unable to connect to discovered cluster nodes. Retrying in ${retryDelay}ms...`);
         await bluebird_1.default.delay(retryDelay);
-        return true;
+    }
+    /**
+     * The ID cards of the other nodes, none of which has this node's IP.
+     *
+     * A card with this IP may be the stale one of a previous run of this node
+     * (a process killed, a container restarted at once): it expires within a
+     * TTL, so it is only a real duplicate if it is still there after that.
+     *
+     * @throws {cluster.join.duplicate_ip} if it is
+     */
+    async discoverNodes() {
+        const nodes = await this.idCardHandler.getRemoteIdCards();
+        if (!nodes.some((node) => node.ip === this.ip)) {
+            return nodes;
+        }
+        // A heartbeat more than the TTL: the card's last renewal may be in flight
+        const delay = this.idCardHandler.ttl + this.heartbeatDelay;
+        this.logger.warn(`[CLUSTER] Another node has the same IP address as this one (${this.ip}). Waiting ${delay}ms for it to expire, in case it is a stale ID card...`);
+        await bluebird_1.default.delay(delay);
+        const remaining = await this.idCardHandler.getRemoteIdCards();
+        const duplicate = remaining.find((node) => node.ip === this.ip);
+        if (duplicate) {
+            throw joinError.get("duplicate_ip", this.ip, duplicate.id);
+        }
+        return remaining;
     }
     /**
      * Discovers other active nodes from the cluster and, if other nodes exist,
      * starts a handshake procedure to sync this node and to make it able to
      * handle new client requests
      *
-     * @return {void}
+     * Rejects with a `cluster.join.*` error, and shuts the node down, when it
+     * cannot join: the node must not go on starting as if it had (a duplicate
+     * IP, a split cluster, `joinTimeout` reached).
      */
     async handshake() {
-        const handshakeTimeout = setTimeout(() => {
-            this.logger.error(`[CLUSTER] Failed to join the cluster: timed out (joinTimeout: ${this.config.joinTimeout}ms)`);
-            global.kuzzle.shutdown();
-        }, this.config.joinTimeout);
-        // NOSONAR (S1874): same deferral as the import above — every node takes
-        // "clusterHandshake" with `Mutex`, so converting one site to `withLock`
-        // would remove the exclusion it exists for. #2894.
-        const lockOptions = { timeout: this.config.joinTimeout };
-        const mutex = new mutex_1.Mutex("clusterHandshake", lockOptions); // NOSONAR
-        try {
-            await mutex.lock();
-            // Create the ID Key AFTER the handshake mutex is actually locked,
+        let timedOut = false;
+        let handshakeTimeout;
+        const timeout = new Promise((_resolve, reject) => {
+            handshakeTimeout = setTimeout(() => {
+                timedOut = true;
+                reject(joinError.get("timeout", this.config.joinTimeout));
+            }, this.config.joinTimeout);
+        });
+        // No retry limit: the handshakes are taken one at a time, and the
+        // `joinTimeout` above is what gives up on a cluster that never lets this
+        // node in.
+        const join = (0, distributedLock_1.withLock)("clusterHandshake", async () => {
+            // Given up on while waiting for the lock: do not join after all
+            if (timedOut) {
+                return;
+            }
+            // Create the ID Key AFTER the handshake lock is actually taken,
             // to prevent race conditions (other nodes attempting to connect to this
             // node while it's still initializing)
             await this.idCardHandler.createIdCard();
@@ -581,37 +644,29 @@ class ClusterNode {
             await this.startHeartbeat(); // NOSONAR: TD-26
             debug("[CLUSTER] Start heartbeat");
             let retried = false;
-            let fullState = null;
+            let fullState;
             let nodes;
             debug("[CLUSTER] Start retrieving full state..");
             do {
-                nodes = await this.idCardHandler.getRemoteIdCards();
+                nodes = await this.discoverNodes();
                 debug("[CLUSTER] %s remote nodes discovered", nodes.length);
                 // No other nodes detected = no handshake required
                 if (nodes.length === 0) {
                     this.trackActivity(this.nodeId, this.ip, nodeActivityEnum.ADDED);
                     return;
                 }
-                // Verify that no other node share the same IP address as this one
-                const duplicate = nodes.filter((node) => node.ip === this.ip);
-                if (duplicate.length > 0) {
-                    this.logger.error(`[CLUSTER] Another node share the same IP address as this one (${this.ip}): ${duplicate[0]?.id}. Shutting down.`);
-                    global.kuzzle.shutdown();
-                    return;
-                }
                 await this.subscribeToNodes(nodes);
                 debug("[CLUSTER] Successfully subscribed to nodes");
                 fullState = await this.command.getFullState(nodes);
                 if (fullState === null) {
-                    if (!(await this.retryFullState(retried))) {
-                        return;
-                    }
+                    // Sequential by design: the retry waits for a heartbeat round
+                    await this.retryFullState(retried); // NOSONAR
                     retried = true;
                 }
             } while (fullState === null);
             debug("[CLUSTER] Fullstate retrieved, loading into node..");
             await this.fullState.loadFullState(fullState); // NOSONAR: TD-26
-            this.activity = fullState.activity ? fullState.activity : this.activity;
+            this.activity = fullState.activity ?? this.activity;
             debug("[CLUSTER] Fullstate loaded.");
             const handshakeResponses = await this.command.broadcastHandshake(nodes);
             debug("[CLUSTER] Successful handshakes with other nodes.");
@@ -625,10 +680,27 @@ class ClusterNode {
                 ? `[CLUSTER] Successfully joined the cluster as "${this.nodeId}" (process node id: ${global.nodeId}).`
                 : `[CLUSTER] Successfully joined the cluster as "${this.nodeId}".`);
             this.trackActivity(this.nodeId, this.ip, nodeActivityEnum.ADDED);
+        }, { retryAttempts: Number.POSITIVE_INFINITY });
+        try {
+            // A timeout does not stop `join`: the node it settles later is shutting
+            // down already
+            await Promise.race([join, timeout]);
+        }
+        catch (error) {
+            // The handshake cannot be stopped where it is: it may still introduce
+            // this node to the others. Leave the cluster rather than stay in it
+            // half-joined, whatever the application does with the failed startup.
+            if (error instanceof distributedLock_1.MutexLockLostError) {
+                this.logger.error("[CLUSTER] Lost the handshake lock while joining the cluster. Shutting down.");
+            }
+            if (error instanceof distributedLock_1.MutexLockLostError ||
+                (error instanceof kuzzleError_1.KuzzleError && error.id.startsWith("cluster.join."))) {
+                void global.kuzzle.shutdown(1);
+            }
+            throw error;
         }
         finally {
             clearTimeout(handshakeTimeout);
-            await mutex.unlock();
         }
     }
     /**

package/dist/lib/cluster/protobuf/command.proto

@@ -1,5 +1,6 @@
 // messages definition for issuing node commands and receiving the
 // corresponding responses
+// /!\ Every singular scalar field is "optional": see sync.proto's header.
 package command;
 
 syntax="proto3";
@@ -10,36 +11,36 @@ import "sync.proto";
 message FullStateResponse {
   // Describes the nodes that have subscribed to a given realtime room
   message NodeSubscribers {
-    string nodeId = 1;
-    uint32 subscribers = 2;
-    uint64 messageId = 3;
+    optional string nodeId = 1;
+    optional uint32 subscribers = 2;
+    optional uint64 messageId = 3;
   }
 
   // Describes a realtime room
   message RealtimeRoom {
-    string roomId = 1;
-    string index = 2;
-    string collection = 3;
+    optional string roomId = 1;
+    optional string index = 2;
+    optional string collection = 3;
 
     // array of polymorphic objects, needs to be stringified beforehand until
     // this PR is merged: https://github.com/protobufjs/protobuf.js/pull/1495
-    string filters = 4;
+    optional string filters = 4;
 
     repeated NodeSubscribers nodes = 5;
   }
 
   // Describes a cluster activity event
   message Activity {
-    string id = 1;
-    string date = 2;
-    string address = 3;
-    uint32 event = 4;
+    optional string id = 1;
+    optional string date = 2;
+    optional string address = 3;
+    optional uint32 event = 4;
     optional string reason = 5;
   }
 
   message NodeState {
-    string id = 1;
-    uint64 lastMessageId = 2;
+    optional string id = 1;
+    optional uint64 lastMessageId = 2;
   }
 
   repeated RealtimeRoom rooms = 1;
@@ -50,30 +51,30 @@ message FullStateResponse {
 
 // Handshake Request
 message HandshakeRequest {
-  string nodeId = 1;
-  uint64 lastMessageId = 2;
-  string ip = 3;
+  optional string nodeId = 1;
+  optional uint64 lastMessageId = 2;
+  optional string ip = 3;
 }
 
 // Handshake response
 message HandshakeResponse {
-  bool added = 1;
-  uint64 lastMessageId = 2;
+  optional bool added = 1;
+  optional uint64 lastMessageId = 2;
 }
 
 // Retransmit request: the messages a subscriber missed from this node's sync
 // stream, by message id, both ends included (#2785)
 message RetransmitRequest {
-  uint64 from = 1;
-  uint64 to = 2;
+  optional uint64 from = 1;
+  optional uint64 to = 2;
 }
 
 // Retransmit response: the missed messages, in order, exactly as they were
 // published (topic + encoded sync message)
 message RetransmitResponse {
   message Frame {
-    string topic = 1;
-    bytes data = 2;
+    optional string topic = 1;
+    optional bytes data = 2;
   }
 
   repeated Frame frames = 1;