kuzzle 2.60.0-beta.2 → 2.60.0-beta.3

package/dist/bin/start-kuzzle-server

@@ -69,6 +69,7 @@ const helpers_1 = require("yargs/helpers");
  * own `main`. Keep it that way.
  */
 const index_1 = require("../index");
+const exitOnFailure_1 = require("../lib/kuzzle/exitOnFailure");
 /**
  * Parses one of the JSON files the command line points at. The type parameter
  * is a *claim about the file*, not a check: `JSON.parse` answers `any` and this
@@ -226,7 +227,7 @@ const run = async () => {
     }
     catch (error) {
         console.error(`[x] [ERROR] ${error instanceof Error ? error.stack : String(error)}`);
-        process.exit(1);
+        await (0, exitOnFailure_1.exitOnFailure)();
     }
 };
 run();

package/dist/lib/cluster/idCardHandler.d.ts

@@ -87,6 +87,10 @@ export declare class ClusterIdCardHandler {
     private disposed;
     private readonly logger;
     constructor(node: ClusterNode);
+    /**
+     * How long an ID card outlives the last renewal of its node, in ms
+     */
+    get ttl(): number;
     /**
      * Generates and reserves a unique ID for this node instance.
      * Makes sure that the ID is not already taken by another node instance.

package/dist/lib/cluster/idCardHandler.js

@@ -94,6 +94,12 @@ class ClusterIdCardHandler {
         this.ip = node.ip;
         this.refreshDelay = node.heartbeatDelay;
     }
+    /**
+     * How long an ID card outlives the last renewal of its node, in ms
+     */
+    get ttl() {
+        return this.refreshDelay * this.refreshMultiplier;
+    }
     /**
      * Generates and reserves a unique ID for this node instance.
      * Makes sure that the ID is not already taken by another node instance.
@@ -139,7 +145,8 @@ class ClusterIdCardHandler {
         const refreshWorker = this.constructWorker(`${__dirname}/workers/IDCardRenewer${(0, node_path_1.extname)(__filename)}`);
         this.refreshWorker = refreshWorker;
         refreshWorker.on("message", async (message) => {
-            if (message.error) {
+            // A renewal racing the disposal finds the card deleted: not a failure
+            if (message.error && !this.disposed) {
                 await this.node.evictSelf(message.error);
             }
         });
@@ -228,6 +235,20 @@ class ClusterIdCardHandler {
                 // It could happens that the worker has been killed before the dispose causing send to fail
             }
         }
+        clearInterval(this.refreshTimer ?? undefined);
+        this.refreshTimer = null;
+        // The worker deletes the card too, but nothing waits for it: the process
+        // usually exits first, and the card then outlives it by a TTL, long
+        // enough for a node restarted on this IP to take it for a live duplicate
+        if (this.nodeIdKey !== null) {
+            try {
+                await global.kuzzle.ask("core:cache:internal:del", this.nodeIdKey);
+                await global.kuzzle.ask("core:cache:internal:execute", "srem", REDIS_ID_CARDS_INDEX, this.nodeIdKey);
+            }
+            catch (error) {
+                this.logger.warn(`Could not delete the ID card ${this.nodeIdKey}: it expires in ${this.ttl}ms (${error instanceof Error ? error.message : String(error)})`);
+            }
+        }
     }
     /**
      * Retrieves the ID cards from other nodes
@@ -309,7 +330,7 @@ class ClusterIdCardHandler {
         if (!this.idCard) {
             return false;
         }
-        return global.kuzzle.ask("core:cache:internal:store", this.nodeIdKey, JSON.stringify(this.idCard.serialize()), { onlyIfNew: creation, ttl: this.refreshDelay * this.refreshMultiplier });
+        return global.kuzzle.ask("core:cache:internal:store", this.nodeIdKey, JSON.stringify(this.idCard.serialize()), { onlyIfNew: creation, ttl: this.ttl });
     }
 }
 exports.ClusterIdCardHandler = ClusterIdCardHandler;

package/dist/lib/cluster/node.d.ts

@@ -143,16 +143,28 @@ declare class ClusterNode {
      *
      * Extracted from `handshake()` unchanged.
      *
-     * @returns false when the caller must stop — the cluster is split and this
-     *          node is shutting down
+     * @throws {cluster.join.full_state_unavailable} on the second failure: the
+     *         cluster is split
      */
     private retryFullState;
+    /**
+     * The ID cards of the other nodes, none of which has this node's IP.
+     *
+     * A card with this IP may be the stale one of a previous run of this node
+     * (a process killed, a container restarted at once): it expires within a
+     * TTL, so it is only a real duplicate if it is still there after that.
+     *
+     * @throws {cluster.join.duplicate_ip} if it is
+     */
+    private discoverNodes;
     /**
      * Discovers other active nodes from the cluster and, if other nodes exist,
      * starts a handshake procedure to sync this node and to make it able to
      * handle new client requests
      *
-     * @return {void}
+     * Rejects with a `cluster.join.*` error, and shuts the node down, when it
+     * cannot join: the node must not go on starting as if it had (a duplicate
+     * IP, a split cluster, `joinTimeout` reached).
      */
     handshake(): Promise<void>;
     /**

package/dist/lib/cluster/node.js

@@ -19,6 +19,39 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -37,6 +70,8 @@ const eventemitter3_1 = __importDefault(require("eventemitter3"));
 const intersection_1 = __importDefault(require("lodash/intersection"));
 const xor_1 = __importDefault(require("lodash/xor"));
 const long_1 = __importDefault(require("long"));
+const kerror = __importStar(require("../kerror"));
+const kuzzleError_1 = require("../kerror/errors/kuzzleError");
 const kuzzleStateEnum_1 = __importDefault(require("../kuzzle/kuzzleStateEnum"));
 const debug_1 = __importDefault(require("../util/debug"));
 const koncordeCompat_1 = require("../util/koncordeCompat");
@@ -47,6 +82,7 @@ const publisher_1 = __importDefault(require("./publisher"));
 const state_1 = __importDefault(require("./state"));
 const subscriber_1 = __importDefault(require("./subscriber"));
 const debug = (0, debug_1.default)("kuzzle:cluster:sync");
+const joinError = kerror.wrap("cluster", "join");
 /** Cluster-wide events carrying the collection write locks (see registerEvents) */
 const COLLECTION_LOCK_EVENT = "core:storage:collection:lock";
 const COLLECTION_UNLOCK_EVENT = "core:storage:collection:unlock";
@@ -332,7 +368,7 @@ class ClusterNode {
         //
         // Exit code 1: this is a failure, and a node that exits 0 is not restarted
         // by an `on-failure` policy (#2785).
-        global.kuzzle.shutdown(1);
+        void global.kuzzle.shutdown(1);
     }
     /**
      * Evicts a remote from the list
@@ -388,14 +424,15 @@ class ClusterNode {
             const candidates = this.electShutdownCandidates(splits, idCards);
             if (candidates.includes(this.nodeId)) {
                 this.logger.error("[CLUSTER] Network split detected. This node is outside the cluster: shutting down.");
-                global.kuzzle.shutdown();
+                // A failure, like `evictSelf`'s: exit non-zero (#2785)
+                void global.kuzzle.shutdown(1);
                 return;
             }
         }
         catch (err) {
             this.logger.error("[CLUSTER] Unexpected exception caught during a cluster consistency check. Shutting down...");
             this.logger.error(err instanceof Error ? err.stack : (0, util_1.inspect)(err));
-            global.kuzzle.shutdown();
+            void global.kuzzle.shutdown(1);
         }
     }
     /**
@@ -523,8 +560,8 @@ class ClusterNode {
      *
      * Extracted from `handshake()` unchanged.
      *
-     * @returns false when the caller must stop — the cluster is split and this
-     *          node is shutting down
+     * @throws {cluster.join.full_state_unavailable} on the second failure: the
+     *         cluster is split
      */
     async retryFullState(retried) {
         // Uh oh... no node was able to give us the full state.
@@ -533,9 +570,7 @@ class ClusterNode {
         // means we're probably facing a network split, and we must then shut
         // down.
         if (retried) {
-            this.logger.error("[CLUSTER] Could not connect to discovered cluster nodes (network split detected). Shutting down.");
-            global.kuzzle.shutdown();
-            return false;
+            throw joinError.get("full_state_unavailable");
         }
         // Disposes all subscribers
         for (const subscriber of this.remoteNodes.values()) {
@@ -546,81 +581,110 @@ class ClusterNode {
         const retryDelay = this.heartbeatDelay * 1.5;
         this.logger.warn(`[CLUSTER] Unable to connect to discovered cluster nodes. Retrying in ${retryDelay}ms...`);
         await bluebird_1.default.delay(retryDelay);
-        return true;
+    }
+    /**
+     * The ID cards of the other nodes, none of which has this node's IP.
+     *
+     * A card with this IP may be the stale one of a previous run of this node
+     * (a process killed, a container restarted at once): it expires within a
+     * TTL, so it is only a real duplicate if it is still there after that.
+     *
+     * @throws {cluster.join.duplicate_ip} if it is
+     */
+    async discoverNodes() {
+        const nodes = await this.idCardHandler.getRemoteIdCards();
+        if (!nodes.some((node) => node.ip === this.ip)) {
+            return nodes;
+        }
+        // A heartbeat more than the TTL: the card's last renewal may be in flight
+        const delay = this.idCardHandler.ttl + this.heartbeatDelay;
+        this.logger.warn(`[CLUSTER] Another node has the same IP address as this one (${this.ip}). Waiting ${delay}ms for it to expire, in case it is a stale ID card...`);
+        await bluebird_1.default.delay(delay);
+        const remaining = await this.idCardHandler.getRemoteIdCards();
+        const duplicate = remaining.find((node) => node.ip === this.ip);
+        if (duplicate) {
+            throw joinError.get("duplicate_ip", this.ip, duplicate.id);
+        }
+        return remaining;
     }
     /**
      * Discovers other active nodes from the cluster and, if other nodes exist,
      * starts a handshake procedure to sync this node and to make it able to
      * handle new client requests
      *
-     * @return {void}
+     * Rejects with a `cluster.join.*` error, and shuts the node down, when it
+     * cannot join: the node must not go on starting as if it had (a duplicate
+     * IP, a split cluster, `joinTimeout` reached).
      */
     async handshake() {
-        const handshakeTimeout = setTimeout(() => {
-            this.logger.error(`[CLUSTER] Failed to join the cluster: timed out (joinTimeout: ${this.config.joinTimeout}ms)`);
-            global.kuzzle.shutdown();
-        }, this.config.joinTimeout);
+        let timedOut = false;
+        let handshakeTimeout;
+        const timeout = new Promise((_resolve, reject) => {
+            handshakeTimeout = setTimeout(() => {
+                timedOut = true;
+                reject(joinError.get("timeout", this.config.joinTimeout));
+            }, this.config.joinTimeout);
+        });
         // No retry limit: the handshakes are taken one at a time, and the
         // `joinTimeout` above is what gives up on a cluster that never lets this
         // node in.
+        const join = (0, distributedLock_1.withLock)("clusterHandshake", async () => {
+            // Given up on while waiting for the lock: do not join after all
+            if (timedOut) {
+                return;
+            }
+            // Create the ID Key AFTER the handshake lock is actually taken,
+            // to prevent race conditions (other nodes attempting to connect to this
+            // node while it's still initializing)
+            await this.idCardHandler.createIdCard();
+            debug("[CLUSTER] ID Card created");
+            const nodeId = this.idCardHandler.nodeId;
+            (0, assert_1.default)(nodeId !== null, "[CLUSTER] createIdCard() returned without reserving a node id");
+            this.nodeId = nodeId;
+            await this.startHeartbeat(); // NOSONAR: TD-26
+            debug("[CLUSTER] Start heartbeat");
+            let retried = false;
+            let fullState;
+            let nodes;
+            debug("[CLUSTER] Start retrieving full state..");
+            do {
+                nodes = await this.discoverNodes();
+                debug("[CLUSTER] %s remote nodes discovered", nodes.length);
+                // No other nodes detected = no handshake required
+                if (nodes.length === 0) {
+                    this.trackActivity(this.nodeId, this.ip, nodeActivityEnum.ADDED);
+                    return;
+                }
+                await this.subscribeToNodes(nodes);
+                debug("[CLUSTER] Successfully subscribed to nodes");
+                fullState = await this.command.getFullState(nodes);
+                if (fullState === null) {
+                    // Sequential by design: the retry waits for a heartbeat round
+                    await this.retryFullState(retried); // NOSONAR
+                    retried = true;
+                }
+            } while (fullState === null);
+            debug("[CLUSTER] Fullstate retrieved, loading into node..");
+            await this.fullState.loadFullState(fullState); // NOSONAR: TD-26
+            this.activity = fullState.activity ?? this.activity;
+            debug("[CLUSTER] Fullstate loaded.");
+            const handshakeResponses = await this.command.broadcastHandshake(nodes);
+            debug("[CLUSTER] Successful handshakes with other nodes.");
+            await this.syncWithPeers(handshakeResponses, fullState);
+            // `createIdCard()` adopts `global.nodeId`, so these are normally the same
+            // string and the line reads as one name (TD-59, #2764). They still differ
+            // in the two cases the adoption cannot cover — a stale IdCard holding the
+            // key, and a Kuzzle running without a Backend — and those are exactly the
+            // cases where a reader needs to be told, so the line says both.
+            this.logger.info(global.nodeId && global.nodeId !== this.nodeId
+                ? `[CLUSTER] Successfully joined the cluster as "${this.nodeId}" (process node id: ${global.nodeId}).`
+                : `[CLUSTER] Successfully joined the cluster as "${this.nodeId}".`);
+            this.trackActivity(this.nodeId, this.ip, nodeActivityEnum.ADDED);
+        }, { retryAttempts: Number.POSITIVE_INFINITY });
         try {
-            await (0, distributedLock_1.withLock)("clusterHandshake", async () => {
-                // Create the ID Key AFTER the handshake lock is actually taken,
-                // to prevent race conditions (other nodes attempting to connect to this
-                // node while it's still initializing)
-                await this.idCardHandler.createIdCard();
-                debug("[CLUSTER] ID Card created");
-                const nodeId = this.idCardHandler.nodeId;
-                (0, assert_1.default)(nodeId !== null, "[CLUSTER] createIdCard() returned without reserving a node id");
-                this.nodeId = nodeId;
-                await this.startHeartbeat(); // NOSONAR: TD-26
-                debug("[CLUSTER] Start heartbeat");
-                let retried = false;
-                let fullState;
-                let nodes;
-                debug("[CLUSTER] Start retrieving full state..");
-                do {
-                    nodes = await this.idCardHandler.getRemoteIdCards();
-                    debug("[CLUSTER] %s remote nodes discovered", nodes.length);
-                    // No other nodes detected = no handshake required
-                    if (nodes.length === 0) {
-                        this.trackActivity(this.nodeId, this.ip, nodeActivityEnum.ADDED);
-                        return;
-                    }
-                    // Verify that no other node share the same IP address as this one
-                    const duplicate = nodes.filter((node) => node.ip === this.ip);
-                    if (duplicate.length > 0) {
-                        this.logger.error(`[CLUSTER] Another node share the same IP address as this one (${this.ip}): ${duplicate[0]?.id}. Shutting down.`);
-                        global.kuzzle.shutdown();
-                        return;
-                    }
-                    await this.subscribeToNodes(nodes);
-                    debug("[CLUSTER] Successfully subscribed to nodes");
-                    fullState = await this.command.getFullState(nodes);
-                    if (fullState === null) {
-                        if (!(await this.retryFullState(retried))) {
-                            return;
-                        }
-                        retried = true;
-                    }
-                } while (fullState === null);
-                debug("[CLUSTER] Fullstate retrieved, loading into node..");
-                await this.fullState.loadFullState(fullState); // NOSONAR: TD-26
-                this.activity = fullState.activity ?? this.activity;
-                debug("[CLUSTER] Fullstate loaded.");
-                const handshakeResponses = await this.command.broadcastHandshake(nodes);
-                debug("[CLUSTER] Successful handshakes with other nodes.");
-                await this.syncWithPeers(handshakeResponses, fullState);
-                // `createIdCard()` adopts `global.nodeId`, so these are normally the same
-                // string and the line reads as one name (TD-59, #2764). They still differ
-                // in the two cases the adoption cannot cover — a stale IdCard holding the
-                // key, and a Kuzzle running without a Backend — and those are exactly the
-                // cases where a reader needs to be told, so the line says both.
-                this.logger.info(global.nodeId && global.nodeId !== this.nodeId
-                    ? `[CLUSTER] Successfully joined the cluster as "${this.nodeId}" (process node id: ${global.nodeId}).`
-                    : `[CLUSTER] Successfully joined the cluster as "${this.nodeId}".`);
-                this.trackActivity(this.nodeId, this.ip, nodeActivityEnum.ADDED);
-            }, { retryAttempts: Number.POSITIVE_INFINITY });
+            // A timeout does not stop `join`: the node it settles later is shutting
+            // down already
+            await Promise.race([join, timeout]);
         }
         catch (error) {
             // The handshake cannot be stopped where it is: it may still introduce
@@ -628,7 +692,10 @@ class ClusterNode {
             // half-joined, whatever the application does with the failed startup.
             if (error instanceof distributedLock_1.MutexLockLostError) {
                 this.logger.error("[CLUSTER] Lost the handshake lock while joining the cluster. Shutting down.");
-                void global.kuzzle.shutdown();
+            }
+            if (error instanceof distributedLock_1.MutexLockLostError ||
+                (error instanceof kuzzleError_1.KuzzleError && error.id.startsWith("cluster.join."))) {
+                void global.kuzzle.shutdown(1);
             }
             throw error;
         }

package/dist/lib/cluster/workers/IDCardRenewer.d.ts

@@ -38,6 +38,12 @@ declare class IDCardRenewer {
     init(config: IDCardRenewerConfig): Promise<void>;
     initRedis(config?: RedisServiceConfig, name?: string): Promise<void>;
     renewIDCard(): Promise<void>;
+    /**
+     * Disposes the worker and tells the parent why, once: the renewals queued
+     * behind a stalled redis all fail together, and each report evicts the node
+     * again (#3017)
+     */
+    private reportFailure;
     dispose(): Promise<void>;
 }
 export { IDCardRenewer };

package/dist/lib/cluster/workers/IDCardRenewer.js

@@ -104,18 +104,24 @@ class IDCardRenewer {
             // Unable to refresh the key in time before it expires
             // => this node is too slow, we need to remove it from the cluster
             if (refreshed === 0) {
-                await this.dispose();
-                process.send?.({
-                    error: "Node too slow: ID card expired",
-                });
+                await this.reportFailure("Node too slow: ID card expired");
             }
         }
         catch (error) {
-            await this.dispose();
-            process.send?.({
-                error: `Failed to refresh ID Card: ${messageOf(error)}`,
-            });
+            await this.reportFailure(`Failed to refresh ID Card: ${messageOf(error)}`);
+        }
+    }
+    /**
+     * Disposes the worker and tells the parent why, once: the renewals queued
+     * behind a stalled redis all fail together, and each report evicts the node
+     * again (#3017)
+     */
+    async reportFailure(error) {
+        if (this.disposed) {
+            return;
         }
+        await this.dispose();
+        process.send?.({ error });
     }
     async dispose() {
         if (this.disposed) {

package/dist/lib/core/auth/passportWrapper.js

@@ -99,7 +99,9 @@ class PassportWrapper {
                     fail(err);
                 }
                 else if (!user) {
-                    const message = typeof info === "object" && "message" in info
+                    const message = 
+                    // `typeof null` is "object" too: a strategy may fail with no info
+                    typeof info === "object" && info !== null && "message" in info
                         ? info.message
                         : undefined;
                     reject(kerror.get("plugin", "strategy", "missing_user", message));

package/dist/lib/kerror/codes/8-cluster.json

@@ -11,6 +11,29 @@
                     "class": "InternalError"
                 }
             }
+        },
+        "join": {
+            "code": 1,
+            "errors": {
+                "duplicate_ip": {
+                    "description": "Another node of the cluster has the same IP address as the one joining it. The joining node does not start.",
+                    "code": 1,
+                    "message": "Another node shares the same IP address as this one (%s): %s.",
+                    "class": "InternalError"
+                },
+                "full_state_unavailable": {
+                    "description": "None of the cluster nodes discovered could give the joining node the cluster state, twice in a row (network split). The joining node does not start.",
+                    "code": 2,
+                    "message": "Could not connect to the discovered cluster nodes (network split detected).",
+                    "class": "InternalError"
+                },
+                "timeout": {
+                    "description": "The node did not join the cluster within the configured cluster.joinTimeout. It does not start.",
+                    "code": 3,
+                    "message": "Failed to join the cluster: timed out (joinTimeout: %sms).",
+                    "class": "InternalError"
+                }
+            }
         }
     }
 }

package/dist/lib/kuzzle/exitOnFailure.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Ends the process with exit code 1, for a start script whose Kuzzle failed
+ * to start or to run.
+ *
+ * A bare `process.exit(1)` cuts Kuzzle's graceful shutdown short: the locks
+ * the node holds and its ID card stay in Redis until they expire (#3022).
+ * Once a Kuzzle instance exists, this goes through `shutdown(1)` instead,
+ * which joins the one a failed start already began (#3020). Before that,
+ * there is nothing to release.
+ *
+ * Internal to Kuzzle: not exported by the package.
+ */
+export declare function exitOnFailure(): Promise<void>;

package/dist/lib/kuzzle/exitOnFailure.js

@@ -0,0 +1,60 @@
+"use strict";
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.exitOnFailure = exitOnFailure;
+const node_util_1 = require("node:util");
+/**
+ * Ends the process with exit code 1, for a start script whose Kuzzle failed
+ * to start or to run.
+ *
+ * A bare `process.exit(1)` cuts Kuzzle's graceful shutdown short: the locks
+ * the node holds and its ID card stay in Redis until they expire (#3022).
+ * Once a Kuzzle instance exists, this goes through `shutdown(1)` instead,
+ * which joins the one a failed start already began (#3020). Before that,
+ * there is nothing to release.
+ *
+ * Internal to Kuzzle: not exported by the package.
+ */
+async function exitOnFailure() {
+    const kuzzle = currentKuzzle();
+    if (kuzzle) {
+        try {
+            // Ends with `process.exit`
+            await kuzzle.shutdown(1);
+            return;
+        }
+        catch (error) {
+            kuzzle.log.error(`Unable to shut down: ${error instanceof Error ? error.message : (0, node_util_1.inspect)(error)}`);
+        }
+    }
+    process.exit(1);
+}
+function currentKuzzle() {
+    try {
+        return global.kuzzle;
+    }
+    catch {
+        // The `global.kuzzle` getter throws until an instance is built
+        return undefined;
+    }
+}
+//# sourceMappingURL=exitOnFailure.js.map

package/dist/lib/kuzzle/kuzzle.d.ts

@@ -17,6 +17,12 @@ import { Logger } from "./Logger";
 declare class Kuzzle extends KuzzleEventEmitter {
     config: IKuzzleConfiguration;
     private _state;
+    /** The running shutdown, which later `shutdown()` calls wait for (#3017) */
+    private shutdownPromise;
+    /** The highest exit code asked for by the `shutdown()` calls so far */
+    private shutdownExitCode;
+    /** Ends a failure's shutdown that does not finish in time */
+    private shutdownWatchdog;
     log: Logger;
     /** Read by `PluginsManager` to locate the plugins directories. */
     rootPath: string;
@@ -117,6 +123,7 @@ declare class Kuzzle extends KuzzleEventEmitter {
      * @returns {Promise}
      */
     shutdown(exitCode?: number): Promise<void>;
+    private runShutdown;
     /**
      * Execute multiple handlers only once on any given environment
      *

package/dist/lib/kuzzle/kuzzle.js

@@ -93,6 +93,12 @@ const Logger_1 = require("./Logger");
 const vault_1 = __importDefault(require("./vault"));
 const name_generator_1 = require("../util/name-generator");
 let _kuzzle = null;
+/**
+ * How long a shutdown caused by a failure may run before the process exits
+ * anyway: a `kuzzle:shutdown` pipe or a lock release that never settles must
+ * not keep a failed node alive, which a bare `process.exit(1)` never did
+ */
+const FAILURE_SHUTDOWN_TIMEOUT = 30000;
 Reflect.defineProperty(global, "kuzzle", {
     configurable: true,
     enumerable: false,
@@ -113,6 +119,12 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
     constructor(config) {
         super(config.plugins.common.maxConcurrentPipes, config.plugins.common.pipesBufferSize);
         this._state = kuzzleStateEnum_1.default.STARTING;
+        /** The running shutdown, which later `shutdown()` calls wait for (#3017) */
+        this.shutdownPromise = null;
+        /** The highest exit code asked for by the `shutdown()` calls so far */
+        this.shutdownExitCode = 0;
+        /** Ends a failure's shutdown that does not finish in time */
+        this.shutdownWatchdog = null;
         global.kuzzle = this;
         this._state = kuzzleStateEnum_1.default.STARTING;
         this.config = config;
@@ -204,6 +216,16 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
         }
         catch (error) {
             this.log.error(`[X] Cannot start Kuzzle ${this.version}: ${error instanceof Error ? error.message : (0, node_util_1.inspect)(error)}`);
+            // A failed start cannot be retried (a second Kuzzle instance cannot be
+            // built), and what it did start keeps the process alive: in production,
+            // a node with no listener and an ID card still renewed (#3020). Not
+            // awaited, so the caller still gets the error before the process exits.
+            this.shutdown(1).catch((shutdownError) => {
+                this.log.error(`Unable to shut down after a failed start: ${shutdownError instanceof Error
+                    ? shutdownError.message
+                    : (0, node_util_1.inspect)(shutdownError)}`);
+                process.exit(1);
+            });
             throw error;
         }
     }
@@ -231,6 +253,22 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
      * @returns {Promise}
      */
     async shutdown(exitCode = 0) {
+        // Several sources can ask for it at once (a signal, a cluster eviction,
+        // the `exit` handler that `process.exit` below fires): they share the one
+        // shutdown, and a failure reported by any of them keeps the exit non-zero
+        this.shutdownExitCode = Math.max(this.shutdownExitCode, exitCode);
+        // Not for a requested shutdown: it waits for the requests still running,
+        // however long they take
+        if (exitCode !== 0) {
+            this.shutdownWatchdog ?? (this.shutdownWatchdog = setTimeout(() => {
+                this.log.error(`[shutdown] Not finished after ${FAILURE_SHUTDOWN_TIMEOUT}ms: exiting now`);
+                process.exit(this.shutdownExitCode);
+            }, FAILURE_SHUTDOWN_TIMEOUT));
+        }
+        this.shutdownPromise ?? (this.shutdownPromise = this.runShutdown());
+        return this.shutdownPromise;
+    }
+    async runShutdown() {
         this._state = kuzzleStateEnum_1.default.SHUTTING_DOWN;
         this.log.info("Initiating shutdown...");
         // Ask the network layer to stop accepting new request
@@ -257,7 +295,10 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
         // `pluginsManager?.application?.log?.flush?.()` was a silent no-op and the
         // application's buffered logs were dropped on shutdown (TD-51).
         await this?.pluginsManager?.application?.instance?.log?.flush?.();
-        process.exit(exitCode);
+        if (this.shutdownWatchdog !== null) {
+            clearTimeout(this.shutdownWatchdog);
+        }
+        process.exit(this.shutdownExitCode);
     }
     /**
      * Execute multiple handlers only once on any given environment
@@ -608,7 +649,9 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
                 // nothing to do with that error
             }
         }
-        await this.shutdown();
+        // Only abnormal terminations get here: exit non-zero, so that an
+        // `on-failure` restart policy brings the node back (#3020)
+        await this.shutdown(1);
     }
 }
 exports.Kuzzle = Kuzzle;

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "kuzzle",
     "author": "The Kuzzle Team <support@kuzzle.io>",
-    "version": "2.60.0-beta.2",
+    "version": "2.60.0-beta.3",
     "description": "Kuzzle is an open-source solution that handles all the data management through a secured API, with a large choice of protocols.",
     "scripts": {
         "build": "rm -Rf ./dist && tsc && node -r ts-node/register/transpile-only ./bin/copy-binaries.ts",

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "kuzzle",
   "author": "The Kuzzle Team <support@kuzzle.io>",
-  "version": "2.60.0-beta.2",
+  "version": "2.60.0-beta.3",
   "description": "Kuzzle is an open-source solution that handles all the data management through a secured API, with a large choice of protocols.",
   "scripts": {
     "build": "rm -Rf ./dist && tsc && node -r ts-node/register/transpile-only ./bin/copy-binaries.ts",