kuzzle 2.60.0-beta.3 → 2.60.0-beta.4

package/dist/lib/api/controllers/documentController.d.ts

@@ -77,7 +77,7 @@ declare class DocumentController extends NativeController {
         errors: any;
         successes: any;
     }>;
-    export(request: KuzzleRequest): Promise<import("../../..").HttpStream>;
+    export(request: KuzzleRequest): Promise<import("../../types").HttpStream>;
     /**
      * @param {Request} request
      * @returns {Promise<Object>}

package/dist/lib/cluster/command.d.ts

@@ -21,6 +21,8 @@ type CommandingNode = {
     };
     ip: string;
     nodeId: string;
+    /** Whether `nodeId` can be read: false until the handshake sets it. */
+    hasNodeId: boolean;
     activity: Activity[];
     /** Only `serialize()`: the command layer never mutates the state. */
     fullState: {
@@ -102,6 +104,11 @@ declare class ClusterCommand {
      * This method can (and should) never return a rejected promise.
      */
     listen(): Promise<void>;
+    /**
+     * Answers one request. Every request gets exactly one reply: REP/REQ
+     * requires it, and a request left unanswered wedges the peer's socket.
+     */
+    private answer;
     /**
      * Closes opened sockets and marks this server as closed.
      */

package/dist/lib/cluster/command.js

@@ -159,32 +159,9 @@ class ClusterCommand {
         while (this.state === stateEnum.RUNNING) {
             try {
                 const [type, data] = await this.server.receive();
-                // A REQ frame carries a topic and, for a handshake, a payload. A
-                // request missing either is answered with DISCARDED, which is what
-                // this socket owes every REQ — the two frames were read unchecked.
-                switch (type?.toString()) {
-                    case commandMessages_1.commandTopic.FULLSTATE:
-                        await this.sendFullState();
-                        break;
-                    case commandMessages_1.commandTopic.HANDSHAKE:
-                        if (data === undefined) {
-                            await this.server.send([commandMessages_1.commandTopic.DISCARDED, null]);
-                            break;
-                        }
-                        await this.handleHandshake(data);
-                        break;
-                    case commandMessages_1.commandTopic.RETRANSMIT:
-                        if (data === undefined) {
-                            await this.server.send([commandMessages_1.commandTopic.DISCARDED, null]);
-                            break;
-                        }
-                        await this.handleRetransmit(data);
-                        break;
-                    default:
-                        // REP/REQ sockets expect a reply to each request made, so we have
-                        // to send some kind of response on an invalid request received
-                        await this.server.send([commandMessages_1.commandTopic.DISCARDED, null]);
-                }
+                // NOSONAR (S9382): a REP socket strictly alternates receive and send,
+                // so a request must be answered before the next one can be read.
+                await this.answer(type, data); // NOSONAR
             }
             catch (e) {
                 if (this.state !== stateEnum.CLOSED) {
@@ -193,6 +170,47 @@ class ClusterCommand {
             }
         }
     }
+    /**
+     * Answers one request. Every request gets exactly one reply: REP/REQ
+     * requires it, and a request left unanswered wedges the peer's socket.
+     */
+    async answer(type, data) {
+        // The socket is bound before the handshake sets this node's id, and a
+        // peer still talking to a dead node on this IP can reach it (#3031). A
+        // node with no id yet has no state to serve and cannot take a peer in:
+        // every request is answered DISCARDED, which the requester treats as no
+        // answer.
+        if (!this.node.hasNodeId) {
+            await this.server.send([commandMessages_1.commandTopic.DISCARDED, null]);
+            return;
+        }
+        // A REQ frame carries a topic and, for a handshake, a payload. A request
+        // missing either is answered with DISCARDED, which is what this socket
+        // owes every REQ — the two frames were read unchecked.
+        switch (type?.toString()) {
+            case commandMessages_1.commandTopic.FULLSTATE:
+                await this.sendFullState();
+                break;
+            case commandMessages_1.commandTopic.HANDSHAKE:
+                if (data === undefined) {
+                    await this.server.send([commandMessages_1.commandTopic.DISCARDED, null]);
+                    break;
+                }
+                await this.handleHandshake(data);
+                break;
+            case commandMessages_1.commandTopic.RETRANSMIT:
+                if (data === undefined) {
+                    await this.server.send([commandMessages_1.commandTopic.DISCARDED, null]);
+                    break;
+                }
+                await this.handleRetransmit(data);
+                break;
+            default:
+                // REP/REQ sockets expect a reply to each request made, so we have
+                // to send some kind of response on an invalid request received
+                await this.server.send([commandMessages_1.commandTopic.DISCARDED, null]);
+        }
+    }
     /**
      * Closes opened sockets and marks this server as closed.
      */
@@ -339,7 +357,19 @@ class ClusterCommand {
             // No payload message for a full state request
             await req.send([commandMessages_1.commandTopic.FULLSTATE, null]);
             try {
-                [, fullState = null] = await req.receive();
+                // NOSONAR (S9382): one node at a time by design, the loop stops at
+                // the first one that serves a full state.
+                const [topic, payload] = await req.receive(); // NOSONAR
+                // DISCARDED carries an empty frame, which would decode as an empty
+                // full state: a node that cannot serve one (#3031) is skipped like
+                // one that does not answer.
+                if (topic?.toString() === commandMessages_1.commandTopic.FULLSTATE && payload) {
+                    fullState = payload;
+                }
+                else {
+                    this.logger.warn(`Unable to fetch a full state from node ${id} (request discarded)`);
+                    idx = (idx + 1) % nodes.length;
+                }
             }
             catch {
                 // no response from the remote node in a timely fashion... retrying
@@ -403,7 +433,16 @@ class ClusterCommand {
         await req.send([commandMessages_1.commandTopic.HANDSHAKE, payload]);
         let response = null;
         try {
-            [, response = null] = await req.receive();
+            const [topic, answer] = await req.receive();
+            // DISCARDED carries an empty frame, which would decode as a handshake
+            // answer with nothing in it: a node that refused the handshake (#3031)
+            // is one that did not answer.
+            if (topic?.toString() === commandMessages_1.commandTopic.HANDSHAKE && answer) {
+                response = answer;
+            }
+            else {
+                this.logger.warn(`Couldn't complete handshake with node ${id}: request discarded`);
+            }
         }
         catch {
             this.logger.warn(`Couldn't complete handshake with node ${id}: no response received`);

package/dist/lib/cluster/node.d.ts

@@ -41,10 +41,21 @@ declare class ClusterNode {
      * The id of the ID card this node reserved during `handshake()`.
      *
      * Null until the handshake gets that far, and read as a `string` at
-     * seventeen sites here plus `subscriber.ts` and `command.ts` — all of which
-     * run after it. The accessor says once where that can fail.
+     * seventeen sites here plus `subscriber.ts` and `command.ts`. The accessor
+     * says once where that can fail: an internal read before the handshake is a
+     * bug, and it throws.
+     *
+     * Network input is not internal: the command socket is bound before the
+     * handshake, and a node restarted on the IP of a dead one can be reached by
+     * peers still talking to that dead node (#3031). The paths network input
+     * reaches check `hasNodeId` first.
      */
     get nodeId(): string;
+    /**
+     * Whether the handshake has set this node's id yet: the non-throwing check
+     * for the paths network input reaches before it has (#3031).
+     */
+    get hasNodeId(): boolean;
     set nodeId(id: string);
     private heartbeatTimer;
     readonly idCardHandler: ClusterIdCardHandler;

package/dist/lib/cluster/node.js

@@ -200,8 +200,14 @@ class ClusterNode {
      * The id of the ID card this node reserved during `handshake()`.
      *
      * Null until the handshake gets that far, and read as a `string` at
-     * seventeen sites here plus `subscriber.ts` and `command.ts` — all of which
-     * run after it. The accessor says once where that can fail.
+     * seventeen sites here plus `subscriber.ts` and `command.ts`. The accessor
+     * says once where that can fail: an internal read before the handshake is a
+     * bug, and it throws.
+     *
+     * Network input is not internal: the command socket is bound before the
+     * handshake, and a node restarted on the IP of a dead one can be reached by
+     * peers still talking to that dead node (#3031). The paths network input
+     * reaches check `hasNodeId` first.
      */
     get nodeId() {
         if (this._nodeId === null) {
@@ -209,6 +215,13 @@ class ClusterNode {
         }
         return this._nodeId;
     }
+    /**
+     * Whether the handshake has set this node's id yet: the non-throwing check
+     * for the paths network input reaches before it has (#3031).
+     */
+    get hasNodeId() {
+        return this._nodeId !== null;
+    }
     set nodeId(id) {
         this._nodeId = id;
     }
@@ -389,8 +402,11 @@ class ClusterNode {
         this.remoteNodes.delete(nodeId);
         this.fullState.removeNode(nodeId);
         subscriber.dispose();
-        if (broadcast) {
-            this.publisher.sendNodeEvicted(this.nodeId, nodeId, reason);
+        // A subscriber can evict its remote node (heartbeat lost, gap, invalid
+        // message) before this node's handshake: there is no evictor id to
+        // broadcast then, and the eviction stays local (#3031).
+        if (broadcast && this._nodeId !== null) {
+            this.publisher.sendNodeEvicted(this._nodeId, nodeId, reason);
         }
         if (this.countActiveNodes() < this.config.minimumNodes) {
             global.kuzzle.state = kuzzleStateEnum_1.default.NOT_ENOUGH_NODES;

package/dist/lib/cluster/subscriber.d.ts

@@ -15,6 +15,8 @@ type SubscribingNode = {
     config: IKuzzleConfiguration["cluster"];
     heartbeatDelay: number;
     nodeId: string;
+    /** Whether `nodeId` can be read: false until the handshake sets it. */
+    hasNodeId: boolean;
     fullState: State;
     eventEmitter: {
         emit(event: string, payload: unknown): void;

package/dist/lib/cluster/subscriber.js

@@ -383,6 +383,12 @@ class ClusterSubscriber {
      * @param message - decoded NodeEvicted protobuf message
      */
     async handleNodeEviction(message) {
+        // Before the handshake this node has no id to compare, and is not part of
+        // the cluster the eviction is about: the message is ignored (#3031).
+        if (!this.localNode.hasNodeId) {
+            debug("[CLUSTER] Eviction of node %s by %s ignored: the handshake has not run", message.nodeId, message.evictor);
+            return;
+        }
         if (message.nodeId === this.localNode.nodeId) {
             this.logger.error(`[CLUSTER] Node evicted by ${message.evictor}. Reason: ${message.reason}`);
             // A failure, like `evictSelf`'s: exit non-zero (#2785).

package/dist/lib/core/backend/backend.d.ts

@@ -1,6 +1,6 @@
 import type { JSONObject } from "../../types/JSONObject";
 import { EmbeddedSDK } from "../shared/sdk/embeddedSdk";
-import type { EventDefinition } from "../../../index";
+import type { EventDefinition } from "../../types/EventHandler";
 import { BackendCluster, BackendConfig, BackendController, BackendHook, BackendImport, BackendPipe, BackendPlugin, BackendStorage, BackendVault, BackendOpenApi, BackendErrors, BackendSubscription } from "./index";
 import { Logger } from "../../kuzzle/Logger";
 import Plugin from "../plugin/plugin";

package/dist/lib/core/backend/backend.js

@@ -61,8 +61,8 @@ const node_fs_1 = __importDefault(require("node:fs"));
 const kuzzle_1 = __importDefault(require("../../kuzzle"));
 const embeddedSdk_1 = require("../shared/sdk/embeddedSdk");
 const kerror = __importStar(require("../../kerror"));
-const index_1 = require("../../../index");
-const index_2 = require("./index");
+const name_generator_1 = require("../../util/name-generator");
+const index_1 = require("./index");
 const Logger_1 = require("../../kuzzle/Logger");
 const plugin_1 = __importDefault(require("../plugin/plugin"));
 const assertionError = kerror.wrap("plugin", "assert");
@@ -130,7 +130,7 @@ class Backend {
             throw assertionError.get("invalid_application_name", name);
         }
         this._name = name;
-        this.nodeId = index_1.NameGenerator.generateRandomName({ prefix: "knode" });
+        this.nodeId = name_generator_1.NameGenerator.generateRandomName({ prefix: "knode" });
         Reflect.defineProperty(this, "_kuzzle", {
             writable: true,
         });
@@ -162,18 +162,18 @@ class Backend {
             // Silent if no version can be found
         }
         global.app = this;
-        this.pipe = new index_2.BackendPipe(this);
-        this.hook = new index_2.BackendHook(this);
-        this.config = new index_2.BackendConfig(this);
-        this.vault = new index_2.BackendVault(this);
-        this.controller = new index_2.BackendController(this);
-        this.plugin = new index_2.BackendPlugin(this);
-        this.storage = new index_2.BackendStorage(this);
-        this.import = new index_2.BackendImport(this);
-        this.cluster = new index_2.BackendCluster();
-        this.openApi = new index_2.BackendOpenApi(this);
-        this.errors = new index_2.BackendErrors(this);
-        this.subscription = new index_2.BackendSubscription(this);
+        this.pipe = new index_1.BackendPipe(this);
+        this.hook = new index_1.BackendHook(this);
+        this.config = new index_1.BackendConfig(this);
+        this.vault = new index_1.BackendVault(this);
+        this.controller = new index_1.BackendController(this);
+        this.plugin = new index_1.BackendPlugin(this);
+        this.storage = new index_1.BackendStorage(this);
+        this.import = new index_1.BackendImport(this);
+        this.cluster = new index_1.BackendCluster();
+        this.openApi = new index_1.BackendOpenApi(this);
+        this.errors = new index_1.BackendErrors(this);
+        this.subscription = new index_1.BackendSubscription(this);
         this.log = new Logger_1.Logger(this.config.content, "kuzzle:app");
         this.kerror = kerror;
         try {

package/dist/lib/core/backend/backendCluster.d.ts

@@ -1,4 +1,4 @@
-import type { EventDefinition, ClusterEventHandler } from "../../../index";
+import type { EventDefinition, ClusterEventHandler } from "../../types/EventHandler";
 export declare class BackendCluster {
     /**
      * Broadcasts an event to listeners across all registered Kuzzle nodes

package/dist/lib/core/shared/store.js

@@ -156,6 +156,10 @@ class Store {
             }
             const exist = await global.kuzzle.ask(`core:storage:${this.scope}:collection:exist`, this.index, collection);
             if (exist) {
+                // Another node created it, maybe so recently that its primary
+                // shards are still initializing: a document request on it would
+                // get a 503 until they are active
+                await global.kuzzle.ask(`core:storage:${this.scope}:collection:shards:wait`, this.index, collection);
                 // @deprecated
                 const dynamicSettings = isConfigDeprecated
                     ? null

package/dist/lib/core/storage/clientAdapter.d.ts

@@ -41,6 +41,12 @@ declare class ClientAdapter {
      * Also checks for duplicated index names.
      */
     populateCache(): Promise<void>;
+    /**
+     * Adds the indexes and collections found in storage to the index cache,
+     * then reloads the collection write locks. Unlike `populateCache`, it never
+     * writes to storage.
+     */
+    reloadCache(): Promise<void>;
     /**
      * Replaces the cached collection write locks with the cluster-wide ones,
      * stored in Redis.

package/dist/lib/core/storage/clientAdapter.js

@@ -56,6 +56,7 @@ const Elasticsearch_1 = require("../../service/storage/Elasticsearch");
 const indexCache_1 = require("./indexCache");
 const safeObject_1 = require("../../util/safeObject");
 const kerror = __importStar(require("../../kerror"));
+const kuzzleError_1 = require("../../kerror/errors/kuzzleError");
 const distributedLock_1 = require("../../util/distributedLock");
 require("../../types/Global");
 const servicesError = kerror.wrap("services", "storage");
@@ -93,10 +94,11 @@ class ClientAdapter {
         this.registerMappingEvents();
         this.registerCacheEvents();
         this.registerLockEvents();
-        // The locks were first loaded with the cache, before this node joined the
-        // cluster: one taken in between was broadcast before this node could hear
-        // it. Reading them again once the cluster is joined closes that gap.
-        global.kuzzle.onPipe("kuzzle:state:live", () => this.loadLocks());
+        // The cache and the locks were first loaded before this node joined the
+        // cluster: an index, a collection or a lock a peer added in between was
+        // broadcast before this node could hear it. Reading them again once the
+        // cluster is joined closes that gap.
+        global.kuzzle.onPipe("kuzzle:state:live", () => this.reloadCache());
         // Global store events registration
         /**
          * Manually refresh the index cache (e.g. after alias creation)
@@ -120,7 +122,20 @@ class ClientAdapter {
             throw servicesError.get("index_already_exists", this.scope, index);
         }
         if (!indexCacheOnly) {
-            await this.client.createIndex(index);
+            try {
+                await this.client.createIndex(index);
+            }
+            catch (error) {
+                // The index exists in storage but not in this node's cache: a peer
+                // created it and its cache broadcast never reached this node (e.g. the
+                // peer stopped before this node joined the cluster). Cache it so that
+                // the next request to that index finds it, then report the conflict.
+                if (error instanceof kuzzleError_1.KuzzleError &&
+                    error.id === "services.storage.index_already_exists") {
+                    this.cache.addIndex(index);
+                }
+                throw error;
+            }
         }
         this.cache.addIndex(index);
         if (propagate) {
@@ -191,6 +206,14 @@ class ClientAdapter {
         if (global.kuzzle.config.services.storageEngine.generateMissingAliases) {
             await this.client.generateMissingAliases();
         }
+        await this.reloadCache();
+    }
+    /**
+     * Adds the indexes and collections found in storage to the index cache,
+     * then reloads the collection write locks. Unlike `populateCache`, it never
+     * writes to storage.
+     */
+    async reloadCache() {
         // `client` is untyped (see the field's comment), so the schema shape is
         // stated here: index name -> its collection names.
         const schema = await this.client.getSchema();
@@ -311,6 +334,13 @@ class ClientAdapter {
             this.cache.assertCollectionExists(index, collection);
             return this.client.getSettings(index, collection);
         });
+        /**
+         * Wait, for a bounded time, until the primary shards of a collection
+         * another node has just created are active
+         * @param {string} index
+         * @param {string} collection
+         */
+        global.kuzzle.onAsk(`core:storage:${this.scope}:collection:shards:wait`, (index, collection) => this.client.waitForCollectionShards(index, collection));
         /**
          * Check a collection existence
          * @param {string} index

package/dist/lib/kuzzle/internalIndexHandler.js

@@ -132,8 +132,11 @@ class InternalIndexHandler extends store_1.Store {
                     return;
                 }
                 await bluebird_1.default.resolve(this._bootstrapSequence()).timeout(this.timeout);
-                await this.create("config", { timestamp: Date.now() }, {
-                    id: this._BOOTSTRAP_DONE_ID,
+                // A node that stopped while holding the lock may still have its
+                // own bootstrap in flight, and write this marker after the check
+                // above: replacing it keeps this node from failing on it
+                await this.createOrReplace("config", this._BOOTSTRAP_DONE_ID, {
+                    timestamp: Date.now(),
                 });
             }
             catch (error) {
@@ -154,9 +157,11 @@ class InternalIndexHandler extends store_1.Store {
         await this.createInitialValidations();
         debug("Bootstrapping JWT secret");
         await this._initSecret();
-        // Create datamodel version
-        await this.create("config", { version: dataModelVersion }, {
-            id: this._DATAMODEL_VERSION_ID,
+        // Replaced rather than created: a bootstrap interrupted after this write
+        // but before the `.done` marker leaves it behind, and the next attempt
+        // must not fail on it (every step above is idempotent too)
+        await this.createOrReplace("config", this._DATAMODEL_VERSION_ID, {
+            version: dataModelVersion,
         });
     }
     /**
@@ -195,7 +200,20 @@ class InternalIndexHandler extends store_1.Store {
         if (!configSeed) {
             if (!storedSeed) {
                 storedSeed = node_crypto_1.default.randomBytes(512).toString("hex");
-                await this.create("config", { seed: storedSeed }, { id: this._JWT_SECRET_ID });
+                try {
+                    await this.create("config", { seed: storedSeed }, { id: this._JWT_SECRET_ID });
+                }
+                catch (error) {
+                    // An interrupted bootstrap stored one in the meantime: it is the
+                    // seed other nodes may already sign with, so keep it, and read it
+                    // back below
+                    const alreadyStored = error instanceof Error &&
+                        "id" in error &&
+                        error.id === "services.storage.document_already_exists";
+                    if (!alreadyStored) {
+                        throw error;
+                    }
+                }
             }
             this.logger.warn("[!] Kuzzle is using a generated seed for authentication. This is suitable for development but should NEVER be used in production. See https://docs.kuzzle.io/core/2/guides/getting-started/deploy-your-application/");
         }

package/dist/lib/model/security/profile.d.ts

@@ -1,6 +1,6 @@
 import type { Policy, OptimizedPolicy, OptimizedPolicyRestrictions } from "../../types/index";
 import type { Role } from "./role";
-import type { KuzzleRequest } from "../../../index";
+import type { KuzzleRequest } from "../../api/request";
 /** @internal */
 type InternalProfilePolicy = {
     role: Role;

package/dist/lib/model/security/role.d.ts

@@ -1,5 +1,5 @@
 import type { ControllerRight, ControllerRights } from "../../types/controllers/ControllerRights";
-import type { KuzzleRequest } from "../../../index";
+import type { KuzzleRequest } from "../../api/request";
 import type { OptimizedPolicyRestrictions } from "../../types/PolicyRestrictions";
 /**
  * @class Role

package/dist/lib/model/security/user.d.ts

@@ -1,5 +1,5 @@
 import type { Profile } from "./profile";
-import type { KuzzleRequest } from "../../../index";
+import type { KuzzleRequest } from "../../api/request";
 /**
  * @class User
  */

package/dist/lib/service/storage/7/elasticsearch.d.ts

@@ -452,6 +452,18 @@ export declare class ES7 {
         mappings?: TypeMapping;
         settings?: JSONObject;
     }): Promise<null>;
+    /**
+     * Waits until the primary shards of a collection are active, for at most
+     * `timeout`, and rejects if they are not by then.
+     *
+     * A collection another node has just created is in the cluster state, so
+     * it exists, before its primary shard is active: until then, Elasticsearch
+     * answers 503 to any document request on it. The node that creates a
+     * collection waits for its shards (`wait_for_active_shards`); a node that
+     * finds it already there calls this instead. It answers at once when the
+     * shards are already active.
+     */
+    waitForCollectionShards(index: string, collection: string, timeout?: string): Promise<void>;
     /**
      * Retrieves settings definition for index/type
      *

package/dist/lib/service/storage/7/elasticsearch.js

@@ -89,6 +89,9 @@ const HIDDEN_COLLECTION = "_kuzzle_keep";
 // is refreshed while held, so a live holder is never overtaken; a crashed
 // one leaves it for at most `ttl` (30 s).
 const HIDDEN_COLLECTION_LOCK = { retryAttempts: Number.POSITIVE_INFINITY };
+// How long `waitForCollectionShards` lets Elasticsearch wait for a
+// collection's primary shards, which is also Elasticsearch's own default
+const COLLECTION_SHARDS_TIMEOUT = "30s";
 const ALIAS_PREFIX = "@"; // @todo next major release: Add ALIAS_PREFIX in FORBIDDEN_CHARS
 const PRIVATE_PREFIX = "%";
 const PUBLIC_PREFIX = "&";
@@ -1146,6 +1149,9 @@ class ES7 {
         this._checkDynamicProperty(mappings);
         const exists = await this.hasCollection(index, collection);
         if (exists) {
+            // Another node created it, maybe so recently that its primary shards
+            // are still initializing
+            await this.waitForCollectionShards(index, collection);
             return this.updateCollection(index, collection, { mappings, settings });
         }
         this._checkMappings(mappings);
@@ -1167,7 +1173,9 @@ class ES7 {
             if (lodash_1.default.get(error, "meta.body.error.type") ===
                 "resource_already_exists_exception") {
                 // race condition: the indice has been created between the "exists"
-                // check above and this "create" attempt
+                // check above and this "create" attempt, by a request that may not
+                // have waited for its shards yet
+                await this.waitForCollectionShards(index, collection);
                 return null;
             }
             throw this._esWrapper.formatESError(error);
@@ -1187,6 +1195,29 @@ class ES7 {
         }
         return null;
     }
+    /**
+     * Waits until the primary shards of a collection are active, for at most
+     * `timeout`, and rejects if they are not by then.
+     *
+     * A collection another node has just created is in the cluster state, so
+     * it exists, before its primary shard is active: until then, Elasticsearch
+     * answers 503 to any document request on it. The node that creates a
+     * collection waits for its shards (`wait_for_active_shards`); a node that
+     * finds it already there calls this instead. It answers at once when the
+     * shards are already active.
+     */
+    async waitForCollectionShards(index, collection, timeout = COLLECTION_SHARDS_TIMEOUT) {
+        try {
+            await this._client.cluster.health({
+                index: this._getAlias(index, collection),
+                timeout,
+                wait_for_status: "yellow",
+            });
+        }
+        catch (error) {
+            throw this._esWrapper.formatESError(error);
+        }
+    }
     /**
      * Retrieves settings definition for index/type
      *

package/dist/lib/service/storage/7/esWrapper.js

@@ -35,6 +35,35 @@ const kerror = (0, kerror_1.wrap)("services", "storage");
 function isResponseError(error) {
     return Boolean(error.meta);
 }
+/**
+ * What the client puts in `message` when the response carried nothing it could
+ * read: this placeholder, or an empty string for a bodyless response.
+ */
+const CLIENT_PLACEHOLDER_MESSAGE = "Response Error";
+function isUsableMessage(message) {
+    return (typeof message === "string" &&
+        message.trim() !== "" &&
+        message !== CLIENT_PLACEHOLDER_MESSAGE);
+}
+/**
+ * A message for an error that came with none worth keeping (typically a 503
+ * with no body while shards initialize), built from what the error still
+ * carries: the client's error name, the HTTP status and the response body.
+ */
+function describeUnreadableError(error) {
+    const statusCode = error.meta?.statusCode;
+    const body = error.meta?.body;
+    const hasBody = body !== undefined && body !== null && body !== "";
+    let description = statusCode === undefined
+        ? "Elasticsearch request failed"
+        : `Elasticsearch responded ${statusCode}`;
+    description += hasBody
+        ? `: ${(0, node_util_1.inspect)(body, { breakLength: Infinity, depth: 3 })}`
+        : " (no body)";
+    return error.name && error.name !== "Error"
+        ? `${error.name}: ${description}`
+        : description;
+}
 const errorMessagesMapping = [
     {
         regex: /^\[es_rejected_execution_exception] rejected execution .*? on EsThreadPoolExecutor\[(.*?), .*$/,
@@ -258,8 +287,8 @@ class ESWrapper {
         return kerror.get("unexpected_bad_request", errorMessage);
     }
     _handleUnknownError(error, message) {
-        debug("unhandled elasticsearch error (unhandled type: %s): %o", (0, get_1.default)(error, "error.meta.statusCode", "<no status code>"), error);
-        return kerror.get("unexpected_error", message);
+        debug("unhandled elasticsearch error (unhandled type: %s): %o", (0, get_1.default)(error, "meta.statusCode", "<no status code>"), error);
+        return kerror.get("unexpected_error", isUsableMessage(message) ? message : describeUnreadableError(error));
     }
 }
 module.exports = ESWrapper;

package/dist/lib/service/storage/8/elasticsearch.d.ts

@@ -443,6 +443,18 @@ export declare class ES8 {
         mappings?: estypes.MappingTypeMapping;
         settings?: estypes.IndicesIndexSettings;
     }): Promise<null>;
+    /**
+     * Waits until the primary shards of a collection are active, for at most
+     * `timeout`, and rejects if they are not by then.
+     *
+     * A collection another node has just created is in the cluster state, so
+     * it exists, before its primary shard is active: until then, Elasticsearch
+     * answers 503 to any document request on it. The node that creates a
+     * collection waits for its shards (`wait_for_active_shards`); a node that
+     * finds it already there calls this instead. It answers at once when the
+     * shards are already active.
+     */
+    waitForCollectionShards(index: string, collection: string, timeout?: string): Promise<void>;
     /**
      * Retrieves settings definition for index/type
      *

package/dist/lib/service/storage/8/elasticsearch.js

@@ -89,6 +89,9 @@ const HIDDEN_COLLECTION = "_kuzzle_keep";
 // is refreshed while held, so a live holder is never overtaken; a crashed
 // one leaves it for at most `ttl` (30 s).
 const HIDDEN_COLLECTION_LOCK = { retryAttempts: Number.POSITIVE_INFINITY };
+// How long `waitForCollectionShards` lets Elasticsearch wait for a
+// collection's primary shards, which is also Elasticsearch's own default
+const COLLECTION_SHARDS_TIMEOUT = "30s";
 const ALIAS_PREFIX = "@"; // @todo next major release: Add ALIAS_PREFIX in FORBIDDEN_CHARS
 const PRIVATE_PREFIX = "%";
 const PUBLIC_PREFIX = "&";
@@ -1168,6 +1171,9 @@ class ES8 {
         this._checkDynamicProperty(mappings);
         const exists = await this.hasCollection(index, collection);
         if (exists) {
+            // Another node created it, maybe so recently that its primary shards
+            // are still initializing
+            await this.waitForCollectionShards(index, collection);
             return this.updateCollection(index, collection, { mappings, settings });
         }
         this._checkMappings(mappings);
@@ -1189,7 +1195,9 @@ class ES8 {
             if (lodash_1.default.get(error, "meta.body.error.type") ===
                 "resource_already_exists_exception") {
                 // race condition: the indice has been created between the "exists"
-                // check above and this "create" attempt
+                // check above and this "create" attempt, by a request that may not
+                // have waited for its shards yet
+                await this.waitForCollectionShards(index, collection);
                 return null;
             }
             throw this._esWrapper.formatESError(error);
@@ -1209,6 +1217,29 @@ class ES8 {
         }
         return null;
     }
+    /**
+     * Waits until the primary shards of a collection are active, for at most
+     * `timeout`, and rejects if they are not by then.
+     *
+     * A collection another node has just created is in the cluster state, so
+     * it exists, before its primary shard is active: until then, Elasticsearch
+     * answers 503 to any document request on it. The node that creates a
+     * collection waits for its shards (`wait_for_active_shards`); a node that
+     * finds it already there calls this instead. It answers at once when the
+     * shards are already active.
+     */
+    async waitForCollectionShards(index, collection, timeout = COLLECTION_SHARDS_TIMEOUT) {
+        try {
+            await this._client.cluster.health({
+                index: this._getAlias(index, collection),
+                timeout,
+                wait_for_status: "yellow",
+            });
+        }
+        catch (error) {
+            throw this._esWrapper.formatESError(error);
+        }
+    }
     /**
      * Retrieves settings definition for index/type
      *

package/dist/lib/service/storage/8/esWrapper.js

@@ -35,6 +35,35 @@ const kerror = (0, kerror_1.wrap)("services", "storage");
 function isResponseError(error) {
     return Boolean(error.meta);
 }
+/**
+ * What the client puts in `message` when the response carried nothing it could
+ * read: this placeholder, or an empty string for a bodyless response.
+ */
+const CLIENT_PLACEHOLDER_MESSAGE = "Response Error";
+function isUsableMessage(message) {
+    return (typeof message === "string" &&
+        message.trim() !== "" &&
+        message !== CLIENT_PLACEHOLDER_MESSAGE);
+}
+/**
+ * A message for an error that came with none worth keeping (typically a 503
+ * with no body while shards initialize), built from what the error still
+ * carries: the client's error name, the HTTP status and the response body.
+ */
+function describeUnreadableError(error) {
+    const statusCode = error.meta?.statusCode;
+    const body = error.meta?.body;
+    const hasBody = body !== undefined && body !== null && body !== "";
+    let description = statusCode === undefined
+        ? "Elasticsearch request failed"
+        : `Elasticsearch responded ${statusCode}`;
+    description += hasBody
+        ? `: ${(0, node_util_1.inspect)(body, { breakLength: Infinity, depth: 3 })}`
+        : " (no body)";
+    return error.name && error.name !== "Error"
+        ? `${error.name}: ${description}`
+        : description;
+}
 const errorMessagesMapping = [
     {
         regex: /^\[es_rejected_execution_exception] rejected execution .*? on EsThreadPoolExecutor\[(.*?), .*$/,
@@ -258,8 +287,8 @@ class ESWrapper {
         return kerror.get("unexpected_bad_request", errorMessage);
     }
     _handleUnknownError(error, message) {
-        debug("unhandled elasticsearch error (unhandled type: %s): %o", (0, get_1.default)(error, "error.meta.statusCode", "<no status code>"), error);
-        return kerror.get("unexpected_error", message);
+        debug("unhandled elasticsearch error (unhandled type: %s): %o", (0, get_1.default)(error, "meta.statusCode", "<no status code>"), error);
+        return kerror.get("unexpected_error", isUsableMessage(message) ? message : describeUnreadableError(error));
     }
 }
 module.exports = ESWrapper;

package/dist/lib/types/events/EventGenericDocument.d.ts

@@ -1,6 +1,7 @@
 import type { KDocument, KDocumentContent } from "kuzzle-types";
 import type { JSONObject } from "../JSONObject";
-import type { KuzzleRequest, PipeEventHandler } from "../../../index";
+import type { KuzzleRequest } from "../../api/request";
+import type { PipeEventHandler } from "../EventHandler";
 /**
  * Events with documents only having the `_id`
  */

package/dist/lib/types/events/EventProtocol.d.ts

@@ -1,5 +1,5 @@
 import type { JSONObject } from "../JSONObject";
-import type { PipeEventHandler } from "../../../index";
+import type { PipeEventHandler } from "../EventHandler";
 import type { ClientConnection } from "../ClientConnection";
 import type { HttpMessage } from "../HttpMessage";
 export type EventHTTPBeforeParsingPayload = {

package/dist/lib/util/mutex.d.ts

@@ -101,7 +101,8 @@ export declare class Mutex {
      *
      * From then on, `lock()` never resolves: the process is about to exit, and
      * answering `false` would send its caller down a path meant for a lock
-     * held by another node.
+     * held by another node. And `unlock()` on a lock it already freed resolves
+     * without doing anything, since its holder may still try to give it back.
      */
     static releaseAllBeforeExit(): Promise<void>;
     get locked(): boolean;

package/dist/lib/util/mutex.js

@@ -165,6 +165,11 @@ class Mutex {
      * @return {Promise}
      */
     async unlock() {
+        // `releaseAllBeforeExit` already gave this lock back: its holder is only
+        // catching up, and throwing would turn a requested shutdown into a crash.
+        if (!this._locked && exiting) {
+            return;
+        }
         if (!this._locked) {
             throw fatal.get("assertion_failed", `tried to unlock the resource "${this.resource}", which is not locked (mutex id: ${this.mutexId})`);
         }
@@ -214,7 +219,8 @@ class Mutex {
      *
      * From then on, `lock()` never resolves: the process is about to exit, and
      * answering `false` would send its caller down a path meant for a lock
-     * held by another node.
+     * held by another node. And `unlock()` on a lock it already freed resolves
+     * without doing anything, since its holder may still try to give it back.
      */
     static async releaseAllBeforeExit() {
         exiting = true;

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "kuzzle",
     "author": "The Kuzzle Team <support@kuzzle.io>",
-    "version": "2.60.0-beta.3",
+    "version": "2.60.0-beta.4",
     "description": "Kuzzle is an open-source solution that handles all the data management through a secured API, with a large choice of protocols.",
     "scripts": {
         "build": "rm -Rf ./dist && tsc && node -r ts-node/register/transpile-only ./bin/copy-binaries.ts",

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "kuzzle",
   "author": "The Kuzzle Team <support@kuzzle.io>",
-  "version": "2.60.0-beta.3",
+  "version": "2.60.0-beta.4",
   "description": "Kuzzle is an open-source solution that handles all the data management through a secured API, with a large choice of protocols.",
   "scripts": {
     "build": "rm -Rf ./dist && tsc && node -r ts-node/register/transpile-only ./bin/copy-binaries.ts",