kuzzle 2.60.0-beta.2 → 2.60.0-beta.4

package/dist/lib/cluster/workers/IDCardRenewer.js

@@ -104,18 +104,24 @@ class IDCardRenewer {
             // Unable to refresh the key in time before it expires
             // => this node is too slow, we need to remove it from the cluster
             if (refreshed === 0) {
-                await this.dispose();
-                process.send?.({
-                    error: "Node too slow: ID card expired",
-                });
+                await this.reportFailure("Node too slow: ID card expired");
             }
         }
         catch (error) {
-            await this.dispose();
-            process.send?.({
-                error: `Failed to refresh ID Card: ${messageOf(error)}`,
-            });
+            await this.reportFailure(`Failed to refresh ID Card: ${messageOf(error)}`);
+        }
+    }
+    /**
+     * Disposes the worker and tells the parent why, once: the renewals queued
+     * behind a stalled redis all fail together, and each report evicts the node
+     * again (#3017)
+     */
+    async reportFailure(error) {
+        if (this.disposed) {
+            return;
         }
+        await this.dispose();
+        process.send?.({ error });
     }
     async dispose() {
         if (this.disposed) {

package/dist/lib/core/auth/passportWrapper.js

@@ -99,7 +99,9 @@ class PassportWrapper {
                     fail(err);
                 }
                 else if (!user) {
-                    const message = typeof info === "object" && "message" in info
+                    const message = 
+                    // `typeof null` is "object" too: a strategy may fail with no info
+                    typeof info === "object" && info !== null && "message" in info
                         ? info.message
                         : undefined;
                     reject(kerror.get("plugin", "strategy", "missing_user", message));

package/dist/lib/core/backend/backend.d.ts

@@ -1,6 +1,6 @@
 import type { JSONObject } from "../../types/JSONObject";
 import { EmbeddedSDK } from "../shared/sdk/embeddedSdk";
-import type { EventDefinition } from "../../../index";
+import type { EventDefinition } from "../../types/EventHandler";
 import { BackendCluster, BackendConfig, BackendController, BackendHook, BackendImport, BackendPipe, BackendPlugin, BackendStorage, BackendVault, BackendOpenApi, BackendErrors, BackendSubscription } from "./index";
 import { Logger } from "../../kuzzle/Logger";
 import Plugin from "../plugin/plugin";

package/dist/lib/core/backend/backend.js

@@ -61,8 +61,8 @@ const node_fs_1 = __importDefault(require("node:fs"));
 const kuzzle_1 = __importDefault(require("../../kuzzle"));
 const embeddedSdk_1 = require("../shared/sdk/embeddedSdk");
 const kerror = __importStar(require("../../kerror"));
-const index_1 = require("../../../index");
-const index_2 = require("./index");
+const name_generator_1 = require("../../util/name-generator");
+const index_1 = require("./index");
 const Logger_1 = require("../../kuzzle/Logger");
 const plugin_1 = __importDefault(require("../plugin/plugin"));
 const assertionError = kerror.wrap("plugin", "assert");
@@ -130,7 +130,7 @@ class Backend {
             throw assertionError.get("invalid_application_name", name);
         }
         this._name = name;
-        this.nodeId = index_1.NameGenerator.generateRandomName({ prefix: "knode" });
+        this.nodeId = name_generator_1.NameGenerator.generateRandomName({ prefix: "knode" });
         Reflect.defineProperty(this, "_kuzzle", {
             writable: true,
         });
@@ -162,18 +162,18 @@ class Backend {
             // Silent if no version can be found
         }
         global.app = this;
-        this.pipe = new index_2.BackendPipe(this);
-        this.hook = new index_2.BackendHook(this);
-        this.config = new index_2.BackendConfig(this);
-        this.vault = new index_2.BackendVault(this);
-        this.controller = new index_2.BackendController(this);
-        this.plugin = new index_2.BackendPlugin(this);
-        this.storage = new index_2.BackendStorage(this);
-        this.import = new index_2.BackendImport(this);
-        this.cluster = new index_2.BackendCluster();
-        this.openApi = new index_2.BackendOpenApi(this);
-        this.errors = new index_2.BackendErrors(this);
-        this.subscription = new index_2.BackendSubscription(this);
+        this.pipe = new index_1.BackendPipe(this);
+        this.hook = new index_1.BackendHook(this);
+        this.config = new index_1.BackendConfig(this);
+        this.vault = new index_1.BackendVault(this);
+        this.controller = new index_1.BackendController(this);
+        this.plugin = new index_1.BackendPlugin(this);
+        this.storage = new index_1.BackendStorage(this);
+        this.import = new index_1.BackendImport(this);
+        this.cluster = new index_1.BackendCluster();
+        this.openApi = new index_1.BackendOpenApi(this);
+        this.errors = new index_1.BackendErrors(this);
+        this.subscription = new index_1.BackendSubscription(this);
         this.log = new Logger_1.Logger(this.config.content, "kuzzle:app");
         this.kerror = kerror;
         try {

package/dist/lib/core/backend/backendCluster.d.ts

@@ -1,4 +1,4 @@
-import type { EventDefinition, ClusterEventHandler } from "../../../index";
+import type { EventDefinition, ClusterEventHandler } from "../../types/EventHandler";
 export declare class BackendCluster {
     /**
      * Broadcasts an event to listeners across all registered Kuzzle nodes

package/dist/lib/core/shared/store.js

@@ -156,6 +156,10 @@ class Store {
             }
             const exist = await global.kuzzle.ask(`core:storage:${this.scope}:collection:exist`, this.index, collection);
             if (exist) {
+                // Another node created it, maybe so recently that its primary
+                // shards are still initializing: a document request on it would
+                // get a 503 until they are active
+                await global.kuzzle.ask(`core:storage:${this.scope}:collection:shards:wait`, this.index, collection);
                 // @deprecated
                 const dynamicSettings = isConfigDeprecated
                     ? null

package/dist/lib/core/storage/clientAdapter.d.ts

@@ -41,6 +41,12 @@ declare class ClientAdapter {
      * Also checks for duplicated index names.
      */
     populateCache(): Promise<void>;
+    /**
+     * Adds the indexes and collections found in storage to the index cache,
+     * then reloads the collection write locks. Unlike `populateCache`, it never
+     * writes to storage.
+     */
+    reloadCache(): Promise<void>;
     /**
      * Replaces the cached collection write locks with the cluster-wide ones,
      * stored in Redis.

package/dist/lib/core/storage/clientAdapter.js

@@ -56,6 +56,7 @@ const Elasticsearch_1 = require("../../service/storage/Elasticsearch");
 const indexCache_1 = require("./indexCache");
 const safeObject_1 = require("../../util/safeObject");
 const kerror = __importStar(require("../../kerror"));
+const kuzzleError_1 = require("../../kerror/errors/kuzzleError");
 const distributedLock_1 = require("../../util/distributedLock");
 require("../../types/Global");
 const servicesError = kerror.wrap("services", "storage");
@@ -93,10 +94,11 @@ class ClientAdapter {
         this.registerMappingEvents();
         this.registerCacheEvents();
         this.registerLockEvents();
-        // The locks were first loaded with the cache, before this node joined the
-        // cluster: one taken in between was broadcast before this node could hear
-        // it. Reading them again once the cluster is joined closes that gap.
-        global.kuzzle.onPipe("kuzzle:state:live", () => this.loadLocks());
+        // The cache and the locks were first loaded before this node joined the
+        // cluster: an index, a collection or a lock a peer added in between was
+        // broadcast before this node could hear it. Reading them again once the
+        // cluster is joined closes that gap.
+        global.kuzzle.onPipe("kuzzle:state:live", () => this.reloadCache());
         // Global store events registration
         /**
          * Manually refresh the index cache (e.g. after alias creation)
@@ -120,7 +122,20 @@ class ClientAdapter {
             throw servicesError.get("index_already_exists", this.scope, index);
         }
         if (!indexCacheOnly) {
-            await this.client.createIndex(index);
+            try {
+                await this.client.createIndex(index);
+            }
+            catch (error) {
+                // The index exists in storage but not in this node's cache: a peer
+                // created it and its cache broadcast never reached this node (e.g. the
+                // peer stopped before this node joined the cluster). Cache it so that
+                // the next request to that index finds it, then report the conflict.
+                if (error instanceof kuzzleError_1.KuzzleError &&
+                    error.id === "services.storage.index_already_exists") {
+                    this.cache.addIndex(index);
+                }
+                throw error;
+            }
         }
         this.cache.addIndex(index);
         if (propagate) {
@@ -191,6 +206,14 @@ class ClientAdapter {
         if (global.kuzzle.config.services.storageEngine.generateMissingAliases) {
             await this.client.generateMissingAliases();
         }
+        await this.reloadCache();
+    }
+    /**
+     * Adds the indexes and collections found in storage to the index cache,
+     * then reloads the collection write locks. Unlike `populateCache`, it never
+     * writes to storage.
+     */
+    async reloadCache() {
         // `client` is untyped (see the field's comment), so the schema shape is
         // stated here: index name -> its collection names.
         const schema = await this.client.getSchema();
@@ -311,6 +334,13 @@ class ClientAdapter {
             this.cache.assertCollectionExists(index, collection);
             return this.client.getSettings(index, collection);
         });
+        /**
+         * Wait, for a bounded time, until the primary shards of a collection
+         * another node has just created are active
+         * @param {string} index
+         * @param {string} collection
+         */
+        global.kuzzle.onAsk(`core:storage:${this.scope}:collection:shards:wait`, (index, collection) => this.client.waitForCollectionShards(index, collection));
         /**
          * Check a collection existence
          * @param {string} index

package/dist/lib/kerror/codes/8-cluster.json

@@ -11,6 +11,29 @@
                     "class": "InternalError"
                 }
             }
+        },
+        "join": {
+            "code": 1,
+            "errors": {
+                "duplicate_ip": {
+                    "description": "Another node of the cluster has the same IP address as the one joining it. The joining node does not start.",
+                    "code": 1,
+                    "message": "Another node shares the same IP address as this one (%s): %s.",
+                    "class": "InternalError"
+                },
+                "full_state_unavailable": {
+                    "description": "None of the cluster nodes discovered could give the joining node the cluster state, twice in a row (network split). The joining node does not start.",
+                    "code": 2,
+                    "message": "Could not connect to the discovered cluster nodes (network split detected).",
+                    "class": "InternalError"
+                },
+                "timeout": {
+                    "description": "The node did not join the cluster within the configured cluster.joinTimeout. It does not start.",
+                    "code": 3,
+                    "message": "Failed to join the cluster: timed out (joinTimeout: %sms).",
+                    "class": "InternalError"
+                }
+            }
         }
     }
 }

package/dist/lib/kuzzle/exitOnFailure.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Ends the process with exit code 1, for a start script whose Kuzzle failed
+ * to start or to run.
+ *
+ * A bare `process.exit(1)` cuts Kuzzle's graceful shutdown short: the locks
+ * the node holds and its ID card stay in Redis until they expire (#3022).
+ * Once a Kuzzle instance exists, this goes through `shutdown(1)` instead,
+ * which joins the one a failed start already began (#3020). Before that,
+ * there is nothing to release.
+ *
+ * Internal to Kuzzle: not exported by the package.
+ */
+export declare function exitOnFailure(): Promise<void>;

package/dist/lib/kuzzle/exitOnFailure.js

@@ -0,0 +1,60 @@
+"use strict";
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.exitOnFailure = exitOnFailure;
+const node_util_1 = require("node:util");
+/**
+ * Ends the process with exit code 1, for a start script whose Kuzzle failed
+ * to start or to run.
+ *
+ * A bare `process.exit(1)` cuts Kuzzle's graceful shutdown short: the locks
+ * the node holds and its ID card stay in Redis until they expire (#3022).
+ * Once a Kuzzle instance exists, this goes through `shutdown(1)` instead,
+ * which joins the one a failed start already began (#3020). Before that,
+ * there is nothing to release.
+ *
+ * Internal to Kuzzle: not exported by the package.
+ */
+async function exitOnFailure() {
+    const kuzzle = currentKuzzle();
+    if (kuzzle) {
+        try {
+            // Ends with `process.exit`
+            await kuzzle.shutdown(1);
+            return;
+        }
+        catch (error) {
+            kuzzle.log.error(`Unable to shut down: ${error instanceof Error ? error.message : (0, node_util_1.inspect)(error)}`);
+        }
+    }
+    process.exit(1);
+}
+function currentKuzzle() {
+    try {
+        return global.kuzzle;
+    }
+    catch {
+        // The `global.kuzzle` getter throws until an instance is built
+        return undefined;
+    }
+}
+//# sourceMappingURL=exitOnFailure.js.map

package/dist/lib/kuzzle/internalIndexHandler.js

@@ -132,8 +132,11 @@ class InternalIndexHandler extends store_1.Store {
                     return;
                 }
                 await bluebird_1.default.resolve(this._bootstrapSequence()).timeout(this.timeout);
-                await this.create("config", { timestamp: Date.now() }, {
-                    id: this._BOOTSTRAP_DONE_ID,
+                // A node that stopped while holding the lock may still have its
+                // own bootstrap in flight, and write this marker after the check
+                // above: replacing it keeps this node from failing on it
+                await this.createOrReplace("config", this._BOOTSTRAP_DONE_ID, {
+                    timestamp: Date.now(),
                 });
             }
             catch (error) {
@@ -154,9 +157,11 @@ class InternalIndexHandler extends store_1.Store {
         await this.createInitialValidations();
         debug("Bootstrapping JWT secret");
         await this._initSecret();
-        // Create datamodel version
-        await this.create("config", { version: dataModelVersion }, {
-            id: this._DATAMODEL_VERSION_ID,
+        // Replaced rather than created: a bootstrap interrupted after this write
+        // but before the `.done` marker leaves it behind, and the next attempt
+        // must not fail on it (every step above is idempotent too)
+        await this.createOrReplace("config", this._DATAMODEL_VERSION_ID, {
+            version: dataModelVersion,
         });
     }
     /**
@@ -195,7 +200,20 @@ class InternalIndexHandler extends store_1.Store {
         if (!configSeed) {
             if (!storedSeed) {
                 storedSeed = node_crypto_1.default.randomBytes(512).toString("hex");
-                await this.create("config", { seed: storedSeed }, { id: this._JWT_SECRET_ID });
+                try {
+                    await this.create("config", { seed: storedSeed }, { id: this._JWT_SECRET_ID });
+                }
+                catch (error) {
+                    // An interrupted bootstrap stored one in the meantime: it is the
+                    // seed other nodes may already sign with, so keep it, and read it
+                    // back below
+                    const alreadyStored = error instanceof Error &&
+                        "id" in error &&
+                        error.id === "services.storage.document_already_exists";
+                    if (!alreadyStored) {
+                        throw error;
+                    }
+                }
             }
             this.logger.warn("[!] Kuzzle is using a generated seed for authentication. This is suitable for development but should NEVER be used in production. See https://docs.kuzzle.io/core/2/guides/getting-started/deploy-your-application/");
         }

package/dist/lib/kuzzle/kuzzle.d.ts

@@ -17,6 +17,12 @@ import { Logger } from "./Logger";
 declare class Kuzzle extends KuzzleEventEmitter {
     config: IKuzzleConfiguration;
     private _state;
+    /** The running shutdown, which later `shutdown()` calls wait for (#3017) */
+    private shutdownPromise;
+    /** The highest exit code asked for by the `shutdown()` calls so far */
+    private shutdownExitCode;
+    /** Ends a failure's shutdown that does not finish in time */
+    private shutdownWatchdog;
     log: Logger;
     /** Read by `PluginsManager` to locate the plugins directories. */
     rootPath: string;
@@ -117,6 +123,7 @@ declare class Kuzzle extends KuzzleEventEmitter {
      * @returns {Promise}
      */
     shutdown(exitCode?: number): Promise<void>;
+    private runShutdown;
     /**
      * Execute multiple handlers only once on any given environment
      *

package/dist/lib/kuzzle/kuzzle.js

@@ -93,6 +93,12 @@ const Logger_1 = require("./Logger");
 const vault_1 = __importDefault(require("./vault"));
 const name_generator_1 = require("../util/name-generator");
 let _kuzzle = null;
+/**
+ * How long a shutdown caused by a failure may run before the process exits
+ * anyway: a `kuzzle:shutdown` pipe or a lock release that never settles must
+ * not keep a failed node alive, which a bare `process.exit(1)` never did
+ */
+const FAILURE_SHUTDOWN_TIMEOUT = 30000;
 Reflect.defineProperty(global, "kuzzle", {
     configurable: true,
     enumerable: false,
@@ -113,6 +119,12 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
     constructor(config) {
         super(config.plugins.common.maxConcurrentPipes, config.plugins.common.pipesBufferSize);
         this._state = kuzzleStateEnum_1.default.STARTING;
+        /** The running shutdown, which later `shutdown()` calls wait for (#3017) */
+        this.shutdownPromise = null;
+        /** The highest exit code asked for by the `shutdown()` calls so far */
+        this.shutdownExitCode = 0;
+        /** Ends a failure's shutdown that does not finish in time */
+        this.shutdownWatchdog = null;
         global.kuzzle = this;
         this._state = kuzzleStateEnum_1.default.STARTING;
         this.config = config;
@@ -204,6 +216,16 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
         }
         catch (error) {
             this.log.error(`[X] Cannot start Kuzzle ${this.version}: ${error instanceof Error ? error.message : (0, node_util_1.inspect)(error)}`);
+            // A failed start cannot be retried (a second Kuzzle instance cannot be
+            // built), and what it did start keeps the process alive: in production,
+            // a node with no listener and an ID card still renewed (#3020). Not
+            // awaited, so the caller still gets the error before the process exits.
+            this.shutdown(1).catch((shutdownError) => {
+                this.log.error(`Unable to shut down after a failed start: ${shutdownError instanceof Error
+                    ? shutdownError.message
+                    : (0, node_util_1.inspect)(shutdownError)}`);
+                process.exit(1);
+            });
             throw error;
         }
     }
@@ -231,6 +253,22 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
      * @returns {Promise}
      */
     async shutdown(exitCode = 0) {
+        // Several sources can ask for it at once (a signal, a cluster eviction,
+        // the `exit` handler that `process.exit` below fires): they share the one
+        // shutdown, and a failure reported by any of them keeps the exit non-zero
+        this.shutdownExitCode = Math.max(this.shutdownExitCode, exitCode);
+        // Not for a requested shutdown: it waits for the requests still running,
+        // however long they take
+        if (exitCode !== 0) {
+            this.shutdownWatchdog ?? (this.shutdownWatchdog = setTimeout(() => {
+                this.log.error(`[shutdown] Not finished after ${FAILURE_SHUTDOWN_TIMEOUT}ms: exiting now`);
+                process.exit(this.shutdownExitCode);
+            }, FAILURE_SHUTDOWN_TIMEOUT));
+        }
+        this.shutdownPromise ?? (this.shutdownPromise = this.runShutdown());
+        return this.shutdownPromise;
+    }
+    async runShutdown() {
         this._state = kuzzleStateEnum_1.default.SHUTTING_DOWN;
         this.log.info("Initiating shutdown...");
         // Ask the network layer to stop accepting new request
@@ -257,7 +295,10 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
         // `pluginsManager?.application?.log?.flush?.()` was a silent no-op and the
         // application's buffered logs were dropped on shutdown (TD-51).
         await this?.pluginsManager?.application?.instance?.log?.flush?.();
-        process.exit(exitCode);
+        if (this.shutdownWatchdog !== null) {
+            clearTimeout(this.shutdownWatchdog);
+        }
+        process.exit(this.shutdownExitCode);
     }
     /**
      * Execute multiple handlers only once on any given environment
@@ -608,7 +649,9 @@ class Kuzzle extends KuzzleEventEmitter_1.default {
                 // nothing to do with that error
             }
         }
-        await this.shutdown();
+        // Only abnormal terminations get here: exit non-zero, so that an
+        // `on-failure` restart policy brings the node back (#3020)
+        await this.shutdown(1);
     }
 }
 exports.Kuzzle = Kuzzle;

package/dist/lib/model/security/profile.d.ts

@@ -1,6 +1,6 @@
 import type { Policy, OptimizedPolicy, OptimizedPolicyRestrictions } from "../../types/index";
 import type { Role } from "./role";
-import type { KuzzleRequest } from "../../../index";
+import type { KuzzleRequest } from "../../api/request";
 /** @internal */
 type InternalProfilePolicy = {
     role: Role;

package/dist/lib/model/security/role.d.ts

@@ -1,5 +1,5 @@
 import type { ControllerRight, ControllerRights } from "../../types/controllers/ControllerRights";
-import type { KuzzleRequest } from "../../../index";
+import type { KuzzleRequest } from "../../api/request";
 import type { OptimizedPolicyRestrictions } from "../../types/PolicyRestrictions";
 /**
  * @class Role

package/dist/lib/model/security/user.d.ts

@@ -1,5 +1,5 @@
 import type { Profile } from "./profile";
-import type { KuzzleRequest } from "../../../index";
+import type { KuzzleRequest } from "../../api/request";
 /**
  * @class User
  */

package/dist/lib/service/storage/7/elasticsearch.d.ts

@@ -452,6 +452,18 @@ export declare class ES7 {
         mappings?: TypeMapping;
         settings?: JSONObject;
     }): Promise<null>;
+    /**
+     * Waits until the primary shards of a collection are active, for at most
+     * `timeout`, and rejects if they are not by then.
+     *
+     * A collection another node has just created is in the cluster state, so
+     * it exists, before its primary shard is active: until then, Elasticsearch
+     * answers 503 to any document request on it. The node that creates a
+     * collection waits for its shards (`wait_for_active_shards`); a node that
+     * finds it already there calls this instead. It answers at once when the
+     * shards are already active.
+     */
+    waitForCollectionShards(index: string, collection: string, timeout?: string): Promise<void>;
     /**
      * Retrieves settings definition for index/type
      *

package/dist/lib/service/storage/7/elasticsearch.js

@@ -89,6 +89,9 @@ const HIDDEN_COLLECTION = "_kuzzle_keep";
 // is refreshed while held, so a live holder is never overtaken; a crashed
 // one leaves it for at most `ttl` (30 s).
 const HIDDEN_COLLECTION_LOCK = { retryAttempts: Number.POSITIVE_INFINITY };
+// How long `waitForCollectionShards` lets Elasticsearch wait for a
+// collection's primary shards, which is also Elasticsearch's own default
+const COLLECTION_SHARDS_TIMEOUT = "30s";
 const ALIAS_PREFIX = "@"; // @todo next major release: Add ALIAS_PREFIX in FORBIDDEN_CHARS
 const PRIVATE_PREFIX = "%";
 const PUBLIC_PREFIX = "&";
@@ -1146,6 +1149,9 @@ class ES7 {
         this._checkDynamicProperty(mappings);
         const exists = await this.hasCollection(index, collection);
         if (exists) {
+            // Another node created it, maybe so recently that its primary shards
+            // are still initializing
+            await this.waitForCollectionShards(index, collection);
             return this.updateCollection(index, collection, { mappings, settings });
         }
         this._checkMappings(mappings);
@@ -1167,7 +1173,9 @@ class ES7 {
             if (lodash_1.default.get(error, "meta.body.error.type") ===
                 "resource_already_exists_exception") {
                 // race condition: the indice has been created between the "exists"
-                // check above and this "create" attempt
+                // check above and this "create" attempt, by a request that may not
+                // have waited for its shards yet
+                await this.waitForCollectionShards(index, collection);
                 return null;
             }
             throw this._esWrapper.formatESError(error);
@@ -1187,6 +1195,29 @@ class ES7 {
         }
         return null;
     }
+    /**
+     * Waits until the primary shards of a collection are active, for at most
+     * `timeout`, and rejects if they are not by then.
+     *
+     * A collection another node has just created is in the cluster state, so
+     * it exists, before its primary shard is active: until then, Elasticsearch
+     * answers 503 to any document request on it. The node that creates a
+     * collection waits for its shards (`wait_for_active_shards`); a node that
+     * finds it already there calls this instead. It answers at once when the
+     * shards are already active.
+     */
+    async waitForCollectionShards(index, collection, timeout = COLLECTION_SHARDS_TIMEOUT) {
+        try {
+            await this._client.cluster.health({
+                index: this._getAlias(index, collection),
+                timeout,
+                wait_for_status: "yellow",
+            });
+        }
+        catch (error) {
+            throw this._esWrapper.formatESError(error);
+        }
+    }
     /**
      * Retrieves settings definition for index/type
      *

package/dist/lib/service/storage/7/esWrapper.js

@@ -35,6 +35,35 @@ const kerror = (0, kerror_1.wrap)("services", "storage");
 function isResponseError(error) {
     return Boolean(error.meta);
 }
+/**
+ * What the client puts in `message` when the response carried nothing it could
+ * read: this placeholder, or an empty string for a bodyless response.
+ */
+const CLIENT_PLACEHOLDER_MESSAGE = "Response Error";
+function isUsableMessage(message) {
+    return (typeof message === "string" &&
+        message.trim() !== "" &&
+        message !== CLIENT_PLACEHOLDER_MESSAGE);
+}
+/**
+ * A message for an error that came with none worth keeping (typically a 503
+ * with no body while shards initialize), built from what the error still
+ * carries: the client's error name, the HTTP status and the response body.
+ */
+function describeUnreadableError(error) {
+    const statusCode = error.meta?.statusCode;
+    const body = error.meta?.body;
+    const hasBody = body !== undefined && body !== null && body !== "";
+    let description = statusCode === undefined
+        ? "Elasticsearch request failed"
+        : `Elasticsearch responded ${statusCode}`;
+    description += hasBody
+        ? `: ${(0, node_util_1.inspect)(body, { breakLength: Infinity, depth: 3 })}`
+        : " (no body)";
+    return error.name && error.name !== "Error"
+        ? `${error.name}: ${description}`
+        : description;
+}
 const errorMessagesMapping = [
     {
         regex: /^\[es_rejected_execution_exception] rejected execution .*? on EsThreadPoolExecutor\[(.*?), .*$/,
@@ -258,8 +287,8 @@ class ESWrapper {
         return kerror.get("unexpected_bad_request", errorMessage);
     }
     _handleUnknownError(error, message) {
-        debug("unhandled elasticsearch error (unhandled type: %s): %o", (0, get_1.default)(error, "error.meta.statusCode", "<no status code>"), error);
-        return kerror.get("unexpected_error", message);
+        debug("unhandled elasticsearch error (unhandled type: %s): %o", (0, get_1.default)(error, "meta.statusCode", "<no status code>"), error);
+        return kerror.get("unexpected_error", isUsableMessage(message) ? message : describeUnreadableError(error));
     }
 }
 module.exports = ESWrapper;

package/dist/lib/service/storage/8/elasticsearch.d.ts

@@ -443,6 +443,18 @@ export declare class ES8 {
         mappings?: estypes.MappingTypeMapping;
         settings?: estypes.IndicesIndexSettings;
     }): Promise<null>;
+    /**
+     * Waits until the primary shards of a collection are active, for at most
+     * `timeout`, and rejects if they are not by then.
+     *
+     * A collection another node has just created is in the cluster state, so
+     * it exists, before its primary shard is active: until then, Elasticsearch
+     * answers 503 to any document request on it. The node that creates a
+     * collection waits for its shards (`wait_for_active_shards`); a node that
+     * finds it already there calls this instead. It answers at once when the
+     * shards are already active.
+     */
+    waitForCollectionShards(index: string, collection: string, timeout?: string): Promise<void>;
     /**
      * Retrieves settings definition for index/type
      *