kuzzle 2.49.1 → 2.50.0-beta.4

package/dist/lib/service/service.js

@@ -0,0 +1,74 @@
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+"use strict";
+const Bluebird = require("bluebird");
+const kerror = require("../kerror");
+/**
+ * Services base class
+ *
+ */
+class Service {
+    constructor(name, config) {
+        this._name = name;
+        this._config = config;
+        this._initTimeout =
+            config.initTimeout ||
+                global.kuzzle.config.services.common.defaultInitTimeout;
+    }
+    get config() {
+        return this._config;
+    }
+    get name() {
+        return this._name;
+    }
+    /**
+     * Call _initSequence to initialize the service
+     * and throw an error if timeout exceed
+     *
+     * @returns {Promise}
+     */
+    init() {
+        return Bluebird.resolve(this._initSequence())
+            .timeout(this._initTimeout)
+            .catch((e) => {
+            if (e instanceof Bluebird.TimeoutError) {
+                throw kerror.get("core", "fatal", "service_timeout", this._name);
+            }
+            throw e;
+        });
+    }
+    /**
+     * @abstract
+     * @returns {Promise}
+     */
+    _initSequence() {
+        throw new Error("Not implemented");
+    }
+    /**
+     * @abstract
+     * @returns {Promise}
+     */
+    info() {
+        throw new Error("Not implemented");
+    }
+}
+module.exports = Service;
+//# sourceMappingURL=service.js.map

package/{lib → dist/lib}/service/storage/7/elasticsearch.d.ts

@@ -1,7 +1,7 @@
 import _ from "lodash";
 import { Client, RequestParams } from "sdk-es7";
 import { TypeMapping } from "sdk-es7/api/types";
-import { JSONObject, KRequestBody, KRequestParams } from "../../../types/storage/7/Elasticsearch";
+import { InfoResult, JSONObject, KRequestBody, KRequestParams } from "../../../types/storage/7/Elasticsearch";
 import { storeScopeEnum } from "../../../core/storage/storeScopeEnum";
 import { QueryTranslator } from "../commons/queryTranslator";
 import ESWrapper from "./esWrapper";
@@ -47,7 +47,7 @@ export declare class ES7 {
      *
      * @returns {Promise.<Object>} service informations
      */
-    info(): Promise<any>;
+    info(): Promise<InfoResult>;
     /**
      * Returns detailed multi-level storage stats data
      *

package/{lib → dist/lib}/service/storage/7/elasticsearch.js

@@ -53,7 +53,6 @@ const assert_1 = __importDefault(require("assert"));
 const bluebird_1 = __importDefault(require("bluebird"));
 const ms_1 = __importDefault(require("ms"));
 const semver_1 = __importDefault(require("semver"));
-const debug_1 = __importDefault(require("../../../util/debug"));
 const storeScopeEnum_1 = require("../../../core/storage/storeScopeEnum");
 const kerror = __importStar(require("../../../kerror"));
 const didYouMean_1 = __importDefault(require("../../../util/didYouMean"));
@@ -64,7 +63,6 @@ const requestAssertions_1 = require("../../../util/requestAssertions");
 const safeObject_1 = require("../../../util/safeObject");
 const queryTranslator_1 = require("../commons/queryTranslator");
 const esWrapper_1 = __importDefault(require("./esWrapper"));
-(0, debug_1.default)("kuzzle:services:elasticsearch");
 const SCROLL_CACHE_PREFIX = "_docscroll_";
 const ROOT_MAPPING_PROPERTIES = [
     "properties",
@@ -270,7 +268,7 @@ class ES7 {
             scroll_id: scrollId,
         };
         const cacheKey = SCROLL_CACHE_PREFIX + global.kuzzle.hash(esRequest.scroll_id);
-        (0, debug_1.default)("Scroll: %o", esRequest);
+        this.logger.debug("Scroll: %o", esRequest);
         if (_scrollTTL) {
             const scrollDuration = (0, ms_1.default)(_scrollTTL);
             if (scrollDuration > this.maxScrollDuration) {
@@ -286,7 +284,7 @@ class ES7 {
             const { body } = await this._client.scroll(esRequest);
             scrollInfo.fetched += body.hits.hits.length;
             if (scrollInfo.fetched >= body.hits.total.value) {
-                (0, debug_1.default)("Last scroll page fetched: deleting scroll %s", body._scroll_id);
+                this.logger.debug("Last scroll page fetched: deleting scroll %s", body._scroll_id);
                 await global.kuzzle.ask("core:cache:internal:del", cacheKey);
                 await this.clearScroll(body._scroll_id);
             }
@@ -341,7 +339,7 @@ class ES7 {
                 throw kerror.get("services", "storage", "scroll_duration_too_great", scroll);
             }
         }
-        (0, debug_1.default)("Search: %j", esRequest);
+        this.logger.debug("Search: %j", esRequest);
         try {
             const { body } = await this._client.search(esRequest);
             if (body._scroll_id) {
@@ -474,7 +472,7 @@ class ES7 {
         if (esRequest.id === "_search") {
             return kerror.reject("services", "storage", "search_as_an_id");
         }
-        (0, debug_1.default)("Get document: %o", esRequest);
+        this.logger.debug("Get document: %o", esRequest);
         try {
             const { body } = await this._client.get(esRequest);
             return {
@@ -510,7 +508,7 @@ class ES7 {
                 })),
             },
         };
-        (0, debug_1.default)("Multi-get documents: %o", esRequest);
+        this.logger.debug("Multi-get documents: %o", esRequest);
         let body;
         try {
             ({ body } = await this._client.mget(esRequest)); // NOSONAR
@@ -548,7 +546,7 @@ class ES7 {
             body: this._sanitizeSearchBody(searchBody),
             index: this._getAlias(index, collection),
         };
-        (0, debug_1.default)("Count: %o", esRequest);
+        this.logger.debug("Count: %o", esRequest);
         try {
             const { body } = await this._client.count(esRequest);
             return body.count;
@@ -588,7 +586,7 @@ class ES7 {
                 updater: null,
             };
         }
-        (0, debug_1.default)("Create document: %o", esRequest);
+        this.logger.debug("Create document: %o", esRequest);
         try {
             const { body } = await this._client.index(esRequest);
             return {
@@ -630,7 +628,7 @@ class ES7 {
                 updater: getKuid(userId),
             };
         }
-        (0, debug_1.default)("Create or replace document: %o", esRequest);
+        this.logger.debug("Create or replace document: %o", esRequest);
         try {
             const { body } = await this._client.index(esRequest);
             return {
@@ -673,7 +671,7 @@ class ES7 {
                 updater: getKuid(userId),
             };
         }
-        (0, debug_1.default)("Update document: %o", esRequest);
+        this.logger.debug("Update document: %o", esRequest);
         try {
             const { body } = await this._client.update(esRequest);
             return {
@@ -725,7 +723,7 @@ class ES7 {
                 createdAt: now,
             };
         }
-        (0, debug_1.default)("Upsert document: %o", esRequest);
+        this.logger.debug("Upsert document: %o", esRequest);
         try {
             const { body } = await this._client.update(esRequest);
             return {
@@ -774,7 +772,7 @@ class ES7 {
             if (!exists) {
                 throw kerror.get("services", "storage", "not_found", id, index, collection);
             }
-            (0, debug_1.default)("Replace document: %o", esRequest);
+            this.logger.debug("Replace document: %o", esRequest);
             const { body } = await this._client.index(esRequest);
             return {
                 _id: id,
@@ -803,7 +801,7 @@ class ES7 {
             refresh,
         };
         assertWellFormedRefresh(esRequest);
-        (0, debug_1.default)("Delete document: %o", esRequest);
+        this.logger.debug("Delete document: %o", esRequest);
         try {
             await this._client.delete(esRequest);
         }
@@ -843,7 +841,7 @@ class ES7 {
             if (fetch) {
                 documents = await this._getAllDocumentsFromQuery(esRequest);
             }
-            (0, debug_1.default)("Delete by query: %o", esRequest);
+            this.logger.debug("Delete by query: %o", esRequest);
             esRequest.refresh = refresh === "wait_for" ? true : refresh;
             const { body } = await this._client.deleteByQuery(esRequest);
             return {
@@ -878,7 +876,7 @@ class ES7 {
             index: alias,
         };
         try {
-            (0, debug_1.default)("DeleteFields document: %o", esRequest);
+            this.logger.debug("DeleteFields document: %o", esRequest);
             const { body } = await this._client.get(esRequest);
             for (const field of fields) {
                 if (lodash_1.default.has(body._source, field)) {
@@ -933,7 +931,7 @@ class ES7 {
                 document._source = undefined;
                 document.body = changes;
             }
-            (0, debug_1.default)("Update by query: %o", esRequest);
+            this.logger.debug("Update by query: %o", esRequest);
             const { errors, items } = await this.mUpdate(index, collection, documents, { refresh, userId });
             return {
                 errors,
@@ -975,7 +973,7 @@ class ES7 {
         if (script.source !== "") {
             esRequest.body.script = script;
         }
-        (0, debug_1.default)("Bulk Update by query: %o", esRequest);
+        this.logger.debug("Bulk Update by query: %o", esRequest);
         let response;
         try {
             response = await this._client.updateByQuery(esRequest);
@@ -1166,7 +1164,7 @@ class ES7 {
         const esRequest = {
             index: indice,
         };
-        (0, debug_1.default)("Get settings: %o", esRequest);
+        this.logger.debug("Get settings: %o", esRequest);
         try {
             const { body } = await this._client.indices.getSettings(esRequest);
             return body[indice].settings.index;
@@ -1189,7 +1187,7 @@ class ES7 {
         const esRequest = {
             index: indice,
         };
-        (0, debug_1.default)("Get mapping: %o", esRequest);
+        this.logger.debug("Get mapping: %o", esRequest);
         try {
             const { body } = await this._client.indices.getMapping(esRequest);
             const properties = includeKuzzleMeta
@@ -1289,7 +1287,7 @@ class ES7 {
             // processing large indexes.
             wait_for_completion: false,
         };
-        (0, debug_1.default)("UpdateByQuery: %o", esRequest);
+        this.logger.debug("UpdateByQuery: %o", esRequest);
         try {
             await this._client.updateByQuery(esRequest);
         }
@@ -1321,7 +1319,7 @@ class ES7 {
             dynamic: mappings.dynamic || collectionMappings.dynamic,
             properties: mappings.properties,
         };
-        (0, debug_1.default)("Update mapping: %o", esRequest);
+        this.logger.debug("Update mapping: %o", esRequest);
         try {
             await this._client.indices.putMapping(esRequest);
         }
@@ -1634,7 +1632,7 @@ class ES7 {
             if (esRequest.index.length === 0) {
                 return [];
             }
-            (0, debug_1.default)("Delete indexes: %o", esRequest);
+            this.logger.debug("Delete indexes: %o", esRequest);
             await this._client.indices.delete(esRequest);
         }
         catch (error) {
@@ -1723,7 +1721,7 @@ class ES7 {
             },
             index: this._getAlias(index, collection),
         };
-        (0, debug_1.default)("mExists: %o", esRequest);
+        this.logger.debug("mExists: %o", esRequest);
         let body;
         try {
             ({ body } = await this._client.mget(esRequest)); // NOSONAR
@@ -2728,7 +2726,7 @@ class ES7 {
      */
     async clearScroll(id) {
         if (id) {
-            (0, debug_1.default)("clearing scroll: %s", id);
+            this.logger.debug("clearing scroll: %s", id);
             await this._client.clearScroll({ scroll_id: id });
         }
     }

package/dist/lib/service/storage/7/esWrapper.d.ts

@@ -0,0 +1,19 @@
+export = ESWrapper;
+declare class ESWrapper {
+    constructor(client: any);
+    client: any;
+    /**
+     * Transforms raw ES errors into a normalized Kuzzle version
+     *
+     * @param {Error} error
+     * @returns {KuzzleError}
+     */
+    formatESError(error: Error): KuzzleError;
+    reject(error: any): Bluebird<never>;
+    _handleConflictError(error: any, message: any): KuzzleError;
+    _handleNotFoundError(error: any, message: any): KuzzleError;
+    _handleBadRequestError(error: any, message: any): KuzzleError;
+    _handleUnknownError(error: any, message: any): KuzzleError;
+}
+import { KuzzleError } from "../../../kerror/errors/kuzzleError";
+import Bluebird = require("bluebird");

package/dist/lib/service/storage/7/esWrapper.js

@@ -0,0 +1,245 @@
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* eslint sort-keys: 0 */
+"use strict";
+const Bluebird = require("bluebird");
+const _ = require("lodash");
+const es = require("sdk-es7");
+const { KuzzleError } = require("../../../kerror/errors");
+const debug = require("../../../util/debug")("kuzzle:services:storage:ESCommon");
+const kerror = require("../../../kerror").wrap("services", "storage");
+const errorMessagesMapping = [
+    {
+        regex: /^\[es_rejected_execution_exception] rejected execution .*? on EsThreadPoolExecutor\[(.*?), .*$/,
+        subCode: "too_many_operations",
+        getPlaceholders: (esError, matches) => [matches[1]],
+    },
+    {
+        // [illegal_argument_exception] object mapping [titi] can't be changed from nested to non-nested
+        regex: /^\[illegal_argument_exception] object mapping \[(.*?)] can't be changed from nested to non-nested$/,
+        subcode: "cannot_change_mapping",
+        getPlaceholders: (esError, matches) => [matches[1]],
+    },
+    {
+        // [illegal_argument_exception] object mapping [baz] can't be changed from non-nested to nested
+        regex: /^\[illegal_argument_exception] object mapping \[(.*?)] can't be changed from non-nested to nested$/,
+        subcode: "cannot_change_mapping",
+        getPlaceholders: (esError, matches) => [matches[1]],
+    },
+    {
+        // [illegal_argument_exception] Can't merge a non object mapping [aeaze] with an object mapping [aeaze]
+        regex: /^\[illegal_argument_exception] Can't merge a non object mapping \[(.*?)] with an object mapping \[(.*?)]$/,
+        subcode: "cannot_change_mapping",
+        getPlaceholders: (esError, matches) => [matches[1]],
+    },
+    {
+        // [illegal_argument_exception] [tutu.tutu] is defined as an object in mapping [aze] but this name is already used for a field in other types
+        regex: /^\[illegal_argument_exception] \[(.*?)] is defined as an object in mapping \[(.*?)] but this name is already used for a field in other types$/,
+        subcode: "duplicate_field_mapping",
+        getPlaceholders: (esError, matches) => [matches[1], matches[2]],
+    },
+    {
+        // [illegal_argument_exception] mapper [source.flags] of different type, current_type [string], merged_type [long]
+        regex: /^mapper \[(.*?)] of different type, current_type \[(.*?)], merged_type \[(.*?)]$/,
+        subcode: "cannot_change_mapping",
+        getPlaceholders: (esError, matches) => [matches[1]],
+    },
+    {
+        // [mapper_parsing_exception] Mapping definition for [flags] has unsupported parameters:  [index : not_analyzed]
+        // eslint-disable-next-line no-regex-spaces
+        regex: /^\[mapper_parsing_exception] Mapping definition for \[(.*?)] has unsupported parameters: \[(.*?)]$/,
+        subcode: "unexpected_properties",
+        getPlaceholders: (esError, matches) => [matches[2], matches[1]],
+    },
+    {
+        // [mapper_parsing_exception] No handler for type [boolean] declared on field [not]
+        regex: /^\[mapper_parsing_exception] No handler for type \[(.*?)] declared on field \[(.*?)]$/,
+        subcode: "invalid_mapping_type",
+        getPlaceholders: (esError, matches) => [matches[2], matches[1]],
+    },
+    {
+        // [mapper_parsing_exception] failed to parse [conditions.host.flags]
+        regex: /^\[mapper_parsing_exception] failed to parse \[(.*?)]$/,
+        subcode: "wrong_mapping_property",
+        getPlaceholders: (esError, matches) => [matches[1]],
+    },
+    {
+        // Failed to parse mapping [_doc]: Expected map for property [fields] on field [enabled] but got a class java.lang.String
+        regex: /^Failed to parse mapping \[.*\]: Expected \w+ for property \[(.*)\] on field \[(.*)\]/,
+        subcode: "wrong_mapping_property",
+        getPlaceholders: (esError, matches) => [`${matches[2]}.${matches[1]}`],
+    },
+    {
+        // [index_not_found_exception] no such index, with { resource.type=index_or_alias & resource.id=foso & index=foso }
+        regex: /^no such index \[([%&])(.*)\.(.*)\]$/,
+        subcode: "unknown_collection",
+        getPlaceholders: (esError, matches) => [matches[2], matches[3]],
+    },
+    {
+        // [mapper_parsing_exception] Expected map for property [fields] on field [foo] but got a class java.lang.String
+        regex: /^\[mapper_parsing_exception] Expected map for property \[(.*?)] on field \[(.*?)] but got a class java\.lang\.String$/,
+        subcode: "wrong_mapping_property",
+        getPlaceholders: (esError, matches) => [`${matches[2]}.${matches[1]}`],
+    },
+    {
+        regex: /^\[version_conflict_engine_exception] \[data]\[(.*?)]: version conflict.*$/,
+        subcode: "too_many_changes",
+        getPlaceholders: (esError, matches) => [matches[1]],
+    },
+    {
+        //[liia]: version conflict, document already exists (current version [2])
+        regex: /^\[(.*)\]: version conflict, document already exists.*/,
+        subcode: "document_already_exists",
+        getPlaceholders: () => [],
+    },
+    {
+        // Unknown key for a START_OBJECT in [term].
+        regex: /^Unknown key for a START_OBJECT in \[(.*)\].*/,
+        subcode: "invalid_search_query",
+        getPlaceholders: (esError, matches) => [matches[1]],
+    },
+    {
+        // mapping set to strict, dynamic introduction of [lehuong] within [_doc] is not allowed
+        regex: /^mapping set to strict, dynamic introduction of \[(.+)\] within \[.+\] is not allowed/,
+        subcode: "strict_mapping_rejection",
+        getPlaceholders: (esError, matches) => {
+            // "/%26index.collection/_doc"
+            const esPath = esError.meta.meta.request.params.path;
+            // keep only "index"
+            const index = esPath.split(".")[0].split("%26")[1];
+            // keep only "collection"
+            const collection = esPath.substr(esPath.indexOf(".") + 1).split("/")[0];
+            return [matches[1], index, collection];
+        },
+    },
+    {
+        // [and] query malformed, no start_object after query name
+        regex: /^\[(.*)\] query malformed, no start_object after query name/,
+        subcode: "unknown_query_keyword",
+        getPlaceholders: (esError, matches) => [matches[1]],
+    },
+    {
+        // no [query] registered for [equals]
+        regex: /^no \[query\] registered for \[(.*)\]/,
+        subcode: "unknown_query_keyword",
+        getPlaceholders: (esError, matches) => [matches[1]],
+    },
+];
+class ESWrapper {
+    constructor(client) {
+        this.client = client;
+    }
+    /**
+     * Transforms raw ES errors into a normalized Kuzzle version
+     *
+     * @param {Error} error
+     * @returns {KuzzleError}
+     */
+    formatESError(error) {
+        if (error instanceof KuzzleError) {
+            return error;
+        }
+        global.kuzzle.emit("services:storage:error", {
+            message: `Elasticsearch Client error: ${error.message}`,
+            // /!\ not all ES error classes have a "meta" property
+            meta: error.meta || null,
+            stack: error.stack,
+        });
+        if (error instanceof es.errors.NoLivingConnectionsError) {
+            throw kerror.get("not_connected");
+        }
+        const message = _.get(error, "meta.body.error.reason", error.message);
+        // Try to match a known elasticsearch error
+        for (const betterError of errorMessagesMapping) {
+            const matches = message.match(betterError.regex);
+            if (matches) {
+                return kerror.get(betterError.subcode, ...betterError.getPlaceholders(error, matches));
+            }
+        }
+        // Try to match using error codes
+        if (error.meta) {
+            switch (error.meta.statusCode) {
+                case 400:
+                    return this._handleBadRequestError(error, message);
+                case 404:
+                    return this._handleNotFoundError(error, message);
+                case 409:
+                    return this._handleConflictError(error, message);
+                default:
+                    break;
+            }
+        }
+        return this._handleUnknownError(error, message);
+    }
+    reject(error) {
+        return Bluebird.reject(this.formatESError(error));
+    }
+    _handleConflictError(error, message) {
+        debug('unhandled "Conflict" elasticsearch error: %a', error);
+        return kerror.get("unexpected_error", message);
+    }
+    _handleNotFoundError(error, message) {
+        let errorMessage = message;
+        if (!error.body._index) {
+            return kerror.get("unexpected_not_found", errorMessage);
+        }
+        // _index= "&nyc-open-data.yellow-taxi"
+        const index = error.body._index.split(".")[0].slice(1);
+        const collection = error.body._index.split(".")[1];
+        // 404 on a GET document
+        if (error.body.found === false) {
+            return kerror.get("not_found", error.body._id, index, collection);
+        }
+        // 404 on DELETE document (ES error payloads are so cool!)
+        if (error.meta.body._id) {
+            return kerror.get("not_found", error.meta.body._id, index, collection);
+        }
+        if (error.meta.body && error.meta.body.error) {
+            errorMessage = error.meta.body.error
+                ? `${error.meta.body.error.reason}: ${error.meta.body.error["resource.id"]}`
+                : `${error.message}: ${error.body._id}`;
+        }
+        debug('unhandled "NotFound" elasticsearch error: %a', error);
+        return kerror.get("unexpected_not_found", errorMessage);
+    }
+    _handleBadRequestError(error, message) {
+        let errorMessage = message;
+        if (error.meta.body && error.meta.body.error) {
+            errorMessage = error.meta.body.error.root_cause
+                ? error.meta.body.error.root_cause[0].reason
+                : error.meta.body.error.reason;
+            // empty query throws exception with ES 7
+            if (error.meta.body.error.type === "parsing_exception" &&
+                _.get(error, "meta.body.error.caused_by.type") ===
+                    "illegal_argument_exception") {
+                errorMessage = error.meta.body.error.caused_by.reason;
+            }
+        }
+        debug('unhandled "BadRequest" elasticsearch error: %a', _.get(error, "meta.body.error.reason", error.message));
+        return kerror.get("unexpected_bad_request", errorMessage);
+    }
+    _handleUnknownError(error, message) {
+        debug("unhandled elasticsearch error (unhandled type: %s): %o", _.get(error, "error.meta.statusCode", "<no status code>"), error);
+        return kerror.get("unexpected_error", message);
+    }
+}
+module.exports = ESWrapper;
+//# sourceMappingURL=esWrapper.js.map