kuzzle 2.31.0 → 2.32.0-beta.1

package/lib/service/storage/8/esWrapper.js

@@ -0,0 +1,303 @@
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* eslint sort-keys: 0 */
+
+"use strict";
+
+const Bluebird = require("bluebird");
+const _ = require("lodash");
+const es = require("sdk-es8");
+
+const { KuzzleError } = require("../../../kerror/errors");
+const debug = require("../../../util/debug")(
+  "kuzzle:services:storage:ESCommon",
+);
+const kerror = require("../../../kerror").wrap("services", "storage");
+
+const errorMessagesMapping = [
+  {
+    regex:
+      /^\[es_rejected_execution_exception] rejected execution .*? on EsThreadPoolExecutor\[(.*?), .*$/,
+    subCode: "too_many_operations",
+    getPlaceholders: (esError, matches) => [matches[1]],
+  },
+  {
+    // [illegal_argument_exception] object mapping [titi] can't be changed from nested to non-nested
+    regex:
+      /^\[illegal_argument_exception] object mapping \[(.*?)] can't be changed from nested to non-nested$/,
+    subcode: "cannot_change_mapping",
+    getPlaceholders: (esError, matches) => [matches[1]],
+  },
+  {
+    // [illegal_argument_exception] object mapping [baz] can't be changed from non-nested to nested
+    regex:
+      /^\[illegal_argument_exception] object mapping \[(.*?)] can't be changed from non-nested to nested$/,
+    subcode: "cannot_change_mapping",
+    getPlaceholders: (esError, matches) => [matches[1]],
+  },
+  {
+    // [illegal_argument_exception] Can't merge a non object mapping [aeaze] with an object mapping [aeaze]
+    regex:
+      /^\[illegal_argument_exception] Can't merge a non object mapping \[(.*?)] with an object mapping \[(.*?)]$/,
+    subcode: "cannot_change_mapping",
+    getPlaceholders: (esError, matches) => [matches[1]],
+  },
+  {
+    // [illegal_argument_exception] [tutu.tutu] is defined as an object in mapping [aze] but this name is already used for a field in other types
+    regex:
+      /^\[illegal_argument_exception] \[(.*?)] is defined as an object in mapping \[(.*?)] but this name is already used for a field in other types$/,
+    subcode: "duplicate_field_mapping",
+    getPlaceholders: (esError, matches) => [matches[1], matches[2]],
+  },
+  {
+    // [illegal_argument_exception] mapper [source.flags] of different type, current_type [string], merged_type [long]
+    regex:
+      /^mapper \[(.*?)] of different type, current_type \[(.*?)], merged_type \[(.*?)]$/,
+    subcode: "cannot_change_mapping",
+    getPlaceholders: (esError, matches) => [matches[1]],
+  },
+  {
+    // [mapper_parsing_exception] Mapping definition for [flags] has unsupported parameters:  [index : not_analyzed]
+    // eslint-disable-next-line no-regex-spaces
+    regex:
+      /^\[mapper_parsing_exception] Mapping definition for \[(.*?)] has unsupported parameters: \[(.*?)]$/,
+    subcode: "unexpected_properties",
+    getPlaceholders: (esError, matches) => [matches[2], matches[1]],
+  },
+  {
+    // [mapper_parsing_exception] No handler for type [boolean] declared on field [not]
+    regex:
+      /^\[mapper_parsing_exception] No handler for type \[(.*?)] declared on field \[(.*?)]$/,
+    subcode: "invalid_mapping_type",
+    getPlaceholders: (esError, matches) => [matches[2], matches[1]],
+  },
+  {
+    // [mapper_parsing_exception] failed to parse [conditions.host.flags]
+    regex: /^\[mapper_parsing_exception] failed to parse \[(.*?)]$/,
+    subcode: "wrong_mapping_property",
+    getPlaceholders: (esError, matches) => [matches[1]],
+  },
+  {
+    // Failed to parse mapping [_doc]: Expected map for property [fields] on field [enabled] but got a class java.lang.String
+    regex:
+      /^Failed to parse mapping \[.*\]: Expected \w+ for property \[(.*)\] on field \[(.*)\]/,
+    subcode: "wrong_mapping_property",
+    getPlaceholders: (esError, matches) => [`${matches[2]}.${matches[1]}`],
+  },
+  {
+    // [index_not_found_exception] no such index, with { resource.type=index_or_alias & resource.id=foso & index=foso }
+    regex: /^no such index \[([%&])(.*)\.(.*)\]$/,
+    subcode: "unknown_collection",
+    getPlaceholders: (esError, matches) => [matches[2], matches[3]],
+  },
+  {
+    // [mapper_parsing_exception] Expected map for property [fields] on field [foo] but got a class java.lang.String
+    regex:
+      /^\[mapper_parsing_exception] Expected map for property \[(.*?)] on field \[(.*?)] but got a class java\.lang\.String$/,
+    subcode: "wrong_mapping_property",
+    getPlaceholders: (esError, matches) => [`${matches[2]}.${matches[1]}`],
+  },
+  {
+    regex:
+      /^\[version_conflict_engine_exception] \[data]\[(.*?)]: version conflict.*$/,
+    subcode: "too_many_changes",
+    getPlaceholders: (esError, matches) => [matches[1]],
+  },
+  {
+    //[liia]: version conflict, document already exists (current version [2])
+    regex: /^\[(.*)\]: version conflict, document already exists.*/,
+    subcode: "document_already_exists",
+    getPlaceholders: () => [],
+  },
+  {
+    // Unknown key for a START_OBJECT in [term].
+    regex: /^Unknown key for a START_OBJECT in \[(.*)\].*/,
+    subcode: "invalid_search_query",
+    getPlaceholders: (esError, matches) => [matches[1]],
+  },
+  {
+    // mapping set to strict, dynamic introduction of [lehuong] within [_doc] is not allowed
+    regex:
+      /^\[(.+)\] mapping set to strict, dynamic introduction of \[(.+)\] within \[.+\] is not allowed/,
+    subcode: "strict_mapping_rejection",
+    getPlaceholders: (esError, matches) => {
+      // "/%26index.collection/_doc"
+      const esPath = esError.meta.meta.request.params.path;
+      // keep only "index"
+      const index = esPath.split(".")[0].split("%26")[1];
+      // keep only "collection"
+      const collection = esPath.substr(esPath.indexOf(".") + 1).split("/")[0];
+
+      return [matches[2], index, collection];
+    },
+  },
+  {
+    // [and] query malformed, no start_object after query name
+    regex: /^\[(.*)\] query malformed, no start_object after query name/,
+    subcode: "unknown_query_keyword",
+    getPlaceholders: (esError, matches) => [matches[1]],
+  },
+  {
+    // no [query] registered for [equals]
+    regex: /^no \[query\] registered for \[(.*)\]/,
+    subcode: "unknown_query_keyword",
+    getPlaceholders: (esError, matches) => [matches[1]],
+  },
+];
+
+class ESWrapper {
+  constructor(client) {
+    this.client = client;
+  }
+
+  /**
+   * Transforms raw ES errors into a normalized Kuzzle version
+   *
+   * @param {Error} error
+   * @returns {KuzzleError}
+   */
+  formatESError(error) {
+    if (error instanceof KuzzleError) {
+      return error;
+    }
+
+    global.kuzzle.emit("services:storage:error", {
+      message: `Elasticsearch Client error: ${error.message}`,
+      // /!\ not all ES error classes have a "meta" property
+      meta: error.meta || null,
+      stack: error.stack,
+    });
+
+    if (error instanceof es.errors.NoLivingConnectionsError) {
+      throw kerror.get("not_connected");
+    }
+    const message = _.get(error, "meta.body.error.reason", error.message);
+
+    // Try to match a known elasticsearch error
+    for (const betterError of errorMessagesMapping) {
+      const matches = message.match(betterError.regex);
+
+      if (matches) {
+        return kerror.get(
+          betterError.subcode,
+          ...betterError.getPlaceholders(error, matches),
+        );
+      }
+    }
+
+    // Try to match using error codes
+    if (error.meta) {
+      switch (error.meta.statusCode) {
+        case 400:
+          return this._handleBadRequestError(error, message);
+        case 404:
+          return this._handleNotFoundError(error, message);
+        case 409:
+          return this._handleConflictError(error, message);
+        default:
+          break;
+      }
+    }
+
+    return this._handleUnknownError(error, message);
+  }
+
+  reject(error) {
+    return Bluebird.reject(this.formatESError(error));
+  }
+
+  _handleConflictError(error, message) {
+    debug('unhandled "Conflict" elasticsearch error: %a', error);
+
+    return kerror.get("unexpected_error", message);
+  }
+
+  _handleNotFoundError(error, message) {
+    let errorMessage = message;
+
+    if (!error.body._index) {
+      return kerror.get("unexpected_not_found", errorMessage);
+    }
+
+    // _index= "&nyc-open-data.yellow-taxi"
+    const index = error.body._index.split(".")[0].slice(1);
+    const collection = error.body._index.split(".")[1];
+
+    // 404 on a GET document
+    if (error.body.found === false) {
+      return kerror.get("not_found", error.body._id, index, collection);
+    }
+
+    // 404 on DELETE document (ES error payloads are so cool!)
+    if (error.meta.body._id) {
+      return kerror.get("not_found", error.meta.body._id, index, collection);
+    }
+
+    if (error.meta.body && error.meta.body.error) {
+      errorMessage = error.meta.body.error
+        ? `${error.meta.body.error.reason}: ${error.meta.body.error["resource.id"]}`
+        : `${error.message}: ${error.body._id}`;
+    }
+
+    debug('unhandled "NotFound" elasticsearch error: %a', error);
+
+    return kerror.get("unexpected_not_found", errorMessage);
+  }
+
+  _handleBadRequestError(error, message) {
+    let errorMessage = message;
+
+    if (error.meta.body && error.meta.body.error) {
+      errorMessage = error.meta.body.error.root_cause
+        ? error.meta.body.error.root_cause[0].reason
+        : error.meta.body.error.reason;
+
+      // empty query throws exception with ES 7
+      if (
+        error.meta.body.error.type === "parsing_exception" &&
+        _.get(error, "meta.body.error.caused_by.type") ===
+          "illegal_argument_exception"
+      ) {
+        errorMessage = error.meta.body.error.caused_by.reason;
+      }
+    }
+
+    debug(
+      'unhandled "BadRequest" elasticsearch error: %a',
+      _.get(error, "meta.body.error.reason", error.message),
+    );
+
+    return kerror.get("unexpected_bad_request", errorMessage);
+  }
+
+  _handleUnknownError(error, message) {
+    debug(
+      "unhandled elasticsearch error (unhandled type: %s): %o",
+      _.get(error, "error.meta.statusCode", "<no status code>"),
+      error,
+    );
+
+    return kerror.get("unexpected_error", message);
+  }
+}
+
+module.exports = ESWrapper;

package/lib/service/storage/Elasticsearch.d.ts

@@ -0,0 +1,9 @@
+import Service from "../service";
+import { storeScopeEnum } from "../../core/storage/storeScopeEnum";
+export declare class Elasticsearch extends Service {
+    client: any;
+    constructor(config: any, scope?: storeScopeEnum);
+    static buildClient(config: any, version?: "7" | "8"): any;
+    _initSequence(): Promise<void>;
+    init(): Promise<void>;
+}

package/lib/service/storage/Elasticsearch.js

@@ -0,0 +1,48 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Elasticsearch = void 0;
+const sdk_es7_1 = require("sdk-es7");
+const sdk_es8_1 = require("sdk-es8");
+const elasticsearch_1 = require("./7/elasticsearch");
+const elasticsearch_2 = require("./8/elasticsearch");
+const service_1 = __importDefault(require("../service"));
+const storeScopeEnum_1 = require("../../core/storage/storeScopeEnum");
+class Elasticsearch extends service_1.default {
+    constructor(config, scope = storeScopeEnum_1.storeScopeEnum.PUBLIC) {
+        super("elasticsearch", config);
+        global.kuzzle.log.info(`[ℹ] Elasticsearch configuration is set to major version : ${config.majorVersion}`);
+        if (config.majorVersion === "7") {
+            this.client = new elasticsearch_1.ES7(config, scope);
+        }
+        else if (config.majorVersion === "8") {
+            this.client = new elasticsearch_2.ES8(config, scope);
+        }
+        else {
+            throw new Error("Invalid Elasticsearch version.");
+        }
+    }
+    static buildClient(config, version) {
+        if (!version) {
+            version = "7";
+        }
+        switch (version) {
+            case "7":
+                return new sdk_es7_1.Client(config);
+            case "8":
+                return new sdk_es8_1.Client(config);
+            default:
+                throw new Error("Invalid Elasticsearch version.");
+        }
+    }
+    async _initSequence() {
+        await this.client._initSequence();
+    }
+    async init() {
+        await super.init();
+    }
+}
+exports.Elasticsearch = Elasticsearch;
+//# sourceMappingURL=Elasticsearch.js.map

package/lib/service/storage/commons/queryTranslator.d.ts

@@ -0,0 +1,5 @@
+export declare class QueryTranslator {
+    translate(filters: any): any;
+    _translateOperator(operator: any, operands: any): any;
+    _translateClause(clause: any, content: any): any;
+}

package/lib/service/storage/commons/queryTranslator.js

@@ -0,0 +1,189 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QueryTranslator = void 0;
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const kerror_1 = require("../../../kerror");
+class KeywordError extends Error {
+    constructor(type, name) {
+        super(`The ${type} "${name}" of Koncorde DSL is not supported for search queries.`);
+        this.keyword = { name, type };
+    }
+}
+const KONCORDE_OPERATORS = ["and", "or", "not", "bool"];
+/**
+ * Parse the Koncorde path to extract the path and the value
+ * path have the form "path.to.field[json_value]"
+ *
+ * @param {string} path
+ * @returns
+ */
+function parseKoncordePath(path) {
+    const firstBracket = path.indexOf("[");
+    if (firstBracket < 0) {
+        return {
+            path,
+        };
+    }
+    const lastBracket = path.lastIndexOf("]");
+    if (lastBracket < 0) {
+        throw (0, kerror_1.get)("services", "koncorde", "elastic_translation_error", `Invalid exists path "${path}": missing closing bracket`);
+    }
+    return {
+        path: path.slice(0, firstBracket),
+        value: JSON.parse(path.slice(firstBracket + 1, lastBracket)),
+    };
+}
+const KONCORDE_CLAUSES_TO_ES = {
+    equals: (content) => ({
+        term: {
+            ...content,
+        },
+    }),
+    exists: (field) => {
+        // Support old syntax { exists: { field: "path" } } and { exists: "path" }
+        const parsedInfo = parseKoncordePath(field.field || field);
+        // If we have a value, we need to use a range query to be sure that the value is the same
+        if (parsedInfo.value) {
+            return {
+                bool: {
+                    filter: [
+                        {
+                            exists: {
+                                field: parsedInfo.path,
+                            },
+                        },
+                        {
+                            range: {
+                                [parsedInfo.path]: {
+                                    gte: parsedInfo.value,
+                                    lte: parsedInfo.value,
+                                },
+                            },
+                        },
+                    ],
+                },
+            };
+        }
+        return {
+            exists: {
+                field: parsedInfo.path,
+            },
+        };
+    },
+    geoBoundingBox: (content) => ({
+        geo_bounding_box: {
+            ...content,
+        },
+    }),
+    geoDistance: (content) => ({
+        geo_distance: {
+            ...content,
+        },
+    }),
+    geoDistanceRange: (content) => ({
+        geo_distance_range: {
+            ...content,
+        },
+    }),
+    geoPolygon: (content) => ({
+        geo_polygon: {
+            ...content,
+        },
+    }),
+    ids: (content) => ({
+        ids: {
+            ...content,
+        },
+    }),
+    in: (content) => ({
+        terms: {
+            ...content,
+        },
+    }),
+    missing: (field) => ({
+        bool: {
+            must_not: [{ exists: { field } }],
+        },
+    }),
+    range: (content) => ({
+        range: {
+            ...content,
+        },
+    }),
+};
+const KONCORDE_OPERATORS_TO_ES = {
+    and: (content) => ({
+        bool: {
+            filter: [...content],
+        },
+    }),
+    bool: undefined,
+    not: (content) => {
+        const [name, value] = Object.entries(content[0])[0];
+        return {
+            bool: {
+                must_not: [{ [name]: value }],
+            },
+        };
+    },
+    or: (content) => ({
+        bool: {
+            should: [...content],
+        },
+    }),
+};
+class QueryTranslator {
+    translate(filters) {
+        const [name, value] = Object.entries(filters)[0];
+        if (KONCORDE_OPERATORS.includes(name)) {
+            return this._translateOperator(name, value);
+        }
+        return this._translateClause(name, value);
+    }
+    _translateOperator(operator, operands) {
+        const converter = KONCORDE_OPERATORS_TO_ES[operator];
+        if (converter === undefined) {
+            throw new KeywordError("operator", operator);
+        }
+        const esOperands = [];
+        if (operator === "not") {
+            esOperands.push(this.translate(operands));
+        }
+        else {
+            for (const operand of operands) {
+                esOperands.push(this.translate(operand));
+            }
+        }
+        return converter(esOperands);
+    }
+    _translateClause(clause, content) {
+        const converter = KONCORDE_CLAUSES_TO_ES[clause];
+        if (converter === undefined) {
+            return {
+                [clause]: content,
+            };
+        }
+        return converter(content);
+    }
+}
+exports.QueryTranslator = QueryTranslator;
+//# sourceMappingURL=queryTranslator.js.map

package/lib/types/EventHandler.d.ts

@@ -14,10 +14,30 @@ export type EventDefinition = {
      */
     args: any[];
 };
+/**
+ * Describe an ask event with it's name and the payload and result types
+ */
+export type AskEventDefinition = {
+    /**
+     * Name of the event
+     *
+     * @example
+     * "core:document:create"
+     */
+    name: string;
+    /**
+     * Payload of the event
+     */
+    payload: any;
+    /**
+     * Result of the event
+     */
+    result: any;
+};
 /**
  * Handler for hook events
  */
-export type HookEventHandler<TEventDefinition extends EventDefinition = EventDefinition> = (...args: TEventDefinition["args"]) => void;
+export type HookEventHandler<TEventDefinition extends EventDefinition = EventDefinition> = (...args: TEventDefinition["args"]) => Promise<void> | void;
 /**
  * Handler for pipe event.
  *
@@ -28,6 +48,14 @@ export type PipeEventHandler<TEventDefinition extends EventDefinition = EventDef
  * Handler for cluster event.
  */
 export type ClusterEventHandler<TEventDefinition extends EventDefinition = EventDefinition> = (...args: TEventDefinition["args"]) => any;
+/**
+ * Handler for ask event.
+ */
+export type AskEventHandler<TAskEventDefinition extends AskEventDefinition = AskEventDefinition> = (...args: [payload?: TAskEventDefinition["payload"], ...rest: any[]]) => Promise<TAskEventDefinition["result"]> | TAskEventDefinition["result"];
+/**
+ * Handler for call event.
+ */
+export type CallEventHandler<TAskEventDefinition extends AskEventDefinition = AskEventDefinition> = (...args: [payload?: TAskEventDefinition["payload"], ...rest: any[]]) => TAskEventDefinition["result"];
 /**
  * @deprecated Use HookEventHandler, PipeEventHandler or ClusterEventHandler
  */

package/lib/types/config/KuzzleConfiguration.d.ts

@@ -1,4 +1,4 @@
-import { ServerConfiguration, ServicesConfiguration, SecurityConfiguration, HttpConfiguration, PluginsConfiguration, LimitsConfiguration, DumpConfiguration } from "../index";
+import { DumpConfiguration, HttpConfiguration, LimitsConfiguration, PluginsConfiguration, SecurityConfiguration, ServerConfiguration, ServicesConfiguration } from "../index";
 export interface IKuzzleConfiguration {
     realtime: {
         /**
@@ -121,6 +121,7 @@ export interface IKuzzleConfiguration {
     };
     internal: {
         hash: any;
+        notifiableProtocols: string[];
     };
     validation: Record<string, unknown>;
 }

package/lib/types/config/storageEngine/StorageEngineElasticsearchConfiguration.d.ts

@@ -1,5 +1,9 @@
-import { ClientOptions } from "@elastic/elasticsearch";
 export type StorageEngineElasticsearch = {
+    /**
+     * Elasticsearch major version
+     * @default "7"
+     */
+    majorVersion: "7" | "8";
     /**
      * @default ['storageEngine']
      */
@@ -22,7 +26,7 @@ export type StorageEngineElasticsearch = {
      * }
      *
      */
-    client: ClientOptions;
+    client: any;
     /**
      * Default policy against new fields that are not referenced in the
      * collection mapping.

package/lib/types/storage/{Elasticsearch.d.ts → 7/Elasticsearch.d.ts}

@@ -1,4 +1,4 @@
-import { ByteSize, ClusterNodesStats } from "@elastic/elasticsearch/api/types";
+import { ByteSize, ClusterNodesStats } from "sdk-es7/api/types";
 export type InfoResult = {
     type: string;
     version: string;