kuzzle 2.29.0 → 2.29.1-beta.1

package/lib/api/httpRoutes.js

@@ -23,22 +23,6 @@
 
 "use strict";
 
-const {
-  OpenApiDocumentCount,
-  OpenApiDocumentDeleteByQuery,
-  OpenApiDocumentDelete,
-  OpenApiDocumentScroll,
-  OpenApiDocumentExists,
-  OpenApiDocumentUpdate,
-  OpenApiDocumentReplace,
-  OpenApiDocumentGet,
-  OpenApiDocumentCreate,
-  OpenApiDocumentCreateOrReplace,
-  OpenApiDocumentValidate,
-  OpenApiSecurityUpsertUser,
-  OpenApiDocumentmCreateOrReplace,
-} = require("./openapi/components");
-
 const routes = [
   // GET (idempotent)
   { verb: "get", path: "/_me", controller: "auth", action: "getCurrentUser" },
@@ -242,7 +226,6 @@ const routes = [
     path: "/:index/:collection/:_id",
     controller: "document",
     action: "get",
-    openapi: OpenApiDocumentGet,
   },
   {
     verb: "get",
@@ -255,7 +238,6 @@ const routes = [
     path: "/:index/:collection/:_id/_exists",
     controller: "document",
     action: "exists",
-    openapi: OpenApiDocumentExists,
   },
   {
     verb: "post",
@@ -268,7 +250,6 @@ const routes = [
     path: "/_scroll/:scrollId",
     controller: "document",
     action: "scroll",
-    openapi: OpenApiDocumentScroll,
   },
 
   {
@@ -770,14 +751,12 @@ const routes = [
     path: "/:index/:collection/_count",
     controller: "document",
     action: "count",
-    openapi: OpenApiDocumentCount,
   },
   {
     verb: "post",
     path: "/:index/:collection/_create",
     controller: "document",
     action: "create",
-    openapi: OpenApiDocumentCreate,
   },
   {
     verb: "post",
@@ -832,7 +811,6 @@ const routes = [
     path: "/:index/:collection/_validate",
     controller: "document",
     action: "validate",
-    openapi: OpenApiDocumentValidate,
   },
 
   {
@@ -955,7 +933,6 @@ const routes = [
     path: "/users/:_id/_upsert",
     controller: "security",
     action: "upsertUser",
-    openapi: OpenApiSecurityUpsertUser,
   },
   {
     verb: "post",
@@ -1241,7 +1218,6 @@ const routes = [
     path: "/:index/:collection/:_id",
     controller: "document",
     action: "delete",
-    openapi: OpenApiDocumentDelete,
   },
   {
     verb: "delete",
@@ -1254,7 +1230,6 @@ const routes = [
     path: "/:index/:collection/_query",
     controller: "document",
     action: "deleteByQuery",
-    openapi: OpenApiDocumentDeleteByQuery,
   },
   {
     verb: "delete",
@@ -1421,21 +1396,18 @@ const routes = [
     path: "/:index/:collection/:_id",
     controller: "document",
     action: "createOrReplace",
-    openapi: OpenApiDocumentCreateOrReplace,
   },
   {
     verb: "put",
     path: "/:index/:collection/_mCreateOrReplace",
     controller: "document",
     action: "mCreateOrReplace",
-    openapi: OpenApiDocumentmCreateOrReplace,
   },
   {
     verb: "put",
     path: "/:index/:collection/:_id/_replace",
     controller: "document",
     action: "replace",
-    openapi: OpenApiDocumentReplace,
   },
   {
     verb: "put",
@@ -1458,7 +1430,6 @@ const routes = [
     path: "/:index/:collection/:_id/_update",
     controller: "document",
     action: "update",
-    openapi: OpenApiDocumentUpdate,
     deprecated: {
       since: "2.11.0",
       message: 'Use "document:update" route with PATCH instead of PUT',

package/lib/api/openapi/OpenApiManager.js

@@ -21,7 +21,6 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.OpenApiManager = void 0;
-const components_1 = require("./components");
 const package_json_1 = require("../../../package.json");
 const openApiGenerator_1 = require("./openApiGenerator");
 class OpenApiManager {
@@ -40,7 +39,6 @@ class OpenApiManager {
                     name: "Kuzzle team",
                     url: "https://kuzzle.io",
                     email: "support@kuzzle.io",
-                    discord: "http://join.discord.kuzzle.io",
                 },
                 license: {
                     name: "Apache 2",
@@ -52,39 +50,9 @@ class OpenApiManager {
                 description: "Kuzzle API Documentation",
                 url: "https://docs.kuzzle.io/core/2/api/",
             },
-            servers: [
-                {
-                    url: "https://{baseUrl}:{port}",
-                    description: "Kuzzle Base Url",
-                    variables: {
-                        baseUrl: { default: "localhost" },
-                        port: { default: 7512 },
-                    },
-                },
-            ],
             tags: [],
             schemes: ["https", "http"],
             paths: {},
-            components: {
-                ...components_1.OpenApiPayloadsDefinitions,
-                document: {
-                    ...components_1.OpenApiDocumentCountComponent,
-                    ...components_1.OpenApiDocumentDeleteByQueryComponent,
-                    ...components_1.OpenApiDocumentDeleteComponent,
-                    ...components_1.OpenApiDocumentScrollComponent,
-                    ...components_1.OpenApiDocumentExistsComponent,
-                    ...components_1.OpenApiDocumentUpdateComponent,
-                    ...components_1.OpenApiDocumentReplaceComponent,
-                    ...components_1.OpenApiDocumentGetComponent,
-                    ...components_1.OpenApiDocumentCreateOrReplaceComponent,
-                    ...components_1.OpenApiDocumentCreateComponent,
-                    ...components_1.OpenApiDocumentValidateComponent,
-                },
-                security: {
-                    ...components_1.OpenApiSecurityUpsertUserComponent,
-                    ...components_1.OpenApiDocumentmCreateOrReplaceComponent,
-                },
-            },
         };
         this.applicationDefinition = applicationDefinition;
         (0, openApiGenerator_1.generateOpenApi)(kuzzleRoutes, this.kuzzleDefinition);

package/lib/api/openapi/openApiGenerator.js

@@ -72,7 +72,7 @@ function generateParameters(route) {
                 in: "path",
                 name: m[1],
                 required: true,
-                schema: { type: "string" },
+                type: "string",
             });
             m = routeUrlMatch.exec(route.path);
         }

package/lib/config/default.config.js

@@ -416,6 +416,19 @@ const defaultConfig = {
                             },
                         },
                     },
+                    imports: {
+                        settings: {
+                            // @deprecated : replace undefined by 1
+                            number_of_shards: 1,
+                            number_of_replicas: 1,
+                        },
+                        mappings: {
+                            dynamic: "strict",
+                            properties: {
+                                hash: { type: "keyword" },
+                            },
+                        },
+                    },
                 },
             },
             maxScrollDuration: "1m",

package/lib/core/backend/backendOpenApi.js

@@ -27,7 +27,7 @@ class BackendOpenApi extends index_1.ApplicationManager {
         super(application);
         /* eslint-disable sort-keys */
         this.definition = {
-            openapi: "3.0.0",
+            openapi: "3.1.0",
             info: {
                 title: `${application.name} API`,
                 description: `${application.name} HTTP API definition`,
@@ -35,7 +35,6 @@ class BackendOpenApi extends index_1.ApplicationManager {
                     name: "Kuzzle team",
                     url: "https://kuzzle.io",
                     email: "support@kuzzle.io",
-                    discord: "http://join.discord.kuzzle.io",
                 },
                 license: {
                     name: "Apache 2",
@@ -47,20 +46,9 @@ class BackendOpenApi extends index_1.ApplicationManager {
                 description: "Kuzzle API Documentation",
                 url: "https://docs.kuzzle.io/core/2/api/",
             },
-            servers: [
-                {
-                    url: "https://{baseUrl}:{port}",
-                    description: `${application.name} Base Url`,
-                    variables: {
-                        baseUrl: { default: "localhost" },
-                        port: { default: 7512 },
-                    },
-                },
-            ],
             tags: [],
             schemes: ["https", "http"],
             paths: {},
-            components: {},
         };
         /* eslint-enable sort-keys */
     }

package/lib/kuzzle/kuzzle.js

@@ -382,6 +382,45 @@ class Kuzzle extends kuzzleEventEmitter_1.default {
             await bluebird_1.default.delay(1000);
         }
     }
+    isConfigsEmpty(importConfig, supportConfig) {
+        if (lodash_1.default.isEmpty(importConfig.mappings) &&
+            lodash_1.default.isEmpty(importConfig.profiles) &&
+            lodash_1.default.isEmpty(importConfig.roles) &&
+            lodash_1.default.isEmpty(importConfig.userMappings) &&
+            lodash_1.default.isEmpty(importConfig.users) &&
+            lodash_1.default.isEmpty(supportConfig.fixtures) &&
+            lodash_1.default.isEmpty(supportConfig.mappings) &&
+            lodash_1.default.isEmpty(supportConfig.securities)) {
+            return true;
+        }
+        return false;
+    }
+    async persistHashedImport({ existingRedisHash, existingESHash, importPayloadHash, type, }) {
+        if (!existingRedisHash && !existingESHash) {
+            // If the import is not initialized in the redis cache and in the ES, we initialize it
+            this.log.info(`${type} import is not initialized, initializing...`);
+            await this.ask("core:storage:private:document:create", "kuzzle", "imports", {
+                hash: importPayloadHash,
+            }, { id: `${exports.BACKEND_IMPORT_KEY}:${type}` });
+            await this.ask("core:cache:internal:store", `${exports.BACKEND_IMPORT_KEY}:${type}`, importPayloadHash);
+        }
+        else if (existingRedisHash && !existingESHash) {
+            // If the import is initialized in the redis cache but not in the ES
+            // We initialize it in the ES
+            this.log.info(`${type} import is not initialized in %kuzzle.imports, initializing...`);
+            const redisCache = await this.ask("core:cache:internal:get", `${exports.BACKEND_IMPORT_KEY}:${type}`);
+            await this.ask("core:storage:private:document:create", "kuzzle", "imports", {
+                hash: redisCache,
+            }, { id: `${exports.BACKEND_IMPORT_KEY}:${type}` });
+        }
+        else if (!existingRedisHash && existingESHash) {
+            // If the import is initialized in the ES but not in the redis cache
+            // We initialize it in the redis cache
+            this.log.info(`${type} import is not initialized in the redis cache, initializing...`);
+            const esDocument = await this.ask("core:storage:private:document:get", "kuzzle", "imports", `${exports.BACKEND_IMPORT_KEY}:${type}`);
+            await this.ask("core:cache:internal:store", `${exports.BACKEND_IMPORT_KEY}:${type}`, esDocument._source.hash);
+        }
+    }
     /**
      * Load into the app several imports
      *
@@ -391,14 +430,7 @@ class Kuzzle extends kuzzleEventEmitter_1.default {
      * @returns {Promise<void>}
      */
     async loadInitialState(toImport = {}, toSupport = {}) {
-        if (lodash_1.default.isEmpty(toImport.mappings) &&
-            lodash_1.default.isEmpty(toImport.profiles) &&
-            lodash_1.default.isEmpty(toImport.roles) &&
-            lodash_1.default.isEmpty(toImport.userMappings) &&
-            lodash_1.default.isEmpty(toImport.users) &&
-            lodash_1.default.isEmpty(toSupport.fixtures) &&
-            lodash_1.default.isEmpty(toSupport.mappings) &&
-            lodash_1.default.isEmpty(toSupport.securities)) {
+        if (this.isConfigsEmpty(toImport, toSupport)) {
             return;
         }
         const lockedMutex = [];
@@ -422,17 +454,32 @@ class Kuzzle extends kuzzleEventEmitter_1.default {
                 }
                 const importPayloadHash = (0, crypto_1.sha256)((0, json_stable_stringify_1.default)(importPayload));
                 const mutex = new mutex_1.Mutex(`backend:import:${type}`, { timeout: 0 });
-                const existingHash = await this.ask("core:cache:internal:get", `${exports.BACKEND_IMPORT_KEY}:${type}`);
-                const initialized = existingHash === importPayloadHash;
+                const existingRedisHash = await this.ask("core:cache:internal:get", `${exports.BACKEND_IMPORT_KEY}:${type}`);
+                const existingESHash = await this.ask("core:storage:private:document:exist", "kuzzle", "imports", `${exports.BACKEND_IMPORT_KEY}:${type}`);
+                let initialized = false;
+                if (existingRedisHash) {
+                    // Check if the import is already initialized inside the redis cache
+                    initialized = existingRedisHash === importPayloadHash;
+                }
+                else if (existingESHash) {
+                    // Check if the import is already initialized inside the ES
+                    const esDocument = await this.ask("core:storage:private:document:get", "kuzzle", "imports", `${exports.BACKEND_IMPORT_KEY}:${type}`);
+                    initialized = esDocument._source.hash === importPayloadHash;
+                }
                 const locked = await mutex.lock();
                 await importMethod({ toImport, toSupport }, {
                     firstCall: !initialized && locked,
-                    initialized,
+                    initialized: initialized,
                     locked,
                 });
-                if (!initialized && locked) {
+                if (locked) {
                     lockedMutex.push(mutex);
-                    await this.ask("core:cache:internal:store", `${exports.BACKEND_IMPORT_KEY}:${type}`, importPayloadHash);
+                    await this.persistHashedImport({
+                        existingESHash,
+                        existingRedisHash,
+                        importPayloadHash,
+                        type,
+                    });
                 }
             }
             await this._waitForImportToFinish();

package/lib/types/OpenApiDefinition.d.ts

@@ -9,7 +9,6 @@ export type OpenApiDefinition = {
             name: string;
             url: string;
             email: string;
-            discord: string;
         };
         license: {
             name: string;
@@ -21,23 +20,11 @@ export type OpenApiDefinition = {
         description: string;
         url: string;
     };
-    servers: Array<{
-        url: string;
-        description: string;
-        variables: {
-            baseUrl: {
-                default: string;
-            };
-            port: {
-                default: number;
-            };
-        };
-    }>;
     tags: Array<{
         description: string;
         name: string;
     }>;
     schemes: string[];
     paths: JSONObject;
-    components: JSONObject;
+    components?: JSONObject;
 };

package/lib/types/config/storageEngine/StorageEngineElasticsearchConfiguration.d.ts

@@ -316,6 +316,26 @@ export type StorageEngineElasticsearch = {
                     };
                 };
             };
+            imports: {
+                settings: {
+                    /**
+                     * @default 1
+                     */
+                    number_of_shards: number;
+                    /**
+                     * @default 1
+                     */
+                    number_of_replicas: number;
+                };
+                mappings: {
+                    dynamic: "strict";
+                    properties: {
+                        hash: {
+                            type: "keyword";
+                        };
+                    };
+                };
+            };
         };
     };
     maxScrollDuration: string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "kuzzle",
   "author": "The Kuzzle Team <support@kuzzle.io>",
-  "version": "2.29.0",
+  "version": "2.29.1-beta.1",
   "description": "Kuzzle is an open-source solution that handles all the data management through a secured API, with a large choice of protocols.",
   "bin": "bin/start-kuzzle-server",
   "scripts": {

package/lib/api/openapi/components/document/count.yaml

@@ -1,47 +0,0 @@
-DocumentCount:
-  summary: "Counts documents in a collection."
-  tags:
-    - document
-  parameters:
-    - in: path
-      name: index
-      schema:
-        type: string
-    - in: path
-      name: collection
-      schema:
-        type: string
-    - in: body
-      name: "body"
-      description: "Counts documents in a collection."
-      required: true
-      schema:
-        $ref: "#/components/document/DocumentCountRequest"
-  responses:
-    200:
-      description: "Counts documents in a collection."
-      schema:
-        $ref: "#/components/document/DocumentCountResponse"
-
-components:
-  schemas:
-    DocumentCountRequest:
-      allOf:
-        - type: "object"
-          properties:
-            query:
-              type: "object"
-              properties:
-                match_all:
-                  type: "object"
-
-    DocumentCountResponse:
-      allOf:
-        - $ref: "#/components/ResponsePayload"
-        - type: "object"
-          properties:
-            result:
-              type: "object"
-              properties:
-                count:
-                  type: "integer"

package/lib/api/openapi/components/document/create.yaml

@@ -1,46 +0,0 @@
-DocumentCreate:
-  summary: "Creates a new document in the persistent data storage."
-  tags:
-    - document
-  parameters:
-    - in: path
-      name: index
-      schema:
-        type: string
-    - in: path
-      name: collection
-      schema:
-        type: string
-    - name: body
-      in: "body"
-      description: "Creates a new document in the persistent data storage."
-      required: true
-      schema:
-        $ref: "#/components/document/DocumentCreateRequest"
-  responses:
-    200:
-      description: "Creates a new document in the persistent data storage."
-      schema:
-        $ref: "#/components/document/DocumentCreateResponse"
-
-components:
-  schemas:
-    DocumentCreateRequest:
-      allOf:
-        - type: "object"
-          description: "document content"
-    DocumentCreateResponse:
-      allOf:
-        - $ref: "#/components/ResponsePayload"
-        - type: "object"
-          properties:
-            result:
-              type: "object"
-              properties:
-                _id:
-                  type: "string"
-                  description: "documentId"
-                _version:
-                  type: "integer"
-                _sources:
-                  type: "object"

package/lib/api/openapi/components/document/createOrReplace.yaml

@@ -1,61 +0,0 @@
-DocumentCreateOrReplace:
-  summary: "Creates a new document in the persistent data storage, or replaces its content if it already exists."
-  tags:
-    - document
-  parameters:
-    - in: path
-      name: index
-      schema:
-        type: string
-      required: true
-    - in: path
-      name: collection
-      schema:
-        type: string
-      required: true
-    - in: path
-      name: _id
-      schema:
-        type: integer
-      required: true
-    - in: path
-      name: refresh
-      schema:
-        type: string
-      required: false
-    - name: body
-      in: "body"
-      description: "Creates a new document in the persistent data storage, or replaces its content if it already exists."
-      required: true
-      schema:
-        $ref: "#/components/document/DocumentCreateOrReplaceRequest"
-  responses:
-    200:
-      description: "Creates a new document in the persistent data storage, or replaces its content if it already exists."
-      schema:
-        $ref: "#/components/document/DocumentCreateOrReplaceResponse"
-
-components:
-  schemas:
-    DocumentCreateOrReplaceRequest:
-      allOf:
-        - type: "object"
-          description: "document content"
-    DocumentCreateOrReplaceResponse:
-      allOf:
-        - $ref: "#/components/ResponsePayload"
-        - type: "object"
-          properties:
-            result:
-              type: "object"
-              properties:
-                _id:
-                  type: "string"
-                  description: "documentId"
-                _version:
-                  type: "integer"
-                _sources:
-                  type: "object"
-                  description: "document content"
-                created:
-                  type: "boolean"

package/lib/api/openapi/components/document/delete.yaml

@@ -1,67 +0,0 @@
-DocumentDelete:
-  summary: "Deletes a document."
-  tags:
-    - document
-  parameters:
-    - in: path
-      name: index
-      schema:
-        type: string
-      required: true
-    - in: path
-      name: collection
-      schema:
-        type: string
-      required: true
-    - in: path
-      name: documentId
-      schema:
-        type: string
-      required: true
-    - in: path
-      name: refresh
-      schema:
-        type: string
-      description: " if set to wait_for, Kuzzle will not respond until the deletion has been indexed"
-      required: false
-    - in: path
-      name: source
-      schema:
-        type: boolean
-      description: "if set to true Kuzzle will return the deleted document body in the response."
-      required: false
-    - in: path
-      name: silent
-      schema:
-        type: boolean
-      description: "if set, then Kuzzle will not generate notifications. Available since 2.9.2"
-      required: false
-  responses:
-    200:
-      description: "Deletes a document."
-      schema:
-        $ref: "#/components/document/DocumentDeleteResponse"
-
-components:
-  schemas:
-    DocumentDeleteRequest:
-      allOf:
-        - type: "object"
-          properties:
-            _id:
-              type: "string"
-              description: "documentId"
-            refresh:
-              type: "string"
-              description: " if set to wait_for, Kuzzle will not respond until the created/replaced document is indexed"
-    DocumentDeleteResponse:
-      allOf:
-        - $ref: "#/components/ResponsePayload"
-        - type: "object"
-          properties:
-            result:
-              type: "object"
-              properties:
-                _id:
-                  type: "string"
-                  description: "documentId"