kuzzle 2.29.0 → 2.29.1-beta.2

package/lib/api/httpRoutes.js

@@ -23,22 +23,6 @@
 
 "use strict";
 
-const {
-  OpenApiDocumentCount,
-  OpenApiDocumentDeleteByQuery,
-  OpenApiDocumentDelete,
-  OpenApiDocumentScroll,
-  OpenApiDocumentExists,
-  OpenApiDocumentUpdate,
-  OpenApiDocumentReplace,
-  OpenApiDocumentGet,
-  OpenApiDocumentCreate,
-  OpenApiDocumentCreateOrReplace,
-  OpenApiDocumentValidate,
-  OpenApiSecurityUpsertUser,
-  OpenApiDocumentmCreateOrReplace,
-} = require("./openapi/components");
-
 const routes = [
   // GET (idempotent)
   { verb: "get", path: "/_me", controller: "auth", action: "getCurrentUser" },
@@ -242,7 +226,6 @@ const routes = [
     path: "/:index/:collection/:_id",
     controller: "document",
     action: "get",
-    openapi: OpenApiDocumentGet,
   },
   {
     verb: "get",
@@ -255,7 +238,6 @@ const routes = [
     path: "/:index/:collection/:_id/_exists",
     controller: "document",
     action: "exists",
-    openapi: OpenApiDocumentExists,
   },
   {
     verb: "post",
@@ -268,7 +250,6 @@ const routes = [
     path: "/_scroll/:scrollId",
     controller: "document",
     action: "scroll",
-    openapi: OpenApiDocumentScroll,
   },
 
   {
@@ -770,14 +751,12 @@ const routes = [
     path: "/:index/:collection/_count",
     controller: "document",
     action: "count",
-    openapi: OpenApiDocumentCount,
   },
   {
     verb: "post",
     path: "/:index/:collection/_create",
     controller: "document",
     action: "create",
-    openapi: OpenApiDocumentCreate,
   },
   {
     verb: "post",
@@ -832,7 +811,6 @@ const routes = [
     path: "/:index/:collection/_validate",
     controller: "document",
     action: "validate",
-    openapi: OpenApiDocumentValidate,
   },
 
   {
@@ -955,7 +933,6 @@ const routes = [
     path: "/users/:_id/_upsert",
     controller: "security",
     action: "upsertUser",
-    openapi: OpenApiSecurityUpsertUser,
   },
   {
     verb: "post",
@@ -1241,7 +1218,6 @@ const routes = [
     path: "/:index/:collection/:_id",
     controller: "document",
     action: "delete",
-    openapi: OpenApiDocumentDelete,
   },
   {
     verb: "delete",
@@ -1254,7 +1230,6 @@ const routes = [
     path: "/:index/:collection/_query",
     controller: "document",
     action: "deleteByQuery",
-    openapi: OpenApiDocumentDeleteByQuery,
   },
   {
     verb: "delete",
@@ -1421,21 +1396,18 @@ const routes = [
     path: "/:index/:collection/:_id",
     controller: "document",
     action: "createOrReplace",
-    openapi: OpenApiDocumentCreateOrReplace,
   },
   {
     verb: "put",
     path: "/:index/:collection/_mCreateOrReplace",
     controller: "document",
     action: "mCreateOrReplace",
-    openapi: OpenApiDocumentmCreateOrReplace,
   },
   {
     verb: "put",
     path: "/:index/:collection/:_id/_replace",
     controller: "document",
     action: "replace",
-    openapi: OpenApiDocumentReplace,
   },
   {
     verb: "put",
@@ -1458,7 +1430,6 @@ const routes = [
     path: "/:index/:collection/:_id/_update",
     controller: "document",
     action: "update",
-    openapi: OpenApiDocumentUpdate,
     deprecated: {
       since: "2.11.0",
       message: 'Use "document:update" route with PATCH instead of PUT',

package/lib/api/openapi/OpenApiManager.js

@@ -21,7 +21,6 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.OpenApiManager = void 0;
-const components_1 = require("./components");
 const package_json_1 = require("../../../package.json");
 const openApiGenerator_1 = require("./openApiGenerator");
 class OpenApiManager {
@@ -40,7 +39,6 @@ class OpenApiManager {
                     name: "Kuzzle team",
                     url: "https://kuzzle.io",
                     email: "support@kuzzle.io",
-                    discord: "http://join.discord.kuzzle.io",
                 },
                 license: {
                     name: "Apache 2",
@@ -52,39 +50,9 @@ class OpenApiManager {
                 description: "Kuzzle API Documentation",
                 url: "https://docs.kuzzle.io/core/2/api/",
             },
-            servers: [
-                {
-                    url: "https://{baseUrl}:{port}",
-                    description: "Kuzzle Base Url",
-                    variables: {
-                        baseUrl: { default: "localhost" },
-                        port: { default: 7512 },
-                    },
-                },
-            ],
             tags: [],
             schemes: ["https", "http"],
             paths: {},
-            components: {
-                ...components_1.OpenApiPayloadsDefinitions,
-                document: {
-                    ...components_1.OpenApiDocumentCountComponent,
-                    ...components_1.OpenApiDocumentDeleteByQueryComponent,
-                    ...components_1.OpenApiDocumentDeleteComponent,
-                    ...components_1.OpenApiDocumentScrollComponent,
-                    ...components_1.OpenApiDocumentExistsComponent,
-                    ...components_1.OpenApiDocumentUpdateComponent,
-                    ...components_1.OpenApiDocumentReplaceComponent,
-                    ...components_1.OpenApiDocumentGetComponent,
-                    ...components_1.OpenApiDocumentCreateOrReplaceComponent,
-                    ...components_1.OpenApiDocumentCreateComponent,
-                    ...components_1.OpenApiDocumentValidateComponent,
-                },
-                security: {
-                    ...components_1.OpenApiSecurityUpsertUserComponent,
-                    ...components_1.OpenApiDocumentmCreateOrReplaceComponent,
-                },
-            },
         };
         this.applicationDefinition = applicationDefinition;
         (0, openApiGenerator_1.generateOpenApi)(kuzzleRoutes, this.kuzzleDefinition);

package/lib/api/openapi/openApiGenerator.js

@@ -72,7 +72,7 @@ function generateParameters(route) {
                 in: "path",
                 name: m[1],
                 required: true,
-                schema: { type: "string" },
+                type: "string",
             });
             m = routeUrlMatch.exec(route.path);
         }

package/lib/config/default.config.js

@@ -416,6 +416,19 @@ const defaultConfig = {
                             },
                         },
                     },
+                    imports: {
+                        settings: {
+                            // @deprecated : replace undefined by 1
+                            number_of_shards: 1,
+                            number_of_replicas: 1,
+                        },
+                        mappings: {
+                            dynamic: "strict",
+                            properties: {
+                                hash: { type: "keyword" },
+                            },
+                        },
+                    },
                 },
             },
             maxScrollDuration: "1m",

package/lib/core/backend/backendOpenApi.js

@@ -27,7 +27,7 @@ class BackendOpenApi extends index_1.ApplicationManager {
         super(application);
         /* eslint-disable sort-keys */
         this.definition = {
-            openapi: "3.0.0",
+            openapi: "3.1.0",
             info: {
                 title: `${application.name} API`,
                 description: `${application.name} HTTP API definition`,
@@ -35,7 +35,6 @@ class BackendOpenApi extends index_1.ApplicationManager {
                     name: "Kuzzle team",
                     url: "https://kuzzle.io",
                     email: "support@kuzzle.io",
-                    discord: "http://join.discord.kuzzle.io",
                 },
                 license: {
                     name: "Apache 2",
@@ -47,20 +46,9 @@ class BackendOpenApi extends index_1.ApplicationManager {
                 description: "Kuzzle API Documentation",
                 url: "https://docs.kuzzle.io/core/2/api/",
             },
-            servers: [
-                {
-                    url: "https://{baseUrl}:{port}",
-                    description: `${application.name} Base Url`,
-                    variables: {
-                        baseUrl: { default: "localhost" },
-                        port: { default: 7512 },
-                    },
-                },
-            ],
             tags: [],
             schemes: ["https", "http"],
             paths: {},
-            components: {},
         };
         /* eslint-enable sort-keys */
     }

package/lib/kuzzle/kuzzle.d.ts

@@ -1 +1,134 @@
+/// <reference types="node" />
+import KuzzleEventEmitter from "./event/kuzzleEventEmitter";
+import { InstallationConfig, ImportConfig, SupportConfig, StartOptions } from "./../types/Kuzzle";
+import { KuzzleConfiguration } from "../types/config/KuzzleConfiguration";
 export declare const BACKEND_IMPORT_KEY = "backend:init:import";
+declare class Kuzzle extends KuzzleEventEmitter {
+    private config;
+    private _state;
+    private log;
+    private rootPath;
+    /**
+     * Internal index bootstrapper and accessor
+     */
+    private internalIndex;
+    private pluginsManager;
+    private tokenManager;
+    private passport;
+    /**
+     * The funnel dispatches messages to API controllers
+     */
+    private funnel;
+    /**
+     * The router listens to client requests and pass them to the funnel
+     */
+    private router;
+    /**
+     * Statistics core component
+     */
+    private statistics;
+    /**
+     * Network entry point
+     */
+    private entryPoint;
+    /**
+     * Validation core component
+     */
+    private validation;
+    /**
+     * Dump generator
+     */
+    private dumpGenerator;
+    /**
+     * Vault component (will be initialized after bootstrap)
+     */
+    private vault;
+    /**
+     * AsyncLocalStorage wrapper
+     */
+    private asyncStore;
+    /**
+     * Kuzzle internal debugger
+     */
+    private debugger;
+    /**
+     * Kuzzle version
+     */
+    private version;
+    private openApiManager;
+    /**
+     * List of differents imports types and their associated method
+     */
+    private importTypes;
+    private koncorde;
+    private secret;
+    /**
+     * Node unique ID amongst other cluster nodes
+     */
+    id: string;
+    constructor(config: KuzzleConfiguration);
+    /**
+     * Initializes all the needed components of Kuzzle.
+     *
+     * @param {Application} - Application Plugin instance
+     * @param {Object} - Additional options (import, installations, plugins, secretsFile, support, vaultKey)
+     *
+     * @this {Kuzzle}
+     */
+    start(application: any, options?: StartOptions): Promise<void>;
+    /**
+     * Generates the node ID.
+     *
+     * This will init the cluster if it's enabled.
+     */
+    private initKuzzleNode;
+    /**
+     * Gracefully exits after processing remaining requests
+     *
+     * @returns {Promise}
+     */
+    shutdown(): Promise<void>;
+    /**
+     * Execute multiple handlers only once on any given environment
+     *
+     * @param {Array<{ id: string, handler: () => void, description?: string }>} installations - Array of unique methods to execute
+     *
+     * @returns {Promise<void>}
+     */
+    install(installations: InstallationConfig[]): Promise<void>;
+    ask(...args: any[]): Promise<any>;
+    emit(...args: any[]): Promise<any>;
+    pipe(...args: any[]): Promise<any>;
+    private importUserMappings;
+    private importMappings;
+    private importFixtures;
+    private importPermissions;
+    /**
+     * Check if every import has been done, if one of them is not finished yet, wait for it
+     */
+    private _waitForImportToFinish;
+    private isConfigsEmpty;
+    private persistHashedImport;
+    /**
+     * Load into the app several imports
+     *
+     * @param {Object} toImport - Contains `mappings`, `onExistingUsers`, `profiles`, `roles`, `userMappings`, `users`
+     * @param {Object} toSupport - Contains `fixtures`, `mappings`, `securities` (`profiles`, `roles`, `users`)
+     *
+     * @returns {Promise<void>}
+     */
+    loadInitialState(toImport?: ImportConfig, toSupport?: SupportConfig): Promise<void>;
+    dump(suffix: any): any;
+    hash(input: any): string | Buffer;
+    get state(): kuzzleStateEnum;
+    set state(value: kuzzleStateEnum);
+    /**
+     * Register handlers and do a kuzzle dump for:
+     * - system signals
+     * - unhandled-rejection
+     * - uncaught-exception
+     */
+    registerSignalHandlers(): void;
+    dumpAndExit(suffix: any): Promise<void>;
+}
+export { Kuzzle };

package/lib/kuzzle/kuzzle.js

@@ -46,7 +46,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BACKEND_IMPORT_KEY = void 0;
+exports.Kuzzle = exports.BACKEND_IMPORT_KEY = void 0;
 const path_1 = __importDefault(require("path"));
 const murmurhash_native_1 = require("murmurhash-native");
 const json_stable_stringify_1 = __importDefault(require("json-stable-stringify"));
@@ -382,6 +382,45 @@ class Kuzzle extends kuzzleEventEmitter_1.default {
             await bluebird_1.default.delay(1000);
         }
     }
+    isConfigsEmpty(importConfig, supportConfig) {
+        if (lodash_1.default.isEmpty(importConfig.mappings) &&
+            lodash_1.default.isEmpty(importConfig.profiles) &&
+            lodash_1.default.isEmpty(importConfig.roles) &&
+            lodash_1.default.isEmpty(importConfig.userMappings) &&
+            lodash_1.default.isEmpty(importConfig.users) &&
+            lodash_1.default.isEmpty(supportConfig.fixtures) &&
+            lodash_1.default.isEmpty(supportConfig.mappings) &&
+            lodash_1.default.isEmpty(supportConfig.securities)) {
+            return true;
+        }
+        return false;
+    }
+    async persistHashedImport({ existingRedisHash, existingESHash, importPayloadHash, type, }) {
+        if (!existingRedisHash && !existingESHash) {
+            // If the import is not initialized in the redis cache and in the ES, we initialize it
+            this.log.info(`${type} import is not initialized, initializing...`);
+            await this.ask("core:storage:private:document:create", "kuzzle", "imports", {
+                hash: importPayloadHash,
+            }, { id: `${exports.BACKEND_IMPORT_KEY}:${type}` });
+            await this.ask("core:cache:internal:store", `${exports.BACKEND_IMPORT_KEY}:${type}`, importPayloadHash);
+        }
+        else if (existingRedisHash && !existingESHash) {
+            // If the import is initialized in the redis cache but not in the ES
+            // We initialize it in the ES
+            this.log.info(`${type} import is not initialized in %kuzzle.imports, initializing...`);
+            const redisCache = await this.ask("core:cache:internal:get", `${exports.BACKEND_IMPORT_KEY}:${type}`);
+            await this.ask("core:storage:private:document:create", "kuzzle", "imports", {
+                hash: redisCache,
+            }, { id: `${exports.BACKEND_IMPORT_KEY}:${type}` });
+        }
+        else if (!existingRedisHash && existingESHash) {
+            // If the import is initialized in the ES but not in the redis cache
+            // We initialize it in the redis cache
+            this.log.info(`${type} import is not initialized in the redis cache, initializing...`);
+            const esDocument = await this.ask("core:storage:private:document:get", "kuzzle", "imports", `${exports.BACKEND_IMPORT_KEY}:${type}`);
+            await this.ask("core:cache:internal:store", `${exports.BACKEND_IMPORT_KEY}:${type}`, esDocument._source.hash);
+        }
+    }
     /**
      * Load into the app several imports
      *
@@ -391,14 +430,7 @@ class Kuzzle extends kuzzleEventEmitter_1.default {
      * @returns {Promise<void>}
      */
     async loadInitialState(toImport = {}, toSupport = {}) {
-        if (lodash_1.default.isEmpty(toImport.mappings) &&
-            lodash_1.default.isEmpty(toImport.profiles) &&
-            lodash_1.default.isEmpty(toImport.roles) &&
-            lodash_1.default.isEmpty(toImport.userMappings) &&
-            lodash_1.default.isEmpty(toImport.users) &&
-            lodash_1.default.isEmpty(toSupport.fixtures) &&
-            lodash_1.default.isEmpty(toSupport.mappings) &&
-            lodash_1.default.isEmpty(toSupport.securities)) {
+        if (this.isConfigsEmpty(toImport, toSupport)) {
             return;
         }
         const lockedMutex = [];
@@ -422,17 +454,32 @@ class Kuzzle extends kuzzleEventEmitter_1.default {
                 }
                 const importPayloadHash = (0, crypto_1.sha256)((0, json_stable_stringify_1.default)(importPayload));
                 const mutex = new mutex_1.Mutex(`backend:import:${type}`, { timeout: 0 });
-                const existingHash = await this.ask("core:cache:internal:get", `${exports.BACKEND_IMPORT_KEY}:${type}`);
-                const initialized = existingHash === importPayloadHash;
+                const existingRedisHash = await this.ask("core:cache:internal:get", `${exports.BACKEND_IMPORT_KEY}:${type}`);
+                const existingESHash = await this.ask("core:storage:private:document:exist", "kuzzle", "imports", `${exports.BACKEND_IMPORT_KEY}:${type}`);
+                let initialized = false;
+                if (existingRedisHash) {
+                    // Check if the import is already initialized inside the redis cache
+                    initialized = existingRedisHash === importPayloadHash;
+                }
+                else if (existingESHash) {
+                    // Check if the import is already initialized inside the ES
+                    const esDocument = await this.ask("core:storage:private:document:get", "kuzzle", "imports", `${exports.BACKEND_IMPORT_KEY}:${type}`);
+                    initialized = esDocument._source.hash === importPayloadHash;
+                }
                 const locked = await mutex.lock();
                 await importMethod({ toImport, toSupport }, {
                     firstCall: !initialized && locked,
-                    initialized,
+                    initialized: initialized,
                     locked,
                 });
-                if (!initialized && locked) {
+                if (locked) {
                     lockedMutex.push(mutex);
-                    await this.ask("core:cache:internal:store", `${exports.BACKEND_IMPORT_KEY}:${type}`, importPayloadHash);
+                    await this.persistHashedImport({
+                        existingESHash,
+                        existingRedisHash,
+                        importPayloadHash,
+                        type,
+                    });
                 }
             }
             await this._waitForImportToFinish();
@@ -536,5 +583,6 @@ class Kuzzle extends kuzzleEventEmitter_1.default {
         await this.shutdown();
     }
 }
+exports.Kuzzle = Kuzzle;
 module.exports = Kuzzle;
 //# sourceMappingURL=kuzzle.js.map

package/lib/types/Global.d.ts

@@ -1,11 +1,13 @@
 import { Backend } from "../core/backend";
+import { Kuzzle } from "../kuzzle";
+/**
+ * This file contains global type declarations for Kuzzle.
+ * We need to use `var` so Typescript extends the globalThis type.
+ * See https://www.typescriptlang.org/docs/handbook/release-notes/typescript-3-4.html#type-checking-for-globalthis
+ */
 declare global {
-    namespace NodeJS {
-        interface Global {
-            kuzzle: any;
-            app: Backend;
-            NODE_ENV: string;
-        }
-    }
+    var app: Backend;
+    var kuzzle: Kuzzle;
+    var NODE_ENV: string;
 }
 export {};

package/lib/types/Global.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 global.NODE_ENV = process.env.NODE_ENV;
-/* eslint-enable @typescript-eslint/no-namespace */
+/* eslint-enable no-var, vars-on-top*/
 //# sourceMappingURL=Global.js.map

package/lib/types/OpenApiDefinition.d.ts

@@ -9,7 +9,6 @@ export type OpenApiDefinition = {
             name: string;
             url: string;
             email: string;
-            discord: string;
         };
         license: {
             name: string;
@@ -21,23 +20,11 @@ export type OpenApiDefinition = {
         description: string;
         url: string;
     };
-    servers: Array<{
-        url: string;
-        description: string;
-        variables: {
-            baseUrl: {
-                default: string;
-            };
-            port: {
-                default: number;
-            };
-        };
-    }>;
     tags: Array<{
         description: string;
         name: string;
     }>;
     schemes: string[];
     paths: JSONObject;
-    components: JSONObject;
+    components?: JSONObject;
 };

package/lib/types/config/storageEngine/StorageEngineElasticsearchConfiguration.d.ts

@@ -316,6 +316,26 @@ export type StorageEngineElasticsearch = {
                     };
                 };
             };
+            imports: {
+                settings: {
+                    /**
+                     * @default 1
+                     */
+                    number_of_shards: number;
+                    /**
+                     * @default 1
+                     */
+                    number_of_replicas: number;
+                };
+                mappings: {
+                    dynamic: "strict";
+                    properties: {
+                        hash: {
+                            type: "keyword";
+                        };
+                    };
+                };
+            };
         };
     };
     maxScrollDuration: string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "kuzzle",
   "author": "The Kuzzle Team <support@kuzzle.io>",
-  "version": "2.29.0",
+  "version": "2.29.1-beta.2",
   "description": "Kuzzle is an open-source solution that handles all the data management through a secured API, with a large choice of protocols.",
   "bin": "bin/start-kuzzle-server",
   "scripts": {

package/lib/api/openapi/components/document/count.yaml

@@ -1,47 +0,0 @@
-DocumentCount:
-  summary: "Counts documents in a collection."
-  tags:
-    - document
-  parameters:
-    - in: path
-      name: index
-      schema:
-        type: string
-    - in: path
-      name: collection
-      schema:
-        type: string
-    - in: body
-      name: "body"
-      description: "Counts documents in a collection."
-      required: true
-      schema:
-        $ref: "#/components/document/DocumentCountRequest"
-  responses:
-    200:
-      description: "Counts documents in a collection."
-      schema:
-        $ref: "#/components/document/DocumentCountResponse"
-
-components:
-  schemas:
-    DocumentCountRequest:
-      allOf:
-        - type: "object"
-          properties:
-            query:
-              type: "object"
-              properties:
-                match_all:
-                  type: "object"
-
-    DocumentCountResponse:
-      allOf:
-        - $ref: "#/components/ResponsePayload"
-        - type: "object"
-          properties:
-            result:
-              type: "object"
-              properties:
-                count:
-                  type: "integer"