kuzzle 2.21.0 → 2.22.0

package/lib/api/controllers/debugController.js

@@ -42,10 +42,14 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DebugController = void 0;
 const baseController_1 = require("./baseController");
 const kerror = __importStar(require("../../kerror"));
+const get_1 = __importDefault(require("lodash/get"));
 /**
  * @class DebugController
  */
@@ -70,12 +74,18 @@ class DebugController extends baseController_1.NativeController {
      * Connect the debugger
      */
     async enable() {
+        if (!(0, get_1.default)(global.kuzzle.config, "security.debug.native_debug_protocol")) {
+            throw kerror.get("core", "debugger", "native_debug_protocol_usage_denied");
+        }
         await global.kuzzle.ask("core:debugger:enable");
     }
     /**
      * Disconnect the debugger and clears all the events listeners
      */
     async disable() {
+        if (!(0, get_1.default)(global.kuzzle.config, "security.debug.native_debug_protocol")) {
+            throw kerror.get("core", "debugger", "native_debug_protocol_usage_denied");
+        }
         await global.kuzzle.ask("core:debugger:disable");
     }
     /**
@@ -83,6 +93,9 @@ class DebugController extends baseController_1.NativeController {
      * See: https://chromedevtools.github.io/devtools-protocol/v8/
      */
     async post(request) {
+        if (!(0, get_1.default)(global.kuzzle.config, "security.debug.native_debug_protocol")) {
+            throw kerror.get("core", "debugger", "native_debug_protocol_usage_denied");
+        }
         const method = request.getBodyString("method");
         const params = request.getBodyObject("params", {});
         return global.kuzzle.ask("core:debugger:post", method, params);
@@ -92,6 +105,9 @@ class DebugController extends baseController_1.NativeController {
      * See events from: https://chromedevtools.github.io/devtools-protocol/v8/
      */
     async addListener(request) {
+        if (!(0, get_1.default)(global.kuzzle.config, "security.debug.native_debug_protocol")) {
+            throw kerror.get("core", "debugger", "native_debug_protocol_usage_denied");
+        }
         if (request.context.connection.protocol !== "websocket") {
             throw kerror.get("api", "assert", "unsupported_protocol", request.context.connection.protocol, "debug:addListener");
         }
@@ -102,6 +118,9 @@ class DebugController extends baseController_1.NativeController {
      * Remove the websocket connection from the events' listeners
      */
     async removeListener(request) {
+        if (!(0, get_1.default)(global.kuzzle.config, "security.debug.native_debug_protocol")) {
+            throw kerror.get("core", "debugger", "native_debug_protocol_usage_denied");
+        }
         if (request.context.connection.protocol !== "websocket") {
             throw kerror.get("api", "assert", "unsupported_protocol", request.context.connection.protocol, "debug:removeListener");
         }

package/lib/api/funnel.js

@@ -51,6 +51,7 @@ const debug = require("../util/debug")("kuzzle:funnel");
 const processError = kerror.wrap("api", "process");
 const { has } = require("../util/safeObject");
 const { HttpStream } = require("../types");
+const get = require("lodash/get");
 
 // Actions of the auth controller that does not necessite to verify the token
 // when cookie auth is active
@@ -179,6 +180,12 @@ class Funnel {
       throw processError.get("not_enough_nodes");
     }
 
+    const isRequestFromDebugSession = get(
+      request,
+      "context.connection.misc.internal.debugSession",
+      false
+    );
+
     if (this.overloaded) {
       const now = Date.now();
 
@@ -226,7 +233,8 @@ class Funnel {
      */
     if (
       this.pendingRequestsQueue.length >=
-      global.kuzzle.config.limits.requestsBufferSize
+        global.kuzzle.config.limits.requestsBufferSize &&
+      !isRequestFromDebugSession
     ) {
       const error = processError.get("overloaded");
       global.kuzzle.emit("log:error", error);
@@ -239,7 +247,13 @@ class Funnel {
         request.internalId,
         new PendingRequest(request, fn, context)
       );
-      this.pendingRequestsQueue.push(request.internalId);
+
+      if (isRequestFromDebugSession) {
+        // Push at the front to prioritize debug requests
+        this.pendingRequestsQueue.unshift(request.internalId);
+      } else {
+        this.pendingRequestsQueue.push(request.internalId);
+      }
 
       if (!this.overloaded) {
         this.overloaded = true;

package/lib/cluster/node.js

@@ -256,6 +256,15 @@ class ClusterNode {
    */
   preventEviction(evictionPrevented) {
     this.publisher.sendNodePreventEviction(evictionPrevented);
+    // This node is subscribed to the other node and might not receive their heartbeat while debugging
+    // so this node should not have the responsability of evicting others when his own eviction is prevented
+    // when debugging.
+    // Otherwise when recovering from a debug session, all the other nodes will be evicted.
+    for (const subscriber of this.remoteNodes.values()) {
+      subscriber.handleNodePreventEviction({
+        evictionPrevented,
+      });
+    }
   }
 
   /**

package/lib/cluster/subscriber.js

@@ -680,7 +680,15 @@ class ClusterSubscriber {
    * to recover, otherwise we evict it from the cluster.
    */
   async checkHeartbeat() {
-    if (this.state === stateEnum.EVICTED || this.remoteNodeEvictionPrevented) {
+    if (this.remoteNodeEvictionPrevented) {
+      // Fake the heartbeat while the node eviction prevention is enabled
+      // otherwise when the node eviction prevention is disabled
+      // the node will be evicted if it did not send a heartbeat before disabling the protection.
+      this.lastHeartbeat = Date.now();
+      return;
+    }
+
+    if (this.state === stateEnum.EVICTED) {
       return;
     }
 

package/lib/cluster/workers/IDCardRenewer.js

@@ -29,6 +29,10 @@ class IDCardRenewer {
       const redisConf = config.redis || {};
       await this.initRedis(redisConf.config, redisConf.name);
     } catch (error) {
+      // eslint-disable-next-line no-console
+      console.error(
+        `Failed to connect to redis, could not refresh ID card: ${error.message}`
+      );
       this.parentPort.postMessage({
         error: `Failed to connect to redis, could not refresh ID card: ${error.message}`,
       });

package/lib/core/backend/backend.d.ts

@@ -1,5 +1,5 @@
 import { EmbeddedSDK } from "../shared/sdk/embeddedSdk";
-import { EventDefinition, JSONObject } from "../../../index";
+import { BackendSubscription, EventDefinition, JSONObject } from "../../../index";
 import { BackendCluster, BackendConfig, BackendController, BackendHook, BackendImport, BackendPipe, BackendPlugin, BackendStorage, BackendVault, BackendOpenApi, InternalLogger, BackendErrors } from "./index";
 export declare class Backend {
     private _kuzzle;
@@ -121,6 +121,7 @@ export declare class Backend {
      * Standard errors
      */
     errors: BackendErrors;
+    subscription: BackendSubscription;
     /**
      * @deprecated
      *

package/lib/core/backend/backend.js

@@ -51,7 +51,8 @@ const fs_1 = __importDefault(require("fs"));
 const kuzzle_1 = __importDefault(require("../../kuzzle"));
 const embeddedSdk_1 = require("../shared/sdk/embeddedSdk");
 const kerror = __importStar(require("../../kerror"));
-const index_1 = require("./index");
+const index_1 = require("../../../index");
+const index_2 = require("./index");
 const assertionError = kerror.wrap("plugin", "assert");
 const runtimeError = kerror.wrap("plugin", "runtime");
 let _app = null;
@@ -139,18 +140,19 @@ class Backend {
             // Silent if no version can be found
         }
         global.app = this;
-        this.pipe = new index_1.BackendPipe(this);
-        this.hook = new index_1.BackendHook(this);
-        this.config = new index_1.BackendConfig(this);
-        this.vault = new index_1.BackendVault(this);
-        this.controller = new index_1.BackendController(this);
-        this.plugin = new index_1.BackendPlugin(this);
-        this.storage = new index_1.BackendStorage(this);
-        this.import = new index_1.BackendImport(this);
-        this.log = new index_1.InternalLogger(this);
-        this.cluster = new index_1.BackendCluster();
-        this.openApi = new index_1.BackendOpenApi(this);
-        this.errors = new index_1.BackendErrors(this);
+        this.pipe = new index_2.BackendPipe(this);
+        this.hook = new index_2.BackendHook(this);
+        this.config = new index_2.BackendConfig(this);
+        this.vault = new index_2.BackendVault(this);
+        this.controller = new index_2.BackendController(this);
+        this.plugin = new index_2.BackendPlugin(this);
+        this.storage = new index_2.BackendStorage(this);
+        this.import = new index_2.BackendImport(this);
+        this.log = new index_2.InternalLogger(this);
+        this.cluster = new index_2.BackendCluster();
+        this.openApi = new index_2.BackendOpenApi(this);
+        this.errors = new index_2.BackendErrors(this);
+        this.subscription = new index_1.BackendSubscription(this);
         this.kerror = kerror;
         try {
             this.commit = this._readCommit();

package/lib/core/backend/backendSubscription.d.ts

@@ -0,0 +1,25 @@
+import { JSONObject } from "kuzzle-sdk";
+import { Connection } from "../../api/request";
+import { ApplicationManager } from "./index";
+export declare class BackendSubscription extends ApplicationManager {
+    /**
+     * Registers a new realtime subscription on the specified connection
+     *
+     * @param connection Connection to register the subscription on
+     * @param index Index name
+     * @param collection Collection name
+     * @param filters Subscription filters
+     * @param options.volatile Volatile data
+     * @param options.scope Subscription scope
+     * @param options.users Option for users notifications
+     */
+    add(connection: Connection, index: string, collection: string, filters: JSONObject, { volatile, scope, users, }?: {
+        volatile?: JSONObject;
+        scope?: "in" | "out" | "all" | "none";
+        users?: "in" | "out" | "all" | "none";
+    }): Promise<{
+        roomId: string;
+        channel: string;
+    }>;
+    remove(connection: Connection, roomId: string): Promise<void>;
+}

package/lib/core/backend/backendSubscription.js

@@ -0,0 +1,90 @@
+"use strict";
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BackendSubscription = void 0;
+const request_1 = require("../../api/request");
+const kerror = __importStar(require("../../kerror"));
+const index_1 = require("./index");
+const runtimeError = kerror.wrap("plugin", "runtime");
+class BackendSubscription extends index_1.ApplicationManager {
+    /**
+     * Registers a new realtime subscription on the specified connection
+     *
+     * @param connection Connection to register the subscription on
+     * @param index Index name
+     * @param collection Collection name
+     * @param filters Subscription filters
+     * @param options.volatile Volatile data
+     * @param options.scope Subscription scope
+     * @param options.users Option for users notifications
+     */
+    async add(connection, index, collection, filters, { volatile, scope, users, } = {}) {
+        if (!this._application.started) {
+            throw runtimeError.get("unavailable_before_start", "subscriptions.add");
+        }
+        const subscriptionRequest = new request_1.KuzzleRequest({
+            action: "subscribe",
+            body: filters,
+            collection,
+            controller: "realtime",
+            index,
+            scope,
+            users,
+        }, {
+            connectionId: connection.id,
+            volatile,
+        });
+        const { channel, roomId } = await global.kuzzle.ask("core:realtime:subscribe", subscriptionRequest);
+        return { channel, roomId };
+    }
+    async remove(connection, roomId) {
+        if (!this._application.started) {
+            throw runtimeError.get("unavailable_before_start", "subscriptions.remove");
+        }
+        await global.kuzzle.ask("core:realtime:unsubscribe", connection.id, roomId);
+    }
+}
+exports.BackendSubscription = BackendSubscription;
+//# sourceMappingURL=backendSubscription.js.map

package/lib/core/backend/index.d.ts

@@ -12,3 +12,4 @@ export * from "./backendVault";
 export * from "./backendOpenApi";
 export * from "./internalLogger";
 export * from "./backendErrors";
+export * from "./backendSubscription";

package/lib/core/backend/index.js

@@ -28,4 +28,5 @@ __exportStar(require("./backendVault"), exports);
 __exportStar(require("./backendOpenApi"), exports);
 __exportStar(require("./internalLogger"), exports);
 __exportStar(require("./backendErrors"), exports);
+__exportStar(require("./backendSubscription"), exports);
 //# sourceMappingURL=index.js.map

package/lib/core/debug/kuzzleDebugger.d.ts

@@ -6,6 +6,7 @@ export declare class KuzzleDebugger {
      * Map<eventName, Set<connectionId>>
      */
     private events;
+    private httpWsProtocol?;
     init(): Promise<void>;
     registerAsks(): Promise<void>;
     /**

package/lib/core/debug/kuzzleDebugger.js

@@ -29,7 +29,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.KuzzleDebugger = void 0;
 const inspector_1 = __importDefault(require("inspector"));
 const kerror = __importStar(require("../../kerror"));
-const get_1 = __importDefault(require("lodash/get"));
 const DEBUGGER_EVENT = "kuzzle-debugger-event";
 class KuzzleDebugger {
     constructor() {
@@ -40,6 +39,7 @@ class KuzzleDebugger {
         this.events = new Map();
     }
     async init() {
+        this.httpWsProtocol = global.kuzzle.entryPoint.protocols.get("websocket");
         this.inspector = new inspector_1.default.Session();
         // Remove connection id from the list of listeners for each event
         global.kuzzle.on("connection:remove", (connectionId) => {
@@ -100,6 +100,17 @@ class KuzzleDebugger {
         this.inspector.disconnect();
         this.debuggerStatus = false;
         await global.kuzzle.ask("cluster:node:preventEviction", false);
+        // Disable debug mode for all connected sockets that still have listeners
+        if (this.httpWsProtocol) {
+            for (const eventName of this.events.keys()) {
+                for (const connectionId of this.events.get(eventName)) {
+                    const socket = this.httpWsProtocol.socketByConnectionId.get(connectionId);
+                    if (socket) {
+                        socket.internal.debugSession = false;
+                    }
+                }
+            }
+        }
         this.events.clear();
     }
     /**
@@ -110,16 +121,33 @@ class KuzzleDebugger {
         if (!this.debuggerStatus) {
             throw kerror.get("core", "debugger", "not_enabled");
         }
-        if (!(0, get_1.default)(global.kuzzle.config, "security.debug.native_debug_protocol")) {
-            throw kerror.get("core", "debugger", "native_debug_protocol_usage_denied");
-        }
-        // Always disable report progress because this params causes a segfault.
+        // Always disable report progress because this parameter causes a segfault.
         // The reason this happens is because the inspector is running inside the same thread
-        // as the Kuzzle Process and reportProgress forces the inspector to send events
-        // to the main thread, while it is being inspected by the HeapProfiler, which causes javascript code
-        // to be executed as the HeapProfiler is running, which causes a segfault.
+        // as the Kuzzle Process and reportProgress forces the inspector to call function in the JS Heap
+        // while it is being inspected by the HeapProfiler, which causes a segfault.
         // See: https://github.com/nodejs/node/issues/44634
-        params.reportProgress = false;
+        if (params.reportProgress) {
+            // We need to send a fake HeapProfiler.reportHeapSnapshotProgress event
+            // to the inspector to make Chrome think that the HeapProfiler is done
+            // otherwise, even though the Chrome Inspector did receive the whole snapshot, it will not be parsed.
+            //
+            // Chrome inspector is waiting for a HeapProfiler.reportHeapSnapshotProgress event with the finished property set to true
+            // The `done` and `total` properties are only used to show a progress bar, so there are not important.
+            // Sending this event before the HeapProfiler.addHeapSnapshotChunk event will not cause any problem,
+            // in fact, Chrome always do that when taking a snapshot, it receives the HeapProfiler.reportHeapSnapshotProgress event
+            // before the HeapProfiler.addHeapSnapshotChunk event.
+            // So this will have no impact and when receiving the HeapProfiler.addHeapSnapshotChunk event, Chrome will wait to receive
+            // a complete snapshot before parsing it if it has received the HeapProfiler.reportHeapSnapshotProgress event with the finished property set to true before.
+            this.inspector.emit("inspectorNotification", {
+                method: "HeapProfiler.reportHeapSnapshotProgress",
+                params: {
+                    done: 0,
+                    finished: true,
+                    total: 0,
+                },
+            });
+            params.reportProgress = false;
+        }
         return this.inspectorPost(method, params);
     }
     /**
@@ -130,6 +158,17 @@ class KuzzleDebugger {
         if (!this.debuggerStatus) {
             throw kerror.get("core", "debugger", "not_enabled");
         }
+        if (this.httpWsProtocol) {
+            const socket = this.httpWsProtocol.socketByConnectionId.get(connectionId);
+            if (socket) {
+                /**
+                 * Mark the socket as a debugging socket
+                 * this will bypass some limitations like the max pressure buffer size,
+                 * which could end the connection when the debugger is sending a lot of data.
+                 */
+                socket.internal.debugSession = true;
+            }
+        }
         let listeners = this.events.get(event);
         if (!listeners) {
             listeners = new Set();
@@ -148,6 +187,28 @@ class KuzzleDebugger {
         if (listeners) {
             listeners.delete(connectionId);
         }
+        if (!this.httpWsProtocol) {
+            return;
+        }
+        const socket = this.httpWsProtocol.socketByConnectionId.get(connectionId);
+        if (!socket) {
+            return;
+        }
+        let removeDebugSessionMarker = true;
+        /**
+         * If the connection doesn't listen to any other events
+         * we can remove the debugSession marker
+         */
+        for (const eventName of this.events.keys()) {
+            const eventListener = this.events.get(eventName);
+            if (eventListener && eventListener.has(connectionId)) {
+                removeDebugSessionMarker = false;
+                break;
+            }
+        }
+        if (removeDebugSessionMarker) {
+            socket.internal.debugSession = false;
+        }
     }
     /**
      * Execute a method using the Chrome Debug Protocol

package/lib/core/network/clientConnection.js

@@ -31,10 +31,11 @@ const uuid = require("uuid");
  * @param {object} [headers] - Optional extra key-value object. I.e., for http, will receive the request headers
  */
 class ClientConnection {
-  constructor(protocol, ips, headers = null) {
+  constructor(protocol, ips, headers = null, internal = null) {
     this.id = uuid.v4();
     this.protocol = protocol;
     this.headers = {};
+    this.internal = {};
 
     if (!Array.isArray(ips)) {
       throw new TypeError(`Expected ips to be an Array, got ${typeof ips}`);
@@ -45,6 +46,10 @@ class ClientConnection {
       this.headers = headers;
     }
 
+    if (isPlainObject(internal)) {
+      this.internal = internal;
+    }
+
     Object.freeze(this);
   }
 }

package/lib/core/network/protocols/httpwsProtocol.js

@@ -282,6 +282,7 @@ class HttpWsProtocol extends Protocol {
     res.upgrade(
       {
         headers,
+        internal: {},
       },
       req.getHeader("sec-websocket-key"),
       req.getHeader("sec-websocket-protocol"),
@@ -292,7 +293,12 @@ class HttpWsProtocol extends Protocol {
 
   wsOnOpenHandler(socket) {
     const ip = Buffer.from(socket.getRemoteAddressAsText()).toString();
-    const connection = new ClientConnection(this.name, [ip], socket.headers);
+    const connection = new ClientConnection(
+      this.name,
+      [ip],
+      socket.headers,
+      socket.internal
+    );
 
     this.entryPoint.newConnection(connection);
     this.connectionBySocket.set(socket, connection);
@@ -457,8 +463,17 @@ class HttpWsProtocol extends Protocol {
       const buffer = this.backpressureBuffer.get(socket);
       buffer.push(payload);
 
-      // Client socket too slow: we need to close it
-      if (buffer.length > WS_BACKPRESSURE_BUFFER_MAX_LENGTH) {
+      /**
+       * Client socket too slow: we need to close it
+       *
+       * If the socket is marked as a debugSession, we don't close it
+       * the debugger might send a lot of messages and we don't want to
+       * loose the connection while debugging and loose important information.
+       */
+      if (
+        !socket.internal.debugSession &&
+        buffer.length > WS_BACKPRESSURE_BUFFER_MAX_LENGTH
+      ) {
         socket.end(WS_FORCED_TERMINATION_CODE, WS_BACKPRESSURE_MESSAGE);
       }
     }

package/lib/kuzzle/kuzzle.js

@@ -148,7 +148,6 @@ class Kuzzle extends kuzzleEventEmitter_1.default {
                 regExpEngine: this.config.realtime.pcreSupport ? "js" : "re2",
                 seed: this.config.internal.hash.seed,
             });
-            await this.debugger.init();
             await new cacheEngine_1.default().init();
             await new storageEngine_1.default().init();
             await new realtime_1.default().init();
@@ -167,6 +166,7 @@ class Kuzzle extends kuzzleEventEmitter_1.default {
             // must be initialized before plugins to allow API requests from plugins
             // before opening connections to external users
             await this.entryPoint.init();
+            await this.debugger.init();
             this.pluginsManager.application = application;
             const pluginImports = await this.pluginsManager.init(options.plugins);
             this.log.info(`[✔] Successfully loaded ${this.pluginsManager.loadedPlugins.length} plugins: ${this.pluginsManager.loadedPlugins.join(", ")}`);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "kuzzle",
   "author": "The Kuzzle Team <support@kuzzle.io>",
-  "version": "2.21.0",
+  "version": "2.22.0",
   "description": "Kuzzle is an open-source solution that handles all the data management through a secured API, with a large choice of protocols.",
   "bin": "bin/start-kuzzle-server",
   "scripts": {