kuzzle 2.20.3 → 2.21.1

package/lib/cluster/publisher.js

@@ -270,6 +270,16 @@ class ClusterPublisher {
     return this.send("NodeShutdown", { nodeId });
   }
 
+  /**
+   * Publishes an event about the node being in debug mode
+   * @param {bool} evictionPrevented
+   *
+   * @returns {Long} ID of the message sent
+   */
+  sendNodePreventEviction(evictionPrevented) {
+    return this.send("NodePreventEviction", { evictionPrevented });
+  }
+
   /**
    * Publishes an event about a node being evicted
    *

package/lib/cluster/subscriber.js

@@ -55,6 +55,8 @@ class ClusterSubscriber {
     this.remoteNodeIP = remoteNodeIP;
     this.remoteNodeAddress = `tcp://${remoteNodeIP}:${this.localNode.config.ports.sync}`;
     this.remoteNodeId = remoteNodeId;
+    // Used in debug mode when the node might be slower
+    this.remoteNodeEvictionPrevented = false;
     this.socket = null;
     this.protoroot = null;
 
@@ -87,6 +89,7 @@ class ClusterSubscriber {
       NewAuthStrategy: this.handleNewAuthStrategy,
       NewRealtimeRoom: this.handleNewRealtimeRoom,
       NodeEvicted: this.handleNodeEviction,
+      NodePreventEviction: this.handleNodePreventEviction,
       NodeShutdown: this.handleNodeShutdown,
       RefreshIndexCache: this.handleRefreshIndexCache,
       RefreshValidators: this.handleRefreshValidators,
@@ -229,6 +232,10 @@ class ClusterSubscriber {
     }
   }
 
+  async handleNodePreventEviction(message) {
+    this.remoteNodeEvictionPrevented = message.evictionPrevented;
+  }
+
   /**
    * Handles a heartbeat from the remote node
    *
@@ -673,6 +680,14 @@ class ClusterSubscriber {
    * to recover, otherwise we evict it from the cluster.
    */
   async checkHeartbeat() {
+    if (this.remoteNodeEvictionPrevented) {
+      // Fake the heartbeat while the node eviction prevention is enabled
+      // otherwise when the node eviction prevention is disabled
+      // the node will be evicted if it did not send a heartbeat before disabling the protection.
+      this.lastHeartbeat = Date.now();
+      return;
+    }
+
     if (this.state === stateEnum.EVICTED) {
       return;
     }

package/lib/cluster/workers/IDCardRenewer.js

@@ -1,11 +1,9 @@
 "use strict";
 
-const { isMainThread, parentPort } = require("worker_threads");
 const Redis = require("../../service/cache/redis");
 
 class IDCardRenewer {
   constructor() {
-    this.parentPort = parentPort;
     this.redis = null;
     this.refreshTimer = null;
     this.nodeIdKey = null;
@@ -31,6 +29,10 @@ class IDCardRenewer {
       const redisConf = config.redis || {};
       await this.initRedis(redisConf.config, redisConf.name);
     } catch (error) {
+      // eslint-disable-next-line no-console
+      console.error(
+        `Failed to connect to redis, could not refresh ID card: ${error.message}`
+      );
       this.parentPort.postMessage({
         error: `Failed to connect to redis, could not refresh ID card: ${error.message}`,
       });
@@ -50,7 +52,7 @@ class IDCardRenewer {
     }
 
     // Notify that the worker is running and updating the ID Card
-    this.parentPort.postMessage({ initialized: true });
+    process.send({ initialized: true });
   }
 
   async initRedis(config, name) {
@@ -73,13 +75,13 @@ class IDCardRenewer {
       // => this node is too slow, we need to remove it from the cluster
       if (refreshed === 0) {
         await this.dispose();
-        this.parentPort.postMessage({
+        process.send({
           error: "Node too slow: ID card expired",
         });
       }
     } catch (error) {
       await this.dispose();
-      this.parentPort.postMessage({
+      process.send({
         error: `Failed to refresh ID Card: ${error.message}`,
       });
     }
@@ -112,26 +114,25 @@ class IDCardRenewer {
   }
 }
 
-if (!isMainThread) {
-  const idCardRenewer = new IDCardRenewer();
-
-  parentPort.on("message", async (message) => {
-    if (message.action === "start") {
-      // Simulate basic global Kuzzle Context
-      global.kuzzle = { ...message.kuzzle };
-      global.kuzzle.log = {
-        debug: console.debug, // eslint-disable-line no-console
-        error: console.error, // eslint-disable-line no-console
-        info: console.info, // eslint-disable-line no-console
-        warn: console.warn, // eslint-disable-line no-console
-      };
-      // Should never throw
-      await idCardRenewer.init(message);
-    } else if (message.action === "dispose") {
-      // Should never throw
-      await idCardRenewer.dispose();
-    }
-  });
-}
+const idCardRenewer = new IDCardRenewer();
+
+process.on("message", async (message) => {
+  if (message.action === "start") {
+    // Simulate basic global Kuzzle Context
+    global.kuzzle = { ...message.kuzzle };
+    global.kuzzle.log = {
+      debug: console.debug, // eslint-disable-line no-console
+      error: console.error, // eslint-disable-line no-console
+      info: console.info, // eslint-disable-line no-console
+      warn: console.warn, // eslint-disable-line no-console
+    };
+    // Should never throw
+    await idCardRenewer.init(message);
+  } else if (message.action === "dispose") {
+    // Should never throw
+    await idCardRenewer.dispose();
+    process.exit(0);
+  }
+});
 
 module.exports = { IDCardRenewer };

package/lib/core/debug/kuzzleDebugger.d.ts

@@ -0,0 +1,46 @@
+import { JSONObject } from "kuzzle-sdk";
+export declare class KuzzleDebugger {
+    private inspector;
+    private debuggerStatus;
+    /**
+     * Map<eventName, Set<connectionId>>
+     */
+    private events;
+    private httpWsProtocol?;
+    init(): Promise<void>;
+    registerAsks(): Promise<void>;
+    /**
+     * Connect the debugger
+     */
+    enable(): Promise<void>;
+    /**
+     * Disconnect the debugger and clears all the events listeners
+     */
+    disable(): Promise<void>;
+    /**
+     * Trigger action from debugger directly following the Chrome Debug Protocol
+     * See: https://chromedevtools.github.io/devtools-protocol/v8/
+     */
+    post(method: string, params?: JSONObject): Promise<JSONObject>;
+    /**
+     * Make the websocket connection listen and receive events from Chrome Debug Protocol
+     * See events from: https://chromedevtools.github.io/devtools-protocol/v8/
+     */
+    addListener(event: string, connectionId: string): Promise<void>;
+    /**
+     * Remove the websocket connection from the events" listeners
+     */
+    removeListener(event: string, connectionId: string): Promise<void>;
+    /**
+     * Execute a method using the Chrome Debug Protocol
+     * @param method Chrome Debug Protocol method to execute
+     * @param params
+     * @returns
+     */
+    private inspectorPost;
+    /**
+     * Sends a direct notification to a websocket connection without having to listen to a specific room
+     */
+    private notifyConnection;
+    private notifyGlobalListeners;
+}

package/lib/core/debug/kuzzleDebugger.js

@@ -0,0 +1,266 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KuzzleDebugger = void 0;
+const inspector_1 = __importDefault(require("inspector"));
+const kerror = __importStar(require("../../kerror"));
+const DEBUGGER_EVENT = "kuzzle-debugger-event";
+class KuzzleDebugger {
+    constructor() {
+        this.debuggerStatus = false;
+        /**
+         * Map<eventName, Set<connectionId>>
+         */
+        this.events = new Map();
+    }
+    async init() {
+        this.httpWsProtocol = global.kuzzle.entryPoint.protocols.get("websocket");
+        this.inspector = new inspector_1.default.Session();
+        // Remove connection id from the list of listeners for each event
+        global.kuzzle.on("connection:remove", (connectionId) => {
+            if (!this.debuggerStatus) {
+                return;
+            }
+            for (const listener of this.events.values()) {
+                listener.delete(connectionId);
+            }
+        });
+        this.inspector.on("inspectorNotification", async (payload) => {
+            if (!this.debuggerStatus) {
+                return;
+            }
+            await this.notifyGlobalListeners(payload.method, payload);
+            const listeners = this.events.get(payload.method);
+            if (!listeners) {
+                return;
+            }
+            const promises = [];
+            for (const connectionId of listeners) {
+                promises.push(this.notifyConnection(connectionId, DEBUGGER_EVENT, {
+                    event: payload.method,
+                    result: payload,
+                }));
+            }
+            // No need to catch, notify is already try-catched
+            await Promise.all(promises);
+        });
+        await this.registerAsks();
+    }
+    async registerAsks() {
+        global.kuzzle.onAsk("core:debugger:enable", () => this.enable());
+        global.kuzzle.onAsk("core:debugger:disable", () => this.disable());
+        global.kuzzle.onAsk("core:debugger:post", (method, params) => this.post(method, params));
+        global.kuzzle.onAsk("core:debugger:isEnabled", () => this.debuggerStatus);
+        global.kuzzle.onAsk("core:debugger:removeListener", (event, connectionId) => this.removeListener(event, connectionId));
+        global.kuzzle.onAsk("core:debugger:addListener", (event, connectionId) => this.addListener(event, connectionId));
+    }
+    /**
+     * Connect the debugger
+     */
+    async enable() {
+        if (this.debuggerStatus) {
+            return;
+        }
+        this.inspector.connect();
+        this.debuggerStatus = true;
+        await global.kuzzle.ask("cluster:node:preventEviction", true);
+    }
+    /**
+     * Disconnect the debugger and clears all the events listeners
+     */
+    async disable() {
+        if (!this.debuggerStatus) {
+            return;
+        }
+        this.inspector.disconnect();
+        this.debuggerStatus = false;
+        await global.kuzzle.ask("cluster:node:preventEviction", false);
+        // Disable debug mode for all connected sockets that still have listeners
+        if (this.httpWsProtocol) {
+            for (const eventName of this.events.keys()) {
+                for (const connectionId of this.events.get(eventName)) {
+                    const socket = this.httpWsProtocol.socketByConnectionId.get(connectionId);
+                    if (socket) {
+                        socket.internal.debugSession = false;
+                    }
+                }
+            }
+        }
+        this.events.clear();
+    }
+    /**
+     * Trigger action from debugger directly following the Chrome Debug Protocol
+     * See: https://chromedevtools.github.io/devtools-protocol/v8/
+     */
+    async post(method, params = {}) {
+        if (!this.debuggerStatus) {
+            throw kerror.get("core", "debugger", "not_enabled");
+        }
+        // Always disable report progress because this parameter causes a segfault.
+        // The reason this happens is because the inspector is running inside the same thread
+        // as the Kuzzle Process and reportProgress forces the inspector to call function in the JS Heap
+        // while it is being inspected by the HeapProfiler, which causes a segfault.
+        // See: https://github.com/nodejs/node/issues/44634
+        if (params.reportProgress) {
+            // We need to send a fake HeapProfiler.reportHeapSnapshotProgress event
+            // to the inspector to make Chrome think that the HeapProfiler is done
+            // otherwise, even though the Chrome Inspector did receive the whole snapshot, it will not be parsed.
+            //
+            // Chrome inspector is waiting for a HeapProfiler.reportHeapSnapshotProgress event with the finished property set to true
+            // The `done` and `total` properties are only used to show a progress bar, so there are not important.
+            // Sending this event before the HeapProfiler.addHeapSnapshotChunk event will not cause any problem,
+            // in fact, Chrome always do that when taking a snapshot, it receives the HeapProfiler.reportHeapSnapshotProgress event
+            // before the HeapProfiler.addHeapSnapshotChunk event.
+            // So this will have no impact and when receiving the HeapProfiler.addHeapSnapshotChunk event, Chrome will wait to receive
+            // a complete snapshot before parsing it if it has received the HeapProfiler.reportHeapSnapshotProgress event with the finished property set to true before.
+            this.inspector.emit("inspectorNotification", {
+                method: "HeapProfiler.reportHeapSnapshotProgress",
+                params: {
+                    done: 0,
+                    finished: true,
+                    total: 0,
+                },
+            });
+            params.reportProgress = false;
+        }
+        return this.inspectorPost(method, params);
+    }
+    /**
+     * Make the websocket connection listen and receive events from Chrome Debug Protocol
+     * See events from: https://chromedevtools.github.io/devtools-protocol/v8/
+     */
+    async addListener(event, connectionId) {
+        if (!this.debuggerStatus) {
+            throw kerror.get("core", "debugger", "not_enabled");
+        }
+        if (this.httpWsProtocol) {
+            const socket = this.httpWsProtocol.socketByConnectionId.get(connectionId);
+            if (socket) {
+                /**
+                 * Mark the socket as a debugging socket
+                 * this will bypass some limitations like the max pressure buffer size,
+                 * which could end the connection when the debugger is sending a lot of data.
+                 */
+                socket.internal.debugSession = true;
+            }
+        }
+        let listeners = this.events.get(event);
+        if (!listeners) {
+            listeners = new Set();
+            this.events.set(event, listeners);
+        }
+        listeners.add(connectionId);
+    }
+    /**
+     * Remove the websocket connection from the events" listeners
+     */
+    async removeListener(event, connectionId) {
+        if (!this.debuggerStatus) {
+            throw kerror.get("core", "debugger", "not_enabled");
+        }
+        const listeners = this.events.get(event);
+        if (listeners) {
+            listeners.delete(connectionId);
+        }
+        if (!this.httpWsProtocol) {
+            return;
+        }
+        const socket = this.httpWsProtocol.socketByConnectionId.get(connectionId);
+        if (!socket) {
+            return;
+        }
+        let removeDebugSessionMarker = true;
+        /**
+         * If the connection doesn't listen to any other events
+         * we can remove the debugSession marker
+         */
+        for (const eventName of this.events.keys()) {
+            const eventListener = this.events.get(eventName);
+            if (eventListener && eventListener.has(connectionId)) {
+                removeDebugSessionMarker = false;
+                break;
+            }
+        }
+        if (removeDebugSessionMarker) {
+            socket.internal.debugSession = false;
+        }
+    }
+    /**
+     * Execute a method using the Chrome Debug Protocol
+     * @param method Chrome Debug Protocol method to execute
+     * @param params
+     * @returns
+     */
+    async inspectorPost(method, params) {
+        if (!this.debuggerStatus) {
+            throw kerror.get("core", "debugger", "not_enabled");
+        }
+        let resolve;
+        const promise = new Promise((res) => {
+            resolve = res;
+        });
+        this.inspector.post(method, params, (err, res) => {
+            if (err) {
+                resolve({
+                    error: JSON.stringify(Object.getOwnPropertyDescriptors(err)),
+                });
+            }
+            else {
+                resolve(res);
+            }
+        });
+        return promise;
+    }
+    /**
+     * Sends a direct notification to a websocket connection without having to listen to a specific room
+     */
+    async notifyConnection(connectionId, event, payload) {
+        global.kuzzle.entryPoint._notify({
+            channels: [event],
+            connectionId,
+            payload,
+        });
+    }
+    async notifyGlobalListeners(event, payload) {
+        const listeners = this.events.get("*");
+        if (!listeners) {
+            return;
+        }
+        const promises = [];
+        for (const connectionId of listeners) {
+            promises.push(this.notifyConnection(connectionId, DEBUGGER_EVENT, {
+                event,
+                result: payload,
+            }));
+        }
+        // No need to catch, notify is already try-catched
+        await Promise.all(promises);
+    }
+}
+exports.KuzzleDebugger = KuzzleDebugger;
+//# sourceMappingURL=kuzzleDebugger.js.map

package/lib/core/network/clientConnection.js

@@ -31,10 +31,11 @@ const uuid = require("uuid");
  * @param {object} [headers] - Optional extra key-value object. I.e., for http, will receive the request headers
  */
 class ClientConnection {
-  constructor(protocol, ips, headers = null) {
+  constructor(protocol, ips, headers = null, internal = null) {
     this.id = uuid.v4();
     this.protocol = protocol;
     this.headers = {};
+    this.internal = {};
 
     if (!Array.isArray(ips)) {
       throw new TypeError(`Expected ips to be an Array, got ${typeof ips}`);
@@ -45,6 +46,10 @@ class ClientConnection {
       this.headers = headers;
     }
 
+    if (isPlainObject(internal)) {
+      this.internal = internal;
+    }
+
     Object.freeze(this);
   }
 }

package/lib/core/network/protocols/httpwsProtocol.js

@@ -282,6 +282,7 @@ class HttpWsProtocol extends Protocol {
     res.upgrade(
       {
         headers,
+        internal: {},
       },
       req.getHeader("sec-websocket-key"),
       req.getHeader("sec-websocket-protocol"),
@@ -292,7 +293,12 @@ class HttpWsProtocol extends Protocol {
 
   wsOnOpenHandler(socket) {
     const ip = Buffer.from(socket.getRemoteAddressAsText()).toString();
-    const connection = new ClientConnection(this.name, [ip], socket.headers);
+    const connection = new ClientConnection(
+      this.name,
+      [ip],
+      socket.headers,
+      socket.internal
+    );
 
     this.entryPoint.newConnection(connection);
     this.connectionBySocket.set(socket, connection);
@@ -457,8 +463,17 @@ class HttpWsProtocol extends Protocol {
       const buffer = this.backpressureBuffer.get(socket);
       buffer.push(payload);
 
-      // Client socket too slow: we need to close it
-      if (buffer.length > WS_BACKPRESSURE_BUFFER_MAX_LENGTH) {
+      /**
+       * Client socket too slow: we need to close it
+       *
+       * If the socket is marked as a debugSession, we don't close it
+       * the debugger might send a lot of messages and we don't want to
+       * loose the connection while debugging and loose important information.
+       */
+      if (
+        !socket.internal.debugSession &&
+        buffer.length > WS_BACKPRESSURE_BUFFER_MAX_LENGTH
+      ) {
         socket.end(WS_FORCED_TERMINATION_CODE, WS_BACKPRESSURE_MESSAGE);
       }
     }

package/lib/kuzzle/dumpGenerator.js

@@ -228,7 +228,7 @@ class DumpGenerator {
     while (dumps.length >= config.history.reports) {
       const dir = dumps.shift().path;
 
-      fs.removeSync(dir);
+      fs.rmdirSync(dir, { recursive: true });
     }
 
     for (let i = 0; i < dumps.length - config.history.coredump; i++) {

package/lib/kuzzle/kuzzle.js

@@ -80,6 +80,7 @@ const package_json_1 = require("../../package.json");
 const name_generator_1 = require("../util/name-generator");
 const openapi_1 = require("../api/openapi");
 const crypto_1 = require("../util/crypto");
+const kuzzleDebugger_1 = require("../core/debug/kuzzleDebugger");
 exports.BACKEND_IMPORT_KEY = "backend:init:import";
 let _kuzzle = null;
 Reflect.defineProperty(global, "kuzzle", {
@@ -119,6 +120,7 @@ class Kuzzle extends kuzzleEventEmitter_1.default {
         this.dumpGenerator = new dumpGenerator_1.default();
         this.vault = null;
         this.asyncStore = new asyncStore_1.default();
+        this.debugger = new kuzzleDebugger_1.KuzzleDebugger();
         this.version = package_json_1.version;
         this.importTypes = {
             fixtures: this.importFixtures.bind(this),
@@ -164,6 +166,7 @@ class Kuzzle extends kuzzleEventEmitter_1.default {
             // must be initialized before plugins to allow API requests from plugins
             // before opening connections to external users
             await this.entryPoint.init();
+            await this.debugger.init();
             this.pluginsManager.application = application;
             const pluginImports = await this.pluginsManager.init(options.plugins);
             this.log.info(`[✔] Successfully loaded ${this.pluginsManager.loadedPlugins.length} plugins: ${this.pluginsManager.loadedPlugins.join(", ")}`);