kuzzle 2.14.16 → 2.16.0

package/lib/cluster/idCardHandler.d.ts

@@ -0,0 +1,140 @@
+import '../types';
+export declare type SerializedIdCard = {
+    id: string;
+    ip: string;
+    birthdate: number;
+    topology: string[];
+};
+export declare class IdCard {
+    /**
+     * Node unique identifier
+     *
+     * @example
+     *
+     * knode-pensive-einstein-844221
+     */
+    private id;
+    /**
+     * Node IP address
+     */
+    private ip;
+    /**
+     * Node creation timestamp
+     */
+    private birthdate;
+    /**
+     * Node known topology composed of node IDs
+     *
+     * Set<node-id>
+     */
+    topology: Set<string>;
+    constructor({ id, ip, birthdate, topology }: SerializedIdCard);
+    serialize(): SerializedIdCard;
+    static unserialize(serialized: SerializedIdCard): IdCard;
+}
+/**
+ * Handles the ID Key stored in Redis, holding node information
+ */
+export declare class ClusterIdCardHandler {
+    /**
+     * Node instance. Represents the local node.
+     */
+    private node;
+    /**
+     * Local node ID Card
+     */
+    private idCard;
+    /**
+     * Local node IP address
+     */
+    private ip;
+    /**
+     * Delay for refreshing the ID Card. The heartbeat timer is run on this delay
+     * and the node ID Card should be available on Redis otherwise it will be evicted.
+     */
+    private refreshDelay;
+    /**
+     * Multiplier used to ensure the node has enough time to refresh it's ID Card
+     * before the ID Card refresh delay
+     */
+    private refreshMultiplier;
+    /**
+     * Worker thread in charge of refreshing the ID Card once the node has started
+     */
+    private refreshWorker;
+    /**
+     * Hold the timer in charge of refreshing the ID Card before the worker starts
+     */
+    private refreshTimer;
+    /**
+     * Local node ID
+     */
+    private nodeId;
+    /**
+     * Local node Redis key
+     */
+    private nodeIdKey;
+    /**
+     * Flag to prevent updating the id card if it has been disposed.
+     * Prevents race condition if a topology update occurs at the same time as
+     * the id card is been disposed because the node is evicting itself from the
+     * cluster
+     */
+    private disposed;
+    constructor(node: any);
+    /**
+     * Generates and reserves a unique ID for this node instance.
+     * Makes sure that the ID is not already taken by another node instance.
+     */
+    createIdCard(): Promise<void>;
+    /**
+     * Helper method to mock worker instantiation in unit tests
+     */
+    private constructWorker;
+    /**
+     * Start refreshing the ID Card before the worker starts to ensure the ID Card
+     * is refreshed.
+     *
+     * Once the worker starts, this timer will be stopped.
+     */
+    private startTemporaryRefresh;
+    dispose(): Promise<void>;
+    /**
+     * Retrieves the ID cards from other nodes
+     *
+     * Each node store it's ID Card under a specific key name. When Redis database
+     * is growing, searching for those keys can be quite expensive and can slow down
+     * the handshake process.
+     *
+     * We are storing a set containing ID Card's keys under a set so when a new node
+     * is started, it can directly get the other nodes ID Cards with SMEMBERS and
+     * then MGET instead of using SCAN.
+     *
+     * When a new node retrieve the ID Card's keys from the set, it try to get them
+     * with MGET, those who cannot be retrieved are expired ID Cards so the node update
+     * the set accordingly.
+     *
+     * @return {Array.<IdCard>}
+     */
+    getRemoteIdCards(): Promise<IdCard[]>;
+    /**
+     * Adds a remote node IdCard to the node known topology
+     */
+    addNode(id: string): Promise<void>;
+    /**
+     * Removes a remote node IdCard from the node known topology
+     */
+    removeNode(id: string): Promise<void>;
+    /**
+     * Store the key under which this node ID Card is stored inside the set.
+     *
+     * This set is an index to retrieve ID Cards faster.
+     */
+    addIdCardToIndex(): Promise<void>;
+    /**
+     * Saves the local node IdCard into Redis
+     *
+     * @returns True if the key was set
+     */
+    private save;
+}

package/lib/cluster/idCardHandler.js

@@ -1,3 +1,4 @@
+"use strict";
 /*
  * Kuzzle, a backend software, self-hostable and ready to use
  * to power modern apps
@@ -18,231 +19,234 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-'use strict';
-
-const { generateRandomName } = require('../util/name-generator');
-const { Worker } = require('worker_threads');
-const Bluebird = require('bluebird');
-
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClusterIdCardHandler = exports.IdCard = void 0;
+const name_generator_1 = require("../util/name-generator");
+const worker_threads_1 = require("worker_threads");
+const bluebird_1 = __importDefault(require("bluebird"));
+require("../types");
 const REDIS_PREFIX = '{cluster/node}/';
 const REDIS_ID_CARDS_INDEX = REDIS_PREFIX + 'id-cards-index';
-
-/**
- * @typedef {IdCard}
- */
 class IdCard {
-  /**
-   * @param {Object} obj - contains the ID card description
-   * @param {string} obj.id - node identifier
-   * @param {string} obj.ip - node IP address
-   * @param {Number} obj.birthdate - node creation timestamp
-   * @param {Array.<string>} obj.topology - node's known topology
-   */
-  constructor (obj) {
-    this.id = obj.id;
-    this.ip = obj.ip;
-    this.birthdate = obj.birthdate;
-    this.topology = new Set(obj.topology);
-  }
-
-  serialize () {
-    return JSON.stringify({
-      birthdate: this.birthdate,
-      id: this.id,
-      ip: this.ip,
-      topology: Array.from(this.topology),
-    });
-  }
-
-  static unserialize (value) {
-    return new IdCard(JSON.parse(value));
-  }
+    constructor({ id, ip, birthdate, topology }) {
+        this.id = id;
+        this.ip = ip;
+        this.birthdate = birthdate;
+        this.topology = new Set(topology);
+    }
+    serialize() {
+        return {
+            birthdate: this.birthdate,
+            id: this.id,
+            ip: this.ip,
+            topology: Array.from(this.topology),
+        };
+    }
+    static unserialize(serialized) {
+        return new IdCard(serialized);
+    }
 }
-
+exports.IdCard = IdCard;
 /**
  * Handles the ID Key stored in Redis, holding node information
  */
 class ClusterIdCardHandler {
-  constructor (node) {
-    this.node = node;
-    this.idCard = null;
-    this.ip = node.ip;
-    this.refreshDelay = node.heartbeatDelay;
-    this.refreshWorker = null;
-    this.nodeId = null;
-    this.nodeIdKey = null;
-
-    // Flag to prevent updating the id card if it has been disposed.
-    // Prevents race condition if a topology update occurs at the same time as
-    // the id card is been disposed because the node is evicting itself from the
-    // cluster
-    this.disposed = false;
-  }
-
-  /**
-   * Generates and reserves a unique ID for this node instance.
-   * Makes sure that the ID is not already taken by another node instance.
-   *
-   * @return {void}
-   */
-  async createIdCard () {
-    let reserved;
-
-    do {
-      this.nodeId = generateRandomName('knode');
-      this.nodeIdKey = `${REDIS_PREFIX}${this.nodeId}`;
-      this.idCard = new IdCard({
-        birthdate: Date.now(),
-        id: this.nodeId,
-        ip: this.ip,
-        topology: [],
-      });
-
-      reserved = await this._save({ creation: true });
-    } while (!reserved);
-
-    await this.addIdCardToIndex();
-
-    this.refreshWorker = this._constructWorker(`${__dirname}/workers/IDCardRenewer.js`);
-    this.refreshWorker.unref();
-
-    this.refreshWorker.on('message', async message => {
-      if (message.error) {
-        await this.node.evictSelf(message.error);
-      }
-    });
-
-    // Transfer informations to the worker
-    this.refreshWorker.postMessage({
-      action: 'start', // start the worker
-      kuzzle: {
-        config: global.kuzzle.config,
-        id: global.kuzzle.id,
-      },
-      nodeIdKey: this.nodeIdKey,
-      // Used to configure a redis the same way as the Cache Engine does
-      redis: {
-        config: global.kuzzle.config.services.internalCache,
-        name: 'internal_adapter',
-      },
-      refreshDelay: this.refreshDelay,
-    });
-  }
-
-  // Used to Mock the creation of a worker for the tests
-  _constructWorker(path) {
-    return new Worker(path);
-  }
-
-  async dispose () {
-    this.disposed = true;
-    if (this.refreshWorker) {
-      this.refreshWorker.postMessage({action: 'dispose'});
+    constructor(node) {
+        /**
+         * Local node ID Card
+         */
+        this.idCard = null;
+        /**
+         * Multiplier used to ensure the node has enough time to refresh it's ID Card
+         * before the ID Card refresh delay
+         */
+        this.refreshMultiplier = 2;
+        /**
+         * Worker thread in charge of refreshing the ID Card once the node has started
+         */
+        this.refreshWorker = null;
+        /**
+         * Hold the timer in charge of refreshing the ID Card before the worker starts
+         */
+        this.refreshTimer = null;
+        /**
+         * Local node ID
+         */
+        this.nodeId = null;
+        /**
+         * Local node Redis key
+         */
+        this.nodeIdKey = null;
+        /**
+         * Flag to prevent updating the id card if it has been disposed.
+         * Prevents race condition if a topology update occurs at the same time as
+         * the id card is been disposed because the node is evicting itself from the
+         * cluster
+         */
+        this.disposed = false;
+        this.node = node;
+        this.ip = node.ip;
+        this.refreshDelay = node.heartbeatDelay;
+    }
+    /**
+     * Generates and reserves a unique ID for this node instance.
+     * Makes sure that the ID is not already taken by another node instance.
+     */
+    async createIdCard() {
+        let reserved = false;
+        do {
+            this.nodeId = (0, name_generator_1.generateRandomName)('knode');
+            this.nodeIdKey = `${REDIS_PREFIX}${this.nodeId}`;
+            this.idCard = new IdCard({
+                birthdate: Date.now(),
+                id: this.nodeId,
+                ip: this.ip,
+                topology: [],
+            });
+            reserved = await this.save({ creation: true });
+        } while (!reserved);
+        await this.addIdCardToIndex();
+        this.refreshWorker = this.constructWorker(`${__dirname}/workers/IDCardRenewer.js`);
+        this.refreshWorker.unref();
+        this.refreshWorker.on('message', async (message) => {
+            if (message.error) {
+                await this.node.evictSelf(message.error);
+            }
+        });
+        // Transfer informations to the worker
+        this.refreshWorker.postMessage({
+            action: 'start',
+            kuzzle: {
+                config: global.kuzzle.config,
+                id: global.kuzzle.id,
+            },
+            nodeIdKey: this.nodeIdKey,
+            // Used to configure a redis the same way as the Cache Engine does
+            redis: {
+                config: global.kuzzle.config.services.internalCache,
+                name: 'internal_adapter',
+            },
+            refreshDelay: this.refreshDelay,
+            refreshMultiplier: this.refreshMultiplier,
+        });
+        this.startTemporaryRefresh();
+    }
+    /**
+     * Helper method to mock worker instantiation in unit tests
+     */
+    constructWorker(path) {
+        return new worker_threads_1.Worker(path);
+    }
+    /**
+     * Start refreshing the ID Card before the worker starts to ensure the ID Card
+     * is refreshed.
+     *
+     * Once the worker starts, this timer will be stopped.
+     */
+    startTemporaryRefresh() {
+        this.refreshTimer = setInterval(async () => {
+            try {
+                await this.save();
+            }
+            catch (error) {
+                global.kuzzle.log.error(`An error occurred while refreshing the ID card during WorkerThread startup: ${error}`);
+            }
+        }, this.refreshDelay * this.refreshMultiplier);
+        this.refreshWorker.on('message', ({ initialized }) => {
+            if (initialized) {
+                clearInterval(this.refreshTimer);
+                this.refreshTimer = null;
+            }
+        });
+    }
+    async dispose() {
+        this.disposed = true;
+        if (this.refreshWorker) {
+            this.refreshWorker.postMessage({ action: 'dispose' });
+        }
+    }
+    /**
+     * Retrieves the ID cards from other nodes
+     *
+     * Each node store it's ID Card under a specific key name. When Redis database
+     * is growing, searching for those keys can be quite expensive and can slow down
+     * the handshake process.
+     *
+     * We are storing a set containing ID Card's keys under a set so when a new node
+     * is started, it can directly get the other nodes ID Cards with SMEMBERS and
+     * then MGET instead of using SCAN.
+     *
+     * When a new node retrieve the ID Card's keys from the set, it try to get them
+     * with MGET, those who cannot be retrieved are expired ID Cards so the node update
+     * the set accordingly.
+     *
+     * @return {Array.<IdCard>}
+     */
+    async getRemoteIdCards() {
+        const idCards = [];
+        let keys = await global.kuzzle.ask('core:cache:internal:execute', 'smembers', REDIS_ID_CARDS_INDEX);
+        keys = keys.filter(nodeIdKey => nodeIdKey !== this.nodeIdKey);
+        if (keys.length === 0) {
+            return idCards;
+        }
+        const rawIdCards = await global.kuzzle.ask('core:cache:internal:mget', keys);
+        const expiredIdCards = [];
+        for (let i = 0; i < keys.length; i++) {
+            // filter keys that might have expired between the key search and their
+            // values retrieval
+            if (rawIdCards[i] !== null) {
+                idCards.push(IdCard.unserialize(JSON.parse(rawIdCards[i])));
+            }
+            else {
+                expiredIdCards.push(keys[i]);
+            }
+        }
+        // Clean expired ID Card's keys in the index
+        await bluebird_1.default.map(expiredIdCards, idCardKey => {
+            return global.kuzzle.ask('core:cache:internal:execute', 'srem', REDIS_ID_CARDS_INDEX, idCardKey);
+        });
+        return idCards;
     }
-  }
-
-  /**
-   * Retrieves the ID cards from other nodes
-   *
-   * Each node store it's ID Card under a specific key name. When Redis database
-   * is growing, searching for those keys can be quite expensive and can slow down
-   * the handshake process.
-   *
-   * We are storing a set containing ID Card's keys under a set so when a new node
-   * is started, it can directly get the other nodes ID Cards with SMEMBERS and
-   * then MGET instead of using SCAN.
-   *
-   * When a new node retrieve the ID Card's keys from the set, it try to get them
-   * with MGET, those who cannot be retrieved are expired ID Cards so the node update
-   * the set accordingly.
-   *
-   * @return {Array.<IdCard>}
-   */
-  async getRemoteIdCards () {
-    const idCards = [];
-
-    let keys = await global.kuzzle.ask(
-      'core:cache:internal:execute',
-      'smembers',
-      REDIS_ID_CARDS_INDEX);
-
-    keys = keys.filter(nodeIdKey => nodeIdKey !== this.nodeIdKey);
-
-    if (keys.length === 0) {
-      return idCards;
+    /**
+     * Adds a remote node IdCard to the node known topology
+     */
+    async addNode(id) {
+        if (this.disposed || this.idCard.topology.has(id)) {
+            return;
+        }
+        this.idCard.topology.add(id);
+        await this.save();
     }
-
-    const values = await global.kuzzle.ask('core:cache:internal:mget', keys);
-    const expiredIdCards = [];
-
-    for (let i = 0; i < keys.length; i++) {
-      // filter keys that might have expired between the key search and their
-      // values retrieval
-      if (values[i] !== null) {
-        idCards.push(IdCard.unserialize(values[i]));
-      }
-      else {
-        expiredIdCards.push(keys[i]);
-      }
+    /**
+     * Removes a remote node IdCard from the node known topology
+     */
+    async removeNode(id) {
+        if (!this.disposed && this.idCard.topology.delete(id)) {
+            await this.save();
+        }
     }
-
-    // Clean expired ID Card's keys in the index
-    await Bluebird.map(expiredIdCards, idCardKey => {
-      return global.kuzzle.ask(
-        'core:cache:internal:execute',
-        'srem',
-        REDIS_ID_CARDS_INDEX,
-        idCardKey);
-    });
-
-    return idCards;
-  }
-
-  /**
-   * Adds a remote node IdCard to the node known topology
-   */
-  async addNode (id) {
-    if (this.disposed || this.idCard.topology.has(id)) {
-      return;
+    /**
+     * Store the key under which this node ID Card is stored inside the set.
+     *
+     * This set is an index to retrieve ID Cards faster.
+     */
+    async addIdCardToIndex() {
+        await global.kuzzle.ask('core:cache:internal:execute', 'sadd', REDIS_ID_CARDS_INDEX, this.nodeIdKey);
     }
-
-    this.idCard.topology.add(id);
-
-    await this._save();
-  }
-
-  /**
-   * Removes a remote node IdCard from the node known topology
-   */
-  async removeNode (id) {
-    if (!this.disposed && this.idCard.topology.delete(id)) {
-      await this._save();
+    /**
+     * Saves the local node IdCard into Redis
+     *
+     * @returns True if the key was set
+     */
+    async save({ creation } = { creation: false }) {
+        if (!this.idCard) {
+            return false;
+        }
+        return await global.kuzzle.ask('core:cache:internal:store', this.nodeIdKey, JSON.stringify(this.idCard.serialize()), { onlyIfNew: creation, ttl: this.refreshDelay * this.refreshMultiplier });
     }
-  }
-
-  /**
-   * Store the key under which this node ID Card is stored inside the set.
-   *
-   * This set is an index to retrieve ID Cards faster.
-   */
-  async addIdCardToIndex () {
-    await global.kuzzle.ask(
-      'core:cache:internal:execute',
-      'sadd',
-      REDIS_ID_CARDS_INDEX, this.nodeIdKey);
-  }
-
-  /**
-   * Saves the local node IdCard into Redis
-   */
-  _save ({ creation } = { creation: false }) {
-    return global.kuzzle.ask(
-      'core:cache:internal:store',
-      this.nodeIdKey,
-      this.idCard.serialize(),
-      { onlyIfNew: creation, ttl: this.refreshDelay * 3 });
-  }
 }
-
-module.exports = { ClusterIdCardHandler, IdCard };
+exports.ClusterIdCardHandler = ClusterIdCardHandler;
+//# sourceMappingURL=idCardHandler.js.map

package/lib/cluster/node.js

@@ -753,6 +753,10 @@ class ClusterNode {
    * @return {void}
    */
   registerEvents () {
+    global.kuzzle.on(
+      'admin:afterRefreshIndexCache',
+      () => this.onIndexCacheRefreshed());
+
     global.kuzzle.on(
       'core:realtime:room:create:after',
       payload => this.onNewRealtimeRoom(payload));
@@ -1123,6 +1127,13 @@ class ClusterNode {
     this.publisher.send('Shutdown', {});
   }
 
+  /**
+   * Triggered when the index cache has been manually refreshed
+   */
+  onIndexCacheRefreshed () {
+    this.publisher.send('RefreshIndexCache', {});
+  }
+
   /**
    * Returns the total number of subscribers on the cluster for the provided
    * room

package/lib/cluster/protobuf/sync.proto

@@ -177,3 +177,7 @@ message ClusterWideEvent {
   string event = 2;
   string payload = 3;
 }
+
+message RefreshIndexCache {
+  uint64 messageId = 1;
+}

package/lib/cluster/subscriber.js

@@ -26,7 +26,6 @@ const protobuf = require('protobufjs');
 const Long = require('long');
 
 const debug = require('../util/debug')('kuzzle:cluster:sync');
-const debugMessages = require('../util/debug')('kuzzle:cluster:messages');
 const DocumentNotification = require('../core/realtime/notification/document');
 const UserNotification = require('../core/realtime/notification/user');
 const { has } = require('../util/safeObject');
@@ -41,8 +40,6 @@ const stateEnum = Object.freeze({
 });
 /* eslint-enable sort-keys */
 
-const debugExcludedTopics = ['Heartbeat', 'UserNotification', 'DocumentNotification'];
-
 // Handles messages received from other nodes
 class ClusterSubscriber {
   /**
@@ -91,6 +88,7 @@ class ClusterSubscriber {
       NewRealtimeRoom: this.handleNewRealtimeRoom,
       NodeEvicted: this.handleNodeEviction,
       NodeShutdown: this.handleNodeShutdown,
+      RefreshIndexCache: this.handleRefreshIndexCache,
       RefreshValidators: this.handleRefreshValidators,
       RemoveAuthStrategy: this.handleAuthStrategyRemoval,
       RemoveCollection: this.handleCollectionRemoval,
@@ -218,16 +216,7 @@ class ClusterSubscriber {
     }
 
     try {
-      if (! debugExcludedTopics.includes(topic)) {
-        debugMessages('[CLUSTER] "%s": %o', topic, message);
-        debugMessages('[CLUSTER] Fullstate before: %j', this.localNode.fullState.serialize());
-      }
-
       await this.handlers[topic].call(this, message);
-
-      if (! debugExcludedTopics.includes(topic)) {
-        debugMessages('[CLUSTER] Fullstate after: %j', this.localNode.fullState.serialize());
-      }
     }
     catch (e) {
       this.localNode.evictSelf(
@@ -509,6 +498,14 @@ class ClusterSubscriber {
     await global.kuzzle.validation.curateSpecification();
   }
 
+  /**
+   * Handles manual refresh of the index cache
+   */
+  async handleRefreshIndexCache () {
+    debug('Index cache manually refresh received from node %s', this.remoteNodeId);
+    await global.kuzzle.ask('core:storage:public:cache:refresh', { from: 'cluster' });
+  }
+
   /**
    * Invalidates a profile to force reloading it from the storage space
    *