kuzzle 2.14.14 → 2.15.2

package/lib/api/controllers/adminController.js

@@ -37,6 +37,7 @@ class AdminController extends NativeController {
       'loadFixtures',
       'loadMappings',
       'loadSecurities',
+      'refreshIndexCache',
       'resetCache',
       'resetDatabase',
       'resetSecurity',
@@ -46,6 +47,10 @@ class AdminController extends NativeController {
     this.shuttingDown = false;
   }
 
+  async refreshIndexCache () {
+    await global.kuzzle.ask('core:storage:public:cache:refresh');
+  }
+
   /**
    * Reset Redis cache
    */

package/lib/api/controllers/documentController.js

@@ -515,11 +515,7 @@ class DocumentController extends NativeController {
         document);
     }
 
-    if (!source) {
-      return { _id: document._id };
-    }
-
-    return document;
+    return source ? document : { _id: document._id };
   }
 
   /**

package/lib/cluster/idCardHandler.d.ts

@@ -0,0 +1,140 @@
+import '../types';
+export declare type SerializedIdCard = {
+    id: string;
+    ip: string;
+    birthdate: number;
+    topology: string[];
+};
+export declare class IdCard {
+    /**
+     * Node unique identifier
+     *
+     * @example
+     *
+     * knode-pensive-einstein-844221
+     */
+    private id;
+    /**
+     * Node IP address
+     */
+    private ip;
+    /**
+     * Node creation timestamp
+     */
+    private birthdate;
+    /**
+     * Node known topology composed of node IDs
+     *
+     * Set<node-id>
+     */
+    topology: Set<string>;
+    constructor({ id, ip, birthdate, topology }: SerializedIdCard);
+    serialize(): SerializedIdCard;
+    static unserialize(serialized: SerializedIdCard): IdCard;
+}
+/**
+ * Handles the ID Key stored in Redis, holding node information
+ */
+export declare class ClusterIdCardHandler {
+    /**
+     * Node instance. Represents the local node.
+     */
+    private node;
+    /**
+     * Local node ID Card
+     */
+    private idCard;
+    /**
+     * Local node IP address
+     */
+    private ip;
+    /**
+     * Delay for refreshing the ID Card. The heartbeat timer is run on this delay
+     * and the node ID Card should be available on Redis otherwise it will be evicted.
+     */
+    private refreshDelay;
+    /**
+     * Multiplier used to ensure the node has enough time to refresh it's ID Card
+     * before the ID Card refresh delay
+     */
+    private refreshMultiplier;
+    /**
+     * Worker thread in charge of refreshing the ID Card once the node has started
+     */
+    private refreshWorker;
+    /**
+     * Hold the timer in charge of refreshing the ID Card before the worker starts
+     */
+    private refreshTimer;
+    /**
+     * Local node ID
+     */
+    private nodeId;
+    /**
+     * Local node Redis key
+     */
+    private nodeIdKey;
+    /**
+     * Flag to prevent updating the id card if it has been disposed.
+     * Prevents race condition if a topology update occurs at the same time as
+     * the id card is been disposed because the node is evicting itself from the
+     * cluster
+     */
+    private disposed;
+    constructor(node: any);
+    /**
+     * Generates and reserves a unique ID for this node instance.
+     * Makes sure that the ID is not already taken by another node instance.
+     */
+    createIdCard(): Promise<void>;
+    /**
+     * Helper method to mock worker instantiation in unit tests
+     */
+    private constructWorker;
+    /**
+     * Start refreshing the ID Card before the worker starts to ensure the ID Card
+     * is refreshed.
+     *
+     * Once the worker starts, this timer will be stopped.
+     */
+    private startTemporaryRefresh;
+    dispose(): Promise<void>;
+    /**
+     * Retrieves the ID cards from other nodes
+     *
+     * Each node store it's ID Card under a specific key name. When Redis database
+     * is growing, searching for those keys can be quite expensive and can slow down
+     * the handshake process.
+     *
+     * We are storing a set containing ID Card's keys under a set so when a new node
+     * is started, it can directly get the other nodes ID Cards with SMEMBERS and
+     * then MGET instead of using SCAN.
+     *
+     * When a new node retrieve the ID Card's keys from the set, it try to get them
+     * with MGET, those who cannot be retrieved are expired ID Cards so the node update
+     * the set accordingly.
+     *
+     * @return {Array.<IdCard>}
+     */
+    getRemoteIdCards(): Promise<IdCard[]>;
+    /**
+     * Adds a remote node IdCard to the node known topology
+     */
+    addNode(id: string): Promise<void>;
+    /**
+     * Removes a remote node IdCard from the node known topology
+     */
+    removeNode(id: string): Promise<void>;
+    /**
+     * Store the key under which this node ID Card is stored inside the set.
+     *
+     * This set is an index to retrieve ID Cards faster.
+     */
+    addIdCardToIndex(): Promise<void>;
+    /**
+     * Saves the local node IdCard into Redis
+     *
+     * @returns True if the key was set
+     */
+    private save;
+}

package/lib/cluster/idCardHandler.js

@@ -1,3 +1,4 @@
+"use strict";
 /*
  * Kuzzle, a backend software, self-hostable and ready to use
  * to power modern apps
@@ -18,188 +19,234 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-'use strict';
-
-const { generateRandomName } = require('../util/name-generator');
-const { Worker } = require('worker_threads');
-
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClusterIdCardHandler = exports.IdCard = void 0;
+const name_generator_1 = require("../util/name-generator");
+const worker_threads_1 = require("worker_threads");
+const bluebird_1 = __importDefault(require("bluebird"));
+require("../types");
 const REDIS_PREFIX = '{cluster/node}/';
-
-/**
- * @typedef {IdCard}
- */
+const REDIS_ID_CARDS_INDEX = REDIS_PREFIX + 'id-cards-index';
 class IdCard {
-  /**
-   * @param {Object} obj - contains the ID card description
-   * @param {string} obj.id - node identifier
-   * @param {string} obj.ip - node IP address
-   * @param {Number} obj.birthdate - node creation timestamp
-   * @param {Array.<string>} obj.topology - node's known topology
-   */
-  constructor (obj) {
-    this.id = obj.id;
-    this.ip = obj.ip;
-    this.birthdate = obj.birthdate;
-    this.topology = new Set(obj.topology);
-  }
-
-  serialize () {
-    return JSON.stringify({
-      birthdate: this.birthdate,
-      id: this.id,
-      ip: this.ip,
-      topology: Array.from(this.topology),
-    });
-  }
-
-  static unserialize (value) {
-    return new IdCard(JSON.parse(value));
-  }
+    constructor({ id, ip, birthdate, topology }) {
+        this.id = id;
+        this.ip = ip;
+        this.birthdate = birthdate;
+        this.topology = new Set(topology);
+    }
+    serialize() {
+        return {
+            birthdate: this.birthdate,
+            id: this.id,
+            ip: this.ip,
+            topology: Array.from(this.topology),
+        };
+    }
+    static unserialize(serialized) {
+        return new IdCard(serialized);
+    }
 }
-
+exports.IdCard = IdCard;
 /**
  * Handles the ID Key stored in Redis, holding node information
  */
 class ClusterIdCardHandler {
-  constructor (node) {
-    this.node = node;
-    this.idCard = null;
-    this.ip = node.ip;
-    this.refreshDelay = node.heartbeatDelay;
-    this.refreshWorker = null;
-    this.nodeId = null;
-    this.nodeIdKey = null;
-
-    // Flag to prevent updating the id card if it has been disposed.
-    // Prevents race condition if a topology update occurs at the same time as
-    // the id card is been disposed because the node is evicting itself from the
-    // cluster
-    this.disposed = false;
-  }
-
-  /**
-   * Generates and reserves a unique ID for this node instance.
-   * Makes sure that the ID is not already taken by another node instance.
-   *
-   * @return {void}
-   */
-  async createIdCard () {
-    let reserved;
-
-    do {
-      this.nodeId = generateRandomName('knode');
-      this.nodeIdKey = `${REDIS_PREFIX}${this.nodeId}`;
-      this.idCard = new IdCard({
-        birthdate: Date.now(),
-        id: this.nodeId,
-        ip: this.ip,
-        topology: [],
-      });
-
-      reserved = await this._save({ creation: true });
-    } while (!reserved);
-
-    this.refreshWorker = this._constructWorker(`${__dirname}/workers/IDCardRenewer.js`);
-    this.refreshWorker.unref();
-
-    this.refreshWorker.on('message', async message => {
-      if (message.error) {
-        await this.node.evictSelf(message.error);
-      }
-    });
-
-    // Transfer informations to the worker
-    this.refreshWorker.postMessage({
-      action: 'start', // start the worker
-      kuzzle: {
-        config: global.kuzzle.config,
-        id: global.kuzzle.id,
-      },
-      nodeIdKey: this.nodeIdKey,
-      // Used to configure a redis the same way as the Cache Engine does
-      redis: {
-        config: global.kuzzle.config.services.internalCache,
-        name: 'internal_adapter',
-      },
-      refreshDelay: this.refreshDelay,
-    });
-  }
-
-  // Used to Mock the creation of a worker for the tests
-  _constructWorker(path) {
-    return new Worker(path);
-  }
-
-  async dispose () {
-    this.disposed = true;
-    if (this.refreshWorker) {
-      this.refreshWorker.postMessage({action: 'dispose'});
+    constructor(node) {
+        /**
+         * Local node ID Card
+         */
+        this.idCard = null;
+        /**
+         * Multiplier used to ensure the node has enough time to refresh it's ID Card
+         * before the ID Card refresh delay
+         */
+        this.refreshMultiplier = 2;
+        /**
+         * Worker thread in charge of refreshing the ID Card once the node has started
+         */
+        this.refreshWorker = null;
+        /**
+         * Hold the timer in charge of refreshing the ID Card before the worker starts
+         */
+        this.refreshTimer = null;
+        /**
+         * Local node ID
+         */
+        this.nodeId = null;
+        /**
+         * Local node Redis key
+         */
+        this.nodeIdKey = null;
+        /**
+         * Flag to prevent updating the id card if it has been disposed.
+         * Prevents race condition if a topology update occurs at the same time as
+         * the id card is been disposed because the node is evicting itself from the
+         * cluster
+         */
+        this.disposed = false;
+        this.node = node;
+        this.ip = node.ip;
+        this.refreshDelay = node.heartbeatDelay;
+    }
+    /**
+     * Generates and reserves a unique ID for this node instance.
+     * Makes sure that the ID is not already taken by another node instance.
+     */
+    async createIdCard() {
+        let reserved = false;
+        do {
+            this.nodeId = (0, name_generator_1.generateRandomName)('knode');
+            this.nodeIdKey = `${REDIS_PREFIX}${this.nodeId}`;
+            this.idCard = new IdCard({
+                birthdate: Date.now(),
+                id: this.nodeId,
+                ip: this.ip,
+                topology: [],
+            });
+            reserved = await this.save({ creation: true });
+        } while (!reserved);
+        await this.addIdCardToIndex();
+        this.refreshWorker = this.constructWorker(`${__dirname}/workers/IDCardRenewer.js`);
+        this.refreshWorker.unref();
+        this.refreshWorker.on('message', async (message) => {
+            if (message.error) {
+                await this.node.evictSelf(message.error);
+            }
+        });
+        // Transfer informations to the worker
+        this.refreshWorker.postMessage({
+            action: 'start',
+            kuzzle: {
+                config: global.kuzzle.config,
+                id: global.kuzzle.id,
+            },
+            nodeIdKey: this.nodeIdKey,
+            // Used to configure a redis the same way as the Cache Engine does
+            redis: {
+                config: global.kuzzle.config.services.internalCache,
+                name: 'internal_adapter',
+            },
+            refreshDelay: this.refreshDelay,
+            refreshMultiplier: this.refreshMultiplier,
+        });
+        this.startTemporaryRefresh();
+    }
+    /**
+     * Helper method to mock worker instantiation in unit tests
+     */
+    constructWorker(path) {
+        return new worker_threads_1.Worker(path);
+    }
+    /**
+     * Start refreshing the ID Card before the worker starts to ensure the ID Card
+     * is refreshed.
+     *
+     * Once the worker starts, this timer will be stopped.
+     */
+    startTemporaryRefresh() {
+        this.refreshTimer = setInterval(async () => {
+            try {
+                await this.save();
+            }
+            catch (error) {
+                global.kuzzle.log.error(`An error occurred while refreshing the ID card during WorkerThread startup: ${error}`);
+            }
+        }, this.refreshDelay * this.refreshMultiplier);
+        this.refreshWorker.on('message', ({ initialized }) => {
+            if (initialized) {
+                clearInterval(this.refreshTimer);
+                this.refreshTimer = null;
+            }
+        });
+    }
+    async dispose() {
+        this.disposed = true;
+        if (this.refreshWorker) {
+            this.refreshWorker.postMessage({ action: 'dispose' });
+        }
+    }
+    /**
+     * Retrieves the ID cards from other nodes
+     *
+     * Each node store it's ID Card under a specific key name. When Redis database
+     * is growing, searching for those keys can be quite expensive and can slow down
+     * the handshake process.
+     *
+     * We are storing a set containing ID Card's keys under a set so when a new node
+     * is started, it can directly get the other nodes ID Cards with SMEMBERS and
+     * then MGET instead of using SCAN.
+     *
+     * When a new node retrieve the ID Card's keys from the set, it try to get them
+     * with MGET, those who cannot be retrieved are expired ID Cards so the node update
+     * the set accordingly.
+     *
+     * @return {Array.<IdCard>}
+     */
+    async getRemoteIdCards() {
+        const idCards = [];
+        let keys = await global.kuzzle.ask('core:cache:internal:execute', 'smembers', REDIS_ID_CARDS_INDEX);
+        keys = keys.filter(nodeIdKey => nodeIdKey !== this.nodeIdKey);
+        if (keys.length === 0) {
+            return idCards;
+        }
+        const rawIdCards = await global.kuzzle.ask('core:cache:internal:mget', keys);
+        const expiredIdCards = [];
+        for (let i = 0; i < keys.length; i++) {
+            // filter keys that might have expired between the key search and their
+            // values retrieval
+            if (rawIdCards[i] !== null) {
+                idCards.push(IdCard.unserialize(JSON.parse(rawIdCards[i])));
+            }
+            else {
+                expiredIdCards.push(keys[i]);
+            }
+        }
+        // Clean expired ID Card's keys in the index
+        await bluebird_1.default.map(expiredIdCards, idCardKey => {
+            return global.kuzzle.ask('core:cache:internal:execute', 'srem', REDIS_ID_CARDS_INDEX, idCardKey);
+        });
+        return idCards;
     }
-  }
-
-  /**
-   * Retrieves the ID cards from other nodes
-   * @return {Array.<IdCard>}
-   */
-  async getRemoteIdCards () {
-    const result = [];
-
-    let keys = await global.kuzzle.ask(
-      'core:cache:internal:searchKeys',
-      `${REDIS_PREFIX}*`);
-
-    keys = keys.filter(nodeIdKey => nodeIdKey !== this.nodeIdKey);
-
-    if (keys.length === 0) {
-      return result;
+    /**
+     * Adds a remote node IdCard to the node known topology
+     */
+    async addNode(id) {
+        if (this.disposed || this.idCard.topology.has(id)) {
+            return;
+        }
+        this.idCard.topology.add(id);
+        await this.save();
     }
-
-    const values = await global.kuzzle.ask('core:cache:internal:mget', keys);
-
-    for (let i = 0; i < keys.length; i++) {
-      // filter keys that might have expired between the key search and their
-      // values retrieval
-      if (values[i] !== null) {
-        result.push(IdCard.unserialize(values[i]));
-      }
+    /**
+     * Removes a remote node IdCard from the node known topology
+     */
+    async removeNode(id) {
+        if (!this.disposed && this.idCard.topology.delete(id)) {
+            await this.save();
+        }
     }
-
-    return result;
-  }
-
-  /**
-   * Adds a remote node IdCard to the node known topology
-   */
-  async addNode (id) {
-    if (this.disposed || this.idCard.topology.has(id)) {
-      return;
+    /**
+     * Store the key under which this node ID Card is stored inside the set.
+     *
+     * This set is an index to retrieve ID Cards faster.
+     */
+    async addIdCardToIndex() {
+        await global.kuzzle.ask('core:cache:internal:execute', 'sadd', REDIS_ID_CARDS_INDEX, this.nodeIdKey);
     }
-
-    this.idCard.topology.add(id);
-
-    await this._save();
-  }
-
-  /**
-   * Removes a remote node IdCard from the node known topology
-   */
-  async removeNode (id) {
-    if (!this.disposed && this.idCard.topology.delete(id)) {
-      await this._save();
+    /**
+     * Saves the local node IdCard into Redis
+     *
+     * @returns True if the key was set
+     */
+    async save({ creation } = { creation: false }) {
+        if (!this.idCard) {
+            return false;
+        }
+        return await global.kuzzle.ask('core:cache:internal:store', this.nodeIdKey, JSON.stringify(this.idCard.serialize()), { onlyIfNew: creation, ttl: this.refreshDelay * this.refreshMultiplier });
     }
-  }
-
-  /**
-   * Saves the local node IdCard into Redis
-   */
-  _save ({ creation } = { creation: false }) {
-    return global.kuzzle.ask(
-      'core:cache:internal:store',
-      this.nodeIdKey,
-      this.idCard.serialize(),
-      { onlyIfNew: creation, ttl: this.refreshDelay * 3 });
-  }
 }
-
-module.exports = { ClusterIdCardHandler, IdCard };
+exports.ClusterIdCardHandler = ClusterIdCardHandler;
+//# sourceMappingURL=idCardHandler.js.map

package/lib/cluster/node.js

@@ -474,21 +474,21 @@ class ClusterNode {
       // to prevent race conditions (other nodes attempting to connect to this
       // node while it's still initializing)
       await this.idCardHandler.createIdCard();
-      global.kuzzle.log.debug('[CLUSTER] ID Card created');
+      debug('[CLUSTER] ID Card created');
 
       this.nodeId = this.idCardHandler.nodeId;
 
       await this.startHeartbeat();
-      global.kuzzle.log.debug('[CLUSTER] Start heartbeat');
+      debug('[CLUSTER] Start heartbeat');
 
       let retried = false;
       let fullState = null;
       let nodes;
 
-      global.kuzzle.log.debug('[CLUSTER] Start retrieving full state..');
+      debug('[CLUSTER] Start retrieving full state..');
       do {
         nodes = await this.idCardHandler.getRemoteIdCards();
-        global.kuzzle.log.debug(`[CLUSTER] ${nodes.length} remote nodes discovered`);
+        debug('[CLUSTER] %s remote nodes discovered', nodes.length);
 
         // No other nodes detected = no handshake required
         if (nodes.length === 0) {
@@ -511,7 +511,7 @@ class ClusterNode {
           this.remoteNodes.set(id, subscriber);
           return subscriber.init();
         });
-        global.kuzzle.log.debug('[CLUSTER] Successfully subscribed to nodes');
+        debug('[CLUSTER] Successfully subscribed to nodes');
 
         fullState = await this.command.getFullState(nodes);
 
@@ -541,18 +541,18 @@ class ClusterNode {
         }
       }
       while (fullState === null);
-      global.kuzzle.log.debug('[CLUSTER] Fullstate retrieved, loading into node..');
+      debug('[CLUSTER] Fullstate retrieved, loading into node..');
 
       await this.fullState.loadFullState(fullState);
       this.activity = fullState.activity
         ? fullState.activity
         : this.activity;
 
-      global.kuzzle.log.debug('[CLUSTER] Fullstate loaded.');
+      debug('[CLUSTER] Fullstate loaded.');
 
       const handshakeResponses = await this.command.broadcastHandshake(nodes);
 
-      global.kuzzle.log.debug('[CLUSTER] Successful handshakes with other nodes.');
+      debug('[CLUSTER] Successful handshakes with other nodes.');
 
       // Update subscribers: start synchronizing, or unsubscribes from nodes who
       // didn't respond
@@ -753,6 +753,10 @@ class ClusterNode {
    * @return {void}
    */
   registerEvents () {
+    global.kuzzle.on(
+      'admin:afterRefreshIndexCache',
+      () => this.onIndexCacheRefreshed());
+
     global.kuzzle.on(
       'core:realtime:room:create:after',
       payload => this.onNewRealtimeRoom(payload));
@@ -1123,6 +1127,13 @@ class ClusterNode {
     this.publisher.send('Shutdown', {});
   }
 
+  /**
+   * Triggered when the index cache has been manually refreshed
+   */
+  onIndexCacheRefreshed () {
+    this.publisher.send('RefreshIndexCache', {});
+  }
+
   /**
    * Returns the total number of subscribers on the cluster for the provided
    * room

package/lib/cluster/protobuf/sync.proto

@@ -177,3 +177,7 @@ message ClusterWideEvent {
   string event = 2;
   string payload = 3;
 }
+
+message RefreshIndexCache {
+  uint64 messageId = 1;
+}