kuzzle 2.56.0 → 2.57.0-beta.2

package/dist/lib/cluster/publisher.js

@@ -1,3 +1,4 @@
+"use strict";
 /*
  * Kuzzle, a backend software, self-hostable and ready to use
  * to power modern apps
@@ -18,24 +19,59 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-"use strict";
-const { Publisher } = require("zeromq");
-const protobuf = require("protobufjs");
-const Long = require("long");
-const Bluebird = require("bluebird");
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+const bluebird_1 = __importDefault(require("bluebird"));
+const long_1 = __importDefault(require("long"));
+const protobuf = __importStar(require("protobufjs"));
+const zeromq_1 = require("zeromq");
 const STATE = Object.freeze({
     READY: 1,
     SENDING: 2,
 });
+/**
+ * Test-only fault injection: when set to N > 0, every Nth **heartbeat** this
+ * node publishes is recorded as sent but never handed to the socket, so that
+ * the functional suites exercise the retransmission path. Unset in production.
+ *
+ * Heartbeats only: a loss is noticed when the sender's *next* message arrives,
+ * up to one heartbeat later, so dropping a state message (a new collection,
+ * say) would let a scenario read another node before the gap is filled — a
+ * timing artefact of the injection, not a retransmission failure.
+ */
+const DROP_HEARTBEAT_EVERY_ENV = "KUZZLE_TEST_CLUSTER_DROP_HEARTBEAT_EVERY";
 // Handles messages publication to other nodes
 class ClusterPublisher {
     /**
-     * @constructor
-     * @param {ClusterNode} node
+     * @param node the cluster node this publisher belongs to
      */
     constructor(node) {
         this.node = node;
-        this.lastMessageId = new Long(0, 0, true);
+        this.lastMessageId = new long_1.default(0, 0, true);
         this.socket = null;
         this.protoroot = null;
         // Only one call to socket.send can be performed at any given time, and it
@@ -46,17 +82,23 @@ class ClusterPublisher {
         // by bufferizing requests to send
         this.state = STATE.READY;
         this.buffer = [];
+        this.history = [];
+        this.historyBytes = 0;
+        const dropEvery = Number.parseInt(process.env[DROP_HEARTBEAT_EVERY_ENV] ?? "", 10);
+        this.dropHeartbeatEvery =
+            Number.isInteger(dropEvery) && dropEvery > 0 ? dropEvery : 0;
+        this.heartbeatsSent = 0;
     }
     async init() {
-        this.socket = new Publisher();
+        this.socket = new zeromq_1.Publisher();
         await this.socket.bind(`tcp://*:${this.node.config.ports.sync}`);
         this.protoroot = await protobuf.load(`${__dirname}/protobuf/sync.proto`);
     }
     /**
      * Publishes an event telling them that this node has created a new room
      *
-     * @param {NormalizedFilter} normalized - Obtained with Koncorde.normalize()
-     * @return {Long} ID of the message sent to other nodes
+     * @param normalized - Obtained with Koncorde.normalize()
+     * @return ID of the message sent to other nodes
      */
     sendNewRealtimeRoom(normalized) {
         const payload = {
@@ -70,16 +112,16 @@ class ClusterPublisher {
      * Publishes an event telling telling them that this node
      * no longer has subscribers on the provided realtime room.
      *
-     * @param  {string} roomId
-     * @return {Long} ID of the message sent to other nodes
+     * @param roomId
+     * @return ID of the message sent to other nodes
      */
     sendRemoveRealtimeRoom(roomId) {
         return this.send("RemoveRealtimeRoom", { roomId });
     }
     /**
      * Publishes an event telling that a room has one less subscriber
-     * @param  {string} roomId
-     * @return {Long} ID of the message sent to other nodes
+     * @param roomId
+     * @return ID of the message sent to other nodes
      */
     sendUnsubscription(roomId) {
         return this.send("Unsubscription", { roomId });
@@ -88,8 +130,8 @@ class ClusterPublisher {
      * Publishes an event telling that a new subscription has been made
      * made to an existing room
      *
-     * @param  {string} roomId
-     * @return {Long} Id of the message sent to other nodes
+     * @param roomId
+     * @return Id of the message sent to other nodes
      */
     sendSubscription(roomId) {
         return this.send("Subscription", { roomId });
@@ -98,9 +140,9 @@ class ClusterPublisher {
      * Publishes an event telling that a document notification must be
      * propagated
      *
-     * @param  {Array.<string>} rooms - Koncorde rooms
-     * @param  {DocumentNotification} notification
-     * @return {Long} Id of the message sent to other nodes
+     * @param rooms - Koncorde rooms
+     * @param notification
+     * @return Id of the message sent to other nodes
      */
     sendDocumentNotification(rooms, notification) {
         return this.send("DocumentNotification", {
@@ -122,9 +164,9 @@ class ClusterPublisher {
      * Publishes an event telling that a user notification must be
      * propagated
      *
-     * @param  {string} room - Koncorde room
-     * @param  {UserNotification} notification
-     * @return {Long} Id of the message sent to other nodes
+     * @param room - Koncorde room
+     * @param notification
+     * @return Id of the message sent to other nodes
      */
     sendUserNotification(room, notification) {
         return this.send("UserNotification", {
@@ -145,10 +187,10 @@ class ClusterPublisher {
      * Publishes an event telling that a new authentication strategy has been
      * dynamically added
      *
-     * @param {string} strategyName
-     * @param {string} pluginName
-     * @param {Object} strategy
-     * @return {Long} Id of the message sent to other nodes
+     * @param strategyName
+     * @param pluginName
+     * @param strategy
+     * @return Id of the message sent to other nodes
      */
     sendNewAuthStrategy(strategyName, pluginName, strategy) {
         return this.send("NewAuthStrategy", {
@@ -161,10 +203,9 @@ class ClusterPublisher {
      * Publishes an event telling that a new authentication strategy has been
      * dynamically added
      *
-     * @param {string} strategyName
-     * @param {string} pluginName
-     * @param {Object} strategyObject
-     * @return {Long} Id of the message sent to other nodes
+     * @param strategyName
+     * @param pluginName
+     * @return Id of the message sent to other nodes
      */
     sendRemoveAuthStrategy(strategyName, pluginName) {
         return this.send("RemoveAuthStrategy", {
@@ -175,8 +216,8 @@ class ClusterPublisher {
     /**
      * Publishes an event telling other nodes to create an info dump
      *
-     * @param  {string} suffix - dump directory suffix name
-     * @returns {Long} ID of the message sent
+     * @param suffix - dump directory suffix name
+     * @returns ID of the message sent
      */
     sendDumpRequest(suffix) {
         return this.send("DumpRequest", { suffix });
@@ -184,9 +225,9 @@ class ClusterPublisher {
     /**
      * Publishes an event about a new index being added
      *
-     * @param  {storeScopeEnum} scope
-     * @param  {string} index
-     * @returns {Long} ID of the message sent
+     * @param scope
+     * @param index
+     * @returns ID of the message sent
      */
     sendAddIndex(scope, index) {
         return this.send("AddIndex", { index, scope });
@@ -194,10 +235,10 @@ class ClusterPublisher {
     /**
      * Publishes an event about a new collection being added
      *
-     * @param  {storeScopeEnum} scope
-     * @param  {string} index
-     * @param  {string} collection
-     * @returns {Long} ID of the message sent
+     * @param scope
+     * @param index
+     * @param collection
+     * @returns ID of the message sent
      */
     sendAddCollection(scope, index, collection) {
         return this.send("AddCollection", { collection, index, scope });
@@ -205,9 +246,9 @@ class ClusterPublisher {
     /**
      * Publishes an event about indexes been removed
      *
-     * @param  {storeScopeEnum} scope
-     * @param  {Array.<string>} indexes
-     * @returns {Long} ID of the message sent
+     * @param scope
+     * @param indexes
+     * @returns ID of the message sent
      */
     sendRemoveIndexes(scope, indexes) {
         return this.send("RemoveIndexes", { indexes, scope });
@@ -215,10 +256,10 @@ class ClusterPublisher {
     /**
      * Publishes an event about a collection been removed
      *
-     * @param  {storeScopeEnum} scope
-     * @param  {string} index
-     * @param  {string} collection
-     * @returns {Long} ID of the message sent
+     * @param scope
+     * @param index
+     * @param collection
+     * @returns ID of the message sent
      */
     sendRemoveCollection(scope, index, collection) {
         return this.send("RemoveCollection", { collection, index, scope });
@@ -226,9 +267,9 @@ class ClusterPublisher {
     /**
      * Publishes an event about a cluster-wide event emission
      *
-     * @param  {string} event name
-     * @param  {Object} payload - event payload
-     * @returns {Long} ID of the message sent
+     * @param event name
+     * @param payload - event payload
+     * @returns ID of the message sent
      */
     sendClusterWideEvent(event, payload) {
         return this.send("ClusterWideEvent", {
@@ -239,18 +280,18 @@ class ClusterPublisher {
     /**
      * Publishes an event about shutdown
      *
-     * @param  {Object} nodeId - node ID
+     * @param nodeId - node ID
      *
-     * @returns {Long} ID of the message sent
+     * @returns ID of the message sent
      */
     sendNodeShutdown(nodeId) {
         return this.send("NodeShutdown", { nodeId });
     }
     /**
      * Publishes an event about the node being in debug mode
-     * @param {bool} evictionPrevented
+     * @param evictionPrevented
      *
-     * @returns {Long} ID of the message sent
+     * @returns ID of the message sent
      */
     sendNodePreventEviction(evictionPrevented) {
         return this.send("NodePreventEviction", { evictionPrevented });
@@ -258,11 +299,11 @@ class ClusterPublisher {
     /**
      * Publishes an event about a node being evicted
      *
-     * @param  {string} evictor node ID of the evictor
-     * @param  {Object} nodeId - node ID of the node being evicted
-     * @param  {Object} reason - reason of the eviction
+     * @param evictor node ID of the evictor
+     * @param nodeId - node ID of the node being evicted
+     * @param reason - reason of the eviction
      *
-     * @returns {Long} ID of the message sent
+     * @returns ID of the message sent
      */
     sendNodeEvicted(evictor, nodeId, reason) {
         return this.send("NodeEvicted", { evictor, nodeId, reason });
@@ -270,9 +311,9 @@ class ClusterPublisher {
     /**
      * Publishes an event about heartbeat
      *
-     * @param  {string} address node sync address
+     * @param address node sync address
      *
-     * @returns {Long} ID of the message sent
+     * @returns ID of the message sent
      */
     sendHeartbeat(address) {
         return this.send("Heartbeat", { address });
@@ -282,19 +323,29 @@ class ClusterPublisher {
      * Returns immediately, but the message to be sent migh be bufferized and send
      * later.
      *
-     * @param  {String} topic name
-     * @param  {Object} data
-     * @returns {Long} ID of the message sent
+     * @param topic name
+     * @param data
+     * @returns ID of the message sent
      * @throws If the topic's protobuf type cannot be found
      */
     send(topic, data) {
-        if (this.socket === null) {
-            return Long.NEG_ONE;
+        // `socket` and `protoroot` are both filled by `init()` and nulled by
+        // `dispose()`, together: one guard answers for both, where the old one
+        // answered for the socket and left `protoroot` to be taken on trust.
+        if (this.socket === null || this.protoroot === null) {
+            return long_1.default.NEG_ONE;
         }
         this.lastMessageId = this.lastMessageId.add(1);
-        const payload = Object.assign({ messageId: this.lastMessageId }, data);
+        const payload = { messageId: this.lastMessageId, ...data };
         const type = this.protoroot.lookupType(topic);
         const buffer = type.encode(type.create(payload)).finish();
+        this.remember(this.lastMessageId, topic, buffer);
+        if (this.dropHeartbeatEvery > 0 && topic === "Heartbeat") {
+            this.heartbeatsSent++;
+            if (this.heartbeatsSent % this.dropHeartbeatEvery === 0) {
+                return this.lastMessageId;
+            }
+        }
         // DO NOT AWAIT: bufferSend is built to bufferize payloads to be sent, and
         // it makes sure that they are sent in order and serially (0mq publisher
         // sockets can only send 1 message at a time, otherwise it throws with a
@@ -306,13 +357,53 @@ class ClusterPublisher {
         this.bufferSend(topic, buffer);
         return this.lastMessageId;
     }
+    /**
+     * The messages sent with ids `from` to `to`, both included, in order — or
+     * `null` if any of them is no longer kept (or never was).
+     */
+    replay(from, to) {
+        const oldest = this.history[0];
+        const newest = this.history.at(-1);
+        if (oldest === undefined ||
+            newest === undefined ||
+            from.greaterThan(to) ||
+            from.lessThan(oldest.messageId) ||
+            to.greaterThan(newest.messageId)) {
+            return null;
+        }
+        // Ids are contiguous, so offsets are plain differences; both are bounded
+        // by the history's length, far below 2^53.
+        const start = from.subtract(oldest.messageId).toNumber();
+        const end = to.subtract(oldest.messageId).toNumber();
+        return this.history
+            .slice(start, end + 1)
+            .map(({ data, topic }) => ({ data, topic }));
+    }
+    /**
+     * Keeps a sent message for `replay()`, then trims the oldest ones until
+     * both of `cluster.retransmitBuffer`'s bounds hold. A bound of 0 keeps
+     * nothing: retransmission is disabled.
+     */
+    remember(messageId, topic, data) {
+        // Always loaded; optional in the public type (see IKuzzleConfiguration).
+        const { bytes, messages } = this.node.config.retransmitBuffer;
+        if (messages <= 0 || bytes <= 0) {
+            return;
+        }
+        this.history.push({ data, messageId, topic });
+        this.historyBytes += data.byteLength;
+        while (this.history.length > messages ||
+            (this.historyBytes > bytes && this.history.length > 0)) {
+            const dropped = this.history.shift();
+            this.historyBytes -= dropped?.data.byteLength ?? 0;
+        }
+    }
     /**
      * Sends the provided message, and then sends all pending ones waiting in
      * the buffer, in order
      *
-     * @param  {string} topic
-     * @param  {Buffer} data
-     * @return {void}
+     * @param topic
+     * @param data
      */
     async bufferSend(topic, data) {
         this.buffer.push({ data, topic });
@@ -323,8 +414,19 @@ class ClusterPublisher {
         do {
             const _buffer = this.buffer;
             this.buffer = [];
-            for (let i = 0; i < _buffer.length; i++) {
-                const payload = _buffer[i];
+            // `_buffer` is a local snapshot — `this.buffer` is reassigned to a new
+            // array above, never mutated in place — so iterating it directly is the
+            // same traversal as the index loop this replaces, without the
+            // `noUncheckedIndexedAccess` noise.
+            for (const payload of _buffer) {
+                // `dispose()` can null the socket while this loop awaits. Before, that
+                // surfaced as a TypeError on the next iteration; there is nothing left
+                // to flush once the socket is gone.
+                if (this.socket === null) {
+                    this.buffer = [];
+                    this.state = STATE.READY;
+                    return;
+                }
                 // This method will never return a rejected promise
                 // http://zeromq.github.io/zeromq.js/classes/publisher.html#send
                 await this.socket.send([payload.topic, payload.data]);
@@ -335,7 +437,7 @@ class ClusterPublisher {
     async dispose() {
         // waits for the buffer to be empty before closing
         while (this.state !== STATE.READY) {
-            await Bluebird.delay(100);
+            await bluebird_1.default.delay(100);
         }
         if (this.socket !== null) {
             this.socket.close();

package/dist/lib/cluster/state.d.ts

@@ -1,7 +1,8 @@
 import { NormalizedFilter } from "koncorde";
-import { JSONObject } from "kuzzle-sdk";
-import { RoomList } from "../types";
-import Long from "long";
+import type { JSONObject } from "../types/JSONObject";
+import type { RoomList } from "../types";
+import type { FullStateAuthStrategy } from "./protobuf/commandMessages";
+import type Long from "long";
 import "../types/Global";
 export type SerializedRoomSubscriptions = {
     nodeId: string;
@@ -26,13 +27,13 @@ export default class State {
      *
      * Map<roomId, RoomState>
      */
-    private realtime;
+    private readonly realtime;
     /**
      * State of authentication strategies
      *
      * Map<strategyName, strategyDefinition>
      */
-    private strategies;
+    private readonly strategies;
     /**
      * Adds a new realtime room to the state
      *
@@ -70,10 +71,10 @@ export default class State {
      * Adds a new dynamic strategy to the full state
      *
      */
-    addAuthStrategy(strategyObject: JSONObject): void;
+    addAuthStrategy(strategyObject: FullStateAuthStrategy): void;
     removeAuthStrategy(strategyName: string): void;
     serialize(): {
-        authStrategies: JSONObject[];
+        authStrategies: FullStateAuthStrategy[];
         rooms: SerializedRoomState[];
     };
     /**
@@ -86,9 +87,14 @@ export default class State {
      *
      * @param serialized POJO object of a full realtime state
      */
-    loadFullState(serialized: SerializedState): void;
+    /**
+     * `Partial`: this reads a full state off the wire, and protobuf leaves an
+     * absent field undefined. Both branches below already guard for it; the
+     * parameter described what `serialize()` produces, not what arrives.
+     */
+    loadFullState(serialized: Partial<SerializedState>): void;
 }
 export type SerializedState = {
-    authStrategies: JSONObject[];
+    authStrategies: FullStateAuthStrategy[];
     rooms: SerializedRoomState[];
 };

package/dist/lib/cluster/state.js

@@ -250,15 +250,15 @@ class State {
         return room.countSubscriptions();
     }
     listRealtimeRooms() {
+        var _a, _b;
         const list = {};
         for (const room of this.realtime.values()) {
-            if (!list[room.index]) {
-                list[room.index] = {};
-            }
-            if (!list[room.index][room.collection]) {
-                list[room.index][room.collection] = {};
-            }
-            list[room.index][room.collection][room.id] = room.countSubscriptions();
+            // Held in locals rather than re-indexed: the compiler cannot see that the
+            // branch above just assigned the entry, so each re-read was a "possibly
+            // undefined". Same lookups, one per level instead of three.
+            const index = (list[_a = room.index] ?? (list[_a] = {}));
+            const collection = (index[_b = room.collection] ?? (index[_b] = {}));
+            collection[room.id] = room.countSubscriptions();
         }
         return list;
     }
@@ -312,6 +312,11 @@ class State {
      *
      * @param serialized POJO object of a full realtime state
      */
+    /**
+     * `Partial`: this reads a full state off the wire, and protobuf leaves an
+     * absent field undefined. Both branches below already guard for it; the
+     * parameter described what `serialize()` produces, not what arrives.
+     */
     loadFullState(serialized) {
         if (serialized.rooms) {
             for (const state of serialized.rooms) {
@@ -329,5 +334,4 @@ class State {
     }
 }
 exports.default = State;
-module.exports = State;
 //# sourceMappingURL=state.js.map