kuzzle 2.56.0 → 2.57.0-beta.2

package/dist/lib/cluster/protobuf/command.proto

@@ -60,3 +60,21 @@ message HandshakeResponse {
   bool added = 1;
   uint64 lastMessageId = 2;
 }
+
+// Retransmit request: the messages a subscriber missed from this node's sync
+// stream, by message id, both ends included (#2785)
+message RetransmitRequest {
+  uint64 from = 1;
+  uint64 to = 2;
+}
+
+// Retransmit response: the missed messages, in order, exactly as they were
+// published (topic + encoded sync message)
+message RetransmitResponse {
+  message Frame {
+    string topic = 1;
+    bytes data = 2;
+  }
+
+  repeated Frame frames = 1;
+}

package/dist/lib/cluster/protobuf/commandMessages.d.ts

@@ -0,0 +1,104 @@
+import type Long from "long";
+import type { NewAuthStrategyMessage } from "./syncMessages";
+/**
+ * The shapes `command.proto` puts on the wire, as `protobufjs`' `toObject()`
+ * returns them.
+ *
+ * Same contract as [`syncMessages.ts`](./syncMessages.ts): `command.proto` is
+ * the source of truth and these types mirror it field for field, so the command
+ * layer is checked against the schema rather than against a reader's memory of
+ * it. Keep them in step with the `.proto`.
+ *
+ * The two decoding details `syncMessages.ts` documents apply here too: a proto
+ * `uint64` comes back as a **`Long`** (`lastMessageId`, `messageId`), a `uint32`
+ * as a plain number (`subscribers`, `event`), and a field the schema comments as
+ * needing to be "stringified beforehand" is a **string** its reader parses.
+ */
+/**
+ * A message as `toObject()` hands it back, before anything has established that
+ * its fields are there. Every proto3 field is optional on the wire, and
+ * `toObject()` returns an index-signature object — so this is what a decode
+ * actually yields, and narrowing it is the reader's job. `subscriber.ts`'s
+ * `DecodedMessage` is the same admission for `sync.proto`.
+ */
+export type Decoded<T> = Partial<T> & Record<string, unknown>;
+/** `FullStateResponse.NodeSubscribers` — who subscribed to a realtime room. */
+export type NodeSubscribers = {
+    nodeId: string;
+    subscribers: number;
+    messageId: Long;
+};
+/** `FullStateResponse.RealtimeRoom`. */
+export type RealtimeRoom = {
+    roomId: string;
+    index: string;
+    collection: string;
+    /**
+     * Serialized JSON. The schema says so explicitly: protobuf cannot carry an
+     * array of polymorphic objects, so the filters travel stringified.
+     */
+    filters: string;
+    nodes: NodeSubscribers[];
+};
+/**
+ * `FullStateResponse.Activity`. `event` is a `nodeActivityEnum` value, which
+ * the schema can only express as a `uint32`.
+ */
+export type Activity = {
+    id: string;
+    date: string;
+    address: string;
+    event: number;
+    reason?: string;
+};
+/** `FullStateResponse.NodeState` — a node and the last message it published. */
+export type NodeState = {
+    id: string;
+    lastMessageId: Long;
+};
+/**
+ * `command.proto` declares `authStrategies` as `repeated sync.NewAuthStrategy`,
+ * but the `messageId` that message carries is **never populated on this path**:
+ * the full state stores the strategy triple exactly as `addAuthStrategy()`
+ * received it, with no message id. The schema reuses a type that has a field
+ * this payload has no value for; the `Omit` says so rather than inventing one.
+ */
+export type FullStateAuthStrategy = Omit<NewAuthStrategyMessage, "messageId">;
+export type FullStateResponse = {
+    rooms: RealtimeRoom[];
+    authStrategies: FullStateAuthStrategy[];
+    activity: Activity[];
+    nodesState: NodeState[];
+};
+export type HandshakeRequest = {
+    nodeId: string;
+    lastMessageId: Long;
+    ip: string;
+};
+export type HandshakeResponse = {
+    added: boolean;
+    lastMessageId: Long;
+};
+/**
+ * The topics the command REP/REQ socket answers on. `DISCARDED` is the reply to
+ * anything else: REQ/REP requires a reply to every request, so an unknown topic
+ * still gets an (empty) answer rather than leaving the peer waiting.
+ */
+export type RetransmitRequest = {
+    from: Long;
+    to: Long;
+};
+/** One sync message as it was published: its topic and its encoded payload. */
+export type RetransmitFrame = {
+    topic: string;
+    data: Uint8Array;
+};
+export type RetransmitResponse = {
+    frames: RetransmitFrame[];
+};
+export declare const commandTopic: Readonly<{
+    DISCARDED: "discarded";
+    FULLSTATE: "fullstate";
+    HANDSHAKE: "handshake";
+    RETRANSMIT: "retransmit";
+}>;

package/dist/lib/cluster/protobuf/commandMessages.js

@@ -0,0 +1,30 @@
+"use strict";
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.commandTopic = void 0;
+exports.commandTopic = Object.freeze({
+    DISCARDED: "discarded",
+    FULLSTATE: "fullstate",
+    HANDSHAKE: "handshake",
+    RETRANSMIT: "retransmit",
+});
+//# sourceMappingURL=commandMessages.js.map

package/dist/lib/cluster/protobuf/syncMessages.d.ts

@@ -0,0 +1,184 @@
+import type Long from "long";
+/**
+ * The shapes `sync.proto` puts on the wire, as `protobufjs`' `toObject()`
+ * returns them.
+ *
+ * `sync.proto` is the source of truth: these types mirror it field for field
+ * and exist so the cluster's message handlers are checked against the schema
+ * rather than against a reader's memory of it. Keep them in step with the
+ * `.proto` — a field added there and forgotten here is a field the compiler
+ * will say does not exist.
+ *
+ * Two decoding details are load-bearing:
+ * - a proto `uint64` comes back as a **`Long`**, not a number, because the
+ *   `long` package is installed (`messageId` and `timestamp`);
+ * - a proto `uint32` comes back as a plain number (`status`);
+ * - fields commented "serialized JSON" in the schema are **strings** here, and
+ *   every handler that reads one calls `JSON.parse` on it.
+ */
+/**
+ * Every sync message carries a `messageId`: it is what lets a subscriber detect
+ * that it missed one. `sync.proto` states the rule in its header comment, and
+ * `validateMessage()` enforces it on the wire.
+ */
+export type SyncMessage = {
+    messageId: Long;
+};
+export type HeartbeatMessage = SyncMessage & {
+    address: string;
+};
+export type NodeEvictedMessage = SyncMessage & {
+    nodeId: string;
+    reason: string;
+    evictor: string;
+};
+export type NodeShutdownMessage = SyncMessage & {
+    nodeId: string;
+};
+export type NodePreventEvictionMessage = SyncMessage & {
+    evictionPrevented: boolean;
+};
+export type NewRealtimeRoomMessage = SyncMessage & {
+    /** Koncorde index, i.e. an index/collection pair encoded as one string */
+    index: string;
+    id: string;
+    /** serialized JSON */
+    filter: string;
+};
+export type RemoveRealtimeRoomMessage = SyncMessage & {
+    roomId: string;
+};
+export type SubscriptionMessage = SyncMessage & {
+    roomId: string;
+};
+export type UnsubscriptionMessage = SyncMessage & {
+    roomId: string;
+};
+export type DocumentNotificationMessage = SyncMessage & {
+    rooms: string[];
+    status: number;
+    requestId: string;
+    timestamp: Long;
+    index: string;
+    collection: string;
+    controller: string;
+    action: string;
+    protocol: string;
+    scope: string;
+    /** serialized JSON */
+    volatile: string;
+    /** serialized JSON */
+    result: string;
+};
+export type UserNotificationMessage = SyncMessage & {
+    room: string;
+    status: number;
+    timestamp: Long;
+    index: string;
+    collection: string;
+    controller: string;
+    action: string;
+    protocol: string;
+    user: string;
+    /** serialized JSON */
+    volatile: string;
+    /** serialized JSON */
+    result: string;
+};
+export type AuthStrategyConfig = {
+    authenticator: string;
+    fields: string[];
+    /** serialized JSON */
+    authenticateOptions: string;
+    /** serialized JSON */
+    strategyOptions: string;
+};
+export type AuthStrategyMethods = {
+    create: string;
+    delete: string;
+    exists: string;
+    update: string;
+    validate: string;
+    verify: string;
+    getById?: string;
+    getInfo?: string;
+    afterRegister?: string;
+};
+export type AuthStrategy = {
+    methods: AuthStrategyMethods;
+    config: AuthStrategyConfig;
+};
+export type NewAuthStrategyMessage = SyncMessage & {
+    strategyName: string;
+    pluginName: string;
+    strategy: AuthStrategy;
+};
+export type RemoveAuthStrategyMessage = SyncMessage & {
+    strategyName: string;
+    pluginName: string;
+};
+export type DumpRequestMessage = SyncMessage & {
+    suffix: string;
+};
+export type InvalidateProfileMessage = SyncMessage & {
+    profileId: string;
+};
+export type InvalidateRoleMessage = SyncMessage & {
+    roleId: string;
+};
+export type AddIndexMessage = SyncMessage & {
+    scope: string;
+    index: string;
+};
+export type AddCollectionMessage = SyncMessage & {
+    scope: string;
+    index: string;
+    collection: string;
+};
+export type RemoveIndexesMessage = SyncMessage & {
+    scope: string;
+    indexes: string[];
+};
+export type RemoveCollectionMessage = SyncMessage & {
+    scope: string;
+    index: string;
+    collection: string;
+};
+export type ClusterWideEventMessage = SyncMessage & {
+    event: string;
+    /** serialized JSON */
+    payload: string;
+};
+/**
+ * Topic name → the message that topic carries. A topic is a protobuf type name,
+ * so this map is the same list as `sync.proto`'s, and `ClusterSubscriber`'s
+ * handler table is typed against it: a handler registered under the wrong topic
+ * stops compiling.
+ */
+export type SyncMessages = {
+    AddCollection: AddCollectionMessage;
+    AddIndex: AddIndexMessage;
+    ClusterWideEvent: ClusterWideEventMessage;
+    DocumentNotification: DocumentNotificationMessage;
+    DumpRequest: DumpRequestMessage;
+    Heartbeat: HeartbeatMessage;
+    InvalidateProfile: InvalidateProfileMessage;
+    InvalidateRole: InvalidateRoleMessage;
+    NewAuthStrategy: NewAuthStrategyMessage;
+    NewRealtimeRoom: NewRealtimeRoomMessage;
+    NodeEvicted: NodeEvictedMessage;
+    NodePreventEviction: NodePreventEvictionMessage;
+    NodeShutdown: NodeShutdownMessage;
+    RefreshIndexCache: SyncMessage;
+    RefreshValidators: SyncMessage;
+    RemoveAuthStrategy: RemoveAuthStrategyMessage;
+    RemoveCollection: RemoveCollectionMessage;
+    RemoveIndexes: RemoveIndexesMessage;
+    RemoveRealtimeRoom: RemoveRealtimeRoomMessage;
+    ResetSecurity: SyncMessage;
+    Shutdown: SyncMessage;
+    Subscription: SubscriptionMessage;
+    Unsubscription: UnsubscriptionMessage;
+    UserNotification: UserNotificationMessage;
+};
+export type SyncTopic = keyof SyncMessages;

package/dist/lib/cluster/protobuf/syncMessages.js

@@ -0,0 +1,23 @@
+"use strict";
+/*
+ * Kuzzle, a backend software, self-hostable and ready to use
+ * to power modern apps
+ *
+ * Copyright 2015-2022 Kuzzle
+ * mailto: support AT kuzzle.io
+ * website: http://kuzzle.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=syncMessages.js.map

package/dist/lib/cluster/publisher.d.ts

@@ -1,164 +1,196 @@
-export = ClusterPublisher;
+import Long from "long";
+import type { JSONObject } from "../types/JSONObject";
+import type { NormalizedFilter } from "koncorde";
+import type DocumentNotification from "../core/realtime/notification/document";
+import type UserNotification from "../core/realtime/notification/user";
+import type { storeScopeEnum } from "../core/storage/storeScopeEnum";
+import type { IKuzzleConfiguration } from "../types/config/KuzzleConfiguration";
+/**
+ * What the publisher needs from the node that owns it. `node.js` is converted
+ * in J3; this is the part of its surface this file actually reads, not a
+ * placeholder for the whole class.
+ */
+type PublishingNode = {
+    config: IKuzzleConfiguration["cluster"];
+};
+type BufferedMessage = {
+    topic: string;
+    /**
+     * protobuf's `finish()` returns a `Uint8Array`, not a `Buffer` — the JSDoc on
+     * `bufferSend` claimed `Buffer` for as long as the file was JavaScript.
+     */
+    data: Uint8Array;
+};
 declare class ClusterPublisher {
+    private readonly node;
     /**
-     * @constructor
-     * @param {ClusterNode} node
+     * ID of the last message sent. Read by `command.js` to answer a handshake,
+     * and by every `send*` caller that needs to name the message it just queued.
      */
-    constructor(node: ClusterNode);
-    node: ClusterNode;
     lastMessageId: Long;
-    socket: Publisher;
-    protoroot: protobuf.Root;
-    state: 1;
-    buffer: any[];
+    private socket;
+    private protoroot;
+    private state;
+    private buffer;
+    /**
+     * The last messages sent, oldest first, with contiguous ids: what
+     * `replay()` answers from. Bounded by `cluster.retransmitBuffer`.
+     */
+    private readonly history;
+    private historyBytes;
+    private readonly dropHeartbeatEvery;
+    private heartbeatsSent;
+    /**
+     * @param node the cluster node this publisher belongs to
+     */
+    constructor(node: PublishingNode);
     init(): Promise<void>;
     /**
      * Publishes an event telling them that this node has created a new room
      *
-     * @param {NormalizedFilter} normalized - Obtained with Koncorde.normalize()
-     * @return {Long} ID of the message sent to other nodes
+     * @param normalized - Obtained with Koncorde.normalize()
+     * @return ID of the message sent to other nodes
      */
     sendNewRealtimeRoom(normalized: NormalizedFilter): Long;
     /**
      * Publishes an event telling telling them that this node
      * no longer has subscribers on the provided realtime room.
      *
-     * @param  {string} roomId
-     * @return {Long} ID of the message sent to other nodes
+     * @param roomId
+     * @return ID of the message sent to other nodes
      */
     sendRemoveRealtimeRoom(roomId: string): Long;
     /**
      * Publishes an event telling that a room has one less subscriber
-     * @param  {string} roomId
-     * @return {Long} ID of the message sent to other nodes
+     * @param roomId
+     * @return ID of the message sent to other nodes
      */
     sendUnsubscription(roomId: string): Long;
     /**
      * Publishes an event telling that a new subscription has been made
      * made to an existing room
      *
-     * @param  {string} roomId
-     * @return {Long} Id of the message sent to other nodes
+     * @param roomId
+     * @return Id of the message sent to other nodes
      */
     sendSubscription(roomId: string): Long;
     /**
      * Publishes an event telling that a document notification must be
      * propagated
      *
-     * @param  {Array.<string>} rooms - Koncorde rooms
-     * @param  {DocumentNotification} notification
-     * @return {Long} Id of the message sent to other nodes
+     * @param rooms - Koncorde rooms
+     * @param notification
+     * @return Id of the message sent to other nodes
      */
-    sendDocumentNotification(rooms: Array<string>, notification: DocumentNotification): Long;
+    sendDocumentNotification(rooms: string[], notification: DocumentNotification): Long;
     /**
      * Publishes an event telling that a user notification must be
      * propagated
      *
-     * @param  {string} room - Koncorde room
-     * @param  {UserNotification} notification
-     * @return {Long} Id of the message sent to other nodes
+     * @param room - Koncorde room
+     * @param notification
+     * @return Id of the message sent to other nodes
      */
     sendUserNotification(room: string, notification: UserNotification): Long;
     /**
      * Publishes an event telling that a new authentication strategy has been
      * dynamically added
      *
-     * @param {string} strategyName
-     * @param {string} pluginName
-     * @param {Object} strategy
-     * @return {Long} Id of the message sent to other nodes
+     * @param strategyName
+     * @param pluginName
+     * @param strategy
+     * @return Id of the message sent to other nodes
      */
-    sendNewAuthStrategy(strategyName: string, pluginName: string, strategy: any): Long;
+    sendNewAuthStrategy(strategyName: string, pluginName: string, strategy: JSONObject): Long;
     /**
      * Publishes an event telling that a new authentication strategy has been
      * dynamically added
      *
-     * @param {string} strategyName
-     * @param {string} pluginName
-     * @param {Object} strategyObject
-     * @return {Long} Id of the message sent to other nodes
+     * @param strategyName
+     * @param pluginName
+     * @return Id of the message sent to other nodes
      */
     sendRemoveAuthStrategy(strategyName: string, pluginName: string): Long;
     /**
      * Publishes an event telling other nodes to create an info dump
      *
-     * @param  {string} suffix - dump directory suffix name
-     * @returns {Long} ID of the message sent
+     * @param suffix - dump directory suffix name
+     * @returns ID of the message sent
      */
     sendDumpRequest(suffix: string): Long;
     /**
      * Publishes an event about a new index being added
      *
-     * @param  {storeScopeEnum} scope
-     * @param  {string} index
-     * @returns {Long} ID of the message sent
+     * @param scope
+     * @param index
+     * @returns ID of the message sent
      */
     sendAddIndex(scope: storeScopeEnum, index: string): Long;
     /**
      * Publishes an event about a new collection being added
      *
-     * @param  {storeScopeEnum} scope
-     * @param  {string} index
-     * @param  {string} collection
-     * @returns {Long} ID of the message sent
+     * @param scope
+     * @param index
+     * @param collection
+     * @returns ID of the message sent
      */
     sendAddCollection(scope: storeScopeEnum, index: string, collection: string): Long;
     /**
      * Publishes an event about indexes been removed
      *
-     * @param  {storeScopeEnum} scope
-     * @param  {Array.<string>} indexes
-     * @returns {Long} ID of the message sent
+     * @param scope
+     * @param indexes
+     * @returns ID of the message sent
      */
-    sendRemoveIndexes(scope: storeScopeEnum, indexes: Array<string>): Long;
+    sendRemoveIndexes(scope: storeScopeEnum, indexes: string[]): Long;
     /**
      * Publishes an event about a collection been removed
      *
-     * @param  {storeScopeEnum} scope
-     * @param  {string} index
-     * @param  {string} collection
-     * @returns {Long} ID of the message sent
+     * @param scope
+     * @param index
+     * @param collection
+     * @returns ID of the message sent
      */
     sendRemoveCollection(scope: storeScopeEnum, index: string, collection: string): Long;
     /**
      * Publishes an event about a cluster-wide event emission
      *
-     * @param  {string} event name
-     * @param  {Object} payload - event payload
-     * @returns {Long} ID of the message sent
+     * @param event name
+     * @param payload - event payload
+     * @returns ID of the message sent
      */
-    sendClusterWideEvent(event: string, payload: any): Long;
+    sendClusterWideEvent(event: string, payload: JSONObject): Long;
     /**
      * Publishes an event about shutdown
      *
-     * @param  {Object} nodeId - node ID
+     * @param nodeId - node ID
      *
-     * @returns {Long} ID of the message sent
+     * @returns ID of the message sent
      */
-    sendNodeShutdown(nodeId: any): Long;
+    sendNodeShutdown(nodeId: string): Long;
     /**
      * Publishes an event about the node being in debug mode
-     * @param {bool} evictionPrevented
+     * @param evictionPrevented
      *
-     * @returns {Long} ID of the message sent
+     * @returns ID of the message sent
      */
-    sendNodePreventEviction(evictionPrevented: bool): Long;
+    sendNodePreventEviction(evictionPrevented: boolean): Long;
     /**
      * Publishes an event about a node being evicted
      *
-     * @param  {string} evictor node ID of the evictor
-     * @param  {Object} nodeId - node ID of the node being evicted
-     * @param  {Object} reason - reason of the eviction
+     * @param evictor node ID of the evictor
+     * @param nodeId - node ID of the node being evicted
+     * @param reason - reason of the eviction
      *
-     * @returns {Long} ID of the message sent
+     * @returns ID of the message sent
      */
-    sendNodeEvicted(evictor: string, nodeId: any, reason: any): Long;
+    sendNodeEvicted(evictor: string, nodeId: string, reason: string): Long;
     /**
      * Publishes an event about heartbeat
      *
-     * @param  {string} address node sync address
+     * @param address node sync address
      *
-     * @returns {Long} ID of the message sent
+     * @returns ID of the message sent
      */
     sendHeartbeat(address: string): Long;
     /**
@@ -166,23 +198,31 @@ declare class ClusterPublisher {
      * Returns immediately, but the message to be sent migh be bufferized and send
      * later.
      *
-     * @param  {String} topic name
-     * @param  {Object} data
-     * @returns {Long} ID of the message sent
+     * @param topic name
+     * @param data
+     * @returns ID of the message sent
      * @throws If the topic's protobuf type cannot be found
      */
-    send(topic: string, data: any): Long;
+    send(topic: string, data: JSONObject): Long;
+    /**
+     * The messages sent with ids `from` to `to`, both included, in order — or
+     * `null` if any of them is no longer kept (or never was).
+     */
+    replay(from: Long, to: Long): BufferedMessage[] | null;
+    /**
+     * Keeps a sent message for `replay()`, then trims the oldest ones until
+     * both of `cluster.retransmitBuffer`'s bounds hold. A bound of 0 keeps
+     * nothing: retransmission is disabled.
+     */
+    private remember;
     /**
      * Sends the provided message, and then sends all pending ones waiting in
      * the buffer, in order
      *
-     * @param  {string} topic
-     * @param  {Buffer} data
-     * @return {void}
+     * @param topic
+     * @param data
      */
-    bufferSend(topic: string, data: Buffer): void;
+    bufferSend(topic: string, data: Uint8Array): Promise<void>;
     dispose(): Promise<void>;
 }
-import Long = require("long/umd/types");
-import { Publisher } from "zeromq";
-import protobuf = require("protobufjs");
+export = ClusterPublisher;