@socket.io/redis-streams-adapter 0.1.0

package/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2023 The Socket.IO team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,80 @@
+# Socket.IO Redis Streams adapter
+
+The `@socket.io/redis-streams-adapter` package allows broadcasting packets between multiple Socket.IO servers.
+
+Supported features:
+
+- [broadcasting](https://socket.io/docs/v4/broadcasting-events/)
+- [utility methods](https://socket.io/docs/v4/server-instance/#Utility-methods)
+  - [`socketsJoin`](https://socket.io/docs/v4/server-instance/#socketsJoin)
+  - [`socketsLeave`](https://socket.io/docs/v4/server-instance/#socketsLeave)
+  - [`disconnectSockets`](https://socket.io/docs/v4/server-instance/#disconnectSockets)
+  - [`fetchSockets`](https://socket.io/docs/v4/server-instance/#fetchSockets)
+  - [`serverSideEmit`](https://socket.io/docs/v4/server-instance/#serverSideEmit)
+- [connection state recovery](https://socket.io/docs/v4/connection-state-recovery)
+
+Related packages:
+
+- Redis adapter: https://github.com/socketio/socket.io-redis-adapter/
+- Redis emitter: https://github.com/socketio/socket.io-redis-emitter/
+- MongoDB adapter: https://github.com/socketio/socket.io-mongo-adapter/
+- MongoDB emitter: https://github.com/socketio/socket.io-mongo-emitter/
+- Postgres adapter: https://github.com/socketio/socket.io-postgres-adapter/
+- Postgres emitter: https://github.com/socketio/socket.io-postgres-emitter/
+
+**Table of contents**
+
+- [Installation](#installation)
+- [Usage](#usage)
+- [Options](#options)
+- [How it works](#how-it-works)
+- [License](#license)
+
+## Installation
+
+```
+npm install @socket.io/redis-streams-adapter redis
+```
+
+## Usage
+
+```js
+import { createClient } from "redis";
+import { Server } from "socket.io";
+import { createAdapter } from "@socket.io/redis-streams-adapter";
+
+const redisClient = createClient({ host: "localhost", port: 6379 });
+
+await redisClient.connect();
+
+const io = new Server({
+  adapter: createAdapter(redisClient)
+});
+
+io.listen(3000);
+```
+
+## Options
+
+| Name                | Description                                                        | Default value |
+|---------------------|--------------------------------------------------------------------|---------------|
+| `streamName`        | The name of the Redis stream.                                      | `socket.io`   |
+| `maxLen`            | The maximum size of the stream. Almost exact trimming (~) is used. | `10_000`      |
+| `readCount`         | The number of elements to fetch per XREAD call.                    | `100`         |
+| `heartbeatInterval` | The number of ms between two heartbeats.                           | `5_000`       |
+| `heartbeatTimeout`  | The number of ms without heartbeat before we consider a node down. | `10_000`      |
+
+## How it works
+
+The adapter will use a [Redis stream](https://redis.io/docs/data-types/streams/) to forward events between the Socket.IO servers.
+
+Notes:
+
+- a single stream is used for all namespaces
+- the `maxLen` option allows to limit the size of the stream
+- unlike the adapter based on Redis PUB/SUB mechanism, this adapter will properly handle any temporary disconnection to the Redis server and resume the stream
+- if [connection state recovery](https://socket.io/docs/v4/connection-state-recovery) is enabled, the sessions will be stored in Redis as a classic key/value pair
+
+## License
+
+[MIT](LICENSE)

package/dist/adapter.d.ts

@@ -0,0 +1,50 @@
+import type { PrivateSessionId, Session } from "socket.io-adapter";
+import { ClusterAdapter, ClusterAdapterOptions, ClusterMessage } from "./cluster-adapter";
+export interface RedisStreamsAdapterOptions extends ClusterAdapterOptions {
+    /**
+     * The name of the Redis stream.
+     */
+    streamName?: string;
+    /**
+     * The maximum size of the stream. Almost exact trimming (~) is used.
+     * @default 10_000
+     */
+    maxLen?: number;
+    /**
+     * The number of elements to fetch per XREAD call.
+     * @default 100
+     */
+    readCount?: number;
+}
+interface RawClusterMessage {
+    uid: string;
+    nsp: string;
+    type: string;
+    data?: string;
+}
+/**
+ * Returns a function that will create a new adapter instance.
+ *
+ * @param redisClient - a Redis client that will be used to publish messages
+ * @param opts - additional options
+ */
+export declare function createAdapter(redisClient: any, opts?: RedisStreamsAdapterOptions): (nsp: any) => RedisStreamsAdapter;
+declare class RedisStreamsAdapter extends ClusterAdapter {
+    #private;
+    constructor(nsp: any, redisClient: any, opts: Required<RedisStreamsAdapterOptions>);
+    doPublish(message: ClusterMessage): any;
+    static encode(message: ClusterMessage): RawClusterMessage;
+    onRawMessage(rawMessage: RawClusterMessage, offset: string): any;
+    static decode(rawMessage: RawClusterMessage): ClusterMessage;
+    persistSession(session: any): void;
+    restoreSession(pid: PrivateSessionId, offset: string): Promise<Session>;
+    /**
+     * Exclusive ranges were added in Redis 6.2, so this is necessary for previous versions.
+     *
+     * @see https://redis.io/commands/xrange/
+     *
+     * @param offset
+     */
+    static nextOffset(offset: any): string;
+}
+export {};

package/dist/adapter.js

@@ -0,0 +1,212 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createAdapter = void 0;
+const msgpack_1 = require("@msgpack/msgpack");
+const redis_1 = require("redis");
+const cluster_adapter_1 = require("./cluster-adapter");
+const debug_1 = require("debug");
+const util_1 = require("./util");
+const debug = (0, debug_1.default)("socket.io-redis-streams-adapter");
+const RESTORE_SESSION_MAX_XRANGE_CALLS = 100;
+/**
+ * Returns a function that will create a new adapter instance.
+ *
+ * @param redisClient - a Redis client that will be used to publish messages
+ * @param opts - additional options
+ */
+function createAdapter(redisClient, opts) {
+    const namespaceToAdapters = new Map();
+    const options = Object.assign({
+        streamName: "socket.io",
+        maxLen: 10000,
+        readCount: 100,
+        heartbeatInterval: 5000,
+        heartbeatTimeout: 10000,
+    }, opts);
+    let offset = "$";
+    let polling = false;
+    async function poll() {
+        try {
+            let response = await redisClient.xRead((0, redis_1.commandOptions)({
+                isolated: true,
+            }), [
+                {
+                    key: options.streamName,
+                    id: offset,
+                },
+            ], {
+                COUNT: options.readCount,
+                BLOCK: 5000,
+            });
+            if (response) {
+                for (const entry of response[0].messages) {
+                    debug("reading entry %s", entry.id);
+                    const message = entry.message;
+                    if (message.nsp) {
+                        namespaceToAdapters
+                            .get(message.nsp)
+                            ?.onRawMessage(message, entry.id);
+                    }
+                    offset = entry.id;
+                }
+            }
+        }
+        catch (e) {
+            debug("something went wrong while consuming the stream: %s", e.message);
+        }
+        if (namespaceToAdapters.size > 0 && redisClient.isOpen) {
+            poll();
+        }
+        else {
+            polling = false;
+        }
+    }
+    return function (nsp) {
+        const adapter = new RedisStreamsAdapter(nsp, redisClient, options);
+        namespaceToAdapters.set(nsp.name, adapter);
+        if (!polling) {
+            polling = true;
+            poll();
+        }
+        const defaultClose = adapter.close;
+        adapter.close = () => {
+            namespaceToAdapters.delete(nsp.name);
+            defaultClose.call(adapter);
+        };
+        return adapter;
+    };
+}
+exports.createAdapter = createAdapter;
+class RedisStreamsAdapter extends cluster_adapter_1.ClusterAdapter {
+    #redisClient;
+    #opts;
+    constructor(nsp, redisClient, opts) {
+        super(nsp, opts);
+        this.#redisClient = redisClient;
+        this.#opts = opts;
+        this.initHeartbeat();
+    }
+    doPublish(message) {
+        debug("publishing %o", message);
+        return this.#redisClient.xAdd(this.#opts.streamName, "*", RedisStreamsAdapter.encode(message), {
+            TRIM: {
+                strategy: "MAXLEN",
+                strategyModifier: "~",
+                threshold: this.#opts.maxLen,
+            },
+        });
+    }
+    static encode(message) {
+        const rawMessage = {
+            uid: message.uid,
+            nsp: message.nsp,
+            type: message.type.toString(),
+        };
+        if (message.data) {
+            const mayContainBinary = [
+                cluster_adapter_1.MessageType.BROADCAST,
+                cluster_adapter_1.MessageType.BROADCAST_ACK,
+                cluster_adapter_1.MessageType.FETCH_SOCKETS_RESPONSE,
+                cluster_adapter_1.MessageType.SERVER_SIDE_EMIT,
+                cluster_adapter_1.MessageType.SERVER_SIDE_EMIT_RESPONSE,
+            ].includes(message.type);
+            if (mayContainBinary && (0, util_1.hasBinary)(message.data)) {
+                rawMessage.data = Buffer.from((0, msgpack_1.encode)(message.data)).toString("base64");
+            }
+            else {
+                rawMessage.data = JSON.stringify(message.data);
+            }
+        }
+        return rawMessage;
+    }
+    onRawMessage(rawMessage, offset) {
+        let message;
+        try {
+            message = RedisStreamsAdapter.decode(rawMessage);
+        }
+        catch (e) {
+            return debug("invalid format: %s", e.message);
+        }
+        this.onMessage(message, offset);
+    }
+    static decode(rawMessage) {
+        const message = {
+            uid: rawMessage.uid,
+            nsp: rawMessage.nsp,
+            type: parseInt(rawMessage.type, 10),
+        };
+        if (rawMessage.data) {
+            if (rawMessage.data.startsWith("{")) {
+                message.data = JSON.parse(rawMessage.data);
+            }
+            else {
+                message.data = (0, msgpack_1.decode)(Buffer.from(rawMessage.data, "base64"));
+            }
+        }
+        return message;
+    }
+    persistSession(session) {
+        debug("persisting session %o", session);
+        const encodedSession = Buffer.from((0, msgpack_1.encode)(session)).toString("base64");
+        this.#redisClient.set(`sio:session:${session.pid}`, encodedSession, {
+            PX: this.nsp.server.opts.connectionStateRecovery.maxDisconnectionDuration,
+        });
+    }
+    async restoreSession(pid, offset) {
+        debug("restoring session %s from offset %s", pid, offset);
+        if (!/^[0-9]+-[0-9]+$/.test(offset)) {
+            return Promise.reject("invalid offset");
+        }
+        const sessionKey = `sio:session:${pid}`;
+        const [rawSession, offsetExists] = await this.#redisClient
+            .multi()
+            .get(sessionKey)
+            .del(sessionKey) // GETDEL was added in Redis version 6.2
+            .xRange(this.#opts.streamName, offset, offset)
+            .exec();
+        if (!rawSession || !offsetExists) {
+            return Promise.reject("session or offset not found");
+        }
+        const session = (0, msgpack_1.decode)(Buffer.from(rawSession, "base64"));
+        debug("found session %o", session);
+        session.missedPackets = [];
+        // FIXME we need to add an arbitrary limit here, because if entries are added faster than what we can consume, then
+        // we will loop endlessly. But if we stop before reaching the end of the stream, we might lose messages.
+        for (let i = 0; i < RESTORE_SESSION_MAX_XRANGE_CALLS; i++) {
+            const entries = await this.#redisClient.xRange(this.#opts.streamName, RedisStreamsAdapter.nextOffset(offset), "+", {
+                COUNT: this.#opts.readCount,
+            });
+            if (entries.length === 0) {
+                break;
+            }
+            for (const entry of entries) {
+                if (entry.message.nsp === this.nsp.name && entry.message.type === "3") {
+                    const message = RedisStreamsAdapter.decode(entry.message);
+                    if (shouldIncludePacket(session.rooms, message.data.opts)) {
+                        // @ts-ignore
+                        session.missedPackets.push(message.data.packet.data);
+                    }
+                }
+                offset = entry.id;
+            }
+        }
+        return session;
+    }
+    /**
+     * Exclusive ranges were added in Redis 6.2, so this is necessary for previous versions.
+     *
+     * @see https://redis.io/commands/xrange/
+     *
+     * @param offset
+     */
+    static nextOffset(offset) {
+        const [timestamp, sequence] = offset.split("-");
+        return timestamp + "-" + (parseInt(sequence) + 1);
+    }
+}
+function shouldIncludePacket(sessionRooms, opts) {
+    const included = opts.rooms.length === 0 ||
+        sessionRooms.some((room) => opts.rooms.indexOf(room) !== -1);
+    const notExcluded = sessionRooms.every((room) => opts.except.indexOf(room) === -1);
+    return included && notExcluded;
+}

package/dist/cluster-adapter.d.ts

@@ -0,0 +1,54 @@
+import { Adapter, BroadcastOptions, Room } from "socket.io-adapter";
+export interface ClusterAdapterOptions {
+    /**
+     * The number of ms between two heartbeats.
+     * @default 5_000
+     */
+    heartbeatInterval?: number;
+    /**
+     * The number of ms without heartbeat before we consider a node down.
+     * @default 10_000
+     */
+    heartbeatTimeout?: number;
+}
+export declare enum MessageType {
+    INITIAL_HEARTBEAT = 1,
+    HEARTBEAT = 2,
+    BROADCAST = 3,
+    SOCKETS_JOIN = 4,
+    SOCKETS_LEAVE = 5,
+    DISCONNECT_SOCKETS = 6,
+    FETCH_SOCKETS = 7,
+    FETCH_SOCKETS_RESPONSE = 8,
+    SERVER_SIDE_EMIT = 9,
+    SERVER_SIDE_EMIT_RESPONSE = 10,
+    BROADCAST_CLIENT_COUNT = 11,
+    BROADCAST_ACK = 12
+}
+export interface ClusterMessage {
+    uid: string;
+    nsp: string;
+    type: MessageType;
+    data?: Record<string, unknown>;
+}
+export declare abstract class ClusterAdapter extends Adapter {
+    #private;
+    protected constructor(nsp: any, opts: Required<ClusterAdapterOptions>);
+    protected initHeartbeat(): void;
+    close(): Promise<void> | void;
+    onMessage(message: ClusterMessage, offset: string): Promise<any>;
+    broadcast(packet: any, opts: BroadcastOptions): Promise<any>;
+    broadcastWithAck(packet: any, opts: BroadcastOptions, clientCountCallback: (clientCount: number) => void, ack: (...args: any[]) => void): void;
+    serverCount(): Promise<number>;
+    /**
+     *
+     * @param opts
+     * @param rooms
+     */
+    addSockets(opts: BroadcastOptions, rooms: Room[]): void;
+    delSockets(opts: BroadcastOptions, rooms: Room[]): void;
+    disconnectSockets(opts: BroadcastOptions, close: boolean): void;
+    fetchSockets(opts: BroadcastOptions): Promise<any[]>;
+    serverSideEmit(packet: any[]): any;
+    abstract doPublish(message: ClusterMessage): Promise<string>;
+}

package/dist/cluster-adapter.js

@@ -0,0 +1,420 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClusterAdapter = exports.MessageType = void 0;
+const socket_io_adapter_1 = require("socket.io-adapter");
+const debug_1 = require("debug");
+const util_1 = require("./util");
+const debug = (0, debug_1.default)("socket.io-adapter");
+const EMITTER_UID = "emitter";
+const DEFAULT_TIMEOUT = 5000;
+var MessageType;
+(function (MessageType) {
+    MessageType[MessageType["INITIAL_HEARTBEAT"] = 1] = "INITIAL_HEARTBEAT";
+    MessageType[MessageType["HEARTBEAT"] = 2] = "HEARTBEAT";
+    MessageType[MessageType["BROADCAST"] = 3] = "BROADCAST";
+    MessageType[MessageType["SOCKETS_JOIN"] = 4] = "SOCKETS_JOIN";
+    MessageType[MessageType["SOCKETS_LEAVE"] = 5] = "SOCKETS_LEAVE";
+    MessageType[MessageType["DISCONNECT_SOCKETS"] = 6] = "DISCONNECT_SOCKETS";
+    MessageType[MessageType["FETCH_SOCKETS"] = 7] = "FETCH_SOCKETS";
+    MessageType[MessageType["FETCH_SOCKETS_RESPONSE"] = 8] = "FETCH_SOCKETS_RESPONSE";
+    MessageType[MessageType["SERVER_SIDE_EMIT"] = 9] = "SERVER_SIDE_EMIT";
+    MessageType[MessageType["SERVER_SIDE_EMIT_RESPONSE"] = 10] = "SERVER_SIDE_EMIT_RESPONSE";
+    MessageType[MessageType["BROADCAST_CLIENT_COUNT"] = 11] = "BROADCAST_CLIENT_COUNT";
+    MessageType[MessageType["BROADCAST_ACK"] = 12] = "BROADCAST_ACK";
+})(MessageType = exports.MessageType || (exports.MessageType = {}));
+function encodeOptions(opts) {
+    return {
+        rooms: [...opts.rooms],
+        except: [...opts.except],
+        flags: opts.flags,
+    };
+}
+function decodeOptions(opts) {
+    return {
+        rooms: new Set(opts.rooms),
+        except: new Set(opts.except),
+        flags: opts.flags,
+    };
+}
+class ClusterAdapter extends socket_io_adapter_1.Adapter {
+    #opts;
+    #uid;
+    #heartbeatTimer;
+    #nodesMap = new Map(); // uid => timestamp of last message
+    #requests = new Map();
+    #ackRequests = new Map();
+    constructor(nsp, opts) {
+        super(nsp);
+        this.#opts = opts;
+        this.#uid = (0, util_1.randomId)();
+    }
+    initHeartbeat() {
+        this.#publish({
+            type: MessageType.INITIAL_HEARTBEAT,
+        });
+    }
+    #scheduleHeartbeat() {
+        if (this.#heartbeatTimer) {
+            clearTimeout(this.#heartbeatTimer);
+        }
+        this.#heartbeatTimer = setTimeout(() => {
+            this.#publish({
+                type: MessageType.HEARTBEAT,
+            });
+        }, this.#opts.heartbeatInterval);
+    }
+    close() {
+        clearTimeout(this.#heartbeatTimer);
+    }
+    async onMessage(message, offset) {
+        if (message.uid === this.#uid) {
+            return debug("ignore message from self");
+        }
+        if (message.uid && message.uid !== EMITTER_UID) {
+            this.#nodesMap.set(message.uid, Date.now());
+        }
+        debug("new event of type %d from %s", message.type, message.uid);
+        switch (message.type) {
+            case MessageType.INITIAL_HEARTBEAT:
+                this.#publish({
+                    type: MessageType.HEARTBEAT,
+                });
+                break;
+            case MessageType.BROADCAST: {
+                const withAck = message.data.requestId !== undefined;
+                if (withAck) {
+                    super.broadcastWithAck(message.data.packet, decodeOptions(message.data.opts), (clientCount) => {
+                        debug("waiting for %d client acknowledgements", clientCount);
+                        this.#publish({
+                            type: MessageType.BROADCAST_CLIENT_COUNT,
+                            data: {
+                                requestId: message.data.requestId,
+                                clientCount,
+                            },
+                        });
+                    }, (arg) => {
+                        debug("received acknowledgement with value %j", arg);
+                        this.#publish({
+                            type: MessageType.BROADCAST_ACK,
+                            data: {
+                                requestId: message.data.requestId,
+                                packet: arg,
+                            },
+                        });
+                    });
+                }
+                else {
+                    const packet = message.data.packet;
+                    const opts = decodeOptions(message.data.opts);
+                    this.#addOffsetIfNecessary(packet, opts, offset);
+                    super.broadcast(packet, opts);
+                }
+                break;
+            }
+            case MessageType.BROADCAST_CLIENT_COUNT: {
+                const request = this.#ackRequests.get(message.data.requestId);
+                request?.clientCountCallback(message.data.clientCount);
+                break;
+            }
+            case MessageType.BROADCAST_ACK: {
+                const request = this.#ackRequests.get(message.data.requestId);
+                request?.ack(message.data.packet);
+                break;
+            }
+            case MessageType.SOCKETS_JOIN:
+                super.addSockets(decodeOptions(message.data.opts), message.data.rooms);
+                break;
+            case MessageType.SOCKETS_LEAVE:
+                super.delSockets(decodeOptions(message.data.opts), message.data.rooms);
+                break;
+            case MessageType.DISCONNECT_SOCKETS:
+                super.disconnectSockets(decodeOptions(message.data.opts), message.data.close);
+                break;
+            case MessageType.FETCH_SOCKETS: {
+                debug("calling fetchSockets with opts %j", message.data.opts);
+                const localSockets = await super.fetchSockets(decodeOptions(message.data.opts));
+                this.#publish({
+                    type: MessageType.FETCH_SOCKETS_RESPONSE,
+                    data: {
+                        requestId: message.data.requestId,
+                        sockets: localSockets.map((socket) => {
+                            // remove sessionStore from handshake, as it may contain circular references
+                            const { sessionStore, ...handshake } = socket.handshake;
+                            return {
+                                id: socket.id,
+                                handshake,
+                                rooms: [...socket.rooms],
+                                data: socket.data,
+                            };
+                        }),
+                    },
+                });
+                break;
+            }
+            case MessageType.FETCH_SOCKETS_RESPONSE: {
+                const requestId = message.data.requestId;
+                const request = this.#requests.get(requestId);
+                if (!request) {
+                    return;
+                }
+                request.current++;
+                message.data.sockets.forEach((socket) => request.responses.push(socket));
+                if (request.current === request.expected) {
+                    clearTimeout(request.timeout);
+                    request.resolve(request.responses);
+                    this.#requests.delete(requestId);
+                }
+                break;
+            }
+            case MessageType.SERVER_SIDE_EMIT: {
+                const packet = message.data.packet;
+                const withAck = message.data.requestId !== undefined;
+                if (!withAck) {
+                    this.nsp._onServerSideEmit(packet);
+                    return;
+                }
+                let called = false;
+                const callback = (arg) => {
+                    // only one argument is expected
+                    if (called) {
+                        return;
+                    }
+                    called = true;
+                    debug("calling acknowledgement with %j", arg);
+                    this.#publish({
+                        type: MessageType.SERVER_SIDE_EMIT_RESPONSE,
+                        data: {
+                            requestId: message.data.requestId,
+                            packet: arg,
+                        },
+                    });
+                };
+                packet.push(callback);
+                this.nsp._onServerSideEmit(packet);
+                break;
+            }
+            case MessageType.SERVER_SIDE_EMIT_RESPONSE: {
+                const requestId = message.data.requestId;
+                const request = this.#requests.get(requestId);
+                if (!request) {
+                    return;
+                }
+                request.current++;
+                request.responses.push(message.data.packet);
+                if (request.current === request.expected) {
+                    clearTimeout(request.timeout);
+                    request.resolve(null, request.responses);
+                    this.#requests.delete(requestId);
+                }
+            }
+        }
+    }
+    async broadcast(packet, opts) {
+        const onlyLocal = opts.flags?.local;
+        if (!onlyLocal) {
+            try {
+                const offset = await this.#publish({
+                    type: MessageType.BROADCAST,
+                    data: {
+                        packet,
+                        opts: encodeOptions(opts),
+                    },
+                });
+                this.#addOffsetIfNecessary(packet, opts, offset);
+            }
+            catch (e) {
+                return debug("error while broadcasting message: %s", e.message);
+            }
+        }
+        super.broadcast(packet, opts);
+    }
+    /**
+     * Adds an offset at the end of the data array in order to allow the client to receive any missed packets when it
+     * reconnects after a temporary disconnection.
+     *
+     * @param packet
+     * @param opts
+     * @param offset
+     * @private
+     */
+    #addOffsetIfNecessary(packet, opts, offset) {
+        if (!this.nsp.server.opts.connectionStateRecovery) {
+            return;
+        }
+        const isEventPacket = packet.type === 2;
+        // packets with acknowledgement are not stored because the acknowledgement function cannot be serialized and
+        // restored on another server upon reconnection
+        const withoutAcknowledgement = packet.id === undefined;
+        const notVolatile = opts.flags?.volatile === undefined;
+        if (isEventPacket && withoutAcknowledgement && notVolatile) {
+            packet.data.push(offset);
+        }
+    }
+    broadcastWithAck(packet, opts, clientCountCallback, ack) {
+        const onlyLocal = opts?.flags?.local;
+        if (!onlyLocal) {
+            const requestId = (0, util_1.randomId)();
+            this.#publish({
+                type: MessageType.BROADCAST,
+                data: {
+                    packet,
+                    requestId,
+                    opts: encodeOptions(opts),
+                },
+            });
+            this.#ackRequests.set(requestId, {
+                clientCountCallback,
+                ack,
+            });
+            // we have no way to know at this level whether the server has received an acknowledgement from each client, so we
+            // will simply clean up the ackRequests map after the given delay
+            setTimeout(() => {
+                this.#ackRequests.delete(requestId);
+            }, opts.flags.timeout);
+        }
+        super.broadcastWithAck(packet, opts, clientCountCallback, ack);
+    }
+    serverCount() {
+        return Promise.resolve(1 + this.#nodesMap.size);
+    }
+    /**
+     *
+     * @param opts
+     * @param rooms
+     */
+    addSockets(opts, rooms) {
+        super.addSockets(opts, rooms);
+        const onlyLocal = opts.flags?.local;
+        if (onlyLocal) {
+            return;
+        }
+        this.#publish({
+            type: MessageType.SOCKETS_JOIN,
+            data: {
+                opts: encodeOptions(opts),
+                rooms,
+            },
+        });
+    }
+    delSockets(opts, rooms) {
+        super.delSockets(opts, rooms);
+        const onlyLocal = opts.flags?.local;
+        if (onlyLocal) {
+            return;
+        }
+        this.#publish({
+            type: MessageType.SOCKETS_LEAVE,
+            data: {
+                opts: encodeOptions(opts),
+                rooms,
+            },
+        });
+    }
+    disconnectSockets(opts, close) {
+        super.disconnectSockets(opts, close);
+        const onlyLocal = opts.flags?.local;
+        if (onlyLocal) {
+            return;
+        }
+        this.#publish({
+            type: MessageType.DISCONNECT_SOCKETS,
+            data: {
+                opts: encodeOptions(opts),
+                close,
+            },
+        });
+    }
+    #getExpectedResponseCount() {
+        this.#nodesMap.forEach((lastSeen, uid) => {
+            const nodeSeemsDown = Date.now() - lastSeen > this.#opts.heartbeatTimeout;
+            if (nodeSeemsDown) {
+                debug("node %s seems down", uid);
+                this.#nodesMap.delete(uid);
+            }
+        });
+        return this.#nodesMap.size;
+    }
+    async fetchSockets(opts) {
+        const localSockets = await super.fetchSockets(opts);
+        const expectedResponseCount = this.#getExpectedResponseCount();
+        if (opts.flags?.local || expectedResponseCount === 0) {
+            return localSockets;
+        }
+        const requestId = (0, util_1.randomId)();
+        return new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                const storedRequest = this.#requests.get(requestId);
+                if (storedRequest) {
+                    reject(new Error(`timeout reached: only ${storedRequest.current} responses received out of ${storedRequest.expected}`));
+                    this.#requests.delete(requestId);
+                }
+            }, opts.flags.timeout || DEFAULT_TIMEOUT);
+            const storedRequest = {
+                type: MessageType.FETCH_SOCKETS,
+                resolve,
+                timeout,
+                current: 0,
+                expected: expectedResponseCount,
+                responses: localSockets,
+            };
+            this.#requests.set(requestId, storedRequest);
+            this.#publish({
+                type: MessageType.FETCH_SOCKETS,
+                data: {
+                    opts: encodeOptions(opts),
+                    requestId,
+                },
+            });
+        });
+    }
+    serverSideEmit(packet) {
+        const withAck = typeof packet[packet.length - 1] === "function";
+        if (!withAck) {
+            return this.#publish({
+                type: MessageType.SERVER_SIDE_EMIT,
+                data: {
+                    packet,
+                },
+            });
+        }
+        const ack = packet.pop();
+        const expectedResponseCount = this.#getExpectedResponseCount();
+        debug('waiting for %d responses to "serverSideEmit" request', expectedResponseCount);
+        if (expectedResponseCount <= 0) {
+            return ack(null, []);
+        }
+        const requestId = (0, util_1.randomId)();
+        const timeout = setTimeout(() => {
+            const storedRequest = this.#requests.get(requestId);
+            if (storedRequest) {
+                ack(new Error(`timeout reached: only ${storedRequest.current} responses received out of ${storedRequest.expected}`), storedRequest.responses);
+                this.#requests.delete(requestId);
+            }
+        }, DEFAULT_TIMEOUT);
+        const storedRequest = {
+            type: MessageType.SERVER_SIDE_EMIT,
+            resolve: ack,
+            timeout,
+            current: 0,
+            expected: expectedResponseCount,
+            responses: [],
+        };
+        this.#requests.set(requestId, storedRequest);
+        this.#publish({
+            type: MessageType.SERVER_SIDE_EMIT,
+            data: {
+                requestId,
+                packet,
+            },
+        });
+    }
+    #publish(message) {
+        this.#scheduleHeartbeat();
+        return this.doPublish({
+            uid: this.#uid,
+            nsp: this.nsp.name,
+            ...message,
+        });
+    }
+}
+exports.ClusterAdapter = ClusterAdapter;

package/dist/index.d.ts

@@ -0,0 +1 @@
+export { createAdapter, RedisStreamsAdapterOptions } from "./adapter";

package/dist/index.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createAdapter = void 0;
+var adapter_1 = require("./adapter");
+Object.defineProperty(exports, "createAdapter", { enumerable: true, get: function () { return adapter_1.createAdapter; } });

package/dist/redis-streams-adapter.d.ts

@@ -0,0 +1,47 @@
+import type { PrivateSessionId, Session } from "socket.io-adapter";
+import { ClusterAdapter, ClusterAdapterOptions, ClusterMessage } from "./cluster-adapter";
+export interface RedisAdapterOptions extends ClusterAdapterOptions {
+    /**
+     * The name of the stream.
+     */
+    streamName?: string;
+    /**
+     * The maximum size of the stream. Almost exact trimming (~) is used.
+     * @default 10_000
+     */
+    maxLen?: number;
+    /**
+     * The number of elements to return per XREAD call
+     * @default 100
+     */
+    readCount?: number;
+}
+interface RawClusterMessage {
+    uid: string;
+    nsp: string;
+    type: string;
+    data?: string;
+}
+/**
+ * Returns a function that will create a RedisAdapter instance.
+ *
+ * @param redisClient - a Redis client that will be used to publish messages
+ * @param opts - additional options
+ */
+export declare function createAdapter(redisClient: any, opts?: RedisAdapterOptions): (nsp: any) => RedisStreamsAdapter;
+declare class RedisStreamsAdapter extends ClusterAdapter {
+    #private;
+    constructor(nsp: any, redisClient: any, opts: Required<RedisAdapterOptions>);
+    doPublish(message: ClusterMessage): any;
+    static encode(message: ClusterMessage): RawClusterMessage;
+    onRawMessage(rawMessage: RawClusterMessage, offset: string): any;
+    static decode(rawMessage: RawClusterMessage): ClusterMessage;
+    persistSession(session: any): void;
+    restoreSession(pid: PrivateSessionId, offset: string): Promise<Session>;
+    /**
+     * Exclusive ranges were added in Redis 6.2, so this is necessary for previous versions.
+     * @param offset
+     */
+    static nextOffset(offset: any): string;
+}
+export {};

package/dist/redis-streams-adapter.js

@@ -0,0 +1,212 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createAdapter = void 0;
+const msgpack_1 = require("@msgpack/msgpack");
+const redis_1 = require("redis");
+const cluster_adapter_1 = require("./cluster-adapter");
+const debug_1 = require("debug");
+const util_1 = require("./util");
+// TODO
+// BROADCAST / other channels
+// MAXLEN
+const debug = (0, debug_1.default)("socket.io-redis-streams-adapter");
+/**
+ * Returns a function that will create a RedisAdapter instance.
+ *
+ * @param redisClient - a Redis client that will be used to publish messages
+ * @param opts - additional options
+ */
+function createAdapter(redisClient, opts) {
+    const namespaceToAdapters = new Map();
+    const options = Object.assign({
+        streamName: "socket.io",
+        maxLen: 1000,
+        readCount: 100,
+        heartbeatInterval: 5000,
+        heartbeatTimeout: 10000,
+    }, opts);
+    let offset = "$";
+    let polling = false;
+    async function poll() {
+        try {
+            let response = await redisClient.xRead((0, redis_1.commandOptions)({
+                isolated: true,
+            }), [
+                {
+                    key: options.streamName,
+                    id: offset,
+                },
+            ], {
+                COUNT: options.readCount,
+                BLOCK: 5000,
+            });
+            if (response) {
+                for (const entry of response[0].messages) {
+                    debug("reading entry %s", entry.id);
+                    const message = entry.message;
+                    if (message.nsp) {
+                        namespaceToAdapters
+                            .get(message.nsp)
+                            ?.onRawMessage(message, entry.id);
+                    }
+                    offset = entry.id;
+                }
+            }
+        }
+        catch (e) {
+            debug("something went wrong while consuming the stream: %s", e.message);
+        }
+        if (namespaceToAdapters.size > 0 && redisClient.isOpen) {
+            poll();
+        }
+        else {
+            polling = false;
+        }
+    }
+    return function (nsp) {
+        const adapter = new RedisStreamsAdapter(nsp, redisClient, options);
+        namespaceToAdapters.set(nsp.name, adapter);
+        if (!polling) {
+            polling = true;
+            poll();
+        }
+        const defaultClose = adapter.close;
+        adapter.close = () => {
+            namespaceToAdapters.delete(nsp.name);
+            defaultClose.call(adapter);
+        };
+        return adapter;
+    };
+}
+exports.createAdapter = createAdapter;
+class RedisStreamsAdapter extends cluster_adapter_1.ClusterAdapter {
+    #redisClient;
+    #opts;
+    constructor(nsp, redisClient, opts) {
+        super(nsp, opts);
+        this.#redisClient = redisClient;
+        this.#opts = opts;
+        this.initHeartbeat();
+    }
+    doPublish(message) {
+        debug("publishing %o", message);
+        return this.#redisClient.xAdd(this.#opts.streamName, "*", RedisStreamsAdapter.encode(message), {
+            TRIM: {
+                strategy: "MAXLEN",
+                strategyModifier: "~",
+                threshold: this.#opts.maxLen,
+            },
+        });
+    }
+    static encode(message) {
+        const rawMessage = {
+            uid: message.uid,
+            nsp: message.nsp,
+            type: message.type.toString(),
+        };
+        if (message.data) {
+            const mayContainBinary = [
+                cluster_adapter_1.MessageType.BROADCAST,
+                cluster_adapter_1.MessageType.BROADCAST_ACK,
+                cluster_adapter_1.MessageType.FETCH_SOCKETS_RESPONSE,
+                cluster_adapter_1.MessageType.SERVER_SIDE_EMIT,
+                cluster_adapter_1.MessageType.SERVER_SIDE_EMIT_RESPONSE,
+            ].includes(message.type);
+            if (mayContainBinary && (0, util_1.hasBinary)(message.data)) {
+                rawMessage.data = Buffer.from((0, msgpack_1.encode)(message.data)).toString("base64");
+            }
+            else {
+                rawMessage.data = JSON.stringify(message.data);
+            }
+        }
+        return rawMessage;
+    }
+    onRawMessage(rawMessage, offset) {
+        let message;
+        try {
+            message = RedisStreamsAdapter.decode(rawMessage);
+        }
+        catch (e) {
+            return debug("invalid format: %s", e.message);
+        }
+        this.onMessage(message, offset);
+    }
+    static decode(rawMessage) {
+        const message = {
+            uid: rawMessage.uid,
+            nsp: rawMessage.nsp,
+            type: parseInt(rawMessage.type, 10),
+        };
+        if (rawMessage.data) {
+            if (rawMessage.data.startsWith("{")) {
+                message.data = JSON.parse(rawMessage.data);
+            }
+            else {
+                message.data = (0, msgpack_1.decode)(Buffer.from(rawMessage.data, "base64"));
+            }
+        }
+        return message;
+    }
+    persistSession(session) {
+        debug("persist session %o", session);
+        this.#redisClient.set(session.pid, JSON.stringify(session), {
+            PX: this.nsp.server.opts.connectionStateRecovery.maxDisconnectionDuration,
+        });
+    }
+    async restoreSession(pid, offset) {
+        console.log("restoreSession", pid);
+        console.log("offset", offset);
+        if (!/^[0-9]+-[0-9]+$/.test(offset)) {
+            console.log("invalid offset");
+            return Promise.reject("invalid offset");
+        }
+        const value = await this.#redisClient.get(pid);
+        if (!value) {
+            console.log("session or offset not found");
+            return Promise.reject("session or offset not found");
+        }
+        const session = JSON.parse(value);
+        console.log("session", session);
+        session.missedPackets = [];
+        // FIXME si les messages sont ajoutés plus vite que consommés, problème !
+        for (let i = 0; i < 10; i++) {
+            const entries = await this.#redisClient.xRange(this.#opts.streamName, RedisStreamsAdapter.nextOffset(offset), "+", {
+                COUNT: 100,
+            });
+            console.log("entries", entries.length);
+            console.log("offset", offset);
+            if (entries.length === 0) {
+                break;
+            }
+            for (const entry of entries) {
+                if (entry.message.nsp === this.nsp.name && entry.message.type === "3") {
+                    const message = RedisStreamsAdapter.decode(entry.message);
+                    console.log("message from the log", message);
+                    if (message.type === cluster_adapter_1.MessageType.BROADCAST &&
+                        shouldIncludePacket(session.rooms, message.data.opts)) {
+                        // TODO match rooms
+                        // @ts-ignore
+                        session.missedPackets.push(message.data.packet.data);
+                    }
+                }
+                offset = entry.id;
+            }
+        }
+        console.log("restored", session);
+        return session;
+    }
+    /**
+     * Exclusive ranges were added in Redis 6.2, so this is necessary for previous versions.
+     * @param offset
+     */
+    static nextOffset(offset) {
+        const [timestamp, sequence] = offset.split("-");
+        return timestamp + "-" + (parseInt(sequence) + 1);
+    }
+}
+function shouldIncludePacket(sessionRooms, opts) {
+    const included = opts.rooms.length === 0 ||
+        sessionRooms.some((room) => opts.rooms.indexOf(room) !== -1);
+    const notExcluded = sessionRooms.every((room) => opts.except.indexOf(room) === -1);
+    return included && notExcluded;
+}

package/dist/test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const socket_io_1 = require("socket.io");
+const redis_1 = require("redis");
+const index_1 = require("./index");
+const io = new socket_io_1.Server();
+const pubClient = (0, redis_1.createClient)({ host: 'localhost', port: 6379 });
+const subClient = pubClient.duplicate();
+Promise.all([pubClient.connect(), subClient.connect()]).then(() => {
+    io.adapter((0, index_1.createAdapter)(pubClient, subClient));
+    io.listen(3000);
+});

package/dist/util.d.ts

@@ -0,0 +1,2 @@
+export declare function hasBinary(obj: any, toJSON?: boolean): boolean;
+export declare function randomId(): string;

package/dist/util.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.randomId = exports.hasBinary = void 0;
+const crypto_1 = require("crypto");
+function hasBinary(obj, toJSON) {
+    if (!obj || typeof obj !== "object") {
+        return false;
+    }
+    if (obj instanceof ArrayBuffer || ArrayBuffer.isView(obj)) {
+        return true;
+    }
+    if (Array.isArray(obj)) {
+        for (let i = 0, l = obj.length; i < l; i++) {
+            if (hasBinary(obj[i])) {
+                return true;
+            }
+        }
+        return false;
+    }
+    for (const key in obj) {
+        if (Object.prototype.hasOwnProperty.call(obj, key) && hasBinary(obj[key])) {
+            return true;
+        }
+    }
+    if (obj.toJSON && typeof obj.toJSON === "function" && !toJSON) {
+        return hasBinary(obj.toJSON(), true);
+    }
+    return false;
+}
+exports.hasBinary = hasBinary;
+function randomId() {
+    return (0, crypto_1.randomBytes)(8).toString("hex");
+}
+exports.randomId = randomId;

package/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@socket.io/redis-streams-adapter",
+  "version": "0.1.0",
+  "description": "The Socket.IO adapter based on Redis Streams, allowing to broadcast events between several Socket.IO servers",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git@github.com:socketio/socket.io-redis-streams-adapter.git"
+  },
+  "files": [
+    "dist/"
+  ],
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "compile": "tsc",
+    "format:check": "prettier --parser typescript --check lib/**/*.ts test/**/*.ts",
+    "format:fix": "prettier --parser typescript --write lib/**/*.ts test/**/*.ts",
+    "prepack": "npm run compile",
+    "test": "npm run format:check && npm run compile && nyc mocha --bail --require ts-node/register test/**/*.ts"
+  },
+  "dependencies": {
+    "@msgpack/msgpack": "~2.8.0",
+    "debug": "~4.3.1"
+  },
+  "peerDependencies": {
+    "socket.io-adapter": "^2.5.2"
+  },
+  "devDependencies": {
+    "@types/expect.js": "^0.3.29",
+    "@types/mocha": "^8.2.1",
+    "@types/node": "^18.15.11",
+    "expect.js": "0.3.1",
+    "mocha": "^10.1.0",
+    "nyc": "^15.1.0",
+    "prettier": "^2.8.7",
+    "redis": "^4.6.5",
+    "socket.io": "^4.6.1",
+    "socket.io-client": "^4.6.1",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.9.5"
+  },
+  "engines": {
+    "node": ">=14.0.0"
+  },
+  "keywords": [
+    "socket.io",
+    "redis",
+    "adapter"
+  ]
+}