ioredis 5.5.0 → 5.6.0

package/README.md

@@ -46,7 +46,7 @@ used in the world's biggest online commerce company [Alibaba](http://www.alibaba
 | Version        | Branch | Node.js Version | Redis Version   |
 | -------------- | ------ | --------------- | --------------- |
 | 5.x.x (latest) | main   | >= 12           | 2.6.12 ~ latest |
-| 4.x.x          | v4     | >= 6            | 2.6.12 ~ 7      |
+| 4.x.x          | v4     | >= 8            | 2.6.12 ~ 7      |
 
 Refer to [CHANGELOG.md](CHANGELOG.md) for features and bug fixes introduced in v5.
 
@@ -1196,6 +1196,38 @@ sub.subscribe("news", () => {
 });
 ```
 
+### Sharded Pub/Sub
+
+For sharded Pub/Sub, use the `spublish` and `ssubscribe` commands instead of the traditional `publish` and `subscribe`. With the old commands, the Redis cluster handles message propagation behind the scenes, allowing you to publish or subscribe to any node without considering sharding. However, this approach has scalability limitations that are addressed with sharded Pub/Sub. Here’s what you need to know:
+
+1. Instead of a single subscriber connection, there is now one subscriber connection per shard. Because of the potential overhead, you can enable or disable the use of the cluster subscriber group with the `shardedSubscribers` option. By default, this option is set to `false`, meaning sharded subscriptions are disabled. You should enable this option when establishing your cluster connection before using `ssubscribe`.
+2. All channel names that you pass to a single `ssubscribe` need to map to the same hash slot. You can call `ssubscribe` multiple times on the same cluster client instance to subscribe to channels across slots. The cluster's subscriber group takes care of forwarding the `ssubscribe` command to the shard that is responsible for the channels.
+
+The following basic example shows you how to use sharded Pub/Sub:
+
+```javascript
+const cluster: Cluster = new Cluster([{host: host, port: port}], {shardedSubscribers: true});
+
+//Register the callback
+cluster.on("smessage", (channel, message) => {
+    console.log(message);
+});
+
+        
+//Subscribe to the channels on the same slot
+cluster.ssubscribe("channel{my}:1", "channel{my}:2").then( ( count: number ) => {
+    console.log(count);
+}).catch( (err) => {
+    console.log(err);
+});
+
+//Publish a message
+cluster.spublish("channel{my}:1", "This is a test message to my first channel.").then((value: number) => {
+    console.log("Published a message to channel{my}:1");
+});
+```
+
+
 ### Events
 
 | Event        | Description                                                                                                                                                                                                |

package/built/cluster/ClusterOptions.d.ts

@@ -97,6 +97,15 @@ export interface ClusterOptions extends CommanderOptions {
      * @default 5000
      */
     slotsRefreshInterval?: number;
+    /**
+     * Use sharded subscribers instead of a single subscriber.
+     *
+     * If sharded subscribers are used, then one additional subscriber connection per master node
+     * is established. If you don't plan to use SPUBLISH/SSUBSCRIBE, then this should be disabled.
+     *
+     * @default false
+     */
+    shardedSubscribers?: boolean;
     /**
      * Passed to the constructor of `Redis`
      *

package/built/cluster/ClusterOptions.js

@@ -18,4 +18,5 @@ exports.DEFAULT_CLUSTER_OPTIONS = {
     dnsLookup: dns_1.lookup,
     enableAutoPipelining: false,
     autoPipeliningIgnoredCommands: [],
+    shardedSubscribers: false,
 };

package/built/cluster/ClusterSubscriber.d.ts

@@ -4,13 +4,26 @@ import ConnectionPool from "./ConnectionPool";
 export default class ClusterSubscriber {
     private connectionPool;
     private emitter;
+    private isSharded;
     private started;
     private subscriber;
     private lastActiveSubscriber;
-    constructor(connectionPool: ConnectionPool, emitter: EventEmitter);
+    private slotRange;
+    constructor(connectionPool: ConnectionPool, emitter: EventEmitter, isSharded?: boolean);
     getInstance(): any;
+    /**
+     * Associate this subscriber to a specific slot range.
+     *
+     * Returns the range or an empty array if the slot range couldn't be associated.
+     *
+     * BTW: This is more for debugging and testing purposes.
+     *
+     * @param range
+     */
+    associateSlotRange(range: number[]): number[];
     start(): void;
     stop(): void;
+    isStarted(): boolean;
     private onSubscriberEnd;
     private selectSubscriber;
 }

package/built/cluster/ClusterSubscriber.js

@@ -5,11 +5,15 @@ const utils_1 = require("../utils");
 const Redis_1 = require("../Redis");
 const debug = (0, utils_1.Debug)("cluster:subscriber");
 class ClusterSubscriber {
-    constructor(connectionPool, emitter) {
+    constructor(connectionPool, emitter, isSharded = false) {
         this.connectionPool = connectionPool;
         this.emitter = emitter;
+        this.isSharded = isSharded;
         this.started = false;
+        //There is only one connection for the entire pool
         this.subscriber = null;
+        //The slot range for which this subscriber is responsible
+        this.slotRange = [];
         this.onSubscriberEnd = () => {
             if (!this.started) {
                 debug("subscriber has disconnected, but ClusterSubscriber is not started, so not reconnecting.");
@@ -49,6 +53,21 @@ class ClusterSubscriber {
     getInstance() {
         return this.subscriber;
     }
+    /**
+     * Associate this subscriber to a specific slot range.
+     *
+     * Returns the range or an empty array if the slot range couldn't be associated.
+     *
+     * BTW: This is more for debugging and testing purposes.
+     *
+     * @param range
+     */
+    associateSlotRange(range) {
+        if (this.isSharded) {
+            this.slotRange = range;
+        }
+        return this.slotRange;
+    }
     start() {
         this.started = true;
         this.selectSubscriber();
@@ -60,7 +79,9 @@ class ClusterSubscriber {
             this.subscriber.disconnect();
             this.subscriber = null;
         }
-        debug("stopped");
+    }
+    isStarted() {
+        return this.started;
     }
     selectSubscriber() {
         const lastActiveSubscriber = this.lastActiveSubscriber;
@@ -91,13 +112,16 @@ class ClusterSubscriber {
          * provided for the subscriber is correct, and if not, the current subscriber
          * will be disconnected and a new subscriber will be selected.
          */
+        let connectionPrefix = "subscriber";
+        if (this.isSharded)
+            connectionPrefix = "ssubscriber";
         this.subscriber = new Redis_1.default({
             port: options.port,
             host: options.host,
             username: options.username,
             password: options.password,
             enableReadyCheck: true,
-            connectionName: (0, util_1.getConnectionName)("subscriber", options.connectionName),
+            connectionName: (0, util_1.getConnectionName)(connectionPrefix, options.connectionName),
             lazyConnect: true,
             tls: options.tls,
             // Don't try to reconnect the subscriber connection. If the connection fails
@@ -153,8 +177,6 @@ class ClusterSubscriber {
         for (const event of [
             "message",
             "messageBuffer",
-            "smessage",
-            "smessageBuffer",
         ]) {
             this.subscriber.on(event, (arg1, arg2) => {
                 this.emitter.emit(event, arg1, arg2);
@@ -165,6 +187,16 @@ class ClusterSubscriber {
                 this.emitter.emit(event, arg1, arg2, arg3);
             });
         }
+        if (this.isSharded == true) {
+            for (const event of [
+                "smessage",
+                "smessageBuffer",
+            ]) {
+                this.subscriber.on(event, (arg1, arg2) => {
+                    this.emitter.emit(event, arg1, arg2);
+                });
+            }
+        }
     }
 }
 exports.default = ClusterSubscriber;

package/built/cluster/ClusterSubscriberGroup.d.ts

@@ -0,0 +1,86 @@
+/// <reference types="node" />
+import ClusterSubscriber from "./ClusterSubscriber";
+import Cluster from "./index";
+/**
+ * Redis differs between "normal" and sharded PubSub. If using the "normal" PubSub feature, exactly one
+ * ClusterSubscriber exists per cluster instance. This works because the Redis cluster bus forwards m
+ * messages between shards. However, this has scalability limitations, which is the reason why the sharded
+ * PubSub feature was added to Redis. With sharded PubSub, each shard is responsible for its own messages.
+ * Given that, we need at least one ClusterSubscriber per master endpoint/node.
+ *
+ * This class leverages the previously exising ClusterSubscriber by adding support for multiple such subscribers
+ * in alignment to the master nodes of the cluster. The ClusterSubscriber class was extended in a non-breaking way
+ * to support this feature.
+ */
+export default class ClusterSubscriberGroup {
+    private cluster;
+    private shardedSubscribers;
+    private clusterSlots;
+    private subscriberToSlotsIndex;
+    private channels;
+    /**
+     * Register callbacks
+     *
+     * @param cluster
+     */
+    constructor(cluster: Cluster);
+    /**
+     * Get the responsible subscriber.
+     *
+     * Returns null if no subscriber was found
+     *
+     * @param slot
+     */
+    getResponsibleSubscriber(slot: number): ClusterSubscriber;
+    /**
+     * Adds a channel for which this subscriber group is responsible
+     *
+     * @param channels
+     */
+    addChannels(channels: (string | Buffer)[]): number;
+    /**
+     * Removes channels for which the subscriber group is responsible by optionally unsubscribing
+     * @param channels
+     */
+    removeChannels(channels: (string | Buffer)[]): number;
+    /**
+     * Disconnect all subscribers
+     */
+    stop(): void;
+    /**
+     * Start all not yet started subscribers
+     */
+    start(): void;
+    /**
+     * Add a subscriber to the group of subscribers
+     *
+     * @param redis
+     */
+    private _addSubscriber;
+    /**
+     * Removes a subscriber from the group
+     * @param redis
+     */
+    private _removeSubscriber;
+    /**
+     * Refreshes the subscriber-related slot ranges
+     *
+     * Returns false if no refresh was needed
+     *
+     * @param cluster
+     */
+    private _refreshSlots;
+    /**
+     * Resubscribes to the previous channels
+     *
+     * @private
+     */
+    private _resubscribe;
+    /**
+     * Deep equality of the cluster slots objects
+     *
+     * @param other
+     * @private
+     */
+    private _slotsAreEqual;
+}

package/built/cluster/ClusterSubscriberGroup.js

@@ -0,0 +1,224 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("../utils");
+const ClusterSubscriber_1 = require("./ClusterSubscriber");
+const ConnectionPool_1 = require("./ConnectionPool");
+const util_1 = require("./util");
+const calculateSlot = require("cluster-key-slot");
+const debug = (0, utils_1.Debug)("cluster:subscriberGroup");
+/**
+ * Redis differs between "normal" and sharded PubSub. If using the "normal" PubSub feature, exactly one
+ * ClusterSubscriber exists per cluster instance. This works because the Redis cluster bus forwards m
+ * messages between shards. However, this has scalability limitations, which is the reason why the sharded
+ * PubSub feature was added to Redis. With sharded PubSub, each shard is responsible for its own messages.
+ * Given that, we need at least one ClusterSubscriber per master endpoint/node.
+ *
+ * This class leverages the previously exising ClusterSubscriber by adding support for multiple such subscribers
+ * in alignment to the master nodes of the cluster. The ClusterSubscriber class was extended in a non-breaking way
+ * to support this feature.
+ */
+class ClusterSubscriberGroup {
+    /**
+     * Register callbacks
+     *
+     * @param cluster
+     */
+    constructor(cluster) {
+        this.cluster = cluster;
+        this.shardedSubscribers = new Map();
+        this.clusterSlots = [];
+        //Simple [min, max] slot ranges aren't enough because you can migrate single slots
+        this.subscriberToSlotsIndex = new Map();
+        this.channels = new Map();
+        cluster.on("+node", (redis) => {
+            this._addSubscriber(redis);
+        });
+        cluster.on("-node", (redis) => {
+            this._removeSubscriber(redis);
+        });
+        cluster.on("refresh", () => {
+            this._refreshSlots(cluster);
+        });
+    }
+    /**
+     * Get the responsible subscriber.
+     *
+     * Returns null if no subscriber was found
+     *
+     * @param slot
+     */
+    getResponsibleSubscriber(slot) {
+        const nodeKey = this.clusterSlots[slot][0];
+        return this.shardedSubscribers.get(nodeKey);
+    }
+    /**
+     * Adds a channel for which this subscriber group is responsible
+     *
+     * @param channels
+     */
+    addChannels(channels) {
+        const slot = calculateSlot(channels[0]);
+        //Check if the all channels belong to the same slot and otherwise reject the operation
+        channels.forEach((c) => {
+            if (calculateSlot(c) != slot)
+                return -1;
+        });
+        const currChannels = this.channels.get(slot);
+        if (!currChannels) {
+            this.channels.set(slot, channels);
+        }
+        else {
+            this.channels.set(slot, currChannels.concat(channels));
+        }
+        return [...this.channels.values()].flatMap(v => v).length;
+    }
+    /**
+     * Removes channels for which the subscriber group is responsible by optionally unsubscribing
+     * @param channels
+     */
+    removeChannels(channels) {
+        const slot = calculateSlot(channels[0]);
+        //Check if the all channels belong to the same slot and otherwise reject the operation
+        channels.forEach((c) => {
+            if (calculateSlot(c) != slot)
+                return -1;
+        });
+        const slotChannels = this.channels.get(slot);
+        if (slotChannels) {
+            const updatedChannels = slotChannels.filter(c => !channels.includes(c));
+            this.channels.set(slot, updatedChannels);
+        }
+        return [...this.channels.values()].flatMap(v => v).length;
+    }
+    /**
+     * Disconnect all subscribers
+     */
+    stop() {
+        for (const s of this.shardedSubscribers.values()) {
+            s.stop();
+        }
+    }
+    /**
+     * Start all not yet started subscribers
+     */
+    start() {
+        for (const s of this.shardedSubscribers.values()) {
+            if (!s.isStarted()) {
+                s.start();
+            }
+        }
+    }
+    /**
+     * Add a subscriber to the group of subscribers
+     *
+     * @param redis
+     */
+    _addSubscriber(redis) {
+        const pool = new ConnectionPool_1.default(redis.options);
+        if (pool.addMasterNode(redis)) {
+            const sub = new ClusterSubscriber_1.default(pool, this.cluster, true);
+            const nodeKey = (0, util_1.getNodeKey)(redis.options);
+            this.shardedSubscribers.set(nodeKey, sub);
+            sub.start();
+            // We need to attempt to resubscribe them in case the new node serves their slot
+            this._resubscribe();
+            this.cluster.emit("+subscriber");
+            return sub;
+        }
+        return null;
+    }
+    /**
+     * Removes a subscriber from the group
+     * @param redis
+     */
+    _removeSubscriber(redis) {
+        const nodeKey = (0, util_1.getNodeKey)(redis.options);
+        const sub = this.shardedSubscribers.get(nodeKey);
+        if (sub) {
+            sub.stop();
+            this.shardedSubscribers.delete(nodeKey);
+            // Even though the subscriber to this node is going down, we might have another subscriber
+            // handling the same slots, so we need to attempt to subscribe the orphaned channels
+            this._resubscribe();
+            this.cluster.emit("-subscriber");
+        }
+        return this.shardedSubscribers;
+    }
+    /**
+     * Refreshes the subscriber-related slot ranges
+     *
+     * Returns false if no refresh was needed
+     *
+     * @param cluster
+     */
+    _refreshSlots(cluster) {
+        //If there was an actual change, then reassign the slot ranges
+        if (this._slotsAreEqual(cluster.slots)) {
+            debug("Nothing to refresh because the new cluster map is equal to the previous one.");
+        }
+        else {
+            debug("Refreshing the slots of the subscriber group.");
+            //Rebuild the slots index
+            this.subscriberToSlotsIndex = new Map();
+            for (let slot = 0; slot < cluster.slots.length; slot++) {
+                const node = cluster.slots[slot][0];
+                if (!this.subscriberToSlotsIndex.has(node)) {
+                    this.subscriberToSlotsIndex.set(node, []);
+                }
+                this.subscriberToSlotsIndex.get(node).push(Number(slot));
+            }
+            //Update the subscribers from the index
+            this._resubscribe();
+            //Update the cached slots map
+            this.clusterSlots = JSON.parse(JSON.stringify(cluster.slots));
+            this.cluster.emit("subscribersReady");
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Resubscribes to the previous channels
+     *
+     * @private
+     */
+    _resubscribe() {
+        if (this.shardedSubscribers) {
+            this.shardedSubscribers.forEach((s, nodeKey) => {
+                const subscriberSlots = this.subscriberToSlotsIndex.get(nodeKey);
+                if (subscriberSlots) {
+                    //More for debugging purposes
+                    s.associateSlotRange(subscriberSlots);
+                    //Resubscribe on the underlying connection
+                    subscriberSlots.forEach((ss) => {
+                        //Might return null if being disconnected
+                        const redis = s.getInstance();
+                        const channels = this.channels.get(ss);
+                        if (channels && channels.length > 0) {
+                            //Try to subscribe now
+                            if (redis) {
+                                redis.ssubscribe(channels);
+                                //If the instance isn't ready yet, then register the re-subscription for later
+                                redis.on("ready", () => {
+                                    redis.ssubscribe(channels);
+                                });
+                            }
+                        }
+                    });
+                }
+            });
+        }
+    }
+    /**
+     * Deep equality of the cluster slots objects
+     *
+     * @param other
+     * @private
+     */
+    _slotsAreEqual(other) {
+        if (this.clusterSlots === undefined)
+            return false;
+        else
+            return JSON.stringify(this.clusterSlots) === JSON.stringify(other);
+    }
+}
+exports.default = ClusterSubscriberGroup;

package/built/cluster/ConnectionPool.d.ts

@@ -10,6 +10,17 @@ export default class ConnectionPool extends EventEmitter {
     getNodes(role?: NodeRole): Redis[];
     getInstanceByKey(key: NodeKey): Redis;
     getSampleInstance(role: NodeRole): Redis;
+    /**
+     * Add a master node to the pool
+     * @param node
+     */
+    addMasterNode(node: RedisOptions): boolean;
+    /**
+     * Creates a Redis connection instance from the node options
+     * @param node
+     * @param readOnly
+     */
+    createRedisFromOptions(node: RedisOptions, readOnly: boolean): Redis;
     /**
      * Find or create a connection to the node
      */

package/built/cluster/ConnectionPool.js

@@ -29,6 +29,40 @@ class ConnectionPool extends events_1.EventEmitter {
         const sampleKey = (0, utils_1.sample)(keys);
         return this.nodes[role][sampleKey];
     }
+    /**
+     * Add a master node to the pool
+     * @param node
+     */
+    addMasterNode(node) {
+        const key = (0, util_1.getNodeKey)(node.options);
+        const redis = this.createRedisFromOptions(node, node.options.readOnly);
+        //Master nodes aren't read-only
+        if (!node.options.readOnly) {
+            this.nodes.all[key] = redis;
+            this.nodes.master[key] = redis;
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Creates a Redis connection instance from the node options
+     * @param node
+     * @param readOnly
+     */
+    createRedisFromOptions(node, readOnly) {
+        const redis = new Redis_1.default((0, utils_1.defaults)({
+            // Never try to reconnect when a node is lose,
+            // instead, waiting for a `MOVED` error and
+            // fetch the slots again.
+            retryStrategy: null,
+            // Offline queue should be enabled so that
+            // we don't need to wait for the `ready` event
+            // before sending commands to the node.
+            enableOfflineQueue: true,
+            readOnly: readOnly,
+        }, node, this.redisOptions, { lazyConnect: true }));
+        return redis;
+    }
     /**
      * Find or create a connection to the node
      */
@@ -60,17 +94,7 @@ class ConnectionPool extends events_1.EventEmitter {
         }
         else {
             debug("Connecting to %s as %s", key, readOnly ? "slave" : "master");
-            redis = new Redis_1.default((0, utils_1.defaults)({
-                // Never try to reconnect when a node is lose,
-                // instead, waiting for a `MOVED` error and
-                // fetch the slots again.
-                retryStrategy: null,
-                // Offline queue should be enabled so that
-                // we don't need to wait for the `ready` event
-                // before sending commands to the node.
-                enableOfflineQueue: true,
-                readOnly: readOnly,
-            }, node, this.redisOptions, { lazyConnect: true }));
+            redis = this.createRedisFromOptions(node, readOnly);
             this.nodes.all[key] = redis;
             this.nodes[readOnly ? "slave" : "master"][key] = redis;
             redis.once("end", () => {

package/built/cluster/index.d.ts

@@ -41,6 +41,7 @@ declare class Cluster extends Commander {
     private delayQueue;
     private offlineQueue;
     private subscriber;
+    private shardedSubscribers;
     private slotsTimer;
     private reconnectTimeout;
     private isRefreshing;

package/built/cluster/index.js

@@ -18,6 +18,7 @@ const ConnectionPool_1 = require("./ConnectionPool");
 const DelayQueue_1 = require("./DelayQueue");
 const util_1 = require("./util");
 const Deque = require("denque");
+const ClusterSubscriberGroup_1 = require("./ClusterSubscriberGroup");
 const debug = (0, utils_1.Debug)("cluster");
 const REJECT_OVERWRITTEN_COMMANDS = new WeakSet();
 /**
@@ -27,6 +28,7 @@ class Cluster extends Commander_1.default {
     /**
      * Creates an instance of Cluster.
      */
+    //TODO: Add an option that enables or disables sharded PubSub
     constructor(startupNodes, options = {}) {
         super();
         this.slots = [];
@@ -60,6 +62,8 @@ class Cluster extends Commander_1.default {
         events_1.EventEmitter.call(this);
         this.startupNodes = startupNodes;
         this.options = (0, utils_1.defaults)({}, options, ClusterOptions_1.DEFAULT_CLUSTER_OPTIONS, this.options);
+        if (this.options.shardedSubscribers == true)
+            this.shardedSubscribers = new ClusterSubscriberGroup_1.default(this);
         if (this.options.redisOptions &&
             this.options.redisOptions.keyPrefix &&
             !this.options.keyPrefix) {
@@ -172,6 +176,9 @@ class Cluster extends Commander_1.default {
                     }
                 });
                 this.subscriber.start();
+                if (this.options.shardedSubscribers) {
+                    this.shardedSubscribers.start();
+                }
             })
                 .catch((err) => {
                 this.setStatus("close");
@@ -197,6 +204,9 @@ class Cluster extends Commander_1.default {
         }
         this.clearNodesRefreshInterval();
         this.subscriber.stop();
+        if (this.options.shardedSubscribers) {
+            this.shardedSubscribers.stop();
+        }
         if (status === "wait") {
             this.setStatus("close");
             this.handleCloseEvent();
@@ -218,6 +228,9 @@ class Cluster extends Commander_1.default {
         }
         this.clearNodesRefreshInterval();
         this.subscriber.stop();
+        if (this.options.shardedSubscribers) {
+            this.shardedSubscribers.stop();
+        }
         if (status === "wait") {
             const ret = (0, standard_as_callback_1.default)(Promise.resolve("OK"), callback);
             // use setImmediate to make sure "close" event
@@ -409,7 +422,24 @@ class Cluster extends Commander_1.default {
                 }
                 else if (Command_1.default.checkFlag("ENTER_SUBSCRIBER_MODE", command.name) ||
                     Command_1.default.checkFlag("EXIT_SUBSCRIBER_MODE", command.name)) {
-                    redis = _this.subscriber.getInstance();
+                    if (_this.options.shardedSubscribers == true &&
+                        (command.name == "ssubscribe" || command.name == "sunsubscribe")) {
+                        const sub = _this.shardedSubscribers.getResponsibleSubscriber(targetSlot);
+                        let status = -1;
+                        if (command.name == "ssubscribe")
+                            status = _this.shardedSubscribers.addChannels(command.getKeys());
+                        if (command.name == "sunsubscribe")
+                            status = _this.shardedSubscribers.removeChannels(command.getKeys());
+                        if (status !== -1) {
+                            redis = sub.getInstance();
+                        }
+                        else {
+                            command.reject(new redis_errors_1.AbortError("Can't add or remove the given channels. Are they in the same slot?"));
+                        }
+                    }
+                    else {
+                        redis = _this.subscriber.getInstance();
+                    }
                     if (!redis) {
                         command.reject(new redis_errors_1.AbortError("No subscriber for the cluster"));
                         return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ioredis",
-  "version": "5.5.0",
+  "version": "5.6.0",
   "description": "A robust, performance-focused and full-featured Redis client for Node.js.",
   "main": "./built/index.js",
   "types": "./built/index.d.ts",