ioredis 4.28.5 → 4.29.1

package/README.md

@@ -12,7 +12,7 @@
 
 A robust, performance-focused and full-featured [Redis](http://redis.io) client for [Node.js](https://nodejs.org).
 
-Supports Redis >= 2.6.12 and (Node.js >= 6). Completely compatible with Redis 6.x.
+Supports Redis >= 2.6.12 and (Node.js >= 8). Completely compatible with Redis 7.x.
 
 # Features
 

package/built/DataHandler.js

@@ -89,6 +89,14 @@ class DataHandler {
                 this.redis.emit("pmessageBuffer", pattern, reply[2], reply[3]);
                 break;
             }
+            case "smessage": {
+                if (this.redis.listeners("smessage").length > 0) {
+                    this.redis.emit("smessage", reply[1].toString(), reply[2] ? reply[2].toString() : "");
+                }
+                this.redis.emit("smessageBuffer", reply[1], reply[2]);
+                break;
+            }
+            case "ssubscribe":
             case "subscribe":
             case "psubscribe": {
                 const channel = reply[1].toString();
@@ -102,6 +110,7 @@ class DataHandler {
                 }
                 break;
             }
+            case "sunsubscribe":
             case "unsubscribe":
             case "punsubscribe": {
                 const channel = reply[1] ? reply[1].toString() : null;

package/built/SubscriptionSet.js

@@ -11,6 +11,7 @@ class SubscriptionSet {
         this.set = {
             subscribe: {},
             psubscribe: {},
+            ssubscribe: {},
         };
     }
     add(set, channel) {
@@ -24,7 +25,8 @@ class SubscriptionSet {
     }
     isEmpty() {
         return (this.channels("subscribe").length === 0 &&
-            this.channels("psubscribe").length === 0);
+            this.channels("psubscribe").length === 0 &&
+            this.channels("ssubscribe").length === 0);
     }
 }
 exports.default = SubscriptionSet;
@@ -35,5 +37,8 @@ function mapSet(set) {
     if (set === "punsubscribe") {
         return "psubscribe";
     }
+    if (set === "sunsubscribe") {
+        return "ssubscribe";
+    }
     return set;
 }

package/built/autoPipelining.js

@@ -16,8 +16,10 @@ exports.notAllowedAutoPipelineCommands = [
     "multi",
     "subscribe",
     "psubscribe",
+    "ssubscribe",
     "unsubscribe",
-    "unpsubscribe",
+    "punsubscribe",
+    "sunsubscribe",
 ];
 function executeAutoPipeline(client, slotKey) {
     /*

package/built/cluster/ClusterOptions.js

@@ -19,4 +19,5 @@ exports.DEFAULT_CLUSTER_OPTIONS = {
     enableAutoPipelining: false,
     autoPipeliningIgnoredCommands: [],
     maxScriptsCachingTime: 60000,
+    shardedSubscribers: false,
 };

package/built/cluster/ClusterSubscriber.js

@@ -5,11 +5,13 @@ const utils_1 = require("../utils");
 const redis_1 = require("../redis");
 const debug = utils_1.Debug("cluster:subscriber");
 class ClusterSubscriber {
-    constructor(connectionPool, emitter) {
+    constructor(connectionPool, emitter, isSharded = false) {
         this.connectionPool = connectionPool;
         this.emitter = emitter;
+        this.isSharded = isSharded;
         this.started = false;
         this.subscriber = null;
+        this.slotRange = [];
         this.connectionPool.on("-node", (_, key) => {
             if (!this.started || !this.subscriber) {
                 return;
@@ -57,31 +59,38 @@ class ClusterSubscriber {
          * provided for the subscriber is correct, and if not, the current subscriber
          * will be disconnected and a new subscriber will be selected.
          */
+        let connectionPrefix = "subscriber";
+        if (this.isSharded)
+            connectionPrefix = "ssubscriber";
         this.subscriber = new redis_1.default({
             port: options.port,
             host: options.host,
             username: options.username,
             password: options.password,
             enableReadyCheck: true,
-            connectionName: util_1.getConnectionName("subscriber", options.connectionName),
+            connectionName: util_1.getConnectionName(connectionPrefix, options.connectionName),
             lazyConnect: true,
             tls: options.tls,
         });
         // Ignore the errors since they're handled in the connection pool.
         this.subscriber.on("error", utils_1.noop);
         // Re-subscribe previous channels
-        const previousChannels = { subscribe: [], psubscribe: [] };
+        const previousChannels = { subscribe: [], psubscribe: [], ssubscribe: [] };
         if (lastActiveSubscriber) {
             const condition = lastActiveSubscriber.condition || lastActiveSubscriber.prevCondition;
             if (condition && condition.subscriber) {
                 previousChannels.subscribe = condition.subscriber.channels("subscribe");
-                previousChannels.psubscribe = condition.subscriber.channels("psubscribe");
+                previousChannels.psubscribe =
+                    condition.subscriber.channels("psubscribe");
+                previousChannels.ssubscribe =
+                    condition.subscriber.channels("ssubscribe");
             }
         }
         if (previousChannels.subscribe.length ||
-            previousChannels.psubscribe.length) {
+            previousChannels.psubscribe.length ||
+            previousChannels.ssubscribe.length) {
             let pending = 0;
-            for (const type of ["subscribe", "psubscribe"]) {
+            for (const type of ["subscribe", "psubscribe", "ssubscribe"]) {
                 const channels = previousChannels[type];
                 if (channels.length) {
                     pending += 1;
@@ -112,6 +121,28 @@ class ClusterSubscriber {
                 this.emitter.emit(event, arg1, arg2, arg3);
             });
         }
+        if (this.isSharded == true) {
+            for (const event of ["smessage", "smessageBuffer"]) {
+                this.subscriber.on(event, (arg1, arg2, arg3) => {
+                    this.emitter.emit(event, arg1, arg2, arg3);
+                });
+            }
+        }
+    }
+    /**
+     * Associate this subscriber to a specific slot range.
+     *
+     * Returns the range or an empty array if the slot range couldn't be associated.
+     *
+     * BTW: This is more for debugging and testing purposes.
+     *
+     * @param range
+     */
+    associateSlotRange(range) {
+        if (this.isSharded) {
+            this.slotRange = range;
+        }
+        return this.slotRange;
     }
     start() {
         this.started = true;
@@ -126,5 +157,8 @@ class ClusterSubscriber {
         }
         debug("stopped");
     }
+    isStarted() {
+        return this.started;
+    }
 }
 exports.default = ClusterSubscriber;

package/built/cluster/ClusterSubscriberGroup.js

@@ -0,0 +1,224 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("../utils");
+const ClusterSubscriber_1 = require("./ClusterSubscriber");
+const ConnectionPool_1 = require("./ConnectionPool");
+const util_1 = require("./util");
+const calculateSlot = require("cluster-key-slot");
+const debug = utils_1.Debug("cluster:subscriberGroup");
+/**
+ * Redis differs between "normal" and sharded PubSub. If using the "normal" PubSub feature, exactly one
+ * ClusterSubscriber exists per cluster instance. This works because the Redis cluster bus forwards m
+ * messages between shards. However, this has scalability limitations, which is the reason why the sharded
+ * PubSub feature was added to Redis. With sharded PubSub, each shard is responsible for its own messages.
+ * Given that, we need at least one ClusterSubscriber per master endpoint/node.
+ *
+ * This class leverages the previously exising ClusterSubscriber by adding support for multiple such subscribers
+ * in alignment to the master nodes of the cluster. The ClusterSubscriber class was extended in a non-breaking way
+ * to support this feature.
+ */
+class ClusterSubscriberGroup {
+    /**
+     * Register callbacks
+     *
+     * @param cluster
+     */
+    constructor(cluster) {
+        this.cluster = cluster;
+        this.shardedSubscribers = new Map();
+        this.clusterSlots = [];
+        //Simple [min, max] slot ranges aren't enough because you can migrate single slots
+        this.subscriberToSlotsIndex = new Map();
+        this.channels = new Map();
+        cluster.on("+node", (redis) => {
+            this._addSubscriber(redis);
+        });
+        cluster.on("-node", (redis) => {
+            this._removeSubscriber(redis);
+        });
+        cluster.on("refresh", () => {
+            this._refreshSlots(cluster);
+        });
+    }
+    /**
+     * Get the responsible subscriber.
+     *
+     * Returns null if no subscriber was found
+     *
+     * @param slot
+     */
+    getResponsibleSubscriber(slot) {
+        const nodeKey = this.clusterSlots[slot][0];
+        return this.shardedSubscribers.get(nodeKey);
+    }
+    /**
+     * Adds a channel for which this subscriber group is responsible
+     *
+     * @param channels
+     */
+    addChannels(channels) {
+        const slot = calculateSlot(channels[0]);
+        //Check if the all channels belong to the same slot and otherwise reject the operation
+        channels.forEach((c) => {
+            if (calculateSlot(c) != slot)
+                return -1;
+        });
+        const currChannels = this.channels.get(slot);
+        if (!currChannels) {
+            this.channels.set(slot, channels);
+        }
+        else {
+            this.channels.set(slot, currChannels.concat(channels));
+        }
+        return Array.from(this.channels.values()).reduce((sum, array) => sum + array.length, 0);
+    }
+    /**
+     * Removes channels for which the subscriber group is responsible by optionally unsubscribing
+     * @param channels
+     */
+    removeChannels(channels) {
+        const slot = calculateSlot(channels[0]);
+        //Check if the all channels belong to the same slot and otherwise reject the operation
+        channels.forEach((c) => {
+            if (calculateSlot(c) != slot)
+                return -1;
+        });
+        const slotChannels = this.channels.get(slot);
+        if (slotChannels) {
+            const updatedChannels = slotChannels.filter((c) => !channels.includes(c));
+            this.channels.set(slot, updatedChannels);
+        }
+        return Array.from(this.channels.values()).reduce((sum, array) => sum + array.length, 0);
+    }
+    /**
+     * Disconnect all subscribers
+     */
+    stop() {
+        for (const s of this.shardedSubscribers.values()) {
+            s.stop();
+        }
+    }
+    /**
+     * Start all not yet started subscribers
+     */
+    start() {
+        for (const s of this.shardedSubscribers.values()) {
+            if (!s.isStarted()) {
+                s.start();
+            }
+        }
+    }
+    /**
+     * Add a subscriber to the group of subscribers
+     *
+     * @param redis
+     */
+    _addSubscriber(redis) {
+        const pool = new ConnectionPool_1.default(redis.options);
+        if (pool.addMasterNode(redis)) {
+            const sub = new ClusterSubscriber_1.default(pool, this.cluster, true);
+            const nodeKey = util_1.getNodeKey(redis.options);
+            this.shardedSubscribers.set(nodeKey, sub);
+            sub.start();
+            // We need to attempt to resubscribe them in case the new node serves their slot
+            this._resubscribe();
+            this.cluster.emit("+subscriber");
+            return sub;
+        }
+        return null;
+    }
+    /**
+     * Removes a subscriber from the group
+     * @param redis
+     */
+    _removeSubscriber(redis) {
+        const nodeKey = util_1.getNodeKey(redis.options);
+        const sub = this.shardedSubscribers.get(nodeKey);
+        if (sub) {
+            sub.stop();
+            this.shardedSubscribers.delete(nodeKey);
+            // Even though the subscriber to this node is going down, we might have another subscriber
+            // handling the same slots, so we need to attempt to subscribe the orphaned channels
+            this._resubscribe();
+            this.cluster.emit("-subscriber");
+        }
+        return this.shardedSubscribers;
+    }
+    /**
+     * Refreshes the subscriber-related slot ranges
+     *
+     * Returns false if no refresh was needed
+     *
+     * @param cluster
+     */
+    _refreshSlots(cluster) {
+        //If there was an actual change, then reassign the slot ranges
+        if (this._slotsAreEqual(cluster.slots)) {
+            debug("Nothing to refresh because the new cluster map is equal to the previous one.");
+        }
+        else {
+            debug("Refreshing the slots of the subscriber group.");
+            //Rebuild the slots index
+            this.subscriberToSlotsIndex = new Map();
+            for (let slot = 0; slot < cluster.slots.length; slot++) {
+                const node = cluster.slots[slot][0];
+                if (!this.subscriberToSlotsIndex.has(node)) {
+                    this.subscriberToSlotsIndex.set(node, []);
+                }
+                this.subscriberToSlotsIndex.get(node).push(Number(slot));
+            }
+            //Update the subscribers from the index
+            this._resubscribe();
+            //Update the cached slots map
+            this.clusterSlots = JSON.parse(JSON.stringify(cluster.slots));
+            this.cluster.emit("subscribersReady");
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Resubscribes to the previous channels
+     *
+     * @private
+     */
+    _resubscribe() {
+        if (this.shardedSubscribers) {
+            this.shardedSubscribers.forEach((s, nodeKey) => {
+                const subscriberSlots = this.subscriberToSlotsIndex.get(nodeKey);
+                if (subscriberSlots) {
+                    //More for debugging purposes
+                    s.associateSlotRange(subscriberSlots);
+                    //Resubscribe on the underlying connection
+                    subscriberSlots.forEach((ss) => {
+                        //Might return null if being disconnected
+                        const redis = s.getInstance();
+                        const channels = this.channels.get(ss);
+                        if (channels && channels.length > 0) {
+                            //Try to subscribe now
+                            if (redis) {
+                                redis.ssubscribe(channels);
+                                //If the instance isn't ready yet, then register the re-subscription for later
+                                redis.on("ready", () => {
+                                    redis.ssubscribe(channels);
+                                });
+                            }
+                        }
+                    });
+                }
+            });
+        }
+    }
+    /**
+     * Deep equality of the cluster slots objects
+     *
+     * @param other
+     * @private
+     */
+    _slotsAreEqual(other) {
+        if (this.clusterSlots === undefined)
+            return false;
+        else
+            return JSON.stringify(this.clusterSlots) === JSON.stringify(other);
+    }
+}
+exports.default = ClusterSubscriberGroup;

package/built/cluster/ConnectionPool.js

@@ -29,6 +29,39 @@ class ConnectionPool extends events_1.EventEmitter {
         const sampleKey = utils_1.sample(keys);
         return this.nodes[role][sampleKey];
     }
+    /**
+     * Add a master node to the pool
+     * @param node
+     */
+    addMasterNode(node) {
+        const key = util_1.getNodeKey(node.options);
+        const redis = this.createRedisFromOptions(node, node.options.readOnly);
+        //Master nodes aren't read-only
+        if (!node.options.readOnly) {
+            this.nodes.all[key] = redis;
+            this.nodes.master[key] = redis;
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Creates a Redis connection instance from the node options
+     * @param node
+     * @param readOnly
+     */
+    createRedisFromOptions(node, readOnly) {
+        return new redis_1.default(utils_1.defaults({
+            // Never try to reconnect when a node is lose,
+            // instead, waiting for a `MOVED` error and
+            // fetch the slots again.
+            retryStrategy: null,
+            // Offline queue should be enabled so that
+            // we don't need to wait for the `ready` event
+            // before sending commands to the node.
+            enableOfflineQueue: true,
+            readOnly: readOnly,
+        }, node, this.redisOptions, { lazyConnect: true }));
+    }
     /**
      * Find or create a connection to the node
      *
@@ -65,17 +98,7 @@ class ConnectionPool extends events_1.EventEmitter {
         }
         else {
             debug("Connecting to %s as %s", key, readOnly ? "slave" : "master");
-            redis = new redis_1.default(utils_1.defaults({
-                // Never try to reconnect when a node is lose,
-                // instead, waiting for a `MOVED` error and
-                // fetch the slots again.
-                retryStrategy: null,
-                // Offline queue should be enabled so that
-                // we don't need to wait for the `ready` event
-                // before sending commands to the node.
-                enableOfflineQueue: true,
-                readOnly: readOnly,
-            }, node, this.redisOptions, { lazyConnect: true }));
+            redis = this.createRedisFromOptions(node, readOnly);
             this.nodes.all[key] = redis;
             this.nodes[readOnly ? "slave" : "master"][key] = redis;
             redis.once("end", () => {

package/built/cluster/index.js

@@ -13,11 +13,12 @@ const standard_as_callback_1 = require("standard-as-callback");
 const PromiseContainer = require("../promiseContainer");
 const ClusterOptions_1 = require("./ClusterOptions");
 const utils_2 = require("../utils");
-const commands = require("redis-commands");
+const commands_1 = require("@ioredis/commands");
 const command_1 = require("../command");
 const redis_1 = require("../redis");
 const commander_1 = require("../commander");
 const Deque = require("denque");
+const ClusterSubscriberGroup_1 = require("./ClusterSubscriberGroup");
 const debug = utils_1.Debug("cluster");
 /**
  * Client for the official Redis Cluster
@@ -61,6 +62,8 @@ class Cluster extends events_1.EventEmitter {
         commander_1.default.call(this);
         this.startupNodes = startupNodes;
         this.options = utils_1.defaults({}, options, ClusterOptions_1.DEFAULT_CLUSTER_OPTIONS, this.options);
+        if (this.options.shardedSubscribers == true)
+            this.shardedSubscribers = new ClusterSubscriberGroup_1.default(this);
         // validate options
         if (typeof this.options.scaleReads !== "function" &&
             ["all", "master", "slave"].indexOf(this.options.scaleReads) === -1) {
@@ -202,6 +205,9 @@ class Cluster extends events_1.EventEmitter {
                     }
                 }.bind(this));
                 this.subscriber.start();
+                if (this.options.shardedSubscribers) {
+                    this.shardedSubscribers.start();
+                }
             })
                 .catch((err) => {
                 this.setStatus("close");
@@ -262,6 +268,9 @@ class Cluster extends events_1.EventEmitter {
         }
         this.clearNodesRefreshInterval();
         this.subscriber.stop();
+        if (this.options.shardedSubscribers) {
+            this.shardedSubscribers.stop();
+        }
         if (status === "wait") {
             this.setStatus("close");
             this.handleCloseEvent();
@@ -475,8 +484,7 @@ class Cluster extends events_1.EventEmitter {
         let to = this.options.scaleReads;
         if (to !== "master") {
             const isCommandReadOnly = command.isReadOnly ||
-                (commands.exists(command.name) &&
-                    commands.hasFlag(command.name, "readonly"));
+                (commands_1.exists(command.name) && commands_1.hasFlag(command.name, "readonly"));
             if (!isCommandReadOnly) {
                 to = "master";
             }
@@ -538,7 +546,24 @@ class Cluster extends events_1.EventEmitter {
                 }
                 else if (command_1.default.checkFlag("ENTER_SUBSCRIBER_MODE", command.name) ||
                     command_1.default.checkFlag("EXIT_SUBSCRIBER_MODE", command.name)) {
-                    redis = _this.subscriber.getInstance();
+                    if (_this.options.shardedSubscribers == true &&
+                        (command.name == "ssubscribe" || command.name == "sunsubscribe")) {
+                        const sub = _this.shardedSubscribers.getResponsibleSubscriber(targetSlot);
+                        let status = -1;
+                        if (command.name == "ssubscribe")
+                            status = _this.shardedSubscribers.addChannels(command.getKeys());
+                        if (command.name == "sunsubscribe")
+                            status = _this.shardedSubscribers.removeChannels(command.getKeys());
+                        if (status !== -1) {
+                            redis = sub.getInstance();
+                        }
+                        else {
+                            command.reject(new redis_errors_1.AbortError("Can't add or remove the given channels. Are they in the same slot?"));
+                        }
+                    }
+                    else {
+                        redis = _this.subscriber.getInstance();
+                    }
                     if (!redis) {
                         command.reject(new redis_errors_1.AbortError("No subscriber for the cluster"));
                         return;

package/built/command.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const commands = require("redis-commands");
+const commands_1 = require("@ioredis/commands");
 const calculateSlot = require("cluster-key-slot");
 const standard_as_callback_1 = require("standard-as-callback");
 const utils_1 = require("./utils");
@@ -131,8 +131,9 @@ class Command {
     _iterateKeys(transform = (key) => key) {
         if (typeof this.keys === "undefined") {
             this.keys = [];
-            if (commands.exists(this.name)) {
-                const keyIndexes = commands.getKeyIndexes(this.name, this.args);
+            if (commands_1.exists(this.name)) {
+                // @ts-ignore
+                const keyIndexes = commands_1.getKeyIndexes(this.name, this.args);
                 for (const index of keyIndexes) {
                     this.args[index] = transform(this.args[index]);
                     this.keys.push(this.args[index]);
@@ -271,12 +272,14 @@ Command.FLAGS = {
         "psubscribe",
         "unsubscribe",
         "punsubscribe",
+        "ssubscribe",
+        "sunsubscribe",
         "ping",
         "quit",
     ],
     VALID_IN_MONITOR_MODE: ["monitor", "auth"],
-    ENTER_SUBSCRIBER_MODE: ["subscribe", "psubscribe"],
-    EXIT_SUBSCRIBER_MODE: ["unsubscribe", "punsubscribe"],
+    ENTER_SUBSCRIBER_MODE: ["subscribe", "psubscribe", "ssubscribe"],
+    EXIT_SUBSCRIBER_MODE: ["unsubscribe", "punsubscribe", "sunsubscribe"],
     WILL_DISCONNECT: ["quit"],
 };
 Command._transformer = {

package/built/commander.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const lodash_1 = require("./utils/lodash");
+const commands_1 = require("@ioredis/commands");
 const command_1 = require("./command");
 const script_1 = require("./script");
 const PromiseContainer = require("./promiseContainer");
@@ -26,9 +27,7 @@ function Commander() {
     this.addedBuiltinSet = new Set();
 }
 exports.default = Commander;
-const commands = require("redis-commands").list.filter(function (command) {
-    return command !== "monitor";
-});
+const commands = commands_1.list.filter((command) => command !== "monitor");
 commands.push("sentinel");
 /**
  * Return supported builtin commands

package/built/pipeline.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const command_1 = require("./command");
 const util_1 = require("util");
 const standard_as_callback_1 = require("standard-as-callback");
-const redis_commands_1 = require("redis-commands");
+const commands_1 = require("@ioredis/commands");
 const calculateSlot = require("cluster-key-slot");
 const pMap = require("p-map");
 const PromiseContainer = require("./promiseContainer");
@@ -104,7 +104,7 @@ Pipeline.prototype.fillResult = function (value, position) {
                 }
             }
             else if (!command.inTransaction) {
-                const isReadOnly = redis_commands_1.exists(command.name) && redis_commands_1.hasFlag(command.name, "readonly");
+                const isReadOnly = commands_1.exists(command.name) && commands_1.hasFlag(command.name, "readonly");
                 if (!isReadOnly) {
                     retriable = false;
                     break;

package/built/redis/event_handler.js

@@ -243,6 +243,11 @@ function readyHandler(self) {
                     debug("psubscribe %d channels", psubscribeChannels.length);
                     self.psubscribe(psubscribeChannels);
                 }
+                const ssubscribeChannels = condition.subscriber.channels("ssubscribe");
+                if (ssubscribeChannels.length) {
+                    debug("ssubscribe %d channels", ssubscribeChannels.length);
+                    self.ssubscribe(ssubscribeChannels);
+                }
             }
         }
         if (self.prevCommandQueue) {

package/built/redis/index.js

@@ -11,7 +11,7 @@ const standard_as_callback_1 = require("standard-as-callback");
 const eventHandler = require("./event_handler");
 const connectors_1 = require("../connectors");
 const ScanStream_1 = require("../ScanStream");
-const commands = require("redis-commands");
+const commands_1 = require("@ioredis/commands");
 const PromiseContainer = require("../promiseContainer");
 const transaction_1 = require("../transaction");
 const RedisOptions_1 = require("./RedisOptions");
@@ -650,8 +650,8 @@ Redis.prototype.sendCommand = function (command, stream) {
     let writable = this.status === "ready" ||
         (!stream &&
             this.status === "connect" &&
-            commands.exists(command.name) &&
-            commands.hasFlag(command.name, "loading"));
+            commands_1.exists(command.name) &&
+            commands_1.hasFlag(command.name, "loading"));
     if (!this.stream) {
         writable = false;
     }