ioredis-om 5.10.2

package/built/cluster/ClusterSubscriberGroup.js

@@ -0,0 +1,373 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("../utils");
+const util_1 = require("./util");
+const calculateSlot = require("cluster-key-slot");
+const ShardedSubscriber_1 = require("./ShardedSubscriber");
+const debug = (0, utils_1.Debug)("cluster:subscriberGroup");
+/**
+ * Redis distinguishes between "normal" and sharded PubSub. When using the normal PubSub feature,
+ * exactly one subscriber exists per cluster instance because the Redis cluster bus forwards
+ * messages between shards. Sharded PubSub removes this limitation by making each shard
+ * responsible for its own messages.
+ *
+ * This class coordinates one ShardedSubscriber per master node in the cluster, providing
+ * sharded PubSub support while keeping the public API backward compatible.
+ */
+class ClusterSubscriberGroup {
+    /**
+     * Register callbacks
+     *
+     * @param cluster
+     */
+    constructor(subscriberGroupEmitter, options) {
+        this.subscriberGroupEmitter = subscriberGroupEmitter;
+        this.options = options;
+        this.shardedSubscribers = new Map();
+        this.clusterSlots = [];
+        // Simple [min, max] slot ranges aren't enough because you can migrate single slots
+        this.subscriberToSlotsIndex = new Map();
+        this.channels = new Map();
+        this.failedAttemptsByNode = new Map();
+        // Only latest pending reset kept; throttled by refreshSlotsCache's isRefreshing + backoff delay
+        this.isResetting = false;
+        this.pendingReset = null;
+        /**
+         * Handles failed subscriber connections by emitting an event to refresh the slots cache
+         * after a backoff period.
+         *
+         * @param error
+         * @param nodeKey
+         */
+        this.handleSubscriberConnectFailed = (error, nodeKey) => {
+            const currentAttempts = this.failedAttemptsByNode.get(nodeKey) || 0;
+            const failedAttempts = currentAttempts + 1;
+            this.failedAttemptsByNode.set(nodeKey, failedAttempts);
+            const attempts = Math.min(failedAttempts, ClusterSubscriberGroup.MAX_RETRY_ATTEMPTS);
+            const backoff = Math.min(ClusterSubscriberGroup.BASE_BACKOFF_MS * 2 ** attempts, ClusterSubscriberGroup.MAX_BACKOFF_MS);
+            const jitter = Math.floor((Math.random() - 0.5) * (backoff * 0.5));
+            const delay = Math.max(0, backoff + jitter);
+            debug("Failed to connect subscriber for %s. Refreshing slots in %dms", nodeKey, delay);
+            this.subscriberGroupEmitter.emit("subscriberConnectFailed", {
+                delay,
+                error,
+            });
+        };
+        /**
+         * Handles successful subscriber connections by resetting the failed attempts counter.
+         *
+         * @param nodeKey
+         */
+        this.handleSubscriberConnectSucceeded = (nodeKey) => {
+            this.failedAttemptsByNode.delete(nodeKey);
+        };
+    }
+    /**
+     * Get the responsible subscriber.
+     *
+     * @param slot
+     */
+    getResponsibleSubscriber(slot) {
+        const nodeKey = this.clusterSlots[slot][0];
+        const sub = this.shardedSubscribers.get(nodeKey);
+        if (sub && sub.subscriberStatus === "idle") {
+            sub
+                .start()
+                .then(() => {
+                this.handleSubscriberConnectSucceeded(sub.getNodeKey());
+            })
+                .catch((err) => {
+                this.handleSubscriberConnectFailed(err, sub.getNodeKey());
+            });
+        }
+        return sub;
+    }
+    /**
+     * Adds a channel for which this subscriber group is responsible
+     *
+     * @param channels
+     */
+    addChannels(channels) {
+        const slot = calculateSlot(channels[0]);
+        // Check if the all channels belong to the same slot and otherwise reject the operation
+        for (const c of channels) {
+            if (calculateSlot(c) !== slot) {
+                return -1;
+            }
+        }
+        const currChannels = this.channels.get(slot);
+        if (!currChannels) {
+            this.channels.set(slot, channels);
+        }
+        else {
+            this.channels.set(slot, currChannels.concat(channels));
+        }
+        return Array.from(this.channels.values()).reduce((sum, array) => sum + array.length, 0);
+    }
+    /**
+     * Removes channels for which the subscriber group is responsible by optionally unsubscribing
+     * @param channels
+     */
+    removeChannels(channels) {
+        const slot = calculateSlot(channels[0]);
+        // Check if the all channels belong to the same slot and otherwise reject the operation
+        for (const c of channels) {
+            if (calculateSlot(c) !== slot) {
+                return -1;
+            }
+        }
+        const slotChannels = this.channels.get(slot);
+        if (slotChannels) {
+            const updatedChannels = slotChannels.filter((c) => !channels.includes(c));
+            this.channels.set(slot, updatedChannels);
+        }
+        return Array.from(this.channels.values()).reduce((sum, array) => sum + array.length, 0);
+    }
+    /**
+     * Disconnect all subscribers and clear some of the internal state.
+     */
+    stop() {
+        for (const s of this.shardedSubscribers.values()) {
+            s.stop();
+        }
+        // Clear subscriber instances and pending operations.
+        // Channels are preserved for resubscription on reconnect.
+        this.pendingReset = null;
+        this.shardedSubscribers.clear();
+        this.subscriberToSlotsIndex.clear();
+    }
+    /**
+     * Start all not yet started subscribers
+     */
+    start() {
+        const startPromises = [];
+        for (const s of this.shardedSubscribers.values()) {
+            if (this.shouldStartSubscriber(s)) {
+                startPromises.push(s
+                    .start()
+                    .then(() => {
+                    this.handleSubscriberConnectSucceeded(s.getNodeKey());
+                })
+                    .catch((err) => {
+                    this.handleSubscriberConnectFailed(err, s.getNodeKey());
+                }));
+                this.subscriberGroupEmitter.emit("+subscriber");
+            }
+        }
+        return Promise.all(startPromises);
+    }
+    /**
+     * Resets the subscriber group by disconnecting all subscribers that are no longer needed and connecting new ones.
+     */
+    async reset(clusterSlots, clusterNodes) {
+        if (this.isResetting) {
+            this.pendingReset = { slots: clusterSlots, nodes: clusterNodes };
+            return;
+        }
+        this.isResetting = true;
+        try {
+            const hasTopologyChanged = this._refreshSlots(clusterSlots);
+            const hasFailedSubscribers = this.hasUnhealthySubscribers();
+            if (!hasTopologyChanged && !hasFailedSubscribers) {
+                debug("No topology change detected or failed subscribers. Skipping reset.");
+                return;
+            }
+            // For each of the sharded subscribers
+            for (const [nodeKey, shardedSubscriber] of this.shardedSubscribers) {
+                if (
+                // If the subscriber is still responsible for a slot range and is healthy then keep it
+                this.subscriberToSlotsIndex.has(nodeKey) &&
+                    shardedSubscriber.isHealthy()) {
+                    debug("Skipping deleting subscriber for %s", nodeKey);
+                    continue;
+                }
+                debug("Removing subscriber for %s", nodeKey);
+                // Otherwise stop the subscriber and remove it
+                shardedSubscriber.stop();
+                this.shardedSubscribers.delete(nodeKey);
+                this.subscriberGroupEmitter.emit("-subscriber");
+            }
+            const startPromises = [];
+            // For each node in slots cache
+            for (const [nodeKey, _] of this.subscriberToSlotsIndex) {
+                const existingSubscriber = this.shardedSubscribers.get(nodeKey);
+                // If we already have a subscriber for this node, only ensure it is healthy
+                // when it now owns slots with active channel subscriptions.
+                if (existingSubscriber && existingSubscriber.isHealthy()) {
+                    debug("Skipping creating new subscriber for %s", nodeKey);
+                    if (!existingSubscriber.isStarted() &&
+                        this.shouldStartSubscriber(existingSubscriber)) {
+                        startPromises.push(existingSubscriber
+                            .start()
+                            .then(() => {
+                            this.handleSubscriberConnectSucceeded(nodeKey);
+                        })
+                            .catch((error) => {
+                            this.handleSubscriberConnectFailed(error, nodeKey);
+                        }));
+                    }
+                    continue;
+                }
+                // If we have an existing subscriber but it is not healthy, stop it
+                if (existingSubscriber && !existingSubscriber.isHealthy()) {
+                    debug("Replacing subscriber for %s", nodeKey);
+                    existingSubscriber.stop();
+                    this.shardedSubscribers.delete(nodeKey);
+                    this.subscriberGroupEmitter.emit("-subscriber");
+                }
+                debug("Creating new subscriber for %s", nodeKey);
+                // Otherwise create a new subscriber
+                const redis = clusterNodes.find((node) => {
+                    return (0, util_1.getNodeKey)(node.options) === nodeKey;
+                });
+                if (!redis) {
+                    debug("Failed to find node for key %s", nodeKey);
+                    continue;
+                }
+                const sub = new ShardedSubscriber_1.default(this.subscriberGroupEmitter, redis.options, this.options.redisOptions);
+                this.shardedSubscribers.set(nodeKey, sub);
+                if (this.shouldStartSubscriber(sub)) {
+                    startPromises.push(sub
+                        .start()
+                        .then(() => {
+                        this.handleSubscriberConnectSucceeded(nodeKey);
+                    })
+                        .catch((error) => {
+                        this.handleSubscriberConnectFailed(error, nodeKey);
+                    }));
+                }
+                this.subscriberGroupEmitter.emit("+subscriber");
+            }
+            // It's vital to await the start promises before resubscribing
+            // Otherwise we might try to resubscribe to a subscriber that is not yet connected
+            // This can cause a race condition
+            await Promise.all(startPromises);
+            this._resubscribe();
+            this.subscriberGroupEmitter.emit("subscribersReady");
+        }
+        finally {
+            this.isResetting = false;
+            if (this.pendingReset) {
+                const { slots, nodes } = this.pendingReset;
+                this.pendingReset = null;
+                await this.reset(slots, nodes);
+            }
+        }
+    }
+    /**
+     * Refreshes the subscriber-related slot ranges
+     *
+     * Returns false if no refresh was needed
+     *
+     * @param targetSlots
+     */
+    _refreshSlots(targetSlots) {
+        //If there was an actual change, then reassign the slot ranges
+        // Also rebuild if subscriberToSlotsIndex is empty (e.g., after stop() was called)
+        if (this._slotsAreEqual(targetSlots) && this.subscriberToSlotsIndex.size > 0) {
+            debug("Nothing to refresh because the new cluster map is equal to the previous one.");
+            return false;
+        }
+        debug("Refreshing the slots of the subscriber group.");
+        //Rebuild the slots index
+        this.subscriberToSlotsIndex = new Map();
+        for (let slot = 0; slot < targetSlots.length; slot++) {
+            const node = targetSlots[slot][0];
+            if (!this.subscriberToSlotsIndex.has(node)) {
+                this.subscriberToSlotsIndex.set(node, []);
+            }
+            this.subscriberToSlotsIndex.get(node).push(Number(slot));
+        }
+        //Update the cached slots map
+        this.clusterSlots = JSON.parse(JSON.stringify(targetSlots));
+        return true;
+    }
+    /**
+     * Resubscribes to the previous channels
+     *
+     * @private
+     */
+    _resubscribe() {
+        if (this.shardedSubscribers) {
+            this.shardedSubscribers.forEach((s, nodeKey) => {
+                const subscriberSlots = this.subscriberToSlotsIndex.get(nodeKey);
+                if (subscriberSlots) {
+                    //Resubscribe on the underlying connection
+                    subscriberSlots.forEach((ss) => {
+                        //Might return null if being disconnected
+                        const redis = s.getInstance();
+                        const channels = this.channels.get(ss);
+                        if (channels && channels.length > 0) {
+                            if (!redis || redis.status === "end") {
+                                return;
+                            }
+                            if (redis.status === "ready") {
+                                redis.ssubscribe(...channels).catch((err) => {
+                                    // TODO: Should we emit an error event here?
+                                    debug("Failed to ssubscribe on node %s: %s", nodeKey, err);
+                                });
+                            }
+                            else {
+                                redis.once("ready", () => {
+                                    redis.ssubscribe(...channels).catch((err) => {
+                                        // TODO: Should we emit an error event here?
+                                        debug("Failed to ssubscribe on node %s: %s", nodeKey, err);
+                                    });
+                                });
+                            }
+                        }
+                    });
+                }
+            });
+        }
+    }
+    /**
+     * Deep equality of the cluster slots objects
+     *
+     * @param other
+     * @private
+     */
+    _slotsAreEqual(other) {
+        if (this.clusterSlots === undefined) {
+            return false;
+        }
+        else {
+            return JSON.stringify(this.clusterSlots) === JSON.stringify(other);
+        }
+    }
+    /**
+     * Checks if any subscribers are in an unhealthy state.
+     *
+     * A subscriber is considered unhealthy if:
+     * - It exists but is not started (failed/disconnected)
+     * - It's missing entirely for a node that should have one
+     *
+     * @returns true if any subscribers need to be recreated
+     */
+    hasUnhealthySubscribers() {
+        const hasFailedSubscribers = Array.from(this.shardedSubscribers.values()).some((sub) => !sub.isHealthy());
+        const hasMissingSubscribers = Array.from(this.subscriberToSlotsIndex.keys()).some((nodeKey) => !this.shardedSubscribers.has(nodeKey));
+        return hasFailedSubscribers || hasMissingSubscribers;
+    }
+    shouldStartSubscriber(sub) {
+        if (sub.isStarted()) {
+            return false;
+        }
+        if (!sub.isLazyConnect()) {
+            return true;
+        }
+        const subscriberSlots = this.subscriberToSlotsIndex.get(sub.getNodeKey());
+        if (!subscriberSlots) {
+            return false;
+        }
+        return subscriberSlots.some((slot) => {
+            const channels = this.channels.get(slot);
+            return Boolean(channels && channels.length > 0);
+        });
+    }
+}
+exports.default = ClusterSubscriberGroup;
+// Retry strategy
+ClusterSubscriberGroup.MAX_RETRY_ATTEMPTS = 10;
+ClusterSubscriberGroup.MAX_BACKOFF_MS = 2000;
+ClusterSubscriberGroup.BASE_BACKOFF_MS = 100;

package/built/cluster/ConnectionPool.d.ts

@@ -0,0 +1,37 @@
+/// <reference types="node" />
+import { EventEmitter } from "events";
+import { RedisOptions, NodeKey, NodeRole } from "./util";
+import Redis from "../Redis";
+export default class ConnectionPool extends EventEmitter {
+    private redisOptions;
+    private nodes;
+    private specifiedOptions;
+    constructor(redisOptions: any);
+    getNodes(role?: NodeRole): Redis[];
+    getInstanceByKey(key: NodeKey): Redis;
+    getSampleInstance(role: NodeRole): Redis;
+    /**
+     * Add a master node to the pool
+     * @param node
+     */
+    addMasterNode(node: RedisOptions): boolean;
+    /**
+     * Creates a Redis connection instance from the node options
+     * @param node
+     * @param readOnly
+     */
+    createRedisFromOptions(node: RedisOptions, readOnly: boolean): Redis;
+    /**
+     * Find or create a connection to the node
+     */
+    findOrCreate(node: RedisOptions, readOnly?: boolean): Redis;
+    /**
+     * Reset the pool with a set of nodes.
+     * The old node will be removed.
+     */
+    reset(nodes: RedisOptions[]): void;
+    /**
+     * Remove a node from the pool.
+     */
+    private removeNode;
+}

package/built/cluster/ConnectionPool.js

@@ -0,0 +1,154 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const events_1 = require("events");
+const utils_1 = require("../utils");
+const util_1 = require("./util");
+const Redis_1 = require("../Redis");
+const debug = (0, utils_1.Debug)("cluster:connectionPool");
+class ConnectionPool extends events_1.EventEmitter {
+    constructor(redisOptions) {
+        super();
+        this.redisOptions = redisOptions;
+        // master + slave = all
+        this.nodes = {
+            all: {},
+            master: {},
+            slave: {},
+        };
+        this.specifiedOptions = {};
+    }
+    getNodes(role = "all") {
+        const nodes = this.nodes[role];
+        return Object.keys(nodes).map((key) => nodes[key]);
+    }
+    getInstanceByKey(key) {
+        return this.nodes.all[key];
+    }
+    getSampleInstance(role) {
+        const keys = Object.keys(this.nodes[role]);
+        const sampleKey = (0, utils_1.sample)(keys);
+        return this.nodes[role][sampleKey];
+    }
+    /**
+     * Add a master node to the pool
+     * @param node
+     */
+    addMasterNode(node) {
+        const key = (0, util_1.getNodeKey)(node.options);
+        const redis = this.createRedisFromOptions(node, node.options.readOnly);
+        //Master nodes aren't read-only
+        if (!node.options.readOnly) {
+            this.nodes.all[key] = redis;
+            this.nodes.master[key] = redis;
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Creates a Redis connection instance from the node options
+     * @param node
+     * @param readOnly
+     */
+    createRedisFromOptions(node, readOnly) {
+        const redis = new Redis_1.default((0, utils_1.defaults)({
+            // Never try to reconnect when a node is lose,
+            // instead, waiting for a `MOVED` error and
+            // fetch the slots again.
+            retryStrategy: null,
+            // Offline queue should be enabled so that
+            // we don't need to wait for the `ready` event
+            // before sending commands to the node.
+            enableOfflineQueue: true,
+            readOnly: readOnly,
+        }, node, this.redisOptions, { lazyConnect: true }));
+        return redis;
+    }
+    /**
+     * Find or create a connection to the node
+     */
+    findOrCreate(node, readOnly = false) {
+        const key = (0, util_1.getNodeKey)(node);
+        readOnly = Boolean(readOnly);
+        if (this.specifiedOptions[key]) {
+            Object.assign(node, this.specifiedOptions[key]);
+        }
+        else {
+            this.specifiedOptions[key] = node;
+        }
+        let redis;
+        if (this.nodes.all[key]) {
+            redis = this.nodes.all[key];
+            if (redis.options.readOnly !== readOnly) {
+                redis.options.readOnly = readOnly;
+                debug("Change role of %s to %s", key, readOnly ? "slave" : "master");
+                redis[readOnly ? "readonly" : "readwrite"]().catch(utils_1.noop);
+                if (readOnly) {
+                    delete this.nodes.master[key];
+                    this.nodes.slave[key] = redis;
+                }
+                else {
+                    delete this.nodes.slave[key];
+                    this.nodes.master[key] = redis;
+                }
+            }
+        }
+        else {
+            debug("Connecting to %s as %s", key, readOnly ? "slave" : "master");
+            redis = this.createRedisFromOptions(node, readOnly);
+            this.nodes.all[key] = redis;
+            this.nodes[readOnly ? "slave" : "master"][key] = redis;
+            redis.once("end", () => {
+                this.removeNode(key);
+                this.emit("-node", redis, key);
+                if (!Object.keys(this.nodes.all).length) {
+                    this.emit("drain");
+                }
+            });
+            this.emit("+node", redis, key);
+            redis.on("error", function (error) {
+                this.emit("nodeError", error, key);
+            });
+        }
+        return redis;
+    }
+    /**
+     * Reset the pool with a set of nodes.
+     * The old node will be removed.
+     */
+    reset(nodes) {
+        debug("Reset with %O", nodes);
+        const newNodes = {};
+        nodes.forEach((node) => {
+            const key = (0, util_1.getNodeKey)(node);
+            // Don't override the existing (master) node
+            // when the current one is slave.
+            if (!(node.readOnly && newNodes[key])) {
+                newNodes[key] = node;
+            }
+        });
+        Object.keys(this.nodes.all).forEach((key) => {
+            if (!newNodes[key]) {
+                debug("Disconnect %s because the node does not hold any slot", key);
+                this.nodes.all[key].disconnect();
+                this.removeNode(key);
+            }
+        });
+        Object.keys(newNodes).forEach((key) => {
+            const node = newNodes[key];
+            this.findOrCreate(node, node.readOnly);
+        });
+    }
+    /**
+     * Remove a node from the pool.
+     */
+    removeNode(key) {
+        const { nodes } = this;
+        if (nodes.all[key]) {
+            debug("Remove %s from the pool", key);
+            delete nodes.all[key];
+        }
+        delete nodes.master[key];
+        delete nodes.slave[key];
+    }
+}
+exports.default = ConnectionPool;

package/built/cluster/DelayQueue.d.ts

@@ -0,0 +1,20 @@
+export interface DelayQueueOptions {
+    callback?: Function;
+    timeout: number;
+}
+/**
+ * Queue that runs items after specified duration
+ */
+export default class DelayQueue {
+    private queues;
+    private timeouts;
+    /**
+     * Add a new item to the queue
+     *
+     * @param bucket bucket name
+     * @param item function that will run later
+     * @param options
+     */
+    push(bucket: string, item: Function, options: DelayQueueOptions): void;
+    private execute;
+}

package/built/cluster/DelayQueue.js

@@ -0,0 +1,53 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("../utils");
+const Deque = require("denque");
+const debug = (0, utils_1.Debug)("delayqueue");
+/**
+ * Queue that runs items after specified duration
+ */
+class DelayQueue {
+    constructor() {
+        this.queues = {};
+        this.timeouts = {};
+    }
+    /**
+     * Add a new item to the queue
+     *
+     * @param bucket bucket name
+     * @param item function that will run later
+     * @param options
+     */
+    push(bucket, item, options) {
+        const callback = options.callback || process.nextTick;
+        if (!this.queues[bucket]) {
+            this.queues[bucket] = new Deque();
+        }
+        const queue = this.queues[bucket];
+        queue.push(item);
+        if (!this.timeouts[bucket]) {
+            this.timeouts[bucket] = setTimeout(() => {
+                callback(() => {
+                    this.timeouts[bucket] = null;
+                    this.execute(bucket);
+                });
+            }, options.timeout);
+        }
+    }
+    execute(bucket) {
+        const queue = this.queues[bucket];
+        if (!queue) {
+            return;
+        }
+        const { length } = queue;
+        if (!length) {
+            return;
+        }
+        debug("send %d commands in %s queue", length, bucket);
+        this.queues[bucket] = null;
+        while (queue.length > 0) {
+            queue.shift()();
+        }
+    }
+}
+exports.default = DelayQueue;

package/built/cluster/ShardedSubscriber.d.ts

@@ -0,0 +1,36 @@
+/// <reference types="node" />
+import EventEmitter = require("events");
+import { RedisOptions } from "./util";
+import Redis from "../Redis";
+import { ClusterOptions } from "./ClusterOptions";
+declare const SubscriberStatus: {
+    readonly IDLE: "idle";
+    readonly STARTING: "starting";
+    readonly CONNECTED: "connected";
+    readonly STOPPING: "stopping";
+    readonly ENDED: "ended";
+};
+declare type SubscriberStatus = typeof SubscriberStatus[keyof typeof SubscriberStatus];
+export default class ShardedSubscriber {
+    private readonly emitter;
+    private readonly nodeKey;
+    private status;
+    private instance;
+    private connectPromise;
+    private lazyConnect;
+    private readonly messageListeners;
+    constructor(emitter: EventEmitter, options: RedisOptions, redisOptions?: ClusterOptions["redisOptions"]);
+    start(): Promise<void>;
+    stop(): void;
+    isStarted(): boolean;
+    get subscriberStatus(): SubscriberStatus;
+    isHealthy(): boolean;
+    getInstance(): Redis | null;
+    getNodeKey(): string;
+    isLazyConnect(): boolean;
+    private onEnd;
+    private onError;
+    private onMoved;
+    private updateStatus;
+}
+export {};