@jetit/publisher 5.5.1 → 5.6.1

package/README.md

@@ -102,6 +102,16 @@ const config: Partial<IStreamsConfig> = {
 const publisher = new Publisher('MyService', config);
 ```
 
+Additionally, the `Publisher` constructor accepts a redisConnectionId parameter, which is used to identify the connection used by the publisher. This is useful when using multiple connections in a environment.
+
+```typescript
+setRedisConfig(options1, 'redis-connection-id');
+const publisher1 = new Publisher('MyService', config, 'redis-connection-id');  // <-- use this connection (options1) for publishing
+
+setRedisConfig(options2, 'another-redis-connection-id');
+const publisher2 = new Publisher('MyService', config, 'another-redis-connection-id');  // <-- use this connection (options2) for publishing
+```
+
 ### Publishing Events
 
 ```typescript

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "5.5.1",
+  "version": "5.6.1",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "^0.0.13",

package/src/lib/publisher.d.ts

@@ -4,4 +4,4 @@ export { ScheduledProcessor as __SCHEDULER_INTERNALS__ } from './redis/scheduler
 export { UTILS as StreamUtilityFunctions } from './redis/utils';
 export { PrometheusAdapter } from './monitoring/adapters/prom';
 export { publishBatch, publishScheduledBatch } from './redis/batch';
-export { IListenOptions, IStreamsConfig, TEventFilter } from './redis/types';
+export type { EventData, IListenOptions, IOptions, IStreamsConfig, PublishData, TEventFilter } from './redis/types';

package/src/lib/redis/registry.d.ts

@@ -4,12 +4,12 @@ export type RedisType = Redis | Cluster;
 export declare class RedisRegistry {
     private static registry;
     private static options;
-    static attemptConnection(connectionKey: string, storeRef?: number): RedisType;
-    static handleDisconnects(connection: RedisType, connectionKey: string, storeRef: number): void;
-    static handlePing(connection: RedisType): void;
-    static getConnection(connectionType?: string, storeRef?: number): RedisType;
-    static setOptions(options: IOptions): void;
-    static _getOptions(): IOptions;
+    static attemptConnection(redisConnectionId: string, connectionKey: string, storeRef?: number): RedisType;
+    static handleDisconnects(redisConnectionId: string, connection: RedisType, connectionKey: string, storeRef: number): void;
+    static handlePing(redisConnectionId: string, connection: RedisType): void;
+    static getConnection(redisConnectionId: string, connectionType?: string, storeRef?: number): RedisType;
+    static setOptions(options: IOptions, redisConnectionId: string): void;
+    static _getOptions(redisConnectionId: string): IOptions;
 }
 /**
  * This function is used to set Redis Connection options per instance. If no
@@ -19,4 +19,4 @@ export declare class RedisRegistry {
  *
  * @param options
  */
-export declare function setRedisConnectionSettings(options: IOptions): void;
+export declare function setRedisConnectionSettings(options: IOptions, redisConnectionId?: string): void;

package/src/lib/redis/registry.js

@@ -5,35 +5,40 @@ exports.setRedisConnectionSettings = setRedisConnectionSettings;
 const ioredis_1 = require("ioredis");
 const logger_1 = require("./logger");
 class RedisRegistry {
-    static attemptConnection(connectionKey, storeRef = 0) {
+    static attemptConnection(redisConnectionId, connectionKey, storeRef = 0) {
         let ref;
-        if (RedisRegistry.options.cluster) {
-            ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, {
-                ...RedisRegistry.options.cluster.options,
+        const options = RedisRegistry.options[redisConnectionId];
+        if (!options)
+            throw new Error(`PUBLISHER: Redis config key ${redisConnectionId} not found`);
+        if (options.cluster) {
+            ref = new ioredis_1.Cluster(options.cluster.nodes, {
+                ...options.cluster.options,
                 redisOptions: {
-                    ...RedisRegistry.options.cluster.options.redisOptions,
+                    ...options.cluster.options.redisOptions,
                     db: storeRef,
                 },
             });
         }
         else
             ref = new ioredis_1.default({
-                ...RedisRegistry.options.redis,
+                ...options.redis,
                 db: storeRef,
             });
-        RedisRegistry.registry.set(connectionKey, ref);
-        RedisRegistry.handleDisconnects(ref, connectionKey, storeRef);
+        if (!RedisRegistry.registry[connectionKey])
+            RedisRegistry.registry[connectionKey] = new Map();
+        RedisRegistry.registry[connectionKey].set(redisConnectionId, ref);
+        RedisRegistry.handleDisconnects(redisConnectionId, ref, connectionKey, storeRef);
         return ref;
     }
-    static handleDisconnects(connection, connectionKey, storeRef) {
+    static handleDisconnects(redisConnectionId, connection, connectionKey, storeRef) {
         connection.on('error', (error) => {
             logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Redis connection error : ${error.message}`);
             connection.removeAllListeners();
             connection.disconnect();
-            RedisRegistry.attemptConnection(connectionKey, storeRef);
+            RedisRegistry.attemptConnection(redisConnectionId, connectionKey, storeRef);
         });
     }
-    static handlePing(connection) {
+    static handlePing(redisConnectionId, connection) {
         setInterval(async () => {
             const res = await Promise.race([
                 connection.ping(),
@@ -45,52 +50,61 @@ class RedisRegistry {
             ]);
             if (res === 'NO_PING') {
                 connection.disconnect(true);
-                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: failed to ping redis, disconnecting and restarting service.');
+                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: failed to ping redis, disconnecting and restarting service for redisConnectionId : ' + redisConnectionId);
                 process.exit(0);
             }
         }, 2000);
     }
-    static getConnection(connectionType = 'primary', storeRef = 0) {
+    static getConnection(redisConnectionId, connectionType = 'primary', storeRef = 0) {
         const connectionKey = `${connectionType}${storeRef}`;
-        let ref = this.registry.get(connectionKey);
+        const options = RedisRegistry.options[redisConnectionId];
+        if (!options)
+            throw new Error(`PUBLISHER: Redis config key ${redisConnectionId} not found`);
+        if (!this.registry[redisConnectionId])
+            this.registry[redisConnectionId] = new Map();
+        let ref = this.registry[redisConnectionId].get(connectionKey);
         if (!ref) {
-            if (RedisRegistry.options.cluster) {
-                ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, {
-                    ...RedisRegistry.options.cluster.options,
+            if (options.cluster) {
+                ref = new ioredis_1.Cluster(options.cluster.nodes, {
+                    ...options.cluster.options,
                     redisOptions: {
-                        ...RedisRegistry.options.cluster.options.redisOptions,
+                        ...options.cluster.options.redisOptions,
                         db: storeRef,
                     },
                 });
             }
-            else if (RedisRegistry.options.sentinels) {
+            else if (options.sentinels) {
                 ref = new ioredis_1.default({
-                    ...RedisRegistry.options.sentinels.options,
-                    sentinels: RedisRegistry.options.sentinels.nodes,
+                    ...options.sentinels.options,
+                    sentinels: options.sentinels.nodes,
                 });
             }
             else {
                 ref = new ioredis_1.default({
-                    ...RedisRegistry.options.redis,
+                    ...options.redis,
                     db: storeRef,
                 });
             }
         }
         return ref;
     }
-    static setOptions(options) {
-        RedisRegistry.options = options;
+    static setOptions(options, redisConnectionId) {
+        RedisRegistry.options[redisConnectionId] = options;
     }
-    static _getOptions() {
-        return RedisRegistry.options;
+    static _getOptions(redisConnectionId) {
+        return RedisRegistry.options[redisConnectionId];
     }
 }
 exports.RedisRegistry = RedisRegistry;
-RedisRegistry.registry = new Map();
+RedisRegistry.registry = {
+    default: new Map(),
+};
 RedisRegistry.options = {
-    redis: {
-        port: parseInt(process.env['REDIS_PORT'] ?? '6379'),
-        host: process.env['REDIS_HOST'] ?? 'localhost',
+    default: {
+        redis: {
+            port: parseInt(process.env['REDIS_PORT'] ?? '6379'),
+            host: process.env['REDIS_HOST'] ?? 'localhost',
+        },
     },
 };
 /**
@@ -101,6 +115,6 @@ RedisRegistry.options = {
  *
  * @param options
  */
-function setRedisConnectionSettings(options) {
-    RedisRegistry.setOptions(options);
+function setRedisConnectionSettings(options, redisConnectionId = 'default') {
+    RedisRegistry.setOptions(options, redisConnectionId);
 }

package/src/lib/redis/scheduler.d.ts

@@ -4,11 +4,12 @@ import { RedisType } from './registry';
  * meant to be used internally by the scheduler application
  */
 export declare class ScheduledProcessor {
+    private redisConnectionId;
     private scheduledMessagesTimer;
     private _redisPublisher?;
     private previousTaskCompleted;
     get redisPublisher(): RedisType;
-    constructor(duration?: number);
+    constructor(duration?: number, redisConnectionId?: string);
     private processScheduledEvents;
     getAllScheduledEvents(): Promise<Array<string>>;
     close(): Promise<void>;

package/src/lib/redis/scheduler.js

@@ -13,10 +13,11 @@ const utils_1 = require("./utils");
 class ScheduledProcessor {
     get redisPublisher() {
         if (!this._redisPublisher)
-            this._redisPublisher = registry_1.RedisRegistry.getConnection('publish');
+            this._redisPublisher = registry_1.RedisRegistry.getConnection(this.redisConnectionId, 'publish');
         return this._redisPublisher;
     }
-    constructor(duration = 1000) {
+    constructor(duration = 1000, redisConnectionId = 'default') {
+        this.redisConnectionId = redisConnectionId;
         this.previousTaskCompleted = true;
         this.scheduledMessagesTimer = (0, rxjs_1.interval)(duration).subscribe(() => {
             logger_1.PUBLISHER_LOGGER.log('Checking Streams messages at ', new Date().toISOString(), '...');
@@ -62,7 +63,7 @@ class ScheduledProcessor {
                     .xadd(streamName, key, 'data', JSON.stringify(eventData))
                     .catch((e) => logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Publishing event ${eventData.eventName} to consumer groups: ${consumerGroups.join(', ')} failed with data ${JSON.stringify(eventData)}, ${e} `));
                 if (key === '*')
-                    key = generatedKey ?? key;
+                    key = `${generatedKey ?? key}`;
             }
             if (eventData.repeatInterval) {
                 const nextEventTime = currentTime + eventData.repeatInterval;

package/src/lib/redis/streams.d.ts

@@ -3,6 +3,7 @@ import { IAggregatedMetrics, TQueryableMetrics } from '../monitoring/types';
 import { CircuitState } from '../performance/circuit_breaker';
 import { EventData, IListenOptions, IStreamsConfig, PublishData } from './types';
 export declare class Streams {
+    private redisConnectionId;
     private _redisPublisher?;
     private _redisGroups?;
     private config;
@@ -33,7 +34,7 @@ export declare class Streams {
      * // Create a new Streams instance for the "POS" service
      * const streams = new Streams('POS');
      */
-    constructor(serviceName: string, config?: Partial<IStreamsConfig>);
+    constructor(serviceName: string, config?: Partial<IStreamsConfig>, redisConnectionId?: string);
     private setupCircuitBreakerListeners;
     private runClear;
     /**

package/src/lib/redis/streams.js

@@ -14,12 +14,12 @@ const utils_1 = require("./utils");
 class Streams {
     get redisPublisher() {
         if (!this._redisPublisher)
-            this._redisPublisher = registry_1.RedisRegistry.getConnection('publish');
+            this._redisPublisher = registry_1.RedisRegistry.getConnection(this.redisConnectionId, 'publish');
         return this._redisPublisher;
     }
     get redisGroups() {
         if (!this._redisGroups)
-            this._redisGroups = registry_1.RedisRegistry.getConnection('groups');
+            this._redisGroups = registry_1.RedisRegistry.getConnection(this.redisConnectionId, 'groups');
         return this._redisGroups;
     }
     /**
@@ -36,7 +36,8 @@ class Streams {
      * // Create a new Streams instance for the "POS" service
      * const streams = new Streams('POS');
      */
-    constructor(serviceName, config = {}) {
+    constructor(serviceName, config = {}, redisConnectionId = 'default') {
+        this.redisConnectionId = redisConnectionId;
         this.eventsListened = [];
         this.subscriptions = new Map();
         this.DEFAULT_STREAMS_CONFIG = {
@@ -367,6 +368,17 @@ class Streams {
                      * they need to be read using XRANGE.
                      */
                     if (multicast || processPending) {
+                        /**
+                         * Very very rare case of this occurring. Cases where the running service is super overloaded
+                         * that causes a messaged to be sent to processing with a delay, but eventually before being
+                         * processed gets picked up by another instance, leading to multiple publications
+                         */
+                        if (processPending) {
+                            const claimed = await redisClient.xclaim(streamName, this.consumerGroupName, this.instanceId, 20000, messageId, 'JUSTID');
+                            if (!claimed || claimed.length === 0) {
+                                return; // Message already claimed or acknowledged by another consumer, so don't repush to the subscriber
+                            }
+                        }
                         const messages = await redisClient.xrange(streamName, messageId, messageId);
                         if (messages?.length) {
                             try {
@@ -468,13 +480,31 @@ class Streams {
                 /** Process Unprocessed Messages with rate limiting */
                 if (!processPending) {
                     const unprocessedMessageIds = await (0, utils_1.getUnacknowledgedMessages)(redisClient, this.consumerGroupName, streamName, this.instanceId);
+                    /**
+                     * Dealing with the case where messages don't get processed due to large
+                     * batch sizes. The previous default was 20ms, which seemed to work in
+                     * most cases but seem to fail when the number of retry messages are high.
+                     *
+                     * The old solution did not take into account the fact that this library
+                     * is dependent on the infrastructure of the app that runs it, so any memory/
+                     * resource/stack overload on the app has an impact in this
+                     */
+                    const getDelay = (count) => {
+                        if (count > 100)
+                            return 500; // 500ms for large backlogs
+                        if (count > 50)
+                            return 300; // 300ms for medium-large
+                        if (count > 20)
+                            return 200; // 200ms for medium
+                        return 100; // 100ms for small
+                    };
                     if (unprocessedMessageIds.countOnThisConsumer &&
                         unprocessedMessageIds.countOnThisConsumer > this.config.unprocessedMessageThreshold) {
                         logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Too many unprocessed events for ${streamName}: count: ${unprocessedMessageIds.count}`);
                     }
                     // Process messages with rate limiting
                     const processWithDelay = async (id, index) => {
-                        await new Promise((resolve) => setTimeout(resolve, index * 20));
+                        await new Promise((resolve) => setTimeout(resolve, index * getDelay(unprocessedMessageIds.countOnThisConsumer ?? 1)));
                         await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
                     };
                     unprocessedMessageIds.messageIds.map((id, index) => processWithDelay(id, index));
@@ -506,7 +536,7 @@ class Streams {
             .then((consumerRegistered) => {
             if (!consumerRegistered)
                 throw new Error('PUBLISHER: Cannot setup consumer');
-            const eventStreamClient = registry_1.RedisRegistry.getConnection(`sub-${eventName}`);
+            const eventStreamClient = registry_1.RedisRegistry.getConnection(this.redisConnectionId, `sub-${eventName}`);
             eventStreamClient.subscribe(eventName).then(() => {
                 logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Redis Subscription connection initiated for ${eventName}`);
             });
@@ -565,7 +595,7 @@ class Streams {
                 await this._redisPublisher.quit();
             }
             for (const eventName of this.eventsListened) {
-                await registry_1.RedisRegistry.getConnection(`sub-${eventName}`).quit();
+                await registry_1.RedisRegistry.getConnection(this.redisConnectionId, `sub-${eventName}`).quit();
             }
             if (this._redisGroups) {
                 await this._redisGroups.quit();