@jetit/publisher 5.4.1 → 5.6.0

package/README.md

@@ -102,6 +102,16 @@ const config: Partial<IStreamsConfig> = {
 const publisher = new Publisher('MyService', config);
 ```
 
+Additionally, the `Publisher` constructor accepts a redisConnectionId parameter, which is used to identify the connection used by the publisher. This is useful when using multiple connections in a environment.
+
+```typescript
+setRedisConfig(options1, 'redis-connection-id');
+const publisher1 = new Publisher('MyService', config, 'redis-connection-id');  // <-- use this connection (options1) for publishing
+
+setRedisConfig(options2, 'another-redis-connection-id');
+const publisher2 = new Publisher('MyService', config, 'another-redis-connection-id');  // <-- use this connection (options2) for publishing
+```
+
 ### Publishing Events
 
 ```typescript

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "5.4.1",
+  "version": "5.6.0",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "^0.0.13",

package/src/lib/monitoring/collector.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MetricsCollector = void 0;
+const logger_1 = require("../redis/logger");
 class MetricsCollector {
     constructor(config, dlq) {
         this.metrics = [];
@@ -116,19 +117,34 @@ class MetricsCollector {
             if (keys.length > 0) {
                 for (const streamKey of keys) {
                     try {
+                        // Verify if the key is a stream before proceeding
+                        const keyType = await this.redisClient.type(streamKey);
+                        if (keyType !== 'stream') {
+                            continue; // Skip non-stream keys
+                        }
                         // Get stream length and pending info
                         const streamLength = await this.redisClient.xlen(streamKey);
                         // Extract consumer group name from stream key (format: eventName:cg-serviceName)
-                        const consumerGroup = streamKey.split(':')[1];
+                        const [eventName, consumerGroup] = streamKey.split(':');
+                        if (!consumerGroup?.startsWith('cg-')) {
+                            continue; // Skip if key format doesn't match expected pattern
+                        }
                         // XPENDING returns [count, min-id, max-id, consumer-list]
-                        const pendingInfo = await this.redisClient.xpending(streamKey, consumerGroup);
-                        const totalPending = pendingInfo ? Number(pendingInfo[0]) : 0;
+                        let totalPending = 0;
+                        try {
+                            const pendingInfo = await this.redisClient.xpending(streamKey, consumerGroup);
+                            totalPending = pendingInfo ? Number(pendingInfo[0]) : 0;
+                        }
+                        catch (error) {
+                            logger_1.PUBLISHER_LOGGER.error(`Error getting pending info for stream ${streamKey}:`, error);
+                            // Continue with totalPending as 0 if XPENDING fails
+                        }
                         const queueDepth = Math.max(0, streamLength - totalPending);
                         totalDepth += queueDepth;
                         individualDepths[streamKey] = queueDepth;
                     }
                     catch (error) {
-                        console.error(`Error processing key ${streamKey}:`, error);
+                        logger_1.PUBLISHER_LOGGER.error(`Error processing key ${streamKey}:`, error);
                         continue;
                     }
                 }

package/src/lib/publisher.d.ts

@@ -4,4 +4,4 @@ export { ScheduledProcessor as __SCHEDULER_INTERNALS__ } from './redis/scheduler
 export { UTILS as StreamUtilityFunctions } from './redis/utils';
 export { PrometheusAdapter } from './monitoring/adapters/prom';
 export { publishBatch, publishScheduledBatch } from './redis/batch';
-export { IListenOptions, IStreamsConfig, TEventFilter } from './redis/types';
+export type { EventData, IListenOptions, IOptions, IStreamsConfig, PublishData, TEventFilter } from './redis/types';

package/src/lib/redis/dlq.js

@@ -18,7 +18,11 @@ class DeadLetterQueue {
                 return;
             }
             const eventJson = JSON.stringify(event);
-            await this.redisClient.multi().hset(DLQ_HASH_KEY, event.eventId, eventJson).zadd(DLQ_ZSET_KEY, event.timestamp, event.eventId).exec();
+            // Execute commands separately to avoid CROSSSLOT error in cluster mode
+            await Promise.all([
+                this.redisClient.hset(DLQ_HASH_KEY, event.eventId, eventJson),
+                this.redisClient.zadd(DLQ_ZSET_KEY, event.timestamp, event.eventId),
+            ]);
             await this.incrementRateLimit();
             logger_1.PUBLISHER_LOGGER.log(`DLQ: Added event ${event.eventId} to Dead Letter Queue`);
         }
@@ -69,8 +73,12 @@ class DeadLetterQueue {
     }
     async removeFromDLQ(eventId) {
         try {
-            const removed = await this.redisClient.multi().hdel(DLQ_HASH_KEY, eventId).zrem(DLQ_ZSET_KEY, eventId).exec();
-            if (removed && removed[0][1] === 1 && removed[1][1] === 1) {
+            // Execute commands separately to avoid CROSSSLOT error in cluster mode
+            const [hdelResult, zremResult] = await Promise.all([
+                this.redisClient.hdel(DLQ_HASH_KEY, eventId),
+                this.redisClient.zrem(DLQ_ZSET_KEY, eventId),
+            ]);
+            if (hdelResult === 1 && zremResult === 1) {
                 logger_1.PUBLISHER_LOGGER.log(`DLQ: Successfully removed event ${eventId} from Dead Letter Queue`);
                 return true;
             }
@@ -90,11 +98,11 @@ class DeadLetterQueue {
         try {
             const expiredEventIds = await this.redisClient.zrangebyscore(DLQ_ZSET_KEY, 0, cutoffTime);
             if (expiredEventIds.length > 0) {
-                await this.redisClient
-                    .multi()
-                    .hdel(DLQ_HASH_KEY, ...expiredEventIds)
-                    .zremrangebyscore(DLQ_ZSET_KEY, 0, cutoffTime)
-                    .exec();
+                // Execute commands separately to avoid CROSSSLOT error in cluster mode
+                await Promise.all([
+                    this.redisClient.hdel(DLQ_HASH_KEY, ...expiredEventIds),
+                    this.redisClient.zremrangebyscore(DLQ_ZSET_KEY, 0, cutoffTime),
+                ]);
             }
             logger_1.PUBLISHER_LOGGER.log(`DLQ: Cleaned up ${expiredEventIds.length} expired events from Dead Letter Queue`);
         }
@@ -104,16 +112,9 @@ class DeadLetterQueue {
     }
     async getDLQStats() {
         try {
-            const results = await this.redisClient.multi().zcard(DLQ_ZSET_KEY).get(DLQ_RATE_LIMIT_KEY).exec();
-            if (!results) {
-                throw new Error('Failed to execute Redis commands');
-            }
-            const [sizeResult, additionRateResult] = results;
-            if (sizeResult[0] || additionRateResult[0]) {
-                throw new Error('Error executing Redis commands');
-            }
-            const size = sizeResult[1];
-            const additionRate = parseInt(additionRateResult[1] || '0', 10);
+            // Execute commands separately to avoid CROSSSLOT error in cluster mode
+            const [size, additionRateStr] = await Promise.all([this.redisClient.zcard(DLQ_ZSET_KEY), this.redisClient.get(DLQ_RATE_LIMIT_KEY)]);
+            const additionRate = parseInt(additionRateStr || '0', 10);
             return { size, additionRate };
         }
         catch (error) {
@@ -126,8 +127,12 @@ class DeadLetterQueue {
         return parseInt(currentRate, 10) >= this.maxEventsThreshold;
     }
     async incrementRateLimit() {
-        await this.redisClient.incr(DLQ_RATE_LIMIT_KEY);
-        await this.redisClient.expire(DLQ_RATE_LIMIT_KEY, 60); // Reset rate limit after 1 minute
+        // These commands operate on the same key so they can be combined in cluster mode
+        await this.redisClient
+            .multi()
+            .incr(DLQ_RATE_LIMIT_KEY)
+            .expire(DLQ_RATE_LIMIT_KEY, 60) // Reset rate limit after 1 minute
+            .exec();
     }
 }
 exports.DeadLetterQueue = DeadLetterQueue;

package/src/lib/redis/registry.d.ts

@@ -4,12 +4,12 @@ export type RedisType = Redis | Cluster;
 export declare class RedisRegistry {
     private static registry;
     private static options;
-    static attemptConnection(connectionKey: string, storeRef?: number): RedisType;
-    static handleDisconnects(connection: RedisType, connectionKey: string, storeRef: number): void;
-    static handlePing(connection: RedisType): void;
-    static getConnection(connectionType?: string, storeRef?: number): RedisType;
-    static setOptions(options: IOptions): void;
-    static _getOptions(): IOptions;
+    static attemptConnection(redisConnectionId: string, connectionKey: string, storeRef?: number): RedisType;
+    static handleDisconnects(redisConnectionId: string, connection: RedisType, connectionKey: string, storeRef: number): void;
+    static handlePing(redisConnectionId: string, connection: RedisType): void;
+    static getConnection(redisConnectionId: string, connectionType?: string, storeRef?: number): RedisType;
+    static setOptions(options: IOptions, redisConnectionId: string): void;
+    static _getOptions(redisConnectionId: string): IOptions;
 }
 /**
  * This function is used to set Redis Connection options per instance. If no
@@ -19,4 +19,4 @@ export declare class RedisRegistry {
  *
  * @param options
  */
-export declare function setRedisConnectionSettings(options: IOptions): void;
+export declare function setRedisConnectionSettings(options: IOptions, redisConnectionId?: string): void;

package/src/lib/redis/registry.js

@@ -5,35 +5,40 @@ exports.setRedisConnectionSettings = setRedisConnectionSettings;
 const ioredis_1 = require("ioredis");
 const logger_1 = require("./logger");
 class RedisRegistry {
-    static attemptConnection(connectionKey, storeRef = 0) {
+    static attemptConnection(redisConnectionId, connectionKey, storeRef = 0) {
         let ref;
-        if (RedisRegistry.options.cluster) {
-            ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, {
-                ...RedisRegistry.options.cluster.options,
+        const options = RedisRegistry.options[redisConnectionId];
+        if (!options)
+            throw new Error(`PUBLISHER: Redis config key ${redisConnectionId} not found`);
+        if (options.cluster) {
+            ref = new ioredis_1.Cluster(options.cluster.nodes, {
+                ...options.cluster.options,
                 redisOptions: {
-                    ...RedisRegistry.options.cluster.options.redisOptions,
+                    ...options.cluster.options.redisOptions,
                     db: storeRef,
                 },
             });
         }
         else
             ref = new ioredis_1.default({
-                ...RedisRegistry.options.redis,
+                ...options.redis,
                 db: storeRef,
             });
-        RedisRegistry.registry.set(connectionKey, ref);
-        RedisRegistry.handleDisconnects(ref, connectionKey, storeRef);
+        if (!RedisRegistry.registry[connectionKey])
+            RedisRegistry.registry[connectionKey] = new Map();
+        RedisRegistry.registry[connectionKey].set(redisConnectionId, ref);
+        RedisRegistry.handleDisconnects(redisConnectionId, ref, connectionKey, storeRef);
         return ref;
     }
-    static handleDisconnects(connection, connectionKey, storeRef) {
+    static handleDisconnects(redisConnectionId, connection, connectionKey, storeRef) {
         connection.on('error', (error) => {
             logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Redis connection error : ${error.message}`);
             connection.removeAllListeners();
             connection.disconnect();
-            RedisRegistry.attemptConnection(connectionKey, storeRef);
+            RedisRegistry.attemptConnection(redisConnectionId, connectionKey, storeRef);
         });
     }
-    static handlePing(connection) {
+    static handlePing(redisConnectionId, connection) {
         setInterval(async () => {
             const res = await Promise.race([
                 connection.ping(),
@@ -45,52 +50,61 @@ class RedisRegistry {
             ]);
             if (res === 'NO_PING') {
                 connection.disconnect(true);
-                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: failed to ping redis, disconnecting and restarting service.');
+                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: failed to ping redis, disconnecting and restarting service for redisConnectionId : ' + redisConnectionId);
                 process.exit(0);
             }
         }, 2000);
     }
-    static getConnection(connectionType = 'primary', storeRef = 0) {
+    static getConnection(redisConnectionId, connectionType = 'primary', storeRef = 0) {
         const connectionKey = `${connectionType}${storeRef}`;
-        let ref = this.registry.get(connectionKey);
+        const options = RedisRegistry.options[redisConnectionId];
+        if (!options)
+            throw new Error(`PUBLISHER: Redis config key ${redisConnectionId} not found`);
+        if (!this.registry[redisConnectionId])
+            this.registry[redisConnectionId] = new Map();
+        let ref = this.registry[redisConnectionId].get(connectionKey);
         if (!ref) {
-            if (RedisRegistry.options.cluster) {
-                ref = new ioredis_1.Cluster(RedisRegistry.options.cluster.nodes, {
-                    ...RedisRegistry.options.cluster.options,
+            if (options.cluster) {
+                ref = new ioredis_1.Cluster(options.cluster.nodes, {
+                    ...options.cluster.options,
                     redisOptions: {
-                        ...RedisRegistry.options.cluster.options.redisOptions,
+                        ...options.cluster.options.redisOptions,
                         db: storeRef,
                     },
                 });
             }
-            else if (RedisRegistry.options.sentinels) {
+            else if (options.sentinels) {
                 ref = new ioredis_1.default({
-                    ...RedisRegistry.options.sentinels.options,
-                    sentinels: RedisRegistry.options.sentinels.nodes,
+                    ...options.sentinels.options,
+                    sentinels: options.sentinels.nodes,
                 });
             }
             else {
                 ref = new ioredis_1.default({
-                    ...RedisRegistry.options.redis,
+                    ...options.redis,
                     db: storeRef,
                 });
             }
         }
         return ref;
     }
-    static setOptions(options) {
-        RedisRegistry.options = options;
+    static setOptions(options, redisConnectionId) {
+        RedisRegistry.options[redisConnectionId] = options;
     }
-    static _getOptions() {
-        return RedisRegistry.options;
+    static _getOptions(redisConnectionId) {
+        return RedisRegistry.options[redisConnectionId];
     }
 }
 exports.RedisRegistry = RedisRegistry;
-RedisRegistry.registry = new Map();
+RedisRegistry.registry = {
+    default: new Map(),
+};
 RedisRegistry.options = {
-    redis: {
-        port: parseInt(process.env['REDIS_PORT'] ?? '6379'),
-        host: process.env['REDIS_HOST'] ?? 'localhost',
+    default: {
+        redis: {
+            port: parseInt(process.env['REDIS_PORT'] ?? '6379'),
+            host: process.env['REDIS_HOST'] ?? 'localhost',
+        },
     },
 };
 /**
@@ -101,6 +115,6 @@ RedisRegistry.options = {
  *
  * @param options
  */
-function setRedisConnectionSettings(options) {
-    RedisRegistry.setOptions(options);
+function setRedisConnectionSettings(options, redisConnectionId = 'default') {
+    RedisRegistry.setOptions(options, redisConnectionId);
 }

package/src/lib/redis/scheduler.d.ts

@@ -4,11 +4,12 @@ import { RedisType } from './registry';
  * meant to be used internally by the scheduler application
  */
 export declare class ScheduledProcessor {
+    private redisConnectionId;
     private scheduledMessagesTimer;
     private _redisPublisher?;
     private previousTaskCompleted;
     get redisPublisher(): RedisType;
-    constructor(duration?: number);
+    constructor(duration?: number, redisConnectionId?: string);
     private processScheduledEvents;
     getAllScheduledEvents(): Promise<Array<string>>;
     close(): Promise<void>;

package/src/lib/redis/scheduler.js

@@ -13,10 +13,11 @@ const utils_1 = require("./utils");
 class ScheduledProcessor {
     get redisPublisher() {
         if (!this._redisPublisher)
-            this._redisPublisher = registry_1.RedisRegistry.getConnection('publish');
+            this._redisPublisher = registry_1.RedisRegistry.getConnection(this.redisConnectionId, 'publish');
         return this._redisPublisher;
     }
-    constructor(duration = 1000) {
+    constructor(duration = 1000, redisConnectionId = 'default') {
+        this.redisConnectionId = redisConnectionId;
         this.previousTaskCompleted = true;
         this.scheduledMessagesTimer = (0, rxjs_1.interval)(duration).subscribe(() => {
             logger_1.PUBLISHER_LOGGER.log('Checking Streams messages at ', new Date().toISOString(), '...');

package/src/lib/redis/streams.d.ts

@@ -3,6 +3,7 @@ import { IAggregatedMetrics, TQueryableMetrics } from '../monitoring/types';
 import { CircuitState } from '../performance/circuit_breaker';
 import { EventData, IListenOptions, IStreamsConfig, PublishData } from './types';
 export declare class Streams {
+    private redisConnectionId;
     private _redisPublisher?;
     private _redisGroups?;
     private config;
@@ -33,7 +34,7 @@ export declare class Streams {
      * // Create a new Streams instance for the "POS" service
      * const streams = new Streams('POS');
      */
-    constructor(serviceName: string, config?: Partial<IStreamsConfig>);
+    constructor(serviceName: string, config?: Partial<IStreamsConfig>, redisConnectionId?: string);
     private setupCircuitBreakerListeners;
     private runClear;
     /**

package/src/lib/redis/streams.js

@@ -14,12 +14,12 @@ const utils_1 = require("./utils");
 class Streams {
     get redisPublisher() {
         if (!this._redisPublisher)
-            this._redisPublisher = registry_1.RedisRegistry.getConnection('publish');
+            this._redisPublisher = registry_1.RedisRegistry.getConnection(this.redisConnectionId, 'publish');
         return this._redisPublisher;
     }
     get redisGroups() {
         if (!this._redisGroups)
-            this._redisGroups = registry_1.RedisRegistry.getConnection('groups');
+            this._redisGroups = registry_1.RedisRegistry.getConnection(this.redisConnectionId, 'groups');
         return this._redisGroups;
     }
     /**
@@ -36,7 +36,8 @@ class Streams {
      * // Create a new Streams instance for the "POS" service
      * const streams = new Streams('POS');
      */
-    constructor(serviceName, config = {}) {
+    constructor(serviceName, config = {}, redisConnectionId = 'default') {
+        this.redisConnectionId = redisConnectionId;
         this.eventsListened = [];
         this.subscriptions = new Map();
         this.DEFAULT_STREAMS_CONFIG = {
@@ -506,7 +507,7 @@ class Streams {
             .then((consumerRegistered) => {
             if (!consumerRegistered)
                 throw new Error('PUBLISHER: Cannot setup consumer');
-            const eventStreamClient = registry_1.RedisRegistry.getConnection(`sub-${eventName}`);
+            const eventStreamClient = registry_1.RedisRegistry.getConnection(this.redisConnectionId, `sub-${eventName}`);
             eventStreamClient.subscribe(eventName).then(() => {
                 logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Redis Subscription connection initiated for ${eventName}`);
             });
@@ -565,7 +566,7 @@ class Streams {
                 await this._redisPublisher.quit();
             }
             for (const eventName of this.eventsListened) {
-                await registry_1.RedisRegistry.getConnection(`sub-${eventName}`).quit();
+                await registry_1.RedisRegistry.getConnection(this.redisConnectionId, `sub-${eventName}`).quit();
             }
             if (this._redisGroups) {
                 await this._redisGroups.quit();