@jetit/publisher 6.0.0 → 6.0.2

package/README.md

@@ -125,6 +125,8 @@ const config: Partial<IStreamsConfig> = {
         halfOpenStateMaxAttempts: 10,
         maxStoredEvents: 5000,
     },
+    maxPendingTasks: 1000, // Max concurrent pending retry tasks per event per consumer (default: 1000, 0 = unbounded)
+    enableMetrics: true, // Set to false to disable MetricsCollector and its periodic SCAN (default: true)
 };
 
 const publisher = new Publisher('MyService', config);
@@ -358,6 +360,8 @@ The Circuit Breaker has three states:
 - Retry logic with exponential backoff for failed operations
 - Circuit Breaker to prevent overwhelming failed services
 - Dead Letter Queue (DLQ) for handling subscription failures
+- **Disable Metrics Collection**: Set `enableMetrics: false` to skip `MetricsCollector` initialization entirely. The collector runs `SCAN *:cg-* COUNT 100` every 60 seconds for queue depth — disabling it eliminates this periodic Redis overhead. Applies to both `Publisher` and `PublisherLite`. When disabled, `getMetrics()` returns `[]` and `getLatestMetrics()` returns `null`.
+- **Pending Retry Task Cap**: Limits the number of concurrent in-flight pending message retry tasks per event per consumer instance. Prevents unbounded memory growth and Redis saturation under sustained load. Configurable via `maxPendingTasks` (default: 1000). Set to `0` to disable the cap (unbounded, pre-6.0.1 behavior). When the cap is reached, a rate-limited warning is logged and skipped messages are retried on the next Pub/Sub trigger.
 - **Adaptive Redis Stream ID Generation (Publisher only)**: The default `Publisher` automatically switches to an optimized ID generation strategy using a Lua script when publishing to many consumer groups (>10 by default) or when ID conflicts are detected. This prevents `XADD` errors related to non-monotonic IDs in high-throughput scenarios. Configurable via `optimizationThreshold` and `optimizationDurationMs`. (`PublisherLite` does not include this complex logic).
   
 ## Cleanup and Graceful Shutdown

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "6.0.0",
+  "version": "6.0.2",
   "type": "commonjs",
   "peerDependencies": {
     "@jetit/id": ">=0.0.14",
@@ -8,8 +8,7 @@
     "rxjs": ">=7.0.0"
   },
   "dependencies": {
-    "nanoid": "^5.0.9",
-    "tslib": "2.7.0"
+    "tslib": "2.8.1"
   },
   "types": "./src/index.d.ts",
   "main": "./src/index.js"

package/src/lib/redis/streams-lite.d.ts

@@ -8,7 +8,7 @@ export declare class StreamsLite {
     private _redisGroups?;
     private config;
     private dlq;
-    private metricsCollector;
+    private metricsCollector?;
     private consumerGroupName;
     private instanceId;
     private instanceUniqueId;

package/src/lib/redis/streams-lite.js

@@ -75,11 +75,13 @@ class StreamsLite {
         }, cleanUpInterval);
         logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Clean Up process setup for ${cleanUpInterval} ms`);
         this.dlq = new dlq_1.DeadLetterQueue(this.redisPublisher, config.dlqEventThreshold);
-        this.metricsCollector = new collector_1.MetricsCollector({
-            redisClient: this.redisPublisher,
-            collectionInterval: 60000,
-            retentionPeriod: 6 * 60 * 60 * 1000,
-        }, this.dlq);
+        if (this.config.enableMetrics !== false) {
+            this.metricsCollector = new collector_1.MetricsCollector({
+                redisClient: this.redisPublisher,
+                collectionInterval: 60000,
+                retentionPeriod: 6 * 60 * 60 * 1000,
+            }, this.dlq);
+        }
         this.duplicateChecker = new duplication_1.ContentBasedDeduplication(this.redisPublisher, this.config.duplicationCheckWindow);
         this.circuitBreaker = new circuit_breaker_1.CircuitBreaker(this.config.circuitBreaker, this.redisPublisher);
         if (this.config.circuitBreaker.enabled)
@@ -667,6 +669,8 @@ class StreamsLite {
      * This will return you the stats of the publisher for the last 6 hours after cleaning
      */
     async getMetrics(startTime, endTime) {
+        if (!this.metricsCollector)
+            return [];
         return this.metricsCollector.getMetrics(startTime, endTime);
     }
     /**
@@ -674,6 +678,8 @@ class StreamsLite {
      * This will return you the latest stats of the publisher
      */
     async getLatestMetrics() {
+        if (!this.metricsCollector)
+            return null;
         return this.metricsCollector.getLatestMetrics();
     }
     /**

package/src/lib/redis/streams.d.ts

@@ -8,7 +8,7 @@ export declare class Streams {
     private _redisGroups?;
     private config;
     private dlq;
-    private metricsCollector;
+    private metricsCollector?;
     private consumerGroupName;
     private instanceId;
     private instanceUniqueId;

package/src/lib/redis/streams.js

@@ -128,6 +128,8 @@ class Streams {
             // New defaults
             optimizationDurationMs: 2 * 60 * 1000, // 2 minutes
             optimizationThreshold: 20, // Enable optimization for >20 consumer groups
+            maxPendingTasks: 1000,
+            enableMetrics: true,
         };
         /** Initialise Config properties */
         this.config = { ...this.config, ...this.DEFAULT_STREAMS_CONFIG, ...config };
@@ -143,11 +145,13 @@ class Streams {
         }, cleanUpInterval);
         logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Clean Up process setup for ${cleanUpInterval} ms`);
         this.dlq = new dlq_1.DeadLetterQueue(this.redisPublisher, config.dlqEventThreshold);
-        this.metricsCollector = new collector_1.MetricsCollector({
-            redisClient: this.redisPublisher,
-            collectionInterval: 60000,
-            retentionPeriod: 6 * 60 * 60 * 1000,
-        }, this.dlq);
+        if (this.config.enableMetrics !== false) {
+            this.metricsCollector = new collector_1.MetricsCollector({
+                redisClient: this.redisPublisher,
+                collectionInterval: 60000,
+                retentionPeriod: 6 * 60 * 60 * 1000,
+            }, this.dlq);
+        }
         this.duplicateChecker = new duplication_1.ContentBasedDeduplication(this.redisPublisher, this.config.duplicationCheckWindow);
         this.circuitBreaker = new circuit_breaker_1.CircuitBreaker(this.config.circuitBreaker, this.redisPublisher);
         if (this.config.circuitBreaker.enabled)
@@ -573,26 +577,19 @@ class Streams {
         if (!isNewSubscription) {
             return bs.asObservable().pipe((0, rxjs_1.skip)(1));
         }
-        const cleanupInterval = 10000; // 10 seconds
-        const timer = (0, rxjs_1.interval)(cleanupInterval).subscribe({
-            next: async () => {
-                try {
-                    await processMessage(this.redisGroups, '0', new tracker_1.MetricsTracker(), false);
-                }
-                catch (error) {
-                    logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error in running recurring cleanup task:', error);
-                }
-            },
-            error: (error) => {
-                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Fatal error in cleanup timer:', error);
-            },
-        });
+        // 10-second cleanup timer disabled — was re-enabled in v6.0.0 but causes
+        // double publishes and triggers unbounded pending retry spawning.
+        // Pending retries are driven by Pub/Sub notifications only.
+        const timer = { unsubscribe: () => { } };
         // Create observable with proper cleanup
         const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1), (0, rxjs_1.finalize)(() => {
             timer.unsubscribe();
             // Clean up subscription on completion
             this.removeSubscription(eventName, subscriptionId);
         }));
+        let activePendingTasks = 0;
+        let lastCapLogTime = 0;
+        const maxPendingTasks = this.config.maxPendingTasks ?? 1000;
         const processMessage = async (redisClient, messageId, tracker, multicast = false, processPending = false) => {
             // Skip processing if subscription was removed. This is needed because the processing is independent of the subscription
             if (!this.subscriptions.has(eventName)) {
@@ -786,12 +783,47 @@ class Streams {
                         unprocessedMessageIds.countOnThisConsumer > this.config.unprocessedMessageThreshold) {
                         logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Too many unprocessed events for ${streamName}: count: ${unprocessedMessageIds.count}`);
                     }
-                    // Process messages with rate limiting
-                    const processWithDelay = async (id, index) => {
-                        await new Promise((resolve) => setTimeout(resolve, index * getDelay(unprocessedMessageIds.countOnThisConsumer ?? 1)));
-                        await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
-                    };
-                    unprocessedMessageIds.messageIds.map((id, index) => processWithDelay(id, index));
+                    if (maxPendingTasks > 0) {
+                        const available = maxPendingTasks - activePendingTasks;
+                        if (available <= 0) {
+                            const now = Date.now();
+                            if (now - lastCapLogTime > 30000) {
+                                logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: Pending task cap reached (${activePendingTasks}/${maxPendingTasks}), skipping batch for ${streamName}`);
+                                lastCapLogTime = now;
+                            }
+                        }
+                        else {
+                            const batch = unprocessedMessageIds.messageIds.slice(0, available);
+                            batch.forEach((id, index) => {
+                                activePendingTasks++;
+                                void (async () => {
+                                    try {
+                                        await new Promise((resolve) => setTimeout(resolve, index * getDelay(unprocessedMessageIds.countOnThisConsumer ?? 1)));
+                                        await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
+                                    }
+                                    catch (err) {
+                                        logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in pending retry task for ${id}`, err);
+                                    }
+                                    finally {
+                                        activePendingTasks--;
+                                    }
+                                })();
+                            });
+                        }
+                    }
+                    else {
+                        unprocessedMessageIds.messageIds.forEach((id, index) => {
+                            void (async () => {
+                                try {
+                                    await new Promise((resolve) => setTimeout(resolve, index * getDelay(unprocessedMessageIds.countOnThisConsumer ?? 1)));
+                                    await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
+                                }
+                                catch (err) {
+                                    logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in pending retry task for ${id}`, err);
+                                }
+                            })();
+                        });
+                    }
                 }
             }
             catch (e) {
@@ -1003,6 +1035,8 @@ class Streams {
      * This will return you the stats of the publisher for the last 6 hours after cleaning
      */
     async getMetrics(startTime, endTime) {
+        if (!this.metricsCollector)
+            return [];
         return this.metricsCollector.getMetrics(startTime, endTime);
     }
     /**
@@ -1010,6 +1044,8 @@ class Streams {
      * This will return you the latest stats of the publisher
      */
     async getLatestMetrics() {
+        if (!this.metricsCollector)
+            return null;
         return this.metricsCollector.getLatestMetrics();
     }
     /**

package/src/lib/redis/types.d.ts

@@ -56,6 +56,8 @@ export interface IStreamsConfig {
     };
     optimizationDurationMs?: number;
     optimizationThreshold?: number;
+    maxPendingTasks?: number;
+    enableMetrics?: boolean;
 }
 export type TEventFilter<T> = (event: EventData<T, string>) => boolean;
 export interface ISubscription<T, TName extends string = string> {