@jetit/publisher 6.0.0 → 6.0.1

package/README.md

@@ -125,6 +125,7 @@ const config: Partial<IStreamsConfig> = {
         halfOpenStateMaxAttempts: 10,
         maxStoredEvents: 5000,
     },
+    maxPendingTasks: 1000, // Max concurrent pending retry tasks per event per consumer (default: 1000, 0 = unbounded)
 };
 
 const publisher = new Publisher('MyService', config);
@@ -358,6 +359,7 @@ The Circuit Breaker has three states:
 - Retry logic with exponential backoff for failed operations
 - Circuit Breaker to prevent overwhelming failed services
 - Dead Letter Queue (DLQ) for handling subscription failures
+- **Pending Retry Task Cap**: Limits the number of concurrent in-flight pending message retry tasks per event per consumer instance. Prevents unbounded memory growth and Redis saturation under sustained load. Configurable via `maxPendingTasks` (default: 1000). Set to `0` to disable the cap (unbounded, pre-6.0.1 behavior). When the cap is reached, a rate-limited warning is logged and skipped messages are retried on the next Pub/Sub trigger.
 - **Adaptive Redis Stream ID Generation (Publisher only)**: The default `Publisher` automatically switches to an optimized ID generation strategy using a Lua script when publishing to many consumer groups (>10 by default) or when ID conflicts are detected. This prevents `XADD` errors related to non-monotonic IDs in high-throughput scenarios. Configurable via `optimizationThreshold` and `optimizationDurationMs`. (`PublisherLite` does not include this complex logic).
   
 ## Cleanup and Graceful Shutdown

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "type": "commonjs",
   "peerDependencies": {
     "@jetit/id": ">=0.0.14",
@@ -8,8 +8,7 @@
     "rxjs": ">=7.0.0"
   },
   "dependencies": {
-    "nanoid": "^5.0.9",
-    "tslib": "2.7.0"
+    "tslib": "2.8.1"
   },
   "types": "./src/index.d.ts",
   "main": "./src/index.js"

package/src/lib/redis/streams.js

@@ -128,6 +128,7 @@ class Streams {
             // New defaults
             optimizationDurationMs: 2 * 60 * 1000, // 2 minutes
             optimizationThreshold: 20, // Enable optimization for >20 consumer groups
+            maxPendingTasks: 1000,
         };
         /** Initialise Config properties */
         this.config = { ...this.config, ...this.DEFAULT_STREAMS_CONFIG, ...config };
@@ -573,26 +574,19 @@ class Streams {
         if (!isNewSubscription) {
             return bs.asObservable().pipe((0, rxjs_1.skip)(1));
         }
-        const cleanupInterval = 10000; // 10 seconds
-        const timer = (0, rxjs_1.interval)(cleanupInterval).subscribe({
-            next: async () => {
-                try {
-                    await processMessage(this.redisGroups, '0', new tracker_1.MetricsTracker(), false);
-                }
-                catch (error) {
-                    logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error in running recurring cleanup task:', error);
-                }
-            },
-            error: (error) => {
-                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Fatal error in cleanup timer:', error);
-            },
-        });
+        // 10-second cleanup timer disabled — was re-enabled in v6.0.0 but causes
+        // double publishes and triggers unbounded pending retry spawning.
+        // Pending retries are driven by Pub/Sub notifications only.
+        const timer = { unsubscribe: () => { } };
         // Create observable with proper cleanup
         const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1), (0, rxjs_1.finalize)(() => {
             timer.unsubscribe();
             // Clean up subscription on completion
             this.removeSubscription(eventName, subscriptionId);
         }));
+        let activePendingTasks = 0;
+        let lastCapLogTime = 0;
+        const maxPendingTasks = this.config.maxPendingTasks ?? 1000;
         const processMessage = async (redisClient, messageId, tracker, multicast = false, processPending = false) => {
             // Skip processing if subscription was removed. This is needed because the processing is independent of the subscription
             if (!this.subscriptions.has(eventName)) {
@@ -786,12 +780,47 @@ class Streams {
                         unprocessedMessageIds.countOnThisConsumer > this.config.unprocessedMessageThreshold) {
                         logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Too many unprocessed events for ${streamName}: count: ${unprocessedMessageIds.count}`);
                     }
-                    // Process messages with rate limiting
-                    const processWithDelay = async (id, index) => {
-                        await new Promise((resolve) => setTimeout(resolve, index * getDelay(unprocessedMessageIds.countOnThisConsumer ?? 1)));
-                        await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
-                    };
-                    unprocessedMessageIds.messageIds.map((id, index) => processWithDelay(id, index));
+                    if (maxPendingTasks > 0) {
+                        const available = maxPendingTasks - activePendingTasks;
+                        if (available <= 0) {
+                            const now = Date.now();
+                            if (now - lastCapLogTime > 30000) {
+                                logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: Pending task cap reached (${activePendingTasks}/${maxPendingTasks}), skipping batch for ${streamName}`);
+                                lastCapLogTime = now;
+                            }
+                        }
+                        else {
+                            const batch = unprocessedMessageIds.messageIds.slice(0, available);
+                            batch.forEach((id, index) => {
+                                activePendingTasks++;
+                                void (async () => {
+                                    try {
+                                        await new Promise((resolve) => setTimeout(resolve, index * getDelay(unprocessedMessageIds.countOnThisConsumer ?? 1)));
+                                        await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
+                                    }
+                                    catch (err) {
+                                        logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in pending retry task for ${id}`, err);
+                                    }
+                                    finally {
+                                        activePendingTasks--;
+                                    }
+                                })();
+                            });
+                        }
+                    }
+                    else {
+                        unprocessedMessageIds.messageIds.forEach((id, index) => {
+                            void (async () => {
+                                try {
+                                    await new Promise((resolve) => setTimeout(resolve, index * getDelay(unprocessedMessageIds.countOnThisConsumer ?? 1)));
+                                    await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
+                                }
+                                catch (err) {
+                                    logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in pending retry task for ${id}`, err);
+                                }
+                            })();
+                        });
+                    }
                 }
             }
             catch (e) {

package/src/lib/redis/types.d.ts

@@ -56,6 +56,7 @@ export interface IStreamsConfig {
     };
     optimizationDurationMs?: number;
     optimizationThreshold?: number;
+    maxPendingTasks?: number;
 }
 export type TEventFilter<T> = (event: EventData<T, string>) => boolean;
 export interface ISubscription<T, TName extends string = string> {