@jetit/publisher 6.0.1 → 6.0.3

package/README.md

@@ -126,6 +126,8 @@ const config: Partial<IStreamsConfig> = {
         maxStoredEvents: 5000,
     },
     maxPendingTasks: 1000, // Max concurrent pending retry tasks per event per consumer (default: 1000, 0 = unbounded)
+    enableMetrics: true, // Set to false to disable MetricsCollector and its periodic SCAN (default: true)
+    recurringCleanupInterval: 10000, // Interval (ms) for the recurring pending-retry timer (default: 10000)
 };
 
 const publisher = new Publisher('MyService', config);
@@ -359,7 +361,8 @@ The Circuit Breaker has three states:
 - Retry logic with exponential backoff for failed operations
 - Circuit Breaker to prevent overwhelming failed services
 - Dead Letter Queue (DLQ) for handling subscription failures
-- **Pending Retry Task Cap**: Limits the number of concurrent in-flight pending message retry tasks per event per consumer instance. Prevents unbounded memory growth and Redis saturation under sustained load. Configurable via `maxPendingTasks` (default: 1000). Set to `0` to disable the cap (unbounded, pre-6.0.1 behavior). When the cap is reached, a rate-limited warning is logged and skipped messages are retried on the next Pub/Sub trigger.
+- **Disable Metrics Collection**: Set `enableMetrics: false` to skip `MetricsCollector` initialization entirely. The collector runs `SCAN *:cg-* COUNT 100` every 60 seconds for queue depth — disabling it eliminates this periodic Redis overhead. Applies to both `Publisher` and `PublisherLite`. When disabled, `getMetrics()` returns `[]` and `getLatestMetrics()` returns `null`.
+- **Pending Retry Task Cap**: Limits the number of concurrent in-flight pending message retry tasks per event per consumer instance. Prevents unbounded memory growth and Redis saturation under sustained load. Configurable via `maxPendingTasks` (default: 1000). Set to `0` to disable the cap (unbounded, pre-6.0.1 behavior). When the cap is reached, a rate-limited warning is logged and skipped messages are retried on the next trigger (Pub/Sub notification or the 10-second cleanup timer).
 - **Adaptive Redis Stream ID Generation (Publisher only)**: The default `Publisher` automatically switches to an optimized ID generation strategy using a Lua script when publishing to many consumer groups (>10 by default) or when ID conflicts are detected. This prevents `XADD` errors related to non-monotonic IDs in high-throughput scenarios. Configurable via `optimizationThreshold` and `optimizationDurationMs`. (`PublisherLite` does not include this complex logic).
   
 ## Cleanup and Graceful Shutdown

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "6.0.1",
+  "version": "6.0.3",
   "type": "commonjs",
   "peerDependencies": {
     "@jetit/id": ">=0.0.14",

package/src/lib/redis/streams-lite.d.ts

@@ -8,7 +8,7 @@ export declare class StreamsLite {
     private _redisGroups?;
     private config;
     private dlq;
-    private metricsCollector;
+    private metricsCollector?;
     private consumerGroupName;
     private instanceId;
     private instanceUniqueId;

package/src/lib/redis/streams-lite.js

@@ -75,11 +75,13 @@ class StreamsLite {
         }, cleanUpInterval);
         logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Clean Up process setup for ${cleanUpInterval} ms`);
         this.dlq = new dlq_1.DeadLetterQueue(this.redisPublisher, config.dlqEventThreshold);
-        this.metricsCollector = new collector_1.MetricsCollector({
-            redisClient: this.redisPublisher,
-            collectionInterval: 60000,
-            retentionPeriod: 6 * 60 * 60 * 1000,
-        }, this.dlq);
+        if (this.config.enableMetrics !== false) {
+            this.metricsCollector = new collector_1.MetricsCollector({
+                redisClient: this.redisPublisher,
+                collectionInterval: 60000,
+                retentionPeriod: 6 * 60 * 60 * 1000,
+            }, this.dlq);
+        }
         this.duplicateChecker = new duplication_1.ContentBasedDeduplication(this.redisPublisher, this.config.duplicationCheckWindow);
         this.circuitBreaker = new circuit_breaker_1.CircuitBreaker(this.config.circuitBreaker, this.redisPublisher);
         if (this.config.circuitBreaker.enabled)
@@ -667,6 +669,8 @@ class StreamsLite {
      * This will return you the stats of the publisher for the last 6 hours after cleaning
      */
     async getMetrics(startTime, endTime) {
+        if (!this.metricsCollector)
+            return [];
         return this.metricsCollector.getMetrics(startTime, endTime);
     }
     /**
@@ -674,6 +678,8 @@ class StreamsLite {
      * This will return you the latest stats of the publisher
      */
     async getLatestMetrics() {
+        if (!this.metricsCollector)
+            return null;
         return this.metricsCollector.getLatestMetrics();
     }
     /**

package/src/lib/redis/streams.d.ts

@@ -8,7 +8,7 @@ export declare class Streams {
     private _redisGroups?;
     private config;
     private dlq;
-    private metricsCollector;
+    private metricsCollector?;
     private consumerGroupName;
     private instanceId;
     private instanceUniqueId;

package/src/lib/redis/streams.js

@@ -129,6 +129,8 @@ class Streams {
             optimizationDurationMs: 2 * 60 * 1000, // 2 minutes
             optimizationThreshold: 20, // Enable optimization for >20 consumer groups
             maxPendingTasks: 1000,
+            enableMetrics: true,
+            recurringCleanupInterval: 10000, // 10 seconds
         };
         /** Initialise Config properties */
         this.config = { ...this.config, ...this.DEFAULT_STREAMS_CONFIG, ...config };
@@ -144,11 +146,13 @@ class Streams {
         }, cleanUpInterval);
         logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Clean Up process setup for ${cleanUpInterval} ms`);
         this.dlq = new dlq_1.DeadLetterQueue(this.redisPublisher, config.dlqEventThreshold);
-        this.metricsCollector = new collector_1.MetricsCollector({
-            redisClient: this.redisPublisher,
-            collectionInterval: 60000,
-            retentionPeriod: 6 * 60 * 60 * 1000,
-        }, this.dlq);
+        if (this.config.enableMetrics !== false) {
+            this.metricsCollector = new collector_1.MetricsCollector({
+                redisClient: this.redisPublisher,
+                collectionInterval: 60000,
+                retentionPeriod: 6 * 60 * 60 * 1000,
+            }, this.dlq);
+        }
         this.duplicateChecker = new duplication_1.ContentBasedDeduplication(this.redisPublisher, this.config.duplicationCheckWindow);
         this.circuitBreaker = new circuit_breaker_1.CircuitBreaker(this.config.circuitBreaker, this.redisPublisher);
         if (this.config.circuitBreaker.enabled)
@@ -574,10 +578,20 @@ class Streams {
         if (!isNewSubscription) {
             return bs.asObservable().pipe((0, rxjs_1.skip)(1));
         }
-        // 10-second cleanup timer disabled — was re-enabled in v6.0.0 but causes
-        // double publishes and triggers unbounded pending retry spawning.
-        // Pending retries are driven by Pub/Sub notifications only.
-        const timer = { unsubscribe: () => { } };
+        const cleanupInterval = this.config.recurringCleanupInterval ?? 10000; // 10 seconds
+        const timer = (0, rxjs_1.interval)(cleanupInterval).subscribe({
+            next: async () => {
+                try {
+                    await processMessage(this.redisGroups, '0', new tracker_1.MetricsTracker(), false);
+                }
+                catch (error) {
+                    logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error in running recurring cleanup task:', error);
+                }
+            },
+            error: (error) => {
+                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Fatal error in cleanup timer:', error);
+            },
+        });
         // Create observable with proper cleanup
         const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1), (0, rxjs_1.finalize)(() => {
             timer.unsubscribe();
@@ -1032,6 +1046,8 @@ class Streams {
      * This will return you the stats of the publisher for the last 6 hours after cleaning
      */
     async getMetrics(startTime, endTime) {
+        if (!this.metricsCollector)
+            return [];
         return this.metricsCollector.getMetrics(startTime, endTime);
     }
     /**
@@ -1039,6 +1055,8 @@ class Streams {
      * This will return you the latest stats of the publisher
      */
     async getLatestMetrics() {
+        if (!this.metricsCollector)
+            return null;
         return this.metricsCollector.getLatestMetrics();
     }
     /**

package/src/lib/redis/types.d.ts

@@ -57,6 +57,8 @@ export interface IStreamsConfig {
     optimizationDurationMs?: number;
     optimizationThreshold?: number;
     maxPendingTasks?: number;
+    enableMetrics?: boolean;
+    recurringCleanupInterval?: number;
 }
 export type TEventFilter<T> = (event: EventData<T, string>) => boolean;
 export interface ISubscription<T, TName extends string = string> {