@jetit/publisher 5.6.0 → 5.6.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "5.6.0",
+  "version": "5.6.3",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "^0.0.13",

package/src/lib/monitoring/adapters/prom.js

@@ -10,7 +10,11 @@ class PrometheusAdapter {
     constructor(streams, promClient) {
         this.streams = streams;
         this.promClient = promClient;
-        this.registry = new this.promClient.Registry({ collectDefaultMetrics: { timeout: 60000 } });
+        this.registry = new this.promClient.Registry();
+        this.promClient.collectDefaultMetrics({
+            register: this.registry,
+            timeout: 60000,
+        });
         this.initializeMetrics();
     }
     initializeMetrics() {
@@ -159,10 +163,10 @@ class PrometheusAdapter {
             }
         });
         Object.entries(metrics.redisCommandLatencies).forEach(([command, latency]) => {
-            this.redisCommandLatency.set({ command }, latency);
+            this.redisCommandLatency.set({ command }, latency?.total / latency?.count || 0);
         });
         Object.entries(metrics.consumerLag).forEach(([consumerGroup, lag]) => {
-            this.consumerLag.set({ consumer_group: consumerGroup }, lag);
+            this.consumerLag.set({ consumer_group: consumerGroup }, lag?.total / lag?.count || 0);
         });
     }
     /**
@@ -171,7 +175,7 @@ class PrometheusAdapter {
     setupEndpoint(app, endPoint = '/metrics') {
         app.get(endPoint, async (_, res) => {
             await this.updateMetrics();
-            res.set('Content-Type', this.registry.contentType);
+            res.header('Content-Type', this.registry.contentType);
             res.send(await this.registry.metrics());
         });
     }

package/src/lib/monitoring/tracker.js

@@ -69,7 +69,7 @@ class MetricsTracker {
                 return i === 0 ? `0-${buckets[i]}` : `${buckets[i - 1] + 1}-${buckets[i]}`;
             }
         }
-        return `>${buckets[buckets.length - 1]}`;
+        return `10000-100000`;
     }
     addRedisCommandLatency(command, time) {
         if (!this.metrics.redisCommandLatencies[command]) {

package/src/lib/redis/scheduler.js

@@ -63,7 +63,7 @@ class ScheduledProcessor {
                     .xadd(streamName, key, 'data', JSON.stringify(eventData))
                     .catch((e) => logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Publishing event ${eventData.eventName} to consumer groups: ${consumerGroups.join(', ')} failed with data ${JSON.stringify(eventData)}, ${e} `));
                 if (key === '*')
-                    key = generatedKey ?? key;
+                    key = `${generatedKey ?? key}`;
             }
             if (eventData.repeatInterval) {
                 const nextEventTime = currentTime + eventData.repeatInterval;

package/src/lib/redis/streams.js

@@ -62,13 +62,14 @@ class Streams {
         this.instanceUniqueId = process.env['INSTANCE_ID'] ?? (0, id_1.generateID)('HEX', 'FE');
         this.instanceId = `${serviceName}:${this.instanceUniqueId}`;
         this.consumerGroupName = `cg-${serviceName}`;
-        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Instance ID: ${this.instanceId}`);
+        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Instance ID: ${this.instanceId} and with config: ${JSON.stringify(this.config)}`);
         const cleanUpInterval = this.config.cleanUpInterval ?? parseInt(process.env['CLEANUP_INTERVAL'] ?? `${this.config.cleanUpInterval}`, 10);
         this.cleanUpTimer = setInterval(() => {
             this.runClear(cleanUpInterval).catch((error) => {
-                logger_1.PUBLISHER_LOGGER.error('Error during cleanup:', error);
+                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error during cleanup:', error);
             });
         }, cleanUpInterval);
+        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Clean Up process setup for ${cleanUpInterval} ms`);
         this.dlq = new dlq_1.DeadLetterQueue(this.redisPublisher, config.dlqEventThreshold);
         this.metricsCollector = new collector_1.MetricsCollector({
             redisClient: this.redisPublisher,
@@ -82,7 +83,7 @@ class Streams {
     }
     setupCircuitBreakerListeners() {
         this.circuitBreaker.on('stateChange', async (newState) => {
-            logger_1.PUBLISHER_LOGGER.log(`Circuit breaker state changed to: ${circuit_breaker_1.CircuitState[newState]}`);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Circuit breaker state changed to: ${circuit_breaker_1.CircuitState[newState]}`);
             if (newState === circuit_breaker_1.CircuitState.CLOSED) {
                 await this.processStoredEvents();
             }
@@ -92,10 +93,10 @@ class Streams {
         logger_1.PUBLISHER_LOGGER.log('PUBLISHER: Running Clearance', this.eventsListened);
         const cleanupPromises = this.eventsListened.map((eventName) => this.cleanupAcknowledgedMessages(eventName, cleanUpInterval)
             .then(() => {
-            logger_1.PUBLISHER_LOGGER.log(`Cleanup process for Acknowledged messages completed for ${eventName}`);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Cleanup process for Acknowledged messages completed for ${eventName}`);
         })
             .catch((error) => {
-            logger_1.PUBLISHER_LOGGER.error(`Error during cleanup for ${eventName}:`, error);
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error during cleanup for ${eventName}:`, error);
         }));
         await Promise.all(cleanupPromises);
     }
@@ -133,7 +134,7 @@ class Streams {
          */
         if (this.config.circuitBreaker.enabled && !this.circuitBreaker.isAllowed()) {
             await this.circuitBreaker.storeEvent(data);
-            logger_1.PUBLISHER_LOGGER.error('Circuit is open, event stored for later processing');
+            logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Circuit is open, event stored for later processing');
             return 'CIRCUIT_BREAKER_FLOW';
         }
         try {
@@ -264,14 +265,14 @@ class Streams {
             // If both exist, this will be a no-op
             try {
                 await this.redisGroups.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
-                logger_1.PUBLISHER_LOGGER.log(`Group created for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Group created for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
             }
             catch (e) {
                 // BUSYGROUP error means group already exists, which is fine
                 if (!e.message.includes('BUSYGROUP')) {
                     throw e;
                 }
-                logger_1.PUBLISHER_LOGGER.log(`Group already exists for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Group already exists for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
             }
             // Create consumer (idempotent operation)
             const createConsumerStatus = await this.redisGroups.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId);
@@ -320,11 +321,11 @@ class Streams {
                     await processMessage(this.redisGroups, '0', new tracker_1.MetricsTracker(), false);
                 }
                 catch (error) {
-                    logger_1.PUBLISHER_LOGGER.error('Error in running recurring cleanup task:', error);
+                    logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error in running recurring cleanup task:', error);
                 }
             },
             error: (error) => {
-                logger_1.PUBLISHER_LOGGER.error('Fatal error in cleanup timer:', error);
+                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Fatal error in cleanup timer:', error);
             },
         });
         // Create observable with proper cleanup
@@ -368,6 +369,17 @@ class Streams {
                      * they need to be read using XRANGE.
                      */
                     if (multicast || processPending) {
+                        /**
+                         * Very very rare case of this occurring. Cases where the running service is super overloaded
+                         * that causes a messaged to be sent to processing with a delay, but eventually before being
+                         * processed gets picked up by another instance, leading to multiple publications
+                         */
+                        if (processPending) {
+                            const claimed = await redisClient.xclaim(streamName, this.consumerGroupName, this.instanceId, 10000, messageId, 'JUSTID');
+                            if (!claimed || claimed.length === 0) {
+                                return; // Message already claimed or acknowledged by another consumer, so don't repush to the subscriber
+                            }
+                        }
                         const messages = await redisClient.xrange(streamName, messageId, messageId);
                         if (messages?.length) {
                             try {
@@ -382,22 +394,30 @@ class Streams {
                     else {
                         const messages = (await redisClient.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'STREAMS', streamName, '>'));
                         if (messages?.length) {
-                            if (messageId === '0') {
-                                messageId = messages[0][1][0][0];
-                                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reprocessing unprocessed message with id: ${messageId}`);
+                            const messageIdRead = messages[0][1][0][0];
+                            if (messageIdRead !== messageId) {
+                                if (messageId === '0') {
+                                    logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reprocessing unprocessed message with id: ${messageId}`);
+                                }
+                                else {
+                                    const [timestamp] = messageIdRead?.split('-').map(Number);
+                                    const [publishedTimestamp] = messageId?.split('-').map(Number);
+                                    logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Message queue processing is ${publishedTimestamp > timestamp ? 'ahead' : 'behind'}`);
+                                }
                             }
+                            messageId = messageIdRead;
                             try {
                                 eventData = JSON.parse(messages[0][1][0][1][1]);
                             }
                             catch (error) {
-                                logger_1.PUBLISHER_LOGGER.error(`JSON parsing failed for message: ${messages[0][1][0][1][1]}`);
+                                logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: JSON parsing failed for message: ${messages[0][1][0][1][1]}`);
                                 return;
                             }
                         }
                     }
                 }
                 catch (error) {
-                    logger_1.PUBLISHER_LOGGER.error('Error retrieving or parsing event data:', error);
+                    logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error retrieving or parsing event data:', error);
                     return;
                 }
                 finally {
@@ -433,7 +453,7 @@ class Streams {
                                 }
                                 catch (error) {
                                     // Log error but don't fail entire processing
-                                    logger_1.PUBLISHER_LOGGER.error(`Error processing subscription ${subId}:`, error);
+                                    logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error processing subscription ${subId}:`, error);
                                 }
                             }));
                         }
@@ -449,7 +469,7 @@ class Streams {
                         tracker.setConsumerLag(this.consumerGroupName, currentTime - eventData.createdAt);
                     }
                     catch (error) {
-                        logger_1.PUBLISHER_LOGGER.error(`Processing error for message ${messageId}:`, error);
+                        logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Processing error for message ${messageId}:`, error);
                         const dlqEvent = {
                             ...eventData,
                             failureReason: error.message,
@@ -469,13 +489,31 @@ class Streams {
                 /** Process Unprocessed Messages with rate limiting */
                 if (!processPending) {
                     const unprocessedMessageIds = await (0, utils_1.getUnacknowledgedMessages)(redisClient, this.consumerGroupName, streamName, this.instanceId);
+                    /**
+                     * Dealing with the case where messages don't get processed due to large
+                     * batch sizes. The previous default was 20ms, which seemed to work in
+                     * most cases but seem to fail when the number of retry messages are high.
+                     *
+                     * The old solution did not take into account the fact that this library
+                     * is dependent on the infrastructure of the app that runs it, so any memory/
+                     * resource/stack overload on the app has an impact in this
+                     */
+                    const getDelay = (count) => {
+                        if (count > 100)
+                            return 500; // 500ms for large backlogs
+                        if (count > 50)
+                            return 300; // 300ms for medium-large
+                        if (count > 20)
+                            return 200; // 200ms for medium
+                        return 100; // 100ms for small
+                    };
                     if (unprocessedMessageIds.countOnThisConsumer &&
                         unprocessedMessageIds.countOnThisConsumer > this.config.unprocessedMessageThreshold) {
                         logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Too many unprocessed events for ${streamName}: count: ${unprocessedMessageIds.count}`);
                     }
                     // Process messages with rate limiting
                     const processWithDelay = async (id, index) => {
-                        await new Promise((resolve) => setTimeout(resolve, index * 20));
+                        await new Promise((resolve) => setTimeout(resolve, index * getDelay(unprocessedMessageIds.countOnThisConsumer ?? 1)));
                         await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
                     };
                     unprocessedMessageIds.messageIds.map((id, index) => processWithDelay(id, index));
@@ -573,7 +611,7 @@ class Streams {
             }
         }
         catch (error) {
-            logger_1.PUBLISHER_LOGGER.error('Error during cleanup:', error);
+            logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error during cleanup:', error);
         }
     }
     async cleanupAcknowledgedMessages(eventName, interval = this.config.acknowledgedMessageCleanupInterval) {
@@ -581,27 +619,34 @@ class Streams {
         const lastAckKey = `last_ack:${streamName}`;
         const oneHourAgo = Date.now() - interval;
         try {
-            // Get consumer group info to check if consumers are active
-            const groupInfo = (await this.redisGroups.xinfo('GROUPS', streamName));
-            // If no active consumers, leave stream as is
-            if (!groupInfo || !groupInfo.some((group) => group.consumers > 0)) {
-                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: No active consumers for ${streamName}, leaving stream as is`);
-                return;
-            }
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Started [cleanupAcknowledgedMessages] for ${streamName} and with cleanUpInterval set as ${interval}`);
             // Get last acknowledged message ID
             const lastAckId = await this.redisGroups.get(lastAckKey);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: [cleanupAcknowledgedMessages] {lastAckId: ${lastAckId}, lastAckKey: ${lastAckKey}} `);
             if (!lastAckId) {
-                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: No acknowledged messages for ${streamName}`);
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: [cleanupAcknowledgedMessages]No acknowledged messages for ${streamName}`);
                 return;
             }
             // Extract timestamp from message ID
             const [timestamp] = lastAckId.split('-').map(Number);
             const cleanupThreshold = Math.min(timestamp, oneHourAgo);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: [cleanupAcknowledgedMessages] {cleanUpThreshold: ${cleanupThreshold}} `);
+            // Get consumer group info to check if consumers are active
+            const groupInfo = (await this.redisGroups.xinfo('GROUPS', streamName))?.map(([, name, , consumers]) => ({
+                name: name,
+                consumers: consumers,
+            }));
+            // If no active consumers, leave stream as is
+            if (groupInfo?.length === 0 || !groupInfo?.some((group) => group.consumers > 0)) {
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: No active consumers for ${streamName}, leaving stream as is`);
+                return;
+            }
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: [cleanupAcknowledgedMessages] XTRIM to be called`);
             await this.redisGroups.xtrim(streamName, 'MINID', cleanupThreshold);
             logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Cleaned up messages before last acknowledged message ${timestamp} from ${streamName}`);
         }
         catch (error) {
-            logger_1.PUBLISHER_LOGGER.error(`Error during cleanup for ${streamName}:`, error);
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error during cleanup for ${streamName}:`, error);
         }
     }
     async getDiagnosticData(events) {
@@ -730,7 +775,7 @@ class Streams {
             ]);
         }
         catch (error) {
-            logger_1.PUBLISHER_LOGGER.error(`Error acknowledging message ${messageId} for ${streamName}:`, error);
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error acknowledging message ${messageId} for ${streamName}:`, error);
             throw error;
         }
     }