@jetit/publisher 5.6.0 → 5.6.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "5.6.0",
+  "version": "5.6.1",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "^0.0.13",

package/src/lib/redis/scheduler.js

@@ -63,7 +63,7 @@ class ScheduledProcessor {
                     .xadd(streamName, key, 'data', JSON.stringify(eventData))
                     .catch((e) => logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Publishing event ${eventData.eventName} to consumer groups: ${consumerGroups.join(', ')} failed with data ${JSON.stringify(eventData)}, ${e} `));
                 if (key === '*')
-                    key = generatedKey ?? key;
+                    key = `${generatedKey ?? key}`;
             }
             if (eventData.repeatInterval) {
                 const nextEventTime = currentTime + eventData.repeatInterval;

package/src/lib/redis/streams.js

@@ -368,6 +368,17 @@ class Streams {
                      * they need to be read using XRANGE.
                      */
                     if (multicast || processPending) {
+                        /**
+                         * Very very rare case of this occurring. Cases where the running service is super overloaded
+                         * that causes a messaged to be sent to processing with a delay, but eventually before being
+                         * processed gets picked up by another instance, leading to multiple publications
+                         */
+                        if (processPending) {
+                            const claimed = await redisClient.xclaim(streamName, this.consumerGroupName, this.instanceId, 20000, messageId, 'JUSTID');
+                            if (!claimed || claimed.length === 0) {
+                                return; // Message already claimed or acknowledged by another consumer, so don't repush to the subscriber
+                            }
+                        }
                         const messages = await redisClient.xrange(streamName, messageId, messageId);
                         if (messages?.length) {
                             try {
@@ -469,13 +480,31 @@ class Streams {
                 /** Process Unprocessed Messages with rate limiting */
                 if (!processPending) {
                     const unprocessedMessageIds = await (0, utils_1.getUnacknowledgedMessages)(redisClient, this.consumerGroupName, streamName, this.instanceId);
+                    /**
+                     * Dealing with the case where messages don't get processed due to large
+                     * batch sizes. The previous default was 20ms, which seemed to work in
+                     * most cases but seem to fail when the number of retry messages are high.
+                     *
+                     * The old solution did not take into account the fact that this library
+                     * is dependent on the infrastructure of the app that runs it, so any memory/
+                     * resource/stack overload on the app has an impact in this
+                     */
+                    const getDelay = (count) => {
+                        if (count > 100)
+                            return 500; // 500ms for large backlogs
+                        if (count > 50)
+                            return 300; // 300ms for medium-large
+                        if (count > 20)
+                            return 200; // 200ms for medium
+                        return 100; // 100ms for small
+                    };
                     if (unprocessedMessageIds.countOnThisConsumer &&
                         unprocessedMessageIds.countOnThisConsumer > this.config.unprocessedMessageThreshold) {
                         logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Too many unprocessed events for ${streamName}: count: ${unprocessedMessageIds.count}`);
                     }
                     // Process messages with rate limiting
                     const processWithDelay = async (id, index) => {
-                        await new Promise((resolve) => setTimeout(resolve, index * 20));
+                        await new Promise((resolve) => setTimeout(resolve, index * getDelay(unprocessedMessageIds.countOnThisConsumer ?? 1)));
                         await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
                     };
                     unprocessedMessageIds.messageIds.map((id, index) => processWithDelay(id, index));