@jetit/publisher 5.6.3 → 6.0.1

package/src/lib/redis/streams.d.ts

@@ -17,6 +17,9 @@ export declare class Streams {
     private subscriptions;
     private duplicateChecker;
     private circuitBreaker;
+    private readonly findHighestStreamIdScript;
+    private findHighestStreamIdScriptSHA;
+    private optimizationActiveUntil;
     private get redisPublisher();
     private get redisGroups();
     private DEFAULT_STREAMS_CONFIG;
@@ -35,6 +38,27 @@ export declare class Streams {
      * const streams = new Streams('POS');
      */
     constructor(serviceName: string, config?: Partial<IStreamsConfig>, redisConnectionId?: string);
+    /**
+     * Loads Lua scripts into Redis server cache
+     */
+    private loadScripts;
+    /**
+     * Executes a cached script, falling back to EVAL if needed
+     */
+    private executeScript;
+    /**
+     * Checks if optimization mode is active for a given event type.
+     * Cleans up expired entries during the check.
+     */
+    private isOptimizationActive;
+    /**
+     * Activates optimization mode for a given event type for a configured duration.
+     */
+    private activateOptimizationFor;
+    /**
+     * Deletes a specific message ID from multiple consumer group streams.
+     */
+    private deleteMessageFromStreams;
     private setupCircuitBreakerListeners;
     private runClear;
     /**

package/src/lib/redis/streams.js

@@ -40,6 +40,75 @@ class Streams {
         this.redisConnectionId = redisConnectionId;
         this.eventsListened = [];
         this.subscriptions = new Map();
+        // Add these properties to store the script SHA and optimization state
+        this.findHighestStreamIdScript = `
+    -- KEYS[1]: event name
+    -- KEYS[2]: maximum streams to check
+    -- ARGV[1..n]: consumer group names
+
+    local eventName = KEYS[1]
+    local maxStreamsToCheck = tonumber(KEYS[2])
+    local highestTimestamp = 0
+    local highestSeq = 0
+
+    -- Calculate how many streams to check (all or max)
+    local streamsToCheck = math.min(#ARGV, maxStreamsToCheck)
+
+    -- If we have more streams than we want to check, use sampling
+    local streamIndices = {}
+    if streamsToCheck < #ARGV then
+        -- Deterministic sampling (every Nth stream)
+        local step = math.floor(#ARGV / streamsToCheck)
+        local offset = math.floor(step / 2) -- Start in the middle of each segment
+
+        for i=1, streamsToCheck do
+            local idx = offset + (i-1) * step
+            if idx <= #ARGV then
+                table.insert(streamIndices, idx)
+            end
+        end
+
+        -- Ensure we also check the last stream in case it's ahead
+        if #streamIndices == 0 or streamIndices[#streamIndices] ~= #ARGV then
+             if #ARGV > 0 then table.insert(streamIndices, #ARGV) end
+        end
+    else
+        -- Check all streams
+        for i=1, #ARGV do
+            streamIndices[i] = i
+        end
+    end
+
+    -- Check selected streams
+    for _, i in ipairs(streamIndices) do
+        local streamName = eventName .. ":" .. ARGV[i]
+        local latestEntries = redis.call('XREVRANGE', streamName, '+', '-', 'COUNT', 1)
+
+        if #latestEntries > 0 then
+            local id = latestEntries[1][1]
+            local dashPos = string.find(id, '-')
+
+            if dashPos then
+                local timestamp = tonumber(string.sub(id, 1, dashPos - 1))
+                local seq = tonumber(string.sub(id, dashPos + 1))
+
+                if timestamp > highestTimestamp then
+                    highestTimestamp = timestamp
+                    highestSeq = seq
+                elseif timestamp == highestTimestamp and seq > highestSeq then
+                    highestSeq = seq
+                end
+            end
+        end
+    end
+
+    -- Add a safety buffer to the sequence number
+    highestSeq = highestSeq + 10
+
+    return { highestTimestamp, highestSeq }
+    `;
+        this.findHighestStreamIdScriptSHA = null;
+        this.optimizationActiveUntil = new Map(); // Tracks optimization state per event
         this.DEFAULT_STREAMS_CONFIG = {
             immediatePublishThreshold: 500,
             unprocessedMessageThreshold: 25,
@@ -56,6 +125,10 @@ class Streams {
                 halfOpenStateMaxAttempts: 10,
                 maxStoredEvents: 10000,
             },
+            // New defaults
+            optimizationDurationMs: 2 * 60 * 1000, // 2 minutes
+            optimizationThreshold: 20, // Enable optimization for >20 consumer groups
+            maxPendingTasks: 1000,
         };
         /** Initialise Config properties */
         this.config = { ...this.config, ...this.DEFAULT_STREAMS_CONFIG, ...config };
@@ -80,6 +153,120 @@ class Streams {
         this.circuitBreaker = new circuit_breaker_1.CircuitBreaker(this.config.circuitBreaker, this.redisPublisher);
         if (this.config.circuitBreaker.enabled)
             this.setupCircuitBreakerListeners();
+        // Load scripts when Redis is available
+        this.loadScripts().catch((error) => {
+            logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Failed to load Redis scripts:', error);
+        });
+    }
+    /**
+     * Loads Lua scripts into Redis server cache
+     */
+    async loadScripts() {
+        try {
+            // Wait until Redis connection is established (simple check)
+            if (!this._redisPublisher) {
+                await new Promise((resolve) => setTimeout(resolve, 100)); // Wait briefly
+                if (!this._redisPublisher) {
+                    logger_1.PUBLISHER_LOGGER.warn('PUBLISHER: Redis connection not ready for script loading.');
+                    return;
+                }
+            }
+            const redis = this.redisPublisher;
+            // Load the script and store its SHA
+            this.findHighestStreamIdScriptSHA = (await redis.script('LOAD', this.findHighestStreamIdScript));
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Successfully loaded stream ID script with SHA: ${this.findHighestStreamIdScriptSHA}`);
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error loading Redis scripts:', error);
+            // We'll try again later if needed via executeScript fallback
+        }
+    }
+    /**
+     * Executes a cached script, falling back to EVAL if needed
+     */
+    async executeScript(script, scriptSHA, numKeys, ...args) {
+        try {
+            // If we have the SHA, try EVALSHA first
+            if (scriptSHA) {
+                try {
+                    // Ensure redisPublisher is available
+                    if (!this._redisPublisher)
+                        throw new Error('Redis publisher connection not available');
+                    return await this.redisPublisher.evalsha(scriptSHA, numKeys, ...args);
+                }
+                catch (error) {
+                    // Add type 'any' to error
+                    // If the script isn't found on the server, fall back to EVAL
+                    if (error?.message && typeof error.message === 'string' && error.message.includes('NOSCRIPT')) {
+                        logger_1.PUBLISHER_LOGGER.warn('PUBLISHER: Script not found on Redis server, reloading...');
+                        if (!this._redisPublisher)
+                            throw new Error('Redis publisher connection not available for script reload');
+                        const newSHA = (await this.redisPublisher.script('LOAD', script)); // Cast result to string
+                        // Update the stored SHA
+                        if (script === this.findHighestStreamIdScript) {
+                            this.findHighestStreamIdScriptSHA = newSHA;
+                        }
+                        // Retry with EVALSHA
+                        return await this.redisPublisher.evalsha(newSHA, numKeys, ...args);
+                    }
+                    throw error;
+                }
+            }
+            // If we don't have the SHA yet, use EVAL directly
+            if (!this._redisPublisher)
+                throw new Error('Redis publisher connection not available for EVAL');
+            return await this.redisPublisher.eval(script, numKeys, ...args);
+        }
+        catch (error) {
+            // Add type 'any' to error
+            logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error executing Redis script:', error);
+            throw error; // Re-throw the original error
+        }
+    }
+    /**
+     * Checks if optimization mode is active for a given event type.
+     * Cleans up expired entries during the check.
+     */
+    isOptimizationActive(eventName) {
+        const expirationTime = this.optimizationActiveUntil.get(eventName);
+        if (!expirationTime)
+            return false;
+        if (Date.now() > expirationTime) {
+            // Clean up expired entry during publish operation
+            this.optimizationActiveUntil.delete(eventName);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Deactivated optimization for ${eventName}`);
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Activates optimization mode for a given event type for a configured duration.
+     */
+    activateOptimizationFor(eventName) {
+        const durationMs = this.config.optimizationDurationMs ?? this.DEFAULT_STREAMS_CONFIG.optimizationDurationMs;
+        const expirationTime = Date.now() + durationMs;
+        this.optimizationActiveUntil.set(eventName, expirationTime);
+        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Activated optimization for ${eventName} until ${new Date(expirationTime).toISOString()}`);
+    }
+    /**
+     * Deletes a specific message ID from multiple consumer group streams.
+     */
+    async deleteMessageFromStreams(eventName, consumerGroups, messageId) {
+        const deletionPromises = consumerGroups.map(async (consumerGroup) => {
+            const streamName = `${eventName}:${consumerGroup}`;
+            try {
+                if (!this._redisPublisher)
+                    throw new Error('Redis publisher connection not available for XDEL');
+                await this.redisPublisher.xdel(streamName, messageId);
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Deleted message ${messageId} from ${streamName} for rollback`);
+            }
+            catch (error) {
+                // Log error but continue trying to delete from other streams
+                logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error deleting message ${messageId} from ${streamName}:`, error);
+            }
+        });
+        // Wait for all deletions to attempt completion
+        await Promise.allSettled(deletionPromises);
     }
     setupCircuitBreakerListeners() {
         this.circuitBreaker.on('stateChange', async (newState) => {
@@ -141,25 +328,98 @@ class Streams {
             tracker.startRedisOperation();
             const consumerGroups = await (0, utils_1.getAllConsumerGroups)(data.eventName, this.redisPublisher);
             tracker.endRedisOperation();
-            let key = '*';
+            let key = '*'; // Default to auto-generated ID
+            const publishedGroups = []; // Track successfully published groups for rollback
             if (consumerGroups.length > 0) {
                 logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
-                for (const consumerGroup of consumerGroups) {
-                    // Publish the event to each consumer group's stream
-                    const streamName = `${data.eventName}:${consumerGroup}`;
+                // Check if optimization is already active or if threshold is met
+                const optimizationThreshold = this.config.optimizationThreshold ?? this.DEFAULT_STREAMS_CONFIG.optimizationThreshold;
+                const useOptimization = this.isOptimizationActive(data.eventName) || consumerGroups.length > optimizationThreshold;
+                // If optimization is needed, generate the optimized ID upfront
+                if (useOptimization) {
+                    const maxStreamsToCheck = Math.max(5, Math.ceil(consumerGroups.length * 0.25));
                     tracker.startRedisOperation();
-                    const generatedKey = await this.redisPublisher.xadd(streamName, key, 'data', JSON.stringify(data));
-                    tracker.endRedisOperation();
-                    tracker.incrementEventCount();
-                    tracker.incrementMessageRate('publish', data.eventName);
-                    if (this.metricsCollector) {
-                        this.metricsCollector.addMetrics(tracker.getMetrics());
+                    try {
+                        const result = (await this.executeScript(this.findHighestStreamIdScript, this.findHighestStreamIdScriptSHA, 2, // Two keys
+                        data.eventName, maxStreamsToCheck.toString(), ...consumerGroups));
+                        const [highestTimestamp, highestSeq] = result;
+                        const currentTime = Date.now();
+                        if (currentTime > highestTimestamp) {
+                            key = `${currentTime}-0`;
+                        }
+                        else {
+                            key = `${highestTimestamp}-${highestSeq}`;
+                        }
+                        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Using optimized ID generation: ${key}`);
+                    }
+                    catch (scriptError) {
+                        logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error in optimized ID generation script, falling back to auto-generated ID:', scriptError);
+                        key = '*';
+                    }
+                    finally {
+                        tracker.endRedisOperation();
                     }
-                    if (key === '*')
-                        key = generatedKey ?? key;
                 }
-                await (0, utils_1.notifySubscribers)(this.redisPublisher, data.eventName, key, multicast);
-                await this.circuitBreaker.recordSuccess();
+                // Publish to each consumer group
+                try {
+                    for (const consumerGroup of consumerGroups) {
+                        const streamName = `${data.eventName}:${consumerGroup}`;
+                        tracker.startRedisOperation();
+                        try {
+                            const generatedKey = await this.redisPublisher.xadd(streamName, key, 'data', JSON.stringify(data));
+                            tracker.endRedisOperation();
+                            // If using auto-generated ID, capture it for the first stream
+                            if (key === '*')
+                                key = generatedKey ?? key;
+                            // Track successful publish
+                            publishedGroups.push(consumerGroup);
+                        }
+                        catch (error) {
+                            tracker.endRedisOperation(); // Ensure tracker ends even on error
+                            // Check if the error is the ID conflict error
+                            if ((typeof error === 'string' && error.includes('equal or smaller than the target stream top item')) ||
+                                (error?.message &&
+                                    typeof error.message === 'string' &&
+                                    error.message.includes('equal or smaller than the target stream top item'))) {
+                                logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: ID conflict detected for ${streamName} (key: ${key}), switching to optimized mode for ${data.eventName} for duration: ${this.config.optimizationDurationMs ?? this.DEFAULT_STREAMS_CONFIG.optimizationDurationMs / 1000}. Consumers: ${consumerGroups.length}`);
+                                // Delete already published messages if any (using the captured key)
+                                if (publishedGroups.length > 0) {
+                                    await this.deleteMessageFromStreams(data.eventName, publishedGroups, key);
+                                    publishedGroups.length = 0;
+                                }
+                                // Activate optimization for this event type
+                                this.activateOptimizationFor(data.eventName);
+                                // Recursively call publish with same data (optimization will be active now)
+                                // Important: Return the result of the recursive call
+                                return this.publish(data, multicast);
+                            }
+                            else {
+                                // For other XADD errors, rethrow to trigger the outer catch block
+                                logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Unhandled XADD error for ${streamName} (key: ${key}): ${error?.message}`);
+                                throw error;
+                            }
+                        }
+                        // Increment metrics only after successful publish
+                        tracker.incrementEventCount();
+                        tracker.incrementMessageRate('publish', data.eventName);
+                        if (this.metricsCollector) {
+                            this.metricsCollector.addMetrics(tracker.getMetrics());
+                        }
+                    } // End of for loop
+                    // Notify subscribers only after all groups are successfully published (or retried)
+                    await (0, utils_1.notifySubscribers)(this.redisPublisher, data.eventName, key, multicast);
+                    await this.circuitBreaker.recordSuccess();
+                }
+                catch (publishError) {
+                    // If we failed during the loop (and it wasn't handled by the adaptive logic),
+                    // try to clean up any messages already sent in this attempt.
+                    if (publishedGroups.length > 0) {
+                        logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error during multi-group publish for ${data.eventName}, rolling back published messages for key ${key}, publishedMessage: ${JSON.stringify(data)}`);
+                        await this.deleteMessageFromStreams(data.eventName, publishedGroups, key);
+                    }
+                    // Rethrow the error that caused the failure
+                    throw publishError;
+                }
             }
             else {
                 logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Event publish failed for event ${data.eventName}, reason: no consumers ${consumerGroups}`);
@@ -314,26 +574,19 @@ class Streams {
         if (!isNewSubscription) {
             return bs.asObservable().pipe((0, rxjs_1.skip)(1));
         }
-        const cleanupInterval = 10000; // 10 seconds
-        const timer = (0, rxjs_1.interval)(cleanupInterval).subscribe({
-            next: async () => {
-                try {
-                    await processMessage(this.redisGroups, '0', new tracker_1.MetricsTracker(), false);
-                }
-                catch (error) {
-                    logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error in running recurring cleanup task:', error);
-                }
-            },
-            error: (error) => {
-                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Fatal error in cleanup timer:', error);
-            },
-        });
+        // 10-second cleanup timer disabled — was re-enabled in v6.0.0 but causes
+        // double publishes and triggers unbounded pending retry spawning.
+        // Pending retries are driven by Pub/Sub notifications only.
+        const timer = { unsubscribe: () => { } };
         // Create observable with proper cleanup
         const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1), (0, rxjs_1.finalize)(() => {
             timer.unsubscribe();
             // Clean up subscription on completion
             this.removeSubscription(eventName, subscriptionId);
         }));
+        let activePendingTasks = 0;
+        let lastCapLogTime = 0;
+        const maxPendingTasks = this.config.maxPendingTasks ?? 1000;
         const processMessage = async (redisClient, messageId, tracker, multicast = false, processPending = false) => {
             // Skip processing if subscription was removed. This is needed because the processing is independent of the subscription
             if (!this.subscriptions.has(eventName)) {
@@ -348,8 +601,8 @@ class Streams {
                 // 3. Pending message reprocessing
                 if (!multicast && messageId !== '0' && !processPending) {
                     try {
-                        const pendingDetails = await redisClient.xpending(streamName, this.consumerGroupName, messageId, messageId, 1);
-                        if (pendingDetails[2] === 0) {
+                        const pendingDetails = (await redisClient.xpending(streamName, this.consumerGroupName, messageId, messageId, 1));
+                        if (pendingDetails && pendingDetails.length > 0 && pendingDetails[0] && pendingDetails[0][3] === 0) {
                             logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: Message ${messageId} already processed for ${streamName}`);
                             return;
                         }
@@ -372,12 +625,28 @@ class Streams {
                         /**
                          * Very very rare case of this occurring. Cases where the running service is super overloaded
                          * that causes a messaged to be sent to processing with a delay, but eventually before being
-                         * processed gets picked up by another instance, leading to multiple publications
+                         * processed gets picked up by another instance, leading to multiple publications. This makes
+                         * sure that there is a check to ensure that the message belongs to this consumer before being
+                         * processed
                          */
                         if (processPending) {
-                            const claimed = await redisClient.xclaim(streamName, this.consumerGroupName, this.instanceId, 10000, messageId, 'JUSTID');
-                            if (!claimed || claimed.length === 0) {
-                                return; // Message already claimed or acknowledged by another consumer, so don't repush to the subscriber
+                            const pendingDetails = (await redisClient.xpending(streamName, this.consumerGroupName, messageId, messageId, 1));
+                            /** Message is not in PEL */
+                            if (!pendingDetails || pendingDetails.length === 0)
+                                return;
+                            const [, instanceId, idleTime, deliveryCount] = pendingDetails[0][1];
+                            /** Message is claimed by another consumer within the group, so do not process */
+                            if (instanceId !== this.instanceId) {
+                                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Process Pending message is claimed by another consumer : ${JSON.stringify({
+                                    instanceId,
+                                    idleTime,
+                                    deliveryCount,
+                                    messageId,
+                                    streamName,
+                                    originalInstanceId: this.instanceId,
+                                    consumerGroupName: this.consumerGroupName,
+                                })}`);
+                                return;
                             }
                         }
                         const messages = await redisClient.xrange(streamName, messageId, messageId);
@@ -511,12 +780,47 @@ class Streams {
                         unprocessedMessageIds.countOnThisConsumer > this.config.unprocessedMessageThreshold) {
                         logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Too many unprocessed events for ${streamName}: count: ${unprocessedMessageIds.count}`);
                     }
-                    // Process messages with rate limiting
-                    const processWithDelay = async (id, index) => {
-                        await new Promise((resolve) => setTimeout(resolve, index * getDelay(unprocessedMessageIds.countOnThisConsumer ?? 1)));
-                        await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
-                    };
-                    unprocessedMessageIds.messageIds.map((id, index) => processWithDelay(id, index));
+                    if (maxPendingTasks > 0) {
+                        const available = maxPendingTasks - activePendingTasks;
+                        if (available <= 0) {
+                            const now = Date.now();
+                            if (now - lastCapLogTime > 30000) {
+                                logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: Pending task cap reached (${activePendingTasks}/${maxPendingTasks}), skipping batch for ${streamName}`);
+                                lastCapLogTime = now;
+                            }
+                        }
+                        else {
+                            const batch = unprocessedMessageIds.messageIds.slice(0, available);
+                            batch.forEach((id, index) => {
+                                activePendingTasks++;
+                                void (async () => {
+                                    try {
+                                        await new Promise((resolve) => setTimeout(resolve, index * getDelay(unprocessedMessageIds.countOnThisConsumer ?? 1)));
+                                        await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
+                                    }
+                                    catch (err) {
+                                        logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in pending retry task for ${id}`, err);
+                                    }
+                                    finally {
+                                        activePendingTasks--;
+                                    }
+                                })();
+                            });
+                        }
+                    }
+                    else {
+                        unprocessedMessageIds.messageIds.forEach((id, index) => {
+                            void (async () => {
+                                try {
+                                    await new Promise((resolve) => setTimeout(resolve, index * getDelay(unprocessedMessageIds.countOnThisConsumer ?? 1)));
+                                    await processMessage(redisClient, id, new tracker_1.MetricsTracker(), multicast, true);
+                                }
+                                catch (err) {
+                                    logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in pending retry task for ${id}`, err);
+                                }
+                            })();
+                        });
+                    }
                 }
             }
             catch (e) {
@@ -597,6 +901,9 @@ class Streams {
      */
     async close() {
         try {
+            // Reset optimization state and script SHA
+            this.optimizationActiveUntil.clear();
+            this.findHighestStreamIdScriptSHA = null;
             if (this.cleanUpTimer) {
                 clearInterval(this.cleanUpTimer);
             }

package/src/lib/redis/types.d.ts

@@ -54,6 +54,9 @@ export interface IStreamsConfig {
         halfOpenStateMaxAttempts: number;
         maxStoredEvents: number;
     };
+    optimizationDurationMs?: number;
+    optimizationThreshold?: number;
+    maxPendingTasks?: number;
 }
 export type TEventFilter<T> = (event: EventData<T, string>) => boolean;
 export interface ISubscription<T, TName extends string = string> {

package/src/lib/redis/utils.js

@@ -12,7 +12,7 @@ exports.decodeScheduledMessage = decodeScheduledMessage;
 const logger_1 = require("./logger");
 async function getAllConsumerGroups(eventName, redisConnection) {
     const consumerGroups = await redisConnection.smembers(`${eventName}`);
-    return consumerGroups;
+    return consumerGroups.sort();
 }
 function* getSummaryOnStreamConsumerGroup(redisClient, consumerGroupName, streamName) {
     const [count, , , consumers] = (yield redisClient.xpending(streamName, consumerGroupName));