@jetit/publisher 5.6.1 → 6.0.0

package/src/lib/redis/streams.d.ts

@@ -17,6 +17,9 @@ export declare class Streams {
     private subscriptions;
     private duplicateChecker;
     private circuitBreaker;
+    private readonly findHighestStreamIdScript;
+    private findHighestStreamIdScriptSHA;
+    private optimizationActiveUntil;
     private get redisPublisher();
     private get redisGroups();
     private DEFAULT_STREAMS_CONFIG;
@@ -35,6 +38,27 @@ export declare class Streams {
      * const streams = new Streams('POS');
      */
     constructor(serviceName: string, config?: Partial<IStreamsConfig>, redisConnectionId?: string);
+    /**
+     * Loads Lua scripts into Redis server cache
+     */
+    private loadScripts;
+    /**
+     * Executes a cached script, falling back to EVAL if needed
+     */
+    private executeScript;
+    /**
+     * Checks if optimization mode is active for a given event type.
+     * Cleans up expired entries during the check.
+     */
+    private isOptimizationActive;
+    /**
+     * Activates optimization mode for a given event type for a configured duration.
+     */
+    private activateOptimizationFor;
+    /**
+     * Deletes a specific message ID from multiple consumer group streams.
+     */
+    private deleteMessageFromStreams;
     private setupCircuitBreakerListeners;
     private runClear;
     /**

package/src/lib/redis/streams.js

@@ -40,6 +40,75 @@ class Streams {
         this.redisConnectionId = redisConnectionId;
         this.eventsListened = [];
         this.subscriptions = new Map();
+        // Add these properties to store the script SHA and optimization state
+        this.findHighestStreamIdScript = `
+    -- KEYS[1]: event name
+    -- KEYS[2]: maximum streams to check
+    -- ARGV[1..n]: consumer group names
+
+    local eventName = KEYS[1]
+    local maxStreamsToCheck = tonumber(KEYS[2])
+    local highestTimestamp = 0
+    local highestSeq = 0
+
+    -- Calculate how many streams to check (all or max)
+    local streamsToCheck = math.min(#ARGV, maxStreamsToCheck)
+
+    -- If we have more streams than we want to check, use sampling
+    local streamIndices = {}
+    if streamsToCheck < #ARGV then
+        -- Deterministic sampling (every Nth stream)
+        local step = math.floor(#ARGV / streamsToCheck)
+        local offset = math.floor(step / 2) -- Start in the middle of each segment
+
+        for i=1, streamsToCheck do
+            local idx = offset + (i-1) * step
+            if idx <= #ARGV then
+                table.insert(streamIndices, idx)
+            end
+        end
+
+        -- Ensure we also check the last stream in case it's ahead
+        if #streamIndices == 0 or streamIndices[#streamIndices] ~= #ARGV then
+             if #ARGV > 0 then table.insert(streamIndices, #ARGV) end
+        end
+    else
+        -- Check all streams
+        for i=1, #ARGV do
+            streamIndices[i] = i
+        end
+    end
+
+    -- Check selected streams
+    for _, i in ipairs(streamIndices) do
+        local streamName = eventName .. ":" .. ARGV[i]
+        local latestEntries = redis.call('XREVRANGE', streamName, '+', '-', 'COUNT', 1)
+
+        if #latestEntries > 0 then
+            local id = latestEntries[1][1]
+            local dashPos = string.find(id, '-')
+
+            if dashPos then
+                local timestamp = tonumber(string.sub(id, 1, dashPos - 1))
+                local seq = tonumber(string.sub(id, dashPos + 1))
+
+                if timestamp > highestTimestamp then
+                    highestTimestamp = timestamp
+                    highestSeq = seq
+                elseif timestamp == highestTimestamp and seq > highestSeq then
+                    highestSeq = seq
+                end
+            end
+        end
+    end
+
+    -- Add a safety buffer to the sequence number
+    highestSeq = highestSeq + 10
+
+    return { highestTimestamp, highestSeq }
+    `;
+        this.findHighestStreamIdScriptSHA = null;
+        this.optimizationActiveUntil = new Map(); // Tracks optimization state per event
         this.DEFAULT_STREAMS_CONFIG = {
             immediatePublishThreshold: 500,
             unprocessedMessageThreshold: 25,
@@ -56,19 +125,23 @@ class Streams {
                 halfOpenStateMaxAttempts: 10,
                 maxStoredEvents: 10000,
             },
+            // New defaults
+            optimizationDurationMs: 2 * 60 * 1000, // 2 minutes
+            optimizationThreshold: 20, // Enable optimization for >20 consumer groups
         };
         /** Initialise Config properties */
         this.config = { ...this.config, ...this.DEFAULT_STREAMS_CONFIG, ...config };
         this.instanceUniqueId = process.env['INSTANCE_ID'] ?? (0, id_1.generateID)('HEX', 'FE');
         this.instanceId = `${serviceName}:${this.instanceUniqueId}`;
         this.consumerGroupName = `cg-${serviceName}`;
-        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Instance ID: ${this.instanceId}`);
+        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Instance ID: ${this.instanceId} and with config: ${JSON.stringify(this.config)}`);
         const cleanUpInterval = this.config.cleanUpInterval ?? parseInt(process.env['CLEANUP_INTERVAL'] ?? `${this.config.cleanUpInterval}`, 10);
         this.cleanUpTimer = setInterval(() => {
             this.runClear(cleanUpInterval).catch((error) => {
-                logger_1.PUBLISHER_LOGGER.error('Error during cleanup:', error);
+                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error during cleanup:', error);
             });
         }, cleanUpInterval);
+        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Clean Up process setup for ${cleanUpInterval} ms`);
         this.dlq = new dlq_1.DeadLetterQueue(this.redisPublisher, config.dlqEventThreshold);
         this.metricsCollector = new collector_1.MetricsCollector({
             redisClient: this.redisPublisher,
@@ -79,10 +152,124 @@ class Streams {
         this.circuitBreaker = new circuit_breaker_1.CircuitBreaker(this.config.circuitBreaker, this.redisPublisher);
         if (this.config.circuitBreaker.enabled)
             this.setupCircuitBreakerListeners();
+        // Load scripts when Redis is available
+        this.loadScripts().catch((error) => {
+            logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Failed to load Redis scripts:', error);
+        });
+    }
+    /**
+     * Loads Lua scripts into Redis server cache
+     */
+    async loadScripts() {
+        try {
+            // Wait until Redis connection is established (simple check)
+            if (!this._redisPublisher) {
+                await new Promise((resolve) => setTimeout(resolve, 100)); // Wait briefly
+                if (!this._redisPublisher) {
+                    logger_1.PUBLISHER_LOGGER.warn('PUBLISHER: Redis connection not ready for script loading.');
+                    return;
+                }
+            }
+            const redis = this.redisPublisher;
+            // Load the script and store its SHA
+            this.findHighestStreamIdScriptSHA = (await redis.script('LOAD', this.findHighestStreamIdScript));
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Successfully loaded stream ID script with SHA: ${this.findHighestStreamIdScriptSHA}`);
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error loading Redis scripts:', error);
+            // We'll try again later if needed via executeScript fallback
+        }
+    }
+    /**
+     * Executes a cached script, falling back to EVAL if needed
+     */
+    async executeScript(script, scriptSHA, numKeys, ...args) {
+        try {
+            // If we have the SHA, try EVALSHA first
+            if (scriptSHA) {
+                try {
+                    // Ensure redisPublisher is available
+                    if (!this._redisPublisher)
+                        throw new Error('Redis publisher connection not available');
+                    return await this.redisPublisher.evalsha(scriptSHA, numKeys, ...args);
+                }
+                catch (error) {
+                    // Add type 'any' to error
+                    // If the script isn't found on the server, fall back to EVAL
+                    if (error?.message && typeof error.message === 'string' && error.message.includes('NOSCRIPT')) {
+                        logger_1.PUBLISHER_LOGGER.warn('PUBLISHER: Script not found on Redis server, reloading...');
+                        if (!this._redisPublisher)
+                            throw new Error('Redis publisher connection not available for script reload');
+                        const newSHA = (await this.redisPublisher.script('LOAD', script)); // Cast result to string
+                        // Update the stored SHA
+                        if (script === this.findHighestStreamIdScript) {
+                            this.findHighestStreamIdScriptSHA = newSHA;
+                        }
+                        // Retry with EVALSHA
+                        return await this.redisPublisher.evalsha(newSHA, numKeys, ...args);
+                    }
+                    throw error;
+                }
+            }
+            // If we don't have the SHA yet, use EVAL directly
+            if (!this._redisPublisher)
+                throw new Error('Redis publisher connection not available for EVAL');
+            return await this.redisPublisher.eval(script, numKeys, ...args);
+        }
+        catch (error) {
+            // Add type 'any' to error
+            logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error executing Redis script:', error);
+            throw error; // Re-throw the original error
+        }
+    }
+    /**
+     * Checks if optimization mode is active for a given event type.
+     * Cleans up expired entries during the check.
+     */
+    isOptimizationActive(eventName) {
+        const expirationTime = this.optimizationActiveUntil.get(eventName);
+        if (!expirationTime)
+            return false;
+        if (Date.now() > expirationTime) {
+            // Clean up expired entry during publish operation
+            this.optimizationActiveUntil.delete(eventName);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Deactivated optimization for ${eventName}`);
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Activates optimization mode for a given event type for a configured duration.
+     */
+    activateOptimizationFor(eventName) {
+        const durationMs = this.config.optimizationDurationMs ?? this.DEFAULT_STREAMS_CONFIG.optimizationDurationMs;
+        const expirationTime = Date.now() + durationMs;
+        this.optimizationActiveUntil.set(eventName, expirationTime);
+        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Activated optimization for ${eventName} until ${new Date(expirationTime).toISOString()}`);
+    }
+    /**
+     * Deletes a specific message ID from multiple consumer group streams.
+     */
+    async deleteMessageFromStreams(eventName, consumerGroups, messageId) {
+        const deletionPromises = consumerGroups.map(async (consumerGroup) => {
+            const streamName = `${eventName}:${consumerGroup}`;
+            try {
+                if (!this._redisPublisher)
+                    throw new Error('Redis publisher connection not available for XDEL');
+                await this.redisPublisher.xdel(streamName, messageId);
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Deleted message ${messageId} from ${streamName} for rollback`);
+            }
+            catch (error) {
+                // Log error but continue trying to delete from other streams
+                logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error deleting message ${messageId} from ${streamName}:`, error);
+            }
+        });
+        // Wait for all deletions to attempt completion
+        await Promise.allSettled(deletionPromises);
     }
     setupCircuitBreakerListeners() {
         this.circuitBreaker.on('stateChange', async (newState) => {
-            logger_1.PUBLISHER_LOGGER.log(`Circuit breaker state changed to: ${circuit_breaker_1.CircuitState[newState]}`);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Circuit breaker state changed to: ${circuit_breaker_1.CircuitState[newState]}`);
             if (newState === circuit_breaker_1.CircuitState.CLOSED) {
                 await this.processStoredEvents();
             }
@@ -92,10 +279,10 @@ class Streams {
         logger_1.PUBLISHER_LOGGER.log('PUBLISHER: Running Clearance', this.eventsListened);
         const cleanupPromises = this.eventsListened.map((eventName) => this.cleanupAcknowledgedMessages(eventName, cleanUpInterval)
             .then(() => {
-            logger_1.PUBLISHER_LOGGER.log(`Cleanup process for Acknowledged messages completed for ${eventName}`);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Cleanup process for Acknowledged messages completed for ${eventName}`);
         })
             .catch((error) => {
-            logger_1.PUBLISHER_LOGGER.error(`Error during cleanup for ${eventName}:`, error);
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error during cleanup for ${eventName}:`, error);
         }));
         await Promise.all(cleanupPromises);
     }
@@ -133,32 +320,105 @@ class Streams {
          */
         if (this.config.circuitBreaker.enabled && !this.circuitBreaker.isAllowed()) {
             await this.circuitBreaker.storeEvent(data);
-            logger_1.PUBLISHER_LOGGER.error('Circuit is open, event stored for later processing');
+            logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Circuit is open, event stored for later processing');
             return 'CIRCUIT_BREAKER_FLOW';
         }
         try {
             tracker.startRedisOperation();
             const consumerGroups = await (0, utils_1.getAllConsumerGroups)(data.eventName, this.redisPublisher);
             tracker.endRedisOperation();
-            let key = '*';
+            let key = '*'; // Default to auto-generated ID
+            const publishedGroups = []; // Track successfully published groups for rollback
             if (consumerGroups.length > 0) {
                 logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
-                for (const consumerGroup of consumerGroups) {
-                    // Publish the event to each consumer group's stream
-                    const streamName = `${data.eventName}:${consumerGroup}`;
+                // Check if optimization is already active or if threshold is met
+                const optimizationThreshold = this.config.optimizationThreshold ?? this.DEFAULT_STREAMS_CONFIG.optimizationThreshold;
+                const useOptimization = this.isOptimizationActive(data.eventName) || consumerGroups.length > optimizationThreshold;
+                // If optimization is needed, generate the optimized ID upfront
+                if (useOptimization) {
+                    const maxStreamsToCheck = Math.max(5, Math.ceil(consumerGroups.length * 0.25));
                     tracker.startRedisOperation();
-                    const generatedKey = await this.redisPublisher.xadd(streamName, key, 'data', JSON.stringify(data));
-                    tracker.endRedisOperation();
-                    tracker.incrementEventCount();
-                    tracker.incrementMessageRate('publish', data.eventName);
-                    if (this.metricsCollector) {
-                        this.metricsCollector.addMetrics(tracker.getMetrics());
+                    try {
+                        const result = (await this.executeScript(this.findHighestStreamIdScript, this.findHighestStreamIdScriptSHA, 2, // Two keys
+                        data.eventName, maxStreamsToCheck.toString(), ...consumerGroups));
+                        const [highestTimestamp, highestSeq] = result;
+                        const currentTime = Date.now();
+                        if (currentTime > highestTimestamp) {
+                            key = `${currentTime}-0`;
+                        }
+                        else {
+                            key = `${highestTimestamp}-${highestSeq}`;
+                        }
+                        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Using optimized ID generation: ${key}`);
+                    }
+                    catch (scriptError) {
+                        logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error in optimized ID generation script, falling back to auto-generated ID:', scriptError);
+                        key = '*';
+                    }
+                    finally {
+                        tracker.endRedisOperation();
                     }
-                    if (key === '*')
-                        key = generatedKey ?? key;
                 }
-                await (0, utils_1.notifySubscribers)(this.redisPublisher, data.eventName, key, multicast);
-                await this.circuitBreaker.recordSuccess();
+                // Publish to each consumer group
+                try {
+                    for (const consumerGroup of consumerGroups) {
+                        const streamName = `${data.eventName}:${consumerGroup}`;
+                        tracker.startRedisOperation();
+                        try {
+                            const generatedKey = await this.redisPublisher.xadd(streamName, key, 'data', JSON.stringify(data));
+                            tracker.endRedisOperation();
+                            // If using auto-generated ID, capture it for the first stream
+                            if (key === '*')
+                                key = generatedKey ?? key;
+                            // Track successful publish
+                            publishedGroups.push(consumerGroup);
+                        }
+                        catch (error) {
+                            tracker.endRedisOperation(); // Ensure tracker ends even on error
+                            // Check if the error is the ID conflict error
+                            if ((typeof error === 'string' && error.includes('equal or smaller than the target stream top item')) ||
+                                (error?.message &&
+                                    typeof error.message === 'string' &&
+                                    error.message.includes('equal or smaller than the target stream top item'))) {
+                                logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: ID conflict detected for ${streamName} (key: ${key}), switching to optimized mode for ${data.eventName} for duration: ${this.config.optimizationDurationMs ?? this.DEFAULT_STREAMS_CONFIG.optimizationDurationMs / 1000}. Consumers: ${consumerGroups.length}`);
+                                // Delete already published messages if any (using the captured key)
+                                if (publishedGroups.length > 0) {
+                                    await this.deleteMessageFromStreams(data.eventName, publishedGroups, key);
+                                    publishedGroups.length = 0;
+                                }
+                                // Activate optimization for this event type
+                                this.activateOptimizationFor(data.eventName);
+                                // Recursively call publish with same data (optimization will be active now)
+                                // Important: Return the result of the recursive call
+                                return this.publish(data, multicast);
+                            }
+                            else {
+                                // For other XADD errors, rethrow to trigger the outer catch block
+                                logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Unhandled XADD error for ${streamName} (key: ${key}): ${error?.message}`);
+                                throw error;
+                            }
+                        }
+                        // Increment metrics only after successful publish
+                        tracker.incrementEventCount();
+                        tracker.incrementMessageRate('publish', data.eventName);
+                        if (this.metricsCollector) {
+                            this.metricsCollector.addMetrics(tracker.getMetrics());
+                        }
+                    } // End of for loop
+                    // Notify subscribers only after all groups are successfully published (or retried)
+                    await (0, utils_1.notifySubscribers)(this.redisPublisher, data.eventName, key, multicast);
+                    await this.circuitBreaker.recordSuccess();
+                }
+                catch (publishError) {
+                    // If we failed during the loop (and it wasn't handled by the adaptive logic),
+                    // try to clean up any messages already sent in this attempt.
+                    if (publishedGroups.length > 0) {
+                        logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error during multi-group publish for ${data.eventName}, rolling back published messages for key ${key}, publishedMessage: ${JSON.stringify(data)}`);
+                        await this.deleteMessageFromStreams(data.eventName, publishedGroups, key);
+                    }
+                    // Rethrow the error that caused the failure
+                    throw publishError;
+                }
             }
             else {
                 logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Event publish failed for event ${data.eventName}, reason: no consumers ${consumerGroups}`);
@@ -264,14 +524,14 @@ class Streams {
             // If both exist, this will be a no-op
             try {
                 await this.redisGroups.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
-                logger_1.PUBLISHER_LOGGER.log(`Group created for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Group created for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
             }
             catch (e) {
                 // BUSYGROUP error means group already exists, which is fine
                 if (!e.message.includes('BUSYGROUP')) {
                     throw e;
                 }
-                logger_1.PUBLISHER_LOGGER.log(`Group already exists for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Group already exists for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
             }
             // Create consumer (idempotent operation)
             const createConsumerStatus = await this.redisGroups.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId);
@@ -320,11 +580,11 @@ class Streams {
                     await processMessage(this.redisGroups, '0', new tracker_1.MetricsTracker(), false);
                 }
                 catch (error) {
-                    logger_1.PUBLISHER_LOGGER.error('Error in running recurring cleanup task:', error);
+                    logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error in running recurring cleanup task:', error);
                 }
             },
             error: (error) => {
-                logger_1.PUBLISHER_LOGGER.error('Fatal error in cleanup timer:', error);
+                logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Fatal error in cleanup timer:', error);
             },
         });
         // Create observable with proper cleanup
@@ -347,8 +607,8 @@ class Streams {
                 // 3. Pending message reprocessing
                 if (!multicast && messageId !== '0' && !processPending) {
                     try {
-                        const pendingDetails = await redisClient.xpending(streamName, this.consumerGroupName, messageId, messageId, 1);
-                        if (pendingDetails[2] === 0) {
+                        const pendingDetails = (await redisClient.xpending(streamName, this.consumerGroupName, messageId, messageId, 1));
+                        if (pendingDetails && pendingDetails.length > 0 && pendingDetails[0] && pendingDetails[0][3] === 0) {
                             logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: Message ${messageId} already processed for ${streamName}`);
                             return;
                         }
@@ -371,12 +631,28 @@ class Streams {
                         /**
                          * Very very rare case of this occurring. Cases where the running service is super overloaded
                          * that causes a messaged to be sent to processing with a delay, but eventually before being
-                         * processed gets picked up by another instance, leading to multiple publications
+                         * processed gets picked up by another instance, leading to multiple publications. This makes
+                         * sure that there is a check to ensure that the message belongs to this consumer before being
+                         * processed
                          */
                         if (processPending) {
-                            const claimed = await redisClient.xclaim(streamName, this.consumerGroupName, this.instanceId, 20000, messageId, 'JUSTID');
-                            if (!claimed || claimed.length === 0) {
-                                return; // Message already claimed or acknowledged by another consumer, so don't repush to the subscriber
+                            const pendingDetails = (await redisClient.xpending(streamName, this.consumerGroupName, messageId, messageId, 1));
+                            /** Message is not in PEL */
+                            if (!pendingDetails || pendingDetails.length === 0)
+                                return;
+                            const [, instanceId, idleTime, deliveryCount] = pendingDetails[0][1];
+                            /** Message is claimed by another consumer within the group, so do not process */
+                            if (instanceId !== this.instanceId) {
+                                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Process Pending message is claimed by another consumer : ${JSON.stringify({
+                                    instanceId,
+                                    idleTime,
+                                    deliveryCount,
+                                    messageId,
+                                    streamName,
+                                    originalInstanceId: this.instanceId,
+                                    consumerGroupName: this.consumerGroupName,
+                                })}`);
+                                return;
                             }
                         }
                         const messages = await redisClient.xrange(streamName, messageId, messageId);
@@ -393,22 +669,30 @@ class Streams {
                     else {
                         const messages = (await redisClient.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'STREAMS', streamName, '>'));
                         if (messages?.length) {
-                            if (messageId === '0') {
-                                messageId = messages[0][1][0][0];
-                                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reprocessing unprocessed message with id: ${messageId}`);
+                            const messageIdRead = messages[0][1][0][0];
+                            if (messageIdRead !== messageId) {
+                                if (messageId === '0') {
+                                    logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reprocessing unprocessed message with id: ${messageId}`);
+                                }
+                                else {
+                                    const [timestamp] = messageIdRead?.split('-').map(Number);
+                                    const [publishedTimestamp] = messageId?.split('-').map(Number);
+                                    logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Message queue processing is ${publishedTimestamp > timestamp ? 'ahead' : 'behind'}`);
+                                }
                             }
+                            messageId = messageIdRead;
                             try {
                                 eventData = JSON.parse(messages[0][1][0][1][1]);
                             }
                             catch (error) {
-                                logger_1.PUBLISHER_LOGGER.error(`JSON parsing failed for message: ${messages[0][1][0][1][1]}`);
+                                logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: JSON parsing failed for message: ${messages[0][1][0][1][1]}`);
                                 return;
                             }
                         }
                     }
                 }
                 catch (error) {
-                    logger_1.PUBLISHER_LOGGER.error('Error retrieving or parsing event data:', error);
+                    logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error retrieving or parsing event data:', error);
                     return;
                 }
                 finally {
@@ -444,7 +728,7 @@ class Streams {
                                 }
                                 catch (error) {
                                     // Log error but don't fail entire processing
-                                    logger_1.PUBLISHER_LOGGER.error(`Error processing subscription ${subId}:`, error);
+                                    logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error processing subscription ${subId}:`, error);
                                 }
                             }));
                         }
@@ -460,7 +744,7 @@ class Streams {
                         tracker.setConsumerLag(this.consumerGroupName, currentTime - eventData.createdAt);
                     }
                     catch (error) {
-                        logger_1.PUBLISHER_LOGGER.error(`Processing error for message ${messageId}:`, error);
+                        logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Processing error for message ${messageId}:`, error);
                         const dlqEvent = {
                             ...eventData,
                             failureReason: error.message,
@@ -588,6 +872,9 @@ class Streams {
      */
     async close() {
         try {
+            // Reset optimization state and script SHA
+            this.optimizationActiveUntil.clear();
+            this.findHighestStreamIdScriptSHA = null;
             if (this.cleanUpTimer) {
                 clearInterval(this.cleanUpTimer);
             }
@@ -602,7 +889,7 @@ class Streams {
             }
         }
         catch (error) {
-            logger_1.PUBLISHER_LOGGER.error('Error during cleanup:', error);
+            logger_1.PUBLISHER_LOGGER.error('PUBLISHER: Error during cleanup:', error);
         }
     }
     async cleanupAcknowledgedMessages(eventName, interval = this.config.acknowledgedMessageCleanupInterval) {
@@ -610,27 +897,34 @@ class Streams {
         const lastAckKey = `last_ack:${streamName}`;
         const oneHourAgo = Date.now() - interval;
         try {
-            // Get consumer group info to check if consumers are active
-            const groupInfo = (await this.redisGroups.xinfo('GROUPS', streamName));
-            // If no active consumers, leave stream as is
-            if (!groupInfo || !groupInfo.some((group) => group.consumers > 0)) {
-                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: No active consumers for ${streamName}, leaving stream as is`);
-                return;
-            }
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Started [cleanupAcknowledgedMessages] for ${streamName} and with cleanUpInterval set as ${interval}`);
             // Get last acknowledged message ID
             const lastAckId = await this.redisGroups.get(lastAckKey);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: [cleanupAcknowledgedMessages] {lastAckId: ${lastAckId}, lastAckKey: ${lastAckKey}} `);
             if (!lastAckId) {
-                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: No acknowledged messages for ${streamName}`);
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: [cleanupAcknowledgedMessages]No acknowledged messages for ${streamName}`);
                 return;
             }
             // Extract timestamp from message ID
             const [timestamp] = lastAckId.split('-').map(Number);
             const cleanupThreshold = Math.min(timestamp, oneHourAgo);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: [cleanupAcknowledgedMessages] {cleanUpThreshold: ${cleanupThreshold}} `);
+            // Get consumer group info to check if consumers are active
+            const groupInfo = (await this.redisGroups.xinfo('GROUPS', streamName))?.map(([, name, , consumers]) => ({
+                name: name,
+                consumers: consumers,
+            }));
+            // If no active consumers, leave stream as is
+            if (groupInfo?.length === 0 || !groupInfo?.some((group) => group.consumers > 0)) {
+                logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: No active consumers for ${streamName}, leaving stream as is`);
+                return;
+            }
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: [cleanupAcknowledgedMessages] XTRIM to be called`);
             await this.redisGroups.xtrim(streamName, 'MINID', cleanupThreshold);
             logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Cleaned up messages before last acknowledged message ${timestamp} from ${streamName}`);
         }
         catch (error) {
-            logger_1.PUBLISHER_LOGGER.error(`Error during cleanup for ${streamName}:`, error);
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error during cleanup for ${streamName}:`, error);
         }
     }
     async getDiagnosticData(events) {
@@ -759,7 +1053,7 @@ class Streams {
             ]);
         }
         catch (error) {
-            logger_1.PUBLISHER_LOGGER.error(`Error acknowledging message ${messageId} for ${streamName}:`, error);
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error acknowledging message ${messageId} for ${streamName}:`, error);
             throw error;
         }
     }

package/src/lib/redis/types.d.ts

@@ -54,6 +54,8 @@ export interface IStreamsConfig {
         halfOpenStateMaxAttempts: number;
         maxStoredEvents: number;
     };
+    optimizationDurationMs?: number;
+    optimizationThreshold?: number;
 }
 export type TEventFilter<T> = (event: EventData<T, string>) => boolean;
 export interface ISubscription<T, TName extends string = string> {

package/src/lib/redis/utils.js

@@ -12,7 +12,7 @@ exports.decodeScheduledMessage = decodeScheduledMessage;
 const logger_1 = require("./logger");
 async function getAllConsumerGroups(eventName, redisConnection) {
     const consumerGroups = await redisConnection.smembers(`${eventName}`);
-    return consumerGroups;
+    return consumerGroups.sort();
 }
 function* getSummaryOnStreamConsumerGroup(redisClient, consumerGroupName, streamName) {
     const [count, , , consumers] = (yield redisClient.xpending(streamName, consumerGroupName));