@jetit/publisher 5.2.1 → 5.3.2

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@jetit/publisher",
-  "version": "5.2.1",
+  "version": "5.3.2",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "^0.0.12",
-    "events": "3.3.0",
     "ioredis": "^5.3.0",
     "rxjs": "^7.8.0",
-    "tslib": "2.5.0"
+    "tslib": "2.7.0"
   },
+  "types": "./src/index.d.ts",
   "main": "./src/index.js"
-}
+}

package/src/lib/monitoring/collector.js

@@ -111,33 +111,26 @@ class MetricsCollector {
         const individualDepths = {};
         let cursor = '0';
         do {
-            const [nextCursor, keys] = await this.redisClient.scan(cursor, 'MATCH', `ack:*:cg-*`, 'COUNT', 100);
+            const [nextCursor, keys] = await this.redisClient.scan(cursor, 'MATCH', '*:cg-*', 'COUNT', 100);
             cursor = nextCursor;
             if (keys.length > 0) {
-                const pipeline = this.redisClient.pipeline();
-                keys.forEach((key) => {
-                    pipeline.xlen(key.slice(4));
-                    pipeline.zcard(key);
-                });
-                const results = await pipeline.exec();
-                if (!results) {
-                    continue;
-                }
-                for (let i = 0; i < results.length; i += 2) {
-                    const key = keys[i / 2];
-                    const [streamLengthErr, streamLength] = results[i];
-                    const [ackCountErr, ackCount] = results[i + 1];
-                    if (streamLengthErr) {
-                        console.error(`Error getting length for key: ${streamLengthErr}`);
-                        continue;
+                for (const streamKey of keys) {
+                    try {
+                        // Get stream length and pending info
+                        const streamLength = await this.redisClient.xlen(streamKey);
+                        // Extract consumer group name from stream key (format: eventName:cg-serviceName)
+                        const consumerGroup = streamKey.split(':')[1];
+                        // XPENDING returns [count, min-id, max-id, consumer-list]
+                        const pendingInfo = await this.redisClient.xpending(streamKey, consumerGroup);
+                        const totalPending = pendingInfo ? Number(pendingInfo[0]) : 0;
+                        const queueDepth = Math.max(0, streamLength - totalPending);
+                        totalDepth += queueDepth;
+                        individualDepths[streamKey] = queueDepth;
                     }
-                    if (ackCountErr) {
-                        console.error(`Error getting ack count for key: ${ackCountErr}`);
+                    catch (error) {
+                        console.error(`Error processing key ${streamKey}:`, error);
                         continue;
                     }
-                    const queueDepth = Math.max(0, streamLength - ackCount);
-                    totalDepth += queueDepth;
-                    individualDepths[key] = queueDepth;
                 }
             }
         } while (cursor !== '0');

package/src/lib/performance/circuit_breaker.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { EventEmitter } from 'events';
 import { RedisType } from '../redis/registry';
 import { EventData, IStreamsConfig } from '../redis/types';

package/src/lib/redis/batch.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.publishScheduledBatch = exports.publishBatch = void 0;
+exports.publishBatch = publishBatch;
+exports.publishScheduledBatch = publishScheduledBatch;
 const id_1 = require("@jetit/id");
 const tracker_1 = require("../monitoring/tracker");
 const logger_1 = require("./logger");
@@ -72,8 +73,6 @@ async function publishBatchWithRetry(streams, events, options) {
 function publishBatch(streams, events, options = {}) {
     return publishBatchWithRetry(streams, events, options);
 }
-exports.publishBatch = publishBatch;
 function publishScheduledBatch(streams, events, options) {
     return publishBatchWithRetry(streams, events, options);
 }
-exports.publishScheduledBatch = publishScheduledBatch;

package/src/lib/redis/duplication.js

@@ -10,7 +10,7 @@ class ContentBasedDeduplication {
         this.hmacSecret = 'content-hash-calculation';
     }
     calculateEventHash(eventData) {
-        const { eventId, timestamp, createdAt, ...hashableData } = eventData;
+        const { eventId, timestamp, createdAt, republishEvent, ...hashableData } = eventData;
         const hmac = (0, crypto_1.createHmac)('sha256', this.hmacSecret);
         return hmac.update(JSON.stringify(hashableData)).digest('hex');
     }

package/src/lib/redis/registry.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.setRedisConnectionSettings = exports.RedisRegistry = void 0;
+exports.RedisRegistry = void 0;
+exports.setRedisConnectionSettings = setRedisConnectionSettings;
 const ioredis_1 = require("ioredis");
 const logger_1 = require("./logger");
 class RedisRegistry {
@@ -103,4 +104,3 @@ RedisRegistry.options = {
 function setRedisConnectionSettings(options) {
     RedisRegistry.setOptions(options);
 }
-exports.setRedisConnectionSettings = setRedisConnectionSettings;

package/src/lib/redis/streams.js

@@ -257,60 +257,40 @@ class Streams {
         const setKeyForK8sHandling = `instance:${this.instanceUniqueId}:consumerGroupName`;
         this.eventsListened.push(eventName);
         try {
-            // Check if the consumer group already exists
-            let groupInfo = [];
+            // Try to create the consumer group and consumer in one go
+            // If group doesn't exist, this will create it with MKSTREAM
+            // If group exists but consumer doesn't, this will create just the consumer
+            // If both exist, this will be a no-op
             try {
-                groupInfo = (await this.redisGroups.xinfo('GROUPS', streamName));
-            }
-            catch (e) {
-                // Do nothing
-            }
-            let groupExists = false;
-            for (const group of groupInfo) {
-                if (group[1] === this.consumerGroupName) {
-                    groupExists = true;
-                    break;
-                }
-            }
-            if (!groupExists) {
                 await this.redisGroups.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
                 logger_1.PUBLISHER_LOGGER.log(`Group created for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
             }
-            else {
+            catch (e) {
+                // BUSYGROUP error means group already exists, which is fine
+                if (!e.message.includes('BUSYGROUP')) {
+                    throw e;
+                }
                 logger_1.PUBLISHER_LOGGER.log(`Group already exists for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
             }
+            // Create consumer (idempotent operation)
+            const createConsumerStatus = await this.redisGroups.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId);
+            // Register event and consumer group in parallel
+            const [addToCGSet, addToFlushSet] = await Promise.all([
+                this.redisGroups.sadd(`${eventName}`, this.consumerGroupName),
+                this.redisGroups.set(setKeyForK8sHandling, this.consumerGroupName),
+                this.redisGroups.sadd(key, eventName),
+            ]);
+            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Consumer Registered with ${this.instanceId} under ${this.consumerGroupName} with status ${JSON.stringify({
+                createConsumerStatus,
+                addToCGSet,
+                addToFlushSet,
+            })}`);
+            return true;
         }
-        catch (e) {
-            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Group creation failed with error ${e.message} for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
-        }
-        // Check if the consumer already exists in the group
-        let consumers = [];
-        try {
-            consumers = (await this.redisGroups.xinfo('CONSUMERS', streamName, this.consumerGroupName));
-        }
-        catch (e) {
-            // Do nothing
-        }
-        let consumerExists = false;
-        for (const consumer of consumers) {
-            if (consumer[1] === this.instanceId) {
-                consumerExists = true;
-                break;
-            }
-        }
-        let createConsumerStatus;
-        if (!consumerExists) {
-            createConsumerStatus = await this.redisGroups.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId);
-        }
-        else {
-            createConsumerStatus = 0; // Consumer already exists
-            logger_1.PUBLISHER_LOGGER.log(`Consumer already exists for ${JSON.stringify({ streamName, cgn: this.consumerGroupName, instanceId: this.instanceId })}`);
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Consumer registration failed for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}:`, error);
+            return false;
         }
-        await this.redisGroups.sadd(key, eventName);
-        const addToCGSet = await this.redisGroups.sadd(`${eventName}`, this.consumerGroupName);
-        const addToFlushSet = await this.redisGroups.set(setKeyForK8sHandling, this.consumerGroupName);
-        logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Consumer Registered and created with ${this.instanceId} under ${this.consumerGroupName} with ${createConsumerStatus} consumers and with the following status ${JSON.stringify({ addToCGSet, addToFlushSet })}`);
-        return createConsumerStatus === 0 || createConsumerStatus === 1;
     }
     listenInternals(eventName, subscriptionId, eventFilter, filterKeepAlive = 24 * 60 * 60 * 1000, publishOnceGuarantee = false, externalAcknowledgement = false) {
         if (!this.subscriptions.has(eventName)) {
@@ -336,21 +316,22 @@ class Streams {
         const processMessage = async (redisClient, messageId, tracker, multicast = false, processPending = false) => {
             logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Processing message ${messageId} for ${streamName}`);
             try {
-                try {
-                    /**
-                     * Check if the message is already acquired by another client and is pending.
-                     * This check should only happen if the message is not a multicast message
-                     */
-                    const pendingDetails = await redisClient.xpending(streamName, this.consumerGroupName, messageId, messageId, 1);
-                    if (pendingDetails[2] === 0 && !multicast) {
-                        logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: MACK ${messageId} for ${streamName}`);
-                        return;
+                // Skip XPENDING check for:
+                // 1. Multicast messages (meant for all consumers)
+                // 2. Initial message processing (messageId = '0')
+                // 3. Pending message reprocessing
+                if (!multicast && messageId !== '0' && !processPending) {
+                    try {
+                        const pendingDetails = await redisClient.xpending(streamName, this.consumerGroupName, messageId, messageId, 1);
+                        if (pendingDetails[2] === 0) {
+                            logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: Message ${messageId} already processed for ${streamName}`);
+                            return;
+                        }
+                    }
+                    catch (e) {
+                        // If XPENDING fails, continue with processing
+                        logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: XPENDING check failed for ${messageId}, continuing with processing`);
                     }
-                }
-                catch (e) {
-                    // Ignore the xpending error and continue
-                    logger_1.PUBLISHER_LOGGER.error('XPENDING ERROR: To be handled');
-                    logger_1.PUBLISHER_LOGGER.warn(JSON.stringify(e));
                 }
                 let eventData;
                 /**
@@ -416,7 +397,7 @@ class Streams {
                             }
                         }
                         if (!externalAcknowledgement)
-                            this.acknowledgeMessage(ackKey);
+                            await this.acknowledgeMessage(ackKey);
                     }
                     catch (processingError) {
                         logger_1.PUBLISHER_LOGGER.error(`Processing error for message ${messageId}:`, processingError);
@@ -551,16 +532,40 @@ class Streams {
     async cleanupAcknowledgedMessages(eventName, interval = this.config.acknowledgedMessageCleanupInterval) {
         const streamName = `${eventName}:${this.consumerGroupName}`;
         const cleanupThreshold = Date.now() - interval;
-        const acknowledgedMessages = await this.redisGroups.zrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
-        if (acknowledgedMessages && acknowledgedMessages.length > 0) {
-            // Remove acknowledged messages from the stream in batches
-            const batchSize = 100;
-            for (let i = 0; i < acknowledgedMessages.length; i += batchSize) {
-                const batch = acknowledgedMessages.slice(i, i + batchSize);
-                await this.redisGroups.xdel(streamName, ...batch);
+        const CHUNK_SIZE = 10000; // Process messages in chunks to avoid memory issues
+        try {
+            // Get pending info for this consumer group once
+            const pendingInfo = await this.redisGroups.xpending(streamName, this.consumerGroupName);
+            const hasPendingMessages = pendingInfo && pendingInfo[0] > 0;
+            let lastId = '-'; // Start from the beginning of the stream
+            while (true) {
+                // Get a chunk of messages
+                const messages = await this.redisGroups.xrange(streamName, lastId, cleanupThreshold.toString(), 'COUNT', CHUNK_SIZE);
+                if (!messages || messages.length === 0)
+                    break;
+                // Update lastId for next iteration (exclusive)
+                lastId = messages[messages.length - 1][0];
+                if (hasPendingMessages) {
+                    // If there are pending messages, we need to check each message
+                    const pendingDetails = (await this.redisGroups.xpending(streamName, this.consumerGroupName, lastId, '+', CHUNK_SIZE, this.instanceId));
+                    const pendingIds = new Set(pendingDetails.map((detail) => detail[0]));
+                    const acknowledgedIds = messages.map((msg) => msg[0]).filter((id) => !pendingIds.has(id));
+                    if (acknowledgedIds.length > 0) {
+                        await this.redisGroups.xdel(streamName, ...acknowledgedIds);
+                    }
+                }
+                else {
+                    // If no pending messages, we can safely delete all messages in this chunk
+                    const messageIds = messages.map((msg) => msg[0]);
+                    await this.redisGroups.xdel(streamName, ...messageIds);
+                }
+                // If we got less than CHUNK_SIZE messages, we've reached the end
+                if (messages.length < CHUNK_SIZE)
+                    break;
             }
-            // Remove acknowledged messages from the Sorted Set
-            await this.redisGroups.zremrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`Error during cleanup for ${streamName}:`, error);
         }
     }
     async getDiagnosticData(events) {
@@ -676,7 +681,6 @@ class Streams {
     async acknowledgeMessage(ackKey) {
         const { streamName, messageId } = this.demergeMessageKey(ackKey);
         await this.redisGroups.xack(streamName, this.consumerGroupName, messageId);
-        await this.redisGroups.zadd(`ack:${streamName}`, Date.now().toString(), messageId);
     }
     frameMessageKey(streamName, messageId) {
         return `${streamName}##${messageId}`;

package/src/lib/redis/utils.js

@@ -1,12 +1,19 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.UTILS = exports.decodeScheduledMessage = exports.encodeScheduledMessage = exports.removedScheduledJob = exports.notifySubscribers = exports.getMessageStatesCount = exports.getUnacknowledgedMessages = exports.getSummaryOnStreamConsumerGroup = exports.getAllConsumerGroups = void 0;
+exports.UTILS = void 0;
+exports.getAllConsumerGroups = getAllConsumerGroups;
+exports.getSummaryOnStreamConsumerGroup = getSummaryOnStreamConsumerGroup;
+exports.getUnacknowledgedMessages = getUnacknowledgedMessages;
+exports.getMessageStatesCount = getMessageStatesCount;
+exports.notifySubscribers = notifySubscribers;
+exports.removedScheduledJob = removedScheduledJob;
+exports.encodeScheduledMessage = encodeScheduledMessage;
+exports.decodeScheduledMessage = decodeScheduledMessage;
 const logger_1 = require("./logger");
 async function getAllConsumerGroups(eventName, redisConnection) {
     const consumerGroups = await redisConnection.smembers(`${eventName}`);
     return consumerGroups;
 }
-exports.getAllConsumerGroups = getAllConsumerGroups;
 function* getSummaryOnStreamConsumerGroup(redisClient, consumerGroupName, streamName) {
     const [count, , , consumers] = (yield redisClient.xpending(streamName, consumerGroupName));
     yield {
@@ -21,7 +28,6 @@ function* getSummaryOnStreamConsumerGroup(redisClient, consumerGroupName, stream
             : [],
     };
 }
-exports.getSummaryOnStreamConsumerGroup = getSummaryOnStreamConsumerGroup;
 async function getUnacknowledgedMessages(redisClient, consumerGroupName, streamName, consumerName, count = 500) {
     try {
         // Get pending messages summary
@@ -32,12 +38,16 @@ async function getUnacknowledgedMessages(redisClient, consumerGroupName, streamN
         }
         // Use the smallest and largest IDs to get a detailed range
         const pendingMessageCount = summary[0];
-        // Get detailed information in the range
-        let pendingMessages = (await redisClient.xpending(streamName, consumerGroupName, '-', '+', count, consumerName));
+        // Create a message ID for 2 seconds ago to exclude recent messages
+        const oneSecondAgo = Date.now() - 2000;
+        const minId = '0-0'; // Start from beginning
+        const maxId = `${oneSecondAgo}-0`; // Only include messages older than 1 second
+        // Get detailed information in the range, excluding recent messages
+        let pendingMessages = (await redisClient.xpending(streamName, consumerGroupName, minId, maxId, count, consumerName));
         /** If no pending messages on consumer, fetch messages from other consumers that haven't been claimed for more than 10s */
         if (count > pendingMessages.length && pendingMessages.length === 0) {
             await redisClient.xautoclaim(streamName, consumerGroupName, consumerName, 10000, '0-0', 'COUNT', 100);
-            pendingMessages = (await redisClient.xpending(streamName, consumerGroupName, '-', '+', count, consumerName));
+            pendingMessages = (await redisClient.xpending(streamName, consumerGroupName, minId, maxId, count, consumerName));
         }
         return {
             count: pendingMessageCount,
@@ -51,7 +61,6 @@ async function getUnacknowledgedMessages(redisClient, consumerGroupName, streamN
         return { count: 0, messageIds: [] };
     }
 }
-exports.getUnacknowledgedMessages = getUnacknowledgedMessages;
 async function getMessageStatesCount(redisClient, streamName, consumerGroup) {
     try {
         const pendingInfo = (await redisClient.xpending(streamName, consumerGroup));
@@ -66,11 +75,9 @@ async function getMessageStatesCount(redisClient, streamName, consumerGroup) {
         return { acknowledged: 0, unacknowledged: 0 };
     }
 }
-exports.getMessageStatesCount = getMessageStatesCount;
 async function notifySubscribers(redisClient, eventName, messageId, multicast = false) {
     await redisClient.publish(eventName, JSON.stringify({ messageId, multicast }));
 }
-exports.notifySubscribers = notifySubscribers;
 async function removedScheduledJob(redisClient, eventString) {
     const currentTime = new Date().getTime();
     const events = await redisClient.zrangebyscore('se', 0, currentTime);
@@ -79,7 +86,6 @@ async function removedScheduledJob(redisClient, eventString) {
     const eventsLater = await redisClient.zrangebyscore('se', 0, currentTime);
     logger_1.PUBLISHER_LOGGER.log(`Total Events in scheduled queue: ${eventsLater.length}`);
 }
-exports.removedScheduledJob = removedScheduledJob;
 function encodeScheduledMessage(data) {
     const eventName = data.eventName;
     const eventData = JSON.stringify(data.data);
@@ -87,7 +93,6 @@ function encodeScheduledMessage(data) {
     const eventDataBuffer = Buffer.from(eventData, 'utf8').toString('base64');
     return `${eventName}%%${eventDataBuffer}%%${repeatInterval}`;
 }
-exports.encodeScheduledMessage = encodeScheduledMessage;
 function decodeScheduledMessage(data) {
     const parts = data.split('%%');
     const eventName = parts[0];
@@ -97,7 +102,6 @@ function decodeScheduledMessage(data) {
         repeatInterval: parseInt(parts[2]),
     };
 }
-exports.decodeScheduledMessage = decodeScheduledMessage;
 exports.UTILS = {
     getMessageStatesCount,
     getUnacknowledgedMessages,