@jetit/publisher 5.1.1 → 5.2.2

package/README.md

@@ -116,11 +116,45 @@ await publisher.publish(eventData);
 ### Subscribing to Events
 
 ```typescript
+// Basic subscription with automatic acknowledgment
 publisher.listen('user-registered').subscribe(event => {
     console.log('New user registered:', event.data);
 });
+
+// Subscription with external acknowledgment
+const options = {
+    externalAcknowledgement: true
+};
+
+publisher.listen('user-registered', options).subscribe(async event => {
+    try {
+        console.log('New user registered:', event.data);
+        // Process the event
+        await processUserRegistration(event.data);
+        
+        // Manually acknowledge the message after successful processing
+        await publisher.acknowledgeMessage(event.ackKey);
+    } catch (error) {
+        // Handle error - message will not be acknowledged and will be reprocessed
+        console.error('Failed to process user registration:', error);
+    }
+});
 ```
 
+The `externalAcknowledgement` option allows you to manually control when messages are acknowledged. This is useful when:
+- You need to ensure message processing is complete before acknowledgment
+- You want to implement custom retry logic
+- You need to coordinate acknowledgment with other operations
+- You want to implement transaction-like behavior
+
+When `externalAcknowledgement` is set to `true`:
+1. Messages won't be automatically acknowledged after delivery
+2. Each message contains an `ackKey` that must be used to acknowledge it
+3. Unacknowledged messages will be redelivered to other consumers
+4. You must explicitly call `acknowledgeMessage(event.ackKey)` after successful processing
+
+**Note:** Be careful with external acknowledgment as failing to acknowledge messages can lead to message redelivery and potential duplicate processing.
+
 ### Scheduled Publishing
 
 ```typescript
@@ -278,4 +312,4 @@ If you encounter issues:
 2. Verify that consumer groups are correctly created
 3. Monitor the DLQ for failed events
 4. Review the performance metrics for any anomalies
-5. Check the logs for detailed error messages
+5. Check the logs for detailed error messages

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "5.1.1",
+  "version": "5.2.2",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "^0.0.12",

package/src/lib/redis/duplication.js

@@ -10,7 +10,7 @@ class ContentBasedDeduplication {
         this.hmacSecret = 'content-hash-calculation';
     }
     calculateEventHash(eventData) {
-        const { eventId, timestamp, createdAt, ...hashableData } = eventData;
+        const { eventId, timestamp, createdAt, republishEvent, ...hashableData } = eventData;
         const hmac = (0, crypto_1.createHmac)('sha256', this.hmacSecret);
         return hmac.update(JSON.stringify(hashableData)).digest('hex');
     }

package/src/lib/redis/registry.js

@@ -62,11 +62,18 @@ class RedisRegistry {
                     },
                 });
             }
-            else
+            else if (RedisRegistry.options.sentinels) {
+                ref = new ioredis_1.default({
+                    ...RedisRegistry.options.sentinels.options,
+                    sentinels: RedisRegistry.options.sentinels.nodes,
+                });
+            }
+            else {
                 ref = new ioredis_1.default({
                     ...RedisRegistry.options.redis,
                     db: storeRef,
                 });
+            }
         }
         return ref;
     }

package/src/lib/redis/streams.d.ts

@@ -200,4 +200,7 @@ export declare class Streams {
      * circuit is OPEN
      */
     processStoredEvents(): Promise<void>;
+    acknowledgeMessage(ackKey: string): Promise<void>;
+    private frameMessageKey;
+    private demergeMessageKey;
 }

package/src/lib/redis/streams.js

@@ -233,10 +233,11 @@ class Streams {
             initialDelay: this.config.initialRetryDelay,
             filterKeepAlive: this.config.filterKeepAlive,
             publishOnceGuarantee: false,
+            externalAcknowledgement: false,
             ...listenerOptions,
         };
         const subscriptionId = (0, id_1.generateID)('HEX');
-        return this.listenInternals(eventName, subscriptionId, options.eventFilter, options.filterKeepAlive, options.publishOnceGuarantee).pipe((0, rxjs_1.retry)({
+        return this.listenInternals(eventName, subscriptionId, options.eventFilter, options.filterKeepAlive, options.publishOnceGuarantee, options.externalAcknowledgement).pipe((0, rxjs_1.retry)({
             count: options.maxRetries,
             delay: (error, retryAttempt) => {
                 const delay = options.initialDelay * Math.pow(2, retryAttempt);
@@ -311,7 +312,7 @@ class Streams {
         logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Consumer Registered and created with ${this.instanceId} under ${this.consumerGroupName} with ${createConsumerStatus} consumers and with the following status ${JSON.stringify({ addToCGSet, addToFlushSet })}`);
         return createConsumerStatus === 0 || createConsumerStatus === 1;
     }
-    listenInternals(eventName, subscriptionId, eventFilter, filterKeepAlive = 24 * 60 * 60 * 1000, publishOnceGuarantee = false) {
+    listenInternals(eventName, subscriptionId, eventFilter, filterKeepAlive = 24 * 60 * 60 * 1000, publishOnceGuarantee = false, externalAcknowledgement = false) {
         if (!this.subscriptions.has(eventName)) {
             this.subscriptions.set(eventName, new Map());
         }
@@ -327,7 +328,6 @@ class Streams {
             /** Clear earlier unprocessed messages. Runs every 10 seconds */
             await processMessage(this.redisGroups, '0', new tracker_1.MetricsTracker(), false);
         });
-        let lastMatchTime = Date.now();
         const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1), (0, rxjs_1.finalize)(() => {
             /** Cleanup timer */
             timer.unsubscribe();
@@ -393,13 +393,14 @@ class Streams {
                         }
                     }
                     try {
+                        const ackKey = this.frameMessageKey(streamName, messageId);
                         const subscriptions = this.subscriptions.get(eventName);
                         if (subscriptions) {
                             const subscriptionEntries = Array.from(subscriptions.entries());
                             for (let i = 0; i < subscriptionEntries.length; i++) {
                                 const [subId, sub] = subscriptionEntries[i];
                                 if (!sub.filter || sub.filter(eventData)) {
-                                    sub.subject.next(eventData);
+                                    sub.subject.next({ ...eventData, ackKey });
                                     sub.lastMatchTime = Date.now();
                                 }
                                 else if (Date.now() - sub.lastMatchTime > sub.keepAlive) {
@@ -414,8 +415,8 @@ class Streams {
                                 }
                             }
                         }
-                        await redisClient.xack(streamName, this.consumerGroupName, messageId);
-                        await redisClient.zadd(`ack:${streamName}`, Date.now().toString(), messageId);
+                        if (!externalAcknowledgement)
+                            this.acknowledgeMessage(ackKey);
                     }
                     catch (processingError) {
                         logger_1.PUBLISHER_LOGGER.error(`Processing error for message ${messageId}:`, processingError);
@@ -497,7 +498,7 @@ class Streams {
                 logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Stream Notification Received for event ${eventName} with message ID ${messageIdRead}`);
                 await processMessage(this.redisGroups, messageIdRead, tracker, multicastRead);
                 const metrics = tracker.getMetrics();
-                logger_1.PERFORMANCE_LOGGER.log(`STIME;${messageIdRead};${data.eventName};${Date.now()};${metrics.totalTime};${metrics.redisOperationTime};${metrics.processingTime}`);
+                logger_1.PERFORMANCE_LOGGER.log(`STIME;${messageIdRead};${eventName};${Date.now()};${metrics.totalTime};${metrics.redisOperationTime};${metrics.processingTime}`);
                 if (this.metricsCollector) {
                     this.metricsCollector.addMetrics(tracker.getMetrics());
                 }
@@ -672,5 +673,17 @@ class Streams {
             await this.circuitBreaker.clearStoredEvents();
         }
     }
+    async acknowledgeMessage(ackKey) {
+        const { streamName, messageId } = this.demergeMessageKey(ackKey);
+        await this.redisGroups.xack(streamName, this.consumerGroupName, messageId);
+        await this.redisGroups.zadd(`ack:${streamName}`, Date.now().toString(), messageId);
+    }
+    frameMessageKey(streamName, messageId) {
+        return `${streamName}##${messageId}`;
+    }
+    demergeMessageKey(messageKey) {
+        const [streamName, messageId] = messageKey.split('##');
+        return { streamName, messageId };
+    }
 }
 exports.Streams = Streams;

package/src/lib/redis/types.d.ts

@@ -5,6 +5,7 @@ export type EventData<TData, TName extends string> = {
     createdAt?: number;
     republishEvent?: string;
     repeatInterval?: number;
+    ackKey?: string;
 };
 export type PublishData<TData, TName extends string> = {
     data: TData;
@@ -13,7 +14,7 @@ export type PublishData<TData, TName extends string> = {
 };
 export type PendingMessages = [never, never, string, string, Array<[string, number]>];
 export type ClaimedMessages = Array<[never, string]>;
-import { ClusterNode, ClusterOptions, RedisOptions } from 'ioredis';
+import { ClusterNode, ClusterOptions, RedisOptions, SentinelAddress, SentinelConnectionOptions } from 'ioredis';
 import { BehaviorSubject } from 'rxjs';
 export interface IOptions {
     redis?: RedisOptions;
@@ -21,6 +22,10 @@ export interface IOptions {
         options: ClusterOptions;
         nodes: Array<ClusterNode>;
     };
+    sentinels?: {
+        options: SentinelConnectionOptions;
+        nodes: Array<SentinelAddress>;
+    };
 }
 export interface IDLQEvent extends EventData<unknown, string> {
     eventId: string;
@@ -63,6 +68,7 @@ export interface IListenOptions<T> {
     eventFilter?: TEventFilter<T>;
     filterKeepAlive: number;
     publishOnceGuarantee: boolean;
+    externalAcknowledgement: boolean;
 }
 export interface BatchPublishOptions {
     batchSize?: number;