@jetit/publisher 5.0.0 → 5.1.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jetit/publisher",
-  "version": "5.0.0",
+  "version": "5.1.1",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "^0.0.12",

package/src/lib/monitoring/adapters/prom.d.ts

@@ -21,6 +21,11 @@ export declare class PrometheusAdapter {
     private subscribeErrorCount;
     private individualQueueDepth;
     private duplicateEventCount;
+    private messageRatePublish;
+    private messageRateSubscribe;
+    private processingTimeHistogram;
+    private redisCommandLatency;
+    private consumerLag;
     /**
      *
      * @param streams [Publisher]

package/src/lib/monitoring/adapters/prom.js

@@ -75,6 +75,36 @@ class PrometheusAdapter {
             labelNames: ['queue_name'],
             registers: [this.registry],
         });
+        this.messageRatePublish = new this.promClient.Gauge({
+            name: 'message_rate_publish',
+            help: 'Number of messages published per event type',
+            labelNames: ['event_type'],
+            registers: [this.registry],
+        });
+        this.messageRateSubscribe = new this.promClient.Gauge({
+            name: 'message_rate_subscribe',
+            help: 'Number of messages subscribed per event type',
+            labelNames: ['event_type'],
+            registers: [this.registry],
+        });
+        this.processingTimeHistogram = new this.promClient.Histogram({
+            name: 'processing_time_histogram',
+            help: 'Distribution of event processing times',
+            buckets: [0, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000],
+            registers: [this.registry],
+        });
+        this.redisCommandLatency = new this.promClient.Gauge({
+            name: 'redis_command_latency',
+            help: 'Latency of Redis commands',
+            labelNames: ['command'],
+            registers: [this.registry],
+        });
+        this.consumerLag = new this.promClient.Gauge({
+            name: 'consumer_lag',
+            help: 'Lag of consumers in milliseconds',
+            labelNames: ['consumer_group'],
+            registers: [this.registry],
+        });
         this.registry.registerMetric(this.queueDepth);
         this.registry.registerMetric(this.dlqSize);
         this.registry.registerMetric(this.dlqRate);
@@ -87,6 +117,11 @@ class PrometheusAdapter {
         this.registry.registerMetric(this.subscribeErrorCount);
         this.registry.registerMetric(this.individualQueueDepth);
         this.registry.registerMetric(this.duplicateEventCount);
+        this.registry.registerMetric(this.messageRatePublish);
+        this.registry.registerMetric(this.messageRateSubscribe);
+        this.registry.registerMetric(this.processingTimeHistogram);
+        this.registry.registerMetric(this.redisCommandLatency);
+        this.registry.registerMetric(this.consumerLag);
     }
     async updateMetrics() {
         const metrics = await this.streams.getLatestMetrics();
@@ -111,6 +146,24 @@ class PrometheusAdapter {
                 this.individualQueueDepth.set({ queue_name: queueName }, depth);
             });
         }
+        Object.entries(metrics.messageRatePublish).forEach(([eventType, rate]) => {
+            this.messageRatePublish.set({ event_type: eventType }, rate);
+        });
+        Object.entries(metrics.messageRateSubscribe).forEach(([eventType, rate]) => {
+            this.messageRateSubscribe.set({ event_type: eventType }, rate);
+        });
+        Object.entries(metrics.processingTimeDistribution).forEach(([bucket, count]) => {
+            const [lower, upper] = bucket.split('-').map(Number);
+            for (let i = 0; i < count; i++) {
+                this.processingTimeHistogram.observe(upper || lower);
+            }
+        });
+        Object.entries(metrics.redisCommandLatencies).forEach(([command, latency]) => {
+            this.redisCommandLatency.set({ command }, latency);
+        });
+        Object.entries(metrics.consumerLag).forEach(([consumerGroup, lag]) => {
+            this.consumerLag.set({ consumer_group: consumerGroup }, lag);
+        });
     }
     /**
      * @param app This needs to be an instance of express or fastify something that supports the express api

package/src/lib/monitoring/collector.d.ts

@@ -10,6 +10,11 @@ export declare class MetricsCollector {
     constructor(config: IMetricsCollectorConfig, dlq: DeadLetterQueue);
     addMetrics(metrics: IPerformanceMetrics): void;
     private aggregateAndStoreMetrics;
+    private aggregateMessageRates;
+    private aggregateHistogram;
+    private aggregateRedisCommandLatencies;
+    private aggregateConsumerLag;
+    private calculateAverages;
     private getQueueDepth;
     private storeMetrics;
     getMetrics(startTime: number, endTime: number): Promise<IAggregatedMetrics[]>;

package/src/lib/monitoring/collector.js

@@ -32,6 +32,11 @@ class MetricsCollector {
             dlqRate: 0,
             operationCount: this.metrics.length,
             duplicateEventsCount: 0,
+            messageRatePublish: {},
+            messageRateSubscribe: {},
+            processingTimeDistribution: {},
+            redisCommandLatencies: {},
+            consumerLag: {},
         };
         for (const metric of this.metrics) {
             aggregated.totalTime += metric.totalTime;
@@ -41,6 +46,11 @@ class MetricsCollector {
             aggregated.publishErrorCount += metric.publishErrorCount;
             aggregated.subscribeErrorCount += metric.subscribeErrorCount;
             aggregated.duplicateEventsCount += metric.duplicateEventsCount;
+            this.aggregateMessageRates(aggregated.messageRatePublish, metric.messageRatePublish);
+            this.aggregateMessageRates(aggregated.messageRateSubscribe, metric.messageRateSubscribe);
+            this.aggregateHistogram(aggregated.processingTimeDistribution, metric.processingTimeDistribution);
+            this.aggregateRedisCommandLatencies(aggregated.redisCommandLatencies, metric.redisCommandLatencies);
+            this.aggregateConsumerLag(aggregated.consumerLag, metric.consumerLag);
         }
         // Calculate averages for time-based metrics
         aggregated.totalTime /= aggregated.operationCount;
@@ -49,21 +59,65 @@ class MetricsCollector {
         const dlqStats = await this.dlq.getDLQStats();
         aggregated.dlqSize = dlqStats.size;
         aggregated.dlqRate = dlqStats.additionRate;
+        this.calculateAverages(aggregated.redisCommandLatencies);
+        this.calculateAverages(aggregated.consumerLag);
         await this.storeMetrics(aggregated);
         this.metrics = [];
     }
+    aggregateMessageRates(target, source) {
+        for (const [eventType, count] of Object.entries(source)) {
+            if (!target[eventType])
+                target[eventType] = 0;
+            target[eventType] += count;
+        }
+    }
+    aggregateHistogram(target, source) {
+        for (const [bucket, count] of Object.entries(source)) {
+            if (!target[bucket])
+                target[bucket] = 0;
+            target[bucket] += count;
+        }
+    }
+    aggregateRedisCommandLatencies(target, source) {
+        for (const [command, { total, count }] of Object.entries(source)) {
+            if (!target[command]) {
+                target[command] = { total: 0, count: 0 };
+            }
+            target[command].total += total;
+            target[command].count += count;
+        }
+    }
+    aggregateConsumerLag(target, source) {
+        for (const [consumerGroup, { total, count }] of Object.entries(source)) {
+            if (!target[consumerGroup]) {
+                target[consumerGroup] = { total: 0, count: 0 };
+            }
+            target[consumerGroup].total += total;
+            target[consumerGroup].count += count;
+        }
+    }
+    calculateAverages(data) {
+        for (const key in data) {
+            if (data[key].count > 0) {
+                data[key].total = data[key].total / data[key].count;
+            }
+            else {
+                data[key].total = 0;
+            }
+        }
+    }
     async getQueueDepth() {
         let totalDepth = 0;
         const individualDepths = {};
         let cursor = '0';
         do {
-            const [nextCursor, keys] = await this.redisClient.scan(cursor, 'MATCH', '*:cg-*', 'COUNT', 100);
+            const [nextCursor, keys] = await this.redisClient.scan(cursor, 'MATCH', `ack:*:cg-*`, 'COUNT', 100);
             cursor = nextCursor;
             if (keys.length > 0) {
                 const pipeline = this.redisClient.pipeline();
                 keys.forEach((key) => {
-                    pipeline.xlen(key);
-                    pipeline.zcard(`ack:${key}`);
+                    pipeline.xlen(key.slice(4));
+                    pipeline.zcard(key);
                 });
                 const results = await pipeline.exec();
                 if (!results) {

package/src/lib/monitoring/tracker.d.ts

@@ -11,5 +11,10 @@ export declare class MetricsTracker {
     incrementDuplicateEvent(): void;
     incrementErrorCount(type: 'publish' | 'subscribe'): void;
     getMetrics(): IPerformanceMetrics;
+    incrementMessageRate(type: 'publish' | 'subscribe', eventType: string): void;
+    addProcessingTime(time: number): void;
+    private getBucket;
+    addRedisCommandLatency(command: string, time: number): void;
+    setConsumerLag(consumerGroup: string, lag: number): void;
     reset(): void;
 }

package/src/lib/monitoring/tracker.js

@@ -12,6 +12,11 @@ class MetricsTracker {
             publishErrorCount: 0,
             subscribeErrorCount: 0,
             duplicateEventsCount: 0,
+            messageRatePublish: {},
+            messageRateSubscribe: {},
+            processingTimeDistribution: {},
+            redisCommandLatencies: {},
+            consumerLag: {},
         };
         this.startTime = (0, node_process_1.hrtime)();
     }
@@ -43,6 +48,43 @@ class MetricsTracker {
         this.metrics.totalTime = totalSeconds * 1000 + totalNanoseconds / 1e6;
         return { ...this.metrics };
     }
+    incrementMessageRate(type, eventType) {
+        const key = `messageRate${type.charAt(0).toUpperCase() + type.slice(1)}`;
+        if (!this.metrics[key][eventType]) {
+            this.metrics[key][eventType] = 0;
+        }
+        this.metrics[key][eventType]++;
+    }
+    addProcessingTime(time) {
+        const bucket = this.getBucket(time);
+        if (!this.metrics.processingTimeDistribution[bucket]) {
+            this.metrics.processingTimeDistribution[bucket] = 0;
+        }
+        this.metrics.processingTimeDistribution[bucket]++;
+    }
+    getBucket(time) {
+        const buckets = [0, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000];
+        for (let i = 0; i < buckets.length; i++) {
+            if (time <= buckets[i]) {
+                return i === 0 ? `0-${buckets[i]}` : `${buckets[i - 1] + 1}-${buckets[i]}`;
+            }
+        }
+        return `>${buckets[buckets.length - 1]}`;
+    }
+    addRedisCommandLatency(command, time) {
+        if (!this.metrics.redisCommandLatencies[command]) {
+            this.metrics.redisCommandLatencies[command] = { total: 0, count: 0 };
+        }
+        this.metrics.redisCommandLatencies[command].total += time;
+        this.metrics.redisCommandLatencies[command].count++;
+    }
+    setConsumerLag(consumerGroup, lag) {
+        if (!this.metrics.consumerLag[consumerGroup]) {
+            this.metrics.consumerLag[consumerGroup] = { total: 0, count: 0 };
+        }
+        this.metrics.consumerLag[consumerGroup].total += lag;
+        this.metrics.consumerLag[consumerGroup].count++;
+    }
     reset() {
         this.startTime = (0, node_process_1.hrtime)();
         this.metrics = {
@@ -53,6 +95,11 @@ class MetricsTracker {
             publishErrorCount: 0,
             subscribeErrorCount: 0,
             duplicateEventsCount: 0,
+            messageRatePublish: {},
+            messageRateSubscribe: {},
+            processingTimeDistribution: {},
+            redisCommandLatencies: {},
+            consumerLag: {},
         };
     }
 }

package/src/lib/monitoring/types.d.ts

@@ -19,7 +19,7 @@ export interface IMetricsCollectorConfig {
     collectionInterval: number;
     retentionPeriod: number;
 }
-export type TQueryableMetrics = Omit<IAggregatedMetrics, 'timestamp' | 'individualQueueDepts'>;
+export type TQueryableMetrics = Omit<IAggregatedMetrics, 'timestamp' | 'individualQueueDepts' | 'messageRatePublish' | 'messageRateSubscribe' | 'processingTimeDistribution' | 'redisCommandLatencies' | 'consumerLag'>;
 export interface IQueueDepths {
     total: number;
     individual: {
@@ -34,4 +34,25 @@ export interface IPerformanceMetrics {
     publishErrorCount: number;
     subscribeErrorCount: number;
     duplicateEventsCount: number;
+    messageRatePublish: {
+        [eventType: string]: number;
+    };
+    messageRateSubscribe: {
+        [eventType: string]: number;
+    };
+    processingTimeDistribution: {
+        [bucket: string]: number;
+    };
+    redisCommandLatencies: {
+        [command: string]: {
+            total: number;
+            count: number;
+        };
+    };
+    consumerLag: {
+        [consumerGroup: string]: {
+            total: number;
+            count: number;
+        };
+    };
 }

package/src/lib/redis/duplication.js

@@ -15,7 +15,9 @@ class ContentBasedDeduplication {
         return hmac.update(JSON.stringify(hashableData)).digest('hex');
     }
     async isDuplicate(event, consumerGroupName) {
-        const eventHash = this.calculateEventHash(event.data);
+        if (event.data === null)
+            return false;
+        const eventHash = this.calculateEventHash(event);
         const key = `processed:${consumerGroupName}:${eventHash}`;
         const result = await this.redisClient.set(key, '1', 'EX', this.ttl, 'NX');
         const isDuplicate = result === null;

package/src/lib/redis/scheduler.js

@@ -59,7 +59,7 @@ class ScheduledProcessor {
                 // Publish the event to each consumer group's stream
                 const streamName = `${eventData.eventName}:${consumerGroup}`;
                 const generatedKey = await this.redisPublisher
-                    .xadd(streamName, '*', 'data', JSON.stringify(eventData))
+                    .xadd(streamName, key, 'data', JSON.stringify(eventData))
                     .catch((e) => logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Publishing event ${eventData.eventName} to consumer groups: ${consumerGroups.join(', ')} failed with data ${JSON.stringify(eventData)}, ${e} `));
                 if (key === '*')
                     key = generatedKey ?? key;

package/src/lib/redis/streams.js

@@ -53,7 +53,7 @@ class Streams {
                 errorThresholdPercentage: 50,
                 openStateDuration: 30000,
                 halfOpenStateMaxAttempts: 10,
-                maxStoredEvents: 5000,
+                maxStoredEvents: 10000,
             },
         };
         /** Initialise Config properties */
@@ -149,6 +149,7 @@ class Streams {
                     const generatedKey = await this.redisPublisher.xadd(streamName, key, 'data', JSON.stringify(data));
                     tracker.endRedisOperation();
                     tracker.incrementEventCount();
+                    tracker.incrementMessageRate('publish', data.eventName);
                     if (this.metricsCollector) {
                         this.metricsCollector.addMetrics(tracker.getMetrics());
                     }
@@ -256,7 +257,13 @@ class Streams {
         this.eventsListened.push(eventName);
         try {
             // Check if the consumer group already exists
-            const groupInfo = (await this.redisGroups.xinfo('GROUPS', streamName));
+            let groupInfo = [];
+            try {
+                groupInfo = (await this.redisGroups.xinfo('GROUPS', streamName));
+            }
+            catch (e) {
+                // Do nothing
+            }
             let groupExists = false;
             for (const group of groupInfo) {
                 if (group[1] === this.consumerGroupName) {
@@ -276,7 +283,13 @@ class Streams {
             logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Group creation failed with error ${e.message} for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
         }
         // Check if the consumer already exists in the group
-        const consumers = (await this.redisGroups.xinfo('CONSUMERS', streamName, this.consumerGroupName));
+        let consumers = [];
+        try {
+            consumers = (await this.redisGroups.xinfo('CONSUMERS', streamName, this.consumerGroupName));
+        }
+        catch (e) {
+            // Do nothing
+        }
         let consumerExists = false;
         for (const consumer of consumers) {
             if (consumer[1] === this.instanceId) {
@@ -361,6 +374,10 @@ class Streams {
                 else {
                     const messages = (await redisClient.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'STREAMS', streamName, '>'));
                     if (messages && messages.length) {
+                        if (messageId === '0') {
+                            messageId = messages[0][1][0][0];
+                            logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reprocessing unprocessed message with id: ${messageId}`);
+                        }
                         eventData = JSON.parse(messages[0][1][0][1][1]);
                     }
                 }
@@ -412,6 +429,11 @@ class Streams {
                         };
                         await this.dlq.addToDLQ(dlqEvent);
                     }
+                    tracker.incrementMessageRate('subscribe', eventData.eventName);
+                    const processingTime = Date.now() - eventData.createdAt;
+                    tracker.addProcessingTime(processingTime);
+                    const lag = Date.now() - eventData.createdAt;
+                    tracker.setConsumerLag(this.consumerGroupName, lag);
                 }
                 else {
                     logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Message ${messageId} not found for ${streamName}`);