@jetit/publisher 4.1.1 → 5.1.1

package/src/lib/publisher.d.ts

@@ -2,3 +2,6 @@ export { setRedisConnectionSettings as setRedisConfig } from './redis/registry';
 export { Streams as Publisher } from './redis/streams';
 export { ScheduledProcessor as __SCHEDULER_INTERNALS__ } from './redis/scheduler';
 export { UTILS as StreamUtilityFunctions } from './redis/utils';
+export { PrometheusAdapter } from './monitoring/adapters/prom';
+export { publishBatch, publishScheduledBatch } from './redis/batch';
+export { IListenOptions, IStreamsConfig, TEventFilter } from './redis/types';

package/src/lib/publisher.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.StreamUtilityFunctions = exports.__SCHEDULER_INTERNALS__ = exports.Publisher = exports.setRedisConfig = void 0;
+exports.publishScheduledBatch = exports.publishBatch = exports.PrometheusAdapter = exports.StreamUtilityFunctions = exports.__SCHEDULER_INTERNALS__ = exports.Publisher = exports.setRedisConfig = void 0;
 var registry_1 = require("./redis/registry");
 Object.defineProperty(exports, "setRedisConfig", { enumerable: true, get: function () { return registry_1.setRedisConnectionSettings; } });
 var streams_1 = require("./redis/streams");
@@ -9,3 +9,8 @@ var scheduler_1 = require("./redis/scheduler");
 Object.defineProperty(exports, "__SCHEDULER_INTERNALS__", { enumerable: true, get: function () { return scheduler_1.ScheduledProcessor; } });
 var utils_1 = require("./redis/utils");
 Object.defineProperty(exports, "StreamUtilityFunctions", { enumerable: true, get: function () { return utils_1.UTILS; } });
+var prom_1 = require("./monitoring/adapters/prom");
+Object.defineProperty(exports, "PrometheusAdapter", { enumerable: true, get: function () { return prom_1.PrometheusAdapter; } });
+var batch_1 = require("./redis/batch");
+Object.defineProperty(exports, "publishBatch", { enumerable: true, get: function () { return batch_1.publishBatch; } });
+Object.defineProperty(exports, "publishScheduledBatch", { enumerable: true, get: function () { return batch_1.publishScheduledBatch; } });

package/src/lib/redis/batch.d.ts

@@ -0,0 +1,6 @@
+import { Streams as Publisher } from './streams';
+import { BatchPublishOptions, BatchPublishResult, PublishData } from './types';
+export declare function publishBatch<TData, TName extends string>(streams: Publisher, events: Array<PublishData<TData, TName>>, options?: BatchPublishOptions): Promise<BatchPublishResult<TData, TName>>;
+export declare function publishScheduledBatch<TData, TName extends string>(streams: Publisher, events: Array<PublishData<TData, TName>>, options: BatchPublishOptions & {
+    scheduledTime: Date;
+}): Promise<BatchPublishResult<TData, TName>>;

package/src/lib/redis/batch.js

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.publishScheduledBatch = exports.publishBatch = void 0;
+const id_1 = require("@jetit/id");
+const tracker_1 = require("../monitoring/tracker");
+const logger_1 = require("./logger");
+async function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function publishSingleEvent(streams, event, batchId, tracker, scheduledTime) {
+    try {
+        tracker.startRedisOperation();
+        let eventId = '';
+        if (scheduledTime) {
+            await streams.scheduledPublish(scheduledTime, { ...event, batchId });
+        }
+        else {
+            eventId = await streams.publish({ ...event, batchId });
+        }
+        tracker.endRedisOperation();
+        return { eventName: event.eventName, eventId, success: true };
+    }
+    catch (error) {
+        return { eventName: event.eventName, eventId: '', success: false, error: error.message };
+    }
+}
+async function publishBatchWithRetry(streams, events, options) {
+    const batchId = (0, id_1.generateID)('HEX');
+    const tracker = new tracker_1.MetricsTracker();
+    let results = [];
+    const batchSize = options.batchSize || 100;
+    const delayBetweenBatches = options.delayBetweenBatches || 1000;
+    /** First attempt: publish all events in chunks */
+    for (let i = 0; i < events.length; i += batchSize) {
+        const chunk = events.slice(i, i + batchSize);
+        tracker.startProcessing();
+        const chunkResults = await Promise.all(chunk.map((event) => publishSingleEvent(streams, event, batchId, tracker, options.scheduledTime)));
+        tracker.endProcessing();
+        results.push(...chunkResults);
+        if (i + batchSize < events.length) {
+            await delay(delayBetweenBatches);
+        }
+    }
+    /** Second attempt: retry failed events once */
+    const failedEvents = results.filter((result) => !result.success);
+    if (failedEvents.length > 0) {
+        logger_1.PERFORMANCE_LOGGER.log(`${options.scheduledTime ? 'SCHEDULED_' : ''}BATCH_RETRY;${batchId};${failedEvents.length}`);
+        const retryResults = await Promise.all(failedEvents.map((result) => {
+            const event = events.find((event) => event.eventName === result.eventName);
+            return publishSingleEvent(streams, event, batchId, tracker, options.scheduledTime);
+        }));
+        results = results.map((result) => {
+            if (!result.success) {
+                const retryResult = retryResults.find((retry) => retry.eventName === result.eventName);
+                return retryResult || result;
+            }
+            return result;
+        });
+    }
+    const metrics = tracker.getMetrics();
+    const averageEventTime = metrics.totalTime / events.length;
+    logger_1.PERFORMANCE_LOGGER.log(`${options.scheduledTime ? 'SCHEDULED_' : ''}BATCH;${batchId};${events.length};${metrics.totalTime};${metrics.redisOperationTime};${metrics.processingTime};${averageEventTime}${options.scheduledTime ? `;${options.scheduledTime.getTime()}` : ''}`);
+    return {
+        batchId,
+        results,
+        performanceMetrics: {
+            ...metrics,
+            averageEventTime,
+        },
+    };
+}
+function publishBatch(streams, events, options = {}) {
+    return publishBatchWithRetry(streams, events, options);
+}
+exports.publishBatch = publishBatch;
+function publishScheduledBatch(streams, events, options) {
+    return publishBatchWithRetry(streams, events, options);
+}
+exports.publishScheduledBatch = publishScheduledBatch;

package/src/lib/redis/dlq.d.ts

@@ -0,0 +1,18 @@
+import { RedisType } from './registry';
+import { IDLQEvent } from './types';
+export declare class DeadLetterQueue {
+    private redisClient;
+    private maxEventsThreshold;
+    constructor(redisClient: RedisType, maxEventsThreshold?: number);
+    addToDLQ(event: IDLQEvent): Promise<void>;
+    getFromDLQ(start?: number, end?: number): Promise<IDLQEvent[]>;
+    retryFromDLQ(eventId: string): Promise<boolean>;
+    removeFromDLQ(eventId: string): Promise<boolean>;
+    cleanupDLQ(): Promise<void>;
+    getDLQStats(): Promise<{
+        size: number;
+        additionRate: number;
+    }>;
+    private isRateLimitExceeded;
+    private incrementRateLimit;
+}

package/src/lib/redis/dlq.js

@@ -0,0 +1,133 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DeadLetterQueue = void 0;
+const logger_1 = require("./logger");
+const DLQ_HASH_KEY = 'dlq_events';
+const DLQ_ZSET_KEY = 'dlq_order';
+const DLQ_RATE_LIMIT_KEY = 'dlq_rate_limit';
+const DLQ_EXPIRATION = 24 * 60 * 60 * 1000; // 24 hours
+class DeadLetterQueue {
+    constructor(redisClient, maxEventsThreshold = 1000) {
+        this.redisClient = redisClient;
+        this.maxEventsThreshold = maxEventsThreshold;
+    }
+    async addToDLQ(event) {
+        try {
+            if (await this.isRateLimitExceeded()) {
+                logger_1.PUBLISHER_LOGGER.warn('DLQ: Rate limit exceeded, not adding event to DLQ');
+                return;
+            }
+            const eventJson = JSON.stringify(event);
+            await this.redisClient.multi().hset(DLQ_HASH_KEY, event.eventId, eventJson).zadd(DLQ_ZSET_KEY, event.timestamp, event.eventId).exec();
+            await this.incrementRateLimit();
+            logger_1.PUBLISHER_LOGGER.log(`DLQ: Added event ${event.eventId} to Dead Letter Queue`);
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error('DLQ: Error adding event to Dead Letter Queue', error);
+        }
+    }
+    async getFromDLQ(start = 0, end = -1) {
+        try {
+            const eventIds = await this.redisClient.zrange(DLQ_ZSET_KEY, start, end);
+            const events = await this.redisClient.hmget(DLQ_HASH_KEY, ...eventIds);
+            return events
+                .filter((x) => !!x)
+                .map((event) => JSON.parse(event ?? ''))
+                .filter((event) => event !== null);
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error('DLQ: Error retrieving events from Dead Letter Queue', error);
+            return [];
+        }
+    }
+    async retryFromDLQ(eventId) {
+        try {
+            const eventJson = await this.redisClient.hget(DLQ_HASH_KEY, eventId);
+            if (!eventJson) {
+                logger_1.PUBLISHER_LOGGER.warn(`DLQ: Event with ID ${eventId} not found in Dead Letter Queue`);
+                return false;
+            }
+            const event = JSON.parse(eventJson);
+            // Attempt to republish the event to its original stream
+            const streamName = event.originalStream;
+            const result = await this.redisClient.xadd(streamName, '*', 'data', JSON.stringify(event));
+            if (result) {
+                // If successful, remove the event from the DLQ
+                await this.removeFromDLQ(eventId);
+                logger_1.PUBLISHER_LOGGER.log(`DLQ: Successfully retried and removed event ${eventId} from Dead Letter Queue`);
+                return true;
+            }
+            else {
+                logger_1.PUBLISHER_LOGGER.error(`DLQ: Failed to republish event ${eventId} to stream ${streamName}`);
+                return false;
+            }
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`DLQ: Error retrying event ${eventId} from Dead Letter Queue`, error);
+            return false;
+        }
+    }
+    async removeFromDLQ(eventId) {
+        try {
+            const removed = await this.redisClient.multi().hdel(DLQ_HASH_KEY, eventId).zrem(DLQ_ZSET_KEY, eventId).exec();
+            if (removed && removed[0][1] === 1 && removed[1][1] === 1) {
+                logger_1.PUBLISHER_LOGGER.log(`DLQ: Successfully removed event ${eventId} from Dead Letter Queue`);
+                return true;
+            }
+            else {
+                logger_1.PUBLISHER_LOGGER.warn(`DLQ: Event with ID ${eventId} not found in Dead Letter Queue for removal`);
+                return false;
+            }
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`DLQ: Error removing event ${eventId} from Dead Letter Queue`, error);
+            return false;
+        }
+    }
+    async cleanupDLQ() {
+        const now = Date.now();
+        const cutoffTime = now - DLQ_EXPIRATION;
+        try {
+            const expiredEventIds = await this.redisClient.zrangebyscore(DLQ_ZSET_KEY, 0, cutoffTime);
+            if (expiredEventIds.length > 0) {
+                await this.redisClient
+                    .multi()
+                    .hdel(DLQ_HASH_KEY, ...expiredEventIds)
+                    .zremrangebyscore(DLQ_ZSET_KEY, 0, cutoffTime)
+                    .exec();
+            }
+            logger_1.PUBLISHER_LOGGER.log(`DLQ: Cleaned up ${expiredEventIds.length} expired events from Dead Letter Queue`);
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error('DLQ: Error cleaning up Dead Letter Queue', error);
+        }
+    }
+    async getDLQStats() {
+        try {
+            const results = await this.redisClient.multi().zcard(DLQ_ZSET_KEY).get(DLQ_RATE_LIMIT_KEY).exec();
+            if (!results) {
+                throw new Error('Failed to execute Redis commands');
+            }
+            const [sizeResult, additionRateResult] = results;
+            if (sizeResult[0] || additionRateResult[0]) {
+                throw new Error('Error executing Redis commands');
+            }
+            const size = sizeResult[1];
+            const additionRate = parseInt(additionRateResult[1] || '0', 10);
+            return { size, additionRate };
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error('DLQ: Error getting Dead Letter Queue stats', error);
+            return { size: 0, additionRate: 0 };
+        }
+    }
+    async isRateLimitExceeded() {
+        const currentRate = (await this.redisClient.get(DLQ_RATE_LIMIT_KEY)) || '0';
+        return parseInt(currentRate, 10) >= this.maxEventsThreshold;
+    }
+    async incrementRateLimit() {
+        await this.redisClient.incr(DLQ_RATE_LIMIT_KEY);
+        await this.redisClient.expire(DLQ_RATE_LIMIT_KEY, 60); // Reset rate limit after 1 minute
+    }
+}
+exports.DeadLetterQueue = DeadLetterQueue;

package/src/lib/redis/duplication.d.ts

@@ -0,0 +1,10 @@
+import { RedisType } from './registry';
+import { EventData } from './types';
+export declare class ContentBasedDeduplication {
+    private redisClient;
+    private ttl;
+    private hmacSecret;
+    constructor(redisClient: RedisType, ttl?: number);
+    private calculateEventHash;
+    isDuplicate(event: EventData<any, string>, consumerGroupName: string): Promise<boolean>;
+}

package/src/lib/redis/duplication.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ContentBasedDeduplication = void 0;
+const crypto_1 = require("crypto");
+const logger_1 = require("./logger");
+class ContentBasedDeduplication {
+    constructor(redisClient, ttl = 86400) {
+        this.redisClient = redisClient;
+        this.ttl = ttl;
+        this.hmacSecret = 'content-hash-calculation';
+    }
+    calculateEventHash(eventData) {
+        const { eventId, timestamp, createdAt, ...hashableData } = eventData;
+        const hmac = (0, crypto_1.createHmac)('sha256', this.hmacSecret);
+        return hmac.update(JSON.stringify(hashableData)).digest('hex');
+    }
+    async isDuplicate(event, consumerGroupName) {
+        if (event.data === null)
+            return false;
+        const eventHash = this.calculateEventHash(event);
+        const key = `processed:${consumerGroupName}:${eventHash}`;
+        const result = await this.redisClient.set(key, '1', 'EX', this.ttl, 'NX');
+        const isDuplicate = result === null;
+        if (isDuplicate) {
+            logger_1.PUBLISHER_LOGGER.log(`Duplicate event detected for consumer group ${consumerGroupName}`);
+        }
+        return isDuplicate;
+    }
+}
+exports.ContentBasedDeduplication = ContentBasedDeduplication;

package/src/lib/redis/scheduler.js

@@ -59,7 +59,7 @@ class ScheduledProcessor {
                 // Publish the event to each consumer group's stream
                 const streamName = `${eventData.eventName}:${consumerGroup}`;
                 const generatedKey = await this.redisPublisher
-                    .xadd(streamName, '*', 'data', JSON.stringify(eventData))
+                    .xadd(streamName, key, 'data', JSON.stringify(eventData))
                     .catch((e) => logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Publishing event ${eventData.eventName} to consumer groups: ${consumerGroups.join(', ')} failed with data ${JSON.stringify(eventData)}, ${e} `));
                 if (key === '*')
                     key = generatedKey ?? key;

package/src/lib/redis/streams.d.ts

@@ -1,15 +1,24 @@
 import { Observable } from 'rxjs';
-import { EventData, PublishData } from './types';
+import { IAggregatedMetrics, TQueryableMetrics } from '../monitoring/types';
+import { CircuitState } from '../performance/circuit_breaker';
+import { EventData, IListenOptions, IStreamsConfig, PublishData } from './types';
 export declare class Streams {
     private _redisPublisher?;
     private _redisGroups?;
+    private config;
+    private dlq;
+    private metricsCollector;
     private consumerGroupName;
     private instanceId;
     private instanceUniqueId;
     private cleanUpTimer;
     private eventsListened;
+    private subscriptions;
+    private duplicateChecker;
+    private circuitBreaker;
     private get redisPublisher();
     private get redisGroups();
+    private DEFAULT_STREAMS_CONFIG;
     /**
      * Creates a new Streams instance for a given service.
      *
@@ -24,7 +33,8 @@ export declare class Streams {
      * // Create a new Streams instance for the "POS" service
      * const streams = new Streams('POS');
      */
-    constructor(serviceName: string);
+    constructor(serviceName: string, config?: Partial<IStreamsConfig>);
+    private setupCircuitBreakerListeners;
     private runClear;
     /**
      * Publishes an event with the given data to the Redis event stream.
@@ -96,7 +106,7 @@ export declare class Streams {
      *   PUBLISHER_LOGGER.log('New order created:', event.data);
      * });
      */
-    listen<T = unknown, const TName extends string = string>(eventName: TName, maxRetries?: number, initialDelay?: number): Observable<EventData<T, TName>>;
+    listen<T = unknown, const TName extends string = string>(eventName: TName, listenerOptions?: IListenOptions<T>): Observable<EventData<T, TName>>;
     private createConsumerAndRegister;
     private listenInternals;
     /**
@@ -142,4 +152,52 @@ export declare class Streams {
         }[];
         message: string;
     }>;
+    private logPerformance;
+    /**
+     * @description
+     * This method is use to retry an event that has ended in the dead letter queue,
+     * which happens after the first retry.
+     */
+    retryFromDLQ(eventId: string): Promise<boolean>;
+    /**
+     * @description
+     * This returns the number of items and the rate at which events are added
+     * to the queue. The queue is global and hence remains as is
+     */
+    getDLQStats(): Promise<{
+        size: number;
+        additionRate: number;
+    }>;
+    private removeSubscription;
+    /**
+     * @description
+     * This is a simple helper utility that can be used externally to create alerts based
+     * on thresholds that can be provided into the function. It returns true/false for each
+     * key that is provided. Not all keys are required
+     */
+    checkThresholds(thresholds: Partial<TQueryableMetrics>): Promise<Record<string, boolean>>;
+    /**
+     * @description
+     * This will return you the stats of the publisher for the last 6 hours after cleaning
+     */
+    getMetrics(startTime: number, endTime: number): Promise<IAggregatedMetrics[]>;
+    /**
+     * @description
+     * This will return you the latest stats of the publisher
+     */
+    getLatestMetrics(): Promise<IAggregatedMetrics | null>;
+    /**
+     * @description
+     * This returns the status of the performance control setup. This includes
+     * the circuit breaker
+     */
+    getPerformanceControlStatus(): Promise<{
+        circuitBreakerState: CircuitState;
+    }>;
+    /**
+     * @description
+     * This is a manual control to process stored events in case the
+     * circuit is OPEN
+     */
+    processStoredEvents(): Promise<void>;
 }