@jetit/publisher 4.1.0 → 5.0.0

package/src/lib/monitoring/collector.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MetricsCollector = void 0;
+class MetricsCollector {
+    constructor(config, dlq) {
+        this.metrics = [];
+        this.metricsKey = 'publisher:metrics';
+        this.redisClient = config.redisClient;
+        this.collectionInterval = config.collectionInterval;
+        this.retentionPeriod = config.retentionPeriod;
+        this.dlq = dlq;
+        setInterval(() => this.aggregateAndStoreMetrics(), this.collectionInterval);
+    }
+    addMetrics(metrics) {
+        this.metrics.push(metrics);
+    }
+    async aggregateAndStoreMetrics() {
+        if (this.metrics.length === 0)
+            return;
+        const queueDepths = await this.getQueueDepth();
+        const aggregated = {
+            timestamp: Date.now(),
+            totalTime: 0,
+            redisOperationTime: 0,
+            processingTime: 0,
+            eventCount: 0,
+            subscribeErrorCount: 0,
+            publishErrorCount: 0,
+            queueDepth: queueDepths.total,
+            individualQueueDepts: queueDepths.individual,
+            dlqSize: 0,
+            dlqRate: 0,
+            operationCount: this.metrics.length,
+            duplicateEventsCount: 0,
+        };
+        for (const metric of this.metrics) {
+            aggregated.totalTime += metric.totalTime;
+            aggregated.redisOperationTime += metric.redisOperationTime;
+            aggregated.processingTime += metric.processingTime;
+            aggregated.eventCount += metric.eventCount;
+            aggregated.publishErrorCount += metric.publishErrorCount;
+            aggregated.subscribeErrorCount += metric.subscribeErrorCount;
+            aggregated.duplicateEventsCount += metric.duplicateEventsCount;
+        }
+        // Calculate averages for time-based metrics
+        aggregated.totalTime /= aggregated.operationCount;
+        aggregated.redisOperationTime /= aggregated.operationCount;
+        aggregated.processingTime /= aggregated.operationCount;
+        const dlqStats = await this.dlq.getDLQStats();
+        aggregated.dlqSize = dlqStats.size;
+        aggregated.dlqRate = dlqStats.additionRate;
+        await this.storeMetrics(aggregated);
+        this.metrics = [];
+    }
+    async getQueueDepth() {
+        let totalDepth = 0;
+        const individualDepths = {};
+        let cursor = '0';
+        do {
+            const [nextCursor, keys] = await this.redisClient.scan(cursor, 'MATCH', '*:cg-*', 'COUNT', 100);
+            cursor = nextCursor;
+            if (keys.length > 0) {
+                const pipeline = this.redisClient.pipeline();
+                keys.forEach((key) => {
+                    pipeline.xlen(key);
+                    pipeline.zcard(`ack:${key}`);
+                });
+                const results = await pipeline.exec();
+                if (!results) {
+                    continue;
+                }
+                for (let i = 0; i < results.length; i += 2) {
+                    const key = keys[i / 2];
+                    const [streamLengthErr, streamLength] = results[i];
+                    const [ackCountErr, ackCount] = results[i + 1];
+                    if (streamLengthErr) {
+                        console.error(`Error getting length for key: ${streamLengthErr}`);
+                        continue;
+                    }
+                    if (ackCountErr) {
+                        console.error(`Error getting ack count for key: ${ackCountErr}`);
+                        continue;
+                    }
+                    const queueDepth = Math.max(0, streamLength - ackCount);
+                    totalDepth += queueDepth;
+                    individualDepths[key] = queueDepth;
+                }
+            }
+        } while (cursor !== '0');
+        return { total: totalDepth, individual: individualDepths };
+    }
+    async storeMetrics(metrics) {
+        const score = metrics.timestamp;
+        const member = JSON.stringify(metrics);
+        await this.redisClient.zadd(this.metricsKey, score, member);
+        // Remove old metrics based on retention period
+        const cutoffTime = Date.now() - this.retentionPeriod;
+        await this.redisClient.zremrangebyscore(this.metricsKey, 0, cutoffTime);
+    }
+    async getMetrics(startTime, endTime) {
+        const metricsData = await this.redisClient.zrangebyscore(this.metricsKey, startTime, endTime);
+        return metricsData.map((data) => JSON.parse(data));
+    }
+    async getLatestMetrics() {
+        const latestMetrics = await this.redisClient.zrevrange(this.metricsKey, 0, 0);
+        return latestMetrics.length > 0 ? JSON.parse(latestMetrics[0]) : null;
+    }
+}
+exports.MetricsCollector = MetricsCollector;

package/src/lib/monitoring/tracker.d.ts

@@ -0,0 +1,15 @@
+import { IPerformanceMetrics } from './types';
+export declare class MetricsTracker {
+    private startTime;
+    private metrics;
+    constructor();
+    startRedisOperation(): void;
+    endRedisOperation(): void;
+    startProcessing(): void;
+    endProcessing(): void;
+    incrementEventCount(): void;
+    incrementDuplicateEvent(): void;
+    incrementErrorCount(type: 'publish' | 'subscribe'): void;
+    getMetrics(): IPerformanceMetrics;
+    reset(): void;
+}

package/src/lib/monitoring/tracker.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MetricsTracker = void 0;
+const node_process_1 = require("node:process");
+class MetricsTracker {
+    constructor() {
+        this.metrics = {
+            totalTime: 0,
+            redisOperationTime: 0,
+            processingTime: 0,
+            eventCount: 0,
+            publishErrorCount: 0,
+            subscribeErrorCount: 0,
+            duplicateEventsCount: 0,
+        };
+        this.startTime = (0, node_process_1.hrtime)();
+    }
+    startRedisOperation() {
+        this.startTime = (0, node_process_1.hrtime)();
+    }
+    endRedisOperation() {
+        const [seconds, nanoseconds] = (0, node_process_1.hrtime)(this.startTime);
+        this.metrics.redisOperationTime += seconds * 1000 + nanoseconds / 1e6;
+    }
+    startProcessing() {
+        this.startTime = (0, node_process_1.hrtime)();
+    }
+    endProcessing() {
+        const [seconds, nanoseconds] = (0, node_process_1.hrtime)(this.startTime);
+        this.metrics.processingTime += seconds * 1000 + nanoseconds / 1e6;
+    }
+    incrementEventCount() {
+        this.metrics.eventCount++;
+    }
+    incrementDuplicateEvent() {
+        this.metrics.duplicateEventsCount++;
+    }
+    incrementErrorCount(type) {
+        this.metrics[`${type}ErrorCount`]++;
+    }
+    getMetrics() {
+        const [totalSeconds, totalNanoseconds] = (0, node_process_1.hrtime)(this.startTime);
+        this.metrics.totalTime = totalSeconds * 1000 + totalNanoseconds / 1e6;
+        return { ...this.metrics };
+    }
+    reset() {
+        this.startTime = (0, node_process_1.hrtime)();
+        this.metrics = {
+            totalTime: 0,
+            redisOperationTime: 0,
+            processingTime: 0,
+            eventCount: 0,
+            publishErrorCount: 0,
+            subscribeErrorCount: 0,
+            duplicateEventsCount: 0,
+        };
+    }
+}
+exports.MetricsTracker = MetricsTracker;

package/src/lib/monitoring/types.d.ts

@@ -0,0 +1,37 @@
+import { RedisType } from '../redis/registry';
+export interface IAggregatedMetrics extends IPerformanceMetrics {
+    timestamp: number;
+    queueDepth: number;
+    dlqSize: number;
+    dlqRate: number;
+    operationCount: number;
+    totalTime: number;
+    redisOperationTime: number;
+    processingTime: number;
+    eventCount: number;
+    publishErrorCount: number;
+    subscribeErrorCount: number;
+    individualQueueDepts: IQueueDepths['individual'];
+    duplicateEventsCount: number;
+}
+export interface IMetricsCollectorConfig {
+    redisClient: RedisType;
+    collectionInterval: number;
+    retentionPeriod: number;
+}
+export type TQueryableMetrics = Omit<IAggregatedMetrics, 'timestamp' | 'individualQueueDepts'>;
+export interface IQueueDepths {
+    total: number;
+    individual: {
+        [key: string]: number;
+    };
+}
+export interface IPerformanceMetrics {
+    totalTime: number;
+    redisOperationTime: number;
+    processingTime: number;
+    eventCount: number;
+    publishErrorCount: number;
+    subscribeErrorCount: number;
+    duplicateEventsCount: number;
+}

package/src/lib/monitoring/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/src/lib/performance/circuit_breaker.d.ts

@@ -0,0 +1,29 @@
+/// <reference types="node" />
+import { EventEmitter } from 'events';
+import { RedisType } from '../redis/registry';
+import { EventData, IStreamsConfig } from '../redis/types';
+export declare enum CircuitState {
+    CLOSED = 0,
+    OPEN = 1,
+    HALF_OPEN = 2
+}
+export declare class CircuitBreaker extends EventEmitter {
+    private readonly config;
+    private state;
+    private failureCount;
+    private successCount;
+    private lastStateChange;
+    private readonly redisClient;
+    constructor(config: IStreamsConfig['circuitBreaker'], redisClient: RedisType);
+    recordSuccess(): Promise<void>;
+    recordFailure(): Promise<void>;
+    isAllowed(): Promise<boolean>;
+    private openCircuit;
+    private closeCircuit;
+    private halfOpenCircuit;
+    getState(): CircuitState;
+    storeEvent(event: EventData<unknown, string>): Promise<void>;
+    getStoredEvents(): Promise<EventData<unknown, string>[]>;
+    clearStoredEvents(): Promise<void>;
+    private startStateCheck;
+}

package/src/lib/performance/circuit_breaker.js

@@ -0,0 +1,103 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CircuitBreaker = exports.CircuitState = void 0;
+const events_1 = require("events");
+var CircuitState;
+(function (CircuitState) {
+    CircuitState[CircuitState["CLOSED"] = 0] = "CLOSED";
+    CircuitState[CircuitState["OPEN"] = 1] = "OPEN";
+    CircuitState[CircuitState["HALF_OPEN"] = 2] = "HALF_OPEN";
+})(CircuitState || (exports.CircuitState = CircuitState = {}));
+class CircuitBreaker extends events_1.EventEmitter {
+    constructor(config, redisClient) {
+        super();
+        this.config = config;
+        this.state = CircuitState.CLOSED;
+        this.failureCount = 0;
+        this.successCount = 0;
+        this.lastStateChange = Date.now();
+        this.redisClient = redisClient;
+        this.startStateCheck();
+    }
+    async recordSuccess() {
+        this.successCount++;
+        if (this.state === CircuitState.HALF_OPEN) {
+            if (this.successCount >= this.config.halfOpenStateMaxAttempts) {
+                await this.closeCircuit();
+            }
+        }
+    }
+    async recordFailure() {
+        this.failureCount++;
+        if (this.state === CircuitState.CLOSED) {
+            const totalAttempts = this.failureCount + this.successCount;
+            const failureRate = (this.failureCount / totalAttempts) * 100;
+            if (this.failureCount >= this.config.errorThreshold || failureRate >= this.config.errorThresholdPercentage) {
+                await this.openCircuit();
+            }
+        }
+        else if (this.state === CircuitState.HALF_OPEN) {
+            await this.openCircuit();
+        }
+    }
+    async isAllowed() {
+        const now = Date.now();
+        if (this.state === CircuitState.OPEN) {
+            if (now - this.lastStateChange >= this.config.openStateDuration) {
+                await this.halfOpenCircuit();
+            }
+            else {
+                return false;
+            }
+        }
+        return true;
+    }
+    openCircuit() {
+        if (this.state !== CircuitState.OPEN) {
+            this.state = CircuitState.OPEN;
+            this.lastStateChange = Date.now();
+            this.emit('stateChange', CircuitState.OPEN);
+        }
+    }
+    closeCircuit() {
+        if (this.state !== CircuitState.CLOSED) {
+            this.state = CircuitState.CLOSED;
+            this.lastStateChange = Date.now();
+            this.failureCount = 0;
+            this.successCount = 0;
+            this.emit('stateChange', CircuitState.CLOSED);
+        }
+    }
+    halfOpenCircuit() {
+        if (this.state !== CircuitState.HALF_OPEN) {
+            this.state = CircuitState.HALF_OPEN;
+            this.lastStateChange = Date.now();
+            this.failureCount = 0;
+            this.successCount = 0;
+            this.emit('stateChange', CircuitState.HALF_OPEN);
+        }
+    }
+    getState() {
+        return this.state;
+    }
+    async storeEvent(event) {
+        const key = `circuit_breaker:stored_events`;
+        await this.redisClient.lpush(key, JSON.stringify(event));
+        await this.redisClient.ltrim(key, 0, this.config.maxStoredEvents - 1);
+    }
+    async getStoredEvents() {
+        const key = `circuit_breaker:stored_events`;
+        const events = await this.redisClient.lrange(key, 0, -1);
+        return events.map((event) => JSON.parse(event));
+    }
+    async clearStoredEvents() {
+        const key = `circuit_breaker:stored_events`;
+        await this.redisClient.del(key);
+    }
+    startStateCheck() {
+        setInterval(() => {
+            this.isAllowed();
+        }, 2000);
+    }
+}
+exports.CircuitBreaker = CircuitBreaker;

package/src/lib/publisher.d.ts

@@ -2,3 +2,6 @@ export { setRedisConnectionSettings as setRedisConfig } from './redis/registry';
 export { Streams as Publisher } from './redis/streams';
 export { ScheduledProcessor as __SCHEDULER_INTERNALS__ } from './redis/scheduler';
 export { UTILS as StreamUtilityFunctions } from './redis/utils';
+export { PrometheusAdapter } from './monitoring/adapters/prom';
+export { publishBatch, publishScheduledBatch } from './redis/batch';
+export { IListenOptions, IStreamsConfig, TEventFilter } from './redis/types';

package/src/lib/publisher.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.StreamUtilityFunctions = exports.__SCHEDULER_INTERNALS__ = exports.Publisher = exports.setRedisConfig = void 0;
+exports.publishScheduledBatch = exports.publishBatch = exports.PrometheusAdapter = exports.StreamUtilityFunctions = exports.__SCHEDULER_INTERNALS__ = exports.Publisher = exports.setRedisConfig = void 0;
 var registry_1 = require("./redis/registry");
 Object.defineProperty(exports, "setRedisConfig", { enumerable: true, get: function () { return registry_1.setRedisConnectionSettings; } });
 var streams_1 = require("./redis/streams");
@@ -9,3 +9,8 @@ var scheduler_1 = require("./redis/scheduler");
 Object.defineProperty(exports, "__SCHEDULER_INTERNALS__", { enumerable: true, get: function () { return scheduler_1.ScheduledProcessor; } });
 var utils_1 = require("./redis/utils");
 Object.defineProperty(exports, "StreamUtilityFunctions", { enumerable: true, get: function () { return utils_1.UTILS; } });
+var prom_1 = require("./monitoring/adapters/prom");
+Object.defineProperty(exports, "PrometheusAdapter", { enumerable: true, get: function () { return prom_1.PrometheusAdapter; } });
+var batch_1 = require("./redis/batch");
+Object.defineProperty(exports, "publishBatch", { enumerable: true, get: function () { return batch_1.publishBatch; } });
+Object.defineProperty(exports, "publishScheduledBatch", { enumerable: true, get: function () { return batch_1.publishScheduledBatch; } });

package/src/lib/redis/batch.d.ts

@@ -0,0 +1,6 @@
+import { Streams as Publisher } from './streams';
+import { BatchPublishOptions, BatchPublishResult, PublishData } from './types';
+export declare function publishBatch<TData, TName extends string>(streams: Publisher, events: Array<PublishData<TData, TName>>, options?: BatchPublishOptions): Promise<BatchPublishResult<TData, TName>>;
+export declare function publishScheduledBatch<TData, TName extends string>(streams: Publisher, events: Array<PublishData<TData, TName>>, options: BatchPublishOptions & {
+    scheduledTime: Date;
+}): Promise<BatchPublishResult<TData, TName>>;

package/src/lib/redis/batch.js

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.publishScheduledBatch = exports.publishBatch = void 0;
+const id_1 = require("@jetit/id");
+const tracker_1 = require("../monitoring/tracker");
+const logger_1 = require("./logger");
+async function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function publishSingleEvent(streams, event, batchId, tracker, scheduledTime) {
+    try {
+        tracker.startRedisOperation();
+        let eventId = '';
+        if (scheduledTime) {
+            await streams.scheduledPublish(scheduledTime, { ...event, batchId });
+        }
+        else {
+            eventId = await streams.publish({ ...event, batchId });
+        }
+        tracker.endRedisOperation();
+        return { eventName: event.eventName, eventId, success: true };
+    }
+    catch (error) {
+        return { eventName: event.eventName, eventId: '', success: false, error: error.message };
+    }
+}
+async function publishBatchWithRetry(streams, events, options) {
+    const batchId = (0, id_1.generateID)('HEX');
+    const tracker = new tracker_1.MetricsTracker();
+    let results = [];
+    const batchSize = options.batchSize || 100;
+    const delayBetweenBatches = options.delayBetweenBatches || 1000;
+    /** First attempt: publish all events in chunks */
+    for (let i = 0; i < events.length; i += batchSize) {
+        const chunk = events.slice(i, i + batchSize);
+        tracker.startProcessing();
+        const chunkResults = await Promise.all(chunk.map((event) => publishSingleEvent(streams, event, batchId, tracker, options.scheduledTime)));
+        tracker.endProcessing();
+        results.push(...chunkResults);
+        if (i + batchSize < events.length) {
+            await delay(delayBetweenBatches);
+        }
+    }
+    /** Second attempt: retry failed events once */
+    const failedEvents = results.filter((result) => !result.success);
+    if (failedEvents.length > 0) {
+        logger_1.PERFORMANCE_LOGGER.log(`${options.scheduledTime ? 'SCHEDULED_' : ''}BATCH_RETRY;${batchId};${failedEvents.length}`);
+        const retryResults = await Promise.all(failedEvents.map((result) => {
+            const event = events.find((event) => event.eventName === result.eventName);
+            return publishSingleEvent(streams, event, batchId, tracker, options.scheduledTime);
+        }));
+        results = results.map((result) => {
+            if (!result.success) {
+                const retryResult = retryResults.find((retry) => retry.eventName === result.eventName);
+                return retryResult || result;
+            }
+            return result;
+        });
+    }
+    const metrics = tracker.getMetrics();
+    const averageEventTime = metrics.totalTime / events.length;
+    logger_1.PERFORMANCE_LOGGER.log(`${options.scheduledTime ? 'SCHEDULED_' : ''}BATCH;${batchId};${events.length};${metrics.totalTime};${metrics.redisOperationTime};${metrics.processingTime};${averageEventTime}${options.scheduledTime ? `;${options.scheduledTime.getTime()}` : ''}`);
+    return {
+        batchId,
+        results,
+        performanceMetrics: {
+            ...metrics,
+            averageEventTime,
+        },
+    };
+}
+function publishBatch(streams, events, options = {}) {
+    return publishBatchWithRetry(streams, events, options);
+}
+exports.publishBatch = publishBatch;
+function publishScheduledBatch(streams, events, options) {
+    return publishBatchWithRetry(streams, events, options);
+}
+exports.publishScheduledBatch = publishScheduledBatch;

package/src/lib/redis/dlq.d.ts

@@ -0,0 +1,18 @@
+import { RedisType } from './registry';
+import { IDLQEvent } from './types';
+export declare class DeadLetterQueue {
+    private redisClient;
+    private maxEventsThreshold;
+    constructor(redisClient: RedisType, maxEventsThreshold?: number);
+    addToDLQ(event: IDLQEvent): Promise<void>;
+    getFromDLQ(start?: number, end?: number): Promise<IDLQEvent[]>;
+    retryFromDLQ(eventId: string): Promise<boolean>;
+    removeFromDLQ(eventId: string): Promise<boolean>;
+    cleanupDLQ(): Promise<void>;
+    getDLQStats(): Promise<{
+        size: number;
+        additionRate: number;
+    }>;
+    private isRateLimitExceeded;
+    private incrementRateLimit;
+}

package/src/lib/redis/dlq.js

@@ -0,0 +1,133 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DeadLetterQueue = void 0;
+const logger_1 = require("./logger");
+const DLQ_HASH_KEY = 'dlq_events';
+const DLQ_ZSET_KEY = 'dlq_order';
+const DLQ_RATE_LIMIT_KEY = 'dlq_rate_limit';
+const DLQ_EXPIRATION = 24 * 60 * 60 * 1000; // 24 hours
+class DeadLetterQueue {
+    constructor(redisClient, maxEventsThreshold = 1000) {
+        this.redisClient = redisClient;
+        this.maxEventsThreshold = maxEventsThreshold;
+    }
+    async addToDLQ(event) {
+        try {
+            if (await this.isRateLimitExceeded()) {
+                logger_1.PUBLISHER_LOGGER.warn('DLQ: Rate limit exceeded, not adding event to DLQ');
+                return;
+            }
+            const eventJson = JSON.stringify(event);
+            await this.redisClient.multi().hset(DLQ_HASH_KEY, event.eventId, eventJson).zadd(DLQ_ZSET_KEY, event.timestamp, event.eventId).exec();
+            await this.incrementRateLimit();
+            logger_1.PUBLISHER_LOGGER.log(`DLQ: Added event ${event.eventId} to Dead Letter Queue`);
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error('DLQ: Error adding event to Dead Letter Queue', error);
+        }
+    }
+    async getFromDLQ(start = 0, end = -1) {
+        try {
+            const eventIds = await this.redisClient.zrange(DLQ_ZSET_KEY, start, end);
+            const events = await this.redisClient.hmget(DLQ_HASH_KEY, ...eventIds);
+            return events
+                .filter((x) => !!x)
+                .map((event) => JSON.parse(event ?? ''))
+                .filter((event) => event !== null);
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error('DLQ: Error retrieving events from Dead Letter Queue', error);
+            return [];
+        }
+    }
+    async retryFromDLQ(eventId) {
+        try {
+            const eventJson = await this.redisClient.hget(DLQ_HASH_KEY, eventId);
+            if (!eventJson) {
+                logger_1.PUBLISHER_LOGGER.warn(`DLQ: Event with ID ${eventId} not found in Dead Letter Queue`);
+                return false;
+            }
+            const event = JSON.parse(eventJson);
+            // Attempt to republish the event to its original stream
+            const streamName = event.originalStream;
+            const result = await this.redisClient.xadd(streamName, '*', 'data', JSON.stringify(event));
+            if (result) {
+                // If successful, remove the event from the DLQ
+                await this.removeFromDLQ(eventId);
+                logger_1.PUBLISHER_LOGGER.log(`DLQ: Successfully retried and removed event ${eventId} from Dead Letter Queue`);
+                return true;
+            }
+            else {
+                logger_1.PUBLISHER_LOGGER.error(`DLQ: Failed to republish event ${eventId} to stream ${streamName}`);
+                return false;
+            }
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`DLQ: Error retrying event ${eventId} from Dead Letter Queue`, error);
+            return false;
+        }
+    }
+    async removeFromDLQ(eventId) {
+        try {
+            const removed = await this.redisClient.multi().hdel(DLQ_HASH_KEY, eventId).zrem(DLQ_ZSET_KEY, eventId).exec();
+            if (removed && removed[0][1] === 1 && removed[1][1] === 1) {
+                logger_1.PUBLISHER_LOGGER.log(`DLQ: Successfully removed event ${eventId} from Dead Letter Queue`);
+                return true;
+            }
+            else {
+                logger_1.PUBLISHER_LOGGER.warn(`DLQ: Event with ID ${eventId} not found in Dead Letter Queue for removal`);
+                return false;
+            }
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error(`DLQ: Error removing event ${eventId} from Dead Letter Queue`, error);
+            return false;
+        }
+    }
+    async cleanupDLQ() {
+        const now = Date.now();
+        const cutoffTime = now - DLQ_EXPIRATION;
+        try {
+            const expiredEventIds = await this.redisClient.zrangebyscore(DLQ_ZSET_KEY, 0, cutoffTime);
+            if (expiredEventIds.length > 0) {
+                await this.redisClient
+                    .multi()
+                    .hdel(DLQ_HASH_KEY, ...expiredEventIds)
+                    .zremrangebyscore(DLQ_ZSET_KEY, 0, cutoffTime)
+                    .exec();
+            }
+            logger_1.PUBLISHER_LOGGER.log(`DLQ: Cleaned up ${expiredEventIds.length} expired events from Dead Letter Queue`);
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error('DLQ: Error cleaning up Dead Letter Queue', error);
+        }
+    }
+    async getDLQStats() {
+        try {
+            const results = await this.redisClient.multi().zcard(DLQ_ZSET_KEY).get(DLQ_RATE_LIMIT_KEY).exec();
+            if (!results) {
+                throw new Error('Failed to execute Redis commands');
+            }
+            const [sizeResult, additionRateResult] = results;
+            if (sizeResult[0] || additionRateResult[0]) {
+                throw new Error('Error executing Redis commands');
+            }
+            const size = sizeResult[1];
+            const additionRate = parseInt(additionRateResult[1] || '0', 10);
+            return { size, additionRate };
+        }
+        catch (error) {
+            logger_1.PUBLISHER_LOGGER.error('DLQ: Error getting Dead Letter Queue stats', error);
+            return { size: 0, additionRate: 0 };
+        }
+    }
+    async isRateLimitExceeded() {
+        const currentRate = (await this.redisClient.get(DLQ_RATE_LIMIT_KEY)) || '0';
+        return parseInt(currentRate, 10) >= this.maxEventsThreshold;
+    }
+    async incrementRateLimit() {
+        await this.redisClient.incr(DLQ_RATE_LIMIT_KEY);
+        await this.redisClient.expire(DLQ_RATE_LIMIT_KEY, 60); // Reset rate limit after 1 minute
+    }
+}
+exports.DeadLetterQueue = DeadLetterQueue;

package/src/lib/redis/duplication.d.ts

@@ -0,0 +1,10 @@
+import { RedisType } from './registry';
+import { EventData } from './types';
+export declare class ContentBasedDeduplication {
+    private redisClient;
+    private ttl;
+    private hmacSecret;
+    constructor(redisClient: RedisType, ttl?: number);
+    private calculateEventHash;
+    isDuplicate(event: EventData<any, string>, consumerGroupName: string): Promise<boolean>;
+}

package/src/lib/redis/duplication.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ContentBasedDeduplication = void 0;
+const crypto_1 = require("crypto");
+const logger_1 = require("./logger");
+class ContentBasedDeduplication {
+    constructor(redisClient, ttl = 86400) {
+        this.redisClient = redisClient;
+        this.ttl = ttl;
+        this.hmacSecret = 'content-hash-calculation';
+    }
+    calculateEventHash(eventData) {
+        const { eventId, timestamp, createdAt, ...hashableData } = eventData;
+        const hmac = (0, crypto_1.createHmac)('sha256', this.hmacSecret);
+        return hmac.update(JSON.stringify(hashableData)).digest('hex');
+    }
+    async isDuplicate(event, consumerGroupName) {
+        const eventHash = this.calculateEventHash(event.data);
+        const key = `processed:${consumerGroupName}:${eventHash}`;
+        const result = await this.redisClient.set(key, '1', 'EX', this.ttl, 'NX');
+        const isDuplicate = result === null;
+        if (isDuplicate) {
+            logger_1.PUBLISHER_LOGGER.log(`Duplicate event detected for consumer group ${consumerGroupName}`);
+        }
+        return isDuplicate;
+    }
+}
+exports.ContentBasedDeduplication = ContentBasedDeduplication;