@jetit/publisher 4.1.1 → 5.0.0

package/README.md

@@ -1,203 +1,281 @@
-# publisher
-
-publisher is a library for implementing an event-driven architecture using Redis PUB/SUB and Redis Streams. It provides a simple and scalable mechanism for publishing and consuming events in real-time, and supports features such as message deduplication, consumer group management, and scheduled event publishing.
-
-## IMPORTANT NOTE
-This project currently does not have a means to clean up inactive consumers. This means that if you have a consumer that is no longer active, it will continue to receive events until it is removed from the consumer group. This is a known issue and will be addressed in a future release. A workaround is to use the following code to remove inactive consumers from the consumer group as part of your process cleanup:
-
-```javascript
-const ioredis = require(`ioredis`);
-
-const env = process.env;
-
-async function bootstrap() {
-    const connection = new ioredis.Redis({
-        host: env.REDIS_HOST,
-        port: parseInt(env.REDIS_PORT),
-    });
-    console.log(`Redis Connection Status ${connection.status}`);
-    await waitForSeconds(0.3);
-    console.log(`Redis Connection Status ${connection.status}`);
-    const instanceUniqueId = env.INSTANCE_ID;
-    if (!instanceUniqueId) {
-        console.log(`Unique instance ID is not available`);
-        return;
-    }
-    console.log(`Instance Unique ID : ${instanceUniqueId}`);
-    const consumerGroupName = await getConsumerGroupName(connection, instanceUniqueId);
-    if (!consumerGroupName) {
-        console.log(`Consumer is not available, so graceful shutdown is not necessary.`);
-        return;
-    }
-    console.log(`Consumer Group Name : ${consumerGroupName}`);
-    const instanceId = getInstanceId(consumerGroupName.slice(3), instanceUniqueId);
-    console.log(`Instance ID : ${instanceId}`);
-    const subscribedEvents = await getAllEventsForInstance(connection, instanceId);
-    console.log(`Subscribed Events : ${JSON.stringify(subscribedEvents)}`);
-    await clearSubscribedEvents(connection, consumerGroupName, instanceId, subscribedEvents);
-    await deleteConsumerGroupNameForInstance(connection, instanceId);
-    await deleteAllEventsFroInstance(connection, instanceId);
-}
+# @jetit/publisher
+
+`@jetit/publisher` is a robust and feature-rich library for implementing an event-driven architecture using Redis PUB/SUB and Redis Streams. It provides a scalable mechanism for publishing and consuming events in real-time, with support for advanced features such as message deduplication, consumer group management, scheduled event publishing, and more.
+
+## Table of Contents
+
+- [@jetit/publisher](#jetitpublisher)
+  - [Table of Contents](#table-of-contents)
+  - [Installation](#installation)
+  - [Key Features](#key-features)
+  - [Usage](#usage)
+    - [Basic Example](#basic-example)
+    - [Configuration](#configuration)
+    - [Publishing Events](#publishing-events)
+    - [Subscribing to Events](#subscribing-to-events)
+    - [Scheduled Publishing](#scheduled-publishing)
+    - [Batch Publishing](#batch-publishing)
+    - [Dead Letter Queue (DLQ)](#dead-letter-queue-dlq)
+    - [Event Filtering](#event-filtering)
+    - [Performance Monitoring](#performance-monitoring)
+    - [Prometheus Integration](#prometheus-integration)
+  - [Advanced Features](#advanced-features)
+    - [Content-Based Deduplication](#content-based-deduplication)
+    - [Multiple Event Subscriptions](#multiple-event-subscriptions)
+    - [Circuit Breaker](#circuit-breaker)
+  - [Performance Optimizations](#performance-optimizations)
+  - [Cleanup and Graceful Shutdown](#cleanup-and-graceful-shutdown)
+  - [Troubleshooting](#troubleshooting)
+
+## Installation
+
+```bash
+npm install @jetit/publisher
+```
 
-/**
- *
- * @param {ioredis.Redis|ioredis.Cluster} connection
- * @param {string} consumerGroupName
- * @param {string} instanceId
- * @param {Array<string>} events
- */
-async function clearSubscribedEvents(connection, consumerGroupName, instanceId, events) {
-    return Promise.all(
-        events.map(async (eventName) => {
-            console.log(`${eventName} is being cleared in publisher`);
-            const streamName = `${eventName}:${consumerGroupName}`;
-            console.log(`${streamName} is being removed.`);
-            await connection.srem(`${eventName}`, consumerGroupName);
-            console.log(`${eventName} is removed from ${consumerGroupName}`);
-            // Releasing all claims based on info from: https://redis.io/commands/xgroup-delconsumer/
-            await releaseAllClaims(connection, streamName, consumerGroupName, instanceId);
-            console.log(`${eventName} removes all claims`);
-            await connection.xgroup(`DELCONSUMER`, streamName, consumerGroupName, instanceId);
-            console.log(`${eventName} is deleted as a consumer from ${consumerGroupName}, ${instanceId}`);
-        })
-    );
-}
+## Key Features
 
-/**
- *
- * @param {ioredis.Redis|ioredis.Cluster} connection
- * @param {string} streamName
- * @param {string} consumerGroupName
- * @param {string} instanceId
- */
-async function releaseAllClaims(connection, streamName, consumerGroupName, instanceId) {
-    /**
-     * Retrieve the pending messages for the consumer. Note this only fetches the last
-     * 10000 events assigned to this consumer. This function has been modified to make sure
-     * that there is a temp instance that claims all this messages
-     */
-    const pendingMessages = await connection.xpending(streamName, consumerGroupName, `-`, `+`, 10000, instanceId);
-
-    if (pendingMessages && pendingMessages.length > 0) {
-        console.log(`${pendingMessages.length} messages to clean up.`);
-        const transaction = connection.multi({ pipeline: true });
-        const tempConsumerId = `${consumerGroupName}-temp`;
-        for (const [messageId] of pendingMessages) {
-            transaction.xclaim(streamName, consumerGroupName, tempConsumerId, 10, messageId);
-        }
-        await transaction.exec();
-    }
-}
+- Real-time event publishing and subscribing
+- Configurable Streams class for flexible usage
+- Improved error handling and reliability
+- Performance tracking with Redis time and operation time metrics
+- Dead Letter Queue (DLQ) for handling subscription failures
+- Event filtering for specialized subscriptions
+- Support for multiple event subscriptions from the same service
+- Batch publishing (regular and scheduled)
+- Basic monitoring with Prometheus export support
+- Content-based one-time guarantee (0-1 semantics support)
+- Optimized cleanup processes for improved performance
+- Circuit Breaker pattern for fault tolerance
 
-/**
- *
- * @param {string} serviceName
- * @param {string} instanceUniqueId
- */
-function getInstanceId(serviceName, instanceUniqueId) {
-    const instanceId = `${serviceName}:${instanceUniqueId}`;
-    console.log(`Generated Instance ID : ${instanceId}`);
-    return instanceId;
-}
+## Usage
 
-/**
- *
- * @param {ioredis.Redis|ioredis.Cluster} serviceName
- * @param {string} instanceId
- * @returns {Promise<string>} consumer group name
- */
-async function getConsumerGroupName(connection, instanceId) {
-    const key = `instance:${instanceId}:consumerGroupName`;
-    console.log(`Get consumer group name called for key : ${key}`);
-    return await connection.get(key);
-}
+### Basic Example
 
-/**
- *
- * @param {ioredis.Redis|ioredis.Cluster} connection
- * @param {string} instanceId
- * @returns
- */
-async function deleteConsumerGroupNameForInstance(connection, instanceId) {
-    return await connection.del(`instance:${instanceId}:consumerGroupName`);
-}
+```typescript
+import { Publisher, EventData } from '@jetit/publisher';
 
-/**
- *
- * @param {ioredis.Redis|ioredis.Cluster} connection
- * @param {string} instanceId
- * @returns {Promise<Array<events>>} subscribed events for this instance
- */
-async function getAllEventsForInstance(connection, instanceId) {
-    const key = `instance:${instanceId}:subscribedEvents`;
-    console.log(`Get consumer group events : ${key}`);
-    return (await connection.sscan(key, 0))[1];
-}
+// Create an instance of the publisher
+const publisher = new Publisher('MyService');
 
-/**
- *
- * @param {ioredis.Redis|ioredis.Cluster} connection
- * @param {string} instanceId
- */
-async function deleteAllEventsFroInstance(connection, instanceId) {
-    return await connection.del(`instance:${instanceId}:subscribedEvents`);
-}
+// Publish an event
+const eventData: EventData<{ message: string }> = {
+    eventName: 'my-event',
+    data: { message: 'Hello, world!' }
+};
 
-async function waitForSeconds(seconds = 10) {
-    return new Promise((res, _) => setTimeout(() => res(), seconds * 1000));
-}
+await publisher.publish(eventData);
 
-/**
- * Start
- */
+// Subscribe to an event
+publisher.listen('my-event').subscribe(event => {
+    console.log(`Received event: ${event.eventName}`, event.data);
+});
+```
 
-bootstrap()
-    .then(() => process.exit(0))
-    .catch((e) => {
-        console.error(e);
-        process.exit(1);
-    });
+### Configuration
 
+The `Publisher` class can be configured with various options, including Circuit Breaker and Backpressure handling:
+
+```typescript
+import { Publisher, IStreamsConfig } from '@jetit/publisher';
+
+const config: Partial<IStreamsConfig> = {
+    cleanUpInterval: 3600000, // 1 hour
+    maxRetries: 5,
+    initialRetryDelay: 1000,
+    immediatePublishThreshold: 500,
+    unprocessedMessageThreshold: 25,
+    acknowledgedMessageCleanupInterval: 3600000, // 1 hour
+    dlqEventThreshold: 2000,
+    filterKeepAlive: 86400000, // 24 hours
+    duplicationCheckWindow: 86400, // 24 hours
+    circuitBreaker: {
+        enabled: true,
+        errorThreshold: 50,
+        errorThresholdPercentage: 50,
+        openStateDuration: 30000, // 30s
+        halfOpenStateMaxAttempts: 10,
+        maxStoredEvents: 5000,
+    },
+};
+
+const publisher = new Publisher('MyService', config);
 ```
 
-## Simple Example
+### Publishing Events
 
 ```typescript
-import { Publisher, EventData } from '@jetit/streams';
+const eventData = {
+    eventName: 'user-registered',
+    data: { userId: '123', email: 'user@example.com' }
+};
 
-// Create an instance of the publisher
-const streams = new Streams('Websockets');
+await publisher.publish(eventData);
+```
 
-// Publish an event
-const eventData: EventData<{ message: string }> = {
-    eventName: 'my-event',
-    data: { message: 'Hello, world!' }
+### Subscribing to Events
+
+```typescript
+publisher.listen('user-registered').subscribe(event => {
+    console.log('New user registered:', event.data);
+});
+```
+
+### Scheduled Publishing
+
+```typescript
+const futureDate = new Date(Date.now() + 60000); // 1 minute from now
+await publisher.scheduledPublish(futureDate, eventData);
+```
+
+### Batch Publishing
+
+```typescript
+import { publishBatch } from '@jetit/publisher';
+
+const events = [
+    { eventName: 'event1', data: { /* ... */ } },
+    { eventName: 'event2', data: { /* ... */ } },
+    // ...
+];
+
+const result = await publishBatch(publisher, events, { batchSize: 100, delayBetweenBatches: 1000 });
+console.log('Batch publish result:', result);
+```
+
+### Dead Letter Queue (DLQ)
+
+```typescript
+// Retry an event from DLQ
+const success = await publisher.retryFromDLQ('eventId');
+
+// Get DLQ stats
+const stats = await publisher.getDLQStats();
+console.log('DLQ stats:', stats);
+```
+
+### Event Filtering
+
+```typescript
+const options = {
+    eventFilter: (event) => event.data.userId === '123',
+    filterKeepAlive: 3600000 // 1 hour
 };
 
-await streams.publish(eventData);
+publisher.listen('user-action', options).subscribe(event => {
+    console.log('Filtered user action:', event);
+});
+```
 
-// Subscribe to an event
-streams.listen('my-event').subscribe(event => {
-    console.log(`Received event: ${event.eventName}`, event.data);
+### Performance Monitoring
+
+```typescript
+// Get metrics for a specific time range
+const metrics = await publisher.getMetrics(startTime, endTime);
+console.log('Performance metrics:', metrics);
+
+// Get latest metrics
+const latestMetrics = await publisher.getLatestMetrics();
+console.log('Latest metrics:', latestMetrics);
+```
+
+### Prometheus Integration
+
+```typescript
+import { PrometheusAdapter } from '@jetit/publisher';
+import promClient from 'prom-client';
+import express from 'express';
+
+const app = express();
+const prometheusAdapter = new PrometheusAdapter(publisher, promClient);
+
+prometheusAdapter.setupEndpoint(app, '/metrics');
+
+app.listen(3000, () => {
+    console.log('Metrics server listening on port 3000');
+});
+```
+
+## Advanced Features
+
+### Content-Based Deduplication
+
+The library supports content-based deduplication to ensure that each unique event is processed only once:
+
+```typescript
+const options = {
+    publishOnceGuarantee: true
+};
+
+publisher.listen('important-event', options).subscribe(event => {
+    console.log('Guaranteed unique event:', event);
 });
 ```
 
-## Possible use cases
+### Multiple Event Subscriptions
 
-1. Microservices communication: If your system is composed of multiple microservices, the publisher can be used to facilitate communication between them by publishing and listening to events.
+You can subscribe to multiple events from the same service:
 
-2. Event sourcing and CQRS: In an event-sourced system, the publisher can be used to store and process events that represent the state changes of the system, enabling Command Query Responsibility Segregation (CQRS) by separating the read and write models.
+```typescript
+const subscription1 = publisher.listen('event1').subscribe(/* ... */);
+const subscription2 = publisher.listen('event2').subscribe(/* ... */);
+```
 
-3. Task queues: The publisher can be used to create task queues for distributing workloads among different worker instances, ensuring that tasks are processed in the order they were created.
+### Circuit Breaker
 
-4. Data streaming and processing: The publisher can be used to ingest and process large volumes of data in real-time, such as log files, clickstream data, or other event-based data.
+The Circuit Breaker pattern is implemented to prevent cascading failures in a distributed system. It helps to gracefully handle failures and allows the system to recover without overwhelming failed services.
 
-5. Distributed system coordination: In a distributed system, the publisher can be used for coordination between different components, such as managing leader election or maintaining configuration information.
+Configuration options:
 
-6. Real-time analytics and monitoring: The publisher can be used to collect and process real-time analytics data, such as user behavior, application performance metrics, or system monitoring information.
+- `enabled`: Enable or disable the Circuit Breaker.
+- `errorThreshold`: Number of errors before opening the circuit.
+- `errorThresholdPercentage`: Percentage of errors to total calls before opening the circuit.
+- `timeWindow`: Time window for error rate calculation (in milliseconds).
+- `openStateDuration`: Duration to keep the circuit open before moving to half-open state (in milliseconds).
+- `halfOpenStateMaxAttempts`: Maximum number of attempts allowed in half-open state.
+
+The Circuit Breaker has three states:
+
+1. Closed: Normal operation, calls pass through.
+2. Open: Calls are immediately rejected without reaching the service.
+3. Half-Open: A limited number of calls are allowed to test if the service has recovered.
+
+
+## Performance Optimizations
+
+- Batched `xdel` operations for improved cleanup performance
+- Configurable cleanup intervals and thresholds
+- Efficient event filtering at the subscription level
+- Retry logic with exponential backoff for failed operations
+- Circuit Breaker to prevent overwhelming failed services
+- Dead Letter Queue (DLQ) for handling subscription failures
+  
+## Cleanup and Graceful Shutdown
+
+To ensure proper cleanup of resources, implement a graceful shutdown:
+
+```typescript
+process.on('SIGTERM', shutdown);
+process.on('SIGINT', shutdown);
+
+async function shutdown() {
+    console.log('Graceful shutdown initiated.');
+    try {
+        await publisher.close();
+        console.log('Resources and connections successfully closed.');
+    } catch (error) {
+        console.error('Error during graceful shutdown:', error);
+    }
+    process.exit(0);
+}
+```
 
-7. Event-driven workflows: You can use the publisher to create event-driven workflows, where each step in the workflow is triggered by the completion of a previous step. This can be useful for orchestrating complex, multi-step processes.
+## Troubleshooting
 
-8. Message broadcasting: The publisher can be used to broadcast messages to multiple consumers or subscribers, allowing for efficient and scalable communication in applications with many components or services.
+If you encounter issues:
 
-9. Multicast Publishing: This is the existing PUB/SUB implementation but with the event data being stored into streams for additional processing
+1. Check the Redis connection settings
+2. Verify that consumer groups are correctly created
+3. Monitor the DLQ for failed events
+4. Review the performance metrics for any anomalies
+5. Check the logs for detailed error messages

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@jetit/publisher",
-  "version": "4.1.1",
+  "version": "5.0.0",
   "type": "commonjs",
   "dependencies": {
     "@jetit/id": "^0.0.12",
+    "events": "3.3.0",
     "ioredis": "^5.3.0",
     "rxjs": "^7.8.0",
-    "tslib": "1.14.1"
+    "tslib": "2.5.0"
   },
   "main": "./src/index.js"
 }

package/src/lib/monitoring/adapters/prom.d.ts

@@ -0,0 +1,37 @@
+/**
+ * One of the horribly typed code I have written. But I dont see
+ * another option without importing other libraries in. So this
+ * works well when the right stuff is passed in for the promClient
+ * and the web server.
+ */
+import { Streams as Publisher } from '../../redis/streams';
+export declare class PrometheusAdapter {
+    private streams;
+    private promClient;
+    private registry;
+    private queueDepth;
+    private dlqSize;
+    private dlqRate;
+    private operationCount;
+    private totalTime;
+    private redisOperationTime;
+    private processingTime;
+    private eventCount;
+    private publishErrorCount;
+    private subscribeErrorCount;
+    private individualQueueDepth;
+    private duplicateEventCount;
+    /**
+     *
+     * @param streams [Publisher]
+     * @param promClient [Prom Client] This needs to be an instance of prom-client
+     */
+    constructor(streams: Publisher, promClient: any);
+    private initializeMetrics;
+    updateMetrics(): Promise<void>;
+    private updatePrometheusMetrics;
+    /**
+     * @param app This needs to be an instance of express or fastify something that supports the express api
+     */
+    setupEndpoint(app: any, endPoint?: string): void;
+}

package/src/lib/monitoring/adapters/prom.js

@@ -0,0 +1,126 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PrometheusAdapter = void 0;
+class PrometheusAdapter {
+    /**
+     *
+     * @param streams [Publisher]
+     * @param promClient [Prom Client] This needs to be an instance of prom-client
+     */
+    constructor(streams, promClient) {
+        this.streams = streams;
+        this.promClient = promClient;
+        this.registry = new this.promClient.Registry({ collectDefaultMetrics: { timeout: 60000 } });
+        this.initializeMetrics();
+    }
+    initializeMetrics() {
+        this.queueDepth = new this.promClient.Gauge({
+            name: 'queue_depth',
+            help: 'Total number of messages in all queues',
+            registers: [this.registry],
+        });
+        this.dlqSize = new this.promClient.Gauge({
+            name: 'dlq_size',
+            help: 'Number of messages in the Dead Letter Queue',
+            registers: [this.registry],
+        });
+        this.duplicateEventCount = new this.promClient.Counter({
+            name: 'duplicate_event_count',
+            help: 'Number of duplicate events detected',
+            registers: [this.registry],
+        });
+        this.dlqRate = new this.promClient.Gauge({
+            name: 'dlq_rate',
+            help: 'Rate of messages being added to the Dead Letter Queue',
+            registers: [this.registry],
+        });
+        this.operationCount = new this.promClient.Counter({
+            name: 'operation_count',
+            help: 'Total number of operations performed',
+            registers: [this.registry],
+        });
+        this.totalTime = new this.promClient.Gauge({
+            name: 'total_time',
+            help: 'Total processing time in milliseconds',
+            registers: [this.registry],
+        });
+        this.redisOperationTime = new this.promClient.Gauge({
+            name: 'redis_operation_time',
+            help: 'Time spent on Redis operations in milliseconds',
+            registers: [this.registry],
+        });
+        this.processingTime = new this.promClient.Gauge({
+            name: 'processing_time',
+            help: 'Time spent on processing events in milliseconds',
+            registers: [this.registry],
+        });
+        this.eventCount = new this.promClient.Counter({
+            name: 'event_count',
+            help: 'Total number of events processed',
+            registers: [this.registry],
+        });
+        this.publishErrorCount = new this.promClient.Counter({
+            name: 'publish_error_count',
+            help: 'Number of errors encountered during publish operations',
+            registers: [this.registry],
+        });
+        this.subscribeErrorCount = new this.promClient.Counter({
+            name: 'subscribe_error_count',
+            help: 'Number of errors encountered during subscribe operations',
+            registers: [this.registry],
+        });
+        this.individualQueueDepth = new this.promClient.Gauge({
+            name: 'individual_queue_depth',
+            help: 'Number of messages in each individual queue',
+            labelNames: ['queue_name'],
+            registers: [this.registry],
+        });
+        this.registry.registerMetric(this.queueDepth);
+        this.registry.registerMetric(this.dlqSize);
+        this.registry.registerMetric(this.dlqRate);
+        this.registry.registerMetric(this.operationCount);
+        this.registry.registerMetric(this.totalTime);
+        this.registry.registerMetric(this.redisOperationTime);
+        this.registry.registerMetric(this.processingTime);
+        this.registry.registerMetric(this.eventCount);
+        this.registry.registerMetric(this.publishErrorCount);
+        this.registry.registerMetric(this.subscribeErrorCount);
+        this.registry.registerMetric(this.individualQueueDepth);
+        this.registry.registerMetric(this.duplicateEventCount);
+    }
+    async updateMetrics() {
+        const metrics = await this.streams.getLatestMetrics();
+        if (metrics) {
+            this.updatePrometheusMetrics(metrics);
+        }
+    }
+    updatePrometheusMetrics(metrics) {
+        this.queueDepth.set(metrics.queueDepth);
+        this.dlqSize.set(metrics.dlqSize);
+        this.dlqRate.set(metrics.dlqRate);
+        this.operationCount.inc(metrics.operationCount);
+        this.totalTime.set(metrics.totalTime);
+        this.redisOperationTime.set(metrics.redisOperationTime);
+        this.processingTime.set(metrics.processingTime);
+        this.eventCount.inc(metrics.eventCount);
+        this.publishErrorCount.inc(metrics.publishErrorCount);
+        this.subscribeErrorCount.inc(metrics.subscribeErrorCount);
+        this.duplicateEventCount.inc(metrics.duplicateEventsCount);
+        if (metrics.individualQueueDepts) {
+            Object.entries(metrics.individualQueueDepts).forEach(([queueName, depth]) => {
+                this.individualQueueDepth.set({ queue_name: queueName }, depth);
+            });
+        }
+    }
+    /**
+     * @param app This needs to be an instance of express or fastify something that supports the express api
+     */
+    setupEndpoint(app, endPoint = '/metrics') {
+        app.get(endPoint, async (_, res) => {
+            await this.updateMetrics();
+            res.set('Content-Type', this.registry.contentType);
+            res.send(await this.registry.metrics());
+        });
+    }
+}
+exports.PrometheusAdapter = PrometheusAdapter;

package/src/lib/monitoring/collector.d.ts

@@ -0,0 +1,17 @@
+import { DeadLetterQueue } from '../redis/dlq';
+import { IAggregatedMetrics, IMetricsCollectorConfig, IPerformanceMetrics } from './types';
+export declare class MetricsCollector {
+    private redisClient;
+    private collectionInterval;
+    private retentionPeriod;
+    private dlq;
+    private metrics;
+    private metricsKey;
+    constructor(config: IMetricsCollectorConfig, dlq: DeadLetterQueue);
+    addMetrics(metrics: IPerformanceMetrics): void;
+    private aggregateAndStoreMetrics;
+    private getQueueDepth;
+    private storeMetrics;
+    getMetrics(startTime: number, endTime: number): Promise<IAggregatedMetrics[]>;
+    getLatestMetrics(): Promise<IAggregatedMetrics | null>;
+}