@jetit/publisher 1.5.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +158 -0
- package/package.json +2 -2
- package/src/lib/redis/streams.d.ts +3 -37
- package/src/lib/redis/streams.js +110 -224
package/README.md
CHANGED
|
@@ -2,6 +2,164 @@
|
|
|
2
2
|
|
|
3
3
|
publisher is a library for implementing an event-driven architecture using Redis PUB/SUB and Redis Streams. It provides a simple and scalable mechanism for publishing and consuming events in real-time, and supports features such as message deduplication, consumer group management, and scheduled event publishing.
|
|
4
4
|
|
|
5
|
+
## IMPORTANT NOTE
|
|
6
|
+
This project currently does not have a means to clean up inactive consumers. This means that if you have a consumer that is no longer active, it will continue to receive events until it is removed from the consumer group. This is a known issue and will be addressed in a future release. A workaround is to use the following code to remove inactive consumers from the consumer group as part of your process cleanup:
|
|
7
|
+
|
|
8
|
+
```javascript
|
|
9
|
+
const ioredis = require(`ioredis`);
|
|
10
|
+
|
|
11
|
+
const env = process.env;
|
|
12
|
+
|
|
13
|
+
async function bootstrap() {
|
|
14
|
+
const connection = new ioredis.Redis({
|
|
15
|
+
host: env.REDIS_HOST,
|
|
16
|
+
port: parseInt(env.REDIS_PORT),
|
|
17
|
+
});
|
|
18
|
+
console.log(`Redis Connection Status ${connection.status}`);
|
|
19
|
+
await waitForSeconds(0.3);
|
|
20
|
+
console.log(`Redis Connection Status ${connection.status}`);
|
|
21
|
+
const instanceUniqueId = env.INSTANCE_ID;
|
|
22
|
+
if (!instanceUniqueId) {
|
|
23
|
+
console.log(`Unique instance ID is not available`);
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
console.log(`Instance Unique ID : ${instanceUniqueId}`);
|
|
27
|
+
const consumerGroupName = await getConsumerGroupName(connection, instanceUniqueId);
|
|
28
|
+
if (!consumerGroupName) {
|
|
29
|
+
console.log(`Consumer is not available, so graceful shutdown is not necessary.`);
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
console.log(`Consumer Group Name : ${consumerGroupName}`);
|
|
33
|
+
const instanceId = getInstanceId(consumerGroupName.slice(3), instanceUniqueId);
|
|
34
|
+
console.log(`Instance ID : ${instanceId}`);
|
|
35
|
+
const subscribedEvents = await getAllEventsForInstance(connection, instanceId);
|
|
36
|
+
console.log(`Subscribed Events : ${JSON.stringify(subscribedEvents)}`);
|
|
37
|
+
await clearSubscribedEvents(connection, consumerGroupName, instanceId, subscribedEvents);
|
|
38
|
+
await deleteConsumerGroupNameForInstance(connection, instanceId);
|
|
39
|
+
await deleteAllEventsFroInstance(connection, instanceId);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
*
|
|
44
|
+
* @param {ioredis.Redis|ioredis.Cluster} connection
|
|
45
|
+
* @param {string} consumerGroupName
|
|
46
|
+
* @param {string} instanceId
|
|
47
|
+
* @param {Array<string>} events
|
|
48
|
+
*/
|
|
49
|
+
async function clearSubscribedEvents(connection, consumerGroupName, instanceId, events) {
|
|
50
|
+
return Promise.all(
|
|
51
|
+
events.map(async (eventName) => {
|
|
52
|
+
console.log(`${eventName} is being cleared in publisher`);
|
|
53
|
+
const streamName = `${eventName}:${consumerGroupName}`;
|
|
54
|
+
console.log(`${streamName} is being removed.`);
|
|
55
|
+
await connection.srem(`${eventName}`, consumerGroupName);
|
|
56
|
+
console.log(`${eventName} is removed from ${consumerGroupName}`);
|
|
57
|
+
// Releasing all claims based on info from: https://redis.io/commands/xgroup-delconsumer/
|
|
58
|
+
await releaseAllClaims(connection, streamName, consumerGroupName, instanceId);
|
|
59
|
+
console.log(`${eventName} removes all claims`);
|
|
60
|
+
await connection.xgroup(`DELCONSUMER`, streamName, consumerGroupName, instanceId);
|
|
61
|
+
console.log(`${eventName} is deleted as a consumer from ${consumerGroupName}, ${instanceId}`);
|
|
62
|
+
})
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
*
|
|
68
|
+
* @param {ioredis.Redis|ioredis.Cluster} connection
|
|
69
|
+
* @param {string} streamName
|
|
70
|
+
* @param {string} consumerGroupName
|
|
71
|
+
* @param {string} instanceId
|
|
72
|
+
*/
|
|
73
|
+
async function releaseAllClaims(connection, streamName, consumerGroupName, instanceId) {
|
|
74
|
+
/**
|
|
75
|
+
* Retrieve the pending messages for the consumer. Note this only fetches the last
|
|
76
|
+
* 10000 events assigned to this consumer. This function has been modified to make sure
|
|
77
|
+
* that there is a temp instance that claims all this messages
|
|
78
|
+
*/
|
|
79
|
+
const pendingMessages = await connection.xpending(streamName, consumerGroupName, `-`, `+`, 10000, instanceId);
|
|
80
|
+
|
|
81
|
+
if (pendingMessages && pendingMessages.length > 0) {
|
|
82
|
+
console.log(`${pendingMessages.length} messages to clean up.`);
|
|
83
|
+
const transaction = connection.multi({ pipeline: true });
|
|
84
|
+
const tempConsumerId = `${consumerGroupName}-temp`;
|
|
85
|
+
for (const [messageId] of pendingMessages) {
|
|
86
|
+
transaction.xclaim(streamName, consumerGroupName, tempConsumerId, 10, messageId);
|
|
87
|
+
}
|
|
88
|
+
await transaction.exec();
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
*
|
|
94
|
+
* @param {string} serviceName
|
|
95
|
+
* @param {string} instanceUniqueId
|
|
96
|
+
*/
|
|
97
|
+
function getInstanceId(serviceName, instanceUniqueId) {
|
|
98
|
+
const instanceId = `${serviceName}:${instanceUniqueId}`;
|
|
99
|
+
console.log(`Generated Instance ID : ${instanceId}`);
|
|
100
|
+
return instanceId;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
*
|
|
105
|
+
* @param {ioredis.Redis|ioredis.Cluster} serviceName
|
|
106
|
+
* @param {string} instanceId
|
|
107
|
+
* @returns {Promise<string>} consumer group name
|
|
108
|
+
*/
|
|
109
|
+
async function getConsumerGroupName(connection, instanceId) {
|
|
110
|
+
const key = `instance:${instanceId}:consumerGroupName`;
|
|
111
|
+
console.log(`Get consumer group name called for key : ${key}`);
|
|
112
|
+
return await connection.get(key);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
*
|
|
117
|
+
* @param {ioredis.Redis|ioredis.Cluster} connection
|
|
118
|
+
* @param {string} instanceId
|
|
119
|
+
* @returns
|
|
120
|
+
*/
|
|
121
|
+
async function deleteConsumerGroupNameForInstance(connection, instanceId) {
|
|
122
|
+
return await connection.del(`instance:${instanceId}:consumerGroupName`);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
*
|
|
127
|
+
* @param {ioredis.Redis|ioredis.Cluster} connection
|
|
128
|
+
* @param {string} instanceId
|
|
129
|
+
* @returns {Promise<Array<events>>} subscribed events for this instance
|
|
130
|
+
*/
|
|
131
|
+
async function getAllEventsForInstance(connection, instanceId) {
|
|
132
|
+
const key = `instance:${instanceId}:subscribedEvents`;
|
|
133
|
+
console.log(`Get consumer group events : ${key}`);
|
|
134
|
+
return (await connection.sscan(key, 0))[1];
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
*
|
|
139
|
+
* @param {ioredis.Redis|ioredis.Cluster} connection
|
|
140
|
+
* @param {string} instanceId
|
|
141
|
+
*/
|
|
142
|
+
async function deleteAllEventsFroInstance(connection, instanceId) {
|
|
143
|
+
return await connection.del(`instance:${instanceId}:subscribedEvents`);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
async function waitForSeconds(seconds = 10) {
|
|
147
|
+
return new Promise((res, _) => setTimeout(() => res(), seconds * 1000));
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Start
|
|
152
|
+
*/
|
|
153
|
+
|
|
154
|
+
bootstrap()
|
|
155
|
+
.then(() => process.exit(0))
|
|
156
|
+
.catch((e) => {
|
|
157
|
+
console.error(e);
|
|
158
|
+
process.exit(1);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
```
|
|
162
|
+
|
|
5
163
|
## Simple Example
|
|
6
164
|
|
|
7
165
|
```typescript
|
package/package.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jetit/publisher",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.6.1",
|
|
4
4
|
"type": "commonjs",
|
|
5
5
|
"dependencies": {
|
|
6
6
|
"@jetit/id": "0.0.11",
|
|
7
7
|
"ioredis": "^5.3.0",
|
|
8
|
-
"rxjs": "^7.
|
|
8
|
+
"rxjs": "^7.8.0"
|
|
9
9
|
},
|
|
10
10
|
"peerDependencies": {
|
|
11
11
|
"tslib": "2.5.0"
|
|
@@ -1,18 +1,15 @@
|
|
|
1
|
-
import { RedisType } from './registry';
|
|
2
1
|
import { EventData, PublishData } from './types';
|
|
3
2
|
import { Observable } from 'rxjs';
|
|
4
3
|
export declare class Streams {
|
|
5
4
|
private _redisPublisher?;
|
|
6
|
-
private _redisSubscriber?;
|
|
7
5
|
private _redisGroups?;
|
|
8
6
|
private consumerGroupName;
|
|
9
7
|
private instanceId;
|
|
10
8
|
private instanceUniqueId;
|
|
11
9
|
private cleanUpTimer;
|
|
12
10
|
private eventsListened;
|
|
13
|
-
get redisPublisher()
|
|
14
|
-
get
|
|
15
|
-
get redisGroups(): RedisType;
|
|
11
|
+
private get redisPublisher();
|
|
12
|
+
private get redisGroups();
|
|
16
13
|
/**
|
|
17
14
|
* Creates a new Streams instance for a given service.
|
|
18
15
|
*
|
|
@@ -29,7 +26,6 @@ export declare class Streams {
|
|
|
29
26
|
*/
|
|
30
27
|
constructor(serviceName: string);
|
|
31
28
|
private runClear;
|
|
32
|
-
private createConsumerGroup;
|
|
33
29
|
private isDuplicateMessage;
|
|
34
30
|
private clearDuplicationCheckKeys;
|
|
35
31
|
/**
|
|
@@ -100,33 +96,8 @@ export declare class Streams {
|
|
|
100
96
|
* });
|
|
101
97
|
*/
|
|
102
98
|
listen<T = unknown, const TName extends string = string>(eventName: TName, maxRetries?: number, initialDelay?: number): Observable<EventData<T, TName>>;
|
|
99
|
+
private createConsumerAndRegister;
|
|
103
100
|
private listenInternals;
|
|
104
|
-
/**
|
|
105
|
-
* This method takes all messages allocated to this instance and republishes them so
|
|
106
|
-
* that other instances of this service can receive and process them.
|
|
107
|
-
*
|
|
108
|
-
* This needs to be handled every 1-2 minutes if the queue becomes too long and messages
|
|
109
|
-
* are not being processed.
|
|
110
|
-
*
|
|
111
|
-
* Ideal implementation would be to wrap this inside a setInterval
|
|
112
|
-
* @param streamName
|
|
113
|
-
*/
|
|
114
|
-
republishUnprocessedEvents(eventName: string): Promise<void>;
|
|
115
|
-
/**
|
|
116
|
-
* This method is used to claim messages in the event of a service crash. This library currently
|
|
117
|
-
* does not detect a service crash. This needs to be built as an extension of Kubernetes and
|
|
118
|
-
* a standalone service that notifies this service to process the events that are marked as
|
|
119
|
-
* pending
|
|
120
|
-
*
|
|
121
|
-
* @param streamName
|
|
122
|
-
* @param idleTimeout
|
|
123
|
-
*
|
|
124
|
-
* * @example
|
|
125
|
-
*
|
|
126
|
-
* // Attempt to recover messages from the "order.created" stream with an idle timeout of 10 seconds
|
|
127
|
-
* await streams.recoverCrashedConsumerMessages('order.created', 10000);
|
|
128
|
-
*/
|
|
129
|
-
recoverCrashedConsumerMessages(eventName: string, idleTimeout?: number): Promise<void>;
|
|
130
101
|
/**
|
|
131
102
|
* This method allows the possibility of a graceful shutdown by cleaning up the
|
|
132
103
|
* redis connections.
|
|
@@ -148,11 +119,6 @@ export declare class Streams {
|
|
|
148
119
|
* }
|
|
149
120
|
*/
|
|
150
121
|
close(): Promise<void>;
|
|
151
|
-
private clearSubscribedEvents;
|
|
152
|
-
private registerSubscribedEvent;
|
|
153
|
-
private registerConsumerGroup;
|
|
154
|
-
private registerConsumerGroupName;
|
|
155
122
|
private scanAndClaimAUnclaimedMessage;
|
|
156
|
-
releaseAllClaims(streamName: string): Promise<void>;
|
|
157
123
|
private cleanupAcknowledgedMessages;
|
|
158
124
|
}
|
package/src/lib/redis/streams.js
CHANGED
|
@@ -7,7 +7,7 @@ const rxjs_1 = require("rxjs");
|
|
|
7
7
|
const id_1 = require("@jetit/id");
|
|
8
8
|
const groups_1 = require("./groups");
|
|
9
9
|
function publisherErrorHandler(error) {
|
|
10
|
-
console.error('
|
|
10
|
+
console.error('PUBLISHER UNHANDLED ERROR: ', error);
|
|
11
11
|
}
|
|
12
12
|
class Streams {
|
|
13
13
|
get redisPublisher() {
|
|
@@ -15,11 +15,6 @@ class Streams {
|
|
|
15
15
|
this._redisPublisher = registry_1.RedisRegistry.getConnection('publish');
|
|
16
16
|
return this._redisPublisher;
|
|
17
17
|
}
|
|
18
|
-
get redisSubscriber() {
|
|
19
|
-
if (!this._redisSubscriber)
|
|
20
|
-
this._redisSubscriber = registry_1.RedisRegistry.getConnection('subscriber');
|
|
21
|
-
return this._redisSubscriber;
|
|
22
|
-
}
|
|
23
18
|
get redisGroups() {
|
|
24
19
|
if (!this._redisGroups)
|
|
25
20
|
this._redisGroups = registry_1.RedisRegistry.getConnection('groups');
|
|
@@ -44,8 +39,8 @@ class Streams {
|
|
|
44
39
|
this.eventsListened = [];
|
|
45
40
|
this.instanceUniqueId = (_a = process.env['INSTANCE_ID']) !== null && _a !== void 0 ? _a : (0, id_1.generateID)('HEX', 'FE');
|
|
46
41
|
this.instanceId = `${serviceName}:${this.instanceUniqueId}`;
|
|
47
|
-
console.log(this.instanceId);
|
|
48
42
|
this.consumerGroupName = `cg-${serviceName}`;
|
|
43
|
+
console.log(`PUBLISHER: Instance ID: ${this.instanceId}`);
|
|
49
44
|
const cleanUpInterval = (_b = parseInt(process.env['CLEANUP_INTERVAL'] || `${1000 * 60 * 60}`, 10)) !== null && _b !== void 0 ? _b : 1000 * 60 * 60;
|
|
50
45
|
setTimeout(() => this.runClear(cleanUpInterval), 60000);
|
|
51
46
|
this.cleanUpTimer = setInterval(() => {
|
|
@@ -54,30 +49,24 @@ class Streams {
|
|
|
54
49
|
}
|
|
55
50
|
runClear(cleanUpInterval) {
|
|
56
51
|
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
57
|
-
console.log('Running
|
|
52
|
+
console.log('PUBLISHER: Running Clearance', this.eventsListened);
|
|
58
53
|
this.clearDuplicationCheckKeys();
|
|
59
54
|
for (const eventName of this.eventsListened) {
|
|
60
55
|
process.nextTick(() => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
56
|
+
/** This removes the messages from the stream after they have been processed according to cleanup interval */
|
|
61
57
|
yield this.cleanupAcknowledgedMessages(eventName, cleanUpInterval).catch(publisherErrorHandler);
|
|
62
|
-
|
|
63
|
-
|
|
58
|
+
console.log(`Cleanup process for Acknowledged messages completed for ${eventName}`);
|
|
59
|
+
/**
|
|
60
|
+
* This process scans and claims any unclaimed message according to the cleanup interval.
|
|
61
|
+
* This triggers a cascaded reaction down the chain as message by message is claimed and processed
|
|
62
|
+
*/
|
|
63
|
+
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
64
|
+
yield this.scanAndClaimAUnclaimedMessage(streamName).catch(publisherErrorHandler);
|
|
65
|
+
console.log(`Unclaimed messages for ${streamName}`);
|
|
64
66
|
}));
|
|
65
67
|
}
|
|
66
68
|
});
|
|
67
69
|
}
|
|
68
|
-
createConsumerGroup(eventName) {
|
|
69
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
70
|
-
try {
|
|
71
|
-
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
72
|
-
yield this.redisGroups.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
|
|
73
|
-
}
|
|
74
|
-
catch (error) {
|
|
75
|
-
if (error.message !== 'BUSYGROUP Consumer Group name already exists') {
|
|
76
|
-
throw error;
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
70
|
isDuplicateMessage(streamName, messageId) {
|
|
82
71
|
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
83
72
|
const processedMessagesKey = `pm:${this.consumerGroupName}:${streamName}`;
|
|
@@ -115,7 +104,7 @@ class Streams {
|
|
|
115
104
|
const transaction = this.redisPublisher.multi({ pipeline: true });
|
|
116
105
|
const consumerGroups = yield (0, groups_1.getAllConsumerGroups)(data.eventName, this.redisPublisher);
|
|
117
106
|
if (consumerGroups.length > 0) {
|
|
118
|
-
console.log(`Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
|
|
107
|
+
console.log(`PUBLISHER: Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
|
|
119
108
|
for (const consumerGroup of consumerGroups) {
|
|
120
109
|
// Publish the event to each consumer group's stream
|
|
121
110
|
const streamName = `${data.eventName}:${consumerGroup}`;
|
|
@@ -123,7 +112,7 @@ class Streams {
|
|
|
123
112
|
}
|
|
124
113
|
transaction.publish(data.eventName, '');
|
|
125
114
|
yield transaction.exec().catch((error) => {
|
|
126
|
-
console.error(`Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
|
|
115
|
+
console.error(`PUBLISHER: Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
|
|
127
116
|
throw new Error('Publisher Error');
|
|
128
117
|
});
|
|
129
118
|
}
|
|
@@ -154,7 +143,7 @@ class Streams {
|
|
|
154
143
|
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
155
144
|
const currentTime = new Date();
|
|
156
145
|
if (scheduledTime < currentTime) {
|
|
157
|
-
throw new Error('Cannot schedule an event in the past');
|
|
146
|
+
throw new Error('PUBLISHER: Cannot schedule an event in the past');
|
|
158
147
|
}
|
|
159
148
|
else if (Math.abs(scheduledTime.getTime() - currentTime.getTime()) <= 500) {
|
|
160
149
|
yield this.publish(eventData);
|
|
@@ -163,7 +152,7 @@ class Streams {
|
|
|
163
152
|
if (uniquePerInstance === true) {
|
|
164
153
|
const existingJob = yield this.redisPublisher.zscore('se', JSON.stringify(eventData));
|
|
165
154
|
if (existingJob) {
|
|
166
|
-
console.log(`Job with data '${eventData}' already exists. Skipping.`);
|
|
155
|
+
console.log(`PUBLISHER: Job with data '${eventData}' already exists. Skipping.`);
|
|
167
156
|
return;
|
|
168
157
|
}
|
|
169
158
|
}
|
|
@@ -199,159 +188,109 @@ class Streams {
|
|
|
199
188
|
* });
|
|
200
189
|
*/
|
|
201
190
|
listen(eventName, maxRetries = 5, initialDelay = 1000) {
|
|
202
|
-
this.registerConsumerGroup(eventName); // Registers the consumer group for listening to the message
|
|
203
|
-
this.registerConsumerGroupName().then();
|
|
204
|
-
this.eventsListened.push(eventName);
|
|
205
191
|
return this.listenInternals(eventName).pipe((0, rxjs_1.retry)({
|
|
206
192
|
count: maxRetries,
|
|
207
193
|
delay: (error, retryAttempt) => {
|
|
208
194
|
const delay = initialDelay * Math.pow(2, retryAttempt);
|
|
209
|
-
console.error(`Error in listen: ${error.message}. Retrying in ${delay}ms (attempt ${retryAttempt + 1})`);
|
|
195
|
+
console.error(`PUBLISHER: Error in listen: ${error.message}. Retrying in ${delay}ms (attempt ${retryAttempt + 1})`);
|
|
210
196
|
return (0, rxjs_1.timer)(delay);
|
|
211
197
|
},
|
|
212
198
|
}), (0, rxjs_1.catchError)((error) => {
|
|
213
|
-
console.error(`Error in listen after ${maxRetries} retries: ${error.message}`);
|
|
199
|
+
console.error(`PUBLISHER: Error in listen after ${maxRetries} retries: ${error.message}`);
|
|
214
200
|
return (0, rxjs_1.throwError)(() => new Error(error.message));
|
|
215
201
|
}));
|
|
216
202
|
}
|
|
217
|
-
|
|
218
|
-
try {
|
|
219
|
-
this.createConsumerGroup(eventName).then();
|
|
220
|
-
this.registerSubscribedEvent(eventName).then();
|
|
221
|
-
const bs = new rxjs_1.BehaviorSubject(null);
|
|
222
|
-
const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1));
|
|
223
|
-
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
224
|
-
this.redisSubscriber.subscribe(eventName);
|
|
225
|
-
this.scanAndClaimAUnclaimedMessage(streamName)
|
|
226
|
-
.then()
|
|
227
|
-
.catch((e) => console.log('PUBLISHER: Err in handling unclaimed Messages ' + e.message));
|
|
228
|
-
const processMessage = () => {
|
|
229
|
-
try {
|
|
230
|
-
process.nextTick(() => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
231
|
-
const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', streamName, '>');
|
|
232
|
-
if (result) {
|
|
233
|
-
const [, streamMessages] = result[0];
|
|
234
|
-
for (const [id, data] of streamMessages) {
|
|
235
|
-
const eventData = JSON.parse(data[1]);
|
|
236
|
-
const messageId = eventData.eventId;
|
|
237
|
-
const isDuplicate = yield this.isDuplicateMessage(streamName, messageId);
|
|
238
|
-
if (isDuplicate) {
|
|
239
|
-
console.warn(`Duplicate message detected: ${messageId}`);
|
|
240
|
-
yield this.redisGroups.xack(streamName, this.consumerGroupName, id);
|
|
241
|
-
continue;
|
|
242
|
-
}
|
|
243
|
-
bs.next(eventData);
|
|
244
|
-
const pmKey = `pm:${this.consumerGroupName}:${streamName}`;
|
|
245
|
-
const currentTime = Date.now();
|
|
246
|
-
const transaction = this.redisGroups.multi({ pipeline: true });
|
|
247
|
-
transaction.zadd(pmKey, currentTime, messageId);
|
|
248
|
-
transaction.xack(streamName, this.consumerGroupName, id);
|
|
249
|
-
transaction.zadd(`ack:${streamName}`, Date.now(), id);
|
|
250
|
-
yield transaction.exec();
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
this.scanAndClaimAUnclaimedMessage(streamName)
|
|
254
|
-
.then()
|
|
255
|
-
.catch((e) => console.log('PUBLISHER: Err in handling unclaimed Messages ' + e.message));
|
|
256
|
-
}));
|
|
257
|
-
}
|
|
258
|
-
catch (e) {
|
|
259
|
-
console.error(JSON.stringify(e));
|
|
260
|
-
}
|
|
261
|
-
};
|
|
262
|
-
this.redisSubscriber.on('message', () => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
263
|
-
yield processMessage();
|
|
264
|
-
}));
|
|
265
|
-
return observable;
|
|
266
|
-
}
|
|
267
|
-
catch (e) {
|
|
268
|
-
console.error(JSON.stringify(e));
|
|
269
|
-
throw e;
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
/**
|
|
273
|
-
* This method takes all messages allocated to this instance and republishes them so
|
|
274
|
-
* that other instances of this service can receive and process them.
|
|
275
|
-
*
|
|
276
|
-
* This needs to be handled every 1-2 minutes if the queue becomes too long and messages
|
|
277
|
-
* are not being processed.
|
|
278
|
-
*
|
|
279
|
-
* Ideal implementation would be to wrap this inside a setInterval
|
|
280
|
-
* @param streamName
|
|
281
|
-
*/
|
|
282
|
-
republishUnprocessedEvents(eventName) {
|
|
203
|
+
createConsumerAndRegister(eventName) {
|
|
283
204
|
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
284
|
-
|
|
205
|
+
const pipeline = this.redisGroups.multi();
|
|
285
206
|
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
286
|
-
const
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
|
|
298
|
-
transaction.publish(eventName, '');
|
|
299
|
-
transaction.xack(streamName, this.consumerGroupName, id);
|
|
300
|
-
yield transaction.exec().catch(publisherErrorHandler);
|
|
301
|
-
console.log(`Event ${eventName} with ID: ${id} published`);
|
|
302
|
-
yield transaction.exec();
|
|
303
|
-
}
|
|
207
|
+
const key = `instance:${this.instanceId}:subscribedEvents`;
|
|
208
|
+
const setKeyForK8sHandling = `instance:${this.instanceUniqueId}:consumerGroupName`;
|
|
209
|
+
this.eventsListened.push(eventName);
|
|
210
|
+
pipeline.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
|
|
211
|
+
pipeline.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId);
|
|
212
|
+
pipeline.sadd(key, eventName);
|
|
213
|
+
pipeline.sadd(`${eventName}`, this.consumerGroupName);
|
|
214
|
+
pipeline.set(setKeyForK8sHandling, this.consumerGroupName);
|
|
215
|
+
const [, createConsumerStatus, , ,] = (yield pipeline.exec());
|
|
216
|
+
console.log(`PUBLISHER: Consumer Registered and created with ${this.instanceId} under ${this.consumerGroupName} with the ${createConsumerStatus[1]} consumers`);
|
|
217
|
+
return createConsumerStatus[1] === 0 || createConsumerStatus[1] === 1;
|
|
304
218
|
});
|
|
305
219
|
}
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
const pendingMessages = (yield this.redisGroups.xpending(streamName, this.consumerGroupName, 'IDLE', 10000, 0));
|
|
325
|
-
if (!pendingMessages)
|
|
326
|
-
return;
|
|
327
|
-
const [, minId, maxId, consumers] = pendingMessages;
|
|
328
|
-
if (!consumers || consumers.length === 0)
|
|
329
|
-
return;
|
|
330
|
-
for (const [consumer, pendingCount] of consumers) {
|
|
331
|
-
if (parseInt(pendingCount) < 0)
|
|
332
|
-
return;
|
|
333
|
-
const pending = (yield this.redisGroups.xpending(streamName, this.consumerGroupName, minId, maxId, Number(pendingCount), consumer));
|
|
334
|
-
if (!pending)
|
|
220
|
+
listenInternals(eventName) {
|
|
221
|
+
/** Create the return observable */
|
|
222
|
+
const bs = new rxjs_1.BehaviorSubject(null);
|
|
223
|
+
const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1));
|
|
224
|
+
/** This gets called the first time the stream is registered to pickup any messages from the previous subscription */
|
|
225
|
+
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
226
|
+
const processMessage = (redisClient) => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
227
|
+
const racePr = new Promise((resolve, _) => {
|
|
228
|
+
setTimeout(resolve, 500, 'RACE');
|
|
229
|
+
});
|
|
230
|
+
console.log(`PUBLISHER: processMessage called for ${streamName} cgn: ${this.consumerGroupName} inst: ${this.instanceId}`);
|
|
231
|
+
try {
|
|
232
|
+
const prResult = yield Promise.race([
|
|
233
|
+
redisClient.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', streamName, '>'),
|
|
234
|
+
racePr,
|
|
235
|
+
]);
|
|
236
|
+
console.log(`PUBLISHER: Promise race resolved with ${JSON.stringify(prResult)}`);
|
|
237
|
+
if (prResult && prResult === 'RACE')
|
|
335
238
|
return;
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
239
|
+
const result = prResult;
|
|
240
|
+
console.log(`PUBLISHER: XREADGROUP returned with ${JSON.stringify(result[0])}`);
|
|
241
|
+
if (result) {
|
|
242
|
+
const [, streamMessages] = result[0];
|
|
243
|
+
for (const [id, data] of streamMessages) {
|
|
341
244
|
const eventData = JSON.parse(data[1]);
|
|
342
|
-
const
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
245
|
+
const messageId = eventData.eventId;
|
|
246
|
+
const isDuplicate = yield this.isDuplicateMessage(streamName, messageId);
|
|
247
|
+
if (isDuplicate) {
|
|
248
|
+
console.warn(`Duplicate message detected: ${messageId}`);
|
|
249
|
+
yield redisClient.xack(streamName, this.consumerGroupName, id);
|
|
250
|
+
continue;
|
|
251
|
+
}
|
|
252
|
+
bs.next(eventData);
|
|
253
|
+
const pmKey = `pm:${this.consumerGroupName}:${streamName}`;
|
|
254
|
+
const currentTime = Date.now();
|
|
255
|
+
const transaction = redisClient.multi({ pipeline: true });
|
|
256
|
+
transaction.zadd(pmKey, currentTime, messageId);
|
|
257
|
+
transaction.xack(streamName, this.consumerGroupName, id);
|
|
258
|
+
transaction.zadd(`ack:${streamName}`, Date.now(), id);
|
|
259
|
+
yield transaction.exec();
|
|
347
260
|
}
|
|
348
261
|
}
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
yield this.redisGroups.xgroup('DELCONSUMER', streamName, this.consumerGroupName, consumer);
|
|
262
|
+
this.scanAndClaimAUnclaimedMessage(streamName)
|
|
263
|
+
.then()
|
|
264
|
+
.catch((e) => console.log('PUBLISHER: Err in handling unclaimed Messages ' + e.message));
|
|
353
265
|
}
|
|
266
|
+
catch (e) {
|
|
267
|
+
console.error(`PUBLISHER: ${JSON.stringify(e)}`);
|
|
268
|
+
}
|
|
269
|
+
});
|
|
270
|
+
this.createConsumerAndRegister(eventName)
|
|
271
|
+
.then((consumerRegistered) => {
|
|
272
|
+
if (!consumerRegistered)
|
|
273
|
+
throw new Error('PUBLISHER: Cannot setup consumer');
|
|
274
|
+
/** Create new REDIS connection and subscribe */
|
|
275
|
+
const eventStreamClient = registry_1.RedisRegistry.getConnection(`sub-${eventName}`);
|
|
276
|
+
eventStreamClient.subscribe(eventName).then(() => {
|
|
277
|
+
console.log(`PUBLISHER: Redis Subscription connection initiated for ${eventName} with ${JSON.stringify({
|
|
278
|
+
cluster: eventStreamClient.isCluster,
|
|
279
|
+
})}`);
|
|
280
|
+
});
|
|
281
|
+
eventStreamClient.on('message', () => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
282
|
+
console.log(`PUBLISHER: Stream Notification Recieved for event ${eventName}`);
|
|
283
|
+
yield processMessage(this.redisGroups);
|
|
284
|
+
}));
|
|
285
|
+
this.scanAndClaimAUnclaimedMessage(streamName)
|
|
286
|
+
.then()
|
|
287
|
+
.catch((e) => console.log('PUBLISHER: Err in handling unclaimed Messages ' + e.message));
|
|
288
|
+
})
|
|
289
|
+
.catch((e) => {
|
|
290
|
+
console.error(`PUBLISHER: ${JSON.stringify(e)}`);
|
|
291
|
+
throw e;
|
|
354
292
|
});
|
|
293
|
+
return observable;
|
|
355
294
|
}
|
|
356
295
|
/**
|
|
357
296
|
* This method allows the possibility of a graceful shutdown by cleaning up the
|
|
@@ -375,85 +314,32 @@ class Streams {
|
|
|
375
314
|
*/
|
|
376
315
|
close() {
|
|
377
316
|
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
378
|
-
|
|
317
|
+
if (this.cleanUpTimer) {
|
|
318
|
+
clearInterval(this.cleanUpTimer);
|
|
319
|
+
}
|
|
379
320
|
if (this.redisPublisher) {
|
|
380
321
|
yield this.redisPublisher.quit();
|
|
381
322
|
}
|
|
382
|
-
|
|
383
|
-
|
|
323
|
+
for (const eventName of this.eventsListened) {
|
|
324
|
+
registry_1.RedisRegistry.getConnection(`sub-${eventName}`).quit();
|
|
384
325
|
}
|
|
385
326
|
if (this.redisGroups) {
|
|
386
327
|
yield this.redisGroups.quit();
|
|
387
328
|
}
|
|
388
|
-
if (this.cleanUpTimer) {
|
|
389
|
-
clearInterval(this.cleanUpTimer);
|
|
390
|
-
}
|
|
391
|
-
});
|
|
392
|
-
}
|
|
393
|
-
clearSubscribedEvents() {
|
|
394
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
395
|
-
console.log(`${this.eventsListened.length} events to be cleared`);
|
|
396
|
-
let x = this.eventsListened.length;
|
|
397
|
-
for (const eventName of this.eventsListened) {
|
|
398
|
-
console.log(`${eventName} is being cleared in publisher`);
|
|
399
|
-
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
400
|
-
yield this.redisGroups.srem(`${eventName}`, this.consumerGroupName);
|
|
401
|
-
console.log(`${eventName} is removed from ${this.consumerGroupName}`);
|
|
402
|
-
// Releasing all claims based on info from: https://redis.io/commands/xgroup-delconsumer/
|
|
403
|
-
yield this.releaseAllClaims(streamName);
|
|
404
|
-
console.log(`${eventName} removes all claims`);
|
|
405
|
-
yield this.redisGroups.xgroup('DELCONSUMER', streamName, this.consumerGroupName, this.instanceId);
|
|
406
|
-
console.log(`${eventName} is deleted as a consumer from ${this.consumerGroupName}, ${this.instanceId}`);
|
|
407
|
-
console.log(x--);
|
|
408
|
-
}
|
|
409
|
-
});
|
|
410
|
-
}
|
|
411
|
-
registerSubscribedEvent(eventName) {
|
|
412
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
413
|
-
const key = `instance:${this.instanceId}:subscribedEvents`;
|
|
414
|
-
console.log(`Registering event name for ${this.consumerGroupName} with key : ${key}`);
|
|
415
|
-
yield this.redisGroups.sadd(key, eventName);
|
|
416
|
-
});
|
|
417
|
-
}
|
|
418
|
-
registerConsumerGroup(eventName) {
|
|
419
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
420
|
-
yield this.redisGroups.sadd(`${eventName}`, this.consumerGroupName);
|
|
421
|
-
});
|
|
422
|
-
}
|
|
423
|
-
registerConsumerGroupName() {
|
|
424
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
425
|
-
const key = `instance:${this.instanceUniqueId}:consumerGroupName`;
|
|
426
|
-
console.log(`Registering service name ${this.consumerGroupName} for key : ${key}`);
|
|
427
|
-
yield this.redisGroups.set(key, this.consumerGroupName);
|
|
428
329
|
});
|
|
429
330
|
}
|
|
430
331
|
scanAndClaimAUnclaimedMessage(streamName) {
|
|
431
332
|
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
432
|
-
const rows = yield this.redisGroups.xautoclaim(streamName, this.consumerGroupName, this.instanceId, 500, 0, 'COUNT', 1);
|
|
433
|
-
if (rows) {
|
|
333
|
+
const rows = yield this.redisGroups.xautoclaim(streamName, this.consumerGroupName, this.instanceId, 500, '0-0', 'COUNT', 1);
|
|
334
|
+
if (rows && rows[0] !== '0-0') {
|
|
335
|
+
console.log(`PUBLISHER: Handling pending unclaimed Message from ${streamName} for ${this.instanceId}`);
|
|
434
336
|
yield this.redisPublisher.publish(streamName.split(':')[0], '');
|
|
435
337
|
return this.scanAndClaimAUnclaimedMessage(streamName);
|
|
436
338
|
}
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
}
|
|
440
|
-
releaseAllClaims(streamName) {
|
|
441
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
442
|
-
/**
|
|
443
|
-
* Retrieve the pending messages for the consumer. Note this only fetches the last
|
|
444
|
-
* 10000 events assigned to this consumer. This function has been modified to make sure
|
|
445
|
-
* that there is a temp instance that claims all this messages
|
|
446
|
-
*/
|
|
447
|
-
const pendingMessages = (yield this.redisGroups.xpending(streamName, this.consumerGroupName, '-', '+', 10000, this.instanceId));
|
|
448
|
-
if (pendingMessages && pendingMessages.length > 0) {
|
|
449
|
-
console.log(`${pendingMessages.length} messages to clean up.`);
|
|
450
|
-
const transaction = this.redisGroups.multi({ pipeline: true });
|
|
451
|
-
const tempConsumerId = `${this.consumerGroupName}-temp`;
|
|
452
|
-
for (const [messageId] of pendingMessages) {
|
|
453
|
-
transaction.xclaim(streamName, this.consumerGroupName, tempConsumerId, 10, messageId);
|
|
454
|
-
}
|
|
455
|
-
yield transaction.exec();
|
|
339
|
+
else {
|
|
340
|
+
console.log(`PUBLISHER: No previous messages found for ${streamName}`);
|
|
456
341
|
}
|
|
342
|
+
return;
|
|
457
343
|
});
|
|
458
344
|
}
|
|
459
345
|
cleanupAcknowledgedMessages(eventName, interval = 60 * 60 * 1000) {
|