@jetit/publisher 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/lib/redis/streams.d.ts +3 -37
- package/src/lib/redis/streams.js +101 -225
package/package.json
CHANGED
|
@@ -1,18 +1,15 @@
|
|
|
1
|
-
import { RedisType } from './registry';
|
|
2
1
|
import { EventData, PublishData } from './types';
|
|
3
2
|
import { Observable } from 'rxjs';
|
|
4
3
|
export declare class Streams {
|
|
5
4
|
private _redisPublisher?;
|
|
6
|
-
private _redisSubscriber?;
|
|
7
5
|
private _redisGroups?;
|
|
8
6
|
private consumerGroupName;
|
|
9
7
|
private instanceId;
|
|
10
8
|
private instanceUniqueId;
|
|
11
9
|
private cleanUpTimer;
|
|
12
10
|
private eventsListened;
|
|
13
|
-
get redisPublisher()
|
|
14
|
-
get
|
|
15
|
-
get redisGroups(): RedisType;
|
|
11
|
+
private get redisPublisher();
|
|
12
|
+
private get redisGroups();
|
|
16
13
|
/**
|
|
17
14
|
* Creates a new Streams instance for a given service.
|
|
18
15
|
*
|
|
@@ -29,7 +26,6 @@ export declare class Streams {
|
|
|
29
26
|
*/
|
|
30
27
|
constructor(serviceName: string);
|
|
31
28
|
private runClear;
|
|
32
|
-
private createConsumerGroup;
|
|
33
29
|
private isDuplicateMessage;
|
|
34
30
|
private clearDuplicationCheckKeys;
|
|
35
31
|
/**
|
|
@@ -100,33 +96,8 @@ export declare class Streams {
|
|
|
100
96
|
* });
|
|
101
97
|
*/
|
|
102
98
|
listen<T = unknown, const TName extends string = string>(eventName: TName, maxRetries?: number, initialDelay?: number): Observable<EventData<T, TName>>;
|
|
99
|
+
private createConsumerAndRegister;
|
|
103
100
|
private listenInternals;
|
|
104
|
-
/**
|
|
105
|
-
* This method takes all messages allocated to this instance and republishes them so
|
|
106
|
-
* that other instances of this service can receive and process them.
|
|
107
|
-
*
|
|
108
|
-
* This needs to be handled every 1-2 minutes if the queue becomes too long and messages
|
|
109
|
-
* are not being processed.
|
|
110
|
-
*
|
|
111
|
-
* Ideal implementation would be to wrap this inside a setInterval
|
|
112
|
-
* @param streamName
|
|
113
|
-
*/
|
|
114
|
-
republishUnprocessedEvents(eventName: string): Promise<void>;
|
|
115
|
-
/**
|
|
116
|
-
* This method is used to claim messages in the event of a service crash. This library currently
|
|
117
|
-
* does not detect a service crash. This needs to be built as an extension of Kubernetes and
|
|
118
|
-
* a standalone service that notifies this service to process the events that are marked as
|
|
119
|
-
* pending
|
|
120
|
-
*
|
|
121
|
-
* @param streamName
|
|
122
|
-
* @param idleTimeout
|
|
123
|
-
*
|
|
124
|
-
* * @example
|
|
125
|
-
*
|
|
126
|
-
* // Attempt to recover messages from the "order.created" stream with an idle timeout of 10 seconds
|
|
127
|
-
* await streams.recoverCrashedConsumerMessages('order.created', 10000);
|
|
128
|
-
*/
|
|
129
|
-
recoverCrashedConsumerMessages(eventName: string, idleTimeout?: number): Promise<void>;
|
|
130
101
|
/**
|
|
131
102
|
* This method allows the possibility of a graceful shutdown by cleaning up the
|
|
132
103
|
* redis connections.
|
|
@@ -148,11 +119,6 @@ export declare class Streams {
|
|
|
148
119
|
* }
|
|
149
120
|
*/
|
|
150
121
|
close(): Promise<void>;
|
|
151
|
-
private clearSubscribedEvents;
|
|
152
|
-
private registerSubscribedEvent;
|
|
153
|
-
private registerConsumerGroup;
|
|
154
|
-
private registerConsumerGroupName;
|
|
155
122
|
private scanAndClaimAUnclaimedMessage;
|
|
156
|
-
releaseAllClaims(streamName: string): Promise<void>;
|
|
157
123
|
private cleanupAcknowledgedMessages;
|
|
158
124
|
}
|
package/src/lib/redis/streams.js
CHANGED
|
@@ -7,7 +7,7 @@ const rxjs_1 = require("rxjs");
|
|
|
7
7
|
const id_1 = require("@jetit/id");
|
|
8
8
|
const groups_1 = require("./groups");
|
|
9
9
|
function publisherErrorHandler(error) {
|
|
10
|
-
console.error('
|
|
10
|
+
console.error('PUBLISHER UNHANDLED ERROR: ', error);
|
|
11
11
|
}
|
|
12
12
|
class Streams {
|
|
13
13
|
get redisPublisher() {
|
|
@@ -15,11 +15,6 @@ class Streams {
|
|
|
15
15
|
this._redisPublisher = registry_1.RedisRegistry.getConnection('publish');
|
|
16
16
|
return this._redisPublisher;
|
|
17
17
|
}
|
|
18
|
-
get redisSubscriber() {
|
|
19
|
-
if (!this._redisSubscriber)
|
|
20
|
-
this._redisSubscriber = registry_1.RedisRegistry.getConnection('subscriber');
|
|
21
|
-
return this._redisSubscriber;
|
|
22
|
-
}
|
|
23
18
|
get redisGroups() {
|
|
24
19
|
if (!this._redisGroups)
|
|
25
20
|
this._redisGroups = registry_1.RedisRegistry.getConnection('groups');
|
|
@@ -44,8 +39,8 @@ class Streams {
|
|
|
44
39
|
this.eventsListened = [];
|
|
45
40
|
this.instanceUniqueId = (_a = process.env['INSTANCE_ID']) !== null && _a !== void 0 ? _a : (0, id_1.generateID)('HEX', 'FE');
|
|
46
41
|
this.instanceId = `${serviceName}:${this.instanceUniqueId}`;
|
|
47
|
-
console.log(this.instanceId);
|
|
48
42
|
this.consumerGroupName = `cg-${serviceName}`;
|
|
43
|
+
console.log(`PUBLISHER: Instance ID: ${this.instanceId}`);
|
|
49
44
|
const cleanUpInterval = (_b = parseInt(process.env['CLEANUP_INTERVAL'] || `${1000 * 60 * 60}`, 10)) !== null && _b !== void 0 ? _b : 1000 * 60 * 60;
|
|
50
45
|
setTimeout(() => this.runClear(cleanUpInterval), 60000);
|
|
51
46
|
this.cleanUpTimer = setInterval(() => {
|
|
@@ -54,30 +49,24 @@ class Streams {
|
|
|
54
49
|
}
|
|
55
50
|
runClear(cleanUpInterval) {
|
|
56
51
|
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
57
|
-
console.log('Running
|
|
52
|
+
console.log('PUBLISHER: Running Clearance', this.eventsListened);
|
|
58
53
|
this.clearDuplicationCheckKeys();
|
|
59
54
|
for (const eventName of this.eventsListened) {
|
|
60
55
|
process.nextTick(() => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
56
|
+
/** This removes the messages from the stream after they have been processed according to cleanup interval */
|
|
61
57
|
yield this.cleanupAcknowledgedMessages(eventName, cleanUpInterval).catch(publisherErrorHandler);
|
|
62
|
-
|
|
63
|
-
|
|
58
|
+
console.log(`Cleanup process for Acknowledged messages completed for ${eventName}`);
|
|
59
|
+
/**
|
|
60
|
+
* This process scans and claims any unclaimed message according to the cleanup interval.
|
|
61
|
+
* This triggers a cascaded reaction down the chain as message by message is claimed and processed
|
|
62
|
+
*/
|
|
63
|
+
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
64
|
+
yield this.scanAndClaimAUnclaimedMessage(streamName).catch(publisherErrorHandler);
|
|
65
|
+
console.log(`Unclaimed messages for ${streamName}`);
|
|
64
66
|
}));
|
|
65
67
|
}
|
|
66
68
|
});
|
|
67
69
|
}
|
|
68
|
-
createConsumerGroup(eventName) {
|
|
69
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
70
|
-
try {
|
|
71
|
-
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
72
|
-
yield this.redisGroups.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
|
|
73
|
-
}
|
|
74
|
-
catch (error) {
|
|
75
|
-
if (error.message !== 'BUSYGROUP Consumer Group name already exists') {
|
|
76
|
-
throw error;
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
70
|
isDuplicateMessage(streamName, messageId) {
|
|
82
71
|
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
83
72
|
const processedMessagesKey = `pm:${this.consumerGroupName}:${streamName}`;
|
|
@@ -115,7 +104,7 @@ class Streams {
|
|
|
115
104
|
const transaction = this.redisPublisher.multi({ pipeline: true });
|
|
116
105
|
const consumerGroups = yield (0, groups_1.getAllConsumerGroups)(data.eventName, this.redisPublisher);
|
|
117
106
|
if (consumerGroups.length > 0) {
|
|
118
|
-
console.log(`Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
|
|
107
|
+
console.log(`PUBLISHER: Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
|
|
119
108
|
for (const consumerGroup of consumerGroups) {
|
|
120
109
|
// Publish the event to each consumer group's stream
|
|
121
110
|
const streamName = `${data.eventName}:${consumerGroup}`;
|
|
@@ -123,7 +112,7 @@ class Streams {
|
|
|
123
112
|
}
|
|
124
113
|
transaction.publish(data.eventName, '');
|
|
125
114
|
yield transaction.exec().catch((error) => {
|
|
126
|
-
console.error(`Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
|
|
115
|
+
console.error(`PUBLISHER: Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
|
|
127
116
|
throw new Error('Publisher Error');
|
|
128
117
|
});
|
|
129
118
|
}
|
|
@@ -154,7 +143,7 @@ class Streams {
|
|
|
154
143
|
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
155
144
|
const currentTime = new Date();
|
|
156
145
|
if (scheduledTime < currentTime) {
|
|
157
|
-
throw new Error('Cannot schedule an event in the past');
|
|
146
|
+
throw new Error('PUBLISHER: Cannot schedule an event in the past');
|
|
158
147
|
}
|
|
159
148
|
else if (Math.abs(scheduledTime.getTime() - currentTime.getTime()) <= 500) {
|
|
160
149
|
yield this.publish(eventData);
|
|
@@ -163,7 +152,7 @@ class Streams {
|
|
|
163
152
|
if (uniquePerInstance === true) {
|
|
164
153
|
const existingJob = yield this.redisPublisher.zscore('se', JSON.stringify(eventData));
|
|
165
154
|
if (existingJob) {
|
|
166
|
-
console.log(`Job with data '${eventData}' already exists. Skipping.`);
|
|
155
|
+
console.log(`PUBLISHER: Job with data '${eventData}' already exists. Skipping.`);
|
|
167
156
|
return;
|
|
168
157
|
}
|
|
169
158
|
}
|
|
@@ -199,159 +188,99 @@ class Streams {
|
|
|
199
188
|
* });
|
|
200
189
|
*/
|
|
201
190
|
listen(eventName, maxRetries = 5, initialDelay = 1000) {
|
|
202
|
-
this.registerConsumerGroup(eventName); // Registers the consumer group for listening to the message
|
|
203
|
-
this.registerConsumerGroupName().then();
|
|
204
|
-
this.eventsListened.push(eventName);
|
|
205
191
|
return this.listenInternals(eventName).pipe((0, rxjs_1.retry)({
|
|
206
192
|
count: maxRetries,
|
|
207
193
|
delay: (error, retryAttempt) => {
|
|
208
194
|
const delay = initialDelay * Math.pow(2, retryAttempt);
|
|
209
|
-
console.error(`Error in listen: ${error.message}. Retrying in ${delay}ms (attempt ${retryAttempt + 1})`);
|
|
195
|
+
console.error(`PUBLISHER: Error in listen: ${error.message}. Retrying in ${delay}ms (attempt ${retryAttempt + 1})`);
|
|
210
196
|
return (0, rxjs_1.timer)(delay);
|
|
211
197
|
},
|
|
212
198
|
}), (0, rxjs_1.catchError)((error) => {
|
|
213
|
-
console.error(`Error in listen after ${maxRetries} retries: ${error.message}`);
|
|
199
|
+
console.error(`PUBLISHER: Error in listen after ${maxRetries} retries: ${error.message}`);
|
|
214
200
|
return (0, rxjs_1.throwError)(() => new Error(error.message));
|
|
215
201
|
}));
|
|
216
202
|
}
|
|
217
|
-
|
|
218
|
-
try {
|
|
219
|
-
this.createConsumerGroup(eventName).then();
|
|
220
|
-
this.registerSubscribedEvent(eventName).then();
|
|
221
|
-
const bs = new rxjs_1.BehaviorSubject(null);
|
|
222
|
-
const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1));
|
|
223
|
-
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
224
|
-
this.redisSubscriber.subscribe(eventName);
|
|
225
|
-
this.scanAndClaimAUnclaimedMessage(streamName)
|
|
226
|
-
.then()
|
|
227
|
-
.catch((e) => console.log('PUBLISHER: Err in handling unclaimed Messages ' + e.message));
|
|
228
|
-
const processMessage = () => {
|
|
229
|
-
try {
|
|
230
|
-
process.nextTick(() => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
231
|
-
const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', streamName, '>');
|
|
232
|
-
if (result) {
|
|
233
|
-
const [, streamMessages] = result[0];
|
|
234
|
-
for (const [id, data] of streamMessages) {
|
|
235
|
-
const eventData = JSON.parse(data[1]);
|
|
236
|
-
const messageId = eventData.eventId;
|
|
237
|
-
const isDuplicate = yield this.isDuplicateMessage(streamName, messageId);
|
|
238
|
-
if (isDuplicate) {
|
|
239
|
-
console.warn(`Duplicate message detected: ${messageId}`);
|
|
240
|
-
yield this.redisGroups.xack(streamName, this.consumerGroupName, id);
|
|
241
|
-
continue;
|
|
242
|
-
}
|
|
243
|
-
bs.next(eventData);
|
|
244
|
-
const pmKey = `pm:${this.consumerGroupName}:${streamName}`;
|
|
245
|
-
const currentTime = Date.now();
|
|
246
|
-
const transaction = this.redisGroups.multi({ pipeline: true });
|
|
247
|
-
transaction.zadd(pmKey, currentTime, messageId);
|
|
248
|
-
transaction.xack(streamName, this.consumerGroupName, id);
|
|
249
|
-
transaction.zadd(`ack:${streamName}`, Date.now(), id);
|
|
250
|
-
yield transaction.exec();
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
this.scanAndClaimAUnclaimedMessage(streamName)
|
|
254
|
-
.then()
|
|
255
|
-
.catch((e) => console.log('PUBLISHER: Err in handling unclaimed Messages ' + e.message));
|
|
256
|
-
}));
|
|
257
|
-
}
|
|
258
|
-
catch (e) {
|
|
259
|
-
console.error(JSON.stringify(e));
|
|
260
|
-
}
|
|
261
|
-
};
|
|
262
|
-
this.redisSubscriber.on('message', () => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
263
|
-
yield processMessage();
|
|
264
|
-
}));
|
|
265
|
-
return observable;
|
|
266
|
-
}
|
|
267
|
-
catch (e) {
|
|
268
|
-
console.error(JSON.stringify(e));
|
|
269
|
-
throw e;
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
/**
|
|
273
|
-
* This method takes all messages allocated to this instance and republishes them so
|
|
274
|
-
* that other instances of this service can receive and process them.
|
|
275
|
-
*
|
|
276
|
-
* This needs to be handled every 1-2 minutes if the queue becomes too long and messages
|
|
277
|
-
* are not being processed.
|
|
278
|
-
*
|
|
279
|
-
* Ideal implementation would be to wrap this inside a setInterval
|
|
280
|
-
* @param streamName
|
|
281
|
-
*/
|
|
282
|
-
republishUnprocessedEvents(eventName) {
|
|
203
|
+
createConsumerAndRegister(eventName) {
|
|
283
204
|
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
284
|
-
|
|
205
|
+
const pipeline = this.redisGroups.multi();
|
|
285
206
|
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
286
|
-
const
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
|
|
298
|
-
transaction.publish(eventName, '');
|
|
299
|
-
transaction.xack(streamName, this.consumerGroupName, id);
|
|
300
|
-
yield transaction.exec().catch(publisherErrorHandler);
|
|
301
|
-
console.log(`Event ${eventName} with ID: ${id} published`);
|
|
302
|
-
yield transaction.exec();
|
|
303
|
-
}
|
|
207
|
+
const key = `instance:${this.instanceId}:subscribedEvents`;
|
|
208
|
+
const setKeyForK8sHandling = `instance:${this.instanceUniqueId}:consumerGroupName`;
|
|
209
|
+
this.eventsListened.push(eventName);
|
|
210
|
+
pipeline.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
|
|
211
|
+
pipeline.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId);
|
|
212
|
+
pipeline.sadd(key, eventName);
|
|
213
|
+
pipeline.sadd(`${eventName}`, this.consumerGroupName);
|
|
214
|
+
pipeline.set(setKeyForK8sHandling, this.consumerGroupName);
|
|
215
|
+
const [, createConsumerStatus, , ,] = (yield pipeline.exec());
|
|
216
|
+
console.log(`PUBLISHER: Consumer Registered and created with ${this.instanceId} under ${this.consumerGroupName} with the ${createConsumerStatus[1]} consumers`);
|
|
217
|
+
return createConsumerStatus[1] === 0 || createConsumerStatus[1] === 1;
|
|
304
218
|
});
|
|
305
219
|
}
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
recoverCrashedConsumerMessages(eventName, idleTimeout = 30000) {
|
|
321
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
322
|
-
console.log(`PUBLISHER: Running recoverCrashedConsumerMessages`);
|
|
323
|
-
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
324
|
-
const pendingMessages = (yield this.redisGroups.xpending(streamName, this.consumerGroupName, 'IDLE', 10000, 0));
|
|
325
|
-
if (!pendingMessages)
|
|
326
|
-
return;
|
|
327
|
-
const [, minId, maxId, consumers] = pendingMessages;
|
|
328
|
-
if (!consumers || consumers.length === 0)
|
|
329
|
-
return;
|
|
330
|
-
for (const [consumer, pendingCount] of consumers) {
|
|
331
|
-
if (parseInt(pendingCount) < 0)
|
|
332
|
-
return;
|
|
333
|
-
const pending = (yield this.redisGroups.xpending(streamName, this.consumerGroupName, minId, maxId, Number(pendingCount), consumer));
|
|
334
|
-
if (!pending)
|
|
335
|
-
return;
|
|
336
|
-
for (const [messageId] of pending) {
|
|
337
|
-
const claimedMessage = (yield this.redisGroups.xclaim(streamName, this.consumerGroupName, this.instanceId, idleTimeout, messageId));
|
|
338
|
-
if (claimedMessage) {
|
|
339
|
-
console.log({ claimedMessage: JSON.stringify(claimedMessage) });
|
|
340
|
-
const [, data] = claimedMessage[0];
|
|
220
|
+
listenInternals(eventName) {
|
|
221
|
+
/** Create the return observable */
|
|
222
|
+
const bs = new rxjs_1.BehaviorSubject(null);
|
|
223
|
+
const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1));
|
|
224
|
+
/** This gets called the first time the stream is registered to pickup any messages from the previous subscription */
|
|
225
|
+
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
226
|
+
const processMessage = () => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
227
|
+
console.log(`PUBLISHER: processMessage called for ${streamName} cgn: ${this.consumerGroupName} inst: ${this.instanceId}`);
|
|
228
|
+
try {
|
|
229
|
+
const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', streamName, '>');
|
|
230
|
+
console.log(`PUBLISHER: XREADGROUP returned with ${JSON.stringify(result[0])}`);
|
|
231
|
+
if (result) {
|
|
232
|
+
const [, streamMessages] = result[0];
|
|
233
|
+
for (const [id, data] of streamMessages) {
|
|
341
234
|
const eventData = JSON.parse(data[1]);
|
|
342
|
-
const
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
235
|
+
const messageId = eventData.eventId;
|
|
236
|
+
const isDuplicate = yield this.isDuplicateMessage(streamName, messageId);
|
|
237
|
+
if (isDuplicate) {
|
|
238
|
+
console.warn(`Duplicate message detected: ${messageId}`);
|
|
239
|
+
yield this.redisGroups.xack(streamName, this.consumerGroupName, id);
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
bs.next(eventData);
|
|
243
|
+
const pmKey = `pm:${this.consumerGroupName}:${streamName}`;
|
|
244
|
+
const currentTime = Date.now();
|
|
245
|
+
const transaction = this.redisGroups.multi({ pipeline: true });
|
|
246
|
+
transaction.zadd(pmKey, currentTime, messageId);
|
|
247
|
+
transaction.xack(streamName, this.consumerGroupName, id);
|
|
248
|
+
transaction.zadd(`ack:${streamName}`, Date.now(), id);
|
|
249
|
+
yield transaction.exec();
|
|
347
250
|
}
|
|
348
251
|
}
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
252
|
+
this.scanAndClaimAUnclaimedMessage(streamName)
|
|
253
|
+
.then()
|
|
254
|
+
.catch((e) => console.log('PUBLISHER: Err in handling unclaimed Messages ' + e.message));
|
|
255
|
+
}
|
|
256
|
+
catch (e) {
|
|
257
|
+
console.error(`PUBLISHER: ${JSON.stringify(e)}`);
|
|
353
258
|
}
|
|
354
259
|
});
|
|
260
|
+
this.createConsumerAndRegister(eventName)
|
|
261
|
+
.then((consumerRegistered) => {
|
|
262
|
+
if (!consumerRegistered)
|
|
263
|
+
throw new Error('PUBLISHER: Cannot setup consumer');
|
|
264
|
+
/** Create new REDIS connection and subscribe */
|
|
265
|
+
const eventStreamClient = registry_1.RedisRegistry.getConnection(`sub-${eventName}`);
|
|
266
|
+
eventStreamClient.subscribe(eventName).then(() => {
|
|
267
|
+
console.log(`PUBLISHER: Redis Subscription connection initiated for ${eventName} with ${JSON.stringify({
|
|
268
|
+
cluster: eventStreamClient.isCluster,
|
|
269
|
+
})}`);
|
|
270
|
+
});
|
|
271
|
+
eventStreamClient.on('message', () => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
272
|
+
console.log(`PUBLISHER: Stream Notification Recieved for event ${eventName}`);
|
|
273
|
+
yield processMessage();
|
|
274
|
+
}));
|
|
275
|
+
this.scanAndClaimAUnclaimedMessage(streamName)
|
|
276
|
+
.then()
|
|
277
|
+
.catch((e) => console.log('PUBLISHER: Err in handling unclaimed Messages ' + e.message));
|
|
278
|
+
})
|
|
279
|
+
.catch((e) => {
|
|
280
|
+
console.error(`PUBLISHER: ${JSON.stringify(e)}`);
|
|
281
|
+
throw e;
|
|
282
|
+
});
|
|
283
|
+
return observable;
|
|
355
284
|
}
|
|
356
285
|
/**
|
|
357
286
|
* This method allows the possibility of a graceful shutdown by cleaning up the
|
|
@@ -375,85 +304,32 @@ class Streams {
|
|
|
375
304
|
*/
|
|
376
305
|
close() {
|
|
377
306
|
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
378
|
-
|
|
307
|
+
if (this.cleanUpTimer) {
|
|
308
|
+
clearInterval(this.cleanUpTimer);
|
|
309
|
+
}
|
|
379
310
|
if (this.redisPublisher) {
|
|
380
311
|
yield this.redisPublisher.quit();
|
|
381
312
|
}
|
|
382
|
-
|
|
383
|
-
|
|
313
|
+
for (const eventName of this.eventsListened) {
|
|
314
|
+
registry_1.RedisRegistry.getConnection(`sub-${eventName}`).quit();
|
|
384
315
|
}
|
|
385
316
|
if (this.redisGroups) {
|
|
386
317
|
yield this.redisGroups.quit();
|
|
387
318
|
}
|
|
388
|
-
if (this.cleanUpTimer) {
|
|
389
|
-
clearInterval(this.cleanUpTimer);
|
|
390
|
-
}
|
|
391
|
-
});
|
|
392
|
-
}
|
|
393
|
-
clearSubscribedEvents() {
|
|
394
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
395
|
-
console.log(`${this.eventsListened.length} events to be cleared`);
|
|
396
|
-
let x = this.eventsListened.length;
|
|
397
|
-
for (const eventName of this.eventsListened) {
|
|
398
|
-
console.log(`${eventName} is being cleared in publisher`);
|
|
399
|
-
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
400
|
-
yield this.redisGroups.srem(`${eventName}`, this.consumerGroupName);
|
|
401
|
-
console.log(`${eventName} is removed from ${this.consumerGroupName}`);
|
|
402
|
-
// Releasing all claims based on info from: https://redis.io/commands/xgroup-delconsumer/
|
|
403
|
-
yield this.releaseAllClaims(streamName);
|
|
404
|
-
console.log(`${eventName} removes all claims`);
|
|
405
|
-
yield this.redisGroups.xgroup('DELCONSUMER', streamName, this.consumerGroupName, this.instanceId);
|
|
406
|
-
console.log(`${eventName} is deleted as a consumer from ${this.consumerGroupName}, ${this.instanceId}`);
|
|
407
|
-
console.log(x--);
|
|
408
|
-
}
|
|
409
|
-
});
|
|
410
|
-
}
|
|
411
|
-
registerSubscribedEvent(eventName) {
|
|
412
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
413
|
-
const key = `instance:${this.instanceId}:subscribedEvents`;
|
|
414
|
-
console.log(`Registering event name for ${this.consumerGroupName} with key : ${key}`);
|
|
415
|
-
yield this.redisGroups.sadd(key, eventName);
|
|
416
|
-
});
|
|
417
|
-
}
|
|
418
|
-
registerConsumerGroup(eventName) {
|
|
419
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
420
|
-
yield this.redisGroups.sadd(`${eventName}`, this.consumerGroupName);
|
|
421
|
-
});
|
|
422
|
-
}
|
|
423
|
-
registerConsumerGroupName() {
|
|
424
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
425
|
-
const key = `instance:${this.instanceUniqueId}:consumerGroupName`;
|
|
426
|
-
console.log(`Registering service name ${this.consumerGroupName} for key : ${key}`);
|
|
427
|
-
yield this.redisGroups.set(key, this.consumerGroupName);
|
|
428
319
|
});
|
|
429
320
|
}
|
|
430
321
|
scanAndClaimAUnclaimedMessage(streamName) {
|
|
431
322
|
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
432
|
-
const rows = yield this.redisGroups.xautoclaim(streamName, this.consumerGroupName, this.instanceId, 500, 0, 'COUNT', 1);
|
|
433
|
-
if (rows) {
|
|
323
|
+
const rows = yield this.redisGroups.xautoclaim(streamName, this.consumerGroupName, this.instanceId, 500, '0-0', 'COUNT', 1);
|
|
324
|
+
if (rows && rows[0] !== '0-0') {
|
|
325
|
+
console.log(`PUBLISHER: Handling pending unclaimed Message from ${streamName} for ${this.instanceId}`);
|
|
434
326
|
yield this.redisPublisher.publish(streamName.split(':')[0], '');
|
|
435
327
|
return this.scanAndClaimAUnclaimedMessage(streamName);
|
|
436
328
|
}
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
}
|
|
440
|
-
releaseAllClaims(streamName) {
|
|
441
|
-
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
442
|
-
/**
|
|
443
|
-
* Retrieve the pending messages for the consumer. Note this only fetches the last
|
|
444
|
-
* 10000 events assigned to this consumer. This function has been modified to make sure
|
|
445
|
-
* that there is a temp instance that claims all this messages
|
|
446
|
-
*/
|
|
447
|
-
const pendingMessages = (yield this.redisGroups.xpending(streamName, this.consumerGroupName, '-', '+', 10000, this.instanceId));
|
|
448
|
-
if (pendingMessages && pendingMessages.length > 0) {
|
|
449
|
-
console.log(`${pendingMessages.length} messages to clean up.`);
|
|
450
|
-
const transaction = this.redisGroups.multi({ pipeline: true });
|
|
451
|
-
const tempConsumerId = `${this.consumerGroupName}-temp`;
|
|
452
|
-
for (const [messageId] of pendingMessages) {
|
|
453
|
-
transaction.xclaim(streamName, this.consumerGroupName, tempConsumerId, 10, messageId);
|
|
454
|
-
}
|
|
455
|
-
yield transaction.exec();
|
|
329
|
+
else {
|
|
330
|
+
console.log(`PUBLISHER: No previous messages found for ${streamName}`);
|
|
456
331
|
}
|
|
332
|
+
return;
|
|
457
333
|
});
|
|
458
334
|
}
|
|
459
335
|
cleanupAcknowledgedMessages(eventName, interval = 60 * 60 * 1000) {
|