@jetit/publisher 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -2
- package/src/README.md +43 -0
- package/src/index.d.ts +1 -0
- package/src/index.js +4 -0
- package/src/lib/publisher.d.ts +3 -0
- package/src/lib/publisher.js +9 -0
- package/src/lib/redis/groups.d.ts +2 -0
- package/src/lib/redis/groups.js +11 -0
- package/src/lib/redis/registry.d.ts +20 -0
- package/src/lib/redis/registry.js +57 -0
- package/src/lib/redis/scheduler.d.ts +15 -0
- package/src/lib/redis/scheduler.js +80 -0
- package/src/lib/redis/streams.d.ts +152 -0
- package/src/lib/redis/streams.js +392 -0
- package/src/lib/redis/types.d.ts +15 -0
- package/src/lib/redis/types.js +2 -0
- package/index.d.ts +0 -222
- package/index.js +0 -553
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Streams = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const registry_1 = require("./registry");
|
|
6
|
+
const rxjs_1 = require("rxjs");
|
|
7
|
+
const id_1 = require("@jetit/id");
|
|
8
|
+
const groups_1 = require("./groups");
|
|
9
|
+
class Streams {
|
|
10
|
+
get redisPublisher() {
|
|
11
|
+
if (!this._redisPublisher)
|
|
12
|
+
this._redisPublisher = registry_1.RedisRegistry.getConnection('publish');
|
|
13
|
+
return this._redisPublisher;
|
|
14
|
+
}
|
|
15
|
+
get redisSubscriber() {
|
|
16
|
+
if (!this._redisSubscriber)
|
|
17
|
+
this._redisSubscriber = registry_1.RedisRegistry.getConnection('subscriber');
|
|
18
|
+
return this._redisSubscriber;
|
|
19
|
+
}
|
|
20
|
+
get redisGroups() {
|
|
21
|
+
if (!this._redisGroups)
|
|
22
|
+
this._redisGroups = registry_1.RedisRegistry.getConnection('groups');
|
|
23
|
+
return this._redisGroups;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Creates a new Streams instance for a given service.
|
|
27
|
+
*
|
|
28
|
+
* The constructor initializes the Redis connections for publishers, subscribers, and consumer groups.
|
|
29
|
+
* It also sets up an interval timer for clearing expired messages from Redis and another interval timer
|
|
30
|
+
* for processing scheduled events at regular intervals.
|
|
31
|
+
*
|
|
32
|
+
* @param serviceName - A unique name for the service that will be using this Streams instance.
|
|
33
|
+
*
|
|
34
|
+
* @example
|
|
35
|
+
*
|
|
36
|
+
* // Create a new Streams instance for the "POS" service
|
|
37
|
+
* const streams = new Streams('POS');
|
|
38
|
+
*/
|
|
39
|
+
constructor(serviceName) {
|
|
40
|
+
var _a;
|
|
41
|
+
this.eventsListened = [];
|
|
42
|
+
this.instanceId = `${serviceName}:${(0, id_1.generateID)('HEX', 'FE')}`;
|
|
43
|
+
this.consumerGroupName = `cg-${serviceName}`;
|
|
44
|
+
const cleanUpInterval = (_a = parseInt(process.env['CLEANUP_INTERVAL'] || '1000 * 60 * 60', 10)) !== null && _a !== void 0 ? _a : 1000 * 60 * 60;
|
|
45
|
+
this.cleanUpTimer = (0, rxjs_1.interval)(cleanUpInterval).subscribe(() => {
|
|
46
|
+
this.clearDuplicationCheckKeys();
|
|
47
|
+
this.eventsListened.forEach((eventName) => this.cleanupAcknowledgedMessages(eventName, cleanUpInterval));
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
createConsumerGroup(eventName) {
|
|
51
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
52
|
+
try {
|
|
53
|
+
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
54
|
+
yield this.redisGroups.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM');
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
if (error.message !== 'BUSYGROUP Consumer Group name already exists') {
|
|
58
|
+
throw error;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
isDuplicateMessage(streamName, messageId) {
|
|
64
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
65
|
+
const processedMessagesKey = `pm:${this.consumerGroupName}:${streamName}`;
|
|
66
|
+
const temp = yield Promise.race([
|
|
67
|
+
this.redisGroups.zscore(processedMessagesKey, messageId),
|
|
68
|
+
/** ioRedis doesnt seem to return the nil event. So waiting for 100ms before moving on */
|
|
69
|
+
new Promise((res) => setTimeout(() => res(null), 100)),
|
|
70
|
+
]);
|
|
71
|
+
return temp !== null;
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
clearDuplicationCheckKeys() {
|
|
75
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
76
|
+
const processedMessagesKeyPattern = `pm:${this.consumerGroupName}:*`;
|
|
77
|
+
let cursor = '0';
|
|
78
|
+
do {
|
|
79
|
+
const [nextCursor, keys] = yield this.redisGroups.scan(cursor, 'MATCH', processedMessagesKeyPattern);
|
|
80
|
+
cursor = nextCursor;
|
|
81
|
+
for (const key of keys) {
|
|
82
|
+
const oneHourAgo = Date.now() - 60 * 60 * 1000;
|
|
83
|
+
yield this.redisGroups.zremrangebyscore(key, '-inf', oneHourAgo);
|
|
84
|
+
}
|
|
85
|
+
} while (cursor !== '0');
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Publishes an event with the given data to the Redis event stream.
|
|
90
|
+
*
|
|
91
|
+
* The method generates a unique event ID for each event, and adds it to the event data to handle message
|
|
92
|
+
* deduplication.
|
|
93
|
+
*
|
|
94
|
+
* @param data - An EventData<T> object containing the data for the event.
|
|
95
|
+
*
|
|
96
|
+
* @returns A Promise that resolves when the event has been published to the Redis stream.
|
|
97
|
+
*
|
|
98
|
+
* @example
|
|
99
|
+
*
|
|
100
|
+
* // Publish an "order.created" event with the given order data
|
|
101
|
+
* const orderData = { id: '123', customerName: 'John Doe', amount: 100 };
|
|
102
|
+
* const eventData = { eventName: 'order.created', data: orderData };
|
|
103
|
+
* await streams.publish(eventData);
|
|
104
|
+
*/
|
|
105
|
+
publish(data) {
|
|
106
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
107
|
+
data.eventId = (0, id_1.generateID)('HEX', 'FF'); // Added a unique Id to handle Message deduplication
|
|
108
|
+
const transaction = this.redisPublisher.multi();
|
|
109
|
+
const consumerGroups = yield (0, groups_1.getAllConsumerGroups)(data.eventName, this.redisGroups);
|
|
110
|
+
if (consumerGroups.length > 0) {
|
|
111
|
+
console.log(`Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
|
|
112
|
+
for (const consumerGroup of consumerGroups) {
|
|
113
|
+
// Publish the event to each consumer group's stream
|
|
114
|
+
const streamName = `${data.eventName}:${consumerGroup}`;
|
|
115
|
+
transaction.xadd(streamName, '*', 'data', JSON.stringify(data));
|
|
116
|
+
}
|
|
117
|
+
transaction.publish(data.eventName, '');
|
|
118
|
+
yield transaction.exec().catch((error) => {
|
|
119
|
+
console.error(`Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
|
|
120
|
+
throw new Error('Publisher Error');
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Schedules an event to be published at a specified future time. Thee event gets published if the
|
|
127
|
+
* differnece between the current time and the scheduled time is less than 500ms.
|
|
128
|
+
*
|
|
129
|
+
* @param scheduledTime - The Date object representing the future time when the event should be published.
|
|
130
|
+
* @param eventData - The event data object, containing the event name and its associated data.
|
|
131
|
+
*
|
|
132
|
+
* @throws Error - Throws an error if the scheduled time is in the past.
|
|
133
|
+
*
|
|
134
|
+
* @example
|
|
135
|
+
*
|
|
136
|
+
* const streams = new Streams('app-service');
|
|
137
|
+
*
|
|
138
|
+
* const futureTime = new Date(Date.now() + 10000); // 10 seconds from now
|
|
139
|
+
* const eventData: EventData<string> = {
|
|
140
|
+
* eventName: 'order.created',
|
|
141
|
+
* data: 'Order data'
|
|
142
|
+
* };
|
|
143
|
+
*
|
|
144
|
+
* await streams.scheduledPublish(futureTime, eventData);
|
|
145
|
+
*/
|
|
146
|
+
scheduledPublish(scheduledTime, eventData, uniquePerInstance = false) {
|
|
147
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
148
|
+
const currentTime = new Date();
|
|
149
|
+
if (scheduledTime < currentTime) {
|
|
150
|
+
throw new Error('Cannot schedule an event in the past');
|
|
151
|
+
}
|
|
152
|
+
else if (Math.abs(scheduledTime.getTime() - currentTime.getTime()) <= 500) {
|
|
153
|
+
yield this.publish(eventData);
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
if (uniquePerInstance === true) {
|
|
157
|
+
const existingJob = yield this.redisPublisher.zscore('se', JSON.stringify(eventData));
|
|
158
|
+
if (existingJob) {
|
|
159
|
+
console.log(`Job with data '${eventData}' already exists. Skipping.`);
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
yield this.redisPublisher.zadd('se', scheduledTime.getTime(), JSON.stringify(eventData));
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Listens for events with the given name and returns an Observable that emits an EventData<T> object
|
|
169
|
+
* each time a new event is received.
|
|
170
|
+
*
|
|
171
|
+
* The method uses a BehaviorSubject to emit the events as Observables. The BehaviorSubject ensures
|
|
172
|
+
* that new subscribers receive the last emitted event, even if they subscribe after the event has been emitted.
|
|
173
|
+
*
|
|
174
|
+
* If an error occurs while subscribing, the method logs the error to the console and throws
|
|
175
|
+
* an error. This is done to prevent the service from continuing without a proper event subscription.
|
|
176
|
+
*
|
|
177
|
+
* There is retry logic with exponential backoff to handle error cases. These are also controllable by the
|
|
178
|
+
* calling service
|
|
179
|
+
*
|
|
180
|
+
* @param eventName - The name of the event to listen for.
|
|
181
|
+
*
|
|
182
|
+
* @returns An Observable that emits an EventData<T> object each time a new event is received.
|
|
183
|
+
*
|
|
184
|
+
* @example
|
|
185
|
+
*
|
|
186
|
+
* // Listen for "order.created" events
|
|
187
|
+
* const orderCreated = streams.listen<OrderCreatedEvent>('order.created');
|
|
188
|
+
*
|
|
189
|
+
* // Subscribe to the Observable and log each new event
|
|
190
|
+
* orderCreated.subscribe((event) => {
|
|
191
|
+
* console.log('New order created:', event.data);
|
|
192
|
+
* });
|
|
193
|
+
*/
|
|
194
|
+
listen(eventName, maxRetries = 5, initialDelay = 1000) {
|
|
195
|
+
this.registerConsumerGroup(eventName); // Registers the consumer group for listening to the message
|
|
196
|
+
this.eventsListened.push(eventName);
|
|
197
|
+
return this.listenInternals(eventName).pipe((0, rxjs_1.retry)({
|
|
198
|
+
count: maxRetries,
|
|
199
|
+
delay: (error, retryAttempt) => {
|
|
200
|
+
const delay = initialDelay * Math.pow(2, retryAttempt);
|
|
201
|
+
console.error(`Error in listen: ${error.message}. Retrying in ${delay}ms (attempt ${retryAttempt + 1})`);
|
|
202
|
+
return (0, rxjs_1.timer)(delay);
|
|
203
|
+
},
|
|
204
|
+
}), (0, rxjs_1.catchError)((error) => {
|
|
205
|
+
console.error(`Error in listen after ${maxRetries} retries: ${error.message}`);
|
|
206
|
+
return (0, rxjs_1.throwError)(() => new Error(error.message));
|
|
207
|
+
}));
|
|
208
|
+
}
|
|
209
|
+
listenInternals(eventName) {
|
|
210
|
+
try {
|
|
211
|
+
this.createConsumerGroup(eventName);
|
|
212
|
+
const bs = new rxjs_1.BehaviorSubject(null);
|
|
213
|
+
const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1));
|
|
214
|
+
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
215
|
+
this.redisSubscriber.subscribe(eventName);
|
|
216
|
+
const processMessage = () => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
217
|
+
try {
|
|
218
|
+
const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'COUNT', 1, 'BLOCK', 0, 'STREAMS', streamName, '>');
|
|
219
|
+
if (result) {
|
|
220
|
+
const [, streamMessages] = result[0];
|
|
221
|
+
for (const [id, data] of streamMessages) {
|
|
222
|
+
const eventData = JSON.parse(data[1]);
|
|
223
|
+
const messageId = eventData.eventId;
|
|
224
|
+
const isDuplicate = yield this.isDuplicateMessage(streamName, messageId);
|
|
225
|
+
if (isDuplicate) {
|
|
226
|
+
console.warn(`Duplicate message detected: ${messageId}`);
|
|
227
|
+
yield this.redisGroups.xack(streamName, this.consumerGroupName, id);
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
bs.next(eventData);
|
|
231
|
+
const pmKey = `pm:${this.consumerGroupName}:${streamName}`;
|
|
232
|
+
const currentTime = Date.now();
|
|
233
|
+
const transaction = this.redisGroups.multi();
|
|
234
|
+
transaction.zadd(pmKey, currentTime, messageId);
|
|
235
|
+
transaction.xack(streamName, this.consumerGroupName, id);
|
|
236
|
+
transaction.zadd(`ack:${streamName}`, Date.now(), id);
|
|
237
|
+
yield transaction.exec();
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
catch (e) {
|
|
242
|
+
console.error(JSON.stringify(e));
|
|
243
|
+
}
|
|
244
|
+
});
|
|
245
|
+
this.redisSubscriber.on('message', () => tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
246
|
+
yield processMessage();
|
|
247
|
+
}));
|
|
248
|
+
return observable;
|
|
249
|
+
}
|
|
250
|
+
catch (e) {
|
|
251
|
+
console.error(JSON.stringify(e));
|
|
252
|
+
throw e;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* This method takes all messages allocated to this instance and republishes them so
|
|
257
|
+
* that other instances of this service can receive and process them.
|
|
258
|
+
*
|
|
259
|
+
* This needs to be handled every 1-2 minutes if the queue becomes too long and messages
|
|
260
|
+
* are not being processed.
|
|
261
|
+
*
|
|
262
|
+
* Ideal implementation would be to wrap this inside a setInterval
|
|
263
|
+
* @param streamName
|
|
264
|
+
*/
|
|
265
|
+
republishUnprocessedEvents(eventName) {
|
|
266
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
267
|
+
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
268
|
+
const result = yield this.redisGroups.xreadgroup('GROUP', this.consumerGroupName, this.instanceId, 'STREAMS', streamName, '>');
|
|
269
|
+
if (result) {
|
|
270
|
+
const [, streamMessages] = result[0];
|
|
271
|
+
for (const [id, data] of streamMessages) {
|
|
272
|
+
const eventData = JSON.parse(data[1]);
|
|
273
|
+
console.log(`Unprocessed event: ${id}, data:`, eventData);
|
|
274
|
+
const transaction = this.redisGroups.multi();
|
|
275
|
+
// Republishing the events
|
|
276
|
+
transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
|
|
277
|
+
transaction.publish(eventName, '');
|
|
278
|
+
transaction.xack(streamName, this.consumerGroupName, id);
|
|
279
|
+
yield transaction.exec();
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* This method is used to claim messages in the event of a service crash. This library currently
|
|
286
|
+
* does not detect a service crash. This needs to be built as an extension of Kubernetes and
|
|
287
|
+
* a standalone service that notifies this service to process the events that are marked as
|
|
288
|
+
* pending
|
|
289
|
+
*
|
|
290
|
+
* @param streamName
|
|
291
|
+
* @param idleTimeout
|
|
292
|
+
*
|
|
293
|
+
* * @example
|
|
294
|
+
*
|
|
295
|
+
* // Attempt to recover messages from the "order.created" stream with an idle timeout of 10 seconds
|
|
296
|
+
* await streams.recoverCrashedConsumerMessages('order.created', 10000);
|
|
297
|
+
*/
|
|
298
|
+
recoverCrashedConsumerMessages(eventName, idleTimeout) {
|
|
299
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
300
|
+
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
301
|
+
const pendingMessages = (yield this.redisGroups.xpending(streamName, this.consumerGroupName));
|
|
302
|
+
if (!pendingMessages)
|
|
303
|
+
return;
|
|
304
|
+
const [, minId, maxId, consumers] = pendingMessages;
|
|
305
|
+
for (const [consumer, pendingCount] of consumers) {
|
|
306
|
+
if (parseInt(pendingCount) > 0) {
|
|
307
|
+
const pending = (yield this.redisGroups.xpending(streamName, this.consumerGroupName, minId, maxId, Number(pendingCount), consumer));
|
|
308
|
+
for (const [messageId] of pending) {
|
|
309
|
+
const claimedMessage = (yield this.redisGroups.xclaim(streamName, this.consumerGroupName, this.instanceId, idleTimeout, messageId));
|
|
310
|
+
if (claimedMessage) {
|
|
311
|
+
const [, data] = claimedMessage[0];
|
|
312
|
+
const eventData = JSON.parse(data[1]);
|
|
313
|
+
const transaction = this.redisGroups.multi();
|
|
314
|
+
transaction.xadd(streamName, '*', 'data', JSON.stringify(eventData));
|
|
315
|
+
transaction.publish(eventName, '');
|
|
316
|
+
transaction.xack(streamName, this.consumerGroupName, messageId);
|
|
317
|
+
yield transaction.exec();
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* This method allows the possibility of a graceful shutdown by cleaning up the
|
|
326
|
+
* redis connections.
|
|
327
|
+
*
|
|
328
|
+
* In all services where the library is used, its better to implement this method
|
|
329
|
+
*
|
|
330
|
+
* process.on('SIGTERM', shutdown);
|
|
331
|
+
* process.on('SIGINT', shutdown);
|
|
332
|
+
*
|
|
333
|
+
* async function shutdown(): Promise<void> {
|
|
334
|
+
* console.log('Graceful shutdown initiated.');
|
|
335
|
+
* try {
|
|
336
|
+
* await streams.close();
|
|
337
|
+
* console.log('Resources and connections successfully closed.');
|
|
338
|
+
* } catch (error) {
|
|
339
|
+
* console.error('Error during graceful shutdown:', error);
|
|
340
|
+
* }
|
|
341
|
+
* process.exit(0);
|
|
342
|
+
* }
|
|
343
|
+
*/
|
|
344
|
+
close() {
|
|
345
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
346
|
+
this.clearSubscribedEvents();
|
|
347
|
+
if (this.redisPublisher) {
|
|
348
|
+
yield this.redisPublisher.quit();
|
|
349
|
+
}
|
|
350
|
+
if (this.redisSubscriber) {
|
|
351
|
+
yield this.redisSubscriber.quit();
|
|
352
|
+
}
|
|
353
|
+
if (this.redisGroups) {
|
|
354
|
+
yield this.redisGroups.quit();
|
|
355
|
+
}
|
|
356
|
+
if (this.cleanUpTimer) {
|
|
357
|
+
this.cleanUpTimer.unsubscribe();
|
|
358
|
+
}
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
clearSubscribedEvents() {
|
|
362
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
363
|
+
console.log(`${this.eventsListened.length} events to be cleared`);
|
|
364
|
+
for (const eventName of this.eventsListened) {
|
|
365
|
+
yield this.redisGroups.srem(`consumerGroups:${eventName}`, this.consumerGroupName);
|
|
366
|
+
}
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
registerConsumerGroup(eventName) {
|
|
370
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
371
|
+
yield this.redisGroups.sadd(`consumerGroups:${eventName}`, this.consumerGroupName);
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
cleanupAcknowledgedMessages(eventName, interval = 60 * 60 * 1000) {
|
|
375
|
+
return tslib_1.__awaiter(this, void 0, void 0, function* () {
|
|
376
|
+
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
377
|
+
const cleanupThreshold = Date.now() - interval;
|
|
378
|
+
const acknowledgedMessages = yield this.redisGroups.zrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
|
|
379
|
+
if (acknowledgedMessages && acknowledgedMessages.length > 0) {
|
|
380
|
+
const transaction = this.redisGroups.multi();
|
|
381
|
+
// Remove acknowledged messages from the stream
|
|
382
|
+
for (const messageId of acknowledgedMessages) {
|
|
383
|
+
transaction.xdel(streamName, messageId);
|
|
384
|
+
}
|
|
385
|
+
// Remove acknowledged messages from the Sorted Set
|
|
386
|
+
transaction.zremrangebyscore(`ack:${streamName}`, '-inf', cleanupThreshold);
|
|
387
|
+
yield transaction.exec();
|
|
388
|
+
}
|
|
389
|
+
});
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
exports.Streams = Streams;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export type EventData<T> = {
|
|
2
|
+
data: T;
|
|
3
|
+
eventName: string;
|
|
4
|
+
eventId?: string;
|
|
5
|
+
};
|
|
6
|
+
export type PendingMessages = [never, never, string, string, Array<[string, number]>];
|
|
7
|
+
export type ClaimedMessages = Array<[never, string]>;
|
|
8
|
+
import { ClusterNode, ClusterOptions, RedisOptions } from 'ioredis';
|
|
9
|
+
export interface IOptions {
|
|
10
|
+
redis?: RedisOptions;
|
|
11
|
+
cluster?: {
|
|
12
|
+
options: ClusterOptions;
|
|
13
|
+
nodes: Array<ClusterNode>;
|
|
14
|
+
};
|
|
15
|
+
}
|
package/index.d.ts
DELETED
|
@@ -1,222 +0,0 @@
|
|
|
1
|
-
declare module "src/lib/redis/types" {
|
|
2
|
-
export type EventData<T> = {
|
|
3
|
-
data: T;
|
|
4
|
-
eventName: string;
|
|
5
|
-
eventId?: string;
|
|
6
|
-
};
|
|
7
|
-
export type PendingMessages = [never, never, string, string, Array<[string, number]>];
|
|
8
|
-
export type ClaimedMessages = Array<[never, string]>;
|
|
9
|
-
import { ClusterNode, ClusterOptions, RedisOptions } from 'ioredis';
|
|
10
|
-
export interface IOptions {
|
|
11
|
-
redis?: RedisOptions;
|
|
12
|
-
cluster?: {
|
|
13
|
-
options: ClusterOptions;
|
|
14
|
-
nodes: Array<ClusterNode>;
|
|
15
|
-
};
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
declare module "src/lib/redis/registry" {
|
|
19
|
-
import Redis, { Cluster } from 'ioredis';
|
|
20
|
-
import { IOptions } from "src/lib/redis/types";
|
|
21
|
-
export type RedisType = Redis | Cluster;
|
|
22
|
-
export class RedisRegistry {
|
|
23
|
-
private static registry;
|
|
24
|
-
private static options;
|
|
25
|
-
static attemptConnection(connectionKey: string, storeRef?: number): Redis;
|
|
26
|
-
static getConnection(connectionType?: string, storeRef?: number): RedisType;
|
|
27
|
-
static setOptions(options: IOptions): void;
|
|
28
|
-
static _getOptions(): IOptions;
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* This function is used to set Redis Connection options per instance. If no
|
|
32
|
-
* options are provided, then the service connects as to a single instance
|
|
33
|
-
* with the environment values from REDIS_PORT and REDIS_HOST. if those
|
|
34
|
-
* environment values are not provided, it attempts to connect to localhost:6379
|
|
35
|
-
*
|
|
36
|
-
* @param options
|
|
37
|
-
*/
|
|
38
|
-
export function setRedisConnectionSettings(options: IOptions): void;
|
|
39
|
-
}
|
|
40
|
-
declare module "src/lib/redis/groups" {
|
|
41
|
-
import { RedisType } from "src/lib/redis/registry";
|
|
42
|
-
export function getAllConsumerGroups(eventName: string, redisConnection: RedisType): Promise<string[]>;
|
|
43
|
-
}
|
|
44
|
-
declare module "src/lib/redis/streams" {
|
|
45
|
-
import { RedisType } from "src/lib/redis/registry";
|
|
46
|
-
import { EventData } from "src/lib/redis/types";
|
|
47
|
-
import { Observable } from 'rxjs';
|
|
48
|
-
export class Streams {
|
|
49
|
-
private _redisPublisher?;
|
|
50
|
-
private _redisSubscriber?;
|
|
51
|
-
private _redisGroups?;
|
|
52
|
-
private consumerGroupName;
|
|
53
|
-
private instanceId;
|
|
54
|
-
private cleanUpTimer;
|
|
55
|
-
private eventsListened;
|
|
56
|
-
get redisPublisher(): RedisType;
|
|
57
|
-
get redisSubscriber(): RedisType;
|
|
58
|
-
get redisGroups(): RedisType;
|
|
59
|
-
/**
|
|
60
|
-
* Creates a new Streams instance for a given service.
|
|
61
|
-
*
|
|
62
|
-
* The constructor initializes the Redis connections for publishers, subscribers, and consumer groups.
|
|
63
|
-
* It also sets up an interval timer for clearing expired messages from Redis and another interval timer
|
|
64
|
-
* for processing scheduled events at regular intervals.
|
|
65
|
-
*
|
|
66
|
-
* @param serviceName - A unique name for the service that will be using this Streams instance.
|
|
67
|
-
*
|
|
68
|
-
* @example
|
|
69
|
-
*
|
|
70
|
-
* // Create a new Streams instance for the "POS" service
|
|
71
|
-
* const streams = new Streams('POS');
|
|
72
|
-
*/
|
|
73
|
-
constructor(serviceName: string);
|
|
74
|
-
private createConsumerGroup;
|
|
75
|
-
private isDuplicateMessage;
|
|
76
|
-
private clearDuplicationCheckKeys;
|
|
77
|
-
/**
|
|
78
|
-
* Publishes an event with the given data to the Redis event stream.
|
|
79
|
-
*
|
|
80
|
-
* The method generates a unique event ID for each event, and adds it to the event data to handle message
|
|
81
|
-
* deduplication.
|
|
82
|
-
*
|
|
83
|
-
* @param data - An EventData<T> object containing the data for the event.
|
|
84
|
-
*
|
|
85
|
-
* @returns A Promise that resolves when the event has been published to the Redis stream.
|
|
86
|
-
*
|
|
87
|
-
* @example
|
|
88
|
-
*
|
|
89
|
-
* // Publish an "order.created" event with the given order data
|
|
90
|
-
* const orderData = { id: '123', customerName: 'John Doe', amount: 100 };
|
|
91
|
-
* const eventData = { eventName: 'order.created', data: orderData };
|
|
92
|
-
* await streams.publish(eventData);
|
|
93
|
-
*/
|
|
94
|
-
publish<T>(data: EventData<T>): Promise<void>;
|
|
95
|
-
/**
|
|
96
|
-
* Schedules an event to be published at a specified future time. Thee event gets published if the
|
|
97
|
-
* differnece between the current time and the scheduled time is less than 500ms.
|
|
98
|
-
*
|
|
99
|
-
* @param scheduledTime - The Date object representing the future time when the event should be published.
|
|
100
|
-
* @param eventData - The event data object, containing the event name and its associated data.
|
|
101
|
-
*
|
|
102
|
-
* @throws Error - Throws an error if the scheduled time is in the past.
|
|
103
|
-
*
|
|
104
|
-
* @example
|
|
105
|
-
*
|
|
106
|
-
* const streams = new Streams('app-service');
|
|
107
|
-
*
|
|
108
|
-
* const futureTime = new Date(Date.now() + 10000); // 10 seconds from now
|
|
109
|
-
* const eventData: EventData<string> = {
|
|
110
|
-
* eventName: 'order.created',
|
|
111
|
-
* data: 'Order data'
|
|
112
|
-
* };
|
|
113
|
-
*
|
|
114
|
-
* await streams.scheduledPublish(futureTime, eventData);
|
|
115
|
-
*/
|
|
116
|
-
scheduledPublish<T>(scheduledTime: Date, eventData: EventData<T>, uniquePerInstance?: boolean): Promise<void>;
|
|
117
|
-
/**
|
|
118
|
-
* Listens for events with the given name and returns an Observable that emits an EventData<T> object
|
|
119
|
-
* each time a new event is received.
|
|
120
|
-
*
|
|
121
|
-
* The method uses a BehaviorSubject to emit the events as Observables. The BehaviorSubject ensures
|
|
122
|
-
* that new subscribers receive the last emitted event, even if they subscribe after the event has been emitted.
|
|
123
|
-
*
|
|
124
|
-
* If an error occurs while subscribing, the method logs the error to the console and throws
|
|
125
|
-
* an error. This is done to prevent the service from continuing without a proper event subscription.
|
|
126
|
-
*
|
|
127
|
-
* There is retry logic with exponential backoff to handle error cases. These are also controllable by the
|
|
128
|
-
* calling service
|
|
129
|
-
*
|
|
130
|
-
* @param eventName - The name of the event to listen for.
|
|
131
|
-
*
|
|
132
|
-
* @returns An Observable that emits an EventData<T> object each time a new event is received.
|
|
133
|
-
*
|
|
134
|
-
* @example
|
|
135
|
-
*
|
|
136
|
-
* // Listen for "order.created" events
|
|
137
|
-
* const orderCreated = streams.listen<OrderCreatedEvent>('order.created');
|
|
138
|
-
*
|
|
139
|
-
* // Subscribe to the Observable and log each new event
|
|
140
|
-
* orderCreated.subscribe((event) => {
|
|
141
|
-
* console.log('New order created:', event.data);
|
|
142
|
-
* });
|
|
143
|
-
*/
|
|
144
|
-
listen<T>(eventName: string, maxRetries?: number, initialDelay?: number): Observable<EventData<T>>;
|
|
145
|
-
private listenInternals;
|
|
146
|
-
/**
|
|
147
|
-
* This method takes all messages allocated to this instance and republishes them so
|
|
148
|
-
* that other instances of this service can receive and process them.
|
|
149
|
-
*
|
|
150
|
-
* This needs to be handled every 1-2 minutes if the queue becomes too long and messages
|
|
151
|
-
* are not being processed.
|
|
152
|
-
*
|
|
153
|
-
* Ideal implementation would be to wrap this inside a setInterval
|
|
154
|
-
* @param streamName
|
|
155
|
-
*/
|
|
156
|
-
republishUnprocessedEvents(eventName: string): Promise<void>;
|
|
157
|
-
/**
|
|
158
|
-
* This method is used to claim messages in the event of a service crash. This library currently
|
|
159
|
-
* does not detect a service crash. This needs to be built as an extension of Kubernetes and
|
|
160
|
-
* a standalone service that notifies this service to process the events that are marked as
|
|
161
|
-
* pending
|
|
162
|
-
*
|
|
163
|
-
* @param streamName
|
|
164
|
-
* @param idleTimeout
|
|
165
|
-
*
|
|
166
|
-
* * @example
|
|
167
|
-
*
|
|
168
|
-
* // Attempt to recover messages from the "order.created" stream with an idle timeout of 10 seconds
|
|
169
|
-
* await streams.recoverCrashedConsumerMessages('order.created', 10000);
|
|
170
|
-
*/
|
|
171
|
-
recoverCrashedConsumerMessages(eventName: string, idleTimeout: number): Promise<void>;
|
|
172
|
-
/**
|
|
173
|
-
* This method allows the possibility of a graceful shutdown by cleaning up the
|
|
174
|
-
* redis connections.
|
|
175
|
-
*
|
|
176
|
-
* In all services where the library is used, its better to implement this method
|
|
177
|
-
*
|
|
178
|
-
* process.on('SIGTERM', shutdown);
|
|
179
|
-
* process.on('SIGINT', shutdown);
|
|
180
|
-
*
|
|
181
|
-
* async function shutdown(): Promise<void> {
|
|
182
|
-
* console.log('Graceful shutdown initiated.');
|
|
183
|
-
* try {
|
|
184
|
-
* await streams.close();
|
|
185
|
-
* console.log('Resources and connections successfully closed.');
|
|
186
|
-
* } catch (error) {
|
|
187
|
-
* console.error('Error during graceful shutdown:', error);
|
|
188
|
-
* }
|
|
189
|
-
* process.exit(0);
|
|
190
|
-
* }
|
|
191
|
-
*/
|
|
192
|
-
close(): Promise<void>;
|
|
193
|
-
private clearSubscribedEvents;
|
|
194
|
-
private registerConsumerGroup;
|
|
195
|
-
private cleanupAcknowledgedMessages;
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
declare module "src/lib/redis/scheduler" {
|
|
199
|
-
import { RedisType } from "src/lib/redis/registry";
|
|
200
|
-
/**
|
|
201
|
-
* DO NOT USE THIS CLASS IF YOU DON'T KNOW WHAT YOU ARE DOING. This class is
|
|
202
|
-
* meant to be used internally by the scheduler application
|
|
203
|
-
*/
|
|
204
|
-
export class ScheduledProcessor {
|
|
205
|
-
private scheduledMessagesTimer;
|
|
206
|
-
private _redisPublisher?;
|
|
207
|
-
private previousTaskCompleted;
|
|
208
|
-
get redisPublisher(): RedisType;
|
|
209
|
-
constructor(duration?: number);
|
|
210
|
-
private processScheduledEvents;
|
|
211
|
-
getAllScheduledEvents(): Promise<Array<string>>;
|
|
212
|
-
close(): Promise<void>;
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
declare module "src/lib/publisher" {
|
|
216
|
-
export { Streams as Publisher } from "src/lib/redis/streams";
|
|
217
|
-
export { setRedisConnectionSettings as setRedisConfig } from "src/lib/redis/registry";
|
|
218
|
-
export { ScheduledProcessor as __SCHEDULER_INTERNALS__ } from "src/lib/redis/scheduler";
|
|
219
|
-
}
|
|
220
|
-
declare module "src/index" {
|
|
221
|
-
export * from "src/lib/publisher";
|
|
222
|
-
}
|