@jetit/publisher 3.3.2 → 3.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/package.json +4 -6
- package/src/lib/redis/logger.d.ts +8 -0
- package/src/lib/redis/logger.js +23 -0
- package/src/lib/redis/registry.js +4 -2
- package/src/lib/redis/scheduler.js +7 -6
- package/src/lib/redis/streams.d.ts +7 -7
- package/src/lib/redis/streams.js +49 -39
- package/src/lib/redis/utils.d.ts +1 -1
- package/src/lib/redis/utils.js +8 -7
package/README.md
CHANGED
|
@@ -198,4 +198,6 @@ streams.listen('my-event').subscribe(event => {
|
|
|
198
198
|
|
|
199
199
|
7. Event-driven workflows: You can use the publisher to create event-driven workflows, where each step in the workflow is triggered by the completion of a previous step. This can be useful for orchestrating complex, multi-step processes.
|
|
200
200
|
|
|
201
|
-
8. Message broadcasting: The publisher can be used to broadcast messages to multiple consumers or subscribers, allowing for efficient and scalable communication in applications with many components or services.
|
|
201
|
+
8. Message broadcasting: The publisher can be used to broadcast messages to multiple consumers or subscribers, allowing for efficient and scalable communication in applications with many components or services.
|
|
202
|
+
|
|
203
|
+
9. Multicast Publishing: This is the existing PUB/SUB implementation but with the event data being stored into streams for additional processing
|
package/package.json
CHANGED
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jetit/publisher",
|
|
3
|
-
"version": "3.3.
|
|
3
|
+
"version": "3.3.5",
|
|
4
4
|
"type": "commonjs",
|
|
5
5
|
"dependencies": {
|
|
6
|
-
"@jetit/id": "0.0.
|
|
6
|
+
"@jetit/id": "^0.0.12",
|
|
7
7
|
"ioredis": "^5.3.0",
|
|
8
|
-
"rxjs": "^7.8.0"
|
|
9
|
-
|
|
10
|
-
"peerDependencies": {
|
|
11
|
-
"tslib": "2.5.3"
|
|
8
|
+
"rxjs": "^7.8.0",
|
|
9
|
+
"tslib": "1.14.1"
|
|
12
10
|
},
|
|
13
11
|
"main": "./src/index.js"
|
|
14
12
|
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PERFORMANCE_LOGGER = exports.PUBLISHER_LOGGER = void 0;
|
|
4
|
+
exports.PUBLISHER_LOGGER = {
|
|
5
|
+
log: (...args) => {
|
|
6
|
+
if (process.env['DEBUG_LOGGING_ENABLED'] === 'TRUE') {
|
|
7
|
+
console.log(...args);
|
|
8
|
+
}
|
|
9
|
+
},
|
|
10
|
+
warn: (...args) => {
|
|
11
|
+
console.warn(...args);
|
|
12
|
+
},
|
|
13
|
+
error: (...args) => {
|
|
14
|
+
console.warn(...args);
|
|
15
|
+
},
|
|
16
|
+
};
|
|
17
|
+
exports.PERFORMANCE_LOGGER = {
|
|
18
|
+
log: (...args) => {
|
|
19
|
+
if (process.env['PERFORMANCE_LOGGING_ENABLED'] !== 'FALSE') {
|
|
20
|
+
console.log(...args);
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
};
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.setRedisConnectionSettings = exports.RedisRegistry = void 0;
|
|
4
4
|
const ioredis_1 = require("ioredis");
|
|
5
|
+
const logger_1 = require("./logger");
|
|
5
6
|
class RedisRegistry {
|
|
6
7
|
static attemptConnection(connectionKey, storeRef = 0) {
|
|
7
8
|
let ref;
|
|
@@ -25,8 +26,9 @@ class RedisRegistry {
|
|
|
25
26
|
}
|
|
26
27
|
static handleDisconnects(connection, connectionKey, storeRef) {
|
|
27
28
|
connection.on('error', (error) => {
|
|
28
|
-
|
|
29
|
+
logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Redis connection error : ${error.message}`);
|
|
29
30
|
connection.removeAllListeners();
|
|
31
|
+
connection.disconnect();
|
|
30
32
|
RedisRegistry.attemptConnection(connectionKey, storeRef);
|
|
31
33
|
});
|
|
32
34
|
}
|
|
@@ -42,7 +44,7 @@ class RedisRegistry {
|
|
|
42
44
|
]);
|
|
43
45
|
if (res === 'NO_PING') {
|
|
44
46
|
connection.disconnect(true);
|
|
45
|
-
|
|
47
|
+
logger_1.PUBLISHER_LOGGER.error('PUBLISHER: failed to ping redis, disconnecting and restarting service.');
|
|
46
48
|
process.exit(0);
|
|
47
49
|
}
|
|
48
50
|
}, 2000);
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.ScheduledProcessor = void 0;
|
|
4
4
|
const id_1 = require("@jetit/id");
|
|
5
5
|
const rxjs_1 = require("rxjs");
|
|
6
|
+
const logger_1 = require("./logger");
|
|
6
7
|
const registry_1 = require("./registry");
|
|
7
8
|
const utils_1 = require("./utils");
|
|
8
9
|
/**
|
|
@@ -18,27 +19,27 @@ class ScheduledProcessor {
|
|
|
18
19
|
constructor(duration = 1000) {
|
|
19
20
|
this.previousTaskCompleted = true;
|
|
20
21
|
this.scheduledMessagesTimer = (0, rxjs_1.interval)(duration).subscribe(() => {
|
|
21
|
-
|
|
22
|
+
logger_1.PUBLISHER_LOGGER.log('Checking Streams messages at ', new Date().toISOString(), '...');
|
|
22
23
|
/** Do not run scheduler if the previous run is not completed */
|
|
23
24
|
if (this.previousTaskCompleted) {
|
|
24
25
|
this.previousTaskCompleted = false;
|
|
25
26
|
this.processScheduledEvents()
|
|
26
27
|
.catch((error) => {
|
|
27
|
-
|
|
28
|
+
logger_1.PUBLISHER_LOGGER.error('Error while processing scheduled events:', error);
|
|
28
29
|
})
|
|
29
30
|
.then(() => {
|
|
30
31
|
this.previousTaskCompleted = true;
|
|
31
32
|
});
|
|
32
33
|
}
|
|
33
34
|
else {
|
|
34
|
-
|
|
35
|
+
logger_1.PUBLISHER_LOGGER.log('Skipping current scheduler run because previous run is in progress');
|
|
35
36
|
}
|
|
36
37
|
});
|
|
37
38
|
}
|
|
38
39
|
async processScheduledEvents() {
|
|
39
40
|
const currentTime = new Date().getTime();
|
|
40
41
|
const events = await this.redisPublisher.zrangebyscore('se', 0, currentTime);
|
|
41
|
-
|
|
42
|
+
logger_1.PUBLISHER_LOGGER.log('Events to process:', events.length);
|
|
42
43
|
for (const eventString of events) {
|
|
43
44
|
const eventData = (0, utils_1.decodeScheduledMessage)(eventString);
|
|
44
45
|
/**
|
|
@@ -52,14 +53,14 @@ class ScheduledProcessor {
|
|
|
52
53
|
eventData.eventId = (0, id_1.generateID)('HEX', 'FF');
|
|
53
54
|
await this.redisPublisher.zrem('se', eventString);
|
|
54
55
|
const consumerGroups = await (0, utils_1.getAllConsumerGroups)(eventData.eventName, this.redisPublisher);
|
|
55
|
-
|
|
56
|
+
logger_1.PUBLISHER_LOGGER.log('Scheduled Publishing to consumer groups: ', consumerGroups, 'with id ', eventData.eventId, '...');
|
|
56
57
|
let key = '*';
|
|
57
58
|
for (const consumerGroup of consumerGroups) {
|
|
58
59
|
// Publish the event to each consumer group's stream
|
|
59
60
|
const streamName = `${eventData.eventName}:${consumerGroup}`;
|
|
60
61
|
const generatedKey = await this.redisPublisher
|
|
61
62
|
.xadd(streamName, '*', 'data', JSON.stringify(eventData))
|
|
62
|
-
.catch((e) =>
|
|
63
|
+
.catch((e) => logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Publishing event ${eventData.eventName} to consumer groups: ${consumerGroups.join(', ')} failed with data ${JSON.stringify(eventData)}, ${e} `));
|
|
63
64
|
if (key === '*')
|
|
64
65
|
key = generatedKey ?? key;
|
|
65
66
|
}
|
|
@@ -43,7 +43,7 @@ export declare class Streams {
|
|
|
43
43
|
* const eventData = { eventName: 'order.created', data: orderData };
|
|
44
44
|
* await streams.publish(eventData);
|
|
45
45
|
*/
|
|
46
|
-
publish<TData = unknown, TName extends string = string>(data: PublishData<TData, TName
|
|
46
|
+
publish<TData = unknown, TName extends string = string>(data: PublishData<TData, TName>, multicast?: boolean): Promise<void>;
|
|
47
47
|
/**
|
|
48
48
|
* Schedules an event to be published at a specified future time. Thee event gets published if the
|
|
49
49
|
* differnece between the current time and the scheduled time is less than 500ms.
|
|
@@ -67,7 +67,7 @@ export declare class Streams {
|
|
|
67
67
|
*
|
|
68
68
|
* await streams.scheduledPublish(futureTime, eventData);
|
|
69
69
|
*/
|
|
70
|
-
scheduledPublish<TData = unknown, TName extends string = string>(scheduledTime: Date, eventData: PublishData<TData, TName>, uniquePerInstance?: boolean, repeatInterval?: number): Promise<void>;
|
|
70
|
+
scheduledPublish<TData = unknown, TName extends string = string>(scheduledTime: Date, eventData: PublishData<TData, TName>, uniquePerInstance?: boolean, repeatInterval?: number, multicast?: boolean): Promise<void>;
|
|
71
71
|
/**
|
|
72
72
|
* Listens for events with the given name and returns an Observable that emits an EventData<T> object
|
|
73
73
|
* each time a new event is received.
|
|
@@ -75,7 +75,7 @@ export declare class Streams {
|
|
|
75
75
|
* The method uses a BehaviorSubject to emit the events as Observables. The BehaviorSubject ensures
|
|
76
76
|
* that new subscribers receive the last emitted event, even if they subscribe after the event has been emitted.
|
|
77
77
|
*
|
|
78
|
-
* If an error occurs while subscribing, the method logs the error to the
|
|
78
|
+
* If an error occurs while subscribing, the method logs the error to the PUBLISHER_LOGGER and throws
|
|
79
79
|
* an error. This is done to prevent the service from continuing without a proper event subscription.
|
|
80
80
|
*
|
|
81
81
|
* There is retry logic with exponential backoff to handle error cases. These are also controllable by the
|
|
@@ -92,7 +92,7 @@ export declare class Streams {
|
|
|
92
92
|
*
|
|
93
93
|
* // Subscribe to the Observable and log each new event
|
|
94
94
|
* orderCreated.subscribe((event) => {
|
|
95
|
-
*
|
|
95
|
+
* PUBLISHER_LOGGER.log('New order created:', event.data);
|
|
96
96
|
* });
|
|
97
97
|
*/
|
|
98
98
|
listen<T = unknown, const TName extends string = string>(eventName: TName, maxRetries?: number, initialDelay?: number): Observable<EventData<T, TName>>;
|
|
@@ -108,12 +108,12 @@ export declare class Streams {
|
|
|
108
108
|
* process.on('SIGINT', shutdown);
|
|
109
109
|
*
|
|
110
110
|
* async function shutdown(): Promise<void> {
|
|
111
|
-
*
|
|
111
|
+
* PUBLISHER_LOGGER.log('Graceful shutdown initiated.');
|
|
112
112
|
* try {
|
|
113
113
|
* await streams.close();
|
|
114
|
-
*
|
|
114
|
+
* PUBLISHER_LOGGER.log('Resources and connections successfully closed.');
|
|
115
115
|
* } catch (error) {
|
|
116
|
-
*
|
|
116
|
+
* PUBLISHER_LOGGER.error('Error during graceful shutdown:', error);
|
|
117
117
|
* }
|
|
118
118
|
* process.exit(0);
|
|
119
119
|
* }
|
package/src/lib/redis/streams.js
CHANGED
|
@@ -3,10 +3,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.Streams = void 0;
|
|
4
4
|
const id_1 = require("@jetit/id");
|
|
5
5
|
const rxjs_1 = require("rxjs");
|
|
6
|
+
const logger_1 = require("./logger");
|
|
6
7
|
const registry_1 = require("./registry");
|
|
7
8
|
const utils_1 = require("./utils");
|
|
8
9
|
function publisherErrorHandler(error) {
|
|
9
|
-
|
|
10
|
+
logger_1.PUBLISHER_LOGGER.error('PUBLISHER UNHANDLED ERROR: ', JSON.stringify(error));
|
|
10
11
|
}
|
|
11
12
|
class Streams {
|
|
12
13
|
get redisPublisher() {
|
|
@@ -38,23 +39,24 @@ class Streams {
|
|
|
38
39
|
this.instanceUniqueId = process.env['INSTANCE_ID'] ?? (0, id_1.generateID)('HEX', 'FE');
|
|
39
40
|
this.instanceId = `${serviceName}:${this.instanceUniqueId}`;
|
|
40
41
|
this.consumerGroupName = `cg-${serviceName}`;
|
|
41
|
-
|
|
42
|
+
logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Instance ID: ${this.instanceId}`);
|
|
42
43
|
const cleanUpInterval = parseInt(process.env['CLEANUP_INTERVAL'] || `${1000 * 60 * 60}`, 10) ?? 1000 * 60 * 60;
|
|
43
44
|
this.cleanUpTimer = setInterval(() => {
|
|
44
45
|
this.runClear(cleanUpInterval);
|
|
45
46
|
}, cleanUpInterval);
|
|
46
47
|
}
|
|
47
48
|
async runClear(cleanUpInterval) {
|
|
48
|
-
|
|
49
|
+
logger_1.PUBLISHER_LOGGER.log('PUBLISHER: Running Clearance', this.eventsListened);
|
|
49
50
|
for (const eventName of this.eventsListened) {
|
|
50
51
|
process.nextTick(async () => {
|
|
51
52
|
/** This removes the messages from the stream after they have been processed according to cleanup interval */
|
|
52
53
|
await this.cleanupAcknowledgedMessages(eventName, cleanUpInterval).catch(publisherErrorHandler);
|
|
53
|
-
|
|
54
|
+
logger_1.PUBLISHER_LOGGER.log(`Cleanup process for Acknowledged messages completed for ${eventName}`);
|
|
54
55
|
});
|
|
55
56
|
}
|
|
56
57
|
}
|
|
57
|
-
async publish(data) {
|
|
58
|
+
async publish(data, multicast = false) {
|
|
59
|
+
const publishStartTime = process.hrtime();
|
|
58
60
|
if (data.eventId)
|
|
59
61
|
data.republishEvent = data.eventId;
|
|
60
62
|
data.eventId = (0, id_1.generateID)('HEX', 'FF');
|
|
@@ -63,28 +65,31 @@ class Streams {
|
|
|
63
65
|
const consumerGroups = await (0, utils_1.getAllConsumerGroups)(data.eventName, this.redisPublisher);
|
|
64
66
|
let key = '*';
|
|
65
67
|
if (consumerGroups.length > 0) {
|
|
66
|
-
|
|
68
|
+
logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')}`);
|
|
67
69
|
try {
|
|
68
70
|
for (const consumerGroup of consumerGroups) {
|
|
69
71
|
// Publish the event to each consumer group's stream
|
|
70
72
|
const streamName = `${data.eventName}:${consumerGroup}`;
|
|
71
73
|
const generatedKey = await this.redisPublisher
|
|
72
74
|
.xadd(streamName, key, 'data', JSON.stringify(data))
|
|
73
|
-
.catch((e) =>
|
|
75
|
+
.catch((e) => logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Publishing event ${data.eventName} to consumer groups: ${consumerGroups.join(', ')} failed with data ${JSON.stringify(data)}, ${e} `));
|
|
74
76
|
if (key === '*')
|
|
75
77
|
key = generatedKey ?? key;
|
|
76
78
|
}
|
|
77
|
-
await (0, utils_1.notifySubscribers)(this.redisPublisher, data.eventName, key);
|
|
79
|
+
await (0, utils_1.notifySubscribers)(this.redisPublisher, data.eventName, key, multicast);
|
|
78
80
|
}
|
|
79
81
|
catch (error) {
|
|
80
|
-
|
|
82
|
+
logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error while publishing event for service ${this.consumerGroupName} with instance ${this.instanceId}: `, error);
|
|
81
83
|
throw new Error('Publisher Error');
|
|
82
84
|
}
|
|
83
85
|
}
|
|
84
86
|
else
|
|
85
|
-
|
|
87
|
+
logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Event publish failed for event ${data.eventName}, reason: no consumers ${consumerGroups}`);
|
|
88
|
+
const publishEndTime = process.hrtime(publishStartTime);
|
|
89
|
+
const elapsedTime = publishEndTime[0] * 1000 + publishEndTime[1] / 1000000;
|
|
90
|
+
logger_1.PERFORMANCE_LOGGER.log(`PTIME;${key};${data.eventName};${Date.now()};${elapsedTime}`);
|
|
86
91
|
}
|
|
87
|
-
async scheduledPublish(scheduledTime, eventData, uniquePerInstance = false, repeatInterval = 0) {
|
|
92
|
+
async scheduledPublish(scheduledTime, eventData, uniquePerInstance = false, repeatInterval = 0, multicast = false) {
|
|
88
93
|
const currentTime = new Date();
|
|
89
94
|
delete eventData.repeatInterval;
|
|
90
95
|
if (repeatInterval > 0) {
|
|
@@ -94,14 +99,14 @@ class Streams {
|
|
|
94
99
|
throw new Error('PUBLISHER: Cannot schedule an event in the past');
|
|
95
100
|
}
|
|
96
101
|
else if (Math.abs(scheduledTime.getTime() - currentTime.getTime()) <= 500) {
|
|
97
|
-
await this.publish(eventData);
|
|
102
|
+
await this.publish(eventData, multicast);
|
|
98
103
|
}
|
|
99
104
|
else {
|
|
100
105
|
const key = (0, utils_1.encodeScheduledMessage)(eventData);
|
|
101
106
|
if (uniquePerInstance === true) {
|
|
102
107
|
const existingJob = await this.redisPublisher.zscore('se', key);
|
|
103
108
|
if (existingJob) {
|
|
104
|
-
|
|
109
|
+
logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Job with data '${eventData}' already exists. Skipping.`);
|
|
105
110
|
return;
|
|
106
111
|
}
|
|
107
112
|
}
|
|
@@ -115,7 +120,7 @@ class Streams {
|
|
|
115
120
|
* The method uses a BehaviorSubject to emit the events as Observables. The BehaviorSubject ensures
|
|
116
121
|
* that new subscribers receive the last emitted event, even if they subscribe after the event has been emitted.
|
|
117
122
|
*
|
|
118
|
-
* If an error occurs while subscribing, the method logs the error to the
|
|
123
|
+
* If an error occurs while subscribing, the method logs the error to the PUBLISHER_LOGGER and throws
|
|
119
124
|
* an error. This is done to prevent the service from continuing without a proper event subscription.
|
|
120
125
|
*
|
|
121
126
|
* There is retry logic with exponential backoff to handle error cases. These are also controllable by the
|
|
@@ -132,7 +137,7 @@ class Streams {
|
|
|
132
137
|
*
|
|
133
138
|
* // Subscribe to the Observable and log each new event
|
|
134
139
|
* orderCreated.subscribe((event) => {
|
|
135
|
-
*
|
|
140
|
+
* PUBLISHER_LOGGER.log('New order created:', event.data);
|
|
136
141
|
* });
|
|
137
142
|
*/
|
|
138
143
|
listen(eventName, maxRetries = 5, initialDelay = 1000) {
|
|
@@ -140,11 +145,11 @@ class Streams {
|
|
|
140
145
|
count: maxRetries,
|
|
141
146
|
delay: (error, retryAttempt) => {
|
|
142
147
|
const delay = initialDelay * Math.pow(2, retryAttempt);
|
|
143
|
-
|
|
148
|
+
logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in listen: ${error.message}. Retrying in ${delay}ms (attempt ${retryAttempt + 1})`);
|
|
144
149
|
return (0, rxjs_1.timer)(delay);
|
|
145
150
|
},
|
|
146
151
|
}), (0, rxjs_1.catchError)((error) => {
|
|
147
|
-
|
|
152
|
+
logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error in listen after ${maxRetries} retries: ${error.message}`);
|
|
148
153
|
return (0, rxjs_1.throwError)(() => new Error(error.message));
|
|
149
154
|
}));
|
|
150
155
|
}
|
|
@@ -156,36 +161,36 @@ class Streams {
|
|
|
156
161
|
await this.redisGroups
|
|
157
162
|
.xgroup('CREATE', streamName, this.consumerGroupName, '0', 'MKSTREAM')
|
|
158
163
|
.then(() => {
|
|
159
|
-
|
|
164
|
+
logger_1.PUBLISHER_LOGGER.log(`Group created created for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
|
|
160
165
|
})
|
|
161
166
|
.catch((e) => {
|
|
162
|
-
|
|
167
|
+
logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Group creation failed with error ${e.message} for ${JSON.stringify({ streamName, cgn: this.consumerGroupName })}`);
|
|
163
168
|
});
|
|
164
169
|
const createConsumerStatus = (await this.redisGroups.xgroup('CREATECONSUMER', streamName, this.consumerGroupName, this.instanceId));
|
|
165
170
|
await this.redisGroups.sadd(key, eventName);
|
|
166
171
|
const addToCGSet = await this.redisGroups.sadd(`${eventName}`, this.consumerGroupName);
|
|
167
172
|
const addToFlushSet = await this.redisGroups.set(setKeyForK8sHandling, this.consumerGroupName);
|
|
168
|
-
|
|
173
|
+
logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Consumer Registered and created with ${this.instanceId} under ${this.consumerGroupName} with ${createConsumerStatus} consumers and with the following status ${JSON.stringify({ addToCGSet, addToFlushSet })}`);
|
|
169
174
|
return createConsumerStatus === 0 || createConsumerStatus === 1;
|
|
170
175
|
}
|
|
171
176
|
listenInternals(eventName) {
|
|
172
177
|
const bs = new rxjs_1.BehaviorSubject(null);
|
|
173
178
|
const observable = bs.asObservable().pipe((0, rxjs_1.skip)(1));
|
|
174
179
|
const streamName = `${eventName}:${this.consumerGroupName}`;
|
|
175
|
-
const processMessage = async (redisClient, messageId, processPending = false) => {
|
|
176
|
-
|
|
180
|
+
const processMessage = async (redisClient, messageId, multicast = false, processPending = false) => {
|
|
181
|
+
logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Processing message ${messageId} for ${streamName}`);
|
|
177
182
|
try {
|
|
178
183
|
try {
|
|
179
184
|
const pendingDetails = await redisClient.xpending(streamName, this.consumerGroupName, messageId, messageId, 1, this.instanceId);
|
|
180
|
-
if (pendingDetails[2] === 0) {
|
|
181
|
-
|
|
185
|
+
if (pendingDetails[2] === 0 && multicast === false) {
|
|
186
|
+
logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: MACK ${messageId} for ${streamName}`);
|
|
182
187
|
return;
|
|
183
188
|
}
|
|
184
189
|
}
|
|
185
190
|
catch (e) {
|
|
186
191
|
// Ignore the xpending error and continue
|
|
187
|
-
|
|
188
|
-
|
|
192
|
+
logger_1.PUBLISHER_LOGGER.error('XPENDING ERROR: To be handled');
|
|
193
|
+
logger_1.PUBLISHER_LOGGER.warn(JSON.stringify(e));
|
|
189
194
|
}
|
|
190
195
|
const messages = await redisClient.xrange(streamName, messageId, messageId);
|
|
191
196
|
if (messages && messages.length) {
|
|
@@ -195,22 +200,22 @@ class Streams {
|
|
|
195
200
|
await redisClient.zadd(`ack:${streamName}`, Date.now().toString(), messageId);
|
|
196
201
|
}
|
|
197
202
|
else {
|
|
198
|
-
|
|
203
|
+
logger_1.PUBLISHER_LOGGER.warn(`PUBLISHER: Message ${messageId} not found for ${streamName}`);
|
|
199
204
|
}
|
|
200
205
|
/** Process Unprocessed Message if this is a main tree, otherwise limit to processing 100 messages that are unacknowledged */
|
|
201
206
|
if (!processPending) {
|
|
202
207
|
const unprocessedMessageIds = await (0, utils_1.getUnacknowledgedMessages)(redisClient, this.consumerGroupName, streamName, 25);
|
|
203
208
|
if (unprocessedMessageIds.count > 25) {
|
|
204
|
-
|
|
209
|
+
logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Too many unprocessed events for ${streamName}: count: ${unprocessedMessageIds.count}`);
|
|
205
210
|
}
|
|
206
211
|
for (const id of unprocessedMessageIds.messageIds) {
|
|
207
|
-
|
|
208
|
-
await processMessage(redisClient, id, true);
|
|
212
|
+
logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Reporcessing unprocessed message with id: ${id}`);
|
|
213
|
+
await processMessage(redisClient, id, multicast, true);
|
|
209
214
|
}
|
|
210
215
|
}
|
|
211
216
|
}
|
|
212
217
|
catch (e) {
|
|
213
|
-
|
|
218
|
+
logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error processing message ${messageId} for ${streamName}`, e);
|
|
214
219
|
}
|
|
215
220
|
};
|
|
216
221
|
/** Register the consumer and setup the Observable */
|
|
@@ -220,15 +225,20 @@ class Streams {
|
|
|
220
225
|
throw new Error('PUBLISHER: Cannot setup consumer');
|
|
221
226
|
const eventStreamClient = registry_1.RedisRegistry.getConnection(`sub-${eventName}`);
|
|
222
227
|
eventStreamClient.subscribe(eventName).then(() => {
|
|
223
|
-
|
|
228
|
+
logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Redis Subscription connection initiated for ${eventName}`);
|
|
224
229
|
});
|
|
225
|
-
eventStreamClient.on('message', async (channel,
|
|
226
|
-
|
|
227
|
-
|
|
230
|
+
eventStreamClient.on('message', async (channel, data) => {
|
|
231
|
+
const subscribeStartTime = process.hrtime();
|
|
232
|
+
const { messageId, multicast } = JSON.parse(data);
|
|
233
|
+
logger_1.PUBLISHER_LOGGER.log(`PUBLISHER: Stream Notification Received for event ${eventName} with message ID ${messageId}`);
|
|
234
|
+
await processMessage(this.redisGroups, messageId, multicast);
|
|
235
|
+
const subscribendTime = process.hrtime(subscribeStartTime);
|
|
236
|
+
const elapsedTime = subscribendTime[0] * 1000 + subscribendTime[1] / 1000000;
|
|
237
|
+
logger_1.PERFORMANCE_LOGGER.log(`STIME;${messageId};${data.eventName};${Date.now()};${elapsedTime}`);
|
|
228
238
|
});
|
|
229
239
|
})
|
|
230
240
|
.catch((e) => {
|
|
231
|
-
|
|
241
|
+
logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error during consumer registration for ${eventName}`, e);
|
|
232
242
|
});
|
|
233
243
|
return observable;
|
|
234
244
|
}
|
|
@@ -242,12 +252,12 @@ class Streams {
|
|
|
242
252
|
* process.on('SIGINT', shutdown);
|
|
243
253
|
*
|
|
244
254
|
* async function shutdown(): Promise<void> {
|
|
245
|
-
*
|
|
255
|
+
* PUBLISHER_LOGGER.log('Graceful shutdown initiated.');
|
|
246
256
|
* try {
|
|
247
257
|
* await streams.close();
|
|
248
|
-
*
|
|
258
|
+
* PUBLISHER_LOGGER.log('Resources and connections successfully closed.');
|
|
249
259
|
* } catch (error) {
|
|
250
|
-
*
|
|
260
|
+
* PUBLISHER_LOGGER.error('Error during graceful shutdown:', error);
|
|
251
261
|
* }
|
|
252
262
|
* process.exit(0);
|
|
253
263
|
* }
|
package/src/lib/redis/utils.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ export declare function getMessageStatesCount(redisClient: RedisType, streamName
|
|
|
10
10
|
acknowledged: number;
|
|
11
11
|
unacknowledged: number;
|
|
12
12
|
}>;
|
|
13
|
-
export declare function notifySubscribers(redisClient: RedisType, eventName: string, messageId: string): Promise<void>;
|
|
13
|
+
export declare function notifySubscribers(redisClient: RedisType, eventName: string, messageId: string, multicast?: boolean): Promise<void>;
|
|
14
14
|
export declare function removedScheduledJob(redisClient: RedisType, eventString: string): Promise<void>;
|
|
15
15
|
export declare function encodeScheduledMessage<TData, TName extends string>(data: EventData<TData, TName>): string;
|
|
16
16
|
export declare function decodeScheduledMessage(data: string): EventData<never, never>;
|
package/src/lib/redis/utils.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.UTILS = exports.decodeScheduledMessage = exports.encodeScheduledMessage = exports.removedScheduledJob = exports.notifySubscribers = exports.getMessageStatesCount = exports.getUnacknowledgedMessages = exports.getAllConsumerGroups = void 0;
|
|
4
|
+
const logger_1 = require("./logger");
|
|
4
5
|
async function getAllConsumerGroups(eventName, redisConnection) {
|
|
5
6
|
const consumerGroups = await redisConnection.smembers(`${eventName}`);
|
|
6
7
|
return consumerGroups;
|
|
@@ -25,7 +26,7 @@ async function getUnacknowledgedMessages(redisClient, consumerGroupName, streamN
|
|
|
25
26
|
};
|
|
26
27
|
}
|
|
27
28
|
catch (error) {
|
|
28
|
-
|
|
29
|
+
logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error fetching unacknowledged messages for ${streamName}`, error);
|
|
29
30
|
return { count: 0, messageIds: [] };
|
|
30
31
|
}
|
|
31
32
|
}
|
|
@@ -40,22 +41,22 @@ async function getMessageStatesCount(redisClient, streamName, consumerGroup) {
|
|
|
40
41
|
};
|
|
41
42
|
}
|
|
42
43
|
catch (error) {
|
|
43
|
-
|
|
44
|
+
logger_1.PUBLISHER_LOGGER.error(`PUBLISHER: Error fetching message states count for ${streamName}`, error);
|
|
44
45
|
return { acknowledged: 0, unacknowledged: 0 };
|
|
45
46
|
}
|
|
46
47
|
}
|
|
47
48
|
exports.getMessageStatesCount = getMessageStatesCount;
|
|
48
|
-
async function notifySubscribers(redisClient, eventName, messageId) {
|
|
49
|
-
await redisClient.publish(eventName, messageId);
|
|
49
|
+
async function notifySubscribers(redisClient, eventName, messageId, multicast = false) {
|
|
50
|
+
await redisClient.publish(eventName, JSON.stringify({ messageId, multicast }));
|
|
50
51
|
}
|
|
51
52
|
exports.notifySubscribers = notifySubscribers;
|
|
52
53
|
async function removedScheduledJob(redisClient, eventString) {
|
|
53
54
|
const currentTime = new Date().getTime();
|
|
54
55
|
const events = await redisClient.zrangebyscore('se', 0, currentTime);
|
|
55
|
-
|
|
56
|
+
logger_1.PUBLISHER_LOGGER.log(`Total Events in scheduled queue: ${events.length}`);
|
|
56
57
|
await redisClient.zrem('se', eventString);
|
|
57
58
|
const eventsLater = await redisClient.zrangebyscore('se', 0, currentTime);
|
|
58
|
-
|
|
59
|
+
logger_1.PUBLISHER_LOGGER.log(`Total Events in scheduled queue: ${eventsLater.length}`);
|
|
59
60
|
}
|
|
60
61
|
exports.removedScheduledJob = removedScheduledJob;
|
|
61
62
|
function encodeScheduledMessage(data) {
|
|
@@ -72,7 +73,7 @@ function decodeScheduledMessage(data) {
|
|
|
72
73
|
return {
|
|
73
74
|
data: JSON.parse(Buffer.from(parts[1], 'base64').toString('utf-8')),
|
|
74
75
|
eventName,
|
|
75
|
-
repeatInterval: parts[2],
|
|
76
|
+
repeatInterval: parseInt(parts[2]),
|
|
76
77
|
};
|
|
77
78
|
}
|
|
78
79
|
exports.decodeScheduledMessage = decodeScheduledMessage;
|