@cap-js-community/event-queue 0.1.58 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/db/Event.cds +1 -0
- package/package.json +5 -4
- package/src/EventQueueError.js +15 -0
- package/src/EventQueueProcessorBase.js +136 -105
- package/src/config.js +77 -51
- package/src/dbHandler.js +11 -2
- package/src/initialize.js +27 -26
- package/src/processEventQueue.js +4 -2
- package/src/publishEvent.js +27 -1
- package/src/redisPubSub.js +15 -27
- package/src/runner.js +23 -1
- package/src/shared/EventScheduler.js +53 -0
- package/src/shared/common.js +12 -1
- package/src/shared/redis.js +23 -2
package/README.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
[](https://www.npmjs.com/package/@cap-js-community/event-queue)
|
|
4
4
|
[](https://www.npmjs.com/package/@cap-js-community/event-queue)
|
|
5
5
|
[](https://api.reuse.software/info/github.com/cap-js-community/event-queue)
|
|
6
|
-
[](https://github.com/cap-js-community/event-queue/commits/main)
|
|
7
7
|
|
|
8
8
|
The Event-Queue is a framework built on top of CAP Node.js, specifically designed to enable efficient and streamlined
|
|
9
9
|
asynchronous event processing in a multi-tenancy environment. With a strong emphasis on load balancing, this package
|
package/db/Event.cds
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cap-js-community/event-queue",
|
|
3
|
-
"version": "0.1
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "An event queue that enables secure transactional processing of asynchronous events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"files": [
|
|
@@ -23,9 +23,10 @@
|
|
|
23
23
|
"test:unit": "jest --testPathIgnorePatterns=\"/test-integration/\"",
|
|
24
24
|
"test:integration": "jest --testPathIgnorePatterns=\"/test/\" --runInBand --forceExit",
|
|
25
25
|
"test:all": "npm run test:unit && npm run test:integration",
|
|
26
|
-
"test:ci": "npm run test:prepare && npm run test:all",
|
|
27
26
|
"test:all:coverage": "jest --runInBand --forceExit --collect-coverage",
|
|
28
27
|
"test:prepare": "npm run build:ci --prefix=./test-integration/_env",
|
|
28
|
+
"test:deploySchema": "node test-integration/_env/srv/hana/deploy.js",
|
|
29
|
+
"test:cleanSchemas": "node test-integration/_env/srv/hana/cleanObsoletSchemas.js",
|
|
29
30
|
"lint": "npm run eslint && npm run prettier",
|
|
30
31
|
"lint:ci": "npm run eslint:ci && npm run prettier:ci",
|
|
31
32
|
"eslint": "eslint --fix .",
|
|
@@ -43,7 +44,7 @@
|
|
|
43
44
|
"dependencies": {
|
|
44
45
|
"redis": "4.6.10",
|
|
45
46
|
"verror": "1.10.1",
|
|
46
|
-
"yaml": "2.3.
|
|
47
|
+
"yaml": "2.3.4"
|
|
47
48
|
},
|
|
48
49
|
"devDependencies": {
|
|
49
50
|
"@sap/cds": "7.3.1",
|
|
@@ -55,7 +56,7 @@
|
|
|
55
56
|
"express": "4.18.2",
|
|
56
57
|
"hdb": "0.19.6",
|
|
57
58
|
"jest": "29.7.0",
|
|
58
|
-
"prettier": "
|
|
59
|
+
"prettier": "2.8.8",
|
|
59
60
|
"sqlite3": "5.1.6"
|
|
60
61
|
},
|
|
61
62
|
"homepage": "https://cap-js-community.github.io/event-queue/",
|
package/src/EventQueueError.js
CHANGED
|
@@ -11,6 +11,7 @@ const ERROR_CODES = {
|
|
|
11
11
|
MISSING_TABLE_DEFINITION: "MISSING_TABLE_DEFINITION",
|
|
12
12
|
MISSING_ELEMENT_IN_TABLE: "MISSING_ELEMENT_IN_TABLE",
|
|
13
13
|
TYPE_MISMATCH_TABLE: "TYPE_MISMATCH_TABLE",
|
|
14
|
+
NO_VALID_DATE: "NO_VALID_DATE",
|
|
14
15
|
};
|
|
15
16
|
|
|
16
17
|
const ERROR_CODES_META = {
|
|
@@ -39,6 +40,9 @@ const ERROR_CODES_META = {
|
|
|
39
40
|
[ERROR_CODES.TYPE_MISMATCH_TABLE]: {
|
|
40
41
|
message: "At least one field in the provided table doesn't have the expected data type.",
|
|
41
42
|
},
|
|
43
|
+
[ERROR_CODES.NO_VALID_DATE]: {
|
|
44
|
+
message: "One or more events contain a date in a malformed format.",
|
|
45
|
+
},
|
|
42
46
|
};
|
|
43
47
|
|
|
44
48
|
class EventQueueError extends VError {
|
|
@@ -131,6 +135,17 @@ class EventQueueError extends VError {
|
|
|
131
135
|
message
|
|
132
136
|
);
|
|
133
137
|
}
|
|
138
|
+
|
|
139
|
+
static malformedDate(date) {
|
|
140
|
+
const { message } = ERROR_CODES_META[ERROR_CODES.NO_VALID_DATE];
|
|
141
|
+
return new EventQueueError(
|
|
142
|
+
{
|
|
143
|
+
name: ERROR_CODES.NO_VALID_DATE,
|
|
144
|
+
info: { date },
|
|
145
|
+
},
|
|
146
|
+
message
|
|
147
|
+
);
|
|
148
|
+
}
|
|
134
149
|
}
|
|
135
150
|
|
|
136
151
|
module.exports = EventQueueError;
|
|
@@ -7,6 +7,7 @@ const { EventProcessingStatus, TransactionMode } = require("./constants");
|
|
|
7
7
|
const distributedLock = require("./shared/distributedLock");
|
|
8
8
|
const EventQueueError = require("./EventQueueError");
|
|
9
9
|
const { arrayToFlatMap } = require("./shared/common");
|
|
10
|
+
const eventScheduler = require("./shared/EventScheduler");
|
|
10
11
|
const eventQueueConfig = require("./config");
|
|
11
12
|
const PerformanceTracer = require("./shared/PerformanceTracer");
|
|
12
13
|
|
|
@@ -20,12 +21,16 @@ const SELECT_LIMIT_EVENTS_PER_TICK = 100;
|
|
|
20
21
|
const DEFAULT_DELETE_FINISHED_EVENTS_AFTER = 0;
|
|
21
22
|
const DAYS_TO_MS = 24 * 60 * 60 * 1000;
|
|
22
23
|
const TRIES_FOR_EXCEEDED_EVENTS = 3;
|
|
24
|
+
const EVENT_START_AFTER_HEADROOM = 3 * 1000;
|
|
23
25
|
|
|
24
26
|
class EventQueueProcessorBase {
|
|
25
27
|
#eventsWithExceededTries = [];
|
|
26
28
|
#exceededTriesExceeded = [];
|
|
27
29
|
#selectedEventMap = {};
|
|
28
30
|
#queueEntriesWithPayloadMap = {};
|
|
31
|
+
#eventType = null;
|
|
32
|
+
#eventSubType = null;
|
|
33
|
+
#config = null;
|
|
29
34
|
|
|
30
35
|
constructor(context, eventType, eventSubType, config) {
|
|
31
36
|
this.__context = context;
|
|
@@ -36,26 +41,27 @@ class EventQueueProcessorBase {
|
|
|
36
41
|
this.__eventProcessingMap = {};
|
|
37
42
|
this.__statusMap = {};
|
|
38
43
|
this.__commitedStatusMap = {};
|
|
39
|
-
this
|
|
40
|
-
this
|
|
41
|
-
this.
|
|
42
|
-
this.__parallelEventProcessing = this.
|
|
44
|
+
this.#eventType = eventType;
|
|
45
|
+
this.#eventSubType = eventSubType;
|
|
46
|
+
this.__eventConfig = config ?? {};
|
|
47
|
+
this.__parallelEventProcessing = this.__eventConfig.parallelEventProcessing ?? DEFAULT_PARALLEL_EVENT_PROCESSING;
|
|
43
48
|
if (this.__parallelEventProcessing > LIMIT_PARALLEL_EVENT_PROCESSING) {
|
|
44
49
|
this.__parallelEventProcessing = LIMIT_PARALLEL_EVENT_PROCESSING;
|
|
45
50
|
}
|
|
46
51
|
// NOTE: keep the feature, this might be needed again
|
|
47
52
|
this.__concurrentEventProcessing = false;
|
|
48
|
-
this.__startTime = this.
|
|
49
|
-
this.__retryAttempts = this.
|
|
50
|
-
this.__selectMaxChunkSize = this.
|
|
51
|
-
this.__selectNextChunk = !!this.
|
|
53
|
+
this.__startTime = this.__eventConfig.startTime ?? new Date();
|
|
54
|
+
this.__retryAttempts = this.__eventConfig.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
|
|
55
|
+
this.__selectMaxChunkSize = this.__eventConfig.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
|
|
56
|
+
this.__selectNextChunk = !!this.__eventConfig.checkForNextChunk;
|
|
52
57
|
this.__keepalivePromises = {};
|
|
53
|
-
this.__outdatedCheckEnabled = this.
|
|
54
|
-
this.__transactionMode = this.
|
|
55
|
-
if (this.
|
|
58
|
+
this.__outdatedCheckEnabled = this.__eventConfig.eventOutdatedCheck ?? true;
|
|
59
|
+
this.__transactionMode = this.__eventConfig.transactionMode ?? TransactionMode.isolated;
|
|
60
|
+
if (this.__eventConfig.deleteFinishedEventsAfterDays) {
|
|
56
61
|
this.__deleteFinishedEventsAfter =
|
|
57
|
-
Number.isInteger(this.
|
|
58
|
-
|
|
62
|
+
Number.isInteger(this.__eventConfig.deleteFinishedEventsAfterDays) &&
|
|
63
|
+
this.__eventConfig.deleteFinishedEventsAfterDays > 0
|
|
64
|
+
? this.__eventConfig.deleteFinishedEventsAfterDays
|
|
59
65
|
: DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
|
|
60
66
|
} else {
|
|
61
67
|
this.__deleteFinishedEventsAfter = DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
|
|
@@ -65,7 +71,7 @@ class EventQueueProcessorBase {
|
|
|
65
71
|
this.__txUsageAllowed = true;
|
|
66
72
|
this.__txMap = {};
|
|
67
73
|
this.__txRollback = {};
|
|
68
|
-
this
|
|
74
|
+
this.#config = eventQueueConfig.getConfigInstance();
|
|
69
75
|
this.__queueEntries = [];
|
|
70
76
|
}
|
|
71
77
|
|
|
@@ -99,8 +105,8 @@ class EventQueueProcessorBase {
|
|
|
99
105
|
this.__performanceLoggerEvents?.endPerformanceTrace(
|
|
100
106
|
{ threshold: 50 },
|
|
101
107
|
{
|
|
102
|
-
eventType: this
|
|
103
|
-
eventSubType: this
|
|
108
|
+
eventType: this.#eventType,
|
|
109
|
+
eventSubType: this.#eventSubType,
|
|
104
110
|
}
|
|
105
111
|
);
|
|
106
112
|
}
|
|
@@ -109,16 +115,16 @@ class EventQueueProcessorBase {
|
|
|
109
115
|
this.__performanceLoggerPreprocessing?.endPerformanceTrace(
|
|
110
116
|
{ threshold: 50 },
|
|
111
117
|
{
|
|
112
|
-
eventType: this
|
|
113
|
-
eventSubType: this
|
|
118
|
+
eventType: this.#eventType,
|
|
119
|
+
eventSubType: this.#eventSubType,
|
|
114
120
|
}
|
|
115
121
|
);
|
|
116
122
|
}
|
|
117
123
|
|
|
118
124
|
logTimeExceeded(iterationCounter) {
|
|
119
125
|
this.logger.info("Exiting event queue processing as max time exceeded", {
|
|
120
|
-
eventType: this
|
|
121
|
-
eventSubType: this
|
|
126
|
+
eventType: this.#eventType,
|
|
127
|
+
eventSubType: this.#eventSubType,
|
|
122
128
|
iterationCounter,
|
|
123
129
|
});
|
|
124
130
|
}
|
|
@@ -127,8 +133,8 @@ class EventQueueProcessorBase {
|
|
|
127
133
|
// TODO: how to handle custom fields
|
|
128
134
|
this.logger.info("Processing queue event", {
|
|
129
135
|
numberQueueEntries: queueEntries.length,
|
|
130
|
-
eventType: this
|
|
131
|
-
eventSubType: this
|
|
136
|
+
eventType: this.#eventType,
|
|
137
|
+
eventSubType: this.#eventSubType,
|
|
132
138
|
});
|
|
133
139
|
}
|
|
134
140
|
|
|
@@ -157,8 +163,8 @@ class EventQueueProcessorBase {
|
|
|
157
163
|
this.logger.error(
|
|
158
164
|
"The supplied queueEntry has not been selected before and should not be processed. Entry will not be processed.",
|
|
159
165
|
{
|
|
160
|
-
eventType: this
|
|
161
|
-
eventSubType: this
|
|
166
|
+
eventType: this.#eventType,
|
|
167
|
+
eventSubType: this.#eventSubType,
|
|
162
168
|
queueEntryId: queueEntry.ID,
|
|
163
169
|
}
|
|
164
170
|
);
|
|
@@ -177,8 +183,8 @@ class EventQueueProcessorBase {
|
|
|
177
183
|
setStatusToDone(queueEntry) {
|
|
178
184
|
this.logger.debug("setting status for queueEntry to done", {
|
|
179
185
|
id: queueEntry.ID,
|
|
180
|
-
eventType: this
|
|
181
|
-
eventSubType: this
|
|
186
|
+
eventType: this.#eventType,
|
|
187
|
+
eventSubType: this.#eventSubType,
|
|
182
188
|
});
|
|
183
189
|
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Done);
|
|
184
190
|
}
|
|
@@ -207,8 +213,8 @@ class EventQueueProcessorBase {
|
|
|
207
213
|
this.logger.debug("add entry to processing map", {
|
|
208
214
|
key,
|
|
209
215
|
queueEntry,
|
|
210
|
-
eventType: this
|
|
211
|
-
eventSubType: this
|
|
216
|
+
eventType: this.#eventType,
|
|
217
|
+
eventSubType: this.#eventSubType,
|
|
212
218
|
});
|
|
213
219
|
this.__eventProcessingMap[key] = this.__eventProcessingMap[key] ?? {
|
|
214
220
|
queueEntries: [],
|
|
@@ -230,8 +236,8 @@ class EventQueueProcessorBase {
|
|
|
230
236
|
setEventStatus(queueEntries, queueEntryProcessingStatusTuple, returnMap = false) {
|
|
231
237
|
this.logger.debug("setting event status for entries", {
|
|
232
238
|
queueEntryProcessingStatusTuple: JSON.stringify(queueEntryProcessingStatusTuple),
|
|
233
|
-
eventType: this
|
|
234
|
-
eventSubType: this
|
|
239
|
+
eventType: this.#eventType,
|
|
240
|
+
eventSubType: this.#eventSubType,
|
|
235
241
|
});
|
|
236
242
|
const statusMap = this.commitOnEventLevel || returnMap ? {} : this.__statusMap;
|
|
237
243
|
try {
|
|
@@ -245,8 +251,8 @@ class EventQueueProcessorBase {
|
|
|
245
251
|
this.logger.error(
|
|
246
252
|
`The supplied status tuple doesn't have the required structure. Setting all entries to error. Error: ${error.toString()}`,
|
|
247
253
|
{
|
|
248
|
-
eventType: this
|
|
249
|
-
eventSubType: this
|
|
254
|
+
eventType: this.#eventType,
|
|
255
|
+
eventSubType: this.#eventSubType,
|
|
250
256
|
}
|
|
251
257
|
);
|
|
252
258
|
}
|
|
@@ -286,8 +292,8 @@ class EventQueueProcessorBase {
|
|
|
286
292
|
this.logger.error(
|
|
287
293
|
`Caught error during event processing - setting queue entry to error. Please catch your promises/exceptions. Error: ${error}`,
|
|
288
294
|
{
|
|
289
|
-
eventType: this
|
|
290
|
-
eventSubType: this
|
|
295
|
+
eventType: this.#eventType,
|
|
296
|
+
eventSubType: this.#eventSubType,
|
|
291
297
|
queueEntriesIds: queueEntries.map(({ ID }) => ID),
|
|
292
298
|
}
|
|
293
299
|
);
|
|
@@ -304,8 +310,8 @@ class EventQueueProcessorBase {
|
|
|
304
310
|
*/
|
|
305
311
|
async persistEventStatus(tx, { skipChecks, statusMap = this.__statusMap } = {}) {
|
|
306
312
|
this.logger.debug("entering persistEventStatus", {
|
|
307
|
-
eventType: this
|
|
308
|
-
eventSubType: this
|
|
313
|
+
eventType: this.#eventType,
|
|
314
|
+
eventSubType: this.#eventSubType,
|
|
309
315
|
});
|
|
310
316
|
this.#ensureOnlySelectedQueueEntries(statusMap);
|
|
311
317
|
if (!skipChecks) {
|
|
@@ -335,8 +341,8 @@ class EventQueueProcessorBase {
|
|
|
335
341
|
}
|
|
336
342
|
);
|
|
337
343
|
this.logger.debug("persistEventStatus for entries", {
|
|
338
|
-
eventType: this
|
|
339
|
-
eventSubType: this
|
|
344
|
+
eventType: this.#eventType,
|
|
345
|
+
eventSubType: this.#eventSubType,
|
|
340
346
|
invalidAttempts,
|
|
341
347
|
failed,
|
|
342
348
|
exceeded,
|
|
@@ -344,7 +350,7 @@ class EventQueueProcessorBase {
|
|
|
344
350
|
});
|
|
345
351
|
if (invalidAttempts.length) {
|
|
346
352
|
await tx.run(
|
|
347
|
-
UPDATE.entity(this.
|
|
353
|
+
UPDATE.entity(this.#config.tableNameEventQueue)
|
|
348
354
|
.set({
|
|
349
355
|
status: EventProcessingStatus.Open,
|
|
350
356
|
lastAttemptTimestamp: new Date().toISOString(),
|
|
@@ -365,7 +371,7 @@ class EventQueueProcessorBase {
|
|
|
365
371
|
continue;
|
|
366
372
|
}
|
|
367
373
|
await tx.run(
|
|
368
|
-
UPDATE.entity(this.
|
|
374
|
+
UPDATE.entity(this.#config.tableNameEventQueue)
|
|
369
375
|
.set({
|
|
370
376
|
status: status,
|
|
371
377
|
lastAttemptTimestamp: ts,
|
|
@@ -374,8 +380,8 @@ class EventQueueProcessorBase {
|
|
|
374
380
|
);
|
|
375
381
|
}
|
|
376
382
|
this.logger.debug("exiting persistEventStatus", {
|
|
377
|
-
eventType: this
|
|
378
|
-
eventSubType: this
|
|
383
|
+
eventType: this.#eventType,
|
|
384
|
+
eventSubType: this.#eventSubType,
|
|
379
385
|
});
|
|
380
386
|
}
|
|
381
387
|
|
|
@@ -384,18 +390,18 @@ class EventQueueProcessorBase {
|
|
|
384
390
|
return;
|
|
385
391
|
}
|
|
386
392
|
const deleteCount = await tx.run(
|
|
387
|
-
DELETE.from(this.
|
|
393
|
+
DELETE.from(this.#config.tableNameEventQueue).where(
|
|
388
394
|
"type =",
|
|
389
|
-
this
|
|
395
|
+
this.#eventType,
|
|
390
396
|
"AND subType=",
|
|
391
|
-
this
|
|
397
|
+
this.#eventSubType,
|
|
392
398
|
"AND lastAttemptTimestamp <=",
|
|
393
399
|
new Date(Date.now() - this.__deleteFinishedEventsAfter * DAYS_TO_MS).toISOString()
|
|
394
400
|
)
|
|
395
401
|
);
|
|
396
402
|
this.logger.debug("Deleted finished events", {
|
|
397
|
-
eventType: this
|
|
398
|
-
eventSubType: this
|
|
403
|
+
eventType: this.#eventType,
|
|
404
|
+
eventSubType: this.#eventSubType,
|
|
399
405
|
deleteFinishedEventsAfter: this.__deleteFinishedEventsAfter,
|
|
400
406
|
deleteCount,
|
|
401
407
|
});
|
|
@@ -407,8 +413,8 @@ class EventQueueProcessorBase {
|
|
|
407
413
|
return;
|
|
408
414
|
}
|
|
409
415
|
this.logger.error("Missing status for selected event entry. Setting status to error", {
|
|
410
|
-
eventType: this
|
|
411
|
-
eventSubType: this
|
|
416
|
+
eventType: this.#eventType,
|
|
417
|
+
eventSubType: this.#eventSubType,
|
|
412
418
|
queueEntry,
|
|
413
419
|
});
|
|
414
420
|
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
|
|
@@ -429,8 +435,8 @@ class EventQueueProcessorBase {
|
|
|
429
435
|
}
|
|
430
436
|
|
|
431
437
|
this.logger.error("Not allowed event status returned. Only Open, Done, Error is allowed!", {
|
|
432
|
-
eventType: this
|
|
433
|
-
eventSubType: this
|
|
438
|
+
eventType: this.#eventType,
|
|
439
|
+
eventSubType: this.#eventSubType,
|
|
434
440
|
queueEntryId,
|
|
435
441
|
status: statusMap[queueEntryId],
|
|
436
442
|
});
|
|
@@ -447,8 +453,8 @@ class EventQueueProcessorBase {
|
|
|
447
453
|
this.logger.error(
|
|
448
454
|
"Status reported for event queue entry which haven't be selected before. Removing the status.",
|
|
449
455
|
{
|
|
450
|
-
eventType: this
|
|
451
|
-
eventSubType: this
|
|
456
|
+
eventType: this.#eventType,
|
|
457
|
+
eventSubType: this.#eventSubType,
|
|
452
458
|
queueEntryId,
|
|
453
459
|
}
|
|
454
460
|
);
|
|
@@ -458,8 +464,8 @@ class EventQueueProcessorBase {
|
|
|
458
464
|
|
|
459
465
|
handleErrorDuringClustering(error) {
|
|
460
466
|
this.logger.error(`Error during clustering of events - setting all queue entries to error. Error: ${error}`, {
|
|
461
|
-
eventType: this
|
|
462
|
-
eventSubType: this
|
|
467
|
+
eventType: this.#eventType,
|
|
468
|
+
eventSubType: this.#eventSubType,
|
|
463
469
|
});
|
|
464
470
|
this.__queueEntries.forEach((queueEntry) => {
|
|
465
471
|
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
|
|
@@ -471,21 +477,13 @@ class EventQueueProcessorBase {
|
|
|
471
477
|
"Undefined payload is not allowed. If status should be done, nulls needs to be returned" +
|
|
472
478
|
" - setting queue entry to error",
|
|
473
479
|
{
|
|
474
|
-
eventType: this
|
|
475
|
-
eventSubType: this
|
|
480
|
+
eventType: this.#eventType,
|
|
481
|
+
eventSubType: this.#eventSubType,
|
|
476
482
|
}
|
|
477
483
|
);
|
|
478
484
|
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
|
|
479
485
|
}
|
|
480
486
|
|
|
481
|
-
static async handleMissingTypeImplementation(context, eventType, eventSubType) {
|
|
482
|
-
const baseInstance = new EventQueueProcessorBase(context, eventType, eventSubType);
|
|
483
|
-
baseInstance.logger.error("No Implementation found in the provided configuration file.", {
|
|
484
|
-
eventType,
|
|
485
|
-
eventSubType,
|
|
486
|
-
});
|
|
487
|
-
}
|
|
488
|
-
|
|
489
487
|
/**
|
|
490
488
|
* This function selects all relevant events based on the eventType and eventSubType supplied through the constructor
|
|
491
489
|
* during initialization of the class.
|
|
@@ -495,17 +493,20 @@ class EventQueueProcessorBase {
|
|
|
495
493
|
*/
|
|
496
494
|
async getQueueEntriesAndSetToInProgress() {
|
|
497
495
|
let result = [];
|
|
496
|
+
const refDateStartAfter = new Date(Date.now() + this.#config.runInterval);
|
|
498
497
|
await executeInNewTransaction(this.__baseContext, "eventQueue-getQueueEntriesAndSetToInProgress", async (tx) => {
|
|
499
498
|
const entries = await tx.run(
|
|
500
|
-
SELECT.from(this.
|
|
501
|
-
.forUpdate({ wait: this.
|
|
499
|
+
SELECT.from(this.#config.tableNameEventQueue)
|
|
500
|
+
.forUpdate({ wait: this.#config.forUpdateTimeout })
|
|
502
501
|
.limit(this.selectMaxChunkSize)
|
|
503
502
|
.where(
|
|
504
503
|
"type =",
|
|
505
|
-
this
|
|
504
|
+
this.#eventType,
|
|
506
505
|
"AND subType=",
|
|
507
|
-
this
|
|
508
|
-
"AND (
|
|
506
|
+
this.#eventSubType,
|
|
507
|
+
"AND ( startAfter IS NULL OR startAfter <=",
|
|
508
|
+
refDateStartAfter.toISOString(),
|
|
509
|
+
" ) AND ( status =",
|
|
509
510
|
EventProcessingStatus.Open,
|
|
510
511
|
"OR ( status =",
|
|
511
512
|
EventProcessingStatus.Error,
|
|
@@ -514,7 +515,7 @@ class EventQueueProcessorBase {
|
|
|
514
515
|
") OR ( status =",
|
|
515
516
|
EventProcessingStatus.InProgress,
|
|
516
517
|
"AND lastAttemptTimestamp <=",
|
|
517
|
-
new Date(new Date().getTime() - this.
|
|
518
|
+
new Date(new Date().getTime() - this.#config.globalTxTimeout).toISOString(),
|
|
518
519
|
") )"
|
|
519
520
|
)
|
|
520
521
|
.orderBy("createdAt", "ID")
|
|
@@ -522,37 +523,44 @@ class EventQueueProcessorBase {
|
|
|
522
523
|
|
|
523
524
|
if (!entries.length) {
|
|
524
525
|
this.logger.debug("no entries available for processing", {
|
|
525
|
-
eventType: this
|
|
526
|
-
eventSubType: this
|
|
526
|
+
eventType: this.#eventType,
|
|
527
|
+
eventSubType: this.#eventSubType,
|
|
527
528
|
});
|
|
528
529
|
this.__emptyChunkSelected = true;
|
|
529
530
|
return;
|
|
530
531
|
}
|
|
531
532
|
|
|
532
|
-
|
|
533
|
-
|
|
533
|
+
const { exceededTries, openEvents, exceededTriesExceeded, delayedEvents } = this.#clusterEvents(
|
|
534
|
+
entries,
|
|
535
|
+
refDateStartAfter
|
|
536
|
+
);
|
|
537
|
+
const eventsForProcessing = exceededTries.concat(openEvents).concat(exceededTriesExceeded);
|
|
538
|
+
this.#selectedEventMap = arrayToFlatMap(eventsForProcessing);
|
|
534
539
|
if (exceededTries.length) {
|
|
535
540
|
this.#eventsWithExceededTries = exceededTries;
|
|
536
541
|
}
|
|
537
542
|
if (exceededTriesExceeded.length) {
|
|
538
543
|
this.#exceededTriesExceeded = exceededTriesExceeded;
|
|
539
544
|
}
|
|
545
|
+
this.#handleDelayedEvents(delayedEvents);
|
|
540
546
|
|
|
541
547
|
result = openEvents;
|
|
548
|
+
this.logger.info("Selected event queue entries for processing", {
|
|
549
|
+
openEvents: openEvents.length,
|
|
550
|
+
...(delayedEvents.length && { delayedEvents: delayedEvents.length }),
|
|
551
|
+
...(exceededTries.length && { exceededTries: exceededTries.length }),
|
|
552
|
+
eventType: this.#eventType,
|
|
553
|
+
eventSubType: this.#eventSubType,
|
|
554
|
+
});
|
|
542
555
|
|
|
543
|
-
if (!
|
|
556
|
+
if (!eventsForProcessing.length) {
|
|
544
557
|
this.__emptyChunkSelected = true;
|
|
558
|
+
return;
|
|
545
559
|
}
|
|
546
560
|
|
|
547
|
-
this.logger.info("Selected event queue entries for processing", {
|
|
548
|
-
queueEntriesCount: result.length,
|
|
549
|
-
eventType: this.__eventType,
|
|
550
|
-
eventSubType: this.__eventSubType,
|
|
551
|
-
});
|
|
552
|
-
|
|
553
561
|
const isoTimestamp = new Date().toISOString();
|
|
554
562
|
await tx.run(
|
|
555
|
-
UPDATE.entity(this.
|
|
563
|
+
UPDATE.entity(this.#config.tableNameEventQueue)
|
|
556
564
|
.with({
|
|
557
565
|
status: EventProcessingStatus.InProgress,
|
|
558
566
|
lastAttemptTimestamp: isoTimestamp,
|
|
@@ -560,10 +568,10 @@ class EventQueueProcessorBase {
|
|
|
560
568
|
})
|
|
561
569
|
.where(
|
|
562
570
|
"ID IN",
|
|
563
|
-
|
|
571
|
+
eventsForProcessing.map(({ ID }) => ID)
|
|
564
572
|
)
|
|
565
573
|
);
|
|
566
|
-
|
|
574
|
+
eventsForProcessing.forEach((entry) => {
|
|
567
575
|
entry.lastAttemptTimestamp = isoTimestamp;
|
|
568
576
|
// NOTE: empty payloads are supported on DB-Level.
|
|
569
577
|
// Behaviour of event queue is: null as payload is treated as obsolete/done
|
|
@@ -578,22 +586,45 @@ class EventQueueProcessorBase {
|
|
|
578
586
|
return result;
|
|
579
587
|
}
|
|
580
588
|
|
|
581
|
-
#
|
|
589
|
+
#handleDelayedEvents(delayedEvents) {
|
|
590
|
+
const eventSchedulerInstance = eventScheduler.getInstance();
|
|
591
|
+
for (const delayedEvent of delayedEvents) {
|
|
592
|
+
eventSchedulerInstance.scheduleEvent(
|
|
593
|
+
this.__context.tenant,
|
|
594
|
+
this.#eventType,
|
|
595
|
+
this.#eventSubType,
|
|
596
|
+
delayedEvent.startAfter
|
|
597
|
+
);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
#clusterEvents(events, refDateStartAfter) {
|
|
602
|
+
const refDate = new Date(refDateStartAfter.getTime() - this.#config.runInterval + EVENT_START_AFTER_HEADROOM);
|
|
582
603
|
return events.reduce(
|
|
583
604
|
(result, event) => {
|
|
584
605
|
if (event.attempts === this.__retryAttempts + TRIES_FOR_EXCEEDED_EVENTS) {
|
|
585
606
|
result.exceededTriesExceeded.push(event);
|
|
586
607
|
} else if (event.attempts >= this.__retryAttempts) {
|
|
587
608
|
result.exceededTries.push(event);
|
|
609
|
+
} else if (this.#isDelayedEvent(event, refDate)) {
|
|
610
|
+
result.delayedEvents.push(event);
|
|
588
611
|
} else {
|
|
589
612
|
result.openEvents.push(event);
|
|
590
613
|
}
|
|
591
614
|
return result;
|
|
592
615
|
},
|
|
593
|
-
{ exceededTries: [], openEvents: [], exceededTriesExceeded: [] }
|
|
616
|
+
{ exceededTries: [], openEvents: [], exceededTriesExceeded: [], delayedEvents: [] }
|
|
594
617
|
);
|
|
595
618
|
}
|
|
596
619
|
|
|
620
|
+
#isDelayedEvent(event, refDate) {
|
|
621
|
+
if (!event.startAfter) {
|
|
622
|
+
return false;
|
|
623
|
+
}
|
|
624
|
+
event.startAfter = new Date(event.startAfter);
|
|
625
|
+
return !(refDate >= event.startAfter);
|
|
626
|
+
}
|
|
627
|
+
|
|
597
628
|
async handleExceededEvents() {
|
|
598
629
|
await this.#handleExceededTriesExceeded();
|
|
599
630
|
if (!this.#eventsWithExceededTries.length) {
|
|
@@ -603,15 +634,15 @@ class EventQueueProcessorBase {
|
|
|
603
634
|
for (const exceededEvent of this.#eventsWithExceededTries) {
|
|
604
635
|
await executeInNewTransaction(
|
|
605
636
|
this.context,
|
|
606
|
-
`eventQueue-handleExceededEvents-${this
|
|
637
|
+
`eventQueue-handleExceededEvents-${this.#eventType}##${this.#eventSubType}`,
|
|
607
638
|
async (tx) => {
|
|
608
639
|
try {
|
|
609
640
|
this.processEventContext = tx.context;
|
|
610
641
|
this.modifyQueueEntry(exceededEvent);
|
|
611
642
|
await this.hookForExceededEvents({ ...exceededEvent });
|
|
612
643
|
this.logger.warn("The retry attempts for the following events are exceeded", {
|
|
613
|
-
eventType: this
|
|
614
|
-
eventSubType: this
|
|
644
|
+
eventType: this.#eventType,
|
|
645
|
+
eventSubType: this.#eventSubType,
|
|
615
646
|
retryAttempts: this.__retryAttempts,
|
|
616
647
|
queueEntriesId: exceededEvent.ID,
|
|
617
648
|
currentAttempt: exceededEvent.attempts,
|
|
@@ -621,8 +652,8 @@ class EventQueueProcessorBase {
|
|
|
621
652
|
this.logger.error(
|
|
622
653
|
`Caught error during hook for exceeded events - setting queue entry to error. Please catch your promises/exceptions. Error: ${err}`,
|
|
623
654
|
{
|
|
624
|
-
eventType: this
|
|
625
|
-
eventSubType: this
|
|
655
|
+
eventType: this.#eventType,
|
|
656
|
+
eventSubType: this.#eventSubType,
|
|
626
657
|
retryAttempts: this.__retryAttempts,
|
|
627
658
|
queueEntriesId: exceededEvent.ID,
|
|
628
659
|
currentAttempt: exceededEvent.attempts,
|
|
@@ -641,8 +672,8 @@ class EventQueueProcessorBase {
|
|
|
641
672
|
async #handleExceededTriesExceeded() {
|
|
642
673
|
if (this.#exceededTriesExceeded.length) {
|
|
643
674
|
this.logger.error("Event hook failure exceeded, status set to 'exceeded' without invoking hook again!", {
|
|
644
|
-
eventType: this
|
|
645
|
-
eventSubType: this
|
|
675
|
+
eventType: this.#eventType,
|
|
676
|
+
eventSubType: this.#eventSubType,
|
|
646
677
|
queueEntriesIds: this.#eventsWithExceededTries.map(({ ID }) => ID),
|
|
647
678
|
});
|
|
648
679
|
await executeInNewTransaction(this.context, "exceededTriesExceeded", async (tx) => {
|
|
@@ -705,8 +736,8 @@ class EventQueueProcessorBase {
|
|
|
705
736
|
const checkAndUpdatePromise = new Promise((resolve, reject) => {
|
|
706
737
|
executeInNewTransaction(this.__baseContext, "eventProcessing-isOutdatedAndKeepalive", async (tx) => {
|
|
707
738
|
const queueEntriesFresh = await tx.run(
|
|
708
|
-
SELECT.from(this.
|
|
709
|
-
.forUpdate({ wait: this.
|
|
739
|
+
SELECT.from(this.#config.tableNameEventQueue)
|
|
740
|
+
.forUpdate({ wait: this.#config.forUpdateTimeout })
|
|
710
741
|
.where(
|
|
711
742
|
"ID IN",
|
|
712
743
|
queueEntries.map(({ ID }) => ID)
|
|
@@ -720,7 +751,7 @@ class EventQueueProcessorBase {
|
|
|
720
751
|
let newTs = new Date().toISOString();
|
|
721
752
|
if (!eventOutdated) {
|
|
722
753
|
await tx.run(
|
|
723
|
-
UPDATE.entity(this.
|
|
754
|
+
UPDATE.entity(this.#config.tableNameEventQueue)
|
|
724
755
|
.set("lastAttemptTimestamp =", newTs)
|
|
725
756
|
.where(
|
|
726
757
|
"ID IN",
|
|
@@ -730,8 +761,8 @@ class EventQueueProcessorBase {
|
|
|
730
761
|
} else {
|
|
731
762
|
newTs = null;
|
|
732
763
|
this.logger.warn("event data has been modified. Processing skipped.", {
|
|
733
|
-
eventType: this
|
|
734
|
-
eventSubType: this
|
|
764
|
+
eventType: this.#eventType,
|
|
765
|
+
eventSubType: this.#eventSubType,
|
|
735
766
|
queueEntriesIds: queueEntries.map(({ ID }) => ID),
|
|
736
767
|
});
|
|
737
768
|
queueEntries.forEach(({ ID: queueEntryId }) => delete this.__queueEntriesMap[queueEntryId]);
|
|
@@ -761,7 +792,7 @@ class EventQueueProcessorBase {
|
|
|
761
792
|
|
|
762
793
|
const lockAcquired = await distributedLock.acquireLock(
|
|
763
794
|
this.context,
|
|
764
|
-
[this
|
|
795
|
+
[this.#eventType, this.#eventSubType].join("##")
|
|
765
796
|
);
|
|
766
797
|
if (!lockAcquired) {
|
|
767
798
|
return false;
|
|
@@ -775,7 +806,7 @@ class EventQueueProcessorBase {
|
|
|
775
806
|
return;
|
|
776
807
|
}
|
|
777
808
|
try {
|
|
778
|
-
await distributedLock.releaseLock(this.context, [this
|
|
809
|
+
await distributedLock.releaseLock(this.context, [this.#eventType, this.#eventSubType].join("##"));
|
|
779
810
|
} catch (err) {
|
|
780
811
|
this.logger.error("Releasing distributed lock failed. Error:", err.toString());
|
|
781
812
|
}
|
|
@@ -822,14 +853,14 @@ class EventQueueProcessorBase {
|
|
|
822
853
|
|
|
823
854
|
get tx() {
|
|
824
855
|
if (!this.__txUsageAllowed && this.__parallelEventProcessing > 1) {
|
|
825
|
-
throw EventQueueError.wrongTxUsage(this
|
|
856
|
+
throw EventQueueError.wrongTxUsage(this.#eventType, this.#eventSubType);
|
|
826
857
|
}
|
|
827
858
|
return this.__processTx ?? this.__tx;
|
|
828
859
|
}
|
|
829
860
|
|
|
830
861
|
get context() {
|
|
831
862
|
if (!this.__txUsageAllowed && this.__parallelEventProcessing > 1) {
|
|
832
|
-
throw EventQueueError.wrongTxUsage(this
|
|
863
|
+
throw EventQueueError.wrongTxUsage(this.#eventType, this.#eventSubType);
|
|
833
864
|
}
|
|
834
865
|
return this.__processContext ?? this.__context;
|
|
835
866
|
}
|
|
@@ -847,11 +878,11 @@ class EventQueueProcessorBase {
|
|
|
847
878
|
}
|
|
848
879
|
|
|
849
880
|
get eventType() {
|
|
850
|
-
return this
|
|
881
|
+
return this.#eventType;
|
|
851
882
|
}
|
|
852
883
|
|
|
853
884
|
get eventSubType() {
|
|
854
|
-
return this
|
|
885
|
+
return this.#eventSubType;
|
|
855
886
|
}
|
|
856
887
|
|
|
857
888
|
get emptyChunkSelected() {
|