@cap-js-community/event-queue 0.1.58 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/Event.cds +1 -0
- package/package.json +5 -4
- package/src/EventQueueError.js +15 -0
- package/src/EventQueueProcessorBase.js +136 -105
- package/src/config.js +77 -51
- package/src/dbHandler.js +11 -2
- package/src/initialize.js +19 -26
- package/src/processEventQueue.js +4 -2
- package/src/publishEvent.js +27 -1
- package/src/redisPubSub.js +5 -29
- package/src/runner.js +23 -1
- package/src/shared/EventScheduler.js +53 -0
- package/src/shared/common.js +12 -1
package/db/Event.cds
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cap-js-community/event-queue",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "An event queue that enables secure transactional processing of asynchronous events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"files": [
|
|
@@ -23,9 +23,10 @@
|
|
|
23
23
|
"test:unit": "jest --testPathIgnorePatterns=\"/test-integration/\"",
|
|
24
24
|
"test:integration": "jest --testPathIgnorePatterns=\"/test/\" --runInBand --forceExit",
|
|
25
25
|
"test:all": "npm run test:unit && npm run test:integration",
|
|
26
|
-
"test:ci": "npm run test:prepare && npm run test:all",
|
|
27
26
|
"test:all:coverage": "jest --runInBand --forceExit --collect-coverage",
|
|
28
27
|
"test:prepare": "npm run build:ci --prefix=./test-integration/_env",
|
|
28
|
+
"test:deploySchema": "node test-integration/_env/srv/hana/deploy.js",
|
|
29
|
+
"test:cleanSchemas": "node test-integration/_env/srv/hana/cleanObsoletSchemas.js",
|
|
29
30
|
"lint": "npm run eslint && npm run prettier",
|
|
30
31
|
"lint:ci": "npm run eslint:ci && npm run prettier:ci",
|
|
31
32
|
"eslint": "eslint --fix .",
|
|
@@ -43,7 +44,7 @@
|
|
|
43
44
|
"dependencies": {
|
|
44
45
|
"redis": "4.6.10",
|
|
45
46
|
"verror": "1.10.1",
|
|
46
|
-
"yaml": "2.3.
|
|
47
|
+
"yaml": "2.3.4"
|
|
47
48
|
},
|
|
48
49
|
"devDependencies": {
|
|
49
50
|
"@sap/cds": "7.3.1",
|
|
@@ -55,7 +56,7 @@
|
|
|
55
56
|
"express": "4.18.2",
|
|
56
57
|
"hdb": "0.19.6",
|
|
57
58
|
"jest": "29.7.0",
|
|
58
|
-
"prettier": "
|
|
59
|
+
"prettier": "2.8.8",
|
|
59
60
|
"sqlite3": "5.1.6"
|
|
60
61
|
},
|
|
61
62
|
"homepage": "https://cap-js-community.github.io/event-queue/",
|
package/src/EventQueueError.js
CHANGED
|
@@ -11,6 +11,7 @@ const ERROR_CODES = {
|
|
|
11
11
|
MISSING_TABLE_DEFINITION: "MISSING_TABLE_DEFINITION",
|
|
12
12
|
MISSING_ELEMENT_IN_TABLE: "MISSING_ELEMENT_IN_TABLE",
|
|
13
13
|
TYPE_MISMATCH_TABLE: "TYPE_MISMATCH_TABLE",
|
|
14
|
+
NO_VALID_DATE: "NO_VALID_DATE",
|
|
14
15
|
};
|
|
15
16
|
|
|
16
17
|
const ERROR_CODES_META = {
|
|
@@ -39,6 +40,9 @@ const ERROR_CODES_META = {
|
|
|
39
40
|
[ERROR_CODES.TYPE_MISMATCH_TABLE]: {
|
|
40
41
|
message: "At least one field in the provided table doesn't have the expected data type.",
|
|
41
42
|
},
|
|
43
|
+
[ERROR_CODES.NO_VALID_DATE]: {
|
|
44
|
+
message: "One or more events contain a date in a malformed format.",
|
|
45
|
+
},
|
|
42
46
|
};
|
|
43
47
|
|
|
44
48
|
class EventQueueError extends VError {
|
|
@@ -131,6 +135,17 @@ class EventQueueError extends VError {
|
|
|
131
135
|
message
|
|
132
136
|
);
|
|
133
137
|
}
|
|
138
|
+
|
|
139
|
+
static malformedDate(date) {
|
|
140
|
+
const { message } = ERROR_CODES_META[ERROR_CODES.NO_VALID_DATE];
|
|
141
|
+
return new EventQueueError(
|
|
142
|
+
{
|
|
143
|
+
name: ERROR_CODES.NO_VALID_DATE,
|
|
144
|
+
info: { date },
|
|
145
|
+
},
|
|
146
|
+
message
|
|
147
|
+
);
|
|
148
|
+
}
|
|
134
149
|
}
|
|
135
150
|
|
|
136
151
|
module.exports = EventQueueError;
|
|
@@ -7,6 +7,7 @@ const { EventProcessingStatus, TransactionMode } = require("./constants");
|
|
|
7
7
|
const distributedLock = require("./shared/distributedLock");
|
|
8
8
|
const EventQueueError = require("./EventQueueError");
|
|
9
9
|
const { arrayToFlatMap } = require("./shared/common");
|
|
10
|
+
const eventScheduler = require("./shared/EventScheduler");
|
|
10
11
|
const eventQueueConfig = require("./config");
|
|
11
12
|
const PerformanceTracer = require("./shared/PerformanceTracer");
|
|
12
13
|
|
|
@@ -20,12 +21,16 @@ const SELECT_LIMIT_EVENTS_PER_TICK = 100;
|
|
|
20
21
|
const DEFAULT_DELETE_FINISHED_EVENTS_AFTER = 0;
|
|
21
22
|
const DAYS_TO_MS = 24 * 60 * 60 * 1000;
|
|
22
23
|
const TRIES_FOR_EXCEEDED_EVENTS = 3;
|
|
24
|
+
const EVENT_START_AFTER_HEADROOM = 3 * 1000;
|
|
23
25
|
|
|
24
26
|
class EventQueueProcessorBase {
|
|
25
27
|
#eventsWithExceededTries = [];
|
|
26
28
|
#exceededTriesExceeded = [];
|
|
27
29
|
#selectedEventMap = {};
|
|
28
30
|
#queueEntriesWithPayloadMap = {};
|
|
31
|
+
#eventType = null;
|
|
32
|
+
#eventSubType = null;
|
|
33
|
+
#config = null;
|
|
29
34
|
|
|
30
35
|
constructor(context, eventType, eventSubType, config) {
|
|
31
36
|
this.__context = context;
|
|
@@ -36,26 +41,27 @@ class EventQueueProcessorBase {
|
|
|
36
41
|
this.__eventProcessingMap = {};
|
|
37
42
|
this.__statusMap = {};
|
|
38
43
|
this.__commitedStatusMap = {};
|
|
39
|
-
this
|
|
40
|
-
this
|
|
41
|
-
this.
|
|
42
|
-
this.__parallelEventProcessing = this.
|
|
44
|
+
this.#eventType = eventType;
|
|
45
|
+
this.#eventSubType = eventSubType;
|
|
46
|
+
this.__eventConfig = config ?? {};
|
|
47
|
+
this.__parallelEventProcessing = this.__eventConfig.parallelEventProcessing ?? DEFAULT_PARALLEL_EVENT_PROCESSING;
|
|
43
48
|
if (this.__parallelEventProcessing > LIMIT_PARALLEL_EVENT_PROCESSING) {
|
|
44
49
|
this.__parallelEventProcessing = LIMIT_PARALLEL_EVENT_PROCESSING;
|
|
45
50
|
}
|
|
46
51
|
// NOTE: keep the feature, this might be needed again
|
|
47
52
|
this.__concurrentEventProcessing = false;
|
|
48
|
-
this.__startTime = this.
|
|
49
|
-
this.__retryAttempts = this.
|
|
50
|
-
this.__selectMaxChunkSize = this.
|
|
51
|
-
this.__selectNextChunk = !!this.
|
|
53
|
+
this.__startTime = this.__eventConfig.startTime ?? new Date();
|
|
54
|
+
this.__retryAttempts = this.__eventConfig.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
|
|
55
|
+
this.__selectMaxChunkSize = this.__eventConfig.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
|
|
56
|
+
this.__selectNextChunk = !!this.__eventConfig.checkForNextChunk;
|
|
52
57
|
this.__keepalivePromises = {};
|
|
53
|
-
this.__outdatedCheckEnabled = this.
|
|
54
|
-
this.__transactionMode = this.
|
|
55
|
-
if (this.
|
|
58
|
+
this.__outdatedCheckEnabled = this.__eventConfig.eventOutdatedCheck ?? true;
|
|
59
|
+
this.__transactionMode = this.__eventConfig.transactionMode ?? TransactionMode.isolated;
|
|
60
|
+
if (this.__eventConfig.deleteFinishedEventsAfterDays) {
|
|
56
61
|
this.__deleteFinishedEventsAfter =
|
|
57
|
-
Number.isInteger(this.
|
|
58
|
-
|
|
62
|
+
Number.isInteger(this.__eventConfig.deleteFinishedEventsAfterDays) &&
|
|
63
|
+
this.__eventConfig.deleteFinishedEventsAfterDays > 0
|
|
64
|
+
? this.__eventConfig.deleteFinishedEventsAfterDays
|
|
59
65
|
: DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
|
|
60
66
|
} else {
|
|
61
67
|
this.__deleteFinishedEventsAfter = DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
|
|
@@ -65,7 +71,7 @@ class EventQueueProcessorBase {
|
|
|
65
71
|
this.__txUsageAllowed = true;
|
|
66
72
|
this.__txMap = {};
|
|
67
73
|
this.__txRollback = {};
|
|
68
|
-
this
|
|
74
|
+
this.#config = eventQueueConfig.getConfigInstance();
|
|
69
75
|
this.__queueEntries = [];
|
|
70
76
|
}
|
|
71
77
|
|
|
@@ -99,8 +105,8 @@ class EventQueueProcessorBase {
|
|
|
99
105
|
this.__performanceLoggerEvents?.endPerformanceTrace(
|
|
100
106
|
{ threshold: 50 },
|
|
101
107
|
{
|
|
102
|
-
eventType: this
|
|
103
|
-
eventSubType: this
|
|
108
|
+
eventType: this.#eventType,
|
|
109
|
+
eventSubType: this.#eventSubType,
|
|
104
110
|
}
|
|
105
111
|
);
|
|
106
112
|
}
|
|
@@ -109,16 +115,16 @@ class EventQueueProcessorBase {
|
|
|
109
115
|
this.__performanceLoggerPreprocessing?.endPerformanceTrace(
|
|
110
116
|
{ threshold: 50 },
|
|
111
117
|
{
|
|
112
|
-
eventType: this
|
|
113
|
-
eventSubType: this
|
|
118
|
+
eventType: this.#eventType,
|
|
119
|
+
eventSubType: this.#eventSubType,
|
|
114
120
|
}
|
|
115
121
|
);
|
|
116
122
|
}
|
|
117
123
|
|
|
118
124
|
logTimeExceeded(iterationCounter) {
|
|
119
125
|
this.logger.info("Exiting event queue processing as max time exceeded", {
|
|
120
|
-
eventType: this
|
|
121
|
-
eventSubType: this
|
|
126
|
+
eventType: this.#eventType,
|
|
127
|
+
eventSubType: this.#eventSubType,
|
|
122
128
|
iterationCounter,
|
|
123
129
|
});
|
|
124
130
|
}
|
|
@@ -127,8 +133,8 @@ class EventQueueProcessorBase {
|
|
|
127
133
|
// TODO: how to handle custom fields
|
|
128
134
|
this.logger.info("Processing queue event", {
|
|
129
135
|
numberQueueEntries: queueEntries.length,
|
|
130
|
-
eventType: this
|
|
131
|
-
eventSubType: this
|
|
136
|
+
eventType: this.#eventType,
|
|
137
|
+
eventSubType: this.#eventSubType,
|
|
132
138
|
});
|
|
133
139
|
}
|
|
134
140
|
|
|
@@ -157,8 +163,8 @@ class EventQueueProcessorBase {
|
|
|
157
163
|
this.logger.error(
|
|
158
164
|
"The supplied queueEntry has not been selected before and should not be processed. Entry will not be processed.",
|
|
159
165
|
{
|
|
160
|
-
eventType: this
|
|
161
|
-
eventSubType: this
|
|
166
|
+
eventType: this.#eventType,
|
|
167
|
+
eventSubType: this.#eventSubType,
|
|
162
168
|
queueEntryId: queueEntry.ID,
|
|
163
169
|
}
|
|
164
170
|
);
|
|
@@ -177,8 +183,8 @@ class EventQueueProcessorBase {
|
|
|
177
183
|
setStatusToDone(queueEntry) {
|
|
178
184
|
this.logger.debug("setting status for queueEntry to done", {
|
|
179
185
|
id: queueEntry.ID,
|
|
180
|
-
eventType: this
|
|
181
|
-
eventSubType: this
|
|
186
|
+
eventType: this.#eventType,
|
|
187
|
+
eventSubType: this.#eventSubType,
|
|
182
188
|
});
|
|
183
189
|
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Done);
|
|
184
190
|
}
|
|
@@ -207,8 +213,8 @@ class EventQueueProcessorBase {
|
|
|
207
213
|
this.logger.debug("add entry to processing map", {
|
|
208
214
|
key,
|
|
209
215
|
queueEntry,
|
|
210
|
-
eventType: this
|
|
211
|
-
eventSubType: this
|
|
216
|
+
eventType: this.#eventType,
|
|
217
|
+
eventSubType: this.#eventSubType,
|
|
212
218
|
});
|
|
213
219
|
this.__eventProcessingMap[key] = this.__eventProcessingMap[key] ?? {
|
|
214
220
|
queueEntries: [],
|
|
@@ -230,8 +236,8 @@ class EventQueueProcessorBase {
|
|
|
230
236
|
setEventStatus(queueEntries, queueEntryProcessingStatusTuple, returnMap = false) {
|
|
231
237
|
this.logger.debug("setting event status for entries", {
|
|
232
238
|
queueEntryProcessingStatusTuple: JSON.stringify(queueEntryProcessingStatusTuple),
|
|
233
|
-
eventType: this
|
|
234
|
-
eventSubType: this
|
|
239
|
+
eventType: this.#eventType,
|
|
240
|
+
eventSubType: this.#eventSubType,
|
|
235
241
|
});
|
|
236
242
|
const statusMap = this.commitOnEventLevel || returnMap ? {} : this.__statusMap;
|
|
237
243
|
try {
|
|
@@ -245,8 +251,8 @@ class EventQueueProcessorBase {
|
|
|
245
251
|
this.logger.error(
|
|
246
252
|
`The supplied status tuple doesn't have the required structure. Setting all entries to error. Error: ${error.toString()}`,
|
|
247
253
|
{
|
|
248
|
-
eventType: this
|
|
249
|
-
eventSubType: this
|
|
254
|
+
eventType: this.#eventType,
|
|
255
|
+
eventSubType: this.#eventSubType,
|
|
250
256
|
}
|
|
251
257
|
);
|
|
252
258
|
}
|
|
@@ -286,8 +292,8 @@ class EventQueueProcessorBase {
|
|
|
286
292
|
this.logger.error(
|
|
287
293
|
`Caught error during event processing - setting queue entry to error. Please catch your promises/exceptions. Error: ${error}`,
|
|
288
294
|
{
|
|
289
|
-
eventType: this
|
|
290
|
-
eventSubType: this
|
|
295
|
+
eventType: this.#eventType,
|
|
296
|
+
eventSubType: this.#eventSubType,
|
|
291
297
|
queueEntriesIds: queueEntries.map(({ ID }) => ID),
|
|
292
298
|
}
|
|
293
299
|
);
|
|
@@ -304,8 +310,8 @@ class EventQueueProcessorBase {
|
|
|
304
310
|
*/
|
|
305
311
|
async persistEventStatus(tx, { skipChecks, statusMap = this.__statusMap } = {}) {
|
|
306
312
|
this.logger.debug("entering persistEventStatus", {
|
|
307
|
-
eventType: this
|
|
308
|
-
eventSubType: this
|
|
313
|
+
eventType: this.#eventType,
|
|
314
|
+
eventSubType: this.#eventSubType,
|
|
309
315
|
});
|
|
310
316
|
this.#ensureOnlySelectedQueueEntries(statusMap);
|
|
311
317
|
if (!skipChecks) {
|
|
@@ -335,8 +341,8 @@ class EventQueueProcessorBase {
|
|
|
335
341
|
}
|
|
336
342
|
);
|
|
337
343
|
this.logger.debug("persistEventStatus for entries", {
|
|
338
|
-
eventType: this
|
|
339
|
-
eventSubType: this
|
|
344
|
+
eventType: this.#eventType,
|
|
345
|
+
eventSubType: this.#eventSubType,
|
|
340
346
|
invalidAttempts,
|
|
341
347
|
failed,
|
|
342
348
|
exceeded,
|
|
@@ -344,7 +350,7 @@ class EventQueueProcessorBase {
|
|
|
344
350
|
});
|
|
345
351
|
if (invalidAttempts.length) {
|
|
346
352
|
await tx.run(
|
|
347
|
-
UPDATE.entity(this.
|
|
353
|
+
UPDATE.entity(this.#config.tableNameEventQueue)
|
|
348
354
|
.set({
|
|
349
355
|
status: EventProcessingStatus.Open,
|
|
350
356
|
lastAttemptTimestamp: new Date().toISOString(),
|
|
@@ -365,7 +371,7 @@ class EventQueueProcessorBase {
|
|
|
365
371
|
continue;
|
|
366
372
|
}
|
|
367
373
|
await tx.run(
|
|
368
|
-
UPDATE.entity(this.
|
|
374
|
+
UPDATE.entity(this.#config.tableNameEventQueue)
|
|
369
375
|
.set({
|
|
370
376
|
status: status,
|
|
371
377
|
lastAttemptTimestamp: ts,
|
|
@@ -374,8 +380,8 @@ class EventQueueProcessorBase {
|
|
|
374
380
|
);
|
|
375
381
|
}
|
|
376
382
|
this.logger.debug("exiting persistEventStatus", {
|
|
377
|
-
eventType: this
|
|
378
|
-
eventSubType: this
|
|
383
|
+
eventType: this.#eventType,
|
|
384
|
+
eventSubType: this.#eventSubType,
|
|
379
385
|
});
|
|
380
386
|
}
|
|
381
387
|
|
|
@@ -384,18 +390,18 @@ class EventQueueProcessorBase {
|
|
|
384
390
|
return;
|
|
385
391
|
}
|
|
386
392
|
const deleteCount = await tx.run(
|
|
387
|
-
DELETE.from(this.
|
|
393
|
+
DELETE.from(this.#config.tableNameEventQueue).where(
|
|
388
394
|
"type =",
|
|
389
|
-
this
|
|
395
|
+
this.#eventType,
|
|
390
396
|
"AND subType=",
|
|
391
|
-
this
|
|
397
|
+
this.#eventSubType,
|
|
392
398
|
"AND lastAttemptTimestamp <=",
|
|
393
399
|
new Date(Date.now() - this.__deleteFinishedEventsAfter * DAYS_TO_MS).toISOString()
|
|
394
400
|
)
|
|
395
401
|
);
|
|
396
402
|
this.logger.debug("Deleted finished events", {
|
|
397
|
-
eventType: this
|
|
398
|
-
eventSubType: this
|
|
403
|
+
eventType: this.#eventType,
|
|
404
|
+
eventSubType: this.#eventSubType,
|
|
399
405
|
deleteFinishedEventsAfter: this.__deleteFinishedEventsAfter,
|
|
400
406
|
deleteCount,
|
|
401
407
|
});
|
|
@@ -407,8 +413,8 @@ class EventQueueProcessorBase {
|
|
|
407
413
|
return;
|
|
408
414
|
}
|
|
409
415
|
this.logger.error("Missing status for selected event entry. Setting status to error", {
|
|
410
|
-
eventType: this
|
|
411
|
-
eventSubType: this
|
|
416
|
+
eventType: this.#eventType,
|
|
417
|
+
eventSubType: this.#eventSubType,
|
|
412
418
|
queueEntry,
|
|
413
419
|
});
|
|
414
420
|
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
|
|
@@ -429,8 +435,8 @@ class EventQueueProcessorBase {
|
|
|
429
435
|
}
|
|
430
436
|
|
|
431
437
|
this.logger.error("Not allowed event status returned. Only Open, Done, Error is allowed!", {
|
|
432
|
-
eventType: this
|
|
433
|
-
eventSubType: this
|
|
438
|
+
eventType: this.#eventType,
|
|
439
|
+
eventSubType: this.#eventSubType,
|
|
434
440
|
queueEntryId,
|
|
435
441
|
status: statusMap[queueEntryId],
|
|
436
442
|
});
|
|
@@ -447,8 +453,8 @@ class EventQueueProcessorBase {
|
|
|
447
453
|
this.logger.error(
|
|
448
454
|
"Status reported for event queue entry which haven't be selected before. Removing the status.",
|
|
449
455
|
{
|
|
450
|
-
eventType: this
|
|
451
|
-
eventSubType: this
|
|
456
|
+
eventType: this.#eventType,
|
|
457
|
+
eventSubType: this.#eventSubType,
|
|
452
458
|
queueEntryId,
|
|
453
459
|
}
|
|
454
460
|
);
|
|
@@ -458,8 +464,8 @@ class EventQueueProcessorBase {
|
|
|
458
464
|
|
|
459
465
|
handleErrorDuringClustering(error) {
|
|
460
466
|
this.logger.error(`Error during clustering of events - setting all queue entries to error. Error: ${error}`, {
|
|
461
|
-
eventType: this
|
|
462
|
-
eventSubType: this
|
|
467
|
+
eventType: this.#eventType,
|
|
468
|
+
eventSubType: this.#eventSubType,
|
|
463
469
|
});
|
|
464
470
|
this.__queueEntries.forEach((queueEntry) => {
|
|
465
471
|
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
|
|
@@ -471,21 +477,13 @@ class EventQueueProcessorBase {
|
|
|
471
477
|
"Undefined payload is not allowed. If status should be done, nulls needs to be returned" +
|
|
472
478
|
" - setting queue entry to error",
|
|
473
479
|
{
|
|
474
|
-
eventType: this
|
|
475
|
-
eventSubType: this
|
|
480
|
+
eventType: this.#eventType,
|
|
481
|
+
eventSubType: this.#eventSubType,
|
|
476
482
|
}
|
|
477
483
|
);
|
|
478
484
|
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
|
|
479
485
|
}
|
|
480
486
|
|
|
481
|
-
static async handleMissingTypeImplementation(context, eventType, eventSubType) {
|
|
482
|
-
const baseInstance = new EventQueueProcessorBase(context, eventType, eventSubType);
|
|
483
|
-
baseInstance.logger.error("No Implementation found in the provided configuration file.", {
|
|
484
|
-
eventType,
|
|
485
|
-
eventSubType,
|
|
486
|
-
});
|
|
487
|
-
}
|
|
488
|
-
|
|
489
487
|
/**
|
|
490
488
|
* This function selects all relevant events based on the eventType and eventSubType supplied through the constructor
|
|
491
489
|
* during initialization of the class.
|
|
@@ -495,17 +493,20 @@ class EventQueueProcessorBase {
|
|
|
495
493
|
*/
|
|
496
494
|
async getQueueEntriesAndSetToInProgress() {
|
|
497
495
|
let result = [];
|
|
496
|
+
const refDateStartAfter = new Date(Date.now() + this.#config.runInterval);
|
|
498
497
|
await executeInNewTransaction(this.__baseContext, "eventQueue-getQueueEntriesAndSetToInProgress", async (tx) => {
|
|
499
498
|
const entries = await tx.run(
|
|
500
|
-
SELECT.from(this.
|
|
501
|
-
.forUpdate({ wait: this.
|
|
499
|
+
SELECT.from(this.#config.tableNameEventQueue)
|
|
500
|
+
.forUpdate({ wait: this.#config.forUpdateTimeout })
|
|
502
501
|
.limit(this.selectMaxChunkSize)
|
|
503
502
|
.where(
|
|
504
503
|
"type =",
|
|
505
|
-
this
|
|
504
|
+
this.#eventType,
|
|
506
505
|
"AND subType=",
|
|
507
|
-
this
|
|
508
|
-
"AND (
|
|
506
|
+
this.#eventSubType,
|
|
507
|
+
"AND ( startAfter IS NULL OR startAfter <=",
|
|
508
|
+
refDateStartAfter.toISOString(),
|
|
509
|
+
" ) AND ( status =",
|
|
509
510
|
EventProcessingStatus.Open,
|
|
510
511
|
"OR ( status =",
|
|
511
512
|
EventProcessingStatus.Error,
|
|
@@ -514,7 +515,7 @@ class EventQueueProcessorBase {
|
|
|
514
515
|
") OR ( status =",
|
|
515
516
|
EventProcessingStatus.InProgress,
|
|
516
517
|
"AND lastAttemptTimestamp <=",
|
|
517
|
-
new Date(new Date().getTime() - this.
|
|
518
|
+
new Date(new Date().getTime() - this.#config.globalTxTimeout).toISOString(),
|
|
518
519
|
") )"
|
|
519
520
|
)
|
|
520
521
|
.orderBy("createdAt", "ID")
|
|
@@ -522,37 +523,44 @@ class EventQueueProcessorBase {
|
|
|
522
523
|
|
|
523
524
|
if (!entries.length) {
|
|
524
525
|
this.logger.debug("no entries available for processing", {
|
|
525
|
-
eventType: this
|
|
526
|
-
eventSubType: this
|
|
526
|
+
eventType: this.#eventType,
|
|
527
|
+
eventSubType: this.#eventSubType,
|
|
527
528
|
});
|
|
528
529
|
this.__emptyChunkSelected = true;
|
|
529
530
|
return;
|
|
530
531
|
}
|
|
531
532
|
|
|
532
|
-
|
|
533
|
-
|
|
533
|
+
const { exceededTries, openEvents, exceededTriesExceeded, delayedEvents } = this.#clusterEvents(
|
|
534
|
+
entries,
|
|
535
|
+
refDateStartAfter
|
|
536
|
+
);
|
|
537
|
+
const eventsForProcessing = exceededTries.concat(openEvents).concat(exceededTriesExceeded);
|
|
538
|
+
this.#selectedEventMap = arrayToFlatMap(eventsForProcessing);
|
|
534
539
|
if (exceededTries.length) {
|
|
535
540
|
this.#eventsWithExceededTries = exceededTries;
|
|
536
541
|
}
|
|
537
542
|
if (exceededTriesExceeded.length) {
|
|
538
543
|
this.#exceededTriesExceeded = exceededTriesExceeded;
|
|
539
544
|
}
|
|
545
|
+
this.#handleDelayedEvents(delayedEvents);
|
|
540
546
|
|
|
541
547
|
result = openEvents;
|
|
548
|
+
this.logger.info("Selected event queue entries for processing", {
|
|
549
|
+
openEvents: openEvents.length,
|
|
550
|
+
...(delayedEvents.length && { delayedEvents: delayedEvents.length }),
|
|
551
|
+
...(exceededTries.length && { exceededTries: exceededTries.length }),
|
|
552
|
+
eventType: this.#eventType,
|
|
553
|
+
eventSubType: this.#eventSubType,
|
|
554
|
+
});
|
|
542
555
|
|
|
543
|
-
if (!
|
|
556
|
+
if (!eventsForProcessing.length) {
|
|
544
557
|
this.__emptyChunkSelected = true;
|
|
558
|
+
return;
|
|
545
559
|
}
|
|
546
560
|
|
|
547
|
-
this.logger.info("Selected event queue entries for processing", {
|
|
548
|
-
queueEntriesCount: result.length,
|
|
549
|
-
eventType: this.__eventType,
|
|
550
|
-
eventSubType: this.__eventSubType,
|
|
551
|
-
});
|
|
552
|
-
|
|
553
561
|
const isoTimestamp = new Date().toISOString();
|
|
554
562
|
await tx.run(
|
|
555
|
-
UPDATE.entity(this.
|
|
563
|
+
UPDATE.entity(this.#config.tableNameEventQueue)
|
|
556
564
|
.with({
|
|
557
565
|
status: EventProcessingStatus.InProgress,
|
|
558
566
|
lastAttemptTimestamp: isoTimestamp,
|
|
@@ -560,10 +568,10 @@ class EventQueueProcessorBase {
|
|
|
560
568
|
})
|
|
561
569
|
.where(
|
|
562
570
|
"ID IN",
|
|
563
|
-
|
|
571
|
+
eventsForProcessing.map(({ ID }) => ID)
|
|
564
572
|
)
|
|
565
573
|
);
|
|
566
|
-
|
|
574
|
+
eventsForProcessing.forEach((entry) => {
|
|
567
575
|
entry.lastAttemptTimestamp = isoTimestamp;
|
|
568
576
|
// NOTE: empty payloads are supported on DB-Level.
|
|
569
577
|
// Behaviour of event queue is: null as payload is treated as obsolete/done
|
|
@@ -578,22 +586,45 @@ class EventQueueProcessorBase {
|
|
|
578
586
|
return result;
|
|
579
587
|
}
|
|
580
588
|
|
|
581
|
-
#
|
|
589
|
+
#handleDelayedEvents(delayedEvents) {
|
|
590
|
+
const eventSchedulerInstance = eventScheduler.getInstance();
|
|
591
|
+
for (const delayedEvent of delayedEvents) {
|
|
592
|
+
eventSchedulerInstance.scheduleEvent(
|
|
593
|
+
this.__context.tenant,
|
|
594
|
+
this.#eventType,
|
|
595
|
+
this.#eventSubType,
|
|
596
|
+
delayedEvent.startAfter
|
|
597
|
+
);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
#clusterEvents(events, refDateStartAfter) {
|
|
602
|
+
const refDate = new Date(refDateStartAfter.getTime() - this.#config.runInterval + EVENT_START_AFTER_HEADROOM);
|
|
582
603
|
return events.reduce(
|
|
583
604
|
(result, event) => {
|
|
584
605
|
if (event.attempts === this.__retryAttempts + TRIES_FOR_EXCEEDED_EVENTS) {
|
|
585
606
|
result.exceededTriesExceeded.push(event);
|
|
586
607
|
} else if (event.attempts >= this.__retryAttempts) {
|
|
587
608
|
result.exceededTries.push(event);
|
|
609
|
+
} else if (this.#isDelayedEvent(event, refDate)) {
|
|
610
|
+
result.delayedEvents.push(event);
|
|
588
611
|
} else {
|
|
589
612
|
result.openEvents.push(event);
|
|
590
613
|
}
|
|
591
614
|
return result;
|
|
592
615
|
},
|
|
593
|
-
{ exceededTries: [], openEvents: [], exceededTriesExceeded: [] }
|
|
616
|
+
{ exceededTries: [], openEvents: [], exceededTriesExceeded: [], delayedEvents: [] }
|
|
594
617
|
);
|
|
595
618
|
}
|
|
596
619
|
|
|
620
|
+
#isDelayedEvent(event, refDate) {
|
|
621
|
+
if (!event.startAfter) {
|
|
622
|
+
return false;
|
|
623
|
+
}
|
|
624
|
+
event.startAfter = new Date(event.startAfter);
|
|
625
|
+
return !(refDate >= event.startAfter);
|
|
626
|
+
}
|
|
627
|
+
|
|
597
628
|
async handleExceededEvents() {
|
|
598
629
|
await this.#handleExceededTriesExceeded();
|
|
599
630
|
if (!this.#eventsWithExceededTries.length) {
|
|
@@ -603,15 +634,15 @@ class EventQueueProcessorBase {
|
|
|
603
634
|
for (const exceededEvent of this.#eventsWithExceededTries) {
|
|
604
635
|
await executeInNewTransaction(
|
|
605
636
|
this.context,
|
|
606
|
-
`eventQueue-handleExceededEvents-${this
|
|
637
|
+
`eventQueue-handleExceededEvents-${this.#eventType}##${this.#eventSubType}`,
|
|
607
638
|
async (tx) => {
|
|
608
639
|
try {
|
|
609
640
|
this.processEventContext = tx.context;
|
|
610
641
|
this.modifyQueueEntry(exceededEvent);
|
|
611
642
|
await this.hookForExceededEvents({ ...exceededEvent });
|
|
612
643
|
this.logger.warn("The retry attempts for the following events are exceeded", {
|
|
613
|
-
eventType: this
|
|
614
|
-
eventSubType: this
|
|
644
|
+
eventType: this.#eventType,
|
|
645
|
+
eventSubType: this.#eventSubType,
|
|
615
646
|
retryAttempts: this.__retryAttempts,
|
|
616
647
|
queueEntriesId: exceededEvent.ID,
|
|
617
648
|
currentAttempt: exceededEvent.attempts,
|
|
@@ -621,8 +652,8 @@ class EventQueueProcessorBase {
|
|
|
621
652
|
this.logger.error(
|
|
622
653
|
`Caught error during hook for exceeded events - setting queue entry to error. Please catch your promises/exceptions. Error: ${err}`,
|
|
623
654
|
{
|
|
624
|
-
eventType: this
|
|
625
|
-
eventSubType: this
|
|
655
|
+
eventType: this.#eventType,
|
|
656
|
+
eventSubType: this.#eventSubType,
|
|
626
657
|
retryAttempts: this.__retryAttempts,
|
|
627
658
|
queueEntriesId: exceededEvent.ID,
|
|
628
659
|
currentAttempt: exceededEvent.attempts,
|
|
@@ -641,8 +672,8 @@ class EventQueueProcessorBase {
|
|
|
641
672
|
async #handleExceededTriesExceeded() {
|
|
642
673
|
if (this.#exceededTriesExceeded.length) {
|
|
643
674
|
this.logger.error("Event hook failure exceeded, status set to 'exceeded' without invoking hook again!", {
|
|
644
|
-
eventType: this
|
|
645
|
-
eventSubType: this
|
|
675
|
+
eventType: this.#eventType,
|
|
676
|
+
eventSubType: this.#eventSubType,
|
|
646
677
|
queueEntriesIds: this.#eventsWithExceededTries.map(({ ID }) => ID),
|
|
647
678
|
});
|
|
648
679
|
await executeInNewTransaction(this.context, "exceededTriesExceeded", async (tx) => {
|
|
@@ -705,8 +736,8 @@ class EventQueueProcessorBase {
|
|
|
705
736
|
const checkAndUpdatePromise = new Promise((resolve, reject) => {
|
|
706
737
|
executeInNewTransaction(this.__baseContext, "eventProcessing-isOutdatedAndKeepalive", async (tx) => {
|
|
707
738
|
const queueEntriesFresh = await tx.run(
|
|
708
|
-
SELECT.from(this.
|
|
709
|
-
.forUpdate({ wait: this.
|
|
739
|
+
SELECT.from(this.#config.tableNameEventQueue)
|
|
740
|
+
.forUpdate({ wait: this.#config.forUpdateTimeout })
|
|
710
741
|
.where(
|
|
711
742
|
"ID IN",
|
|
712
743
|
queueEntries.map(({ ID }) => ID)
|
|
@@ -720,7 +751,7 @@ class EventQueueProcessorBase {
|
|
|
720
751
|
let newTs = new Date().toISOString();
|
|
721
752
|
if (!eventOutdated) {
|
|
722
753
|
await tx.run(
|
|
723
|
-
UPDATE.entity(this.
|
|
754
|
+
UPDATE.entity(this.#config.tableNameEventQueue)
|
|
724
755
|
.set("lastAttemptTimestamp =", newTs)
|
|
725
756
|
.where(
|
|
726
757
|
"ID IN",
|
|
@@ -730,8 +761,8 @@ class EventQueueProcessorBase {
|
|
|
730
761
|
} else {
|
|
731
762
|
newTs = null;
|
|
732
763
|
this.logger.warn("event data has been modified. Processing skipped.", {
|
|
733
|
-
eventType: this
|
|
734
|
-
eventSubType: this
|
|
764
|
+
eventType: this.#eventType,
|
|
765
|
+
eventSubType: this.#eventSubType,
|
|
735
766
|
queueEntriesIds: queueEntries.map(({ ID }) => ID),
|
|
736
767
|
});
|
|
737
768
|
queueEntries.forEach(({ ID: queueEntryId }) => delete this.__queueEntriesMap[queueEntryId]);
|
|
@@ -761,7 +792,7 @@ class EventQueueProcessorBase {
|
|
|
761
792
|
|
|
762
793
|
const lockAcquired = await distributedLock.acquireLock(
|
|
763
794
|
this.context,
|
|
764
|
-
[this
|
|
795
|
+
[this.#eventType, this.#eventSubType].join("##")
|
|
765
796
|
);
|
|
766
797
|
if (!lockAcquired) {
|
|
767
798
|
return false;
|
|
@@ -775,7 +806,7 @@ class EventQueueProcessorBase {
|
|
|
775
806
|
return;
|
|
776
807
|
}
|
|
777
808
|
try {
|
|
778
|
-
await distributedLock.releaseLock(this.context, [this
|
|
809
|
+
await distributedLock.releaseLock(this.context, [this.#eventType, this.#eventSubType].join("##"));
|
|
779
810
|
} catch (err) {
|
|
780
811
|
this.logger.error("Releasing distributed lock failed. Error:", err.toString());
|
|
781
812
|
}
|
|
@@ -822,14 +853,14 @@ class EventQueueProcessorBase {
|
|
|
822
853
|
|
|
823
854
|
get tx() {
|
|
824
855
|
if (!this.__txUsageAllowed && this.__parallelEventProcessing > 1) {
|
|
825
|
-
throw EventQueueError.wrongTxUsage(this
|
|
856
|
+
throw EventQueueError.wrongTxUsage(this.#eventType, this.#eventSubType);
|
|
826
857
|
}
|
|
827
858
|
return this.__processTx ?? this.__tx;
|
|
828
859
|
}
|
|
829
860
|
|
|
830
861
|
get context() {
|
|
831
862
|
if (!this.__txUsageAllowed && this.__parallelEventProcessing > 1) {
|
|
832
|
-
throw EventQueueError.wrongTxUsage(this
|
|
863
|
+
throw EventQueueError.wrongTxUsage(this.#eventType, this.#eventSubType);
|
|
833
864
|
}
|
|
834
865
|
return this.__processContext ?? this.__context;
|
|
835
866
|
}
|
|
@@ -847,11 +878,11 @@ class EventQueueProcessorBase {
|
|
|
847
878
|
}
|
|
848
879
|
|
|
849
880
|
get eventType() {
|
|
850
|
-
return this
|
|
881
|
+
return this.#eventType;
|
|
851
882
|
}
|
|
852
883
|
|
|
853
884
|
get eventSubType() {
|
|
854
|
-
return this
|
|
885
|
+
return this.#eventSubType;
|
|
855
886
|
}
|
|
856
887
|
|
|
857
888
|
get emptyChunkSelected() {
|
package/src/config.js
CHANGED
|
@@ -13,38 +13,56 @@ const REDIS_CONFIG_CHANNEL = "EVENT_QUEUE_CONFIG_CHANNEL";
|
|
|
13
13
|
const COMPONENT_NAME = "eventQueue/config";
|
|
14
14
|
|
|
15
15
|
class Config {
|
|
16
|
+
#logger;
|
|
17
|
+
#config;
|
|
18
|
+
#forUpdateTimeout;
|
|
19
|
+
#globalTxTimeout;
|
|
20
|
+
#runInterval;
|
|
21
|
+
#redisEnabled;
|
|
22
|
+
#initialized;
|
|
23
|
+
#parallelTenantProcessing;
|
|
24
|
+
#tableNameEventQueue;
|
|
25
|
+
#tableNameEventLock;
|
|
26
|
+
#isRunnerDeactivated;
|
|
27
|
+
#configFilePath;
|
|
28
|
+
#processEventsAfterPublish;
|
|
29
|
+
#skipCsnCheck;
|
|
30
|
+
#disableRedis;
|
|
31
|
+
#env;
|
|
32
|
+
#eventMap;
|
|
16
33
|
constructor() {
|
|
17
|
-
this
|
|
18
|
-
this
|
|
19
|
-
this
|
|
20
|
-
this
|
|
21
|
-
this
|
|
22
|
-
this
|
|
23
|
-
this
|
|
24
|
-
this
|
|
25
|
-
this
|
|
26
|
-
this
|
|
27
|
-
this
|
|
28
|
-
this
|
|
29
|
-
this
|
|
30
|
-
this
|
|
31
|
-
this
|
|
34
|
+
this.#logger = cds.log(COMPONENT_NAME);
|
|
35
|
+
this.#config = null;
|
|
36
|
+
this.#forUpdateTimeout = FOR_UPDATE_TIMEOUT;
|
|
37
|
+
this.#globalTxTimeout = GLOBAL_TX_TIMEOUT;
|
|
38
|
+
this.#runInterval = null;
|
|
39
|
+
this.#redisEnabled = null;
|
|
40
|
+
this.#initialized = false;
|
|
41
|
+
this.#parallelTenantProcessing = null;
|
|
42
|
+
this.#tableNameEventQueue = null;
|
|
43
|
+
this.#tableNameEventLock = null;
|
|
44
|
+
this.#isRunnerDeactivated = false;
|
|
45
|
+
this.#configFilePath = null;
|
|
46
|
+
this.#processEventsAfterPublish = null;
|
|
47
|
+
this.#skipCsnCheck = null;
|
|
48
|
+
this.#disableRedis = null;
|
|
49
|
+
this.#env = getEnvInstance();
|
|
32
50
|
}
|
|
33
51
|
|
|
34
52
|
getEventConfig(type, subType) {
|
|
35
|
-
return this
|
|
53
|
+
return this.#eventMap[[type, subType].join("##")];
|
|
36
54
|
}
|
|
37
55
|
|
|
38
56
|
hasEventAfterCommitFlag(type, subType) {
|
|
39
|
-
return this
|
|
57
|
+
return this.#eventMap[[type, subType].join("##")]?.processAfterCommit ?? true;
|
|
40
58
|
}
|
|
41
59
|
|
|
42
60
|
_checkRedisIsBound() {
|
|
43
|
-
return !!this.
|
|
61
|
+
return !!this.#env.getRedisCredentialsFromEnv();
|
|
44
62
|
}
|
|
45
63
|
|
|
46
64
|
checkRedisEnabled() {
|
|
47
|
-
this
|
|
65
|
+
this.#redisEnabled = !this.#disableRedis && this._checkRedisIsBound() && this.#env.isOnCF;
|
|
48
66
|
}
|
|
49
67
|
|
|
50
68
|
attachConfigChangeHandler() {
|
|
@@ -52,11 +70,11 @@ class Config {
|
|
|
52
70
|
try {
|
|
53
71
|
const { key, value } = JSON.parse(messageData);
|
|
54
72
|
if (this[key] !== value) {
|
|
55
|
-
this.
|
|
73
|
+
this.#logger.info("received config change", { key, value });
|
|
56
74
|
this[key] = value;
|
|
57
75
|
}
|
|
58
76
|
} catch (err) {
|
|
59
|
-
this.
|
|
77
|
+
this.#logger.error("could not parse event config change", {
|
|
60
78
|
messageData,
|
|
61
79
|
});
|
|
62
80
|
}
|
|
@@ -65,124 +83,132 @@ class Config {
|
|
|
65
83
|
|
|
66
84
|
publishConfigChange(key, value) {
|
|
67
85
|
if (!this.redisEnabled) {
|
|
68
|
-
this.
|
|
86
|
+
this.#logger.info("redis not connected, config change won't be published", { key, value });
|
|
69
87
|
return;
|
|
70
88
|
}
|
|
71
89
|
redis.publishMessage(REDIS_CONFIG_CHANNEL, JSON.stringify({ key, value })).catch((error) => {
|
|
72
|
-
this.
|
|
90
|
+
this.#logger.error(`publishing config change failed key: ${key}, value: ${value}`, error);
|
|
73
91
|
});
|
|
74
92
|
}
|
|
75
93
|
|
|
76
94
|
get isRunnerDeactivated() {
|
|
77
|
-
return this
|
|
95
|
+
return this.#isRunnerDeactivated;
|
|
78
96
|
}
|
|
79
97
|
|
|
80
98
|
set isRunnerDeactivated(value) {
|
|
81
|
-
this
|
|
99
|
+
this.#isRunnerDeactivated = value;
|
|
82
100
|
}
|
|
83
101
|
|
|
84
102
|
set fileContent(config) {
|
|
85
|
-
this
|
|
86
|
-
this
|
|
103
|
+
this.#config = config;
|
|
104
|
+
this.#eventMap = config.events.reduce((result, event) => {
|
|
87
105
|
result[[event.type, event.subType].join("##")] = event;
|
|
88
106
|
return result;
|
|
89
107
|
}, {});
|
|
90
108
|
}
|
|
91
109
|
|
|
92
110
|
get fileContent() {
|
|
93
|
-
return this
|
|
111
|
+
return this.#config;
|
|
94
112
|
}
|
|
95
113
|
|
|
96
114
|
get events() {
|
|
97
|
-
return this.
|
|
115
|
+
return this.#config.events;
|
|
98
116
|
}
|
|
99
117
|
|
|
100
118
|
get forUpdateTimeout() {
|
|
101
|
-
return this
|
|
119
|
+
return this.#forUpdateTimeout;
|
|
102
120
|
}
|
|
103
121
|
|
|
104
122
|
get globalTxTimeout() {
|
|
105
|
-
return this
|
|
123
|
+
return this.#globalTxTimeout;
|
|
106
124
|
}
|
|
107
125
|
|
|
108
126
|
set forUpdateTimeout(value) {
|
|
109
|
-
this
|
|
127
|
+
this.#forUpdateTimeout = value;
|
|
110
128
|
}
|
|
111
129
|
|
|
112
130
|
set globalTxTimeout(value) {
|
|
113
|
-
this
|
|
131
|
+
this.#globalTxTimeout = value;
|
|
114
132
|
}
|
|
115
133
|
|
|
116
134
|
get runInterval() {
|
|
117
|
-
return this
|
|
135
|
+
return this.#runInterval;
|
|
118
136
|
}
|
|
119
137
|
|
|
120
138
|
set runInterval(value) {
|
|
121
|
-
this
|
|
139
|
+
this.#runInterval = value;
|
|
122
140
|
}
|
|
123
141
|
|
|
124
142
|
get redisEnabled() {
|
|
125
|
-
return this
|
|
143
|
+
return this.#redisEnabled;
|
|
126
144
|
}
|
|
127
145
|
|
|
128
146
|
set redisEnabled(value) {
|
|
129
|
-
this
|
|
147
|
+
this.#redisEnabled = value;
|
|
130
148
|
}
|
|
131
149
|
|
|
132
150
|
get initialized() {
|
|
133
|
-
return this
|
|
151
|
+
return this.#initialized;
|
|
134
152
|
}
|
|
135
153
|
|
|
136
154
|
set initialized(value) {
|
|
137
|
-
this
|
|
155
|
+
this.#initialized = value;
|
|
138
156
|
}
|
|
139
157
|
|
|
140
158
|
get parallelTenantProcessing() {
|
|
141
|
-
return this
|
|
159
|
+
return this.#parallelTenantProcessing;
|
|
142
160
|
}
|
|
143
161
|
|
|
144
162
|
set parallelTenantProcessing(value) {
|
|
145
|
-
this
|
|
163
|
+
this.#parallelTenantProcessing = value;
|
|
146
164
|
}
|
|
147
165
|
|
|
148
166
|
get tableNameEventQueue() {
|
|
149
|
-
return this
|
|
167
|
+
return this.#tableNameEventQueue;
|
|
150
168
|
}
|
|
151
169
|
|
|
152
170
|
set tableNameEventQueue(value) {
|
|
153
|
-
this
|
|
171
|
+
this.#tableNameEventQueue = value;
|
|
154
172
|
}
|
|
155
173
|
|
|
156
174
|
get tableNameEventLock() {
|
|
157
|
-
return this
|
|
175
|
+
return this.#tableNameEventLock;
|
|
158
176
|
}
|
|
159
177
|
|
|
160
178
|
set tableNameEventLock(value) {
|
|
161
|
-
this
|
|
179
|
+
this.#tableNameEventLock = value;
|
|
162
180
|
}
|
|
163
181
|
|
|
164
182
|
set configFilePath(value) {
|
|
165
|
-
this
|
|
183
|
+
this.#configFilePath = value;
|
|
166
184
|
}
|
|
167
185
|
|
|
168
186
|
get configFilePath() {
|
|
169
|
-
return this
|
|
187
|
+
return this.#configFilePath;
|
|
170
188
|
}
|
|
171
189
|
|
|
172
190
|
set processEventsAfterPublish(value) {
|
|
173
|
-
this
|
|
191
|
+
this.#processEventsAfterPublish = value;
|
|
174
192
|
}
|
|
175
193
|
|
|
176
194
|
get processEventsAfterPublish() {
|
|
177
|
-
return this
|
|
195
|
+
return this.#processEventsAfterPublish;
|
|
178
196
|
}
|
|
179
197
|
|
|
180
198
|
set skipCsnCheck(value) {
|
|
181
|
-
this
|
|
199
|
+
this.#skipCsnCheck = value;
|
|
182
200
|
}
|
|
183
201
|
|
|
184
202
|
get skipCsnCheck() {
|
|
185
|
-
return this
|
|
203
|
+
return this.#skipCsnCheck;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
set disableRedis(value) {
|
|
207
|
+
this.#disableRedis = value;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
get disableRedis() {
|
|
211
|
+
return this.#disableRedis;
|
|
186
212
|
}
|
|
187
213
|
|
|
188
214
|
get isMultiTenancy() {
|
package/src/dbHandler.js
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
-
const
|
|
3
|
+
const cds = require("@sap/cds");
|
|
4
|
+
|
|
5
|
+
const { broadcastEvent } = require("./redisPubSub");
|
|
4
6
|
const config = require("./config");
|
|
5
7
|
|
|
8
|
+
const COMPONENT_NAME = "eventQueue/dbHandler";
|
|
9
|
+
|
|
6
10
|
const registerEventQueueDbHandler = (dbService) => {
|
|
7
11
|
const configInstance = config.getConfigInstance();
|
|
8
12
|
const def = dbService.model.definitions[configInstance.tableNameEventQueue];
|
|
@@ -26,7 +30,12 @@ const registerEventQueueDbHandler = (dbService) => {
|
|
|
26
30
|
eventCombinations.length &&
|
|
27
31
|
req.on("succeeded", () => {
|
|
28
32
|
for (const eventCombination of eventCombinations) {
|
|
29
|
-
|
|
33
|
+
broadcastEvent(req.tenant, ...eventCombination.split("##")).catch((err) => {
|
|
34
|
+
cds.log(COMPONENT_NAME).error("db handler failure during broadcasting event", err, {
|
|
35
|
+
tenant: req.tenant,
|
|
36
|
+
eventCombination,
|
|
37
|
+
});
|
|
38
|
+
});
|
|
30
39
|
}
|
|
31
40
|
});
|
|
32
41
|
});
|
package/src/initialize.js
CHANGED
|
@@ -5,7 +5,6 @@ const fs = require("fs");
|
|
|
5
5
|
const path = require("path");
|
|
6
6
|
|
|
7
7
|
const cds = require("@sap/cds");
|
|
8
|
-
|
|
9
8
|
const yaml = require("yaml");
|
|
10
9
|
const VError = require("verror");
|
|
11
10
|
|
|
@@ -23,6 +22,18 @@ const BASE_TABLES = {
|
|
|
23
22
|
EVENT: "sap.eventqueue.Event",
|
|
24
23
|
LOCK: "sap.eventqueue.Lock",
|
|
25
24
|
};
|
|
25
|
+
const CONFIG_VARS = [
|
|
26
|
+
["configFilePath", null],
|
|
27
|
+
["registerAsEventProcessor", true],
|
|
28
|
+
["processEventsAfterPublish", true],
|
|
29
|
+
["isRunnerDeactivated", false],
|
|
30
|
+
["runInterval", 5 * 60 * 1000],
|
|
31
|
+
["parallelTenantProcessing", 5],
|
|
32
|
+
["tableNameEventQueue", BASE_TABLES.EVENT],
|
|
33
|
+
["tableNameEventLock", BASE_TABLES.LOCK],
|
|
34
|
+
["disableRedis", false],
|
|
35
|
+
["skipCsnCheck", false],
|
|
36
|
+
];
|
|
26
37
|
|
|
27
38
|
const initialize = async ({
|
|
28
39
|
configFilePath,
|
|
@@ -33,6 +44,7 @@ const initialize = async ({
|
|
|
33
44
|
parallelTenantProcessing,
|
|
34
45
|
tableNameEventQueue,
|
|
35
46
|
tableNameEventLock,
|
|
47
|
+
disableRedis,
|
|
36
48
|
skipCsnCheck,
|
|
37
49
|
} = {}) => {
|
|
38
50
|
// TODO: initialize check:
|
|
@@ -54,6 +66,7 @@ const initialize = async ({
|
|
|
54
66
|
parallelTenantProcessing,
|
|
55
67
|
tableNameEventQueue,
|
|
56
68
|
tableNameEventLock,
|
|
69
|
+
disableRedis,
|
|
57
70
|
skipCsnCheck
|
|
58
71
|
);
|
|
59
72
|
|
|
@@ -160,32 +173,12 @@ const checkCustomTable = (baseCsn, customCsn) => {
|
|
|
160
173
|
}
|
|
161
174
|
};
|
|
162
175
|
|
|
163
|
-
const mixConfigVarsWithEnv = (
|
|
164
|
-
configFilePath,
|
|
165
|
-
registerAsEventProcessor,
|
|
166
|
-
processEventsAfterPublish,
|
|
167
|
-
isRunnerDeactivated,
|
|
168
|
-
runInterval,
|
|
169
|
-
parallelTenantProcessing,
|
|
170
|
-
tableNameEventQueue,
|
|
171
|
-
tableNameEventLock,
|
|
172
|
-
skipCsnCheck
|
|
173
|
-
) => {
|
|
176
|
+
const mixConfigVarsWithEnv = (...args) => {
|
|
174
177
|
const configInstance = getConfigInstance();
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
configInstance.isRunnerDeactivated = isRunnerDeactivated ?? cds.env.eventQueue?.isRunnerDeactivated ?? false;
|
|
180
|
-
configInstance.processEventsAfterPublish =
|
|
181
|
-
processEventsAfterPublish ?? cds.env.eventQueue?.processEventsAfterPublish ?? true;
|
|
182
|
-
configInstance.runInterval = runInterval ?? cds.env.eventQueue?.runInterval ?? 5 * 60 * 1000;
|
|
183
|
-
configInstance.parallelTenantProcessing =
|
|
184
|
-
parallelTenantProcessing ?? cds.env.eventQueue?.parallelTenantProcessing ?? 5;
|
|
185
|
-
configInstance.tableNameEventQueue =
|
|
186
|
-
tableNameEventQueue ?? cds.env.eventQueue?.tableNameEventQueue ?? BASE_TABLES.EVENT;
|
|
187
|
-
configInstance.tableNameEventLock = tableNameEventLock ?? cds.env.eventQueue?.tableNameEventLock ?? BASE_TABLES.LOCK;
|
|
188
|
-
configInstance.skipCsnCheck = skipCsnCheck ?? cds.env.eventQueue?.skipCsnCheck ?? false;
|
|
178
|
+
CONFIG_VARS.forEach(([configName, defaultValue], index) => {
|
|
179
|
+
const configValue = args[index];
|
|
180
|
+
configInstance[configName] = configValue ?? cds.env.eventQueue?.[configName] ?? defaultValue;
|
|
181
|
+
});
|
|
189
182
|
};
|
|
190
183
|
|
|
191
184
|
module.exports = {
|
package/src/processEventQueue.js
CHANGED
|
@@ -8,7 +8,6 @@ const { getConfigInstance } = require("./config");
|
|
|
8
8
|
const { TransactionMode } = require("./constants");
|
|
9
9
|
const { limiter, Funnel } = require("./shared/common");
|
|
10
10
|
|
|
11
|
-
const EventQueueBase = require("./EventQueueProcessorBase");
|
|
12
11
|
const { executeInNewTransaction, TriggerRollback } = require("./shared/cdsHelper");
|
|
13
12
|
|
|
14
13
|
const COMPONENT_NAME = "eventQueue/processEventQueue";
|
|
@@ -33,7 +32,10 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
|
|
|
33
32
|
const eventConfig = getConfigInstance().getEventConfig(eventType, eventSubType);
|
|
34
33
|
const [err, EventTypeClass] = resilientRequire(eventConfig?.impl);
|
|
35
34
|
if (!eventConfig || err || !(typeof EventTypeClass.constructor === "function")) {
|
|
36
|
-
|
|
35
|
+
cds.log(COMPONENT_NAME).error("No Implementation found in the provided configuration file.", {
|
|
36
|
+
eventType,
|
|
37
|
+
eventSubType,
|
|
38
|
+
});
|
|
37
39
|
return;
|
|
38
40
|
}
|
|
39
41
|
baseInstance = new EventTypeClass(context, eventType, eventSubType, eventConfig);
|
package/src/publishEvent.js
CHANGED
|
@@ -1,19 +1,45 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
const config = require("./config");
|
|
4
|
+
const common = require("./shared/common");
|
|
4
5
|
const EventQueueError = require("./EventQueueError");
|
|
5
6
|
|
|
7
|
+
/**
|
|
8
|
+
* Asynchronously publishes a series of events to the event queue.
|
|
9
|
+
*
|
|
10
|
+
* @param {Transaction} tx - The transaction object to be used for database operations.
|
|
11
|
+
* @param {Array|Object} events - An array of event objects or a single event object. Each event object should match the Event table structure:
|
|
12
|
+
* {
|
|
13
|
+
* type: String, // Event type. This is a required field.
|
|
14
|
+
* subType: String, // Event subtype. This is a required field.
|
|
15
|
+
* referenceEntity: String, // Reference entity associated with the event.
|
|
16
|
+
* referenceEntityKey: UUID, // UUID key of the reference entity.
|
|
17
|
+
* status: Status, // Status of the event, defaults to 0.
|
|
18
|
+
* payload: LargeString, // Payload of the event.
|
|
19
|
+
* attempts: Integer, // The number of attempts made, defaults to 0.
|
|
20
|
+
* lastAttemptTimestamp: Timestamp, // Timestamp of the last attempt.
|
|
21
|
+
* createdAt: Timestamp, // Timestamp of event creation. This field is automatically set on insert.
|
|
22
|
+
* startAfter: Timestamp, // Timestamp indicating when the event should start after.
|
|
23
|
+
* }
|
|
24
|
+
* @throws {EventQueueError} Throws an error if the configuration is not initialized.
|
|
25
|
+
* @throws {EventQueueError} Throws an error if the event type is unknown.
|
|
26
|
+
* @throws {EventQueueError} Throws an error if the startAfter field is not a valid date.
|
|
27
|
+
* @returns {Promise} Returns a promise which resolves to the result of the database insert operation.
|
|
28
|
+
*/
|
|
6
29
|
const publishEvent = async (tx, events) => {
|
|
7
30
|
const configInstance = config.getConfigInstance();
|
|
8
31
|
if (!configInstance.initialized) {
|
|
9
32
|
throw EventQueueError.notInitialized();
|
|
10
33
|
}
|
|
11
34
|
const eventsForProcessing = Array.isArray(events) ? events : [events];
|
|
12
|
-
for (const { type, subType } of eventsForProcessing) {
|
|
35
|
+
for (const { type, subType, startAfter } of eventsForProcessing) {
|
|
13
36
|
const eventConfig = configInstance.getEventConfig(type, subType);
|
|
14
37
|
if (!eventConfig) {
|
|
15
38
|
throw EventQueueError.unknownEventType(type, subType);
|
|
16
39
|
}
|
|
40
|
+
if (startAfter && !common.isValidDate(startAfter)) {
|
|
41
|
+
throw EventQueueError.malformedDate(startAfter);
|
|
42
|
+
}
|
|
17
43
|
}
|
|
18
44
|
return await tx.run(INSERT.into(configInstance.tableNameEventQueue).entries(eventsForProcessing));
|
|
19
45
|
};
|
package/src/redisPubSub.js
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
const redis = require("./shared/redis");
|
|
4
|
-
const { processEventQueue } = require("./processEventQueue");
|
|
5
|
-
const { getSubdomainForTenantId } = require("./shared/cdsHelper");
|
|
6
4
|
const { checkLockExistsAndReturnValue } = require("./shared/distributedLock");
|
|
7
5
|
const config = require("./config");
|
|
8
|
-
const {
|
|
6
|
+
const { runEventCombinationForTenant } = require("./runner");
|
|
9
7
|
|
|
10
8
|
const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
|
|
11
9
|
const COMPONENT_NAME = "eventQueue/redisPubSub";
|
|
@@ -23,19 +21,12 @@ const messageHandlerProcessEvents = async (messageData) => {
|
|
|
23
21
|
const logger = cds.log(COMPONENT_NAME);
|
|
24
22
|
try {
|
|
25
23
|
const { tenantId, type, subType } = JSON.parse(messageData);
|
|
26
|
-
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
27
|
-
const context = new cds.EventContext({
|
|
28
|
-
tenant: tenantId,
|
|
29
|
-
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
30
|
-
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
31
|
-
});
|
|
32
|
-
cds.context = context;
|
|
33
24
|
logger.debug("received redis event", {
|
|
34
25
|
tenantId,
|
|
35
26
|
type,
|
|
36
27
|
subType,
|
|
37
28
|
});
|
|
38
|
-
|
|
29
|
+
await runEventCombinationForTenant(tenantId, type, subType);
|
|
39
30
|
} catch (err) {
|
|
40
31
|
logger.error("could not parse event information", {
|
|
41
32
|
messageData,
|
|
@@ -43,12 +34,12 @@ const messageHandlerProcessEvents = async (messageData) => {
|
|
|
43
34
|
}
|
|
44
35
|
};
|
|
45
36
|
|
|
46
|
-
const
|
|
37
|
+
const broadcastEvent = async (tenantId, type, subType) => {
|
|
47
38
|
const logger = cds.log(COMPONENT_NAME);
|
|
48
39
|
const configInstance = config.getConfigInstance();
|
|
49
40
|
if (!configInstance.redisEnabled) {
|
|
50
41
|
if (configInstance.registerAsEventProcessor) {
|
|
51
|
-
await
|
|
42
|
+
await runEventCombinationForTenant(tenantId, type, subType);
|
|
52
43
|
}
|
|
53
44
|
return;
|
|
54
45
|
}
|
|
@@ -76,22 +67,7 @@ const publishEvent = async (tenantId, type, subType) => {
|
|
|
76
67
|
}
|
|
77
68
|
};
|
|
78
69
|
|
|
79
|
-
const _handleEventInternally = async (tenantId, type, subType) => {
|
|
80
|
-
cds.log(COMPONENT_NAME).info("processEventQueue internally", {
|
|
81
|
-
tenantId,
|
|
82
|
-
type,
|
|
83
|
-
subType,
|
|
84
|
-
});
|
|
85
|
-
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
86
|
-
const context = new cds.EventContext({
|
|
87
|
-
tenant: tenantId,
|
|
88
|
-
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
89
|
-
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
90
|
-
});
|
|
91
|
-
getWorkerPoolInstance().addToQueue(async () => processEventQueue(context, type, subType));
|
|
92
|
-
};
|
|
93
|
-
|
|
94
70
|
module.exports = {
|
|
95
71
|
initEventQueueRedisSubscribe,
|
|
96
|
-
|
|
72
|
+
broadcastEvent,
|
|
97
73
|
};
|
package/src/runner.js
CHANGED
|
@@ -3,11 +3,12 @@
|
|
|
3
3
|
const { randomUUID } = require("crypto");
|
|
4
4
|
|
|
5
5
|
const eventQueueConfig = require("./config");
|
|
6
|
-
const { eventQueueRunner } = require("./processEventQueue");
|
|
6
|
+
const { eventQueueRunner, processEventQueue } = require("./processEventQueue");
|
|
7
7
|
const { getWorkerPoolInstance } = require("./shared/WorkerQueue");
|
|
8
8
|
const cdsHelper = require("./shared/cdsHelper");
|
|
9
9
|
const distributedLock = require("./shared/distributedLock");
|
|
10
10
|
const SetIntervalDriftSafe = require("./shared/SetIntervalDriftSafe");
|
|
11
|
+
const { getSubdomainForTenantId } = require("./shared/cdsHelper");
|
|
11
12
|
|
|
12
13
|
const COMPONENT_NAME = "eventQueue/runner";
|
|
13
14
|
const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
|
|
@@ -190,10 +191,31 @@ const _calculateOffsetForFirstRun = async () => {
|
|
|
190
191
|
return offsetDependingOnLastRun;
|
|
191
192
|
};
|
|
192
193
|
|
|
194
|
+
const runEventCombinationForTenant = async (tenantId, type, subType) => {
|
|
195
|
+
try {
|
|
196
|
+
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
197
|
+
const context = new cds.EventContext({
|
|
198
|
+
tenant: tenantId,
|
|
199
|
+
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
200
|
+
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
201
|
+
});
|
|
202
|
+
cds.context = context;
|
|
203
|
+
getWorkerPoolInstance().addToQueue(async () => await processEventQueue(context, type, subType));
|
|
204
|
+
} catch (err) {
|
|
205
|
+
const logger = cds.log(COMPONENT_NAME);
|
|
206
|
+
logger.error("error executing event combination for tenant", err, {
|
|
207
|
+
tenantId,
|
|
208
|
+
type,
|
|
209
|
+
subType,
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
};
|
|
213
|
+
|
|
193
214
|
module.exports = {
|
|
194
215
|
singleTenant,
|
|
195
216
|
multiTenancyDb,
|
|
196
217
|
multiTenancyRedis,
|
|
218
|
+
runEventCombinationForTenant,
|
|
197
219
|
_: {
|
|
198
220
|
_multiTenancyRedis,
|
|
199
221
|
_multiTenancyDb,
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const cds = require("@sap/cds");
|
|
4
|
+
|
|
5
|
+
const { broadcastEvent } = require("../redisPubSub");
|
|
6
|
+
|
|
7
|
+
const COMPONENT_NAME = "eventQueue/shared/EventScheduler";
|
|
8
|
+
|
|
9
|
+
let instance;
|
|
10
|
+
class EventScheduler {
|
|
11
|
+
#scheduledEvents = {};
|
|
12
|
+
constructor() {}
|
|
13
|
+
|
|
14
|
+
scheduleEvent(tenantId, type, subType, startAfter) {
|
|
15
|
+
const startAfterSeconds = startAfter.getSeconds();
|
|
16
|
+
const secondsUntilNextTen = 10 - (startAfterSeconds % 10);
|
|
17
|
+
const roundUpDate = new Date(startAfter.getTime() + secondsUntilNextTen * 1000);
|
|
18
|
+
const key = [tenantId, type, subType, roundUpDate.toISOString()].join("##");
|
|
19
|
+
if (this.#scheduledEvents[key]) {
|
|
20
|
+
return; // event combination already scheduled
|
|
21
|
+
}
|
|
22
|
+
this.#scheduledEvents[key] = true;
|
|
23
|
+
cds.log(COMPONENT_NAME).info("scheduling event queue run for delayed event", {
|
|
24
|
+
type,
|
|
25
|
+
subType,
|
|
26
|
+
delaySeconds: (roundUpDate.getTime() - Date.now()) / 1000,
|
|
27
|
+
});
|
|
28
|
+
setTimeout(() => {
|
|
29
|
+
delete this.#scheduledEvents[key];
|
|
30
|
+
broadcastEvent(tenantId, type, subType).catch((err) => {
|
|
31
|
+
cds.log(COMPONENT_NAME).error("could not execute scheduled event", err, {
|
|
32
|
+
tenantId,
|
|
33
|
+
type,
|
|
34
|
+
subType,
|
|
35
|
+
scheduledFor: roundUpDate.toISOString(),
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
}, roundUpDate.getTime() - Date.now()).unref();
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
clearScheduledEvents() {
|
|
42
|
+
this.#scheduledEvents = {};
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
module.exports = {
|
|
47
|
+
getInstance: () => {
|
|
48
|
+
if (!instance) {
|
|
49
|
+
instance = new EventScheduler();
|
|
50
|
+
}
|
|
51
|
+
return instance;
|
|
52
|
+
},
|
|
53
|
+
};
|
package/src/shared/common.js
CHANGED
|
@@ -107,4 +107,15 @@ const limiter = async (limit, payloads, iterator) => {
|
|
|
107
107
|
return Promise.allSettled(returnPromises);
|
|
108
108
|
};
|
|
109
109
|
|
|
110
|
-
|
|
110
|
+
const isValidDate = (value) => {
|
|
111
|
+
if (typeof value === "string") {
|
|
112
|
+
const date = Date.parse(value);
|
|
113
|
+
return !isNaN(date);
|
|
114
|
+
} else if (value instanceof Date) {
|
|
115
|
+
return !isNaN(value.getTime());
|
|
116
|
+
} else {
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
module.exports = { arrayToFlatMap, Funnel, limiter, isValidDate };
|