@cap-js-community/event-queue 0.1.50 → 0.1.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cds-plugin.js +1 -1
- package/package.json +9 -2
- package/src/EventQueueError.js +5 -10
- package/src/EventQueueProcessorBase.js +291 -324
- package/src/config.js +1 -4
- package/src/dbHandler.js +1 -4
- package/src/initialize.js +11 -39
- package/src/processEventQueue.js +64 -155
- package/src/publishEvent.js +1 -3
- package/src/redisPubSub.js +6 -20
- package/src/runner.js +40 -78
- package/src/shared/PerformanceTracer.js +1 -3
- package/src/shared/SetIntervalDriftSafe.js +37 -0
- package/src/shared/WorkerQueue.js +3 -14
- package/src/shared/cdsHelper.js +2 -11
- package/src/shared/common.js +2 -7
- package/src/shared/distributedLock.js +41 -85
- package/src/shared/redis.js +1 -3
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
const cds = require("@sap/cds");
|
|
4
4
|
|
|
5
|
-
const { executeInNewTransaction } = require("./shared/cdsHelper");
|
|
5
|
+
const { executeInNewTransaction, TriggerRollback } = require("./shared/cdsHelper");
|
|
6
6
|
const { EventProcessingStatus, TransactionMode } = require("./constants");
|
|
7
7
|
const distributedLock = require("./shared/distributedLock");
|
|
8
8
|
const EventQueueError = require("./EventQueueError");
|
|
@@ -17,8 +17,15 @@ const DEFAULT_RETRY_ATTEMPTS = 3;
|
|
|
17
17
|
const DEFAULT_PARALLEL_EVENT_PROCESSING = 1;
|
|
18
18
|
const LIMIT_PARALLEL_EVENT_PROCESSING = 10;
|
|
19
19
|
const SELECT_LIMIT_EVENTS_PER_TICK = 100;
|
|
20
|
+
const DEFAULT_DELETE_FINISHED_EVENTS_AFTER = 0;
|
|
21
|
+
const DAYS_TO_MS = 24 * 60 * 60 * 1000;
|
|
22
|
+
const TRIES_FOR_EXCEEDED_EVENTS = 3;
|
|
20
23
|
|
|
21
24
|
class EventQueueProcessorBase {
|
|
25
|
+
#eventsWithExceededTries = [];
|
|
26
|
+
#exceededTriesExceeded = [];
|
|
27
|
+
#selectedEventMap = {};
|
|
28
|
+
|
|
22
29
|
constructor(context, eventType, eventSubType, config) {
|
|
23
30
|
this.__context = context;
|
|
24
31
|
this.__baseContext = context;
|
|
@@ -32,31 +39,34 @@ class EventQueueProcessorBase {
|
|
|
32
39
|
this.__eventSubType = eventSubType;
|
|
33
40
|
this.__queueEntriesWithPayloadMap = {};
|
|
34
41
|
this.__config = config ?? {};
|
|
35
|
-
this.__parallelEventProcessing =
|
|
36
|
-
this.__config.parallelEventProcessing ??
|
|
37
|
-
DEFAULT_PARALLEL_EVENT_PROCESSING;
|
|
42
|
+
this.__parallelEventProcessing = this.__config.parallelEventProcessing ?? DEFAULT_PARALLEL_EVENT_PROCESSING;
|
|
38
43
|
if (this.__parallelEventProcessing > LIMIT_PARALLEL_EVENT_PROCESSING) {
|
|
39
44
|
this.__parallelEventProcessing = LIMIT_PARALLEL_EVENT_PROCESSING;
|
|
40
45
|
}
|
|
41
46
|
// NOTE: keep the feature, this might be needed again
|
|
42
47
|
this.__concurrentEventProcessing = false;
|
|
43
48
|
this.__startTime = this.__config.startTime ?? new Date();
|
|
44
|
-
this.__retryAttempts =
|
|
45
|
-
|
|
46
|
-
this.__selectMaxChunkSize =
|
|
47
|
-
this.__config.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
|
|
49
|
+
this.__retryAttempts = this.__config.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
|
|
50
|
+
this.__selectMaxChunkSize = this.__config.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
|
|
48
51
|
this.__selectNextChunk = !!this.__config.checkForNextChunk;
|
|
49
52
|
this.__keepalivePromises = {};
|
|
50
53
|
this.__outdatedCheckEnabled = this.__config.eventOutdatedCheck ?? true;
|
|
51
|
-
this.__transactionMode =
|
|
52
|
-
|
|
53
|
-
|
|
54
|
+
this.__transactionMode = this.__config.transactionMode ?? TransactionMode.isolated;
|
|
55
|
+
if (this.__config.deleteFinishedEventsAfterDays) {
|
|
56
|
+
this.__deleteFinishedEventsAfter =
|
|
57
|
+
Number.isInteger(this.__config.deleteFinishedEventsAfterDays) && this.__config.deleteFinishedEventsAfterDays > 0
|
|
58
|
+
? this.__config.deleteFinishedEventsAfterDays
|
|
59
|
+
: DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
|
|
60
|
+
} else {
|
|
61
|
+
this.__deleteFinishedEventsAfter = DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
|
|
62
|
+
}
|
|
54
63
|
this.__emptyChunkSelected = false;
|
|
55
64
|
this.__lockAcquired = false;
|
|
56
65
|
this.__txUsageAllowed = true;
|
|
57
66
|
this.__txMap = {};
|
|
58
67
|
this.__txRollback = {};
|
|
59
68
|
this.__eventQueueConfig = eventQueueConfig.getConfigInstance();
|
|
69
|
+
this.__queueEntries = [];
|
|
60
70
|
}
|
|
61
71
|
|
|
62
72
|
/**
|
|
@@ -78,17 +88,11 @@ class EventQueueProcessorBase {
|
|
|
78
88
|
}
|
|
79
89
|
|
|
80
90
|
startPerformanceTracerEvents() {
|
|
81
|
-
this.__performanceLoggerEvents = new PerformanceTracer(
|
|
82
|
-
this.logger,
|
|
83
|
-
"Processing events"
|
|
84
|
-
);
|
|
91
|
+
this.__performanceLoggerEvents = new PerformanceTracer(this.logger, "Processing events");
|
|
85
92
|
}
|
|
86
93
|
|
|
87
94
|
startPerformanceTracerPreprocessing() {
|
|
88
|
-
this.__performanceLoggerPreprocessing = new PerformanceTracer(
|
|
89
|
-
this.logger,
|
|
90
|
-
"Preprocessing events"
|
|
91
|
-
);
|
|
95
|
+
this.__performanceLoggerPreprocessing = new PerformanceTracer(this.logger, "Preprocessing events");
|
|
92
96
|
}
|
|
93
97
|
|
|
94
98
|
endPerformanceTracerEvents() {
|
|
@@ -176,10 +180,7 @@ class EventQueueProcessorBase {
|
|
|
176
180
|
eventType: this.__eventType,
|
|
177
181
|
eventSubType: this.__eventSubType,
|
|
178
182
|
});
|
|
179
|
-
this.#determineAndAddEventStatusToMap(
|
|
180
|
-
queueEntry.ID,
|
|
181
|
-
EventProcessingStatus.Done
|
|
182
|
-
);
|
|
183
|
+
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Done);
|
|
183
184
|
}
|
|
184
185
|
|
|
185
186
|
/**
|
|
@@ -190,11 +191,9 @@ class EventQueueProcessorBase {
|
|
|
190
191
|
* In this case the events should be clustered together and only one mail should be sent.
|
|
191
192
|
*/
|
|
192
193
|
clusterQueueEntries() {
|
|
193
|
-
Object.entries(this.__queueEntriesWithPayloadMap).forEach(
|
|
194
|
-
(
|
|
195
|
-
|
|
196
|
-
}
|
|
197
|
-
);
|
|
194
|
+
Object.entries(this.__queueEntriesWithPayloadMap).forEach(([key, { queueEntry, payload }]) => {
|
|
195
|
+
this.addEntryToProcessingMap(key, queueEntry, payload);
|
|
196
|
+
});
|
|
198
197
|
}
|
|
199
198
|
|
|
200
199
|
/**
|
|
@@ -225,28 +224,23 @@ class EventQueueProcessorBase {
|
|
|
225
224
|
* event processing.
|
|
226
225
|
* @param {Array} queueEntries which has been selected from event queue table and been modified by modifyQueueEntry
|
|
227
226
|
* @param {Array<Object>} queueEntryProcessingStatusTuple Array of tuple <queueEntryId, processingStatus>
|
|
227
|
+
* @param {boolean} returnMap Allows the function to allow the result as map
|
|
228
228
|
* @return {Object} statusMap Map which contains all events for which a status has been set so far
|
|
229
229
|
*/
|
|
230
|
-
setEventStatus(queueEntries, queueEntryProcessingStatusTuple) {
|
|
230
|
+
setEventStatus(queueEntries, queueEntryProcessingStatusTuple, returnMap = false) {
|
|
231
231
|
this.logger.debug("setting event status for entries", {
|
|
232
|
-
queueEntryProcessingStatusTuple: JSON.stringify(
|
|
233
|
-
queueEntryProcessingStatusTuple
|
|
234
|
-
),
|
|
232
|
+
queueEntryProcessingStatusTuple: JSON.stringify(queueEntryProcessingStatusTuple),
|
|
235
233
|
eventType: this.__eventType,
|
|
236
234
|
eventSubType: this.__eventSubType,
|
|
237
235
|
});
|
|
238
|
-
const statusMap = this.commitOnEventLevel ? {} : this.__statusMap;
|
|
236
|
+
const statusMap = this.commitOnEventLevel || returnMap ? {} : this.__statusMap;
|
|
239
237
|
try {
|
|
240
238
|
queueEntryProcessingStatusTuple.forEach(([id, processingStatus]) =>
|
|
241
239
|
this.#determineAndAddEventStatusToMap(id, processingStatus, statusMap)
|
|
242
240
|
);
|
|
243
241
|
} catch (error) {
|
|
244
242
|
queueEntries.forEach((queueEntry) =>
|
|
245
|
-
this.#determineAndAddEventStatusToMap(
|
|
246
|
-
queueEntry.ID,
|
|
247
|
-
EventProcessingStatus.Error,
|
|
248
|
-
statusMap
|
|
249
|
-
)
|
|
243
|
+
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error, statusMap)
|
|
250
244
|
);
|
|
251
245
|
this.logger.error(
|
|
252
246
|
`The supplied status tuple doesn't have the required structure. Setting all entries to error. Error: ${error.toString()}`,
|
|
@@ -269,24 +263,16 @@ class EventQueueProcessorBase {
|
|
|
269
263
|
try {
|
|
270
264
|
queueEntry.payload = JSON.parse(queueEntry.payload);
|
|
271
265
|
} catch {
|
|
272
|
-
|
|
266
|
+
/* empty */
|
|
273
267
|
}
|
|
274
268
|
}
|
|
275
269
|
|
|
276
|
-
#determineAndAddEventStatusToMap(
|
|
277
|
-
id,
|
|
278
|
-
processingStatus,
|
|
279
|
-
statusMap = this.__statusMap
|
|
280
|
-
) {
|
|
270
|
+
#determineAndAddEventStatusToMap(id, processingStatus, statusMap = this.__statusMap) {
|
|
281
271
|
if (!statusMap[id]) {
|
|
282
272
|
statusMap[id] = processingStatus;
|
|
283
273
|
return;
|
|
284
274
|
}
|
|
285
|
-
if (
|
|
286
|
-
[EventProcessingStatus.Error, EventProcessingStatus.Exceeded].includes(
|
|
287
|
-
statusMap[id]
|
|
288
|
-
)
|
|
289
|
-
) {
|
|
275
|
+
if ([EventProcessingStatus.Error, EventProcessingStatus.Exceeded].includes(statusMap[id])) {
|
|
290
276
|
// NOTE: worst aggregation --> if already error|exceeded keep this state
|
|
291
277
|
return;
|
|
292
278
|
}
|
|
@@ -306,17 +292,9 @@ class EventQueueProcessorBase {
|
|
|
306
292
|
}
|
|
307
293
|
);
|
|
308
294
|
queueEntries.forEach((queueEntry) =>
|
|
309
|
-
this.#determineAndAddEventStatusToMap(
|
|
310
|
-
queueEntry.ID,
|
|
311
|
-
EventProcessingStatus.Error
|
|
312
|
-
)
|
|
313
|
-
);
|
|
314
|
-
return Object.fromEntries(
|
|
315
|
-
queueEntries.map((queueEntry) => [
|
|
316
|
-
queueEntry.ID,
|
|
317
|
-
EventProcessingStatus.Error,
|
|
318
|
-
])
|
|
295
|
+
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error)
|
|
319
296
|
);
|
|
297
|
+
return Object.fromEntries(queueEntries.map((queueEntry) => [queueEntry.ID, EventProcessingStatus.Error]));
|
|
320
298
|
}
|
|
321
299
|
|
|
322
300
|
/**
|
|
@@ -324,10 +302,7 @@ class EventQueueProcessorBase {
|
|
|
324
302
|
* selected events a status has been submitted. Persisting the status of events is done in a dedicated database tx.
|
|
325
303
|
* The function accepts no arguments as there are dedicated functions to set the status of events (e.g. setEventStatus)
|
|
326
304
|
*/
|
|
327
|
-
async persistEventStatus(
|
|
328
|
-
tx,
|
|
329
|
-
{ skipChecks, statusMap = this.__statusMap } = {}
|
|
330
|
-
) {
|
|
305
|
+
async persistEventStatus(tx, { skipChecks, statusMap = this.__statusMap } = {}) {
|
|
331
306
|
this.logger.debug("entering persistEventStatus", {
|
|
332
307
|
eventType: this.__eventType,
|
|
333
308
|
eventSubType: this.__eventSubType,
|
|
@@ -338,9 +313,7 @@ class EventQueueProcessorBase {
|
|
|
338
313
|
}
|
|
339
314
|
this.#ensureEveryStatusIsAllowed(statusMap);
|
|
340
315
|
|
|
341
|
-
const { success, failed, exceeded, invalidAttempts } = Object.entries(
|
|
342
|
-
statusMap
|
|
343
|
-
).reduce(
|
|
316
|
+
const { success, failed, exceeded, invalidAttempts } = Object.entries(statusMap).reduce(
|
|
344
317
|
(result, [notificationEntityId, processingStatus]) => {
|
|
345
318
|
this.__commitedStatusMap[notificationEntityId] = processingStatus;
|
|
346
319
|
if (processingStatus === EventProcessingStatus.Open) {
|
|
@@ -380,34 +353,24 @@ class EventQueueProcessorBase {
|
|
|
380
353
|
.where("ID IN", invalidAttempts)
|
|
381
354
|
);
|
|
382
355
|
}
|
|
383
|
-
|
|
356
|
+
const ts = new Date().toISOString();
|
|
357
|
+
const updateTuples = [
|
|
358
|
+
[success, EventProcessingStatus.Done],
|
|
359
|
+
[failed, EventProcessingStatus.Error],
|
|
360
|
+
[exceeded, EventProcessingStatus.Exceeded],
|
|
361
|
+
];
|
|
362
|
+
|
|
363
|
+
for (const [eventIds, status] of updateTuples) {
|
|
364
|
+
if (!eventIds.length) {
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
384
367
|
await tx.run(
|
|
385
368
|
UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
|
|
386
369
|
.set({
|
|
387
|
-
status:
|
|
388
|
-
lastAttemptTimestamp:
|
|
389
|
-
})
|
|
390
|
-
.where("ID IN", success)
|
|
391
|
-
);
|
|
392
|
-
}
|
|
393
|
-
if (failed.length) {
|
|
394
|
-
await tx.run(
|
|
395
|
-
UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
|
|
396
|
-
.where("ID IN", failed)
|
|
397
|
-
.with({
|
|
398
|
-
status: EventProcessingStatus.Error,
|
|
399
|
-
lastAttemptTimestamp: new Date().toISOString(),
|
|
400
|
-
})
|
|
401
|
-
);
|
|
402
|
-
}
|
|
403
|
-
if (exceeded.length) {
|
|
404
|
-
await tx.run(
|
|
405
|
-
UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
|
|
406
|
-
.where("ID IN", exceeded)
|
|
407
|
-
.with({
|
|
408
|
-
status: EventProcessingStatus.Exceeded,
|
|
409
|
-
lastAttemptTimestamp: new Date().toISOString(),
|
|
370
|
+
status: status,
|
|
371
|
+
lastAttemptTimestamp: ts,
|
|
410
372
|
})
|
|
373
|
+
.where("ID IN", eventIds)
|
|
411
374
|
);
|
|
412
375
|
}
|
|
413
376
|
this.logger.debug("exiting persistEventStatus", {
|
|
@@ -416,26 +379,39 @@ class EventQueueProcessorBase {
|
|
|
416
379
|
});
|
|
417
380
|
}
|
|
418
381
|
|
|
382
|
+
async deleteFinishedEvents(tx) {
|
|
383
|
+
if (!this.__deleteFinishedEventsAfter) {
|
|
384
|
+
return;
|
|
385
|
+
}
|
|
386
|
+
const deleteCount = await tx.run(
|
|
387
|
+
DELETE.from(this.__eventQueueConfig.tableNameEventQueue).where(
|
|
388
|
+
"type =",
|
|
389
|
+
this.eventType,
|
|
390
|
+
"AND subType=",
|
|
391
|
+
this.eventSubType,
|
|
392
|
+
"AND lastAttemptTimestamp <=",
|
|
393
|
+
new Date(Date.now() - this.__deleteFinishedEventsAfter * DAYS_TO_MS).toISOString()
|
|
394
|
+
)
|
|
395
|
+
);
|
|
396
|
+
this.logger.debug("Deleted finished events", {
|
|
397
|
+
eventType: this.eventType,
|
|
398
|
+
eventSubType: this.eventSubType,
|
|
399
|
+
deleteFinishedEventsAfter: this.__deleteFinishedEventsAfter,
|
|
400
|
+
deleteCount,
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
|
|
419
404
|
#ensureEveryQueueEntryHasStatus() {
|
|
420
405
|
this.__queueEntries.forEach((queueEntry) => {
|
|
421
|
-
if (
|
|
422
|
-
queueEntry.ID in this.__statusMap ||
|
|
423
|
-
queueEntry.ID in this.__commitedStatusMap
|
|
424
|
-
) {
|
|
406
|
+
if (queueEntry.ID in this.__statusMap || queueEntry.ID in this.__commitedStatusMap) {
|
|
425
407
|
return;
|
|
426
408
|
}
|
|
427
|
-
this.logger.error(
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
}
|
|
434
|
-
);
|
|
435
|
-
this.#determineAndAddEventStatusToMap(
|
|
436
|
-
queueEntry.ID,
|
|
437
|
-
EventProcessingStatus.Error
|
|
438
|
-
);
|
|
409
|
+
this.logger.error("Missing status for selected event entry. Setting status to error", {
|
|
410
|
+
eventType: this.__eventType,
|
|
411
|
+
eventSubType: this.__eventSubType,
|
|
412
|
+
queueEntry,
|
|
413
|
+
});
|
|
414
|
+
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
|
|
439
415
|
});
|
|
440
416
|
}
|
|
441
417
|
|
|
@@ -452,22 +428,19 @@ class EventQueueProcessorBase {
|
|
|
452
428
|
return;
|
|
453
429
|
}
|
|
454
430
|
|
|
455
|
-
this.logger.error(
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
status: statusMap[queueEntryId],
|
|
462
|
-
}
|
|
463
|
-
);
|
|
431
|
+
this.logger.error("Not allowed event status returned. Only Open, Done, Error is allowed!", {
|
|
432
|
+
eventType: this.__eventType,
|
|
433
|
+
eventSubType: this.__eventSubType,
|
|
434
|
+
queueEntryId,
|
|
435
|
+
status: statusMap[queueEntryId],
|
|
436
|
+
});
|
|
464
437
|
delete statusMap[queueEntryId];
|
|
465
438
|
});
|
|
466
439
|
}
|
|
467
440
|
|
|
468
441
|
#ensureOnlySelectedQueueEntries(statusMap) {
|
|
469
442
|
Object.keys(statusMap).forEach((queueEntryId) => {
|
|
470
|
-
if (this
|
|
443
|
+
if (this.#selectedEventMap[queueEntryId]) {
|
|
471
444
|
return;
|
|
472
445
|
}
|
|
473
446
|
|
|
@@ -484,18 +457,12 @@ class EventQueueProcessorBase {
|
|
|
484
457
|
}
|
|
485
458
|
|
|
486
459
|
handleErrorDuringClustering(error) {
|
|
487
|
-
this.logger.error(
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
eventSubType: this.__eventSubType,
|
|
492
|
-
}
|
|
493
|
-
);
|
|
460
|
+
this.logger.error(`Error during clustering of events - setting all queue entries to error. Error: ${error}`, {
|
|
461
|
+
eventType: this.__eventType,
|
|
462
|
+
eventSubType: this.__eventSubType,
|
|
463
|
+
});
|
|
494
464
|
this.__queueEntries.forEach((queueEntry) => {
|
|
495
|
-
this.#determineAndAddEventStatusToMap(
|
|
496
|
-
queueEntry.ID,
|
|
497
|
-
EventProcessingStatus.Error
|
|
498
|
-
);
|
|
465
|
+
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
|
|
499
466
|
});
|
|
500
467
|
}
|
|
501
468
|
|
|
@@ -508,29 +475,15 @@ class EventQueueProcessorBase {
|
|
|
508
475
|
eventSubType: this.__eventSubType,
|
|
509
476
|
}
|
|
510
477
|
);
|
|
511
|
-
this.#determineAndAddEventStatusToMap(
|
|
512
|
-
queueEntry.ID,
|
|
513
|
-
EventProcessingStatus.Error
|
|
514
|
-
);
|
|
478
|
+
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
|
|
515
479
|
}
|
|
516
480
|
|
|
517
|
-
static async handleMissingTypeImplementation(
|
|
518
|
-
context,
|
|
519
|
-
|
|
520
|
-
eventSubType
|
|
521
|
-
) {
|
|
522
|
-
const baseInstance = new EventQueueProcessorBase(
|
|
523
|
-
context,
|
|
481
|
+
static async handleMissingTypeImplementation(context, eventType, eventSubType) {
|
|
482
|
+
const baseInstance = new EventQueueProcessorBase(context, eventType, eventSubType);
|
|
483
|
+
baseInstance.logger.error("No Implementation found in the provided configuration file.", {
|
|
524
484
|
eventType,
|
|
525
|
-
eventSubType
|
|
526
|
-
);
|
|
527
|
-
baseInstance.logger.error(
|
|
528
|
-
"No Implementation found in the provided configuration file.",
|
|
529
|
-
{
|
|
530
|
-
eventType,
|
|
531
|
-
eventSubType,
|
|
532
|
-
}
|
|
533
|
-
);
|
|
485
|
+
eventSubType,
|
|
486
|
+
});
|
|
534
487
|
}
|
|
535
488
|
|
|
536
489
|
/**
|
|
@@ -542,137 +495,179 @@ class EventQueueProcessorBase {
|
|
|
542
495
|
*/
|
|
543
496
|
async getQueueEntriesAndSetToInProgress() {
|
|
544
497
|
let result = [];
|
|
545
|
-
await executeInNewTransaction(
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
).toISOString(),
|
|
570
|
-
") )"
|
|
571
|
-
)
|
|
572
|
-
.orderBy("createdAt", "ID")
|
|
573
|
-
);
|
|
574
|
-
|
|
575
|
-
if (!entries.length) {
|
|
576
|
-
this.logger.debug("no entries available for processing", {
|
|
577
|
-
eventType: this.__eventType,
|
|
578
|
-
eventSubType: this.__eventSubType,
|
|
579
|
-
});
|
|
580
|
-
this.__emptyChunkSelected = true;
|
|
581
|
-
return;
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
const { exceededTries, openEvents } =
|
|
585
|
-
this.#filterExceededEvents(entries);
|
|
586
|
-
if (exceededTries.length) {
|
|
587
|
-
this.__eventsWithExceededTries = exceededTries;
|
|
588
|
-
}
|
|
589
|
-
result = openEvents;
|
|
590
|
-
|
|
591
|
-
if (!result.length) {
|
|
592
|
-
this.__emptyChunkSelected = true;
|
|
593
|
-
return;
|
|
594
|
-
}
|
|
498
|
+
await executeInNewTransaction(this.__baseContext, "eventQueue-getQueueEntriesAndSetToInProgress", async (tx) => {
|
|
499
|
+
const entries = await tx.run(
|
|
500
|
+
SELECT.from(this.__eventQueueConfig.tableNameEventQueue)
|
|
501
|
+
.forUpdate({ wait: this.__eventQueueConfig.forUpdateTimeout })
|
|
502
|
+
.limit(this.selectMaxChunkSize)
|
|
503
|
+
.where(
|
|
504
|
+
"type =",
|
|
505
|
+
this.__eventType,
|
|
506
|
+
"AND subType=",
|
|
507
|
+
this.__eventSubType,
|
|
508
|
+
"AND ( status =",
|
|
509
|
+
EventProcessingStatus.Open,
|
|
510
|
+
"OR ( status =",
|
|
511
|
+
EventProcessingStatus.Error,
|
|
512
|
+
"AND lastAttemptTimestamp <=",
|
|
513
|
+
this.__startTime.toISOString(),
|
|
514
|
+
") OR ( status =",
|
|
515
|
+
EventProcessingStatus.InProgress,
|
|
516
|
+
"AND lastAttemptTimestamp <=",
|
|
517
|
+
new Date(new Date().getTime() - this.__eventQueueConfig.globalTxTimeout).toISOString(),
|
|
518
|
+
") )"
|
|
519
|
+
)
|
|
520
|
+
.orderBy("createdAt", "ID")
|
|
521
|
+
);
|
|
595
522
|
|
|
596
|
-
|
|
597
|
-
|
|
523
|
+
if (!entries.length) {
|
|
524
|
+
this.logger.debug("no entries available for processing", {
|
|
598
525
|
eventType: this.__eventType,
|
|
599
526
|
eventSubType: this.__eventSubType,
|
|
600
527
|
});
|
|
528
|
+
this.__emptyChunkSelected = true;
|
|
529
|
+
return;
|
|
530
|
+
}
|
|
601
531
|
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
status: EventProcessingStatus.InProgress,
|
|
607
|
-
lastAttemptTimestamp: isoTimestamp,
|
|
608
|
-
attempts: { "+=": 1 },
|
|
609
|
-
})
|
|
610
|
-
.where(
|
|
611
|
-
"ID IN",
|
|
612
|
-
result.map(({ ID }) => ID)
|
|
613
|
-
)
|
|
614
|
-
);
|
|
615
|
-
result.forEach((entry) => {
|
|
616
|
-
entry.lastAttemptTimestamp = isoTimestamp;
|
|
617
|
-
// NOTE: empty payloads are supported on DB-Level.
|
|
618
|
-
// Behaviour of event queue is: null as payload is treated as obsolete/done
|
|
619
|
-
// For supporting this convert null to empty string --> "" as payload will be processed normally
|
|
620
|
-
if (entry.payload === null) {
|
|
621
|
-
entry.payload = "";
|
|
622
|
-
}
|
|
623
|
-
});
|
|
532
|
+
this.#selectedEventMap = arrayToFlatMap(entries);
|
|
533
|
+
const { exceededTries, openEvents, exceededTriesExceeded } = this.#filterExceededEvents(entries);
|
|
534
|
+
if (exceededTries.length) {
|
|
535
|
+
this.#eventsWithExceededTries = exceededTries;
|
|
624
536
|
}
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
537
|
+
if (exceededTriesExceeded.length) {
|
|
538
|
+
this.#exceededTriesExceeded = exceededTriesExceeded;
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
result = openEvents;
|
|
542
|
+
|
|
543
|
+
if (!result.length) {
|
|
544
|
+
this.__emptyChunkSelected = true;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
this.logger.info("Selected event queue entries for processing", {
|
|
548
|
+
queueEntriesCount: result.length,
|
|
549
|
+
eventType: this.__eventType,
|
|
550
|
+
eventSubType: this.__eventSubType,
|
|
551
|
+
});
|
|
552
|
+
|
|
553
|
+
const isoTimestamp = new Date().toISOString();
|
|
554
|
+
await tx.run(
|
|
555
|
+
UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
|
|
556
|
+
.with({
|
|
557
|
+
status: EventProcessingStatus.InProgress,
|
|
558
|
+
lastAttemptTimestamp: isoTimestamp,
|
|
559
|
+
attempts: { "+=": 1 },
|
|
560
|
+
})
|
|
561
|
+
.where(
|
|
562
|
+
"ID IN",
|
|
563
|
+
entries.map(({ ID }) => ID)
|
|
564
|
+
)
|
|
565
|
+
);
|
|
566
|
+
entries.forEach((entry) => {
|
|
567
|
+
entry.lastAttemptTimestamp = isoTimestamp;
|
|
568
|
+
// NOTE: empty payloads are supported on DB-Level.
|
|
569
|
+
// Behaviour of event queue is: null as payload is treated as obsolete/done
|
|
570
|
+
// For supporting this convert null to empty string --> "" as payload will be processed normally
|
|
571
|
+
if (entry.payload === null) {
|
|
572
|
+
entry.payload = "";
|
|
573
|
+
}
|
|
574
|
+
});
|
|
575
|
+
this.__queueEntries = result;
|
|
576
|
+
this.__queueEntriesMap = arrayToFlatMap(result);
|
|
577
|
+
});
|
|
628
578
|
return result;
|
|
629
579
|
}
|
|
630
580
|
|
|
631
581
|
#filterExceededEvents(events) {
|
|
632
582
|
return events.reduce(
|
|
633
583
|
(result, event) => {
|
|
634
|
-
if (event.attempts === this.__retryAttempts) {
|
|
584
|
+
if (event.attempts === this.__retryAttempts + TRIES_FOR_EXCEEDED_EVENTS) {
|
|
585
|
+
result.exceededTriesExceeded.push(event);
|
|
586
|
+
} else if (event.attempts >= this.__retryAttempts) {
|
|
635
587
|
result.exceededTries.push(event);
|
|
636
588
|
} else {
|
|
637
589
|
result.openEvents.push(event);
|
|
638
590
|
}
|
|
639
591
|
return result;
|
|
640
592
|
},
|
|
641
|
-
{ exceededTries: [], openEvents: [] }
|
|
593
|
+
{ exceededTries: [], openEvents: [], exceededTriesExceeded: [] }
|
|
642
594
|
);
|
|
643
595
|
}
|
|
644
596
|
|
|
645
|
-
async handleExceededEvents(
|
|
646
|
-
await this
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
597
|
+
async handleExceededEvents() {
|
|
598
|
+
await this.#handleExceededTriesExceeded();
|
|
599
|
+
if (!this.#eventsWithExceededTries.length) {
|
|
600
|
+
return;
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
for (const exceededEvent of this.#eventsWithExceededTries) {
|
|
604
|
+
await executeInNewTransaction(
|
|
605
|
+
this.context,
|
|
606
|
+
`eventQueue-handleExceededEvents-${this.eventType}##${this.eventSubType}`,
|
|
607
|
+
async (tx) => {
|
|
608
|
+
try {
|
|
609
|
+
this.processEventContext = tx.context;
|
|
610
|
+
this.modifyQueueEntry(exceededEvent);
|
|
611
|
+
await this.hookForExceededEvents({ ...exceededEvent });
|
|
612
|
+
this.logger.warn("The retry attempts for the following events are exceeded", {
|
|
613
|
+
eventType: this.__eventType,
|
|
614
|
+
eventSubType: this.__eventSubType,
|
|
615
|
+
retryAttempts: this.__retryAttempts,
|
|
616
|
+
queueEntriesId: exceededEvent.ID,
|
|
617
|
+
currentAttempt: exceededEvent.attempts,
|
|
618
|
+
});
|
|
619
|
+
await this.#persistEventQueueStatusForExceeded(this.tx, [exceededEvent], EventProcessingStatus.Exceeded);
|
|
620
|
+
} catch (err) {
|
|
621
|
+
this.logger.error(
|
|
622
|
+
`Caught error during hook for exceeded events - setting queue entry to error. Please catch your promises/exceptions. Error: ${err}`,
|
|
623
|
+
{
|
|
624
|
+
eventType: this.__eventType,
|
|
625
|
+
eventSubType: this.__eventSubType,
|
|
626
|
+
retryAttempts: this.__retryAttempts,
|
|
627
|
+
queueEntriesId: exceededEvent.ID,
|
|
628
|
+
currentAttempt: exceededEvent.attempts,
|
|
629
|
+
}
|
|
630
|
+
);
|
|
631
|
+
await executeInNewTransaction(this.context, "error-hookForExceededEvents", async (tx) =>
|
|
632
|
+
this.#persistEventQueueStatusForExceeded(tx, [exceededEvent], EventProcessingStatus.Error)
|
|
633
|
+
);
|
|
634
|
+
throw new TriggerRollback();
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
async #handleExceededTriesExceeded() {
|
|
642
|
+
if (this.#exceededTriesExceeded.length) {
|
|
643
|
+
this.logger.error("Event hook failure exceeded, status set to 'exceeded' without invoking hook again!", {
|
|
659
644
|
eventType: this.__eventType,
|
|
660
645
|
eventSubType: this.__eventSubType,
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
646
|
+
queueEntriesIds: this.#eventsWithExceededTries.map(({ ID }) => ID),
|
|
647
|
+
});
|
|
648
|
+
await executeInNewTransaction(this.context, "exceededTriesExceeded", async (tx) => {
|
|
649
|
+
await this.#persistEventQueueStatusForExceeded(tx, this.#exceededTriesExceeded, EventProcessingStatus.Exceeded);
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
async #persistEventQueueStatusForExceeded(tx, events, status) {
|
|
655
|
+
const statusMap = this.setEventStatus(
|
|
656
|
+
events,
|
|
657
|
+
events.map((e) => [e.ID, status]),
|
|
658
|
+
true
|
|
664
659
|
);
|
|
665
|
-
await this.
|
|
660
|
+
await this.persistEventStatus(tx, { statusMap, skipChecks: true });
|
|
666
661
|
}
|
|
667
662
|
|
|
668
663
|
/**
|
|
669
664
|
* This function enables the possibility to execute custom actions for events for which the retry attempts have been
|
|
670
665
|
* exceeded. As always a valid transaction is available with this.tx. This transaction will be committed after the
|
|
671
666
|
* execution of this function.
|
|
672
|
-
* @param {Object}
|
|
667
|
+
* @param {Object} exceededEvent exceeded event queue entry
|
|
673
668
|
*/
|
|
674
669
|
// eslint-disable-next-line no-unused-vars
|
|
675
|
-
async hookForExceededEvents(
|
|
670
|
+
async hookForExceededEvents(exceededEvent) {}
|
|
676
671
|
|
|
677
672
|
/**
|
|
678
673
|
* This function serves the purpose of mass enabled preloading data for processing the events which are added with
|
|
@@ -695,9 +690,7 @@ class EventQueueProcessorBase {
|
|
|
695
690
|
return false;
|
|
696
691
|
}
|
|
697
692
|
let eventOutdated;
|
|
698
|
-
const runningChecks = queueEntries
|
|
699
|
-
.map((queueEntry) => this.__keepalivePromises[queueEntry.ID])
|
|
700
|
-
.filter((p) => p);
|
|
693
|
+
const runningChecks = queueEntries.map((queueEntry) => this.__keepalivePromises[queueEntry.ID]).filter((p) => p);
|
|
701
694
|
if (runningChecks.length === queueEntries.length) {
|
|
702
695
|
const results = await Promise.allSettled(runningChecks);
|
|
703
696
|
for (const { value } of results) {
|
|
@@ -710,70 +703,54 @@ class EventQueueProcessorBase {
|
|
|
710
703
|
await Promise.allSettled(runningChecks);
|
|
711
704
|
}
|
|
712
705
|
const checkAndUpdatePromise = new Promise((resolve) => {
|
|
713
|
-
executeInNewTransaction(
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
.
|
|
706
|
+
executeInNewTransaction(this.__baseContext, "eventProcessing-isOutdatedAndKeepalive", async (tx) => {
|
|
707
|
+
const queueEntriesFresh = await tx.run(
|
|
708
|
+
SELECT.from(this.__eventQueueConfig.tableNameEventQueue)
|
|
709
|
+
.forUpdate({ wait: this.__eventQueueConfig.forUpdateTimeout })
|
|
710
|
+
.where(
|
|
711
|
+
"ID IN",
|
|
712
|
+
queueEntries.map(({ ID }) => ID)
|
|
713
|
+
)
|
|
714
|
+
.columns("ID", "lastAttemptTimestamp")
|
|
715
|
+
);
|
|
716
|
+
eventOutdated = queueEntriesFresh.some((queueEntryFresh) => {
|
|
717
|
+
const queueEntry = this.__queueEntriesMap[queueEntryFresh.ID];
|
|
718
|
+
return queueEntry?.lastAttemptTimestamp !== queueEntryFresh.lastAttemptTimestamp;
|
|
719
|
+
});
|
|
720
|
+
let newTs = new Date().toISOString();
|
|
721
|
+
if (!eventOutdated) {
|
|
722
|
+
await tx.run(
|
|
723
|
+
UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
|
|
724
|
+
.set("lastAttemptTimestamp =", newTs)
|
|
720
725
|
.where(
|
|
721
726
|
"ID IN",
|
|
722
727
|
queueEntries.map(({ ID }) => ID)
|
|
723
728
|
)
|
|
724
|
-
.columns("ID", "lastAttemptTimestamp")
|
|
725
729
|
);
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
)
|
|
732
|
-
});
|
|
733
|
-
let newTs = new Date().toISOString();
|
|
734
|
-
if (!eventOutdated) {
|
|
735
|
-
await tx.run(
|
|
736
|
-
UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
|
|
737
|
-
.set("lastAttemptTimestamp =", newTs)
|
|
738
|
-
.where(
|
|
739
|
-
"ID IN",
|
|
740
|
-
queueEntries.map(({ ID }) => ID)
|
|
741
|
-
)
|
|
742
|
-
);
|
|
743
|
-
} else {
|
|
744
|
-
newTs = null;
|
|
745
|
-
this.logger.warn(
|
|
746
|
-
"event data has been modified. Processing skipped.",
|
|
747
|
-
{
|
|
748
|
-
eventType: this.__eventType,
|
|
749
|
-
eventSubType: this.__eventSubType,
|
|
750
|
-
queueEntriesIds: queueEntries.map(({ ID }) => ID),
|
|
751
|
-
}
|
|
752
|
-
);
|
|
753
|
-
queueEntries.forEach(
|
|
754
|
-
({ ID: queueEntryId }) =>
|
|
755
|
-
delete this.__queueEntriesMap[queueEntryId]
|
|
756
|
-
);
|
|
757
|
-
}
|
|
758
|
-
this.__queueEntries = Object.values(this.__queueEntriesMap);
|
|
759
|
-
queueEntriesFresh.forEach((queueEntryFresh) => {
|
|
760
|
-
if (this.__queueEntriesMap[queueEntryFresh.ID]) {
|
|
761
|
-
const queueEntry = this.__queueEntriesMap[queueEntryFresh.ID];
|
|
762
|
-
if (newTs) {
|
|
763
|
-
queueEntry.lastAttemptTimestamp = newTs;
|
|
764
|
-
}
|
|
765
|
-
}
|
|
766
|
-
delete this.__keepalivePromises[queueEntryFresh.ID];
|
|
730
|
+
} else {
|
|
731
|
+
newTs = null;
|
|
732
|
+
this.logger.warn("event data has been modified. Processing skipped.", {
|
|
733
|
+
eventType: this.__eventType,
|
|
734
|
+
eventSubType: this.__eventSubType,
|
|
735
|
+
queueEntriesIds: queueEntries.map(({ ID }) => ID),
|
|
767
736
|
});
|
|
768
|
-
|
|
737
|
+
queueEntries.forEach(({ ID: queueEntryId }) => delete this.__queueEntriesMap[queueEntryId]);
|
|
769
738
|
}
|
|
770
|
-
|
|
739
|
+
this.__queueEntries = Object.values(this.__queueEntriesMap);
|
|
740
|
+
queueEntriesFresh.forEach((queueEntryFresh) => {
|
|
741
|
+
if (this.__queueEntriesMap[queueEntryFresh.ID]) {
|
|
742
|
+
const queueEntry = this.__queueEntriesMap[queueEntryFresh.ID];
|
|
743
|
+
if (newTs) {
|
|
744
|
+
queueEntry.lastAttemptTimestamp = newTs;
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
delete this.__keepalivePromises[queueEntryFresh.ID];
|
|
748
|
+
});
|
|
749
|
+
resolve(eventOutdated);
|
|
750
|
+
});
|
|
771
751
|
});
|
|
772
752
|
|
|
773
|
-
queueEntries.forEach(
|
|
774
|
-
(queueEntry) =>
|
|
775
|
-
(this.__keepalivePromises[queueEntry.ID] = checkAndUpdatePromise)
|
|
776
|
-
);
|
|
753
|
+
queueEntries.forEach((queueEntry) => (this.__keepalivePromises[queueEntry.ID] = checkAndUpdatePromise));
|
|
777
754
|
return await checkAndUpdatePromise;
|
|
778
755
|
}
|
|
779
756
|
|
|
@@ -798,15 +775,9 @@ class EventQueueProcessorBase {
|
|
|
798
775
|
return;
|
|
799
776
|
}
|
|
800
777
|
try {
|
|
801
|
-
await distributedLock.releaseLock(
|
|
802
|
-
this.context,
|
|
803
|
-
[this.eventType, this.eventSubType].join("##")
|
|
804
|
-
);
|
|
778
|
+
await distributedLock.releaseLock(this.context, [this.eventType, this.eventSubType].join("##"));
|
|
805
779
|
} catch (err) {
|
|
806
|
-
this.logger.error(
|
|
807
|
-
"Releasing distributed lock failed. Error:",
|
|
808
|
-
err.toString()
|
|
809
|
-
);
|
|
780
|
+
this.logger.error("Releasing distributed lock failed. Error:", err.toString());
|
|
810
781
|
}
|
|
811
782
|
}
|
|
812
783
|
|
|
@@ -814,14 +785,6 @@ class EventQueueProcessorBase {
|
|
|
814
785
|
return Object.values(statusMap).includes(EventProcessingStatus.Error);
|
|
815
786
|
}
|
|
816
787
|
|
|
817
|
-
getSelectNextChunk() {
|
|
818
|
-
return this.__selectNextChunk;
|
|
819
|
-
}
|
|
820
|
-
|
|
821
|
-
getSelectMaxChunkSize() {
|
|
822
|
-
return this.__selectMaxChunkSize;
|
|
823
|
-
}
|
|
824
|
-
|
|
825
788
|
clearEventProcessingContext() {
|
|
826
789
|
this.__processContext = null;
|
|
827
790
|
this.__processTx = null;
|
|
@@ -891,14 +854,18 @@ class EventQueueProcessorBase {
|
|
|
891
854
|
return this.__eventSubType;
|
|
892
855
|
}
|
|
893
856
|
|
|
894
|
-
get exceededEvents() {
|
|
895
|
-
return this.__eventsWithExceededTries;
|
|
896
|
-
}
|
|
897
|
-
|
|
898
857
|
get emptyChunkSelected() {
|
|
899
858
|
return this.__emptyChunkSelected;
|
|
900
859
|
}
|
|
901
860
|
|
|
861
|
+
get selectNextChunk() {
|
|
862
|
+
return this.__selectNextChunk;
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
get selectMaxChunkSize() {
|
|
866
|
+
return this.__selectMaxChunkSize;
|
|
867
|
+
}
|
|
868
|
+
|
|
902
869
|
set txUsageAllowed(value) {
|
|
903
870
|
this.__txUsageAllowed = value;
|
|
904
871
|
}
|