@cap-js-community/event-queue 0.1.50 → 0.1.52

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@
2
2
 
3
3
  const cds = require("@sap/cds");
4
4
 
5
- const { executeInNewTransaction } = require("./shared/cdsHelper");
5
+ const { executeInNewTransaction, TriggerRollback } = require("./shared/cdsHelper");
6
6
  const { EventProcessingStatus, TransactionMode } = require("./constants");
7
7
  const distributedLock = require("./shared/distributedLock");
8
8
  const EventQueueError = require("./EventQueueError");
@@ -17,8 +17,15 @@ const DEFAULT_RETRY_ATTEMPTS = 3;
17
17
  const DEFAULT_PARALLEL_EVENT_PROCESSING = 1;
18
18
  const LIMIT_PARALLEL_EVENT_PROCESSING = 10;
19
19
  const SELECT_LIMIT_EVENTS_PER_TICK = 100;
20
+ const DEFAULT_DELETE_FINISHED_EVENTS_AFTER = 0;
21
+ const DAYS_TO_MS = 24 * 60 * 60 * 1000;
22
+ const TRIES_FOR_EXCEEDED_EVENTS = 3;
20
23
 
21
24
  class EventQueueProcessorBase {
25
+ #eventsWithExceededTries = [];
26
+ #exceededTriesExceeded = [];
27
+ #selectedEventMap = {};
28
+
22
29
  constructor(context, eventType, eventSubType, config) {
23
30
  this.__context = context;
24
31
  this.__baseContext = context;
@@ -32,31 +39,34 @@ class EventQueueProcessorBase {
32
39
  this.__eventSubType = eventSubType;
33
40
  this.__queueEntriesWithPayloadMap = {};
34
41
  this.__config = config ?? {};
35
- this.__parallelEventProcessing =
36
- this.__config.parallelEventProcessing ??
37
- DEFAULT_PARALLEL_EVENT_PROCESSING;
42
+ this.__parallelEventProcessing = this.__config.parallelEventProcessing ?? DEFAULT_PARALLEL_EVENT_PROCESSING;
38
43
  if (this.__parallelEventProcessing > LIMIT_PARALLEL_EVENT_PROCESSING) {
39
44
  this.__parallelEventProcessing = LIMIT_PARALLEL_EVENT_PROCESSING;
40
45
  }
41
46
  // NOTE: keep the feature, this might be needed again
42
47
  this.__concurrentEventProcessing = false;
43
48
  this.__startTime = this.__config.startTime ?? new Date();
44
- this.__retryAttempts =
45
- this.__config.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
46
- this.__selectMaxChunkSize =
47
- this.__config.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
49
+ this.__retryAttempts = this.__config.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
50
+ this.__selectMaxChunkSize = this.__config.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
48
51
  this.__selectNextChunk = !!this.__config.checkForNextChunk;
49
52
  this.__keepalivePromises = {};
50
53
  this.__outdatedCheckEnabled = this.__config.eventOutdatedCheck ?? true;
51
- this.__transactionMode =
52
- this.__config.transactionMode ?? TransactionMode.isolated;
53
- this.__eventsWithExceededTries = [];
54
+ this.__transactionMode = this.__config.transactionMode ?? TransactionMode.isolated;
55
+ if (this.__config.deleteFinishedEventsAfterDays) {
56
+ this.__deleteFinishedEventsAfter =
57
+ Number.isInteger(this.__config.deleteFinishedEventsAfterDays) && this.__config.deleteFinishedEventsAfterDays > 0
58
+ ? this.__config.deleteFinishedEventsAfterDays
59
+ : DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
60
+ } else {
61
+ this.__deleteFinishedEventsAfter = DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
62
+ }
54
63
  this.__emptyChunkSelected = false;
55
64
  this.__lockAcquired = false;
56
65
  this.__txUsageAllowed = true;
57
66
  this.__txMap = {};
58
67
  this.__txRollback = {};
59
68
  this.__eventQueueConfig = eventQueueConfig.getConfigInstance();
69
+ this.__queueEntries = [];
60
70
  }
61
71
 
62
72
  /**
@@ -78,17 +88,11 @@ class EventQueueProcessorBase {
78
88
  }
79
89
 
80
90
  startPerformanceTracerEvents() {
81
- this.__performanceLoggerEvents = new PerformanceTracer(
82
- this.logger,
83
- "Processing events"
84
- );
91
+ this.__performanceLoggerEvents = new PerformanceTracer(this.logger, "Processing events");
85
92
  }
86
93
 
87
94
  startPerformanceTracerPreprocessing() {
88
- this.__performanceLoggerPreprocessing = new PerformanceTracer(
89
- this.logger,
90
- "Preprocessing events"
91
- );
95
+ this.__performanceLoggerPreprocessing = new PerformanceTracer(this.logger, "Preprocessing events");
92
96
  }
93
97
 
94
98
  endPerformanceTracerEvents() {
@@ -176,10 +180,7 @@ class EventQueueProcessorBase {
176
180
  eventType: this.__eventType,
177
181
  eventSubType: this.__eventSubType,
178
182
  });
179
- this.#determineAndAddEventStatusToMap(
180
- queueEntry.ID,
181
- EventProcessingStatus.Done
182
- );
183
+ this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Done);
183
184
  }
184
185
 
185
186
  /**
@@ -190,11 +191,9 @@ class EventQueueProcessorBase {
190
191
  * In this case the events should be clustered together and only one mail should be sent.
191
192
  */
192
193
  clusterQueueEntries() {
193
- Object.entries(this.__queueEntriesWithPayloadMap).forEach(
194
- ([key, { queueEntry, payload }]) => {
195
- this.addEntryToProcessingMap(key, queueEntry, payload);
196
- }
197
- );
194
+ Object.entries(this.__queueEntriesWithPayloadMap).forEach(([key, { queueEntry, payload }]) => {
195
+ this.addEntryToProcessingMap(key, queueEntry, payload);
196
+ });
198
197
  }
199
198
 
200
199
  /**
@@ -225,28 +224,23 @@ class EventQueueProcessorBase {
225
224
  * event processing.
226
225
  * @param {Array} queueEntries which has been selected from event queue table and been modified by modifyQueueEntry
227
226
  * @param {Array<Object>} queueEntryProcessingStatusTuple Array of tuple <queueEntryId, processingStatus>
227
+ * @param {boolean} returnMap Allows the function to allow the result as map
228
228
  * @return {Object} statusMap Map which contains all events for which a status has been set so far
229
229
  */
230
- setEventStatus(queueEntries, queueEntryProcessingStatusTuple) {
230
+ setEventStatus(queueEntries, queueEntryProcessingStatusTuple, returnMap = false) {
231
231
  this.logger.debug("setting event status for entries", {
232
- queueEntryProcessingStatusTuple: JSON.stringify(
233
- queueEntryProcessingStatusTuple
234
- ),
232
+ queueEntryProcessingStatusTuple: JSON.stringify(queueEntryProcessingStatusTuple),
235
233
  eventType: this.__eventType,
236
234
  eventSubType: this.__eventSubType,
237
235
  });
238
- const statusMap = this.commitOnEventLevel ? {} : this.__statusMap;
236
+ const statusMap = this.commitOnEventLevel || returnMap ? {} : this.__statusMap;
239
237
  try {
240
238
  queueEntryProcessingStatusTuple.forEach(([id, processingStatus]) =>
241
239
  this.#determineAndAddEventStatusToMap(id, processingStatus, statusMap)
242
240
  );
243
241
  } catch (error) {
244
242
  queueEntries.forEach((queueEntry) =>
245
- this.#determineAndAddEventStatusToMap(
246
- queueEntry.ID,
247
- EventProcessingStatus.Error,
248
- statusMap
249
- )
243
+ this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error, statusMap)
250
244
  );
251
245
  this.logger.error(
252
246
  `The supplied status tuple doesn't have the required structure. Setting all entries to error. Error: ${error.toString()}`,
@@ -269,24 +263,16 @@ class EventQueueProcessorBase {
269
263
  try {
270
264
  queueEntry.payload = JSON.parse(queueEntry.payload);
271
265
  } catch {
272
- return queueEntry.payload;
266
+ /* empty */
273
267
  }
274
268
  }
275
269
 
276
- #determineAndAddEventStatusToMap(
277
- id,
278
- processingStatus,
279
- statusMap = this.__statusMap
280
- ) {
270
+ #determineAndAddEventStatusToMap(id, processingStatus, statusMap = this.__statusMap) {
281
271
  if (!statusMap[id]) {
282
272
  statusMap[id] = processingStatus;
283
273
  return;
284
274
  }
285
- if (
286
- [EventProcessingStatus.Error, EventProcessingStatus.Exceeded].includes(
287
- statusMap[id]
288
- )
289
- ) {
275
+ if ([EventProcessingStatus.Error, EventProcessingStatus.Exceeded].includes(statusMap[id])) {
290
276
  // NOTE: worst aggregation --> if already error|exceeded keep this state
291
277
  return;
292
278
  }
@@ -306,17 +292,9 @@ class EventQueueProcessorBase {
306
292
  }
307
293
  );
308
294
  queueEntries.forEach((queueEntry) =>
309
- this.#determineAndAddEventStatusToMap(
310
- queueEntry.ID,
311
- EventProcessingStatus.Error
312
- )
313
- );
314
- return Object.fromEntries(
315
- queueEntries.map((queueEntry) => [
316
- queueEntry.ID,
317
- EventProcessingStatus.Error,
318
- ])
295
+ this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error)
319
296
  );
297
+ return Object.fromEntries(queueEntries.map((queueEntry) => [queueEntry.ID, EventProcessingStatus.Error]));
320
298
  }
321
299
 
322
300
  /**
@@ -324,10 +302,7 @@ class EventQueueProcessorBase {
324
302
  * selected events a status has been submitted. Persisting the status of events is done in a dedicated database tx.
325
303
  * The function accepts no arguments as there are dedicated functions to set the status of events (e.g. setEventStatus)
326
304
  */
327
- async persistEventStatus(
328
- tx,
329
- { skipChecks, statusMap = this.__statusMap } = {}
330
- ) {
305
+ async persistEventStatus(tx, { skipChecks, statusMap = this.__statusMap } = {}) {
331
306
  this.logger.debug("entering persistEventStatus", {
332
307
  eventType: this.__eventType,
333
308
  eventSubType: this.__eventSubType,
@@ -338,9 +313,7 @@ class EventQueueProcessorBase {
338
313
  }
339
314
  this.#ensureEveryStatusIsAllowed(statusMap);
340
315
 
341
- const { success, failed, exceeded, invalidAttempts } = Object.entries(
342
- statusMap
343
- ).reduce(
316
+ const { success, failed, exceeded, invalidAttempts } = Object.entries(statusMap).reduce(
344
317
  (result, [notificationEntityId, processingStatus]) => {
345
318
  this.__commitedStatusMap[notificationEntityId] = processingStatus;
346
319
  if (processingStatus === EventProcessingStatus.Open) {
@@ -380,34 +353,24 @@ class EventQueueProcessorBase {
380
353
  .where("ID IN", invalidAttempts)
381
354
  );
382
355
  }
383
- if (success.length) {
356
+ const ts = new Date().toISOString();
357
+ const updateTuples = [
358
+ [success, EventProcessingStatus.Done],
359
+ [failed, EventProcessingStatus.Error],
360
+ [exceeded, EventProcessingStatus.Exceeded],
361
+ ];
362
+
363
+ for (const [eventIds, status] of updateTuples) {
364
+ if (!eventIds.length) {
365
+ continue;
366
+ }
384
367
  await tx.run(
385
368
  UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
386
369
  .set({
387
- status: EventProcessingStatus.Done,
388
- lastAttemptTimestamp: new Date().toISOString(),
389
- })
390
- .where("ID IN", success)
391
- );
392
- }
393
- if (failed.length) {
394
- await tx.run(
395
- UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
396
- .where("ID IN", failed)
397
- .with({
398
- status: EventProcessingStatus.Error,
399
- lastAttemptTimestamp: new Date().toISOString(),
400
- })
401
- );
402
- }
403
- if (exceeded.length) {
404
- await tx.run(
405
- UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
406
- .where("ID IN", exceeded)
407
- .with({
408
- status: EventProcessingStatus.Exceeded,
409
- lastAttemptTimestamp: new Date().toISOString(),
370
+ status: status,
371
+ lastAttemptTimestamp: ts,
410
372
  })
373
+ .where("ID IN", eventIds)
411
374
  );
412
375
  }
413
376
  this.logger.debug("exiting persistEventStatus", {
@@ -416,26 +379,39 @@ class EventQueueProcessorBase {
416
379
  });
417
380
  }
418
381
 
382
+ async deleteFinishedEvents(tx) {
383
+ if (!this.__deleteFinishedEventsAfter) {
384
+ return;
385
+ }
386
+ const deleteCount = await tx.run(
387
+ DELETE.from(this.__eventQueueConfig.tableNameEventQueue).where(
388
+ "type =",
389
+ this.eventType,
390
+ "AND subType=",
391
+ this.eventSubType,
392
+ "AND lastAttemptTimestamp <=",
393
+ new Date(Date.now() - this.__deleteFinishedEventsAfter * DAYS_TO_MS).toISOString()
394
+ )
395
+ );
396
+ this.logger.debug("Deleted finished events", {
397
+ eventType: this.eventType,
398
+ eventSubType: this.eventSubType,
399
+ deleteFinishedEventsAfter: this.__deleteFinishedEventsAfter,
400
+ deleteCount,
401
+ });
402
+ }
403
+
419
404
  #ensureEveryQueueEntryHasStatus() {
420
405
  this.__queueEntries.forEach((queueEntry) => {
421
- if (
422
- queueEntry.ID in this.__statusMap ||
423
- queueEntry.ID in this.__commitedStatusMap
424
- ) {
406
+ if (queueEntry.ID in this.__statusMap || queueEntry.ID in this.__commitedStatusMap) {
425
407
  return;
426
408
  }
427
- this.logger.error(
428
- "Missing status for selected event entry. Setting status to error",
429
- {
430
- eventType: this.__eventType,
431
- eventSubType: this.__eventSubType,
432
- queueEntry,
433
- }
434
- );
435
- this.#determineAndAddEventStatusToMap(
436
- queueEntry.ID,
437
- EventProcessingStatus.Error
438
- );
409
+ this.logger.error("Missing status for selected event entry. Setting status to error", {
410
+ eventType: this.__eventType,
411
+ eventSubType: this.__eventSubType,
412
+ queueEntry,
413
+ });
414
+ this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
439
415
  });
440
416
  }
441
417
 
@@ -452,22 +428,19 @@ class EventQueueProcessorBase {
452
428
  return;
453
429
  }
454
430
 
455
- this.logger.error(
456
- "Not allowed event status returned. Only Open, Done, Error is allowed!",
457
- {
458
- eventType: this.__eventType,
459
- eventSubType: this.__eventSubType,
460
- queueEntryId,
461
- status: statusMap[queueEntryId],
462
- }
463
- );
431
+ this.logger.error("Not allowed event status returned. Only Open, Done, Error is allowed!", {
432
+ eventType: this.__eventType,
433
+ eventSubType: this.__eventSubType,
434
+ queueEntryId,
435
+ status: statusMap[queueEntryId],
436
+ });
464
437
  delete statusMap[queueEntryId];
465
438
  });
466
439
  }
467
440
 
468
441
  #ensureOnlySelectedQueueEntries(statusMap) {
469
442
  Object.keys(statusMap).forEach((queueEntryId) => {
470
- if (this.__queueEntriesMap[queueEntryId]) {
443
+ if (this.#selectedEventMap[queueEntryId]) {
471
444
  return;
472
445
  }
473
446
 
@@ -484,18 +457,12 @@ class EventQueueProcessorBase {
484
457
  }
485
458
 
486
459
  handleErrorDuringClustering(error) {
487
- this.logger.error(
488
- `Error during clustering of events - setting all queue entries to error. Error: ${error}`,
489
- {
490
- eventType: this.__eventType,
491
- eventSubType: this.__eventSubType,
492
- }
493
- );
460
+ this.logger.error(`Error during clustering of events - setting all queue entries to error. Error: ${error}`, {
461
+ eventType: this.__eventType,
462
+ eventSubType: this.__eventSubType,
463
+ });
494
464
  this.__queueEntries.forEach((queueEntry) => {
495
- this.#determineAndAddEventStatusToMap(
496
- queueEntry.ID,
497
- EventProcessingStatus.Error
498
- );
465
+ this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
499
466
  });
500
467
  }
501
468
 
@@ -508,29 +475,15 @@ class EventQueueProcessorBase {
508
475
  eventSubType: this.__eventSubType,
509
476
  }
510
477
  );
511
- this.#determineAndAddEventStatusToMap(
512
- queueEntry.ID,
513
- EventProcessingStatus.Error
514
- );
478
+ this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
515
479
  }
516
480
 
517
- static async handleMissingTypeImplementation(
518
- context,
519
- eventType,
520
- eventSubType
521
- ) {
522
- const baseInstance = new EventQueueProcessorBase(
523
- context,
481
+ static async handleMissingTypeImplementation(context, eventType, eventSubType) {
482
+ const baseInstance = new EventQueueProcessorBase(context, eventType, eventSubType);
483
+ baseInstance.logger.error("No Implementation found in the provided configuration file.", {
524
484
  eventType,
525
- eventSubType
526
- );
527
- baseInstance.logger.error(
528
- "No Implementation found in the provided configuration file.",
529
- {
530
- eventType,
531
- eventSubType,
532
- }
533
- );
485
+ eventSubType,
486
+ });
534
487
  }
535
488
 
536
489
  /**
@@ -542,137 +495,179 @@ class EventQueueProcessorBase {
542
495
  */
543
496
  async getQueueEntriesAndSetToInProgress() {
544
497
  let result = [];
545
- await executeInNewTransaction(
546
- this.__baseContext,
547
- "eventQueue-getQueueEntriesAndSetToInProgress",
548
- async (tx) => {
549
- const entries = await tx.run(
550
- SELECT.from(this.__eventQueueConfig.tableNameEventQueue)
551
- .forUpdate({ wait: this.__eventQueueConfig.forUpdateTimeout })
552
- .limit(this.getSelectMaxChunkSize())
553
- .where(
554
- "type =",
555
- this.__eventType,
556
- "AND subType=",
557
- this.__eventSubType,
558
- "AND ( status =",
559
- EventProcessingStatus.Open,
560
- "OR ( status =",
561
- EventProcessingStatus.Error,
562
- "AND lastAttemptTimestamp <=",
563
- this.__startTime.toISOString(),
564
- ") OR ( status =",
565
- EventProcessingStatus.InProgress,
566
- "AND lastAttemptTimestamp <=",
567
- new Date(
568
- new Date().getTime() - this.__eventQueueConfig.globalTxTimeout
569
- ).toISOString(),
570
- ") )"
571
- )
572
- .orderBy("createdAt", "ID")
573
- );
574
-
575
- if (!entries.length) {
576
- this.logger.debug("no entries available for processing", {
577
- eventType: this.__eventType,
578
- eventSubType: this.__eventSubType,
579
- });
580
- this.__emptyChunkSelected = true;
581
- return;
582
- }
583
-
584
- const { exceededTries, openEvents } =
585
- this.#filterExceededEvents(entries);
586
- if (exceededTries.length) {
587
- this.__eventsWithExceededTries = exceededTries;
588
- }
589
- result = openEvents;
590
-
591
- if (!result.length) {
592
- this.__emptyChunkSelected = true;
593
- return;
594
- }
498
+ await executeInNewTransaction(this.__baseContext, "eventQueue-getQueueEntriesAndSetToInProgress", async (tx) => {
499
+ const entries = await tx.run(
500
+ SELECT.from(this.__eventQueueConfig.tableNameEventQueue)
501
+ .forUpdate({ wait: this.__eventQueueConfig.forUpdateTimeout })
502
+ .limit(this.selectMaxChunkSize)
503
+ .where(
504
+ "type =",
505
+ this.__eventType,
506
+ "AND subType=",
507
+ this.__eventSubType,
508
+ "AND ( status =",
509
+ EventProcessingStatus.Open,
510
+ "OR ( status =",
511
+ EventProcessingStatus.Error,
512
+ "AND lastAttemptTimestamp <=",
513
+ this.__startTime.toISOString(),
514
+ ") OR ( status =",
515
+ EventProcessingStatus.InProgress,
516
+ "AND lastAttemptTimestamp <=",
517
+ new Date(new Date().getTime() - this.__eventQueueConfig.globalTxTimeout).toISOString(),
518
+ ") )"
519
+ )
520
+ .orderBy("createdAt", "ID")
521
+ );
595
522
 
596
- this.logger.info("Selected event queue entries for processing", {
597
- queueEntriesCount: result.length,
523
+ if (!entries.length) {
524
+ this.logger.debug("no entries available for processing", {
598
525
  eventType: this.__eventType,
599
526
  eventSubType: this.__eventSubType,
600
527
  });
528
+ this.__emptyChunkSelected = true;
529
+ return;
530
+ }
601
531
 
602
- const isoTimestamp = new Date().toISOString();
603
- await tx.run(
604
- UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
605
- .with({
606
- status: EventProcessingStatus.InProgress,
607
- lastAttemptTimestamp: isoTimestamp,
608
- attempts: { "+=": 1 },
609
- })
610
- .where(
611
- "ID IN",
612
- result.map(({ ID }) => ID)
613
- )
614
- );
615
- result.forEach((entry) => {
616
- entry.lastAttemptTimestamp = isoTimestamp;
617
- // NOTE: empty payloads are supported on DB-Level.
618
- // Behaviour of event queue is: null as payload is treated as obsolete/done
619
- // For supporting this convert null to empty string --> "" as payload will be processed normally
620
- if (entry.payload === null) {
621
- entry.payload = "";
622
- }
623
- });
532
+ this.#selectedEventMap = arrayToFlatMap(entries);
533
+ const { exceededTries, openEvents, exceededTriesExceeded } = this.#filterExceededEvents(entries);
534
+ if (exceededTries.length) {
535
+ this.#eventsWithExceededTries = exceededTries;
624
536
  }
625
- );
626
- this.__queueEntries = result;
627
- this.__queueEntriesMap = arrayToFlatMap(result);
537
+ if (exceededTriesExceeded.length) {
538
+ this.#exceededTriesExceeded = exceededTriesExceeded;
539
+ }
540
+
541
+ result = openEvents;
542
+
543
+ if (!result.length) {
544
+ this.__emptyChunkSelected = true;
545
+ }
546
+
547
+ this.logger.info("Selected event queue entries for processing", {
548
+ queueEntriesCount: result.length,
549
+ eventType: this.__eventType,
550
+ eventSubType: this.__eventSubType,
551
+ });
552
+
553
+ const isoTimestamp = new Date().toISOString();
554
+ await tx.run(
555
+ UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
556
+ .with({
557
+ status: EventProcessingStatus.InProgress,
558
+ lastAttemptTimestamp: isoTimestamp,
559
+ attempts: { "+=": 1 },
560
+ })
561
+ .where(
562
+ "ID IN",
563
+ entries.map(({ ID }) => ID)
564
+ )
565
+ );
566
+ entries.forEach((entry) => {
567
+ entry.lastAttemptTimestamp = isoTimestamp;
568
+ // NOTE: empty payloads are supported on DB-Level.
569
+ // Behaviour of event queue is: null as payload is treated as obsolete/done
570
+ // For supporting this convert null to empty string --> "" as payload will be processed normally
571
+ if (entry.payload === null) {
572
+ entry.payload = "";
573
+ }
574
+ });
575
+ this.__queueEntries = result;
576
+ this.__queueEntriesMap = arrayToFlatMap(result);
577
+ });
628
578
  return result;
629
579
  }
630
580
 
631
581
  #filterExceededEvents(events) {
632
582
  return events.reduce(
633
583
  (result, event) => {
634
- if (event.attempts === this.__retryAttempts) {
584
+ if (event.attempts === this.__retryAttempts + TRIES_FOR_EXCEEDED_EVENTS) {
585
+ result.exceededTriesExceeded.push(event);
586
+ } else if (event.attempts >= this.__retryAttempts) {
635
587
  result.exceededTries.push(event);
636
588
  } else {
637
589
  result.openEvents.push(event);
638
590
  }
639
591
  return result;
640
592
  },
641
- { exceededTries: [], openEvents: [] }
593
+ { exceededTries: [], openEvents: [], exceededTriesExceeded: [] }
642
594
  );
643
595
  }
644
596
 
645
- async handleExceededEvents(exceededEvents) {
646
- await this.tx.run(
647
- UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
648
- .with({
649
- status: EventProcessingStatus.Exceeded,
650
- })
651
- .where(
652
- "ID IN",
653
- exceededEvents.map(({ ID }) => ID)
654
- )
655
- );
656
- this.logger.error(
657
- "The retry attempts for the following events are exceeded",
658
- {
597
+ async handleExceededEvents() {
598
+ await this.#handleExceededTriesExceeded();
599
+ if (!this.#eventsWithExceededTries.length) {
600
+ return;
601
+ }
602
+
603
+ for (const exceededEvent of this.#eventsWithExceededTries) {
604
+ await executeInNewTransaction(
605
+ this.context,
606
+ `eventQueue-handleExceededEvents-${this.eventType}##${this.eventSubType}`,
607
+ async (tx) => {
608
+ try {
609
+ this.processEventContext = tx.context;
610
+ this.modifyQueueEntry(exceededEvent);
611
+ await this.hookForExceededEvents({ ...exceededEvent });
612
+ this.logger.warn("The retry attempts for the following events are exceeded", {
613
+ eventType: this.__eventType,
614
+ eventSubType: this.__eventSubType,
615
+ retryAttempts: this.__retryAttempts,
616
+ queueEntriesId: exceededEvent.ID,
617
+ currentAttempt: exceededEvent.attempts,
618
+ });
619
+ await this.#persistEventQueueStatusForExceeded(this.tx, [exceededEvent], EventProcessingStatus.Exceeded);
620
+ } catch (err) {
621
+ this.logger.error(
622
+ `Caught error during hook for exceeded events - setting queue entry to error. Please catch your promises/exceptions. Error: ${err}`,
623
+ {
624
+ eventType: this.__eventType,
625
+ eventSubType: this.__eventSubType,
626
+ retryAttempts: this.__retryAttempts,
627
+ queueEntriesId: exceededEvent.ID,
628
+ currentAttempt: exceededEvent.attempts,
629
+ }
630
+ );
631
+ await executeInNewTransaction(this.context, "error-hookForExceededEvents", async (tx) =>
632
+ this.#persistEventQueueStatusForExceeded(tx, [exceededEvent], EventProcessingStatus.Error)
633
+ );
634
+ throw new TriggerRollback();
635
+ }
636
+ }
637
+ );
638
+ }
639
+ }
640
+
641
+ async #handleExceededTriesExceeded() {
642
+ if (this.#exceededTriesExceeded.length) {
643
+ this.logger.error("Event hook failure exceeded, status set to 'exceeded' without invoking hook again!", {
659
644
  eventType: this.__eventType,
660
645
  eventSubType: this.__eventSubType,
661
- retryAttempts: this.__retryAttempts,
662
- queueEntriesIds: exceededEvents.map(({ ID }) => ID),
663
- }
646
+ queueEntriesIds: this.#eventsWithExceededTries.map(({ ID }) => ID),
647
+ });
648
+ await executeInNewTransaction(this.context, "exceededTriesExceeded", async (tx) => {
649
+ await this.#persistEventQueueStatusForExceeded(tx, this.#exceededTriesExceeded, EventProcessingStatus.Exceeded);
650
+ });
651
+ }
652
+ }
653
+
654
+ async #persistEventQueueStatusForExceeded(tx, events, status) {
655
+ const statusMap = this.setEventStatus(
656
+ events,
657
+ events.map((e) => [e.ID, status]),
658
+ true
664
659
  );
665
- await this.hookForExceededEvents(exceededEvents);
660
+ await this.persistEventStatus(tx, { statusMap, skipChecks: true });
666
661
  }
667
662
 
668
663
  /**
669
664
  * This function enables the possibility to execute custom actions for events for which the retry attempts have been
670
665
  * exceeded. As always a valid transaction is available with this.tx. This transaction will be committed after the
671
666
  * execution of this function.
672
- * @param {Object} exceededEvents exceeded event queue entries
667
+ * @param {Object} exceededEvent exceeded event queue entry
673
668
  */
674
669
  // eslint-disable-next-line no-unused-vars
675
- async hookForExceededEvents(exceededEvents) {}
670
+ async hookForExceededEvents(exceededEvent) {}
676
671
 
677
672
  /**
678
673
  * This function serves the purpose of mass enabled preloading data for processing the events which are added with
@@ -695,9 +690,7 @@ class EventQueueProcessorBase {
695
690
  return false;
696
691
  }
697
692
  let eventOutdated;
698
- const runningChecks = queueEntries
699
- .map((queueEntry) => this.__keepalivePromises[queueEntry.ID])
700
- .filter((p) => p);
693
+ const runningChecks = queueEntries.map((queueEntry) => this.__keepalivePromises[queueEntry.ID]).filter((p) => p);
701
694
  if (runningChecks.length === queueEntries.length) {
702
695
  const results = await Promise.allSettled(runningChecks);
703
696
  for (const { value } of results) {
@@ -709,71 +702,55 @@ class EventQueueProcessorBase {
709
702
  } else if (runningChecks.length) {
710
703
  await Promise.allSettled(runningChecks);
711
704
  }
712
- const checkAndUpdatePromise = new Promise((resolve) => {
713
- executeInNewTransaction(
714
- this.__baseContext,
715
- "eventProcessing-isOutdatedAndKeepalive",
716
- async (tx) => {
717
- const queueEntriesFresh = await tx.run(
718
- SELECT.from(this.__eventQueueConfig.tableNameEventQueue)
719
- .forUpdate({ wait: this.__eventQueueConfig.forUpdateTimeout })
705
+ const checkAndUpdatePromise = new Promise((resolve, reject) => {
706
+ executeInNewTransaction(this.__baseContext, "eventProcessing-isOutdatedAndKeepalive", async (tx) => {
707
+ const queueEntriesFresh = await tx.run(
708
+ SELECT.from(this.__eventQueueConfig.tableNameEventQueue)
709
+ .forUpdate({ wait: this.__eventQueueConfig.forUpdateTimeout })
710
+ .where(
711
+ "ID IN",
712
+ queueEntries.map(({ ID }) => ID)
713
+ )
714
+ .columns("ID", "lastAttemptTimestamp")
715
+ );
716
+ eventOutdated = queueEntriesFresh.some((queueEntryFresh) => {
717
+ const queueEntry = this.__queueEntriesMap[queueEntryFresh.ID];
718
+ return queueEntry?.lastAttemptTimestamp !== queueEntryFresh.lastAttemptTimestamp;
719
+ });
720
+ let newTs = new Date().toISOString();
721
+ if (!eventOutdated) {
722
+ await tx.run(
723
+ UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
724
+ .set("lastAttemptTimestamp =", newTs)
720
725
  .where(
721
726
  "ID IN",
722
727
  queueEntries.map(({ ID }) => ID)
723
728
  )
724
- .columns("ID", "lastAttemptTimestamp")
725
729
  );
726
- eventOutdated = queueEntriesFresh.some((queueEntryFresh) => {
727
- const queueEntry = this.__queueEntriesMap[queueEntryFresh.ID];
728
- return (
729
- queueEntry?.lastAttemptTimestamp !==
730
- queueEntryFresh.lastAttemptTimestamp
731
- );
732
- });
733
- let newTs = new Date().toISOString();
734
- if (!eventOutdated) {
735
- await tx.run(
736
- UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
737
- .set("lastAttemptTimestamp =", newTs)
738
- .where(
739
- "ID IN",
740
- queueEntries.map(({ ID }) => ID)
741
- )
742
- );
743
- } else {
744
- newTs = null;
745
- this.logger.warn(
746
- "event data has been modified. Processing skipped.",
747
- {
748
- eventType: this.__eventType,
749
- eventSubType: this.__eventSubType,
750
- queueEntriesIds: queueEntries.map(({ ID }) => ID),
751
- }
752
- );
753
- queueEntries.forEach(
754
- ({ ID: queueEntryId }) =>
755
- delete this.__queueEntriesMap[queueEntryId]
756
- );
757
- }
758
- this.__queueEntries = Object.values(this.__queueEntriesMap);
759
- queueEntriesFresh.forEach((queueEntryFresh) => {
760
- if (this.__queueEntriesMap[queueEntryFresh.ID]) {
761
- const queueEntry = this.__queueEntriesMap[queueEntryFresh.ID];
762
- if (newTs) {
763
- queueEntry.lastAttemptTimestamp = newTs;
764
- }
765
- }
766
- delete this.__keepalivePromises[queueEntryFresh.ID];
730
+ } else {
731
+ newTs = null;
732
+ this.logger.warn("event data has been modified. Processing skipped.", {
733
+ eventType: this.__eventType,
734
+ eventSubType: this.__eventSubType,
735
+ queueEntriesIds: queueEntries.map(({ ID }) => ID),
767
736
  });
768
- resolve(eventOutdated);
737
+ queueEntries.forEach(({ ID: queueEntryId }) => delete this.__queueEntriesMap[queueEntryId]);
769
738
  }
770
- );
739
+ this.__queueEntries = Object.values(this.__queueEntriesMap);
740
+ queueEntriesFresh.forEach((queueEntryFresh) => {
741
+ if (this.__queueEntriesMap[queueEntryFresh.ID]) {
742
+ const queueEntry = this.__queueEntriesMap[queueEntryFresh.ID];
743
+ if (newTs) {
744
+ queueEntry.lastAttemptTimestamp = newTs;
745
+ }
746
+ }
747
+ delete this.__keepalivePromises[queueEntryFresh.ID];
748
+ });
749
+ resolve(eventOutdated);
750
+ }).catch(reject);
771
751
  });
772
752
 
773
- queueEntries.forEach(
774
- (queueEntry) =>
775
- (this.__keepalivePromises[queueEntry.ID] = checkAndUpdatePromise)
776
- );
753
+ queueEntries.forEach((queueEntry) => (this.__keepalivePromises[queueEntry.ID] = checkAndUpdatePromise));
777
754
  return await checkAndUpdatePromise;
778
755
  }
779
756
 
@@ -798,15 +775,9 @@ class EventQueueProcessorBase {
798
775
  return;
799
776
  }
800
777
  try {
801
- await distributedLock.releaseLock(
802
- this.context,
803
- [this.eventType, this.eventSubType].join("##")
804
- );
778
+ await distributedLock.releaseLock(this.context, [this.eventType, this.eventSubType].join("##"));
805
779
  } catch (err) {
806
- this.logger.error(
807
- "Releasing distributed lock failed. Error:",
808
- err.toString()
809
- );
780
+ this.logger.error("Releasing distributed lock failed. Error:", err.toString());
810
781
  }
811
782
  }
812
783
 
@@ -814,14 +785,6 @@ class EventQueueProcessorBase {
814
785
  return Object.values(statusMap).includes(EventProcessingStatus.Error);
815
786
  }
816
787
 
817
- getSelectNextChunk() {
818
- return this.__selectNextChunk;
819
- }
820
-
821
- getSelectMaxChunkSize() {
822
- return this.__selectMaxChunkSize;
823
- }
824
-
825
788
  clearEventProcessingContext() {
826
789
  this.__processContext = null;
827
790
  this.__processTx = null;
@@ -891,14 +854,18 @@ class EventQueueProcessorBase {
891
854
  return this.__eventSubType;
892
855
  }
893
856
 
894
- get exceededEvents() {
895
- return this.__eventsWithExceededTries;
896
- }
897
-
898
857
  get emptyChunkSelected() {
899
858
  return this.__emptyChunkSelected;
900
859
  }
901
860
 
861
+ get selectNextChunk() {
862
+ return this.__selectNextChunk;
863
+ }
864
+
865
+ get selectMaxChunkSize() {
866
+ return this.__selectMaxChunkSize;
867
+ }
868
+
902
869
  set txUsageAllowed(value) {
903
870
  this.__txUsageAllowed = value;
904
871
  }