@cap-js-community/event-queue 0.1.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,922 @@
1
+ "use strict";
2
+
3
+ const cds = require("@sap/cds");
4
+
5
+ const { executeInNewTransaction } = require("./shared/cdsHelper");
6
+ const { EventProcessingStatus } = require("./constants");
7
+ const distributedLock = require("./shared/distributedLock");
8
+ const EventQueueError = require("./EventQueueError");
9
+ const { arrayToFlatMap } = require("./shared/common");
10
+ const eventQueueConfig = require("./config");
11
+ const PerformanceTracer = require("./shared/PerformanceTracer");
12
+
13
+ const IMPLEMENT_ERROR_MESSAGE = "needs to be reimplemented";
14
+ const COMPONENT_NAME = "eventQueue/EventQueueProcessorBase";
15
+
16
+ const DEFAULT_RETRY_ATTEMPTS = 3;
17
+ const DEFAULT_PARALLEL_EVENT_PROCESSING = 1;
18
+ const LIMIT_PARALLEL_EVENT_PROCESSING = 10;
19
+ const SELECT_LIMIT_EVENTS_PER_TICK = 100;
20
+
21
+ class EventQueueProcessorBase {
22
+ constructor(context, eventType, eventSubType, config) {
23
+ this.__context = context;
24
+ this.__baseContext = context;
25
+ this.__tx = cds.tx(context);
26
+ this.__baseLogger = cds.log(COMPONENT_NAME);
27
+ this.__logger = null;
28
+ this.__eventProcessingMap = {};
29
+ this.__statusMap = {};
30
+ this.__commitedStatusMap = {};
31
+ this.__eventType = eventType;
32
+ this.__eventSubType = eventSubType;
33
+ this.__queueEntriesWithPayloadMap = {};
34
+ this.__config = config ?? {};
35
+ this.__parallelEventProcessing =
36
+ this.__config.parallelEventProcessing ??
37
+ DEFAULT_PARALLEL_EVENT_PROCESSING;
38
+ if (this.__parallelEventProcessing > LIMIT_PARALLEL_EVENT_PROCESSING) {
39
+ this.__parallelEventProcessing = LIMIT_PARALLEL_EVENT_PROCESSING;
40
+ }
41
+ // NOTE: keep the feature, this might be needed again
42
+ this.__concurrentEventProcessing = false;
43
+ this.__startTime = this.__config.startTime ?? new Date();
44
+ this.__retryAttempts =
45
+ this.__config.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
46
+ this.__selectMaxChunkSize =
47
+ this.__config.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
48
+ this.__selectNextChunk = !!this.__config.checkForNextChunk;
49
+ this.__keepalivePromises = {};
50
+ this.__outdatedCheckEnabled = this.__config.eventOutdatedCheck ?? true;
51
+ this.__commitOnEventLevel = this.__config.commitOnEventLevel ?? true;
52
+ this.__eventsWithExceededTries = [];
53
+ this.__emptyChunkSelected = false;
54
+ this.__lockAcquired = false;
55
+ this.__txUsageAllowed = true;
56
+ this.__txMap = {};
57
+ this.__txRollback = {};
58
+ this.__eventQueueConfig = eventQueueConfig.getConfigInstance();
59
+ }
60
+
61
+ /**
62
+ * Process one or multiple events depending on the clustering algorithm by default there it's one event
63
+ * @param processContext the context valid for the event processing. This context is associated with a valid transaction
64
+ * Access to the context is also possible with this.getContextForEventProcessing(key).
65
+ * The associated tx can be accessed with this.getTxForEventProcessing(key).
66
+ * @param {string} key cluster key generated during the clustering step. By default, this is ID of the event queue entry
67
+ * @param {Array<Object>} queueEntries this are the queueEntries which are collected during the clustering step for the given
68
+ * clustering key
69
+ * @param {Object} payload resulting from the functions checkEventAndGeneratePayload and the clustering function
70
+ * @returns {Promise<Array <Array <String, Number>>>} Must return an array of the length of passed queueEntries
71
+ * This array needs to be nested based on the following structure: [ ["eventId1", EventProcessingStatus.Done],
72
+ * ["eventId2", EventProcessingStatus.Error] ]
73
+ */
74
+ // eslint-disable-next-line no-unused-vars
75
+ async processEvent(processContext, key, queueEntries, payload) {
76
+ throw new Error(IMPLEMENT_ERROR_MESSAGE);
77
+ }
78
+
79
+ startPerformanceTracerEvents() {
80
+ this.__performanceLoggerEvents = new PerformanceTracer(
81
+ this.logger,
82
+ "Processing events"
83
+ );
84
+ }
85
+
86
+ startPerformanceTracerPreprocessing() {
87
+ this.__performanceLoggerPreprocessing = new PerformanceTracer(
88
+ this.logger,
89
+ "Preprocessing events"
90
+ );
91
+ }
92
+
93
+ endPerformanceTracerEvents() {
94
+ this.__performanceLoggerEvents?.endPerformanceTrace(
95
+ { threshold: 50 },
96
+ {
97
+ eventType: this.eventType,
98
+ eventSubType: this.eventSubType,
99
+ }
100
+ );
101
+ }
102
+
103
+ endPerformanceTracerPreprocessing() {
104
+ this.__performanceLoggerPreprocessing?.endPerformanceTrace(
105
+ { threshold: 50 },
106
+ {
107
+ eventType: this.eventType,
108
+ eventSubType: this.eventSubType,
109
+ }
110
+ );
111
+ }
112
+
113
+ logTimeExceeded(iterationCounter) {
114
+ this.logger.info("Exiting event queue processing as max time exceeded", {
115
+ eventType: this.eventType,
116
+ eventSubType: this.eventSubType,
117
+ iterationCounter,
118
+ });
119
+ }
120
+
121
+ logStartMessage(queueEntries) {
122
+ // TODO: how to handle custom fields
123
+ this.logger.info("Processing queue event", {
124
+ numberQueueEntries: queueEntries.length,
125
+ eventType: this.__eventType,
126
+ eventSubType: this.__eventSubType,
127
+ });
128
+ }
129
+
130
+ /**
131
+ * This function will be called for every event which should to be processed. Within this function basic validations
132
+ * should be done, e.g. is the event still valid and should be processed. Also, this step should be used to gather the
133
+ * required data for the clustering step. Keep in mind that this function will be called for every event and not once
134
+ * for all events. Mass data select should be done later (beforeProcessingEvents).
135
+ * If no payload is returned the status will be set to done. Transaction is available with this.tx;
136
+ * this transaction will always be rollbacked so do not use this transaction persisting data.
137
+ * @param {Object} queueEntry which has been selected from event queue table and been modified by modifyQueueEntry
138
+ * @returns {Promise<Object>} payload which is needed for clustering the events.
139
+ */
140
+ async checkEventAndGeneratePayload(queueEntry) {
141
+ return queueEntry.payload;
142
+ }
143
+
144
+ /**
145
+ * This function will be called for every event which should to be processed. This functions sets for every event
146
+ * the payload which will be passed to the clustering functions.
147
+ * @param {Object} queueEntry which has been selected from event queue table and been modified by modifyQueueEntry
148
+ * @param {Object} payload which is the result of checkEventAndGeneratePayload
149
+ */
150
+ addEventWithPayloadForProcessing(queueEntry, payload) {
151
+ if (!this.__queueEntriesMap[queueEntry.ID]) {
152
+ this.logger.error(
153
+ "The supplied queueEntry has not been selected before and should not be processed. Entry will not be processed.",
154
+ {
155
+ eventType: this.__eventType,
156
+ eventSubType: this.__eventSubType,
157
+ queueEntryId: queueEntry.ID,
158
+ }
159
+ );
160
+ return;
161
+ }
162
+ this.__queueEntriesWithPayloadMap[queueEntry.ID] = {
163
+ queueEntry,
164
+ payload,
165
+ };
166
+ }
167
+
168
+ /**
169
+ * This function sets the status of an queueEntry to done
170
+ * @param {Object} queueEntry which has been selected from event queue table and been modified by modifyQueueEntry
171
+ */
172
+ setStatusToDone(queueEntry) {
173
+ this.logger.debug("setting status for queueEntry to done", {
174
+ id: queueEntry.ID,
175
+ eventType: this.__eventType,
176
+ eventSubType: this.__eventSubType,
177
+ });
178
+ this._determineAndAddEventStatusToMap(
179
+ queueEntry.ID,
180
+ EventProcessingStatus.Done
181
+ );
182
+ }
183
+
184
+ /**
185
+ * This function allows to cluster multiple events so that they will be processed together. By default, there is no
186
+ * clustering happening. Therefore, the cluster key is the ID of the event. If an alternative clustering is needed
187
+ * this function should be overwritten. For every cluster-key the function processEvent will be called once.
188
+ * This can be useful for e.g. multiple tasks have been scheduled and always the same user should be informed.
189
+ * In this case the events should be clustered together and only one mail should be sent.
190
+ */
191
+ clusterQueueEntries() {
192
+ Object.entries(this.__queueEntriesWithPayloadMap).forEach(
193
+ ([key, { queueEntry, payload }]) => {
194
+ this._addEntryToProcessingMap(key, queueEntry, payload);
195
+ }
196
+ );
197
+ }
198
+
199
+ _addEntryToProcessingMap(key, queueEntry, payload) {
200
+ this.logger.debug("add entry to processing map", {
201
+ key,
202
+ queueEntry,
203
+ eventType: this.__eventType,
204
+ eventSubType: this.__eventSubType,
205
+ });
206
+ this.__eventProcessingMap[key] = this.__eventProcessingMap[key] ?? {
207
+ queueEntries: [],
208
+ payload,
209
+ };
210
+ this.__eventProcessingMap[key].queueEntries.push(queueEntry);
211
+ }
212
+
213
+ /**
214
+ * This function sets the status of multiple events to a given status. If the structure of queueEntryProcessingStatusTuple
215
+ * is not as expected all events will be set to error. The function respects the config commitOnEventLevel. If
216
+ * commitOnEventLevel is true the status will be written to a dedicated map and returned afterwards to handle concurrent
217
+ * event processing.
218
+ * @param {Array} queueEntries which has been selected from event queue table and been modified by modifyQueueEntry
219
+ * @param {Array<Object>} queueEntryProcessingStatusTuple Array of tuple <queueEntryId, processingStatus>
220
+ * @return {Object} statusMap Map which contains all events for which a status has been set so far
221
+ */
222
+ setEventStatus(queueEntries, queueEntryProcessingStatusTuple) {
223
+ this.logger.debug("setting event status for entries", {
224
+ queueEntryProcessingStatusTuple: JSON.stringify(
225
+ queueEntryProcessingStatusTuple
226
+ ),
227
+ eventType: this.__eventType,
228
+ eventSubType: this.__eventSubType,
229
+ });
230
+ const statusMap = this.__commitOnEventLevel ? {} : this.__statusMap;
231
+ try {
232
+ queueEntryProcessingStatusTuple.forEach(([id, processingStatus]) =>
233
+ this._determineAndAddEventStatusToMap(id, processingStatus, statusMap)
234
+ );
235
+ } catch (error) {
236
+ queueEntries.forEach((queueEntry) =>
237
+ this._determineAndAddEventStatusToMap(
238
+ queueEntry.ID,
239
+ EventProcessingStatus.Error,
240
+ statusMap
241
+ )
242
+ );
243
+ this.logger.error(
244
+ `The supplied status tuple doesn't have the required structure. Setting all entries to error. Error: ${error.toString()}`,
245
+ {
246
+ eventType: this.__eventType,
247
+ eventSubType: this.__eventSubType,
248
+ }
249
+ );
250
+ }
251
+ return statusMap;
252
+ }
253
+
254
+ /**
255
+ * This function allows to modify a select queueEntry (event) before processing. By default, the payload of the event
256
+ * is parsed. The return value of the function is ignored, it's required to modify the reference which is passed into
257
+ * the function.
258
+ * @param {Object} queueEntry which has been selected from event queue table
259
+ */
260
+ modifyQueueEntry(queueEntry) {
261
+ try {
262
+ queueEntry.payload = JSON.parse(queueEntry.payload);
263
+ } catch {
264
+ return queueEntry.payload;
265
+ }
266
+ }
267
+
268
+ _determineAndAddEventStatusToMap(
269
+ id,
270
+ processingStatus,
271
+ statusMap = this.__statusMap
272
+ ) {
273
+ if (!statusMap[id]) {
274
+ statusMap[id] = processingStatus;
275
+ return;
276
+ }
277
+ if (
278
+ [EventProcessingStatus.Error, EventProcessingStatus.Exceeded].includes(
279
+ statusMap[id]
280
+ )
281
+ ) {
282
+ // NOTE: worst aggregation --> if already error|exceeded keep this state
283
+ return;
284
+ }
285
+ if (statusMap[id] >= 0) {
286
+ statusMap[id] = processingStatus;
287
+ }
288
+ }
289
+
290
+ handleErrorDuringProcessing(error, queueEntries) {
291
+ queueEntries = Array.isArray(queueEntries) ? queueEntries : [queueEntries];
292
+ this.logger.error(
293
+ `Caught error during event processing - setting queue entry to error. Please catch your promises/exceptions. Error: ${error}`,
294
+ {
295
+ eventType: this.__eventType,
296
+ eventSubType: this.__eventSubType,
297
+ queueEntriesIds: queueEntries.map(({ ID }) => ID),
298
+ }
299
+ );
300
+ queueEntries.forEach((queueEntry) =>
301
+ this._determineAndAddEventStatusToMap(
302
+ queueEntry.ID,
303
+ EventProcessingStatus.Error
304
+ )
305
+ );
306
+ return Object.fromEntries(
307
+ queueEntries.map((queueEntry) => [
308
+ queueEntry.ID,
309
+ EventProcessingStatus.Error,
310
+ ])
311
+ );
312
+ }
313
+
314
+ /**
315
+ * This function validates for all selected events one status has been submitted. It's also validated that only for
316
+ * selected events a status has been submitted. Persisting the status of events is done in a dedicated database tx.
317
+ * The function accepts no arguments as there are dedicated functions to set the status of events (e.g. setEventStatus)
318
+ */
319
+ async persistEventStatus(
320
+ tx,
321
+ { skipChecks, statusMap = this.__statusMap } = {}
322
+ ) {
323
+ this.logger.debug("entering persistEventStatus", {
324
+ eventType: this.__eventType,
325
+ eventSubType: this.__eventSubType,
326
+ });
327
+ this._ensureOnlySelectedQueueEntries(statusMap);
328
+ if (!skipChecks) {
329
+ this._ensureEveryQueueEntryHasStatus();
330
+ }
331
+ this._ensureEveryStatusIsAllowed(statusMap);
332
+
333
+ const { success, failed, exceeded, invalidAttempts } = Object.entries(
334
+ statusMap
335
+ ).reduce(
336
+ (result, [notificationEntityId, processingStatus]) => {
337
+ this.__commitedStatusMap[notificationEntityId] = processingStatus;
338
+ if (processingStatus === EventProcessingStatus.Open) {
339
+ result.invalidAttempts.push(notificationEntityId);
340
+ } else if (processingStatus === EventProcessingStatus.Done) {
341
+ result.success.push(notificationEntityId);
342
+ } else if (processingStatus === EventProcessingStatus.Error) {
343
+ result.failed.push(notificationEntityId);
344
+ } else if (processingStatus === EventProcessingStatus.Exceeded) {
345
+ result.exceeded.push(notificationEntityId);
346
+ }
347
+ return result;
348
+ },
349
+ {
350
+ success: [],
351
+ failed: [],
352
+ exceeded: [],
353
+ invalidAttempts: [],
354
+ }
355
+ );
356
+ this.logger.debug("persistEventStatus for entries", {
357
+ eventType: this.__eventType,
358
+ eventSubType: this.__eventSubType,
359
+ invalidAttempts,
360
+ failed,
361
+ exceeded,
362
+ success,
363
+ });
364
+ if (invalidAttempts.length) {
365
+ await tx.run(
366
+ UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
367
+ .set({
368
+ status: EventProcessingStatus.Open,
369
+ lastAttemptTimestamp: new Date().toISOString(),
370
+ attempts: { "-=": 1 },
371
+ })
372
+ .where("ID IN", invalidAttempts)
373
+ );
374
+ }
375
+ if (success.length) {
376
+ await tx.run(
377
+ UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
378
+ .set({
379
+ status: EventProcessingStatus.Done,
380
+ lastAttemptTimestamp: new Date().toISOString(),
381
+ })
382
+ .where("ID IN", success)
383
+ );
384
+ }
385
+ if (failed.length) {
386
+ await tx.run(
387
+ UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
388
+ .where("ID IN", failed)
389
+ .with({
390
+ status: EventProcessingStatus.Error,
391
+ lastAttemptTimestamp: new Date().toISOString(),
392
+ })
393
+ );
394
+ }
395
+ if (exceeded.length) {
396
+ await tx.run(
397
+ UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
398
+ .where("ID IN", exceeded)
399
+ .with({
400
+ status: EventProcessingStatus.Exceeded,
401
+ lastAttemptTimestamp: new Date().toISOString(),
402
+ })
403
+ );
404
+ }
405
+ this.logger.debug("exiting persistEventStatus", {
406
+ eventType: this.__eventType,
407
+ eventSubType: this.__eventSubType,
408
+ });
409
+ }
410
+
411
+ _ensureEveryQueueEntryHasStatus() {
412
+ this.__queueEntries.forEach((queueEntry) => {
413
+ if (
414
+ queueEntry.ID in this.__statusMap ||
415
+ queueEntry.ID in this.__commitedStatusMap
416
+ ) {
417
+ return;
418
+ }
419
+ this.logger.error(
420
+ "Missing status for selected event entry. Setting status to error",
421
+ {
422
+ eventType: this.__eventType,
423
+ eventSubType: this.__eventSubType,
424
+ queueEntry,
425
+ }
426
+ );
427
+ this._determineAndAddEventStatusToMap(
428
+ queueEntry.ID,
429
+ EventProcessingStatus.Error
430
+ );
431
+ });
432
+ }
433
+
434
+ _ensureEveryStatusIsAllowed(statusMap) {
435
+ Object.entries(statusMap).forEach(([queueEntryId, status]) => {
436
+ if (
437
+ [
438
+ EventProcessingStatus.Open,
439
+ EventProcessingStatus.Done,
440
+ EventProcessingStatus.Error,
441
+ EventProcessingStatus.Exceeded,
442
+ ].includes(status)
443
+ ) {
444
+ return;
445
+ }
446
+
447
+ this.logger.error(
448
+ "Not allowed event status returned. Only Open, Done, Error is allowed!",
449
+ {
450
+ eventType: this.__eventType,
451
+ eventSubType: this.__eventSubType,
452
+ queueEntryId,
453
+ status: statusMap[queueEntryId],
454
+ }
455
+ );
456
+ delete statusMap[queueEntryId];
457
+ });
458
+ }
459
+
460
+ _ensureOnlySelectedQueueEntries(statusMap) {
461
+ Object.keys(statusMap).forEach((queueEntryId) => {
462
+ if (this.__queueEntriesMap[queueEntryId]) {
463
+ return;
464
+ }
465
+
466
+ this.logger.error(
467
+ "Status reported for event queue entry which haven't be selected before. Removing the status.",
468
+ {
469
+ eventType: this.__eventType,
470
+ eventSubType: this.__eventSubType,
471
+ queueEntryId,
472
+ }
473
+ );
474
+ delete statusMap[queueEntryId];
475
+ });
476
+ }
477
+
478
+ handleErrorDuringClustering(error) {
479
+ this.logger.error(
480
+ `Error during clustering of events - setting all queue entries to error. Error: ${error}`,
481
+ {
482
+ eventType: this.__eventType,
483
+ eventSubType: this.__eventSubType,
484
+ }
485
+ );
486
+ this.__queueEntries.forEach((queueEntry) => {
487
+ this._determineAndAddEventStatusToMap(
488
+ queueEntry.ID,
489
+ EventProcessingStatus.Error
490
+ );
491
+ });
492
+ }
493
+
494
+ handleInvalidPayloadReturned(queueEntry) {
495
+ this.logger.error(
496
+ "Undefined payload is not allowed. If status should be done, nulls needs to be returned" +
497
+ " - setting queue entry to error",
498
+ {
499
+ eventType: this.__eventType,
500
+ eventSubType: this.__eventSubType,
501
+ }
502
+ );
503
+ this._determineAndAddEventStatusToMap(
504
+ queueEntry.ID,
505
+ EventProcessingStatus.Error
506
+ );
507
+ }
508
+
509
+ static async handleMissingTypeImplementation(
510
+ context,
511
+ eventType,
512
+ eventSubType
513
+ ) {
514
+ const baseInstance = new EventQueueProcessorBase(
515
+ context,
516
+ eventType,
517
+ eventSubType
518
+ );
519
+ baseInstance.logger.error(
520
+ "No Implementation found in the provided configuration file.",
521
+ {
522
+ eventType,
523
+ eventSubType,
524
+ }
525
+ );
526
+ }
527
+
528
+ /**
529
+ * This function selects all relevant events based on the eventType and eventSubType supplied through the constructor
530
+ * during initialization of the class.
531
+ * Relevant Events for selection are: open events, error events if the number retry attempts has not been succeeded or
532
+ * events which are in progress for longer than 30 minutes.
533
+ * @return {Promise<Array<Object>>} all relevant events for processing for the given eventType and eventSubType
534
+ */
535
+ async getQueueEntriesAndSetToInProgress() {
536
+ let result = [];
537
+ await executeInNewTransaction(
538
+ this.__baseContext,
539
+ "eventQueue-getQueueEntriesAndSetToInProgress",
540
+ async (tx) => {
541
+ const entries = await tx.run(
542
+ SELECT.from(this.__eventQueueConfig.tableNameEventQueue)
543
+ .forUpdate({ wait: this.__eventQueueConfig.forUpdateTimeout })
544
+ .limit(this.getSelectMaxChunkSize())
545
+ .where(
546
+ "type =",
547
+ this.__eventType,
548
+ "AND subType=",
549
+ this.__eventSubType,
550
+ "AND ( status =",
551
+ EventProcessingStatus.Open,
552
+ "OR ( status =",
553
+ EventProcessingStatus.Error,
554
+ "AND lastAttemptTimestamp <=",
555
+ this.__startTime.toISOString(),
556
+ ") OR ( status =",
557
+ EventProcessingStatus.InProgress,
558
+ "AND lastAttemptTimestamp <=",
559
+ new Date(
560
+ new Date().getTime() - this.__eventQueueConfig.globalTxTimeout
561
+ ).toISOString(),
562
+ ") )"
563
+ )
564
+ .orderBy("createdAt", "ID")
565
+ );
566
+
567
+ if (!entries.length) {
568
+ this.logger.debug("no entries available for processing", {
569
+ eventType: this.__eventType,
570
+ eventSubType: this.__eventSubType,
571
+ });
572
+ this.__emptyChunkSelected = true;
573
+ return;
574
+ }
575
+
576
+ const { exceededTries, openEvents } =
577
+ this._filterExceededEvents(entries);
578
+ if (exceededTries.length) {
579
+ this.__eventsWithExceededTries = exceededTries;
580
+ }
581
+ result = openEvents;
582
+
583
+ if (!result.length) {
584
+ this.__emptyChunkSelected = true;
585
+ return;
586
+ }
587
+
588
+ this.logger.info("Selected event queue entries for processing", {
589
+ queueEntriesCount: result.length,
590
+ eventType: this.__eventType,
591
+ eventSubType: this.__eventSubType,
592
+ });
593
+
594
+ const isoTimestamp = new Date().toISOString();
595
+ await tx.run(
596
+ UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
597
+ .with({
598
+ status: EventProcessingStatus.InProgress,
599
+ lastAttemptTimestamp: isoTimestamp,
600
+ attempts: { "+=": 1 },
601
+ })
602
+ .where(
603
+ "ID IN",
604
+ result.map(({ ID }) => ID)
605
+ )
606
+ );
607
+ result.forEach((entry) => {
608
+ entry.lastAttemptTimestamp = isoTimestamp;
609
+ // NOTE: empty payloads are supported on DB-Level.
610
+ // Behaviour of event queue is: null as payload is treated as obsolete/done
611
+ // For supporting this convert null to empty string --> "" as payload will be processed normally
612
+ if (entry.payload === null) {
613
+ entry.payload = "";
614
+ }
615
+ });
616
+ }
617
+ );
618
+ this.__queueEntries = result;
619
+ this.__queueEntriesMap = arrayToFlatMap(result);
620
+ return result;
621
+ }
622
+
623
+ _filterExceededEvents(events) {
624
+ return events.reduce(
625
+ (result, event) => {
626
+ if (event.attempts === this.__retryAttempts) {
627
+ result.exceededTries.push(event);
628
+ } else {
629
+ result.openEvents.push(event);
630
+ }
631
+ return result;
632
+ },
633
+ { exceededTries: [], openEvents: [] }
634
+ );
635
+ }
636
+
637
+ async handleExceededEvents(exceededEvents) {
638
+ await this.tx.run(
639
+ UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
640
+ .with({
641
+ status: EventProcessingStatus.Exceeded,
642
+ })
643
+ .where(
644
+ "ID IN",
645
+ exceededEvents.map(({ ID }) => ID)
646
+ )
647
+ );
648
+ this.logger.error(
649
+ "The retry attempts for the following events are exceeded",
650
+ {
651
+ eventType: this.__eventType,
652
+ eventSubType: this.__eventSubType,
653
+ retryAttempts: this.__retryAttempts,
654
+ queueEntriesIds: exceededEvents.map(({ ID }) => ID),
655
+ }
656
+ );
657
+ await this.hookForExceededEvents(exceededEvents);
658
+ }
659
+
660
+ /**
661
+ * This function enables the possibility to execute custom actions for events for which the retry attempts have been
662
+ * exceeded. As always a valid transaction is available with this.tx. This transaction will be committed after the
663
+ * execution of this function.
664
+ * @param {Object} exceededEvents exceeded event queue entries
665
+ */
666
+ // eslint-disable-next-line no-unused-vars
667
+ async hookForExceededEvents(exceededEvents) {}
668
+
669
+ /**
670
+ * This function serves the purpose of mass enabled preloading data for processing the events which are added with
671
+ * the function 'addEventWithPayloadForProcessing'. This function is called after the clustering and before the
672
+ * process-events-steps. The event data is available with this.eventProcessingMap.
673
+ */
674
+ // eslint-disable-next-line no-unused-vars
675
+ async beforeProcessingEvents() {}
676
+
677
+ /**
678
+ * This function checks if the db records of events have been modified since the selection (beginning of processing)
679
+ * If the db records are unmodified the field lastAttemptTimestamp of the records is updated to
680
+ * "send a keep alive signal". This extends the allowed processing time of the events as events which are in progress
681
+ * for more than 30 minutes (global tx timeout) are selected with the next tick.
682
+ * If events are outdated/modified these events are not being processed and no status will be persisted.
683
+ * @return {Promise<boolean>} true if the db record of the event has been modified since selection
684
+ */
685
+ async isOutdatedAndKeepalive(queueEntries) {
686
+ if (!this.__outdatedCheckEnabled) {
687
+ return false;
688
+ }
689
+ let eventOutdated;
690
+ const runningChecks = queueEntries
691
+ .map((queueEntry) => this.__keepalivePromises[queueEntry.ID])
692
+ .filter((p) => p);
693
+ if (runningChecks.length === queueEntries.length) {
694
+ const results = await Promise.allSettled(runningChecks);
695
+ for (const { value } of results) {
696
+ if (value) {
697
+ return true;
698
+ }
699
+ }
700
+ return false;
701
+ } else if (runningChecks.length) {
702
+ await Promise.allSettled(runningChecks);
703
+ }
704
+ const checkAndUpdatePromise = new Promise((resolve) => {
705
+ executeInNewTransaction(
706
+ this.__baseContext,
707
+ "eventProcessing-isOutdatedAndKeepalive",
708
+ async (tx) => {
709
+ const queueEntriesFresh = await tx.run(
710
+ SELECT.from(this.__eventQueueConfig.tableNameEventQueue)
711
+ .forUpdate({ wait: this.__eventQueueConfig.forUpdateTimeout })
712
+ .where(
713
+ "ID IN",
714
+ queueEntries.map(({ ID }) => ID)
715
+ )
716
+ .columns("ID", "lastAttemptTimestamp")
717
+ );
718
+ eventOutdated = queueEntriesFresh.some((queueEntryFresh) => {
719
+ const queueEntry = this.__queueEntriesMap[queueEntryFresh.ID];
720
+ return (
721
+ queueEntry?.lastAttemptTimestamp !==
722
+ queueEntryFresh.lastAttemptTimestamp
723
+ );
724
+ });
725
+ let newTs = new Date().toISOString();
726
+ if (!eventOutdated) {
727
+ await tx.run(
728
+ UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
729
+ .set("lastAttemptTimestamp =", newTs)
730
+ .where(
731
+ "ID IN",
732
+ queueEntries.map(({ ID }) => ID)
733
+ )
734
+ );
735
+ } else {
736
+ newTs = null;
737
+ this.logger.warn(
738
+ "event data has been modified. Processing skipped.",
739
+ {
740
+ eventType: this.__eventType,
741
+ eventSubType: this.__eventSubType,
742
+ queueEntriesIds: queueEntries.map(({ ID }) => ID),
743
+ }
744
+ );
745
+ queueEntries.forEach(
746
+ ({ ID: queueEntryId }) =>
747
+ delete this.__queueEntriesMap[queueEntryId]
748
+ );
749
+ }
750
+ this.__queueEntries = Object.values(this.__queueEntriesMap);
751
+ queueEntriesFresh.forEach((queueEntryFresh) => {
752
+ if (this.__queueEntriesMap[queueEntryFresh.ID]) {
753
+ const queueEntry = this.__queueEntriesMap[queueEntryFresh.ID];
754
+ if (newTs) {
755
+ queueEntry.lastAttemptTimestamp = newTs;
756
+ }
757
+ }
758
+ delete this.__keepalivePromises[queueEntryFresh.ID];
759
+ });
760
+ resolve(eventOutdated);
761
+ }
762
+ );
763
+ });
764
+
765
+ queueEntries.forEach(
766
+ (queueEntry) =>
767
+ (this.__keepalivePromises[queueEntry.ID] = checkAndUpdatePromise)
768
+ );
769
+ return await checkAndUpdatePromise;
770
+ }
771
+
772
+ async handleDistributedLock() {
773
+ if (this.concurrentEventProcessing) {
774
+ return true;
775
+ }
776
+
777
+ const lockAcquired = await distributedLock.acquireLock(
778
+ this.context,
779
+ [this.eventType, this.eventSubType].join("##")
780
+ );
781
+ if (!lockAcquired) {
782
+ return false;
783
+ }
784
+ this.__lockAcquired = true;
785
+ return true;
786
+ }
787
+
788
+ async handleReleaseLock() {
789
+ if (!this.__lockAcquired) {
790
+ return;
791
+ }
792
+ try {
793
+ await distributedLock.releaseLock(
794
+ this.context,
795
+ [this.eventType, this.eventSubType].join("##")
796
+ );
797
+ } catch (err) {
798
+ this.logger.error(
799
+ "Releasing distributed lock failed. Error:",
800
+ err.toString()
801
+ );
802
+ }
803
+ }
804
+
805
+ statusMapContainsError(statusMap) {
806
+ return Object.values(statusMap).includes(EventProcessingStatus.Error);
807
+ }
808
+
809
+ getSelectNextChunk() {
810
+ return this.__selectNextChunk;
811
+ }
812
+
813
+ getSelectMaxChunkSize() {
814
+ return this.__selectMaxChunkSize;
815
+ }
816
+
817
+ clearEventProcessingContext() {
818
+ this.__processContext = null;
819
+ this.__processTx = null;
820
+ }
821
+
822
+ get shouldTriggerRollback() {
823
+ return (
824
+ this.statusMapContainsError(this.__statusMap) ||
825
+ this.statusMapContainsError(this.__commitedStatusMap)
826
+ );
827
+ }
828
+
829
+ get logger() {
830
+ return this.__logger ?? this.__baseLogger;
831
+ }
832
+
833
+ get queueEntriesWithPayloadMap() {
834
+ return this.__queueEntriesWithPayloadMap;
835
+ }
836
+
837
+ get eventProcessingMap() {
838
+ return this.__eventProcessingMap;
839
+ }
840
+
841
+ get parallelEventProcessing() {
842
+ return this.__parallelEventProcessing;
843
+ }
844
+
845
+ get concurrentEventProcessing() {
846
+ return this.__concurrentEventProcessing;
847
+ }
848
+
849
+ set processEventContext(context) {
850
+ if (!context) {
851
+ this.__processContext = null;
852
+ this.__processTx = null;
853
+ return;
854
+ }
855
+ this.__processContext = context;
856
+ this.__processTx = cds.tx(context);
857
+ }
858
+
859
+ get tx() {
860
+ if (!this.__txUsageAllowed && this.__parallelEventProcessing > 1) {
861
+ throw EventQueueError.wrongTxUsage(this.eventType, this.eventSubType);
862
+ }
863
+ return this.__processTx ?? this.__tx;
864
+ }
865
+
866
+ get context() {
867
+ if (!this.__txUsageAllowed && this.__parallelEventProcessing > 1) {
868
+ throw EventQueueError.wrongTxUsage(this.eventType, this.eventSubType);
869
+ }
870
+ return this.__processContext ?? this.__context;
871
+ }
872
+
873
+ get baseContext() {
874
+ return this.__baseContext;
875
+ }
876
+
877
+ get commitOnEventLevel() {
878
+ return this.__commitOnEventLevel;
879
+ }
880
+
881
+ get eventType() {
882
+ return this.__eventType;
883
+ }
884
+
885
+ get eventSubType() {
886
+ return this.__eventSubType;
887
+ }
888
+
889
+ get exceededEvents() {
890
+ return this.__eventsWithExceededTries;
891
+ }
892
+
893
+ get emptyChunkSelected() {
894
+ return this.__emptyChunkSelected;
895
+ }
896
+
897
+ set txUsageAllowed(value) {
898
+ this.__txUsageAllowed = value;
899
+ }
900
+
901
+ getContextForEventProcessing(key) {
902
+ return this.__txMap[key]?.context;
903
+ }
904
+
905
+ getTxForEventProcessing(key) {
906
+ return this.__txMap[key];
907
+ }
908
+
909
+ setShouldRollbackTransaction(key) {
910
+ this.__txRollback[key] = true;
911
+ }
912
+
913
+ shouldRollbackTransaction(key) {
914
+ return this.__txRollback[key];
915
+ }
916
+
917
+ setTxForEventProcessing(key, tx) {
918
+ this.__txMap[key] = tx;
919
+ }
920
+ }
921
+
922
+ module.exports = EventQueueProcessorBase;