@cap-js-community/event-queue 0.1.57 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/db/Event.cds CHANGED
@@ -21,4 +21,5 @@ entity Event: cuid {
21
21
  attempts: Integer default 0 not null;
22
22
  lastAttemptTimestamp: Timestamp;
23
23
  createdAt: Timestamp @cds.on.insert : $now;
24
+ startAfter: Timestamp;
24
25
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cap-js-community/event-queue",
3
- "version": "0.1.57",
3
+ "version": "0.2.0",
4
4
  "description": "An event queue that enables secure transactional processing of asynchronous events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
5
5
  "main": "src/index.js",
6
6
  "files": [
@@ -23,14 +23,15 @@
23
23
  "test:unit": "jest --testPathIgnorePatterns=\"/test-integration/\"",
24
24
  "test:integration": "jest --testPathIgnorePatterns=\"/test/\" --runInBand --forceExit",
25
25
  "test:all": "npm run test:unit && npm run test:integration",
26
- "test:ci": "npm run test:prepare && npm run test:all",
27
26
  "test:all:coverage": "jest --runInBand --forceExit --collect-coverage",
28
27
  "test:prepare": "npm run build:ci --prefix=./test-integration/_env",
28
+ "test:deploySchema": "node test-integration/_env/srv/hana/deploy.js",
29
+ "test:cleanSchemas": "node test-integration/_env/srv/hana/cleanObsoletSchemas.js",
29
30
  "lint": "npm run eslint && npm run prettier",
30
31
  "lint:ci": "npm run eslint:ci && npm run prettier:ci",
31
32
  "eslint": "eslint --fix .",
32
33
  "eslint:ci": "eslint .",
33
- "prettier": "prettier --write --loglevel error .",
34
+ "prettier": "prettier --write --log-level error .",
34
35
  "prettier:ci": "prettier --check .",
35
36
  "prepareRelease": "npm prune --production",
36
37
  "docs": "cd docs && bundle exec jekyll serve",
@@ -41,24 +42,22 @@
41
42
  "node": ">=16"
42
43
  },
43
44
  "dependencies": {
44
- "uuid": "9.0.1",
45
45
  "redis": "4.6.10",
46
46
  "verror": "1.10.1",
47
- "yaml": "2.3.2"
47
+ "yaml": "2.3.4"
48
48
  },
49
49
  "devDependencies": {
50
- "@sap/eslint-plugin-cds": "2.6.3",
51
- "@sap/cds-dk": "7.1.1",
52
- "hdb": "0.19.6",
50
+ "@sap/cds": "7.3.1",
51
+ "@sap/cds-dk": "7.3.1",
52
+ "eslint": "8.52.0",
53
+ "eslint-config-prettier": "9.0.0",
54
+ "eslint-plugin-jest": "27.6.0",
53
55
  "eslint-plugin-node": "11.1.0",
54
- "sqlite3": "5.1.6",
55
56
  "express": "4.18.2",
56
- "@sap/cds": "7.1.2",
57
- "eslint": "8.50.0",
58
- "eslint-config-prettier": "9.0.0",
59
- "eslint-plugin-jest": "27.4.0",
57
+ "hdb": "0.19.6",
60
58
  "jest": "29.7.0",
61
- "prettier": "3.0.3"
59
+ "prettier": "2.8.8",
60
+ "sqlite3": "5.1.6"
62
61
  },
63
62
  "homepage": "https://cap-js-community.github.io/event-queue/",
64
63
  "repository": {
@@ -11,6 +11,7 @@ const ERROR_CODES = {
11
11
  MISSING_TABLE_DEFINITION: "MISSING_TABLE_DEFINITION",
12
12
  MISSING_ELEMENT_IN_TABLE: "MISSING_ELEMENT_IN_TABLE",
13
13
  TYPE_MISMATCH_TABLE: "TYPE_MISMATCH_TABLE",
14
+ NO_VALID_DATE: "NO_VALID_DATE",
14
15
  };
15
16
 
16
17
  const ERROR_CODES_META = {
@@ -39,6 +40,9 @@ const ERROR_CODES_META = {
39
40
  [ERROR_CODES.TYPE_MISMATCH_TABLE]: {
40
41
  message: "At least one field in the provided table doesn't have the expected data type.",
41
42
  },
43
+ [ERROR_CODES.NO_VALID_DATE]: {
44
+ message: "One or more events contain a date in a malformed format.",
45
+ },
42
46
  };
43
47
 
44
48
  class EventQueueError extends VError {
@@ -131,6 +135,17 @@ class EventQueueError extends VError {
131
135
  message
132
136
  );
133
137
  }
138
+
139
+ static malformedDate(date) {
140
+ const { message } = ERROR_CODES_META[ERROR_CODES.NO_VALID_DATE];
141
+ return new EventQueueError(
142
+ {
143
+ name: ERROR_CODES.NO_VALID_DATE,
144
+ info: { date },
145
+ },
146
+ message
147
+ );
148
+ }
134
149
  }
135
150
 
136
151
  module.exports = EventQueueError;
@@ -7,6 +7,7 @@ const { EventProcessingStatus, TransactionMode } = require("./constants");
7
7
  const distributedLock = require("./shared/distributedLock");
8
8
  const EventQueueError = require("./EventQueueError");
9
9
  const { arrayToFlatMap } = require("./shared/common");
10
+ const eventScheduler = require("./shared/EventScheduler");
10
11
  const eventQueueConfig = require("./config");
11
12
  const PerformanceTracer = require("./shared/PerformanceTracer");
12
13
 
@@ -20,12 +21,16 @@ const SELECT_LIMIT_EVENTS_PER_TICK = 100;
20
21
  const DEFAULT_DELETE_FINISHED_EVENTS_AFTER = 0;
21
22
  const DAYS_TO_MS = 24 * 60 * 60 * 1000;
22
23
  const TRIES_FOR_EXCEEDED_EVENTS = 3;
24
+ const EVENT_START_AFTER_HEADROOM = 3 * 1000;
23
25
 
24
26
  class EventQueueProcessorBase {
25
27
  #eventsWithExceededTries = [];
26
28
  #exceededTriesExceeded = [];
27
29
  #selectedEventMap = {};
28
30
  #queueEntriesWithPayloadMap = {};
31
+ #eventType = null;
32
+ #eventSubType = null;
33
+ #config = null;
29
34
 
30
35
  constructor(context, eventType, eventSubType, config) {
31
36
  this.__context = context;
@@ -36,26 +41,27 @@ class EventQueueProcessorBase {
36
41
  this.__eventProcessingMap = {};
37
42
  this.__statusMap = {};
38
43
  this.__commitedStatusMap = {};
39
- this.__eventType = eventType;
40
- this.__eventSubType = eventSubType;
41
- this.__config = config ?? {};
42
- this.__parallelEventProcessing = this.__config.parallelEventProcessing ?? DEFAULT_PARALLEL_EVENT_PROCESSING;
44
+ this.#eventType = eventType;
45
+ this.#eventSubType = eventSubType;
46
+ this.__eventConfig = config ?? {};
47
+ this.__parallelEventProcessing = this.__eventConfig.parallelEventProcessing ?? DEFAULT_PARALLEL_EVENT_PROCESSING;
43
48
  if (this.__parallelEventProcessing > LIMIT_PARALLEL_EVENT_PROCESSING) {
44
49
  this.__parallelEventProcessing = LIMIT_PARALLEL_EVENT_PROCESSING;
45
50
  }
46
51
  // NOTE: keep the feature, this might be needed again
47
52
  this.__concurrentEventProcessing = false;
48
- this.__startTime = this.__config.startTime ?? new Date();
49
- this.__retryAttempts = this.__config.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
50
- this.__selectMaxChunkSize = this.__config.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
51
- this.__selectNextChunk = !!this.__config.checkForNextChunk;
53
+ this.__startTime = this.__eventConfig.startTime ?? new Date();
54
+ this.__retryAttempts = this.__eventConfig.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
55
+ this.__selectMaxChunkSize = this.__eventConfig.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
56
+ this.__selectNextChunk = !!this.__eventConfig.checkForNextChunk;
52
57
  this.__keepalivePromises = {};
53
- this.__outdatedCheckEnabled = this.__config.eventOutdatedCheck ?? true;
54
- this.__transactionMode = this.__config.transactionMode ?? TransactionMode.isolated;
55
- if (this.__config.deleteFinishedEventsAfterDays) {
58
+ this.__outdatedCheckEnabled = this.__eventConfig.eventOutdatedCheck ?? true;
59
+ this.__transactionMode = this.__eventConfig.transactionMode ?? TransactionMode.isolated;
60
+ if (this.__eventConfig.deleteFinishedEventsAfterDays) {
56
61
  this.__deleteFinishedEventsAfter =
57
- Number.isInteger(this.__config.deleteFinishedEventsAfterDays) && this.__config.deleteFinishedEventsAfterDays > 0
58
- ? this.__config.deleteFinishedEventsAfterDays
62
+ Number.isInteger(this.__eventConfig.deleteFinishedEventsAfterDays) &&
63
+ this.__eventConfig.deleteFinishedEventsAfterDays > 0
64
+ ? this.__eventConfig.deleteFinishedEventsAfterDays
59
65
  : DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
60
66
  } else {
61
67
  this.__deleteFinishedEventsAfter = DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
@@ -65,7 +71,7 @@ class EventQueueProcessorBase {
65
71
  this.__txUsageAllowed = true;
66
72
  this.__txMap = {};
67
73
  this.__txRollback = {};
68
- this.__eventQueueConfig = eventQueueConfig.getConfigInstance();
74
+ this.#config = eventQueueConfig.getConfigInstance();
69
75
  this.__queueEntries = [];
70
76
  }
71
77
 
@@ -99,8 +105,8 @@ class EventQueueProcessorBase {
99
105
  this.__performanceLoggerEvents?.endPerformanceTrace(
100
106
  { threshold: 50 },
101
107
  {
102
- eventType: this.eventType,
103
- eventSubType: this.eventSubType,
108
+ eventType: this.#eventType,
109
+ eventSubType: this.#eventSubType,
104
110
  }
105
111
  );
106
112
  }
@@ -109,16 +115,16 @@ class EventQueueProcessorBase {
109
115
  this.__performanceLoggerPreprocessing?.endPerformanceTrace(
110
116
  { threshold: 50 },
111
117
  {
112
- eventType: this.eventType,
113
- eventSubType: this.eventSubType,
118
+ eventType: this.#eventType,
119
+ eventSubType: this.#eventSubType,
114
120
  }
115
121
  );
116
122
  }
117
123
 
118
124
  logTimeExceeded(iterationCounter) {
119
125
  this.logger.info("Exiting event queue processing as max time exceeded", {
120
- eventType: this.eventType,
121
- eventSubType: this.eventSubType,
126
+ eventType: this.#eventType,
127
+ eventSubType: this.#eventSubType,
122
128
  iterationCounter,
123
129
  });
124
130
  }
@@ -127,8 +133,8 @@ class EventQueueProcessorBase {
127
133
  // TODO: how to handle custom fields
128
134
  this.logger.info("Processing queue event", {
129
135
  numberQueueEntries: queueEntries.length,
130
- eventType: this.__eventType,
131
- eventSubType: this.__eventSubType,
136
+ eventType: this.#eventType,
137
+ eventSubType: this.#eventSubType,
132
138
  });
133
139
  }
134
140
 
@@ -157,8 +163,8 @@ class EventQueueProcessorBase {
157
163
  this.logger.error(
158
164
  "The supplied queueEntry has not been selected before and should not be processed. Entry will not be processed.",
159
165
  {
160
- eventType: this.__eventType,
161
- eventSubType: this.__eventSubType,
166
+ eventType: this.#eventType,
167
+ eventSubType: this.#eventSubType,
162
168
  queueEntryId: queueEntry.ID,
163
169
  }
164
170
  );
@@ -177,8 +183,8 @@ class EventQueueProcessorBase {
177
183
  setStatusToDone(queueEntry) {
178
184
  this.logger.debug("setting status for queueEntry to done", {
179
185
  id: queueEntry.ID,
180
- eventType: this.__eventType,
181
- eventSubType: this.__eventSubType,
186
+ eventType: this.#eventType,
187
+ eventSubType: this.#eventSubType,
182
188
  });
183
189
  this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Done);
184
190
  }
@@ -207,8 +213,8 @@ class EventQueueProcessorBase {
207
213
  this.logger.debug("add entry to processing map", {
208
214
  key,
209
215
  queueEntry,
210
- eventType: this.__eventType,
211
- eventSubType: this.__eventSubType,
216
+ eventType: this.#eventType,
217
+ eventSubType: this.#eventSubType,
212
218
  });
213
219
  this.__eventProcessingMap[key] = this.__eventProcessingMap[key] ?? {
214
220
  queueEntries: [],
@@ -230,8 +236,8 @@ class EventQueueProcessorBase {
230
236
  setEventStatus(queueEntries, queueEntryProcessingStatusTuple, returnMap = false) {
231
237
  this.logger.debug("setting event status for entries", {
232
238
  queueEntryProcessingStatusTuple: JSON.stringify(queueEntryProcessingStatusTuple),
233
- eventType: this.__eventType,
234
- eventSubType: this.__eventSubType,
239
+ eventType: this.#eventType,
240
+ eventSubType: this.#eventSubType,
235
241
  });
236
242
  const statusMap = this.commitOnEventLevel || returnMap ? {} : this.__statusMap;
237
243
  try {
@@ -245,8 +251,8 @@ class EventQueueProcessorBase {
245
251
  this.logger.error(
246
252
  `The supplied status tuple doesn't have the required structure. Setting all entries to error. Error: ${error.toString()}`,
247
253
  {
248
- eventType: this.__eventType,
249
- eventSubType: this.__eventSubType,
254
+ eventType: this.#eventType,
255
+ eventSubType: this.#eventSubType,
250
256
  }
251
257
  );
252
258
  }
@@ -286,8 +292,8 @@ class EventQueueProcessorBase {
286
292
  this.logger.error(
287
293
  `Caught error during event processing - setting queue entry to error. Please catch your promises/exceptions. Error: ${error}`,
288
294
  {
289
- eventType: this.__eventType,
290
- eventSubType: this.__eventSubType,
295
+ eventType: this.#eventType,
296
+ eventSubType: this.#eventSubType,
291
297
  queueEntriesIds: queueEntries.map(({ ID }) => ID),
292
298
  }
293
299
  );
@@ -304,8 +310,8 @@ class EventQueueProcessorBase {
304
310
  */
305
311
  async persistEventStatus(tx, { skipChecks, statusMap = this.__statusMap } = {}) {
306
312
  this.logger.debug("entering persistEventStatus", {
307
- eventType: this.__eventType,
308
- eventSubType: this.__eventSubType,
313
+ eventType: this.#eventType,
314
+ eventSubType: this.#eventSubType,
309
315
  });
310
316
  this.#ensureOnlySelectedQueueEntries(statusMap);
311
317
  if (!skipChecks) {
@@ -335,8 +341,8 @@ class EventQueueProcessorBase {
335
341
  }
336
342
  );
337
343
  this.logger.debug("persistEventStatus for entries", {
338
- eventType: this.__eventType,
339
- eventSubType: this.__eventSubType,
344
+ eventType: this.#eventType,
345
+ eventSubType: this.#eventSubType,
340
346
  invalidAttempts,
341
347
  failed,
342
348
  exceeded,
@@ -344,7 +350,7 @@ class EventQueueProcessorBase {
344
350
  });
345
351
  if (invalidAttempts.length) {
346
352
  await tx.run(
347
- UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
353
+ UPDATE.entity(this.#config.tableNameEventQueue)
348
354
  .set({
349
355
  status: EventProcessingStatus.Open,
350
356
  lastAttemptTimestamp: new Date().toISOString(),
@@ -365,7 +371,7 @@ class EventQueueProcessorBase {
365
371
  continue;
366
372
  }
367
373
  await tx.run(
368
- UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
374
+ UPDATE.entity(this.#config.tableNameEventQueue)
369
375
  .set({
370
376
  status: status,
371
377
  lastAttemptTimestamp: ts,
@@ -374,8 +380,8 @@ class EventQueueProcessorBase {
374
380
  );
375
381
  }
376
382
  this.logger.debug("exiting persistEventStatus", {
377
- eventType: this.__eventType,
378
- eventSubType: this.__eventSubType,
383
+ eventType: this.#eventType,
384
+ eventSubType: this.#eventSubType,
379
385
  });
380
386
  }
381
387
 
@@ -384,18 +390,18 @@ class EventQueueProcessorBase {
384
390
  return;
385
391
  }
386
392
  const deleteCount = await tx.run(
387
- DELETE.from(this.__eventQueueConfig.tableNameEventQueue).where(
393
+ DELETE.from(this.#config.tableNameEventQueue).where(
388
394
  "type =",
389
- this.eventType,
395
+ this.#eventType,
390
396
  "AND subType=",
391
- this.eventSubType,
397
+ this.#eventSubType,
392
398
  "AND lastAttemptTimestamp <=",
393
399
  new Date(Date.now() - this.__deleteFinishedEventsAfter * DAYS_TO_MS).toISOString()
394
400
  )
395
401
  );
396
402
  this.logger.debug("Deleted finished events", {
397
- eventType: this.eventType,
398
- eventSubType: this.eventSubType,
403
+ eventType: this.#eventType,
404
+ eventSubType: this.#eventSubType,
399
405
  deleteFinishedEventsAfter: this.__deleteFinishedEventsAfter,
400
406
  deleteCount,
401
407
  });
@@ -407,8 +413,8 @@ class EventQueueProcessorBase {
407
413
  return;
408
414
  }
409
415
  this.logger.error("Missing status for selected event entry. Setting status to error", {
410
- eventType: this.__eventType,
411
- eventSubType: this.__eventSubType,
416
+ eventType: this.#eventType,
417
+ eventSubType: this.#eventSubType,
412
418
  queueEntry,
413
419
  });
414
420
  this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
@@ -429,8 +435,8 @@ class EventQueueProcessorBase {
429
435
  }
430
436
 
431
437
  this.logger.error("Not allowed event status returned. Only Open, Done, Error is allowed!", {
432
- eventType: this.__eventType,
433
- eventSubType: this.__eventSubType,
438
+ eventType: this.#eventType,
439
+ eventSubType: this.#eventSubType,
434
440
  queueEntryId,
435
441
  status: statusMap[queueEntryId],
436
442
  });
@@ -447,8 +453,8 @@ class EventQueueProcessorBase {
447
453
  this.logger.error(
448
454
  "Status reported for event queue entry which haven't be selected before. Removing the status.",
449
455
  {
450
- eventType: this.__eventType,
451
- eventSubType: this.__eventSubType,
456
+ eventType: this.#eventType,
457
+ eventSubType: this.#eventSubType,
452
458
  queueEntryId,
453
459
  }
454
460
  );
@@ -458,8 +464,8 @@ class EventQueueProcessorBase {
458
464
 
459
465
  handleErrorDuringClustering(error) {
460
466
  this.logger.error(`Error during clustering of events - setting all queue entries to error. Error: ${error}`, {
461
- eventType: this.__eventType,
462
- eventSubType: this.__eventSubType,
467
+ eventType: this.#eventType,
468
+ eventSubType: this.#eventSubType,
463
469
  });
464
470
  this.__queueEntries.forEach((queueEntry) => {
465
471
  this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
@@ -471,21 +477,13 @@ class EventQueueProcessorBase {
471
477
  "Undefined payload is not allowed. If status should be done, nulls needs to be returned" +
472
478
  " - setting queue entry to error",
473
479
  {
474
- eventType: this.__eventType,
475
- eventSubType: this.__eventSubType,
480
+ eventType: this.#eventType,
481
+ eventSubType: this.#eventSubType,
476
482
  }
477
483
  );
478
484
  this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error);
479
485
  }
480
486
 
481
- static async handleMissingTypeImplementation(context, eventType, eventSubType) {
482
- const baseInstance = new EventQueueProcessorBase(context, eventType, eventSubType);
483
- baseInstance.logger.error("No Implementation found in the provided configuration file.", {
484
- eventType,
485
- eventSubType,
486
- });
487
- }
488
-
489
487
  /**
490
488
  * This function selects all relevant events based on the eventType and eventSubType supplied through the constructor
491
489
  * during initialization of the class.
@@ -495,17 +493,20 @@ class EventQueueProcessorBase {
495
493
  */
496
494
  async getQueueEntriesAndSetToInProgress() {
497
495
  let result = [];
496
+ const refDateStartAfter = new Date(Date.now() + this.#config.runInterval);
498
497
  await executeInNewTransaction(this.__baseContext, "eventQueue-getQueueEntriesAndSetToInProgress", async (tx) => {
499
498
  const entries = await tx.run(
500
- SELECT.from(this.__eventQueueConfig.tableNameEventQueue)
501
- .forUpdate({ wait: this.__eventQueueConfig.forUpdateTimeout })
499
+ SELECT.from(this.#config.tableNameEventQueue)
500
+ .forUpdate({ wait: this.#config.forUpdateTimeout })
502
501
  .limit(this.selectMaxChunkSize)
503
502
  .where(
504
503
  "type =",
505
- this.__eventType,
504
+ this.#eventType,
506
505
  "AND subType=",
507
- this.__eventSubType,
508
- "AND ( status =",
506
+ this.#eventSubType,
507
+ "AND ( startAfter IS NULL OR startAfter <=",
508
+ refDateStartAfter.toISOString(),
509
+ " ) AND ( status =",
509
510
  EventProcessingStatus.Open,
510
511
  "OR ( status =",
511
512
  EventProcessingStatus.Error,
@@ -514,7 +515,7 @@ class EventQueueProcessorBase {
514
515
  ") OR ( status =",
515
516
  EventProcessingStatus.InProgress,
516
517
  "AND lastAttemptTimestamp <=",
517
- new Date(new Date().getTime() - this.__eventQueueConfig.globalTxTimeout).toISOString(),
518
+ new Date(new Date().getTime() - this.#config.globalTxTimeout).toISOString(),
518
519
  ") )"
519
520
  )
520
521
  .orderBy("createdAt", "ID")
@@ -522,37 +523,44 @@ class EventQueueProcessorBase {
522
523
 
523
524
  if (!entries.length) {
524
525
  this.logger.debug("no entries available for processing", {
525
- eventType: this.__eventType,
526
- eventSubType: this.__eventSubType,
526
+ eventType: this.#eventType,
527
+ eventSubType: this.#eventSubType,
527
528
  });
528
529
  this.__emptyChunkSelected = true;
529
530
  return;
530
531
  }
531
532
 
532
- this.#selectedEventMap = arrayToFlatMap(entries);
533
- const { exceededTries, openEvents, exceededTriesExceeded } = this.#filterExceededEvents(entries);
533
+ const { exceededTries, openEvents, exceededTriesExceeded, delayedEvents } = this.#clusterEvents(
534
+ entries,
535
+ refDateStartAfter
536
+ );
537
+ const eventsForProcessing = exceededTries.concat(openEvents).concat(exceededTriesExceeded);
538
+ this.#selectedEventMap = arrayToFlatMap(eventsForProcessing);
534
539
  if (exceededTries.length) {
535
540
  this.#eventsWithExceededTries = exceededTries;
536
541
  }
537
542
  if (exceededTriesExceeded.length) {
538
543
  this.#exceededTriesExceeded = exceededTriesExceeded;
539
544
  }
545
+ this.#handleDelayedEvents(delayedEvents);
540
546
 
541
547
  result = openEvents;
548
+ this.logger.info("Selected event queue entries for processing", {
549
+ openEvents: openEvents.length,
550
+ ...(delayedEvents.length && { delayedEvents: delayedEvents.length }),
551
+ ...(exceededTries.length && { exceededTries: exceededTries.length }),
552
+ eventType: this.#eventType,
553
+ eventSubType: this.#eventSubType,
554
+ });
542
555
 
543
- if (!result.length) {
556
+ if (!eventsForProcessing.length) {
544
557
  this.__emptyChunkSelected = true;
558
+ return;
545
559
  }
546
560
 
547
- this.logger.info("Selected event queue entries for processing", {
548
- queueEntriesCount: result.length,
549
- eventType: this.__eventType,
550
- eventSubType: this.__eventSubType,
551
- });
552
-
553
561
  const isoTimestamp = new Date().toISOString();
554
562
  await tx.run(
555
- UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
563
+ UPDATE.entity(this.#config.tableNameEventQueue)
556
564
  .with({
557
565
  status: EventProcessingStatus.InProgress,
558
566
  lastAttemptTimestamp: isoTimestamp,
@@ -560,10 +568,10 @@ class EventQueueProcessorBase {
560
568
  })
561
569
  .where(
562
570
  "ID IN",
563
- entries.map(({ ID }) => ID)
571
+ eventsForProcessing.map(({ ID }) => ID)
564
572
  )
565
573
  );
566
- entries.forEach((entry) => {
574
+ eventsForProcessing.forEach((entry) => {
567
575
  entry.lastAttemptTimestamp = isoTimestamp;
568
576
  // NOTE: empty payloads are supported on DB-Level.
569
577
  // Behaviour of event queue is: null as payload is treated as obsolete/done
@@ -578,22 +586,45 @@ class EventQueueProcessorBase {
578
586
  return result;
579
587
  }
580
588
 
581
- #filterExceededEvents(events) {
589
+ #handleDelayedEvents(delayedEvents) {
590
+ const eventSchedulerInstance = eventScheduler.getInstance();
591
+ for (const delayedEvent of delayedEvents) {
592
+ eventSchedulerInstance.scheduleEvent(
593
+ this.__context.tenant,
594
+ this.#eventType,
595
+ this.#eventSubType,
596
+ delayedEvent.startAfter
597
+ );
598
+ }
599
+ }
600
+
601
+ #clusterEvents(events, refDateStartAfter) {
602
+ const refDate = new Date(refDateStartAfter.getTime() - this.#config.runInterval + EVENT_START_AFTER_HEADROOM);
582
603
  return events.reduce(
583
604
  (result, event) => {
584
605
  if (event.attempts === this.__retryAttempts + TRIES_FOR_EXCEEDED_EVENTS) {
585
606
  result.exceededTriesExceeded.push(event);
586
607
  } else if (event.attempts >= this.__retryAttempts) {
587
608
  result.exceededTries.push(event);
609
+ } else if (this.#isDelayedEvent(event, refDate)) {
610
+ result.delayedEvents.push(event);
588
611
  } else {
589
612
  result.openEvents.push(event);
590
613
  }
591
614
  return result;
592
615
  },
593
- { exceededTries: [], openEvents: [], exceededTriesExceeded: [] }
616
+ { exceededTries: [], openEvents: [], exceededTriesExceeded: [], delayedEvents: [] }
594
617
  );
595
618
  }
596
619
 
620
+ #isDelayedEvent(event, refDate) {
621
+ if (!event.startAfter) {
622
+ return false;
623
+ }
624
+ event.startAfter = new Date(event.startAfter);
625
+ return !(refDate >= event.startAfter);
626
+ }
627
+
597
628
  async handleExceededEvents() {
598
629
  await this.#handleExceededTriesExceeded();
599
630
  if (!this.#eventsWithExceededTries.length) {
@@ -603,15 +634,15 @@ class EventQueueProcessorBase {
603
634
  for (const exceededEvent of this.#eventsWithExceededTries) {
604
635
  await executeInNewTransaction(
605
636
  this.context,
606
- `eventQueue-handleExceededEvents-${this.eventType}##${this.eventSubType}`,
637
+ `eventQueue-handleExceededEvents-${this.#eventType}##${this.#eventSubType}`,
607
638
  async (tx) => {
608
639
  try {
609
640
  this.processEventContext = tx.context;
610
641
  this.modifyQueueEntry(exceededEvent);
611
642
  await this.hookForExceededEvents({ ...exceededEvent });
612
643
  this.logger.warn("The retry attempts for the following events are exceeded", {
613
- eventType: this.__eventType,
614
- eventSubType: this.__eventSubType,
644
+ eventType: this.#eventType,
645
+ eventSubType: this.#eventSubType,
615
646
  retryAttempts: this.__retryAttempts,
616
647
  queueEntriesId: exceededEvent.ID,
617
648
  currentAttempt: exceededEvent.attempts,
@@ -621,8 +652,8 @@ class EventQueueProcessorBase {
621
652
  this.logger.error(
622
653
  `Caught error during hook for exceeded events - setting queue entry to error. Please catch your promises/exceptions. Error: ${err}`,
623
654
  {
624
- eventType: this.__eventType,
625
- eventSubType: this.__eventSubType,
655
+ eventType: this.#eventType,
656
+ eventSubType: this.#eventSubType,
626
657
  retryAttempts: this.__retryAttempts,
627
658
  queueEntriesId: exceededEvent.ID,
628
659
  currentAttempt: exceededEvent.attempts,
@@ -641,8 +672,8 @@ class EventQueueProcessorBase {
641
672
  async #handleExceededTriesExceeded() {
642
673
  if (this.#exceededTriesExceeded.length) {
643
674
  this.logger.error("Event hook failure exceeded, status set to 'exceeded' without invoking hook again!", {
644
- eventType: this.__eventType,
645
- eventSubType: this.__eventSubType,
675
+ eventType: this.#eventType,
676
+ eventSubType: this.#eventSubType,
646
677
  queueEntriesIds: this.#eventsWithExceededTries.map(({ ID }) => ID),
647
678
  });
648
679
  await executeInNewTransaction(this.context, "exceededTriesExceeded", async (tx) => {
@@ -705,8 +736,8 @@ class EventQueueProcessorBase {
705
736
  const checkAndUpdatePromise = new Promise((resolve, reject) => {
706
737
  executeInNewTransaction(this.__baseContext, "eventProcessing-isOutdatedAndKeepalive", async (tx) => {
707
738
  const queueEntriesFresh = await tx.run(
708
- SELECT.from(this.__eventQueueConfig.tableNameEventQueue)
709
- .forUpdate({ wait: this.__eventQueueConfig.forUpdateTimeout })
739
+ SELECT.from(this.#config.tableNameEventQueue)
740
+ .forUpdate({ wait: this.#config.forUpdateTimeout })
710
741
  .where(
711
742
  "ID IN",
712
743
  queueEntries.map(({ ID }) => ID)
@@ -720,7 +751,7 @@ class EventQueueProcessorBase {
720
751
  let newTs = new Date().toISOString();
721
752
  if (!eventOutdated) {
722
753
  await tx.run(
723
- UPDATE.entity(this.__eventQueueConfig.tableNameEventQueue)
754
+ UPDATE.entity(this.#config.tableNameEventQueue)
724
755
  .set("lastAttemptTimestamp =", newTs)
725
756
  .where(
726
757
  "ID IN",
@@ -730,8 +761,8 @@ class EventQueueProcessorBase {
730
761
  } else {
731
762
  newTs = null;
732
763
  this.logger.warn("event data has been modified. Processing skipped.", {
733
- eventType: this.__eventType,
734
- eventSubType: this.__eventSubType,
764
+ eventType: this.#eventType,
765
+ eventSubType: this.#eventSubType,
735
766
  queueEntriesIds: queueEntries.map(({ ID }) => ID),
736
767
  });
737
768
  queueEntries.forEach(({ ID: queueEntryId }) => delete this.__queueEntriesMap[queueEntryId]);
@@ -761,7 +792,7 @@ class EventQueueProcessorBase {
761
792
 
762
793
  const lockAcquired = await distributedLock.acquireLock(
763
794
  this.context,
764
- [this.eventType, this.eventSubType].join("##")
795
+ [this.#eventType, this.#eventSubType].join("##")
765
796
  );
766
797
  if (!lockAcquired) {
767
798
  return false;
@@ -775,7 +806,7 @@ class EventQueueProcessorBase {
775
806
  return;
776
807
  }
777
808
  try {
778
- await distributedLock.releaseLock(this.context, [this.eventType, this.eventSubType].join("##"));
809
+ await distributedLock.releaseLock(this.context, [this.#eventType, this.#eventSubType].join("##"));
779
810
  } catch (err) {
780
811
  this.logger.error("Releasing distributed lock failed. Error:", err.toString());
781
812
  }
@@ -822,14 +853,14 @@ class EventQueueProcessorBase {
822
853
 
823
854
  get tx() {
824
855
  if (!this.__txUsageAllowed && this.__parallelEventProcessing > 1) {
825
- throw EventQueueError.wrongTxUsage(this.eventType, this.eventSubType);
856
+ throw EventQueueError.wrongTxUsage(this.#eventType, this.#eventSubType);
826
857
  }
827
858
  return this.__processTx ?? this.__tx;
828
859
  }
829
860
 
830
861
  get context() {
831
862
  if (!this.__txUsageAllowed && this.__parallelEventProcessing > 1) {
832
- throw EventQueueError.wrongTxUsage(this.eventType, this.eventSubType);
863
+ throw EventQueueError.wrongTxUsage(this.#eventType, this.#eventSubType);
833
864
  }
834
865
  return this.__processContext ?? this.__context;
835
866
  }
@@ -847,11 +878,11 @@ class EventQueueProcessorBase {
847
878
  }
848
879
 
849
880
  get eventType() {
850
- return this.__eventType;
881
+ return this.#eventType;
851
882
  }
852
883
 
853
884
  get eventSubType() {
854
- return this.__eventSubType;
885
+ return this.#eventSubType;
855
886
  }
856
887
 
857
888
  get emptyChunkSelected() {
package/src/config.js CHANGED
@@ -13,38 +13,56 @@ const REDIS_CONFIG_CHANNEL = "EVENT_QUEUE_CONFIG_CHANNEL";
13
13
  const COMPONENT_NAME = "eventQueue/config";
14
14
 
15
15
  class Config {
16
+ #logger;
17
+ #config;
18
+ #forUpdateTimeout;
19
+ #globalTxTimeout;
20
+ #runInterval;
21
+ #redisEnabled;
22
+ #initialized;
23
+ #parallelTenantProcessing;
24
+ #tableNameEventQueue;
25
+ #tableNameEventLock;
26
+ #isRunnerDeactivated;
27
+ #configFilePath;
28
+ #processEventsAfterPublish;
29
+ #skipCsnCheck;
30
+ #disableRedis;
31
+ #env;
32
+ #eventMap;
16
33
  constructor() {
17
- this.__logger = cds.log(COMPONENT_NAME);
18
- this.__config = null;
19
- this.__forUpdateTimeout = FOR_UPDATE_TIMEOUT;
20
- this.__globalTxTimeout = GLOBAL_TX_TIMEOUT;
21
- this.__runInterval = null;
22
- this.__redisEnabled = null;
23
- this.__initialized = false;
24
- this.__parallelTenantProcessing = null;
25
- this.__tableNameEventQueue = null;
26
- this.__tableNameEventLock = null;
27
- this.__isRunnerDeactivated = false;
28
- this.__configFilePath = null;
29
- this.__processEventsAfterPublish = null;
30
- this.__skipCsnCheck = null;
31
- this.__env = getEnvInstance();
34
+ this.#logger = cds.log(COMPONENT_NAME);
35
+ this.#config = null;
36
+ this.#forUpdateTimeout = FOR_UPDATE_TIMEOUT;
37
+ this.#globalTxTimeout = GLOBAL_TX_TIMEOUT;
38
+ this.#runInterval = null;
39
+ this.#redisEnabled = null;
40
+ this.#initialized = false;
41
+ this.#parallelTenantProcessing = null;
42
+ this.#tableNameEventQueue = null;
43
+ this.#tableNameEventLock = null;
44
+ this.#isRunnerDeactivated = false;
45
+ this.#configFilePath = null;
46
+ this.#processEventsAfterPublish = null;
47
+ this.#skipCsnCheck = null;
48
+ this.#disableRedis = null;
49
+ this.#env = getEnvInstance();
32
50
  }
33
51
 
34
52
  getEventConfig(type, subType) {
35
- return this.__eventMap[[type, subType].join("##")];
53
+ return this.#eventMap[[type, subType].join("##")];
36
54
  }
37
55
 
38
56
  hasEventAfterCommitFlag(type, subType) {
39
- return this.__eventMap[[type, subType].join("##")]?.processAfterCommit ?? true;
57
+ return this.#eventMap[[type, subType].join("##")]?.processAfterCommit ?? true;
40
58
  }
41
59
 
42
60
  _checkRedisIsBound() {
43
- return !!this.__env.getRedisCredentialsFromEnv();
61
+ return !!this.#env.getRedisCredentialsFromEnv();
44
62
  }
45
63
 
46
64
  checkRedisEnabled() {
47
- this.__redisEnabled = this._checkRedisIsBound() && this.__env.isOnCF;
65
+ this.#redisEnabled = !this.#disableRedis && this._checkRedisIsBound() && this.#env.isOnCF;
48
66
  }
49
67
 
50
68
  attachConfigChangeHandler() {
@@ -52,11 +70,11 @@ class Config {
52
70
  try {
53
71
  const { key, value } = JSON.parse(messageData);
54
72
  if (this[key] !== value) {
55
- this.__logger.info("received config change", { key, value });
73
+ this.#logger.info("received config change", { key, value });
56
74
  this[key] = value;
57
75
  }
58
76
  } catch (err) {
59
- this.__logger.error("could not parse event config change", {
77
+ this.#logger.error("could not parse event config change", {
60
78
  messageData,
61
79
  });
62
80
  }
@@ -65,124 +83,132 @@ class Config {
65
83
 
66
84
  publishConfigChange(key, value) {
67
85
  if (!this.redisEnabled) {
68
- this.__logger.info("redis not connected, config change won't be published", { key, value });
86
+ this.#logger.info("redis not connected, config change won't be published", { key, value });
69
87
  return;
70
88
  }
71
89
  redis.publishMessage(REDIS_CONFIG_CHANNEL, JSON.stringify({ key, value })).catch((error) => {
72
- this.__logger.error(`publishing config change failed key: ${key}, value: ${value}`, error);
90
+ this.#logger.error(`publishing config change failed key: ${key}, value: ${value}`, error);
73
91
  });
74
92
  }
75
93
 
76
94
  get isRunnerDeactivated() {
77
- return this.__isRunnerDeactivated;
95
+ return this.#isRunnerDeactivated;
78
96
  }
79
97
 
80
98
  set isRunnerDeactivated(value) {
81
- this.__isRunnerDeactivated = value;
99
+ this.#isRunnerDeactivated = value;
82
100
  }
83
101
 
84
102
  set fileContent(config) {
85
- this.__config = config;
86
- this.__eventMap = config.events.reduce((result, event) => {
103
+ this.#config = config;
104
+ this.#eventMap = config.events.reduce((result, event) => {
87
105
  result[[event.type, event.subType].join("##")] = event;
88
106
  return result;
89
107
  }, {});
90
108
  }
91
109
 
92
110
  get fileContent() {
93
- return this.__config;
111
+ return this.#config;
94
112
  }
95
113
 
96
114
  get events() {
97
- return this.__config.events;
115
+ return this.#config.events;
98
116
  }
99
117
 
100
118
  get forUpdateTimeout() {
101
- return this.__forUpdateTimeout;
119
+ return this.#forUpdateTimeout;
102
120
  }
103
121
 
104
122
  get globalTxTimeout() {
105
- return this.__globalTxTimeout;
123
+ return this.#globalTxTimeout;
106
124
  }
107
125
 
108
126
  set forUpdateTimeout(value) {
109
- this.__forUpdateTimeout = value;
127
+ this.#forUpdateTimeout = value;
110
128
  }
111
129
 
112
130
  set globalTxTimeout(value) {
113
- this.__globalTxTimeout = value;
131
+ this.#globalTxTimeout = value;
114
132
  }
115
133
 
116
134
  get runInterval() {
117
- return this.__runInterval;
135
+ return this.#runInterval;
118
136
  }
119
137
 
120
138
  set runInterval(value) {
121
- this.__runInterval = value;
139
+ this.#runInterval = value;
122
140
  }
123
141
 
124
142
  get redisEnabled() {
125
- return this.__redisEnabled;
143
+ return this.#redisEnabled;
126
144
  }
127
145
 
128
146
  set redisEnabled(value) {
129
- this.__redisEnabled = value;
147
+ this.#redisEnabled = value;
130
148
  }
131
149
 
132
150
  get initialized() {
133
- return this.__initialized;
151
+ return this.#initialized;
134
152
  }
135
153
 
136
154
  set initialized(value) {
137
- this.__initialized = value;
155
+ this.#initialized = value;
138
156
  }
139
157
 
140
158
  get parallelTenantProcessing() {
141
- return this.__parallelTenantProcessing;
159
+ return this.#parallelTenantProcessing;
142
160
  }
143
161
 
144
162
  set parallelTenantProcessing(value) {
145
- this.__parallelTenantProcessing = value;
163
+ this.#parallelTenantProcessing = value;
146
164
  }
147
165
 
148
166
  get tableNameEventQueue() {
149
- return this.__tableNameEventQueue;
167
+ return this.#tableNameEventQueue;
150
168
  }
151
169
 
152
170
  set tableNameEventQueue(value) {
153
- this.__tableNameEventQueue = value;
171
+ this.#tableNameEventQueue = value;
154
172
  }
155
173
 
156
174
  get tableNameEventLock() {
157
- return this.__tableNameEventLock;
175
+ return this.#tableNameEventLock;
158
176
  }
159
177
 
160
178
  set tableNameEventLock(value) {
161
- this.__tableNameEventLock = value;
179
+ this.#tableNameEventLock = value;
162
180
  }
163
181
 
164
182
  set configFilePath(value) {
165
- this.__configFilePath = value;
183
+ this.#configFilePath = value;
166
184
  }
167
185
 
168
186
  get configFilePath() {
169
- return this.__configFilePath;
187
+ return this.#configFilePath;
170
188
  }
171
189
 
172
190
  set processEventsAfterPublish(value) {
173
- this.__processEventsAfterPublish = value;
191
+ this.#processEventsAfterPublish = value;
174
192
  }
175
193
 
176
194
  get processEventsAfterPublish() {
177
- return this.__processEventsAfterPublish;
195
+ return this.#processEventsAfterPublish;
178
196
  }
179
197
 
180
198
  set skipCsnCheck(value) {
181
- this.__skipCsnCheck = value;
199
+ this.#skipCsnCheck = value;
182
200
  }
183
201
 
184
202
  get skipCsnCheck() {
185
- return this.__skipCsnCheck;
203
+ return this.#skipCsnCheck;
204
+ }
205
+
206
+ set disableRedis(value) {
207
+ this.#disableRedis = value;
208
+ }
209
+
210
+ get disableRedis() {
211
+ return this.#disableRedis;
186
212
  }
187
213
 
188
214
  get isMultiTenancy() {
package/src/dbHandler.js CHANGED
@@ -1,8 +1,12 @@
1
1
  "use strict";
2
2
 
3
- const { publishEvent } = require("./redisPubSub");
3
+ const cds = require("@sap/cds");
4
+
5
+ const { broadcastEvent } = require("./redisPubSub");
4
6
  const config = require("./config");
5
7
 
8
+ const COMPONENT_NAME = "eventQueue/dbHandler";
9
+
6
10
  const registerEventQueueDbHandler = (dbService) => {
7
11
  const configInstance = config.getConfigInstance();
8
12
  const def = dbService.model.definitions[configInstance.tableNameEventQueue];
@@ -26,7 +30,12 @@ const registerEventQueueDbHandler = (dbService) => {
26
30
  eventCombinations.length &&
27
31
  req.on("succeeded", () => {
28
32
  for (const eventCombination of eventCombinations) {
29
- publishEvent(req.tenant, ...eventCombination.split("##"));
33
+ broadcastEvent(req.tenant, ...eventCombination.split("##")).catch((err) => {
34
+ cds.log(COMPONENT_NAME).error("db handler failure during broadcasting event", err, {
35
+ tenant: req.tenant,
36
+ eventCombination,
37
+ });
38
+ });
30
39
  }
31
40
  });
32
41
  });
package/src/initialize.js CHANGED
@@ -5,7 +5,6 @@ const fs = require("fs");
5
5
  const path = require("path");
6
6
 
7
7
  const cds = require("@sap/cds");
8
-
9
8
  const yaml = require("yaml");
10
9
  const VError = require("verror");
11
10
 
@@ -23,6 +22,18 @@ const BASE_TABLES = {
23
22
  EVENT: "sap.eventqueue.Event",
24
23
  LOCK: "sap.eventqueue.Lock",
25
24
  };
25
+ const CONFIG_VARS = [
26
+ ["configFilePath", null],
27
+ ["registerAsEventProcessor", true],
28
+ ["processEventsAfterPublish", true],
29
+ ["isRunnerDeactivated", false],
30
+ ["runInterval", 5 * 60 * 1000],
31
+ ["parallelTenantProcessing", 5],
32
+ ["tableNameEventQueue", BASE_TABLES.EVENT],
33
+ ["tableNameEventLock", BASE_TABLES.LOCK],
34
+ ["disableRedis", false],
35
+ ["skipCsnCheck", false],
36
+ ];
26
37
 
27
38
  const initialize = async ({
28
39
  configFilePath,
@@ -33,6 +44,7 @@ const initialize = async ({
33
44
  parallelTenantProcessing,
34
45
  tableNameEventQueue,
35
46
  tableNameEventLock,
47
+ disableRedis,
36
48
  skipCsnCheck,
37
49
  } = {}) => {
38
50
  // TODO: initialize check:
@@ -54,6 +66,7 @@ const initialize = async ({
54
66
  parallelTenantProcessing,
55
67
  tableNameEventQueue,
56
68
  tableNameEventLock,
69
+ disableRedis,
57
70
  skipCsnCheck
58
71
  );
59
72
 
@@ -160,32 +173,12 @@ const checkCustomTable = (baseCsn, customCsn) => {
160
173
  }
161
174
  };
162
175
 
163
- const mixConfigVarsWithEnv = (
164
- configFilePath,
165
- registerAsEventProcessor,
166
- processEventsAfterPublish,
167
- isRunnerDeactivated,
168
- runInterval,
169
- parallelTenantProcessing,
170
- tableNameEventQueue,
171
- tableNameEventLock,
172
- skipCsnCheck
173
- ) => {
176
+ const mixConfigVarsWithEnv = (...args) => {
174
177
  const configInstance = getConfigInstance();
175
-
176
- configInstance.configFilePath = configFilePath ?? cds.env.eventQueue?.configFilePath;
177
- configInstance.registerAsEventProcessor =
178
- registerAsEventProcessor ?? cds.env.eventQueue?.registerAsEventProcessor ?? true;
179
- configInstance.isRunnerDeactivated = isRunnerDeactivated ?? cds.env.eventQueue?.isRunnerDeactivated ?? false;
180
- configInstance.processEventsAfterPublish =
181
- processEventsAfterPublish ?? cds.env.eventQueue?.processEventsAfterPublish ?? true;
182
- configInstance.runInterval = runInterval ?? cds.env.eventQueue?.runInterval ?? 5 * 60 * 1000;
183
- configInstance.parallelTenantProcessing =
184
- parallelTenantProcessing ?? cds.env.eventQueue?.parallelTenantProcessing ?? 5;
185
- configInstance.tableNameEventQueue =
186
- tableNameEventQueue ?? cds.env.eventQueue?.tableNameEventQueue ?? BASE_TABLES.EVENT;
187
- configInstance.tableNameEventLock = tableNameEventLock ?? cds.env.eventQueue?.tableNameEventLock ?? BASE_TABLES.LOCK;
188
- configInstance.skipCsnCheck = skipCsnCheck ?? cds.env.eventQueue?.skipCsnCheck ?? false;
178
+ CONFIG_VARS.forEach(([configName, defaultValue], index) => {
179
+ const configValue = args[index];
180
+ configInstance[configName] = configValue ?? cds.env.eventQueue?.[configName] ?? defaultValue;
181
+ });
189
182
  };
190
183
 
191
184
  module.exports = {
@@ -8,7 +8,6 @@ const { getConfigInstance } = require("./config");
8
8
  const { TransactionMode } = require("./constants");
9
9
  const { limiter, Funnel } = require("./shared/common");
10
10
 
11
- const EventQueueBase = require("./EventQueueProcessorBase");
12
11
  const { executeInNewTransaction, TriggerRollback } = require("./shared/cdsHelper");
13
12
 
14
13
  const COMPONENT_NAME = "eventQueue/processEventQueue";
@@ -33,7 +32,10 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
33
32
  const eventConfig = getConfigInstance().getEventConfig(eventType, eventSubType);
34
33
  const [err, EventTypeClass] = resilientRequire(eventConfig?.impl);
35
34
  if (!eventConfig || err || !(typeof EventTypeClass.constructor === "function")) {
36
- await EventQueueBase.handleMissingTypeImplementation(context, eventType, eventSubType);
35
+ cds.log(COMPONENT_NAME).error("No Implementation found in the provided configuration file.", {
36
+ eventType,
37
+ eventSubType,
38
+ });
37
39
  return;
38
40
  }
39
41
  baseInstance = new EventTypeClass(context, eventType, eventSubType, eventConfig);
@@ -1,19 +1,45 @@
1
1
  "use strict";
2
2
 
3
3
  const config = require("./config");
4
+ const common = require("./shared/common");
4
5
  const EventQueueError = require("./EventQueueError");
5
6
 
7
+ /**
8
+ * Asynchronously publishes a series of events to the event queue.
9
+ *
10
+ * @param {Transaction} tx - The transaction object to be used for database operations.
11
+ * @param {Array|Object} events - An array of event objects or a single event object. Each event object should match the Event table structure:
12
+ * {
13
+ * type: String, // Event type. This is a required field.
14
+ * subType: String, // Event subtype. This is a required field.
15
+ * referenceEntity: String, // Reference entity associated with the event.
16
+ * referenceEntityKey: UUID, // UUID key of the reference entity.
17
+ * status: Status, // Status of the event, defaults to 0.
18
+ * payload: LargeString, // Payload of the event.
19
+ * attempts: Integer, // The number of attempts made, defaults to 0.
20
+ * lastAttemptTimestamp: Timestamp, // Timestamp of the last attempt.
21
+ * createdAt: Timestamp, // Timestamp of event creation. This field is automatically set on insert.
22
+ * startAfter: Timestamp, // Timestamp indicating when the event should start after.
23
+ * }
24
+ * @throws {EventQueueError} Throws an error if the configuration is not initialized.
25
+ * @throws {EventQueueError} Throws an error if the event type is unknown.
26
+ * @throws {EventQueueError} Throws an error if the startAfter field is not a valid date.
27
+ * @returns {Promise} Returns a promise which resolves to the result of the database insert operation.
28
+ */
6
29
  const publishEvent = async (tx, events) => {
7
30
  const configInstance = config.getConfigInstance();
8
31
  if (!configInstance.initialized) {
9
32
  throw EventQueueError.notInitialized();
10
33
  }
11
34
  const eventsForProcessing = Array.isArray(events) ? events : [events];
12
- for (const { type, subType } of eventsForProcessing) {
35
+ for (const { type, subType, startAfter } of eventsForProcessing) {
13
36
  const eventConfig = configInstance.getEventConfig(type, subType);
14
37
  if (!eventConfig) {
15
38
  throw EventQueueError.unknownEventType(type, subType);
16
39
  }
40
+ if (startAfter && !common.isValidDate(startAfter)) {
41
+ throw EventQueueError.malformedDate(startAfter);
42
+ }
17
43
  }
18
44
  return await tx.run(INSERT.into(configInstance.tableNameEventQueue).entries(eventsForProcessing));
19
45
  };
@@ -1,11 +1,9 @@
1
1
  "use strict";
2
2
 
3
3
  const redis = require("./shared/redis");
4
- const { processEventQueue } = require("./processEventQueue");
5
- const { getSubdomainForTenantId } = require("./shared/cdsHelper");
6
4
  const { checkLockExistsAndReturnValue } = require("./shared/distributedLock");
7
5
  const config = require("./config");
8
- const { getWorkerPoolInstance } = require("./shared/WorkerQueue");
6
+ const { runEventCombinationForTenant } = require("./runner");
9
7
 
10
8
  const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
11
9
  const COMPONENT_NAME = "eventQueue/redisPubSub";
@@ -23,19 +21,12 @@ const messageHandlerProcessEvents = async (messageData) => {
23
21
  const logger = cds.log(COMPONENT_NAME);
24
22
  try {
25
23
  const { tenantId, type, subType } = JSON.parse(messageData);
26
- const subdomain = await getSubdomainForTenantId(tenantId);
27
- const context = new cds.EventContext({
28
- tenant: tenantId,
29
- // NOTE: we need this because of logging otherwise logs would not contain the subdomain
30
- http: { req: { authInfo: { getSubdomain: () => subdomain } } },
31
- });
32
- cds.context = context;
33
24
  logger.debug("received redis event", {
34
25
  tenantId,
35
26
  type,
36
27
  subType,
37
28
  });
38
- getWorkerPoolInstance().addToQueue(async () => processEventQueue(context, type, subType));
29
+ await runEventCombinationForTenant(tenantId, type, subType);
39
30
  } catch (err) {
40
31
  logger.error("could not parse event information", {
41
32
  messageData,
@@ -43,12 +34,12 @@ const messageHandlerProcessEvents = async (messageData) => {
43
34
  }
44
35
  };
45
36
 
46
- const publishEvent = async (tenantId, type, subType) => {
37
+ const broadcastEvent = async (tenantId, type, subType) => {
47
38
  const logger = cds.log(COMPONENT_NAME);
48
39
  const configInstance = config.getConfigInstance();
49
40
  if (!configInstance.redisEnabled) {
50
41
  if (configInstance.registerAsEventProcessor) {
51
- await _handleEventInternally(tenantId, type, subType);
42
+ await runEventCombinationForTenant(tenantId, type, subType);
52
43
  }
53
44
  return;
54
45
  }
@@ -76,22 +67,7 @@ const publishEvent = async (tenantId, type, subType) => {
76
67
  }
77
68
  };
78
69
 
79
- const _handleEventInternally = async (tenantId, type, subType) => {
80
- cds.log(COMPONENT_NAME).info("processEventQueue internally", {
81
- tenantId,
82
- type,
83
- subType,
84
- });
85
- const subdomain = await getSubdomainForTenantId(tenantId);
86
- const context = new cds.EventContext({
87
- tenant: tenantId,
88
- // NOTE: we need this because of logging otherwise logs would not contain the subdomain
89
- http: { req: { authInfo: { getSubdomain: () => subdomain } } },
90
- });
91
- getWorkerPoolInstance().addToQueue(async () => processEventQueue(context, type, subType));
92
- };
93
-
94
70
  module.exports = {
95
71
  initEventQueueRedisSubscribe,
96
- publishEvent,
72
+ broadcastEvent,
97
73
  };
package/src/runner.js CHANGED
@@ -1,13 +1,14 @@
1
1
  "use strict";
2
2
 
3
- const uuid = require("uuid");
3
+ const { randomUUID } = require("crypto");
4
4
 
5
5
  const eventQueueConfig = require("./config");
6
- const { eventQueueRunner } = require("./processEventQueue");
6
+ const { eventQueueRunner, processEventQueue } = require("./processEventQueue");
7
7
  const { getWorkerPoolInstance } = require("./shared/WorkerQueue");
8
8
  const cdsHelper = require("./shared/cdsHelper");
9
9
  const distributedLock = require("./shared/distributedLock");
10
10
  const SetIntervalDriftSafe = require("./shared/SetIntervalDriftSafe");
11
+ const { getSubdomainForTenantId } = require("./shared/cdsHelper");
11
12
 
12
13
  const COMPONENT_NAME = "eventQueue/runner";
13
14
  const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
@@ -130,7 +131,7 @@ const _executeRunForTenant = async (tenantId, runId) => {
130
131
 
131
132
  const _acquireRunId = async (context) => {
132
133
  const configInstance = eventQueueConfig.getConfigInstance();
133
- let runId = uuid.v4();
134
+ let runId = randomUUID();
134
135
  const couldSetValue = await distributedLock.setValueWithExpire(context, EVENT_QUEUE_RUN_ID, runId, {
135
136
  tenantScoped: false,
136
137
  expiryTime: configInstance.runInterval * 0.95,
@@ -190,10 +191,31 @@ const _calculateOffsetForFirstRun = async () => {
190
191
  return offsetDependingOnLastRun;
191
192
  };
192
193
 
194
+ const runEventCombinationForTenant = async (tenantId, type, subType) => {
195
+ try {
196
+ const subdomain = await getSubdomainForTenantId(tenantId);
197
+ const context = new cds.EventContext({
198
+ tenant: tenantId,
199
+ // NOTE: we need this because of logging otherwise logs would not contain the subdomain
200
+ http: { req: { authInfo: { getSubdomain: () => subdomain } } },
201
+ });
202
+ cds.context = context;
203
+ getWorkerPoolInstance().addToQueue(async () => await processEventQueue(context, type, subType));
204
+ } catch (err) {
205
+ const logger = cds.log(COMPONENT_NAME);
206
+ logger.error("error executing event combination for tenant", err, {
207
+ tenantId,
208
+ type,
209
+ subType,
210
+ });
211
+ }
212
+ };
213
+
193
214
  module.exports = {
194
215
  singleTenant,
195
216
  multiTenancyDb,
196
217
  multiTenancyRedis,
218
+ runEventCombinationForTenant,
197
219
  _: {
198
220
  _multiTenancyRedis,
199
221
  _multiTenancyDb,
@@ -0,0 +1,53 @@
1
+ "use strict";
2
+
3
+ const cds = require("@sap/cds");
4
+
5
+ const { broadcastEvent } = require("../redisPubSub");
6
+
7
+ const COMPONENT_NAME = "eventQueue/shared/EventScheduler";
8
+
9
+ let instance;
10
+ class EventScheduler {
11
+ #scheduledEvents = {};
12
+ constructor() {}
13
+
14
+ scheduleEvent(tenantId, type, subType, startAfter) {
15
+ const startAfterSeconds = startAfter.getSeconds();
16
+ const secondsUntilNextTen = 10 - (startAfterSeconds % 10);
17
+ const roundUpDate = new Date(startAfter.getTime() + secondsUntilNextTen * 1000);
18
+ const key = [tenantId, type, subType, roundUpDate.toISOString()].join("##");
19
+ if (this.#scheduledEvents[key]) {
20
+ return; // event combination already scheduled
21
+ }
22
+ this.#scheduledEvents[key] = true;
23
+ cds.log(COMPONENT_NAME).info("scheduling event queue run for delayed event", {
24
+ type,
25
+ subType,
26
+ delaySeconds: (roundUpDate.getTime() - Date.now()) / 1000,
27
+ });
28
+ setTimeout(() => {
29
+ delete this.#scheduledEvents[key];
30
+ broadcastEvent(tenantId, type, subType).catch((err) => {
31
+ cds.log(COMPONENT_NAME).error("could not execute scheduled event", err, {
32
+ tenantId,
33
+ type,
34
+ subType,
35
+ scheduledFor: roundUpDate.toISOString(),
36
+ });
37
+ });
38
+ }, roundUpDate.getTime() - Date.now()).unref();
39
+ }
40
+
41
+ clearScheduledEvents() {
42
+ this.#scheduledEvents = {};
43
+ }
44
+ }
45
+
46
+ module.exports = {
47
+ getInstance: () => {
48
+ if (!instance) {
49
+ instance = new EventScheduler();
50
+ }
51
+ return instance;
52
+ },
53
+ };
@@ -107,4 +107,15 @@ const limiter = async (limit, payloads, iterator) => {
107
107
  return Promise.allSettled(returnPromises);
108
108
  };
109
109
 
110
- module.exports = { arrayToFlatMap, Funnel, limiter };
110
+ const isValidDate = (value) => {
111
+ if (typeof value === "string") {
112
+ const date = Date.parse(value);
113
+ return !isNaN(date);
114
+ } else if (value instanceof Date) {
115
+ return !isNaN(value.getTime());
116
+ } else {
117
+ return false;
118
+ }
119
+ };
120
+
121
+ module.exports = { arrayToFlatMap, Funnel, limiter, isValidDate };