@cap-js-community/event-queue 1.9.3 → 1.10.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -2
- package/src/EventQueueProcessorBase.js +1 -1
- package/src/config.js +163 -40
- package/src/index.d.ts +1 -1
- package/src/outbox/EventQueueGenericOutboxHandler.js +214 -13
- package/src/outbox/eventQueueAsOutbox.js +17 -8
- package/src/processEventQueue.js +1 -1
- package/src/publishEvent.js +6 -0
- package/src/runner/openEvents.js +6 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cap-js-community/event-queue",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.10.0-beta.1",
|
|
4
4
|
"description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
"scripts": {
|
|
24
24
|
"test:unit": "jest --testPathIgnorePatterns=\"/test-integration/\"",
|
|
25
25
|
"test:integration": "jest --testPathIgnorePatterns=\"/test/\" --runInBand --forceExit",
|
|
26
|
+
"voter:test:integration": "jest --testPathIgnorePatterns=\"/test/\" --forceExit",
|
|
26
27
|
"test": "npm run test:unit && npm run test:integration",
|
|
27
28
|
"test:all:coverage": "jest --runInBand --forceExit --collect-coverage",
|
|
28
29
|
"test:prepare": "npm run build:ci --prefix=./test-integration/_env",
|
|
@@ -64,7 +65,8 @@
|
|
|
64
65
|
"jest": "^29.7.0",
|
|
65
66
|
"prettier": "^2.8.8",
|
|
66
67
|
"sqlite3": "^5.1.7",
|
|
67
|
-
"@opentelemetry/api": "^1.9.0"
|
|
68
|
+
"@opentelemetry/api": "^1.9.0",
|
|
69
|
+
"@actions/core": "^1.11.1"
|
|
68
70
|
},
|
|
69
71
|
"homepage": "https://cap-js-community.github.io/event-queue/",
|
|
70
72
|
"repository": {
|
|
@@ -243,7 +243,7 @@ class EventQueueProcessorBase {
|
|
|
243
243
|
* This can be useful for e.g. multiple tasks have been scheduled and always the same user should be informed.
|
|
244
244
|
* In this case the events should be clustered together and only one mail should be sent.
|
|
245
245
|
*/
|
|
246
|
-
clusterQueueEntries(queueEntriesWithPayloadMap) {
|
|
246
|
+
async clusterQueueEntries(queueEntriesWithPayloadMap) {
|
|
247
247
|
Object.entries(queueEntriesWithPayloadMap).forEach(([key, { queueEntry, payload }]) => {
|
|
248
248
|
this.addEntryToProcessingMap(key, queueEntry, payload);
|
|
249
249
|
});
|
package/src/config.js
CHANGED
|
@@ -36,6 +36,45 @@ const BASE_TABLES = {
|
|
|
36
36
|
LOCK: "sap.eventqueue.Lock",
|
|
37
37
|
};
|
|
38
38
|
|
|
39
|
+
const ALLOWED_EVENT_OPTIONS_BASE = [
|
|
40
|
+
"type",
|
|
41
|
+
"subType",
|
|
42
|
+
"load",
|
|
43
|
+
"impl",
|
|
44
|
+
"transactionMode",
|
|
45
|
+
"deleteFinishedEventsAfterDays",
|
|
46
|
+
"useEventQueueUser",
|
|
47
|
+
"priority",
|
|
48
|
+
"keepAliveInterval",
|
|
49
|
+
"increasePriorityOverTime",
|
|
50
|
+
"keepAliveMaxInProgressTime",
|
|
51
|
+
"appNames",
|
|
52
|
+
"appInstances",
|
|
53
|
+
"internalEvent",
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
const ALLOWED_EVENT_OPTIONS_AD_HOC = [
|
|
57
|
+
...ALLOWED_EVENT_OPTIONS_BASE,
|
|
58
|
+
"inheritTraceContext",
|
|
59
|
+
"selectMaxChunkSize",
|
|
60
|
+
"parallelEventProcessing",
|
|
61
|
+
"retryAttempts",
|
|
62
|
+
"processAfterCommit",
|
|
63
|
+
"checkForNextChunk",
|
|
64
|
+
"retryFailedAfter",
|
|
65
|
+
"multiInstanceProcessing",
|
|
66
|
+
"kind",
|
|
67
|
+
"timeBucket",
|
|
68
|
+
];
|
|
69
|
+
|
|
70
|
+
const ALLOWED_EVENT_OPTIONS_PERIODIC_EVENT = [
|
|
71
|
+
...ALLOWED_EVENT_OPTIONS_BASE,
|
|
72
|
+
"interval",
|
|
73
|
+
"cron",
|
|
74
|
+
"utc",
|
|
75
|
+
"useCronTimezone",
|
|
76
|
+
];
|
|
77
|
+
|
|
39
78
|
class Config {
|
|
40
79
|
#logger;
|
|
41
80
|
#config;
|
|
@@ -286,7 +325,7 @@ class Config {
|
|
|
286
325
|
});
|
|
287
326
|
}
|
|
288
327
|
|
|
289
|
-
|
|
328
|
+
addCAPOutboxEventBase(serviceName, config) {
|
|
290
329
|
if (this.#eventMap[this.generateKey(CAP_EVENT_TYPE, serviceName)]) {
|
|
291
330
|
const index = this.#config.events.findIndex(
|
|
292
331
|
(event) => event.type === CAP_EVENT_TYPE && event.subType === serviceName
|
|
@@ -294,36 +333,49 @@ class Config {
|
|
|
294
333
|
this.#config.events.splice(index, 1);
|
|
295
334
|
}
|
|
296
335
|
|
|
297
|
-
const eventConfig = {
|
|
336
|
+
const eventConfig = this.#sanitizeParamsAdHocEvent({
|
|
298
337
|
type: CAP_EVENT_TYPE,
|
|
299
338
|
subType: serviceName,
|
|
300
|
-
load: config.load,
|
|
301
339
|
impl: "./outbox/EventQueueGenericOutboxHandler",
|
|
302
|
-
|
|
340
|
+
kind: config.kind ?? "persistent-outbox",
|
|
341
|
+
selectMaxChunkSize: config.selectMaxChunkSize ?? config.chunkSize,
|
|
303
342
|
parallelEventProcessing: config.parallelEventProcessing ?? (config.parallel && CAP_PARALLEL_DEFAULT),
|
|
304
|
-
retryAttempts: config.maxAttempts,
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
deleteFinishedEventsAfterDays: config.deleteFinishedEventsAfterDays,
|
|
309
|
-
appNames: config.appNames,
|
|
310
|
-
appInstances: config.appInstances,
|
|
311
|
-
useEventQueueUser: config.useEventQueueUser,
|
|
312
|
-
retryFailedAfter: config.retryFailedAfter,
|
|
313
|
-
priority: config.priority,
|
|
314
|
-
multiInstanceProcessing: config.multiInstanceProcessing,
|
|
315
|
-
increasePriorityOverTime: config.increasePriorityOverTime,
|
|
316
|
-
keepAliveInterval: config.keepAliveInterval,
|
|
317
|
-
inheritTraceContext: true,
|
|
318
|
-
internalEvent: true,
|
|
319
|
-
};
|
|
343
|
+
retryAttempts: config.retryAttempts ?? config.maxAttempts,
|
|
344
|
+
...config,
|
|
345
|
+
});
|
|
346
|
+
eventConfig.internalEvent = true;
|
|
320
347
|
|
|
321
348
|
this.#basicEventTransformation(eventConfig);
|
|
349
|
+
this.#validateAdHocEvents(this.#eventMap, eventConfig, false);
|
|
322
350
|
this.#basicEventTransformationAfterValidate(eventConfig);
|
|
323
351
|
this.#config.events.push(eventConfig);
|
|
324
352
|
this.#eventMap[this.generateKey(CAP_EVENT_TYPE, serviceName)] = eventConfig;
|
|
325
353
|
}
|
|
326
354
|
|
|
355
|
+
addCAPOutboxEventSpecificAction(serviceName, actionName) {
|
|
356
|
+
const subType = [serviceName, actionName].join(".");
|
|
357
|
+
if (this.#eventMap[this.generateKey(CAP_EVENT_TYPE, subType)]) {
|
|
358
|
+
const index = this.#config.events.findIndex(
|
|
359
|
+
(event) => event.type === CAP_EVENT_TYPE && event.subType === serviceName
|
|
360
|
+
);
|
|
361
|
+
this.#config.events.splice(index, 1);
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
const eventConfig = this.#sanitizeParamsAdHocEvent({
|
|
365
|
+
...this.getEventConfig(CAP_EVENT_TYPE, serviceName),
|
|
366
|
+
...this.getCdsOutboxEventSpecificConfig(serviceName, actionName),
|
|
367
|
+
subType,
|
|
368
|
+
});
|
|
369
|
+
eventConfig.internalEvent = true;
|
|
370
|
+
|
|
371
|
+
this.#basicEventTransformation(eventConfig);
|
|
372
|
+
this.#validateAdHocEvents(this.#eventMap, eventConfig, false);
|
|
373
|
+
this.#basicEventTransformationAfterValidate(eventConfig);
|
|
374
|
+
this.#config.events.push(eventConfig);
|
|
375
|
+
this.#eventMap[this.generateKey(CAP_EVENT_TYPE, subType)] = eventConfig;
|
|
376
|
+
return eventConfig;
|
|
377
|
+
}
|
|
378
|
+
|
|
327
379
|
#unblockEventLocalState(key, tenant) {
|
|
328
380
|
const map = this.#blockedEvents[key];
|
|
329
381
|
if (!map) {
|
|
@@ -356,11 +408,59 @@ class Config {
|
|
|
356
408
|
fileContent.periodicEvents ??= [];
|
|
357
409
|
const events = this.#configEvents ?? {};
|
|
358
410
|
const periodicEvents = this.#configPeriodicEvents ?? {};
|
|
411
|
+
const periodicCapServiceEvents = this.#cdsPeriodicOutboxServicesFromEnv();
|
|
359
412
|
fileContent.events = fileContent.events.concat(this.#mapEnvEvents(events));
|
|
360
|
-
fileContent.periodicEvents = fileContent.periodicEvents
|
|
413
|
+
fileContent.periodicEvents = fileContent.periodicEvents
|
|
414
|
+
.concat(this.#mapEnvEvents(periodicEvents))
|
|
415
|
+
.concat(this.#mapCapOutboxPeriodicEvent(periodicCapServiceEvents));
|
|
361
416
|
this.fileContent = fileContent;
|
|
362
417
|
}
|
|
363
418
|
|
|
419
|
+
#mapCapOutboxPeriodicEvent(periodicEventMap) {
|
|
420
|
+
return Object.values(periodicEventMap);
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
#cdsPeriodicOutboxServicesFromEnv() {
|
|
424
|
+
return Object.entries(cds.env.requires).reduce((result, [name, value]) => {
|
|
425
|
+
if (value.outbox?.events) {
|
|
426
|
+
for (const fnName in value.outbox.events) {
|
|
427
|
+
const base = { ...value.outbox };
|
|
428
|
+
const fnConfig = value.outbox.events[fnName];
|
|
429
|
+
if (fnConfig.interval || fnConfig.cron) {
|
|
430
|
+
if ("interval" in base || "cron" in base) {
|
|
431
|
+
this.#logger.error(
|
|
432
|
+
"The properties interval|cron must be defined in the event section and will be ignored in the outbox section.",
|
|
433
|
+
{ serviceName: name }
|
|
434
|
+
);
|
|
435
|
+
delete base.cron;
|
|
436
|
+
delete base.interval;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
result[fnName] = Object.assign(
|
|
440
|
+
{
|
|
441
|
+
type: CAP_EVENT_TYPE,
|
|
442
|
+
subType: `${name}.${fnName}`,
|
|
443
|
+
impl: "./outbox/EventQueueGenericOutboxHandler",
|
|
444
|
+
internalEvent: true,
|
|
445
|
+
},
|
|
446
|
+
base,
|
|
447
|
+
fnConfig
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
return result;
|
|
453
|
+
}, {});
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
getCdsOutboxEventSpecificConfig(serviceName, action) {
|
|
457
|
+
if (cds.env.requires[serviceName]?.outbox?.events?.[action]) {
|
|
458
|
+
return cds.env.requires[serviceName].outbox.events[action];
|
|
459
|
+
} else {
|
|
460
|
+
return null;
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
364
464
|
#mapEnvEvents(events) {
|
|
365
465
|
return Object.entries(events)
|
|
366
466
|
.map(([key, event]) => {
|
|
@@ -379,19 +479,21 @@ class Config {
|
|
|
379
479
|
config.events = config.events ?? [];
|
|
380
480
|
config.periodicEvents = config.periodicEvents ?? [];
|
|
381
481
|
this.#eventMap = config.events.reduce((result, event) => {
|
|
382
|
-
this.#
|
|
383
|
-
this.#
|
|
384
|
-
this.#
|
|
385
|
-
|
|
482
|
+
const eventSanitized = this.#sanitizeParamsAdHocEvent(event);
|
|
483
|
+
this.#basicEventTransformation(eventSanitized);
|
|
484
|
+
this.#validateAdHocEvents(result, eventSanitized);
|
|
485
|
+
this.#basicEventTransformationAfterValidate(eventSanitized);
|
|
486
|
+
result[this.generateKey(eventSanitized.type, eventSanitized.subType)] = eventSanitized;
|
|
386
487
|
return result;
|
|
387
488
|
}, {});
|
|
388
489
|
this.#eventMap = config.periodicEvents.reduce((result, event) => {
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
this.#
|
|
393
|
-
this.#
|
|
394
|
-
|
|
490
|
+
const eventSanitized = this.#sanitizeParamsPeriodicEventEvent(event);
|
|
491
|
+
eventSanitized.type = `${eventSanitized.type}${SUFFIX_PERIODIC}`;
|
|
492
|
+
eventSanitized.isPeriodic = true;
|
|
493
|
+
this.#basicEventTransformation(eventSanitized);
|
|
494
|
+
this.#validatePeriodicConfig(result, eventSanitized);
|
|
495
|
+
this.#basicEventTransformationAfterValidate(eventSanitized);
|
|
496
|
+
result[this.generateKey(eventSanitized.type, eventSanitized.subType)] = eventSanitized;
|
|
395
497
|
return result;
|
|
396
498
|
}, this.#eventMap);
|
|
397
499
|
this.#config = config;
|
|
@@ -405,6 +507,23 @@ class Config {
|
|
|
405
507
|
event.keepAliveMaxInProgressTime = event.keepAliveInterval * DEFAULT_MAX_FACTOR_STUCK_2_KEEP_ALIVE_INTERVAL;
|
|
406
508
|
}
|
|
407
509
|
|
|
510
|
+
#sanitizeParamsBase(config, allowList) {
|
|
511
|
+
return Object.entries(config).reduce((result, [name, value]) => {
|
|
512
|
+
if (allowList.includes(name)) {
|
|
513
|
+
result[name] = value;
|
|
514
|
+
}
|
|
515
|
+
return result;
|
|
516
|
+
}, {});
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
#sanitizeParamsAdHocEvent(config) {
|
|
520
|
+
return this.#sanitizeParamsBase(config, ALLOWED_EVENT_OPTIONS_AD_HOC);
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
#sanitizeParamsPeriodicEventEvent(config) {
|
|
524
|
+
return this.#sanitizeParamsBase(config, ALLOWED_EVENT_OPTIONS_PERIODIC_EVENT);
|
|
525
|
+
}
|
|
526
|
+
|
|
408
527
|
#basicEventTransformationAfterValidate(event) {
|
|
409
528
|
event._appNameMap = event.appNames ? Object.fromEntries(new Map(event.appNames.map((a) => [a, true]))) : null;
|
|
410
529
|
event._appInstancesMap = event.appInstances
|
|
@@ -498,16 +617,12 @@ class Config {
|
|
|
498
617
|
throw EventQueueError.invalidInterval(event.type, event.subType, event.interval);
|
|
499
618
|
}
|
|
500
619
|
|
|
501
|
-
if (event.multiInstanceProcessing) {
|
|
502
|
-
throw EventQueueError.multiInstanceProcessingNotAllowed(event.type, event.subType);
|
|
503
|
-
}
|
|
504
|
-
|
|
505
620
|
this.#basicEventValidation(event);
|
|
506
621
|
}
|
|
507
622
|
|
|
508
|
-
#validateAdHocEvents(eventMap, event) {
|
|
623
|
+
#validateAdHocEvents(eventMap, event, checkForDuplication = true) {
|
|
509
624
|
const key = this.generateKey(event.type, event.subType);
|
|
510
|
-
if (eventMap[key] && !eventMap[key].isPeriodic) {
|
|
625
|
+
if (eventMap[key] && !eventMap[key].isPeriodic && checkForDuplication) {
|
|
511
626
|
throw EventQueueError.duplicateEventRegistration(event.type, event.subType);
|
|
512
627
|
}
|
|
513
628
|
|
|
@@ -516,6 +631,14 @@ class Config {
|
|
|
516
631
|
}
|
|
517
632
|
event.inheritTraceContext = event.inheritTraceContext ?? DEFAULT_INHERIT_TRACE_CONTEXT;
|
|
518
633
|
|
|
634
|
+
if (event.timeBucket) {
|
|
635
|
+
try {
|
|
636
|
+
CronExpressionParser.parse(event.timeBucket);
|
|
637
|
+
} catch {
|
|
638
|
+
throw EventQueueError.cantParseCronExpression(event.type, event.subType, event.timeBucket);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
|
|
519
642
|
this.#basicEventValidation(event);
|
|
520
643
|
}
|
|
521
644
|
|
|
@@ -540,7 +663,7 @@ class Config {
|
|
|
540
663
|
}
|
|
541
664
|
|
|
542
665
|
get events() {
|
|
543
|
-
return this.#
|
|
666
|
+
return Object.values(this.#eventMap).filter((e) => !e.isPeriodic);
|
|
544
667
|
}
|
|
545
668
|
|
|
546
669
|
set configEvents(value) {
|
|
@@ -556,7 +679,7 @@ class Config {
|
|
|
556
679
|
}
|
|
557
680
|
|
|
558
681
|
get periodicEvents() {
|
|
559
|
-
return this.#
|
|
682
|
+
return Object.values(this.#eventMap).filter((e) => e.isPeriodic);
|
|
560
683
|
}
|
|
561
684
|
|
|
562
685
|
isPeriodicEvent(type, subType) {
|
|
@@ -564,7 +687,7 @@ class Config {
|
|
|
564
687
|
}
|
|
565
688
|
|
|
566
689
|
get allEvents() {
|
|
567
|
-
return
|
|
690
|
+
return Object.values(this.#eventMap);
|
|
568
691
|
}
|
|
569
692
|
|
|
570
693
|
get forUpdateTimeout() {
|
package/src/index.d.ts
CHANGED
|
@@ -194,7 +194,7 @@ declare class Config {
|
|
|
194
194
|
blockEvent(type: any, subType: any, isPeriodic: any, tenant?: string): void;
|
|
195
195
|
clearPeriodicEventBlockList(): void;
|
|
196
196
|
unblockEvent(type: any, subType: any, isPeriodic: any, tenant?: string): void;
|
|
197
|
-
|
|
197
|
+
addCAPOutboxEventBase(serviceName: any, config: any): void;
|
|
198
198
|
isEventBlocked(type: any, subType: any, isPeriodicEvent: any, tenant: any): any;
|
|
199
199
|
set isEventQueueActive(value: boolean);
|
|
200
200
|
get isEventQueueActive(): boolean;
|
|
@@ -15,21 +15,222 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
|
|
|
15
15
|
this.logger = cds.log(`${COMPONENT_NAME}/${eventSubType}`);
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
+
async getQueueEntriesAndSetToInProgress() {
|
|
19
|
+
const [serviceName] = this.eventSubType.split(".");
|
|
20
|
+
this.__srv = await cds.connect.to(serviceName);
|
|
21
|
+
this.__srvUnboxed = cds.unboxed(this.__srv);
|
|
22
|
+
const { handlers, clusterRelevant, specificClusterRelevant } = this.__srvUnboxed.handlers.on.reduce(
|
|
23
|
+
(result, handler) => {
|
|
24
|
+
if (handler.on.startsWith("clusterQueueEntries")) {
|
|
25
|
+
if (handler.on.split(".").length === 2) {
|
|
26
|
+
result.specificClusterRelevant = true;
|
|
27
|
+
} else {
|
|
28
|
+
result.clusterRelevant = true;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
result.handlers[handler.on] = handler.on;
|
|
33
|
+
return result;
|
|
34
|
+
},
|
|
35
|
+
{ handlers: {}, clusterRelevant: false, specificClusterRelevant: false }
|
|
36
|
+
);
|
|
37
|
+
this.__onHandlers = handlers;
|
|
38
|
+
this.__genericClusterHandler = clusterRelevant;
|
|
39
|
+
this.__specificClusterHandler = specificClusterRelevant;
|
|
40
|
+
await this.#setContextUser(this.context, config.userId);
|
|
41
|
+
return super.getQueueEntriesAndSetToInProgress();
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// NOTE: issue here: if events are not sorted before it might not be unique here:
|
|
45
|
+
// - we have service events
|
|
46
|
+
// - we have action specific events
|
|
47
|
+
// 1. I need to collect all action names
|
|
48
|
+
// 2. I need to check for which actions a handler exits
|
|
49
|
+
// 3. For actions with specific existing handler --> call specific action
|
|
50
|
+
// 4. For all others call generic handler
|
|
51
|
+
// NOTE: OVERALL idea is that the handler returns the cluster map and MUST not call any baseImpl functions!
|
|
52
|
+
// --> structure is a map of { key: { queueEntries: [], payload: {} }
|
|
53
|
+
// TODO: document that clusterQueueEntries is now async!!!
|
|
54
|
+
// TODO: validate that return structure is as expected
|
|
55
|
+
async clusterQueueEntries(queueEntriesWithPayloadMap) {
|
|
56
|
+
if (!this.__genericClusterRelevantAndAvailable && !this.__specificClusterRelevantAndAvailable) {
|
|
57
|
+
return super.clusterQueueEntries(queueEntriesWithPayloadMap);
|
|
58
|
+
}
|
|
59
|
+
const { genericClusterEvents, specificClusterEvents } = this.#clusterByAction(queueEntriesWithPayloadMap);
|
|
60
|
+
if (Object.keys(genericClusterEvents).length) {
|
|
61
|
+
if (!this.__genericClusterRelevantAndAvailable) {
|
|
62
|
+
await super.clusterQueueEntries(genericClusterEvents);
|
|
63
|
+
} else {
|
|
64
|
+
const msg = new cds.Request({
|
|
65
|
+
event: "clusterQueueEntries",
|
|
66
|
+
data: { queueEntriesWithPayloadMap: genericClusterEvents },
|
|
67
|
+
eventQueue: {
|
|
68
|
+
processor: this,
|
|
69
|
+
clusterByPayloadProperty: (propertyName) =>
|
|
70
|
+
EventQueueGenericOutboxHandler.clusterByPayloadProperty(genericClusterEvents, propertyName),
|
|
71
|
+
clusterByEventProperty: (propertyName) =>
|
|
72
|
+
EventQueueGenericOutboxHandler.clusterByEventProperty(genericClusterEvents, propertyName),
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
|
|
76
|
+
this.#addToProcessingMap(handlerCluster);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
for (const actionName in specificClusterEvents) {
|
|
81
|
+
const msg = new cds.Request({
|
|
82
|
+
event: `clusterQueueEntries.${actionName}`,
|
|
83
|
+
data: { queueEntriesWithPayloadMap: specificClusterEvents[actionName] },
|
|
84
|
+
eventQueue: {
|
|
85
|
+
processor: this,
|
|
86
|
+
clusterByPayloadProperty: (propertyName) =>
|
|
87
|
+
EventQueueGenericOutboxHandler.clusterByPayloadProperty(specificClusterEvents[actionName], propertyName),
|
|
88
|
+
clusterByEventProperty: (propertyName) =>
|
|
89
|
+
EventQueueGenericOutboxHandler.clusterByEventProperty(specificClusterEvents[actionName], propertyName),
|
|
90
|
+
},
|
|
91
|
+
});
|
|
92
|
+
const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
|
|
93
|
+
this.#addToProcessingMap(handlerCluster);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
static clusterByPayloadProperty(queueEntriesWithPayloadMap, propertyName) {
|
|
98
|
+
return Object.entries(queueEntriesWithPayloadMap).reduce((result, [, { queueEntry, payload }]) => {
|
|
99
|
+
result[payload[propertyName]] ??= {
|
|
100
|
+
queueEntries: [],
|
|
101
|
+
payload,
|
|
102
|
+
};
|
|
103
|
+
result[payload[propertyName]].queueEntries.push(queueEntry);
|
|
104
|
+
return result;
|
|
105
|
+
}, {});
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
static clusterByEventProperty(queueEntriesWithPayloadMap, propertyName) {
|
|
109
|
+
return Object.entries(queueEntriesWithPayloadMap).reduce((result, [, { queueEntry, payload }]) => {
|
|
110
|
+
result[queueEntry[propertyName]] ??= {
|
|
111
|
+
queueEntries: [],
|
|
112
|
+
payload,
|
|
113
|
+
};
|
|
114
|
+
result[queueEntry[propertyName]].queueEntries.push(queueEntry);
|
|
115
|
+
return result;
|
|
116
|
+
}, {});
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
#clusterByAction(queueEntriesWithPayloadMap) {
|
|
120
|
+
return Object.entries(queueEntriesWithPayloadMap).reduce(
|
|
121
|
+
(result, [eventId, clusterData]) => {
|
|
122
|
+
const hasSpecificClusterHandler = this.#hasEventSpecificClusterHandler(clusterData.queueEntry);
|
|
123
|
+
if (hasSpecificClusterHandler && this.__specificClusterRelevantAndAvailable) {
|
|
124
|
+
result.specificClusterEvents[clusterData.payload.event] ??= {};
|
|
125
|
+
result.specificClusterEvents[clusterData.payload.event][eventId] = clusterData;
|
|
126
|
+
} else {
|
|
127
|
+
result.genericClusterEvents[eventId] = clusterData;
|
|
128
|
+
}
|
|
129
|
+
return result;
|
|
130
|
+
},
|
|
131
|
+
{ genericClusterEvents: {}, specificClusterEvents: {} }
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
#addToProcessingMap(handlerCluster) {
|
|
136
|
+
for (const clusterKey in handlerCluster) {
|
|
137
|
+
const { payload, queueEntries } = handlerCluster[clusterKey];
|
|
138
|
+
for (const queueEntry of queueEntries) {
|
|
139
|
+
this.addEntryToProcessingMap(clusterKey, queueEntry, payload);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// NOTE: Currently not exposed to CAP service; I don't see any valid use case at this time
|
|
145
|
+
modifyQueueEntry(queueEntry) {
|
|
146
|
+
super.modifyQueueEntry(queueEntry);
|
|
147
|
+
const hasSpecificClusterHandler = this.#hasEventSpecificClusterHandler(queueEntry);
|
|
148
|
+
if (this.__specificClusterHandler && hasSpecificClusterHandler) {
|
|
149
|
+
this.__specificClusterRelevantAndAvailable = true;
|
|
150
|
+
}
|
|
151
|
+
if (this.__genericClusterHandler && !hasSpecificClusterHandler) {
|
|
152
|
+
this.__genericClusterRelevantAndAvailable = true;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
#hasEventSpecificClusterHandler(queueEntry) {
|
|
157
|
+
return !!this.__onHandlers[["clusterQueueEntries", queueEntry.payload.event].join(".")];
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
async checkEventAndGeneratePayload(queueEntry) {
|
|
161
|
+
const payload = await super.checkEventAndGeneratePayload(queueEntry);
|
|
162
|
+
const { event } = payload;
|
|
163
|
+
const handlerName = this.#checkHandlerExists("checkEventAndGeneratePayload", event);
|
|
164
|
+
if (!handlerName) {
|
|
165
|
+
return payload;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const { msg, userId } = this.#buildDispatchData(this.context, payload, {
|
|
169
|
+
queueEntries: [queueEntry],
|
|
170
|
+
});
|
|
171
|
+
msg.event = handlerName;
|
|
172
|
+
await this.#setContextUser(this.context, userId);
|
|
173
|
+
const data = await this.__srvUnboxed.tx(this.context).send(msg);
|
|
174
|
+
if (data) {
|
|
175
|
+
payload.data = data;
|
|
176
|
+
return payload;
|
|
177
|
+
} else {
|
|
178
|
+
return null;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// simple here as per entry
|
|
183
|
+
async hookForExceededEvents(exceededEvent) {
|
|
184
|
+
return await super.hookForExceededEvents(exceededEvent);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
async beforeProcessingEvents() {
|
|
188
|
+
return await super.beforeProcessingEvents();
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// maybe async getter on req.data // only for periodic events
|
|
192
|
+
// getLastSuccessfulRunTimestamp
|
|
193
|
+
|
|
194
|
+
#checkHandlerExists(eventQueueFn, event) {
|
|
195
|
+
const specificHandler = this.__onHandlers[[eventQueueFn, event].join(".")];
|
|
196
|
+
if (specificHandler) {
|
|
197
|
+
return specificHandler;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
const genericHandler = this.__onHandlers[eventQueueFn];
|
|
201
|
+
return genericHandler ?? null;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
async processPeriodicEvent(processContext, key, queueEntry) {
|
|
205
|
+
const [, action] = this.eventSubType.split(".");
|
|
206
|
+
const msg = new cds.Event({ event: action, eventQueue: { processor: this, key, queueEntries: [queueEntry] } });
|
|
207
|
+
await this.#setContextUser(processContext, config.userId);
|
|
208
|
+
await this.__srvUnboxed.tx(processContext).emit(msg);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
#buildDispatchData(context, payload, { key, queueEntries } = {}) {
|
|
212
|
+
const { useEventQueueUser } = this.eventConfig;
|
|
213
|
+
const userId = useEventQueueUser ? config.userId : payload.contextUser;
|
|
214
|
+
const msg = payload._fromSend ? new cds.Request(payload) : new cds.Event(payload);
|
|
215
|
+
const invocationFn = payload._fromSend ? "send" : "emit";
|
|
216
|
+
delete msg._fromSend; // TODO: this changes the source object --> check after multiple invocations
|
|
217
|
+
delete msg.contextUser;
|
|
218
|
+
msg.eventQueue = { processor: this, key, queueEntries, payload };
|
|
219
|
+
return { msg, userId, invocationFn };
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
async #setContextUser(context, userId) {
|
|
223
|
+
context.user = new cds.User.Privileged({
|
|
224
|
+
id: userId,
|
|
225
|
+
authInfo: await common.getTokenInfo(this.baseContext.tenant),
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
|
|
18
229
|
async processEvent(processContext, key, queueEntries, payload) {
|
|
19
230
|
try {
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
const
|
|
23
|
-
const msg = payload._fromSend ? new cds.Request(payload) : new cds.Event(payload);
|
|
24
|
-
const invocationFn = payload._fromSend ? "send" : "emit";
|
|
25
|
-
delete msg._fromSend;
|
|
26
|
-
delete msg.contextUser;
|
|
27
|
-
processContext.user = new cds.User.Privileged({
|
|
28
|
-
id: userId,
|
|
29
|
-
authInfo: await common.getTokenInfo(processContext.tenant),
|
|
30
|
-
});
|
|
31
|
-
processContext._eventQueue = { processor: this, key, queueEntries, payload };
|
|
32
|
-
const result = await cds.unboxed(service).tx(processContext)[invocationFn](msg);
|
|
231
|
+
const { userId, invocationFn, msg } = this.#buildDispatchData(processContext, payload, { key, queueEntries });
|
|
232
|
+
await this.#setContextUser(processContext, userId);
|
|
233
|
+
const result = await this.__srvUnboxed.tx(processContext)[invocationFn](msg);
|
|
33
234
|
return this.#determineResultStatus(result, queueEntries);
|
|
34
235
|
} catch (err) {
|
|
35
236
|
this.logger.error("error processing outboxed service call", err, {
|
|
@@ -20,16 +20,14 @@ function outboxed(srv, customOpts) {
|
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
const logger = cds.log(COMPONENT_NAME);
|
|
23
|
-
|
|
23
|
+
let outboxOpts = Object.assign(
|
|
24
24
|
{},
|
|
25
25
|
(typeof cds.requires.outbox === "object" && cds.requires.outbox) || {},
|
|
26
26
|
(typeof srv.options?.outbox === "object" && srv.options.outbox) || {},
|
|
27
27
|
customOpts || {}
|
|
28
28
|
);
|
|
29
29
|
|
|
30
|
-
|
|
31
|
-
config.addCAPOutboxEvent(srv.name, outboxOpts);
|
|
32
|
-
}
|
|
30
|
+
config.addCAPOutboxEventBase(srv.name, outboxOpts);
|
|
33
31
|
|
|
34
32
|
const originalSrv = srv[UNBOXED] || srv;
|
|
35
33
|
const outboxedSrv = Object.create(originalSrv);
|
|
@@ -42,9 +40,20 @@ function outboxed(srv, customOpts) {
|
|
|
42
40
|
}
|
|
43
41
|
outboxedSrv.handle = async function (req) {
|
|
44
42
|
const context = req.context || cds.context;
|
|
43
|
+
outboxOpts = Object.assign(
|
|
44
|
+
{},
|
|
45
|
+
(typeof cds.requires.outbox === "object" && cds.requires.outbox) || {},
|
|
46
|
+
(typeof srv.options?.outbox === "object" && srv.options.outbox) || {},
|
|
47
|
+
customOpts || {}
|
|
48
|
+
);
|
|
49
|
+
config.addCAPOutboxEventBase(srv.name, outboxOpts);
|
|
50
|
+
const specificSettings = config.getCdsOutboxEventSpecificConfig(srv.name, req.method);
|
|
51
|
+
if (specificSettings) {
|
|
52
|
+
outboxOpts = config.addCAPOutboxEventSpecificAction(srv.name, req.event);
|
|
53
|
+
}
|
|
54
|
+
|
|
45
55
|
if (outboxOpts.kind === "persistent-outbox") {
|
|
46
|
-
|
|
47
|
-
await _mapToEventAndPublish(context, srv.name, req);
|
|
56
|
+
await _mapToEventAndPublish(context, srv.name, req, !!specificSettings);
|
|
48
57
|
return;
|
|
49
58
|
}
|
|
50
59
|
context.on("succeeded", async () => {
|
|
@@ -70,7 +79,7 @@ function unboxed(srv) {
|
|
|
70
79
|
return srv[UNBOXED] || srv;
|
|
71
80
|
}
|
|
72
81
|
|
|
73
|
-
const _mapToEventAndPublish = async (context, name, req) => {
|
|
82
|
+
const _mapToEventAndPublish = async (context, name, req, actionSpecific) => {
|
|
74
83
|
const eventQueueSpecificValues = {};
|
|
75
84
|
for (const header in req.headers ?? {}) {
|
|
76
85
|
for (const field of EVENT_QUEUE_SPECIFIC_FIELDS) {
|
|
@@ -93,7 +102,7 @@ const _mapToEventAndPublish = async (context, name, req) => {
|
|
|
93
102
|
|
|
94
103
|
await publishEvent(cds.tx(context), {
|
|
95
104
|
type: CDS_EVENT_TYPE,
|
|
96
|
-
subType: name,
|
|
105
|
+
subType: actionSpecific ? [name, req.event].join(".") : name,
|
|
97
106
|
payload: JSON.stringify(event),
|
|
98
107
|
...eventQueueSpecificValues,
|
|
99
108
|
});
|
package/src/processEventQueue.js
CHANGED
|
@@ -79,7 +79,7 @@ const processEventQueue = async (context, eventType, eventSubType) => {
|
|
|
79
79
|
await executeInNewTransaction(context, `eventQueue-processing-${eventType}##${eventSubType}`, async (tx) => {
|
|
80
80
|
eventTypeInstance.processEventContext = tx.context;
|
|
81
81
|
try {
|
|
82
|
-
eventTypeInstance.clusterQueueEntries(eventTypeInstance.queueEntriesWithPayloadMap);
|
|
82
|
+
await eventTypeInstance.clusterQueueEntries(eventTypeInstance.queueEntriesWithPayloadMap);
|
|
83
83
|
await processEventMap(eventTypeInstance);
|
|
84
84
|
} catch (err) {
|
|
85
85
|
eventTypeInstance.handleErrorDuringClustering(err);
|
package/src/publishEvent.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
+
const { CronExpressionParser } = require("cron-parser");
|
|
4
|
+
|
|
3
5
|
const config = require("./config");
|
|
4
6
|
const common = require("./shared/common");
|
|
5
7
|
const EventQueueError = require("./EventQueueError");
|
|
@@ -60,6 +62,10 @@ const publishEvent = async (
|
|
|
60
62
|
if (addTraceContext) {
|
|
61
63
|
event.context = JSON.stringify({ traceContext: openTelemetry.getCurrentTraceContext() });
|
|
62
64
|
}
|
|
65
|
+
|
|
66
|
+
if (eventConfig.timeBucket) {
|
|
67
|
+
event.startAfter ??= CronExpressionParser.parse(eventConfig.timeBucket).next().toISOString();
|
|
68
|
+
}
|
|
63
69
|
}
|
|
64
70
|
if (config.insertEventsBeforeCommit && !skipInsertEventsBeforeCommit) {
|
|
65
71
|
_registerHandlerAndAddEvents(tx, events);
|
package/src/runner/openEvents.js
CHANGED
|
@@ -4,6 +4,7 @@ const cds = require("@sap/cds");
|
|
|
4
4
|
|
|
5
5
|
const eventConfig = require("../config");
|
|
6
6
|
const { EventProcessingStatus } = require("../constants");
|
|
7
|
+
const config = require("../config");
|
|
7
8
|
|
|
8
9
|
const MS_IN_DAYS = 24 * 60 * 60 * 1000;
|
|
9
10
|
|
|
@@ -36,8 +37,9 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
|
|
|
36
37
|
const result = [];
|
|
37
38
|
for (const { type, subType } of entries) {
|
|
38
39
|
if (eventConfig.isCapOutboxEvent(type)) {
|
|
40
|
+
const [srvName, actionName] = subType.split(".");
|
|
39
41
|
cds.connect
|
|
40
|
-
.to(
|
|
42
|
+
.to(srvName)
|
|
41
43
|
.then((service) => {
|
|
42
44
|
if (!filterAppSpecificEvents) {
|
|
43
45
|
return; // will be done in finally
|
|
@@ -47,6 +49,9 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
|
|
|
47
49
|
return;
|
|
48
50
|
}
|
|
49
51
|
cds.outboxed(service);
|
|
52
|
+
if (actionName) {
|
|
53
|
+
config.addCAPOutboxEventSpecificAction(srvName, actionName);
|
|
54
|
+
}
|
|
50
55
|
if (filterAppSpecificEvents) {
|
|
51
56
|
if (eventConfig.shouldBeProcessedInThisApplication(type, subType)) {
|
|
52
57
|
result.push({ type, subType });
|