@cap-js-community/event-queue 1.6.7 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cds-plugin.js +13 -1
- package/package.json +7 -5
- package/src/EventQueueError.js +60 -0
- package/src/EventQueueProcessorBase.js +26 -4
- package/src/config.js +48 -1
- package/src/index.d.ts +11 -0
- package/src/index.js +2 -1
- package/src/initialize.js +29 -36
- package/src/periodicEvents.js +83 -32
- package/src/redis/redisPub.js +32 -2
package/cds-plugin.js
CHANGED
|
@@ -1,10 +1,22 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
const cds = require("@sap/cds");
|
|
4
|
+
const cdsPackage = require("@sap/cds/package.json");
|
|
4
5
|
|
|
5
6
|
const eventQueue = require("./src");
|
|
6
7
|
const COMPONENT_NAME = "/eventQueue/plugin";
|
|
8
|
+
const SERVE_COMMAND = "serve";
|
|
7
9
|
|
|
8
|
-
|
|
10
|
+
const isServe = cds.cli?.command === SERVE_COMMAND;
|
|
11
|
+
const isBuild = cds.build?.register;
|
|
12
|
+
// NOTE: for sap/cds < 8.2.3 there was no consistent way to detect cds is running as a server, not for build, compile,
|
|
13
|
+
// etc...
|
|
14
|
+
const doLegacyBuildDetection =
|
|
15
|
+
cdsPackage.version.localeCompare("8.2.3", undefined, { numeric: true, sensitivity: "base" }) < 0;
|
|
16
|
+
if ((doLegacyBuildDetection && isBuild) || (!doLegacyBuildDetection && !isServe)) {
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
if (Object.keys(cds.env.eventQueue ?? {}).length) {
|
|
9
21
|
module.exports = eventQueue.initialize().catch((err) => cds.log(COMPONENT_NAME).error(err));
|
|
10
22
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cap-js-community/event-queue",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.7.1",
|
|
4
4
|
"description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
@@ -44,15 +44,16 @@
|
|
|
44
44
|
},
|
|
45
45
|
"dependencies": {
|
|
46
46
|
"@sap/xssec": "^4.2.4",
|
|
47
|
+
"cron-parser": "^4.9.0",
|
|
47
48
|
"redis": "^4.7.0",
|
|
48
49
|
"verror": "^1.10.1",
|
|
49
50
|
"yaml": "^2.5.1"
|
|
50
51
|
},
|
|
51
52
|
"devDependencies": {
|
|
52
|
-
"@sap/cds": "^8.3.0",
|
|
53
|
-
"@sap/cds-dk": "^8.3.0",
|
|
54
53
|
"@cap-js/hana": "^1.3.0",
|
|
55
54
|
"@cap-js/sqlite": "^1.7.3",
|
|
55
|
+
"@sap/cds": "^8.3.0",
|
|
56
|
+
"@sap/cds-dk": "^8.3.0",
|
|
56
57
|
"eslint": "^8.57.0",
|
|
57
58
|
"eslint-config-prettier": "^9.1.0",
|
|
58
59
|
"eslint-plugin-jest": "^28.6.0",
|
|
@@ -74,9 +75,10 @@
|
|
|
74
75
|
"disableRedis": false
|
|
75
76
|
},
|
|
76
77
|
"[test]": {
|
|
77
|
-
"registerAsEventProcessor": false,
|
|
78
78
|
"isRunnerDeactivated": true,
|
|
79
|
-
"
|
|
79
|
+
"registerAsEventProcessor": false,
|
|
80
|
+
"updatePeriodicEvents": false,
|
|
81
|
+
"insertEventsBeforeCommit": false
|
|
80
82
|
}
|
|
81
83
|
},
|
|
82
84
|
"requires": {
|
package/src/EventQueueError.js
CHANGED
|
@@ -13,6 +13,10 @@ const ERROR_CODES = {
|
|
|
13
13
|
TYPE_MISMATCH_TABLE: "TYPE_MISMATCH_TABLE",
|
|
14
14
|
NO_VALID_DATE: "NO_VALID_DATE",
|
|
15
15
|
INVALID_INTERVAL: "INVALID_INTERVAL",
|
|
16
|
+
CANT_PARSE_CRON: "CANT_PARSE_CRON",
|
|
17
|
+
CRON_INTERVAL: "CRON_INTERVAL",
|
|
18
|
+
NO_INTERVAL_OR_CRON: "NO_INTERVAL_OR_CRON",
|
|
19
|
+
INTERVAL_AND_CRON: "INTERVAL_AND_CRON",
|
|
16
20
|
MISSING_IMPL: "MISSING_IMPL",
|
|
17
21
|
DUPLICATE_EVENT_REGISTRATION: "DUPLICATE_EVENT_REGISTRATION",
|
|
18
22
|
NO_MANUEL_INSERT_OF_PERIODIC: "NO_MANUEL_INSERT_OF_PERIODIC",
|
|
@@ -75,6 +79,18 @@ const ERROR_CODES_META = {
|
|
|
75
79
|
[ERROR_CODES.APP_INSTANCES_FORMAT]: {
|
|
76
80
|
message: "The app instances property must be an array and only contain numbers.",
|
|
77
81
|
},
|
|
82
|
+
[ERROR_CODES.CANT_PARSE_CRON]: {
|
|
83
|
+
message: "The cron expression is syntactically not correct and can't be parsed!",
|
|
84
|
+
},
|
|
85
|
+
[ERROR_CODES.CRON_INTERVAL]: {
|
|
86
|
+
message: "The difference between two cron execution must be greater than 10 seconds.",
|
|
87
|
+
},
|
|
88
|
+
[ERROR_CODES.NO_INTERVAL_OR_CRON]: {
|
|
89
|
+
message: "For periodic events either the cron or interval parameter must be defined!",
|
|
90
|
+
},
|
|
91
|
+
[ERROR_CODES.INTERVAL_AND_CRON]: {
|
|
92
|
+
message: "For periodic events only the cron or interval parameter can be defined!",
|
|
93
|
+
},
|
|
78
94
|
};
|
|
79
95
|
|
|
80
96
|
class EventQueueError extends VError {
|
|
@@ -190,6 +206,50 @@ class EventQueueError extends VError {
|
|
|
190
206
|
);
|
|
191
207
|
}
|
|
192
208
|
|
|
209
|
+
static cantParseCronExpression(type, subType, expression) {
|
|
210
|
+
const { message } = ERROR_CODES_META[ERROR_CODES.CANT_PARSE_CRON];
|
|
211
|
+
return new EventQueueError(
|
|
212
|
+
{
|
|
213
|
+
name: ERROR_CODES.CANT_PARSE_CRON,
|
|
214
|
+
info: { type, subType, expression },
|
|
215
|
+
},
|
|
216
|
+
message
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
static invalidIntervalBetweenCron(type, subType, interval) {
|
|
221
|
+
const { message } = ERROR_CODES_META[ERROR_CODES.CRON_INTERVAL];
|
|
222
|
+
return new EventQueueError(
|
|
223
|
+
{
|
|
224
|
+
name: ERROR_CODES.CRON_INTERVAL,
|
|
225
|
+
info: { type, subType, interval },
|
|
226
|
+
},
|
|
227
|
+
message
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
static noCronOrInterval(type, subType) {
|
|
232
|
+
const { message } = ERROR_CODES_META[ERROR_CODES.NO_INTERVAL_OR_CRON];
|
|
233
|
+
return new EventQueueError(
|
|
234
|
+
{
|
|
235
|
+
name: ERROR_CODES.CRON_INTERVAL,
|
|
236
|
+
info: { type, subType },
|
|
237
|
+
},
|
|
238
|
+
message
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
static cronAndInterval(type, subType) {
|
|
243
|
+
const { message } = ERROR_CODES_META[ERROR_CODES.INTERVAL_AND_CRON];
|
|
244
|
+
return new EventQueueError(
|
|
245
|
+
{
|
|
246
|
+
name: ERROR_CODES.INTERVAL_AND_CRON,
|
|
247
|
+
info: { type, subType },
|
|
248
|
+
},
|
|
249
|
+
message
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
|
|
193
253
|
static missingImpl(type, subType) {
|
|
194
254
|
const { message } = ERROR_CODES_META[ERROR_CODES.MISSING_IMPL];
|
|
195
255
|
return new EventQueueError(
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
const cds = require("@sap/cds");
|
|
4
|
+
const cronParser = require("cron-parser");
|
|
4
5
|
|
|
5
6
|
const { executeInNewTransaction, TriggerRollback } = require("./shared/cdsHelper");
|
|
6
7
|
const { EventProcessingStatus, TransactionMode } = require("./constants");
|
|
@@ -944,12 +945,27 @@ class EventQueueProcessorBase {
|
|
|
944
945
|
}
|
|
945
946
|
}
|
|
946
947
|
|
|
948
|
+
#calculateCronDates() {
|
|
949
|
+
if (!this.#eventConfig.cron) {
|
|
950
|
+
return null;
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
// NOTE: do not pass current date as we always want to calc. a future date
|
|
954
|
+
const cronExpression = cronParser.parseExpression(this.#eventConfig.cron, {
|
|
955
|
+
utc: this.#eventConfig.utc,
|
|
956
|
+
...(this.#eventConfig.useCronTimezone && { tz: this.#config.cronTimezone }),
|
|
957
|
+
});
|
|
958
|
+
return cronExpression.next();
|
|
959
|
+
}
|
|
960
|
+
|
|
947
961
|
async scheduleNextPeriodEvent(queueEntry) {
|
|
948
|
-
const intervalInMs = this.#eventConfig.interval * 1000;
|
|
962
|
+
const intervalInMs = this.#eventConfig.cron ? null : this.#eventConfig.interval * 1000;
|
|
963
|
+
const next = this.#calculateCronDates();
|
|
964
|
+
|
|
949
965
|
const newEvent = {
|
|
950
966
|
type: this.#eventType,
|
|
951
967
|
subType: this.#eventSubType,
|
|
952
|
-
startAfter: new Date(new Date(queueEntry.startAfter).getTime() + intervalInMs),
|
|
968
|
+
startAfter: next ?? new Date(new Date(queueEntry.startAfter).getTime() + intervalInMs),
|
|
953
969
|
};
|
|
954
970
|
const { relative } = this.#eventSchedulerInstance.calculateOffset(
|
|
955
971
|
this.#eventType,
|
|
@@ -958,6 +974,7 @@ class EventQueueProcessorBase {
|
|
|
958
974
|
);
|
|
959
975
|
|
|
960
976
|
// more than one interval behind - shift tick to keep up
|
|
977
|
+
// cron package always calc the next future date --> not needed for crone
|
|
961
978
|
if (relative < 0 && Math.abs(relative) >= intervalInMs) {
|
|
962
979
|
const plannedStartAfter = newEvent.startAfter;
|
|
963
980
|
newEvent.startAfter = new Date(Date.now() + 5 * 1000);
|
|
@@ -970,7 +987,12 @@ class EventQueueProcessorBase {
|
|
|
970
987
|
}
|
|
971
988
|
|
|
972
989
|
this.tx._skipEventQueueBroadcase = true;
|
|
973
|
-
await this.tx.run(
|
|
990
|
+
await this.tx.run(
|
|
991
|
+
INSERT.into(this.#config.tableNameEventQueue).entries({
|
|
992
|
+
...newEvent,
|
|
993
|
+
startAfter: newEvent.startAfter.toISOString(),
|
|
994
|
+
})
|
|
995
|
+
);
|
|
974
996
|
this.tx._skipEventQueueBroadcase = false;
|
|
975
997
|
if (intervalInMs < this.#config.runInterval * 1.5) {
|
|
976
998
|
this.#handleDelayedEvents([newEvent]);
|
|
@@ -979,7 +1001,7 @@ class EventQueueProcessorBase {
|
|
|
979
1001
|
this.#eventSubType,
|
|
980
1002
|
newEvent.startAfter
|
|
981
1003
|
);
|
|
982
|
-
// next tick is already behind schedule --> execute direct
|
|
1004
|
+
// NOTE: can only happen for interval events: next tick is already behind schedule --> execute direct
|
|
983
1005
|
if (relativeAfterSchedule <= 0) {
|
|
984
1006
|
this.logger.info("running behind schedule - executing next tick immediately", {
|
|
985
1007
|
eventType: this.#eventType,
|
package/src/config.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
const cds = require("@sap/cds");
|
|
4
|
+
const cronParser = require("cron-parser");
|
|
4
5
|
|
|
5
6
|
const { getEnvInstance } = require("./shared/env");
|
|
6
7
|
const redis = require("./shared/redis");
|
|
@@ -23,6 +24,8 @@ const CAP_EVENT_TYPE = "CAP_OUTBOX";
|
|
|
23
24
|
const CAP_PARALLEL_DEFAULT = 5;
|
|
24
25
|
const DELETE_TENANT_BLOCK_AFTER_MS = 5 * 60 * 1000;
|
|
25
26
|
const PRIORITIES = Object.values(Priorities);
|
|
27
|
+
const UTC_DEFAULT = false;
|
|
28
|
+
const USE_CRON_TZ_DEFAULT = true;
|
|
26
29
|
|
|
27
30
|
const BASE_PERIODIC_EVENTS = [
|
|
28
31
|
{
|
|
@@ -72,6 +75,8 @@ class Config {
|
|
|
72
75
|
#enableCAPTelemetry;
|
|
73
76
|
#unsubscribeHandlers = [];
|
|
74
77
|
#unsubscribedTenants = {};
|
|
78
|
+
#cronTimezone;
|
|
79
|
+
#publishEventBlockList;
|
|
75
80
|
static #instance;
|
|
76
81
|
constructor() {
|
|
77
82
|
this.#logger = cds.log(COMPONENT_NAME);
|
|
@@ -236,7 +241,7 @@ class Config {
|
|
|
236
241
|
}
|
|
237
242
|
const key = this.generateKey(typeWithSuffix, subType);
|
|
238
243
|
this.#blockEventLocalState(key, tenant);
|
|
239
|
-
if (!this.redisEnabled) {
|
|
244
|
+
if (!this.redisEnabled || !this.publishEventBlockList) {
|
|
240
245
|
return;
|
|
241
246
|
}
|
|
242
247
|
|
|
@@ -409,6 +414,32 @@ class Config {
|
|
|
409
414
|
throw EventQueueError.duplicateEventRegistration(event.type, event.subType);
|
|
410
415
|
}
|
|
411
416
|
|
|
417
|
+
if (!event.cron && !event.interval) {
|
|
418
|
+
throw EventQueueError.noCronOrInterval(event.type, event.subType);
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
if (event.cron && event.interval) {
|
|
422
|
+
throw EventQueueError.cronAndInterval(event.type, event.subType);
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
if (event.cron) {
|
|
426
|
+
let cron;
|
|
427
|
+
event.utc = event.utc ?? UTC_DEFAULT;
|
|
428
|
+
event.useCronTimezone = event.useCronTimezone ?? USE_CRON_TZ_DEFAULT;
|
|
429
|
+
try {
|
|
430
|
+
cron = cronParser.parseExpression(event.cron);
|
|
431
|
+
} catch {
|
|
432
|
+
throw EventQueueError.cantParseCronExpression(event.type, event.subType, event.cron);
|
|
433
|
+
}
|
|
434
|
+
const next = cron.next();
|
|
435
|
+
const afterNext = cron.next();
|
|
436
|
+
const diffInSeconds = (afterNext.getTime() - next.getTime()) / 1000;
|
|
437
|
+
if (diffInSeconds <= MIN_INTERVAL_SEC) {
|
|
438
|
+
throw EventQueueError.invalidIntervalBetweenCron(event.type, event.subType, diffInSeconds);
|
|
439
|
+
}
|
|
440
|
+
return this.#basicEventValidation(event);
|
|
441
|
+
}
|
|
442
|
+
|
|
412
443
|
if (!event.interval || event.interval <= MIN_INTERVAL_SEC) {
|
|
413
444
|
throw EventQueueError.invalidInterval(event.type, event.subType, event.interval);
|
|
414
445
|
}
|
|
@@ -471,6 +502,14 @@ class Config {
|
|
|
471
502
|
this.#forUpdateTimeout = value;
|
|
472
503
|
}
|
|
473
504
|
|
|
505
|
+
get publishEventBlockList() {
|
|
506
|
+
return this.#publishEventBlockList;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
set publishEventBlockList(value) {
|
|
510
|
+
this.#publishEventBlockList = value;
|
|
511
|
+
}
|
|
512
|
+
|
|
474
513
|
set globalTxTimeout(value) {
|
|
475
514
|
this.#globalTxTimeout = value;
|
|
476
515
|
}
|
|
@@ -502,6 +541,14 @@ class Config {
|
|
|
502
541
|
this.#initialized = value;
|
|
503
542
|
}
|
|
504
543
|
|
|
544
|
+
get cronTimezone() {
|
|
545
|
+
return this.#cronTimezone;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
set cronTimezone(value) {
|
|
549
|
+
this.#cronTimezone = value;
|
|
550
|
+
}
|
|
551
|
+
|
|
505
552
|
get instanceLoadLimit() {
|
|
506
553
|
return this.#instanceLoadLimit;
|
|
507
554
|
}
|
package/src/index.d.ts
CHANGED
|
@@ -106,6 +106,11 @@ interface EventEntityPublish {
|
|
|
106
106
|
payload?: string;
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
+
interface EventTriggerProcessing {
|
|
110
|
+
type: string;
|
|
111
|
+
subType: string;
|
|
112
|
+
}
|
|
113
|
+
|
|
109
114
|
interface QueueEntriesPayloadMap {
|
|
110
115
|
[key: string]: {
|
|
111
116
|
queueEntry: EventEntity;
|
|
@@ -163,6 +168,12 @@ export function processEventQueue(
|
|
|
163
168
|
startTime: Date
|
|
164
169
|
): Promise<any>;
|
|
165
170
|
|
|
171
|
+
export function triggerEventProcessingRedis(
|
|
172
|
+
tenantId: string,
|
|
173
|
+
events: EventTriggerProcessing[],
|
|
174
|
+
forceBroadcast?: boolean
|
|
175
|
+
): Promise<any>;
|
|
176
|
+
|
|
166
177
|
declare class Config {
|
|
167
178
|
constructor();
|
|
168
179
|
|
package/src/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
const redisPubSub = require("./redis/redisPub");
|
|
4
4
|
|
|
5
5
|
module.exports = {
|
|
6
6
|
...require("./initialize"),
|
|
@@ -11,4 +11,5 @@ module.exports = {
|
|
|
11
11
|
...require("./publishEvent"),
|
|
12
12
|
EventQueueProcessorBase: require("./EventQueueProcessorBase"),
|
|
13
13
|
WorkerQueue: require("./shared/WorkerQueue"),
|
|
14
|
+
triggerEventProcessingRedis: redisPubSub.broadcastEvent,
|
|
14
15
|
};
|
package/src/initialize.js
CHANGED
|
@@ -35,47 +35,40 @@ const CONFIG_VARS = [
|
|
|
35
35
|
["userId", null],
|
|
36
36
|
["cleanupLocksAndEventsForDev", false],
|
|
37
37
|
["redisOptions", {}],
|
|
38
|
-
["insertEventsBeforeCommit",
|
|
38
|
+
["insertEventsBeforeCommit", true],
|
|
39
39
|
["enableCAPTelemetry", false],
|
|
40
|
+
["cronTimezone", null],
|
|
41
|
+
["publishEventBlockList", true],
|
|
40
42
|
];
|
|
41
43
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
} =
|
|
44
|
+
/**
|
|
45
|
+
* Initializes the event queue with the provided options.
|
|
46
|
+
*
|
|
47
|
+
* @param {Object} options - The configuration options.
|
|
48
|
+
* @param {string} [options.configFilePath=null] - Path to the configuration file.
|
|
49
|
+
* @param {boolean} [options.registerAsEventProcessor=true] - Register the instance as an event processor.
|
|
50
|
+
* @param {boolean} [options.processEventsAfterPublish=true] - Process events immediately after publishing.
|
|
51
|
+
* @param {boolean} [options.isEventQueueActive=true] - Flag to activate/deactivate the event queue.
|
|
52
|
+
* @param {number} [options.runInterval=1500000] - Interval for running event queue processing (in milliseconds).
|
|
53
|
+
* @param {boolean} [options.disableRedis=true] - Disable Redis usage for event handling.
|
|
54
|
+
* @param {boolean} [options.updatePeriodicEvents=true] - Automatically update periodic events.
|
|
55
|
+
* @param {number} [options.thresholdLoggingEventProcessing=50] - Threshold for logging event processing time (in milliseconds).
|
|
56
|
+
* @param {boolean} [options.useAsCAPOutbox=false] - Use the event queue as a CAP Outbox.
|
|
57
|
+
* @param {string} [options.userId=null] - ID of the user initiating the process.
|
|
58
|
+
* @param {boolean} [options.cleanupLocksAndEventsForDev=false] - Cleanup locks and events for development environments.
|
|
59
|
+
* @param {Object} [options.redisOptions={}] - Configuration options for Redis.
|
|
60
|
+
* @param {boolean} [options.insertEventsBeforeCommit=true] - Insert events into the queue before committing the transaction.
|
|
61
|
+
* @param {boolean} [options.enableCAPTelemetry=false] - Enable telemetry for CAP.
|
|
62
|
+
* @param {string} [options.cronTimezone=null] - Default timezone for cron jobs.
|
|
63
|
+
* @param {string} [options.publishEventBlockList=true] - If redis is available event blocklist is distributed to all application instances
|
|
64
|
+
*/
|
|
65
|
+
const initialize = async (options = {}) => {
|
|
58
66
|
if (config.initialized) {
|
|
59
67
|
return;
|
|
60
68
|
}
|
|
61
69
|
config.initialized = true;
|
|
62
70
|
|
|
63
|
-
mixConfigVarsWithEnv(
|
|
64
|
-
configFilePath,
|
|
65
|
-
registerAsEventProcessor,
|
|
66
|
-
processEventsAfterPublish,
|
|
67
|
-
isEventQueueActive,
|
|
68
|
-
runInterval,
|
|
69
|
-
disableRedis,
|
|
70
|
-
updatePeriodicEvents,
|
|
71
|
-
thresholdLoggingEventProcessing,
|
|
72
|
-
useAsCAPOutbox,
|
|
73
|
-
userId,
|
|
74
|
-
cleanupLocksAndEventsForDev,
|
|
75
|
-
redisOptions,
|
|
76
|
-
insertEventsBeforeCommit,
|
|
77
|
-
enableCAPTelemetry
|
|
78
|
-
);
|
|
71
|
+
mixConfigVarsWithEnv(options);
|
|
79
72
|
|
|
80
73
|
const logger = cds.log(COMPONENT);
|
|
81
74
|
const redisEnabled = config.checkRedisEnabled();
|
|
@@ -167,9 +160,9 @@ const monkeyPatchCAPOutbox = () => {
|
|
|
167
160
|
}
|
|
168
161
|
};
|
|
169
162
|
|
|
170
|
-
const mixConfigVarsWithEnv = (
|
|
171
|
-
CONFIG_VARS.forEach(([configName, defaultValue]
|
|
172
|
-
const configValue =
|
|
163
|
+
const mixConfigVarsWithEnv = (options) => {
|
|
164
|
+
CONFIG_VARS.forEach(([configName, defaultValue]) => {
|
|
165
|
+
const configValue = options[configName];
|
|
173
166
|
config[configName] = configValue ?? cds.env.eventQueue?.[configName] ?? defaultValue;
|
|
174
167
|
});
|
|
175
168
|
};
|
package/src/periodicEvents.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
const cds = require("@sap/cds");
|
|
4
|
+
const cronParser = require("cron-parser");
|
|
4
5
|
|
|
5
6
|
const { EventProcessingStatus } = require("./constants");
|
|
6
7
|
const { processChunkedSync } = require("./shared/common");
|
|
@@ -10,6 +11,7 @@ const COMPONENT_NAME = "/eventQueue/periodicEvents";
|
|
|
10
11
|
const CHUNK_SIZE_INSERT_PERIODIC_EVENTS = 4;
|
|
11
12
|
|
|
12
13
|
const checkAndInsertPeriodicEvents = async (context) => {
|
|
14
|
+
const now = new Date();
|
|
13
15
|
const tx = cds.tx(context);
|
|
14
16
|
const baseCqn = SELECT.from(eventConfig.tableNameEventQueue)
|
|
15
17
|
.where([
|
|
@@ -27,12 +29,15 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
27
29
|
list: [{ val: EventProcessingStatus.Open }, { val: EventProcessingStatus.InProgress }],
|
|
28
30
|
},
|
|
29
31
|
])
|
|
30
|
-
.
|
|
32
|
+
.groupBy("type", "subType", "createdAt")
|
|
33
|
+
.columns(["type", "subType", "createdAt", "max(startAfter) as startAfter"]);
|
|
31
34
|
const currentPeriodEvents = await tx.run(baseCqn);
|
|
35
|
+
currentPeriodEvents.length &&
|
|
36
|
+
(await tx.run(_addWhere(SELECT.from(eventConfig.tableNameEventQueue).columns("ID"), currentPeriodEvents)));
|
|
32
37
|
|
|
33
38
|
if (!currentPeriodEvents.length) {
|
|
34
39
|
// fresh insert all
|
|
35
|
-
return await
|
|
40
|
+
return await _insertPeriodEvents(tx, eventConfig.periodicEvents, now);
|
|
36
41
|
}
|
|
37
42
|
|
|
38
43
|
const exitingEventMap = currentPeriodEvents.reduce((result, current) => {
|
|
@@ -41,26 +46,27 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
41
46
|
return result;
|
|
42
47
|
}, {});
|
|
43
48
|
|
|
44
|
-
const { newEvents,
|
|
49
|
+
const { newEvents, existingEventsCron, existingEventsInterval } = eventConfig.periodicEvents.reduce(
|
|
45
50
|
(result, event) => {
|
|
46
|
-
|
|
47
|
-
|
|
51
|
+
const existingEvent = exitingEventMap[_generateKey(event)];
|
|
52
|
+
if (existingEvent) {
|
|
53
|
+
const config = eventConfig.getEventConfig(existingEvent.type, existingEvent.subType);
|
|
54
|
+
if (config.cron) {
|
|
55
|
+
result.existingEventsCron.push(exitingEventMap[_generateKey(event)]);
|
|
56
|
+
} else {
|
|
57
|
+
result.existingEventsInterval.push(exitingEventMap[_generateKey(event)]);
|
|
58
|
+
}
|
|
48
59
|
} else {
|
|
49
60
|
result.newEvents.push(event);
|
|
50
61
|
}
|
|
51
62
|
return result;
|
|
52
63
|
},
|
|
53
|
-
{ newEvents: [],
|
|
64
|
+
{ newEvents: [], existingEventsCron: [], existingEventsInterval: [] }
|
|
54
65
|
);
|
|
55
66
|
|
|
56
|
-
const
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
const eventStartAfter = new Date(existingEvent.startAfter);
|
|
60
|
-
// check if to far in future
|
|
61
|
-
const dueInWithNewInterval = new Date(currentDate.getTime() + config.interval * 1000);
|
|
62
|
-
return eventStartAfter >= dueInWithNewInterval;
|
|
63
|
-
});
|
|
67
|
+
const exitingWithNotMatchingInterval = []
|
|
68
|
+
.concat(_determineChangedInterval(existingEventsInterval, now))
|
|
69
|
+
.concat(_determineChangedCron(existingEventsCron, now));
|
|
64
70
|
|
|
65
71
|
exitingWithNotMatchingInterval.length &&
|
|
66
72
|
cds.log(COMPONENT_NAME).info("deleting periodic events because they have changed", {
|
|
@@ -68,12 +74,14 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
68
74
|
});
|
|
69
75
|
|
|
70
76
|
if (exitingWithNotMatchingInterval.length) {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
)
|
|
76
|
-
|
|
77
|
+
const cqnBase = DELETE.from(eventConfig.tableNameEventQueue);
|
|
78
|
+
_addWhere(cqnBase, exitingWithNotMatchingInterval);
|
|
79
|
+
const deleteCount = await tx.run(cqnBase);
|
|
80
|
+
if (deleteCount !== exitingWithNotMatchingInterval.length) {
|
|
81
|
+
cds.log(COMPONENT_NAME).warn("deletion count doesn't match expected count", {
|
|
82
|
+
deleteCount,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
77
85
|
}
|
|
78
86
|
|
|
79
87
|
const newOrChangedEvents = newEvents.concat(exitingWithNotMatchingInterval);
|
|
@@ -82,31 +90,74 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
82
90
|
return;
|
|
83
91
|
}
|
|
84
92
|
|
|
85
|
-
return await
|
|
93
|
+
return await _insertPeriodEvents(tx, newOrChangedEvents, now);
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
const _addWhere = (cqnBase, events) => {
|
|
97
|
+
let or = false;
|
|
98
|
+
for (const { type, subType, createdAt, startAfter } of events) {
|
|
99
|
+
cqnBase[or ? "or" : "where"]({ type, subType, createdAt, startAfter });
|
|
100
|
+
or = true;
|
|
101
|
+
}
|
|
102
|
+
return cqnBase;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
const _determineChangedInterval = (existingEvents, currentDate) => {
|
|
106
|
+
return existingEvents.filter((existingEvent) => {
|
|
107
|
+
const config = eventConfig.getEventConfig(existingEvent.type, existingEvent.subType);
|
|
108
|
+
const eventStartAfter = new Date(existingEvent.startAfter);
|
|
109
|
+
// check if too far in future
|
|
110
|
+
const dueInWithNewInterval = new Date(currentDate.getTime() + config.interval * 1000);
|
|
111
|
+
return eventStartAfter >= dueInWithNewInterval;
|
|
112
|
+
});
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
const _determineChangedCron = (existingEventsCron) => {
|
|
116
|
+
return existingEventsCron.filter((event) => {
|
|
117
|
+
const config = eventConfig.getEventConfig(event.type, event.subType);
|
|
118
|
+
const eventStartAfter = new Date(event.startAfter);
|
|
119
|
+
const eventCreatedAt = new Date(event.createdAt);
|
|
120
|
+
const cronExpression = cronParser.parseExpression(config.cron, {
|
|
121
|
+
currentDate: eventCreatedAt,
|
|
122
|
+
utc: config.utc,
|
|
123
|
+
...(config.useCronTimezone && { tz: eventConfig.cronTimezone }),
|
|
124
|
+
});
|
|
125
|
+
return cronExpression.next().getTime() - eventStartAfter.getTime() > 30 * 1000; // report as changed if diff created than 30 seconds
|
|
126
|
+
});
|
|
86
127
|
};
|
|
87
128
|
|
|
88
|
-
const
|
|
89
|
-
const startAfter = new Date();
|
|
129
|
+
const _insertPeriodEvents = async (tx, events, now) => {
|
|
90
130
|
let counter = 1;
|
|
91
131
|
const chunks = Math.ceil(events.length / CHUNK_SIZE_INSERT_PERIODIC_EVENTS);
|
|
92
132
|
const logger = cds.log(COMPONENT_NAME);
|
|
93
|
-
|
|
133
|
+
const eventsToBeInserted = events.map((event) => {
|
|
134
|
+
const base = { type: event.type, subType: event.subType };
|
|
135
|
+
let startTime = now;
|
|
136
|
+
if (event.cron) {
|
|
137
|
+
startTime = cronParser
|
|
138
|
+
.parseExpression(event.cron, {
|
|
139
|
+
currentDate: now,
|
|
140
|
+
utc: event.utc,
|
|
141
|
+
...(event.useCronTimezone && { tz: eventConfig.cronTimezone }),
|
|
142
|
+
})
|
|
143
|
+
.next();
|
|
144
|
+
}
|
|
145
|
+
base.startAfter = startTime.toISOString();
|
|
146
|
+
return base;
|
|
147
|
+
}, []);
|
|
148
|
+
|
|
149
|
+
processChunkedSync(eventsToBeInserted, CHUNK_SIZE_INSERT_PERIODIC_EVENTS, (chunk) => {
|
|
94
150
|
logger.info(`${counter}/${chunks} | inserting chunk of changed or new periodic events`, {
|
|
95
|
-
events: chunk.map(({ type, subType }) => {
|
|
151
|
+
events: chunk.map(({ type, subType, startAfter }) => {
|
|
96
152
|
const { interval } = eventConfig.getEventConfig(type, subType);
|
|
97
|
-
return { type, subType, interval };
|
|
153
|
+
return { type, subType, interval, ...(startAfter && { startAfter }) };
|
|
98
154
|
}),
|
|
99
155
|
});
|
|
100
156
|
counter++;
|
|
101
157
|
});
|
|
102
|
-
const periodEventsInsert = events.map((periodicEvent) => ({
|
|
103
|
-
type: periodicEvent.type,
|
|
104
|
-
subType: periodicEvent.subType,
|
|
105
|
-
startAfter: startAfter,
|
|
106
|
-
}));
|
|
107
158
|
|
|
108
159
|
tx._skipEventQueueBroadcase = true;
|
|
109
|
-
await tx.run(INSERT.into(eventConfig.tableNameEventQueue).entries(
|
|
160
|
+
await tx.run(INSERT.into(eventConfig.tableNameEventQueue).entries(eventsToBeInserted));
|
|
110
161
|
tx._skipEventQueueBroadcase = false;
|
|
111
162
|
};
|
|
112
163
|
|
package/src/redis/redisPub.js
CHANGED
|
@@ -18,7 +18,37 @@ const SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT = 30 * 1000;
|
|
|
18
18
|
|
|
19
19
|
const wait = promisify(setTimeout);
|
|
20
20
|
|
|
21
|
-
|
|
21
|
+
/**
|
|
22
|
+
* Broadcasts events to the event queue, either locally or through Redis.
|
|
23
|
+
*
|
|
24
|
+
* This function checks if the event queue is active before proceeding to broadcast the events.
|
|
25
|
+
* If the event queue is deactivated, broadcasting is skipped. If Redis is not enabled,
|
|
26
|
+
* events will be processed locally without Redis. The function handles periodic events
|
|
27
|
+
* by checking for locks and only publishing when locks are available.
|
|
28
|
+
*
|
|
29
|
+
* @async
|
|
30
|
+
* @param {string} tenantId - The ID of the tenant for which the events are being broadcasted.
|
|
31
|
+
* @param {Array<{ type: string; subType: string }>} events - An array of event objects, each containing
|
|
32
|
+
* a type and a subtype that specify the kind of event to be broadcasted.
|
|
33
|
+
* @param {boolean} [forceBroadcast=false] - If true, forces the broadcast of periodic events even
|
|
34
|
+
* when locks are not available. Defaults to false.
|
|
35
|
+
* @returns {Promise<void>} A promise that resolves when the events have been successfully broadcasted.
|
|
36
|
+
*
|
|
37
|
+
* @throws {Error} Throws an error if publishing events fails.
|
|
38
|
+
*
|
|
39
|
+
* @example
|
|
40
|
+
* // Example usage of broadcastEvent function
|
|
41
|
+
* const tenantId = '12345';
|
|
42
|
+
* const events = [
|
|
43
|
+
* { type: 'orderCreated', subType: 'online' },
|
|
44
|
+
* { type: 'paymentProcessed', subType: 'creditCard' }
|
|
45
|
+
* ];
|
|
46
|
+
*
|
|
47
|
+
* broadcastEvent(tenantId, events)
|
|
48
|
+
* .then(() => console.log('Events broadcasted successfully!'))
|
|
49
|
+
* .catch(err => console.error('Failed to broadcast events:', err));
|
|
50
|
+
*/
|
|
51
|
+
const broadcastEvent = async (tenantId, events, forceBroadcast = false) => {
|
|
22
52
|
const logger = cds.log(COMPONENT_NAME);
|
|
23
53
|
|
|
24
54
|
if (!config.isEventQueueActive) {
|
|
@@ -46,7 +76,7 @@ const broadcastEvent = async (tenantId, events) => {
|
|
|
46
76
|
isPeriodic: eventConfig.isPeriodic,
|
|
47
77
|
waitInterval: SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT,
|
|
48
78
|
});
|
|
49
|
-
if (!eventConfig.isPeriodic) {
|
|
79
|
+
if (!eventConfig.isPeriodic && !forceBroadcast) {
|
|
50
80
|
break;
|
|
51
81
|
}
|
|
52
82
|
await wait(SLEEP_TIME_FOR_PUBLISH_PERIODIC_EVENT);
|