@cap-js-community/event-queue 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -15
- package/package.json +6 -6
- package/src/EventQueueProcessorBase.js +11 -7
- package/src/config.js +7 -7
- package/src/initialize.js +1 -5
- package/src/periodicEvents.js +7 -2
- package/src/processEventQueue.js +1 -1
- package/src/redisPubSub.js +28 -7
- package/src/runner.js +52 -49
- package/src/shared/WorkerQueue.js +4 -4
- package/src/shared/eventScheduler.js +1 -1
package/README.md
CHANGED
|
@@ -5,22 +5,24 @@
|
|
|
5
5
|
[](https://api.reuse.software/info/github.com/cap-js-community/event-queue)
|
|
6
6
|
[](https://github.com/cap-js-community/event-queue/commits/main)
|
|
7
7
|
|
|
8
|
-
The Event-Queue is a framework built on top of CAP Node.js,
|
|
9
|
-
asynchronous event processing
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
8
|
+
The Event-Queue is a framework built on top of CAP Node.js, designed specifically for efficient and
|
|
9
|
+
streamlined asynchronous event processing. With a focus on load balancing, this package ensures optimal
|
|
10
|
+
event distribution across all available application instances. By providing managed transactions similar to CAP
|
|
11
|
+
handlers, the Event-Queue framework simplifies event processing, enhancing the overall performance of your application.
|
|
12
|
+
|
|
13
|
+
Additionally, Event-Queue provides support for [periodic events](https://cap-js-community.github.io/event-queue/configure-event/#periodic-events),
|
|
14
|
+
allowing for processing at defined intervals. This feature further extends its capabilities in load balancing and
|
|
15
|
+
transaction management, ensuring that even regularly occurring tasks are handled efficiently and effectively without
|
|
16
|
+
overloading any single instance. This makes it an ideal solution for applications needing consistent, reliable event processing.
|
|
13
17
|
|
|
14
18
|
## Getting started
|
|
15
19
|
|
|
16
20
|
- Run `npm add @cap-js-community/event-queue` in `@sap/cds` project
|
|
17
|
-
- Activate the cds-plugin in the cds section of the package.
|
|
21
|
+
- Activate the cds-plugin in the cds section of the package.json.
|
|
18
22
|
|
|
19
23
|
### As cds-plugin
|
|
20
24
|
|
|
21
|
-
|
|
22
|
-
cds-plugin concept.
|
|
23
|
-
https://cap.cloud.sap/docs/releases/march23#new-cds-plugin-technique
|
|
25
|
+
For detailed information check out the [documentation](https://cap-js-community.github.io/event-queue/setup).
|
|
24
26
|
|
|
25
27
|
```json
|
|
26
28
|
{
|
|
@@ -35,13 +37,16 @@ https://cap.cloud.sap/docs/releases/march23#new-cds-plugin-technique
|
|
|
35
37
|
|
|
36
38
|
## Features
|
|
37
39
|
|
|
38
|
-
|
|
39
|
-
-
|
|
40
|
-
|
|
40
|
+
Learn more about features in the [documentation](https://cap-js-community.github.io/event-queue/#functionality-and-problem-solutions). To compare the
|
|
41
|
+
event-queue with other SAP products head over to [Distinction from other solutions](https://cap-js-community.github.io/event-queue/diff-to-outbox/).
|
|
42
|
+
|
|
43
|
+
- [load balancing](https://cap-js-community.github.io/event-queue/load-balancing) and concurrency control of event processing across app instances
|
|
44
|
+
- [periodic events](https://cap-js-community.github.io/event-queue/configure-event/#periodic-events) similar to running cron jobs for business processes
|
|
45
|
+
- [managed transactions](https://cap-js-community.github.io/event-queue/transaction-handling) for event processing
|
|
41
46
|
- async processing of processing intensive tasks for better UI responsiveness
|
|
42
47
|
- push/pull mechanism for reducing delay between publish an event and processing
|
|
43
48
|
- cluster published events during processing (e.g. for combining multiple E-Mail events to one E-Mail)
|
|
44
|
-
- plug and play via cds-plugin
|
|
49
|
+
- [plug and play](https://cap-js-community.github.io/event-queue/setup) via cds-plugin
|
|
45
50
|
|
|
46
51
|
## Documentation
|
|
47
52
|
|
|
@@ -50,7 +55,7 @@ Head over to our [Documentation](https://cap-js-community.github.io/event-queue/
|
|
|
50
55
|
## Support, Feedback, Contributing
|
|
51
56
|
|
|
52
57
|
This project is open to feature requests/suggestions, bug reports etc.
|
|
53
|
-
via [GitHub issues](https://github.com/cap-js-
|
|
58
|
+
via [GitHub issues](https://github.com/cap-js-communityevent-queue/issues). Contribution and feedback are encouraged
|
|
54
59
|
and always welcome. For more information about how to contribute, the project structure, as well as additional
|
|
55
60
|
contribution information, see our [Contribution Guidelines](CONTRIBUTING.md).
|
|
56
61
|
|
|
@@ -62,7 +67,7 @@ times.
|
|
|
62
67
|
|
|
63
68
|
## Licensing
|
|
64
69
|
|
|
65
|
-
Copyright 2023 SAP SE or an SAP affiliate company and `@cap-js-community/event-queue contributors
|
|
70
|
+
Copyright 2023 SAP SE or an SAP affiliate company and `@cap-js-community/event-queue` contributors. Please see
|
|
66
71
|
our [LICENSE](LICENSE) for copyright and license information. Detailed information including third-party components and
|
|
67
72
|
their licensing/copyright information is
|
|
68
73
|
available [via the REUSE tool](https://api.reuse.software/info/github.com/cap-js-community/<your-project>).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cap-js-community/event-queue",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "An event queue that enables secure transactional processing of asynchronous events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"files": [
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"lint:ci": "npm run eslint:ci && npm run prettier:ci",
|
|
32
32
|
"eslint": "eslint --fix .",
|
|
33
33
|
"eslint:ci": "eslint .",
|
|
34
|
-
"prettier": "prettier --write --
|
|
34
|
+
"prettier": "prettier --write --loglevel error .",
|
|
35
35
|
"prettier:ci": "prettier --check .",
|
|
36
36
|
"prepareRelease": "npm prune --production",
|
|
37
37
|
"docs": "cd docs && bundle exec jekyll serve",
|
|
@@ -42,14 +42,14 @@
|
|
|
42
42
|
"node": ">=16"
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
|
-
"redis": "4.6.
|
|
45
|
+
"redis": "4.6.11",
|
|
46
46
|
"verror": "1.10.1",
|
|
47
47
|
"yaml": "2.3.4"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@sap/cds": "7.
|
|
51
|
-
"@sap/cds-dk": "7.
|
|
52
|
-
"eslint": "8.
|
|
50
|
+
"@sap/cds": "7.4.0",
|
|
51
|
+
"@sap/cds-dk": "7.4.0",
|
|
52
|
+
"eslint": "8.54.0",
|
|
53
53
|
"eslint-config-prettier": "9.0.0",
|
|
54
54
|
"eslint-plugin-jest": "27.6.0",
|
|
55
55
|
"eslint-plugin-node": "11.1.0",
|
|
@@ -534,7 +534,7 @@ class EventQueueProcessorBase {
|
|
|
534
534
|
*/
|
|
535
535
|
async getQueueEntriesAndSetToInProgress() {
|
|
536
536
|
let result = [];
|
|
537
|
-
const refDateStartAfter = new Date(Date.now() + this.#config.runInterval);
|
|
537
|
+
const refDateStartAfter = new Date(Date.now() + this.#config.runInterval * 1.2);
|
|
538
538
|
await executeInNewTransaction(this.__baseContext, "eventQueue-getQueueEntriesAndSetToInProgress", async (tx) => {
|
|
539
539
|
const entries = await tx.run(
|
|
540
540
|
SELECT.from(this.#config.tableNameEventQueue)
|
|
@@ -586,7 +586,7 @@ class EventQueueProcessorBase {
|
|
|
586
586
|
this.#handleDelayedEvents(delayedEvents);
|
|
587
587
|
|
|
588
588
|
result = openEvents;
|
|
589
|
-
this.logger.info("Selected event queue entries for processing", {
|
|
589
|
+
this.logger[eventsForProcessing.length ? "info" : "debug"]("Selected event queue entries for processing", {
|
|
590
590
|
openEvents: openEvents.length,
|
|
591
591
|
...(delayedEvents.length && { delayedEvents: delayedEvents.length }),
|
|
592
592
|
...(exceededTries.length && { exceededTries: exceededTries.length }),
|
|
@@ -639,7 +639,7 @@ class EventQueueProcessorBase {
|
|
|
639
639
|
}
|
|
640
640
|
|
|
641
641
|
#clusterEvents(events, refDateStartAfter) {
|
|
642
|
-
const refDate = new Date(refDateStartAfter.getTime() - this.#config.runInterval + EVENT_START_AFTER_HEADROOM);
|
|
642
|
+
const refDate = new Date(refDateStartAfter.getTime() - this.#config.runInterval * 1.2 + EVENT_START_AFTER_HEADROOM);
|
|
643
643
|
return events.reduce(
|
|
644
644
|
(result, event) => {
|
|
645
645
|
if (event.attempts === this.__retryAttempts + TRIES_FOR_EXCEEDED_EVENTS) {
|
|
@@ -858,11 +858,11 @@ class EventQueueProcessorBase {
|
|
|
858
858
|
}
|
|
859
859
|
|
|
860
860
|
async scheduleNextPeriodEvent(queueEntry) {
|
|
861
|
-
const
|
|
861
|
+
const intervalInSec = this.#eventConfig.interval * 1000;
|
|
862
862
|
const newEvent = {
|
|
863
863
|
type: this.#eventType,
|
|
864
864
|
subType: this.#eventSubType,
|
|
865
|
-
startAfter: new Date(new Date(queueEntry.startAfter).getTime() +
|
|
865
|
+
startAfter: new Date(new Date(queueEntry.startAfter).getTime() + intervalInSec),
|
|
866
866
|
};
|
|
867
867
|
const { relative } = this.#eventSchedulerInstance.calculateOffset(
|
|
868
868
|
this.#eventType,
|
|
@@ -871,7 +871,7 @@ class EventQueueProcessorBase {
|
|
|
871
871
|
);
|
|
872
872
|
|
|
873
873
|
// more than one interval behind - shift tick to keep up
|
|
874
|
-
if (relative < 0 && Math.abs(relative) >=
|
|
874
|
+
if (relative < 0 && Math.abs(relative) >= intervalInSec) {
|
|
875
875
|
newEvent.startAfter = new Date(Date.now() + 5 * 1000);
|
|
876
876
|
this.logger.info("interval adjusted because shifted more than one interval", {
|
|
877
877
|
eventType: this.#eventType,
|
|
@@ -883,7 +883,7 @@ class EventQueueProcessorBase {
|
|
|
883
883
|
this.tx._skipEventQueueBroadcase = true;
|
|
884
884
|
await this.tx.run(INSERT.into(this.#config.tableNameEventQueue).entries({ ...newEvent }));
|
|
885
885
|
this.tx._skipEventQueueBroadcase = false;
|
|
886
|
-
if (
|
|
886
|
+
if (intervalInSec < this.#config.runInterval * 1.5) {
|
|
887
887
|
this.#handleDelayedEvents([newEvent]);
|
|
888
888
|
const { relative: relativeAfterSchedule } = this.#eventSchedulerInstance.calculateOffset(
|
|
889
889
|
this.#eventType,
|
|
@@ -941,6 +941,10 @@ class EventQueueProcessorBase {
|
|
|
941
941
|
return this.__logger ?? this.__baseLogger;
|
|
942
942
|
}
|
|
943
943
|
|
|
944
|
+
set logger(value) {
|
|
945
|
+
this.__logger = value;
|
|
946
|
+
}
|
|
947
|
+
|
|
944
948
|
get queueEntriesWithPayloadMap() {
|
|
945
949
|
return this.#queueEntriesWithPayloadMap;
|
|
946
950
|
}
|
package/src/config.js
CHANGED
|
@@ -12,6 +12,7 @@ const REDIS_CONFIG_CHANNEL = "EVENT_QUEUE_CONFIG_CHANNEL";
|
|
|
12
12
|
const COMPONENT_NAME = "eventQueue/config";
|
|
13
13
|
const MIN_INTERVAL_SEC = 10;
|
|
14
14
|
const DEFAULT_LOAD = 1;
|
|
15
|
+
const SUFFIX_PERIODIC = "_PERIODIC";
|
|
15
16
|
|
|
16
17
|
class Config {
|
|
17
18
|
#logger;
|
|
@@ -21,7 +22,7 @@ class Config {
|
|
|
21
22
|
#runInterval;
|
|
22
23
|
#redisEnabled;
|
|
23
24
|
#initialized;
|
|
24
|
-
#
|
|
25
|
+
#instanceLoadLimit;
|
|
25
26
|
#tableNameEventQueue;
|
|
26
27
|
#tableNameEventLock;
|
|
27
28
|
#isRunnerDeactivated;
|
|
@@ -42,7 +43,7 @@ class Config {
|
|
|
42
43
|
this.#runInterval = null;
|
|
43
44
|
this.#redisEnabled = null;
|
|
44
45
|
this.#initialized = false;
|
|
45
|
-
this.#
|
|
46
|
+
this.#instanceLoadLimit = 100;
|
|
46
47
|
this.#tableNameEventQueue = null;
|
|
47
48
|
this.#tableNameEventLock = null;
|
|
48
49
|
this.#isRunnerDeactivated = false;
|
|
@@ -115,7 +116,6 @@ class Config {
|
|
|
115
116
|
}, {});
|
|
116
117
|
this.#eventMap = config.periodicEvents.reduce((result, event) => {
|
|
117
118
|
event.load = event.load ?? DEFAULT_LOAD;
|
|
118
|
-
const SUFFIX_PERIODIC = "_PERIODIC";
|
|
119
119
|
event.type = `${event.type}${SUFFIX_PERIODIC}`;
|
|
120
120
|
event.isPeriodic = true;
|
|
121
121
|
this.validatePeriodicConfig(result, event);
|
|
@@ -214,12 +214,12 @@ class Config {
|
|
|
214
214
|
this.#initialized = value;
|
|
215
215
|
}
|
|
216
216
|
|
|
217
|
-
get
|
|
218
|
-
return this.#
|
|
217
|
+
get instanceLoadLimit() {
|
|
218
|
+
return this.#instanceLoadLimit;
|
|
219
219
|
}
|
|
220
220
|
|
|
221
|
-
set
|
|
222
|
-
this.#
|
|
221
|
+
set instanceLoadLimit(value) {
|
|
222
|
+
this.#instanceLoadLimit = value;
|
|
223
223
|
}
|
|
224
224
|
|
|
225
225
|
get tableNameEventQueue() {
|
package/src/initialize.js
CHANGED
|
@@ -29,7 +29,6 @@ const CONFIG_VARS = [
|
|
|
29
29
|
["processEventsAfterPublish", true],
|
|
30
30
|
["isRunnerDeactivated", false],
|
|
31
31
|
["runInterval", 5 * 60 * 1000],
|
|
32
|
-
["parallelTenantProcessing", 5],
|
|
33
32
|
["tableNameEventQueue", BASE_TABLES.EVENT],
|
|
34
33
|
["tableNameEventLock", BASE_TABLES.LOCK],
|
|
35
34
|
["disableRedis", false],
|
|
@@ -43,7 +42,6 @@ const initialize = async ({
|
|
|
43
42
|
processEventsAfterPublish,
|
|
44
43
|
isRunnerDeactivated,
|
|
45
44
|
runInterval,
|
|
46
|
-
parallelTenantProcessing,
|
|
47
45
|
tableNameEventQueue,
|
|
48
46
|
tableNameEventLock,
|
|
49
47
|
disableRedis,
|
|
@@ -52,7 +50,7 @@ const initialize = async ({
|
|
|
52
50
|
} = {}) => {
|
|
53
51
|
// TODO: initialize check:
|
|
54
52
|
// - content of yaml check
|
|
55
|
-
// - betweenRuns
|
|
53
|
+
// - betweenRuns
|
|
56
54
|
|
|
57
55
|
if (config.initialized) {
|
|
58
56
|
return;
|
|
@@ -65,7 +63,6 @@ const initialize = async ({
|
|
|
65
63
|
processEventsAfterPublish,
|
|
66
64
|
isRunnerDeactivated,
|
|
67
65
|
runInterval,
|
|
68
|
-
parallelTenantProcessing,
|
|
69
66
|
tableNameEventQueue,
|
|
70
67
|
tableNameEventLock,
|
|
71
68
|
disableRedis,
|
|
@@ -92,7 +89,6 @@ const initialize = async ({
|
|
|
92
89
|
multiTenancyEnabled: config.isMultiTenancy,
|
|
93
90
|
redisEnabled: config.redisEnabled,
|
|
94
91
|
runInterval: config.runInterval,
|
|
95
|
-
config: config.parallelTenantProcessing,
|
|
96
92
|
});
|
|
97
93
|
};
|
|
98
94
|
|
package/src/periodicEvents.js
CHANGED
|
@@ -7,6 +7,7 @@ const { processChunkedSync } = require("./shared/common");
|
|
|
7
7
|
const eventConfig = require("./config");
|
|
8
8
|
|
|
9
9
|
const COMPONENT_NAME = "eventQueue/periodicEvents";
|
|
10
|
+
const CHUNK_SIZE_INSERT_PERIODIC_EVENTS = 4;
|
|
10
11
|
|
|
11
12
|
const checkAndInsertPeriodicEvents = async (context) => {
|
|
12
13
|
const tx = cds.tx(context);
|
|
@@ -84,13 +85,17 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
84
85
|
|
|
85
86
|
const insertPeriodEvents = async (tx, events) => {
|
|
86
87
|
const startAfter = new Date();
|
|
87
|
-
|
|
88
|
-
|
|
88
|
+
let counter = 1;
|
|
89
|
+
const chunks = Math.ceil(events.length / CHUNK_SIZE_INSERT_PERIODIC_EVENTS);
|
|
90
|
+
const logger = cds.log(COMPONENT_NAME);
|
|
91
|
+
processChunkedSync(events, CHUNK_SIZE_INSERT_PERIODIC_EVENTS, (chunk) => {
|
|
92
|
+
logger.info(`${counter}/${chunks} | inserting chunk of changed or new periodic events`, {
|
|
89
93
|
events: chunk.map(({ type, subType }) => {
|
|
90
94
|
const { interval } = eventConfig.getEventConfig(type, subType);
|
|
91
95
|
return { type, subType, interval };
|
|
92
96
|
}),
|
|
93
97
|
});
|
|
98
|
+
counter++;
|
|
94
99
|
});
|
|
95
100
|
const periodEventsInsert = events.map((periodicEvent) => ({
|
|
96
101
|
type: periodicEvent.type,
|
package/src/processEventQueue.js
CHANGED
|
@@ -166,7 +166,7 @@ const processPeriodicEvent = async (eventTypeInstance) => {
|
|
|
166
166
|
throw new TriggerRollback();
|
|
167
167
|
}
|
|
168
168
|
if (
|
|
169
|
-
eventTypeInstance.transactionMode
|
|
169
|
+
eventTypeInstance.transactionMode === TransactionMode.alwaysRollback ||
|
|
170
170
|
eventTypeInstance.shouldRollbackTransaction(queueEntry.ID)
|
|
171
171
|
) {
|
|
172
172
|
throw new TriggerRollback();
|
package/src/redisPubSub.js
CHANGED
|
@@ -4,6 +4,7 @@ const redis = require("./shared/redis");
|
|
|
4
4
|
const { checkLockExistsAndReturnValue } = require("./shared/distributedLock");
|
|
5
5
|
const config = require("./config");
|
|
6
6
|
const { runEventCombinationForTenant } = require("./runner");
|
|
7
|
+
const { getSubdomainForTenantId } = require("./shared/cdsHelper");
|
|
7
8
|
|
|
8
9
|
const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
|
|
9
10
|
const COMPONENT_NAME = "eventQueue/redisPubSub";
|
|
@@ -26,7 +27,15 @@ const messageHandlerProcessEvents = async (messageData) => {
|
|
|
26
27
|
type,
|
|
27
28
|
subType,
|
|
28
29
|
});
|
|
29
|
-
await
|
|
30
|
+
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
31
|
+
const tenantContext = {
|
|
32
|
+
tenant: tenantId,
|
|
33
|
+
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
34
|
+
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
35
|
+
};
|
|
36
|
+
return await cds.tx(tenantContext, async ({ context }) => {
|
|
37
|
+
return await runEventCombinationForTenant(context, type, subType);
|
|
38
|
+
});
|
|
30
39
|
} catch (err) {
|
|
31
40
|
logger.error("could not parse event information", {
|
|
32
41
|
messageData,
|
|
@@ -36,13 +45,25 @@ const messageHandlerProcessEvents = async (messageData) => {
|
|
|
36
45
|
|
|
37
46
|
const broadcastEvent = async (tenantId, type, subType) => {
|
|
38
47
|
const logger = cds.log(COMPONENT_NAME);
|
|
39
|
-
if (!config.redisEnabled) {
|
|
40
|
-
if (config.registerAsEventProcessor) {
|
|
41
|
-
await runEventCombinationForTenant(tenantId, type, subType);
|
|
42
|
-
}
|
|
43
|
-
return;
|
|
44
|
-
}
|
|
45
48
|
try {
|
|
49
|
+
if (!config.redisEnabled) {
|
|
50
|
+
if (config.registerAsEventProcessor) {
|
|
51
|
+
let context = {};
|
|
52
|
+
if (tenantId) {
|
|
53
|
+
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
54
|
+
context = {
|
|
55
|
+
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
56
|
+
tenant: tenantId,
|
|
57
|
+
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return await cds.tx(context, async ({ context }) => {
|
|
62
|
+
return await runEventCombinationForTenant(context, type, subType);
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
46
67
|
const result = await checkLockExistsAndReturnValue(
|
|
47
68
|
new cds.EventContext({ tenant: tenantId }),
|
|
48
69
|
[type, subType].join("##")
|
package/src/runner.js
CHANGED
|
@@ -94,45 +94,61 @@ const _checkAndTriggerPeriodicEventUpdate = (tenantIds) => {
|
|
|
94
94
|
|
|
95
95
|
const _executeEventsAllTenants = (tenantIds, runId) => {
|
|
96
96
|
const events = eventQueueConfig.allEvents;
|
|
97
|
-
const
|
|
98
|
-
tenantIds.forEach((tenantId) => {
|
|
97
|
+
const product = tenantIds.reduce((result, tenantId) => {
|
|
99
98
|
events.forEach((event) => {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
99
|
+
result.push([tenantId, event]);
|
|
100
|
+
});
|
|
101
|
+
return result;
|
|
102
|
+
}, []);
|
|
103
|
+
|
|
104
|
+
return product.map(async ([tenantId, event]) => {
|
|
105
|
+
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
106
|
+
const tenantContext = {
|
|
107
|
+
tenant: tenantId,
|
|
108
|
+
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
109
|
+
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
110
|
+
};
|
|
111
|
+
return await cds.tx(tenantContext, async ({ context }) => {
|
|
112
|
+
return await WorkerQueue.instance.addToQueue(event.load, async () => {
|
|
113
|
+
try {
|
|
114
|
+
const lockId = `${runId}_${event.type}_${event.subType}`;
|
|
115
|
+
const couldAcquireLock = await distributedLock.acquireLock(context, lockId, {
|
|
116
|
+
expiryTime: eventQueueConfig.runInterval * 0.95,
|
|
117
|
+
});
|
|
118
|
+
if (!couldAcquireLock) {
|
|
119
|
+
return;
|
|
116
120
|
}
|
|
117
|
-
|
|
118
|
-
|
|
121
|
+
await runEventCombinationForTenant(context, event.type, event.subType, true);
|
|
122
|
+
} catch (err) {
|
|
123
|
+
cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
|
|
124
|
+
tenantId,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
});
|
|
119
128
|
});
|
|
120
129
|
});
|
|
121
|
-
return promises;
|
|
122
130
|
};
|
|
123
131
|
|
|
124
132
|
const _executePeriodicEventsAllTenants = (tenantIds, runId) => {
|
|
125
133
|
tenantIds.forEach((tenantId) => {
|
|
126
134
|
WorkerQueue.instance.addToQueue(1, async () => {
|
|
127
135
|
try {
|
|
128
|
-
const
|
|
129
|
-
const
|
|
130
|
-
|
|
136
|
+
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
137
|
+
const tenantContext = {
|
|
138
|
+
tenant: tenantId,
|
|
139
|
+
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
140
|
+
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
return await cds.tx(tenantContext, async ({ context }) => {
|
|
144
|
+
const couldAcquireLock = await distributedLock.acquireLock(context, runId, {
|
|
145
|
+
expiryTime: eventQueueConfig.runInterval * 0.95,
|
|
146
|
+
});
|
|
147
|
+
if (!couldAcquireLock) {
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
await _checkPeriodicEventsSingleTenant(context);
|
|
131
151
|
});
|
|
132
|
-
if (!couldAcquireLock) {
|
|
133
|
-
return;
|
|
134
|
-
}
|
|
135
|
-
await _checkPeriodicEventsSingleTenant(tenantId);
|
|
136
152
|
} catch (err) {
|
|
137
153
|
cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
|
|
138
154
|
tenantId,
|
|
@@ -147,7 +163,8 @@ const _singleTenantDb = async (tenantId) => {
|
|
|
147
163
|
events.forEach((event) => {
|
|
148
164
|
WorkerQueue.instance.addToQueue(event.load, async () => {
|
|
149
165
|
try {
|
|
150
|
-
|
|
166
|
+
const context = new cds.EventContext({ tenant: tenantId });
|
|
167
|
+
await runEventCombinationForTenant(context, event.type, event.subType, true);
|
|
151
168
|
} catch (err) {
|
|
152
169
|
cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
|
|
153
170
|
tenantId,
|
|
@@ -215,15 +232,8 @@ const _calculateOffsetForFirstRun = async () => {
|
|
|
215
232
|
return offsetDependingOnLastRun;
|
|
216
233
|
};
|
|
217
234
|
|
|
218
|
-
const runEventCombinationForTenant = async (
|
|
235
|
+
const runEventCombinationForTenant = async (context, type, subType, skipWorkerPool) => {
|
|
219
236
|
try {
|
|
220
|
-
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
221
|
-
const context = new cds.EventContext({
|
|
222
|
-
tenant: tenantId,
|
|
223
|
-
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
224
|
-
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
225
|
-
});
|
|
226
|
-
cds.context = context;
|
|
227
237
|
if (skipWorkerPool) {
|
|
228
238
|
return await processEventQueue(context, type, subType);
|
|
229
239
|
} else {
|
|
@@ -236,7 +246,7 @@ const runEventCombinationForTenant = async (tenantId, type, subType, skipWorkerP
|
|
|
236
246
|
} catch (err) {
|
|
237
247
|
const logger = cds.log(COMPONENT_NAME);
|
|
238
248
|
logger.error("error executing event combination for tenant", err, {
|
|
239
|
-
tenantId,
|
|
249
|
+
tenantId: context.tenant,
|
|
240
250
|
type,
|
|
241
251
|
subType,
|
|
242
252
|
});
|
|
@@ -266,7 +276,7 @@ const _multiTenancyPeriodicEvents = async () => {
|
|
|
266
276
|
}
|
|
267
277
|
};
|
|
268
278
|
|
|
269
|
-
const _checkPeriodicEventsSingleTenant = async (
|
|
279
|
+
const _checkPeriodicEventsSingleTenant = async (context = {}) => {
|
|
270
280
|
const logger = cds.log(COMPONENT_NAME);
|
|
271
281
|
if (!eventQueueConfig.updatePeriodicEvents || !eventQueueConfig.periodicEvents.length) {
|
|
272
282
|
logger.info("updating of periodic events is disabled or no periodic events configured", {
|
|
@@ -276,23 +286,16 @@ const _checkPeriodicEventsSingleTenant = async (tenantId) => {
|
|
|
276
286
|
return;
|
|
277
287
|
}
|
|
278
288
|
try {
|
|
279
|
-
const subdomain = await cdsHelper.getSubdomainForTenantId(tenantId);
|
|
280
|
-
const context = new cds.EventContext({
|
|
281
|
-
tenant: tenantId,
|
|
282
|
-
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
283
|
-
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
284
|
-
});
|
|
285
|
-
cds.context = context;
|
|
286
289
|
logger.info("executing updating periotic events", {
|
|
287
|
-
tenantId,
|
|
288
|
-
subdomain,
|
|
290
|
+
tenantId: context.tenant,
|
|
291
|
+
subdomain: context.http?.req.authInfo.getSubdomain(),
|
|
289
292
|
});
|
|
290
293
|
await cdsHelper.executeInNewTransaction(context, "update-periodic-events", async (tx) => {
|
|
291
294
|
await periodicEvents.checkAndInsertPeriodicEvents(tx.context);
|
|
292
295
|
});
|
|
293
296
|
} catch (err) {
|
|
294
297
|
logger.error("Couldn't update periodic events for tenant! Next try after defined interval.", err, {
|
|
295
|
-
tenantId,
|
|
298
|
+
tenantId: context.tenant,
|
|
296
299
|
redisEnabled: eventQueueConfig.redisEnabled,
|
|
297
300
|
});
|
|
298
301
|
}
|
|
@@ -8,9 +8,9 @@ const EventQueueError = require("../EventQueueError");
|
|
|
8
8
|
const COMPONENT_NAME = "eventQueue/WorkerQueue";
|
|
9
9
|
const NANO_TO_MS = 1e6;
|
|
10
10
|
const THRESHOLD = {
|
|
11
|
-
INFO:
|
|
12
|
-
WARN:
|
|
13
|
-
ERROR:
|
|
11
|
+
INFO: 15 * 1000,
|
|
12
|
+
WARN: 30 * 1000,
|
|
13
|
+
ERROR: 45 * 1000,
|
|
14
14
|
};
|
|
15
15
|
|
|
16
16
|
class WorkerQueue {
|
|
@@ -82,7 +82,7 @@ class WorkerQueue {
|
|
|
82
82
|
**/
|
|
83
83
|
static get instance() {
|
|
84
84
|
if (!WorkerQueue.#instance) {
|
|
85
|
-
WorkerQueue.#instance = new WorkerQueue(config.
|
|
85
|
+
WorkerQueue.#instance = new WorkerQueue(config.instanceLoadLimit);
|
|
86
86
|
}
|
|
87
87
|
return WorkerQueue.#instance;
|
|
88
88
|
}
|
|
@@ -19,7 +19,7 @@ class EventScheduler {
|
|
|
19
19
|
return; // event combination already scheduled
|
|
20
20
|
}
|
|
21
21
|
this.#scheduledEvents[key] = true;
|
|
22
|
-
cds.log(COMPONENT_NAME).
|
|
22
|
+
cds.log(COMPONENT_NAME).debug("scheduling event queue run for delayed event", {
|
|
23
23
|
type,
|
|
24
24
|
subType,
|
|
25
25
|
delaySeconds: (date.getTime() - Date.now()) / 1000,
|