@cap-js-community/event-queue 1.0.3 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cds-plugin.js +4 -4
- package/package.json +5 -5
- package/src/EventQueueProcessorBase.js +2 -2
- package/src/config.js +60 -3
- package/src/dbHandler.js +1 -1
- package/src/initialize.js +46 -22
- package/src/outbox/EventQueueGenericOutboxHandler.js +37 -0
- package/src/outbox/eventQueueAsOutbox.js +110 -0
- package/src/periodicEvents.js +1 -1
- package/src/processEventQueue.js +2 -2
- package/src/redisPubSub.js +35 -6
- package/src/runner.js +9 -2
- package/src/shared/WorkerQueue.js +1 -1
- package/src/shared/cdsHelper.js +5 -3
- package/src/shared/eventScheduler.js +1 -1
- package/src/shared/redis.js +1 -1
package/cds-plugin.js
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
const cds = require("@sap/cds");
|
|
4
4
|
|
|
5
5
|
const eventQueue = require("./src");
|
|
6
|
+
const COMPONENT_NAME = "/eventQueue/plugin";
|
|
6
7
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
});
|
|
8
|
+
const eventQueueConfig = cds.env.eventQueue;
|
|
9
|
+
if (!(cds.build.register || (!eventQueueConfig?.config && !eventQueueConfig?.configFilePath))) {
|
|
10
|
+
eventQueue.initialize().catch((err) => cds.log(COMPONENT_NAME).error(err));
|
|
11
11
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cap-js-community/event-queue",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "An event queue that enables secure transactional processing of asynchronous events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"files": [
|
|
@@ -47,17 +47,17 @@
|
|
|
47
47
|
"yaml": "2.3.4"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@sap/cds": "7.5.
|
|
51
|
-
"@sap/cds-dk": "7.5.
|
|
50
|
+
"@sap/cds": "^7.5.3",
|
|
51
|
+
"@sap/cds-dk": "^7.5.1",
|
|
52
52
|
"eslint": "8.56.0",
|
|
53
53
|
"eslint-config-prettier": "9.1.0",
|
|
54
|
-
"eslint-plugin-jest": "27.6.
|
|
54
|
+
"eslint-plugin-jest": "27.6.3",
|
|
55
55
|
"eslint-plugin-node": "11.1.0",
|
|
56
56
|
"express": "4.18.2",
|
|
57
57
|
"hdb": "0.19.7",
|
|
58
58
|
"jest": "29.7.0",
|
|
59
59
|
"prettier": "2.8.8",
|
|
60
|
-
"sqlite3": "5.1.7
|
|
60
|
+
"sqlite3": "5.1.7"
|
|
61
61
|
},
|
|
62
62
|
"homepage": "https://cap-js-community.github.io/event-queue/",
|
|
63
63
|
"repository": {
|
|
@@ -12,7 +12,7 @@ const eventConfig = require("./config");
|
|
|
12
12
|
const PerformanceTracer = require("./shared/PerformanceTracer");
|
|
13
13
|
|
|
14
14
|
const IMPLEMENT_ERROR_MESSAGE = "needs to be reimplemented";
|
|
15
|
-
const COMPONENT_NAME = "eventQueue/EventQueueProcessorBase";
|
|
15
|
+
const COMPONENT_NAME = "/eventQueue/EventQueueProcessorBase";
|
|
16
16
|
|
|
17
17
|
const DEFAULT_RETRY_ATTEMPTS = 3;
|
|
18
18
|
const DEFAULT_PARALLEL_EVENT_PROCESSING = 1;
|
|
@@ -869,7 +869,7 @@ class EventQueueProcessorBase {
|
|
|
869
869
|
return await checkAndUpdatePromise;
|
|
870
870
|
}
|
|
871
871
|
|
|
872
|
-
async
|
|
872
|
+
async acquireDistributedLock() {
|
|
873
873
|
if (this.concurrentEventProcessing) {
|
|
874
874
|
return true;
|
|
875
875
|
}
|
package/src/config.js
CHANGED
|
@@ -10,12 +10,15 @@ const FOR_UPDATE_TIMEOUT = 10;
|
|
|
10
10
|
const GLOBAL_TX_TIMEOUT = 30 * 60 * 1000;
|
|
11
11
|
const REDIS_CONFIG_CHANNEL = "EVENT_QUEUE_CONFIG_CHANNEL";
|
|
12
12
|
const REDIS_CONFIG_BLOCKLIST_CHANNEL = "REDIS_CONFIG_BLOCKLIST_CHANNEL";
|
|
13
|
-
const COMPONENT_NAME = "eventQueue/config";
|
|
13
|
+
const COMPONENT_NAME = "/eventQueue/config";
|
|
14
14
|
const MIN_INTERVAL_SEC = 10;
|
|
15
15
|
const DEFAULT_LOAD = 1;
|
|
16
16
|
const SUFFIX_PERIODIC = "_PERIODIC";
|
|
17
17
|
const COMMAND_BLOCK = "EVENT_QUEUE_EVENT_BLOCK";
|
|
18
18
|
const COMMAND_UNBLOCK = "EVENT_QUEUE_EVENT_UNBLOCK";
|
|
19
|
+
const CAP_EVENT_TYPE = "CAP_OUTBOX";
|
|
20
|
+
|
|
21
|
+
const CAP_PARALLEL_DEFAULT = 5;
|
|
19
22
|
|
|
20
23
|
const BASE_PERIODIC_EVENTS = [
|
|
21
24
|
{
|
|
@@ -51,6 +54,8 @@ class Config {
|
|
|
51
54
|
#blockedPeriodicEvents;
|
|
52
55
|
#isPeriodicEventBlockedCb;
|
|
53
56
|
#thresholdLoggingEventProcessing;
|
|
57
|
+
#useAsCAPOutbox;
|
|
58
|
+
#userId;
|
|
54
59
|
static #instance;
|
|
55
60
|
constructor() {
|
|
56
61
|
this.#logger = cds.log(COMPONENT_NAME);
|
|
@@ -76,6 +81,10 @@ class Config {
|
|
|
76
81
|
return this.#eventMap[this.generateKey(type, subType)];
|
|
77
82
|
}
|
|
78
83
|
|
|
84
|
+
isCapOutboxEvent(type) {
|
|
85
|
+
return type === CAP_EVENT_TYPE;
|
|
86
|
+
}
|
|
87
|
+
|
|
79
88
|
hasEventAfterCommitFlag(type, subType) {
|
|
80
89
|
return this.#eventMap[this.generateKey(type, subType)]?.processAfterCommit ?? true;
|
|
81
90
|
}
|
|
@@ -180,6 +189,30 @@ class Config {
|
|
|
180
189
|
});
|
|
181
190
|
}
|
|
182
191
|
|
|
192
|
+
addCAPOutboxEvent(serviceName, config) {
|
|
193
|
+
if (this.#eventMap[this.generateKey(CAP_EVENT_TYPE, serviceName)]) {
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const eventConfig = {
|
|
198
|
+
type: CAP_EVENT_TYPE,
|
|
199
|
+
subType: serviceName,
|
|
200
|
+
load: config.load ?? DEFAULT_LOAD,
|
|
201
|
+
impl: "./outbox/EventQueueGenericOutboxHandler",
|
|
202
|
+
selectMaxChunkSize: config.chunkSize,
|
|
203
|
+
parallelEventProcessing: config.parallelEventProcessing ?? (config.parallel && CAP_PARALLEL_DEFAULT),
|
|
204
|
+
retryAttempts: config.maxAttempts,
|
|
205
|
+
transactionMode: config.transactionMode,
|
|
206
|
+
processAfterCommit: config.processAfterCommit,
|
|
207
|
+
eventOutdatedCheck: config.eventOutdatedCheck,
|
|
208
|
+
checkForNextChunk: config.checkForNextChunk,
|
|
209
|
+
deleteFinishedEventsAfterDays: config.deleteFinishedEventsAfterDays,
|
|
210
|
+
internalEvent: true,
|
|
211
|
+
};
|
|
212
|
+
this.#config.events.push(eventConfig);
|
|
213
|
+
this.#eventMap[this.generateKey(CAP_EVENT_TYPE, serviceName)] = eventConfig;
|
|
214
|
+
}
|
|
215
|
+
|
|
183
216
|
#unblockPeriodicEventLocalState(key, tenant) {
|
|
184
217
|
const map = this.#blockedPeriodicEvents[key];
|
|
185
218
|
if (!map) {
|
|
@@ -214,7 +247,7 @@ class Config {
|
|
|
214
247
|
this.#eventMap = config.events.reduce((result, event) => {
|
|
215
248
|
event.load = event.load ?? DEFAULT_LOAD;
|
|
216
249
|
this.validateAdHocEvents(result, event);
|
|
217
|
-
result[
|
|
250
|
+
result[this.generateKey(event.type, event.subType)] = event;
|
|
218
251
|
return result;
|
|
219
252
|
}, {});
|
|
220
253
|
this.#eventMap = config.periodicEvents.reduce((result, event) => {
|
|
@@ -222,7 +255,7 @@ class Config {
|
|
|
222
255
|
event.type = `${event.type}${SUFFIX_PERIODIC}`;
|
|
223
256
|
event.isPeriodic = true;
|
|
224
257
|
this.validatePeriodicConfig(result, event);
|
|
225
|
-
result[
|
|
258
|
+
result[this.generateKey(event.type, event.subType)] = event;
|
|
226
259
|
return result;
|
|
227
260
|
}, this.#eventMap);
|
|
228
261
|
}
|
|
@@ -257,6 +290,14 @@ class Config {
|
|
|
257
290
|
return [type, subType].join("##");
|
|
258
291
|
}
|
|
259
292
|
|
|
293
|
+
removeEvent(type, subType) {
|
|
294
|
+
const index = this.#config.events.findIndex((event) => event.type === "CAP_OUTBOX");
|
|
295
|
+
if (index >= 0) {
|
|
296
|
+
this.#config.events.splice(index, 1);
|
|
297
|
+
}
|
|
298
|
+
delete this.#eventMap[this.generateKey(type, subType)];
|
|
299
|
+
}
|
|
300
|
+
|
|
260
301
|
get fileContent() {
|
|
261
302
|
return this.#config;
|
|
262
303
|
}
|
|
@@ -405,6 +446,22 @@ class Config {
|
|
|
405
446
|
return this.#thresholdLoggingEventProcessing;
|
|
406
447
|
}
|
|
407
448
|
|
|
449
|
+
set useAsCAPOutbox(value) {
|
|
450
|
+
this.#useAsCAPOutbox = value;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
get useAsCAPOutbox() {
|
|
454
|
+
return this.#useAsCAPOutbox;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
set userId(value) {
|
|
458
|
+
this.#userId = value;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
get userId() {
|
|
462
|
+
return this.#userId;
|
|
463
|
+
}
|
|
464
|
+
|
|
408
465
|
get isMultiTenancy() {
|
|
409
466
|
return !!cds.requires.multitenancy;
|
|
410
467
|
}
|
package/src/dbHandler.js
CHANGED
|
@@ -5,7 +5,7 @@ const cds = require("@sap/cds");
|
|
|
5
5
|
const { broadcastEvent } = require("./redisPubSub");
|
|
6
6
|
const config = require("./config");
|
|
7
7
|
|
|
8
|
-
const COMPONENT_NAME = "eventQueue/dbHandler";
|
|
8
|
+
const COMPONENT_NAME = "/eventQueue/dbHandler";
|
|
9
9
|
|
|
10
10
|
const registerEventQueueDbHandler = (dbService) => {
|
|
11
11
|
const def = dbService.model.definitions[config.tableNameEventQueue];
|
package/src/initialize.js
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
const { promisify } = require("util");
|
|
4
4
|
const fs = require("fs");
|
|
5
|
-
const path = require("path");
|
|
6
5
|
|
|
7
6
|
const cds = require("@sap/cds");
|
|
8
7
|
const yaml = require("yaml");
|
|
@@ -14,6 +13,7 @@ const dbHandler = require("./dbHandler");
|
|
|
14
13
|
const config = require("./config");
|
|
15
14
|
const { initEventQueueRedisSubscribe, closeSubscribeClient } = require("./redisPubSub");
|
|
16
15
|
const { closeMainClient } = require("./shared/redis");
|
|
16
|
+
const eventQueueAsOutbox = require("./outbox/eventQueueAsOutbox");
|
|
17
17
|
|
|
18
18
|
const readFileAsync = promisify(fs.readFile);
|
|
19
19
|
|
|
@@ -35,6 +35,8 @@ const CONFIG_VARS = [
|
|
|
35
35
|
["skipCsnCheck", false],
|
|
36
36
|
["updatePeriodicEvents", true],
|
|
37
37
|
["thresholdLoggingEventProcessing", 50],
|
|
38
|
+
["useAsCAPOutbox", false],
|
|
39
|
+
["userId", null],
|
|
38
40
|
];
|
|
39
41
|
|
|
40
42
|
const initialize = async ({
|
|
@@ -49,6 +51,8 @@ const initialize = async ({
|
|
|
49
51
|
skipCsnCheck,
|
|
50
52
|
updatePeriodicEvents,
|
|
51
53
|
thresholdLoggingEventProcessing,
|
|
54
|
+
useAsCAPOutbox,
|
|
55
|
+
userId,
|
|
52
56
|
} = {}) => {
|
|
53
57
|
// TODO: initialize check:
|
|
54
58
|
// - content of yaml check
|
|
@@ -70,20 +74,25 @@ const initialize = async ({
|
|
|
70
74
|
disableRedis,
|
|
71
75
|
skipCsnCheck,
|
|
72
76
|
updatePeriodicEvents,
|
|
73
|
-
thresholdLoggingEventProcessing
|
|
77
|
+
thresholdLoggingEventProcessing,
|
|
78
|
+
useAsCAPOutbox,
|
|
79
|
+
userId
|
|
74
80
|
);
|
|
75
81
|
|
|
76
82
|
const logger = cds.log(COMPONENT);
|
|
77
83
|
config.fileContent = await readConfigFromFile(config.configFilePath);
|
|
78
84
|
config.checkRedisEnabled();
|
|
79
85
|
|
|
80
|
-
const dbService = await cds.connect.to("db");
|
|
81
|
-
await (cds.model ? Promise.resolve() : new Promise((resolve) => cds.on("serving", resolve)));
|
|
82
|
-
!config.skipCsnCheck && (await csnCheck());
|
|
83
86
|
if (config.processEventsAfterPublish) {
|
|
84
|
-
|
|
87
|
+
cds.on("connect", (service) => {
|
|
88
|
+
if (service.name === "db ") {
|
|
89
|
+
dbHandler.registerEventQueueDbHandler(service);
|
|
90
|
+
}
|
|
91
|
+
});
|
|
85
92
|
}
|
|
93
|
+
!config.skipCsnCheck && (await csnCheck());
|
|
86
94
|
|
|
95
|
+
monkeyPatchCAPOutbox();
|
|
87
96
|
registerEventProcessors();
|
|
88
97
|
registerCdsShutdown();
|
|
89
98
|
logger.info("event queue initialized", {
|
|
@@ -133,27 +142,42 @@ const registerEventProcessors = () => {
|
|
|
133
142
|
}
|
|
134
143
|
};
|
|
135
144
|
|
|
136
|
-
const
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
145
|
+
const monkeyPatchCAPOutbox = () => {
|
|
146
|
+
if (config.useAsCAPOutbox) {
|
|
147
|
+
Object.defineProperty(cds, "outboxed", {
|
|
148
|
+
get: () => eventQueueAsOutbox.outboxed,
|
|
149
|
+
});
|
|
150
|
+
Object.defineProperty(cds, "unboxed", {
|
|
151
|
+
get: () => eventQueueAsOutbox.unboxed,
|
|
152
|
+
});
|
|
140
153
|
}
|
|
154
|
+
};
|
|
141
155
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
156
|
+
const csnCheck = async () => {
|
|
157
|
+
cds.on("loaded", async (csn) => {
|
|
158
|
+
if (csn.namespace === "cds.xt") {
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
const eventCsn = csn.definitions[config.tableNameEventQueue];
|
|
162
|
+
if (!eventCsn) {
|
|
163
|
+
throw EventQueueError.missingTableInCsn(config.tableNameEventQueue);
|
|
164
|
+
}
|
|
146
165
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
166
|
+
const lockCsn = csn.definitions[config.tableNameEventLock];
|
|
167
|
+
if (!lockCsn) {
|
|
168
|
+
throw EventQueueError.missingTableInCsn(config.tableNameEventLock);
|
|
169
|
+
}
|
|
150
170
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
171
|
+
if (config.tableNameEventQueue === BASE_TABLES.EVENT && config.tableNameEventLock === BASE_TABLES.LOCK) {
|
|
172
|
+
return; // no need to check base tables
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const baseEvent = csn.definitions["sap.eventqueue.Event"];
|
|
176
|
+
const baseLock = csn.definitions["sap.eventqueue.Lock"];
|
|
154
177
|
|
|
155
|
-
|
|
156
|
-
|
|
178
|
+
checkCustomTable(baseEvent, eventCsn);
|
|
179
|
+
checkCustomTable(baseLock, lockCsn);
|
|
180
|
+
});
|
|
157
181
|
};
|
|
158
182
|
|
|
159
183
|
const checkCustomTable = (baseCsn, customCsn) => {
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const cds = require("@sap/cds");
|
|
4
|
+
|
|
5
|
+
const EventQueueBaseClass = require("../EventQueueProcessorBase");
|
|
6
|
+
const { EventProcessingStatus } = require("../constants");
|
|
7
|
+
|
|
8
|
+
const COMPONENT_NAME = "/eventQueue/outbox/generic";
|
|
9
|
+
|
|
10
|
+
class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
|
|
11
|
+
constructor(context, eventType, eventSubType, config) {
|
|
12
|
+
super(context, eventType, eventSubType, config);
|
|
13
|
+
this.logger = cds.log(`${COMPONENT_NAME}/${eventSubType}`);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
async processEvent(processContext, key, queueEntries, payload) {
|
|
17
|
+
let status = EventProcessingStatus.Done;
|
|
18
|
+
try {
|
|
19
|
+
const service = await cds.connect.to(this.eventSubType);
|
|
20
|
+
const userId = payload.contextUser;
|
|
21
|
+
const msg = payload._fromSend ? new cds.Request(payload) : new cds.Event(payload);
|
|
22
|
+
const invocationFn = payload._fromSend ? "send" : "emit";
|
|
23
|
+
delete msg._fromSend;
|
|
24
|
+
delete msg.contextUser;
|
|
25
|
+
processContext.user = new cds.User.Privileged(userId);
|
|
26
|
+
await cds.unboxed(service).tx(processContext)[invocationFn](msg);
|
|
27
|
+
} catch (err) {
|
|
28
|
+
status = EventProcessingStatus.Error;
|
|
29
|
+
this.logger.error("error processing outboxed service call", err, {
|
|
30
|
+
serviceName: this.eventSubType,
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
return queueEntries.map((queueEntry) => [queueEntry.ID, status]);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
module.exports = EventQueueGenericOutboxHandler;
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const cds = require("@sap/cds");
|
|
4
|
+
|
|
5
|
+
const { publishEvent } = require("../publishEvent");
|
|
6
|
+
const config = require("../config");
|
|
7
|
+
|
|
8
|
+
const OUTBOXED = Symbol("outboxed");
|
|
9
|
+
const UNBOXED = Symbol("unboxed");
|
|
10
|
+
|
|
11
|
+
const CDS_EVENT_TYPE = "CAP_OUTBOX";
|
|
12
|
+
|
|
13
|
+
const COMPONENT_NAME = "/eventQueue/eventQueueAsOutbox";
|
|
14
|
+
|
|
15
|
+
function outboxed(srv, customOpts) {
|
|
16
|
+
// outbox max. once
|
|
17
|
+
const logger = cds.log(COMPONENT_NAME);
|
|
18
|
+
if (!new.target) {
|
|
19
|
+
const former = srv[OUTBOXED];
|
|
20
|
+
if (former) {
|
|
21
|
+
return former;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const originalSrv = srv[UNBOXED] || srv;
|
|
26
|
+
const outboxedSrv = Object.create(originalSrv);
|
|
27
|
+
outboxedSrv[UNBOXED] = originalSrv;
|
|
28
|
+
|
|
29
|
+
if (!new.target) {
|
|
30
|
+
Object.defineProperty(srv, OUTBOXED, { value: outboxedSrv });
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const outboxOpts = Object.assign(
|
|
34
|
+
{},
|
|
35
|
+
(typeof cds.requires.outbox === "object" && cds.requires.outbox) || {},
|
|
36
|
+
(typeof srv.options?.outbox === "object" && srv.options.outbox) || {},
|
|
37
|
+
customOpts || {}
|
|
38
|
+
);
|
|
39
|
+
|
|
40
|
+
config.addCAPOutboxEvent(srv.name, outboxOpts);
|
|
41
|
+
outboxedSrv.handle = async function (req) {
|
|
42
|
+
const context = req.context || cds.context;
|
|
43
|
+
if (outboxOpts.kind === "persistent-outbox") {
|
|
44
|
+
config.addCAPOutboxEvent(srv.name, outboxOpts);
|
|
45
|
+
await _mapToEventAndPublish(context, srv.name, req);
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
context.on("succeeded", async () => {
|
|
49
|
+
try {
|
|
50
|
+
if (req.reply) {
|
|
51
|
+
await originalSrv.send(req);
|
|
52
|
+
} else {
|
|
53
|
+
await originalSrv.emit(req);
|
|
54
|
+
}
|
|
55
|
+
} catch (err) {
|
|
56
|
+
logger.error("In memory processing failed", { event: req.event, cause: err });
|
|
57
|
+
if (isUnrecoverable(originalSrv, err) && outboxOpts.crashOnError !== false) {
|
|
58
|
+
cds.exit(1);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
return outboxedSrv;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function unboxed(srv) {
|
|
68
|
+
return srv[UNBOXED] || srv;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const _mapToEventAndPublish = async (context, name, msg) => {
|
|
72
|
+
const event = {
|
|
73
|
+
contextUser: context.user.id,
|
|
74
|
+
...(msg._fromSend || (msg.reply && { _fromSend: true })), // send or emit
|
|
75
|
+
...(msg.inbound && { inbound: msg.inbound }),
|
|
76
|
+
...(msg.event && { event: msg.event }),
|
|
77
|
+
...(msg.data && { data: msg.data }),
|
|
78
|
+
...(msg.headers && { headers: msg.headers }),
|
|
79
|
+
...(msg.query && { query: msg.query }),
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
await publishEvent(cds.tx(context), {
|
|
83
|
+
type: CDS_EVENT_TYPE,
|
|
84
|
+
subType: name,
|
|
85
|
+
payload: JSON.stringify(event),
|
|
86
|
+
});
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
const isUnrecoverable = (service, error) => {
|
|
90
|
+
let unrecoverable = service.isUnrecoverableError && service.isUnrecoverableError(error);
|
|
91
|
+
if (unrecoverable === undefined) {
|
|
92
|
+
unrecoverable = error.unrecoverable;
|
|
93
|
+
}
|
|
94
|
+
return unrecoverable || isStandardError(error);
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
const isStandardError = (err) => {
|
|
98
|
+
return (
|
|
99
|
+
err instanceof TypeError ||
|
|
100
|
+
err instanceof ReferenceError ||
|
|
101
|
+
err instanceof SyntaxError ||
|
|
102
|
+
err instanceof RangeError ||
|
|
103
|
+
err instanceof URIError
|
|
104
|
+
);
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
module.exports = {
|
|
108
|
+
outboxed,
|
|
109
|
+
unboxed,
|
|
110
|
+
};
|
package/src/periodicEvents.js
CHANGED
|
@@ -6,7 +6,7 @@ const { EventProcessingStatus } = require("./constants");
|
|
|
6
6
|
const { processChunkedSync } = require("./shared/common");
|
|
7
7
|
const eventConfig = require("./config");
|
|
8
8
|
|
|
9
|
-
const COMPONENT_NAME = "eventQueue/periodicEvents";
|
|
9
|
+
const COMPONENT_NAME = "/eventQueue/periodicEvents";
|
|
10
10
|
const CHUNK_SIZE_INSERT_PERIODIC_EVENTS = 4;
|
|
11
11
|
|
|
12
12
|
const checkAndInsertPeriodicEvents = async (context) => {
|
package/src/processEventQueue.js
CHANGED
|
@@ -10,7 +10,7 @@ const { limiter } = require("./shared/common");
|
|
|
10
10
|
|
|
11
11
|
const { executeInNewTransaction, TriggerRollback } = require("./shared/cdsHelper");
|
|
12
12
|
|
|
13
|
-
const COMPONENT_NAME = "eventQueue/processEventQueue";
|
|
13
|
+
const COMPONENT_NAME = "/eventQueue/processEventQueue";
|
|
14
14
|
const MAX_EXECUTION_TIME = 5 * 60 * 1000;
|
|
15
15
|
|
|
16
16
|
const processEventQueue = async (context, eventType, eventSubType, startTime = new Date()) => {
|
|
@@ -29,7 +29,7 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
|
|
|
29
29
|
return;
|
|
30
30
|
}
|
|
31
31
|
baseInstance = new EventTypeClass(context, eventType, eventSubType, eventConfig);
|
|
32
|
-
const continueProcessing = await baseInstance.
|
|
32
|
+
const continueProcessing = await baseInstance.acquireDistributedLock();
|
|
33
33
|
if (!continueProcessing) {
|
|
34
34
|
return;
|
|
35
35
|
}
|
package/src/redisPubSub.js
CHANGED
|
@@ -5,11 +5,11 @@ const cds = require("@sap/cds");
|
|
|
5
5
|
const redis = require("./shared/redis");
|
|
6
6
|
const { checkLockExistsAndReturnValue } = require("./shared/distributedLock");
|
|
7
7
|
const config = require("./config");
|
|
8
|
-
const
|
|
8
|
+
const runner = require("./runner");
|
|
9
9
|
const { getSubdomainForTenantId } = require("./shared/cdsHelper");
|
|
10
10
|
|
|
11
11
|
const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
|
|
12
|
-
const COMPONENT_NAME = "eventQueue/redisPubSub";
|
|
12
|
+
const COMPONENT_NAME = "/eventQueue/redisPubSub";
|
|
13
13
|
|
|
14
14
|
let subscriberClientPromise;
|
|
15
15
|
|
|
@@ -17,10 +17,10 @@ const initEventQueueRedisSubscribe = () => {
|
|
|
17
17
|
if (subscriberClientPromise || !config.redisEnabled) {
|
|
18
18
|
return;
|
|
19
19
|
}
|
|
20
|
-
redis.subscribeRedisChannel(EVENT_MESSAGE_CHANNEL,
|
|
20
|
+
redis.subscribeRedisChannel(EVENT_MESSAGE_CHANNEL, _messageHandlerProcessEvents);
|
|
21
21
|
};
|
|
22
22
|
|
|
23
|
-
const
|
|
23
|
+
const _messageHandlerProcessEvents = async (messageData) => {
|
|
24
24
|
const logger = cds.log(COMPONENT_NAME);
|
|
25
25
|
try {
|
|
26
26
|
const { tenantId, type, subType } = JSON.parse(messageData);
|
|
@@ -38,13 +38,37 @@ const messageHandlerProcessEvents = async (messageData) => {
|
|
|
38
38
|
}
|
|
39
39
|
|
|
40
40
|
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
41
|
+
const user = new cds.User.Privileged(config.userId);
|
|
41
42
|
const tenantContext = {
|
|
42
43
|
tenant: tenantId,
|
|
44
|
+
user,
|
|
43
45
|
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
44
46
|
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
45
47
|
};
|
|
48
|
+
|
|
49
|
+
if (!config.getEventConfig(type, subType)) {
|
|
50
|
+
if (config.isCapOutboxEvent(type)) {
|
|
51
|
+
try {
|
|
52
|
+
const service = await cds.connect.to(subType);
|
|
53
|
+
cds.outboxed(service);
|
|
54
|
+
} catch (err) {
|
|
55
|
+
logger.error("could not connect to outboxed service", err, {
|
|
56
|
+
type,
|
|
57
|
+
subType,
|
|
58
|
+
});
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
} else {
|
|
62
|
+
logger.error("cannot find configuration for published event. Event won't be processed", {
|
|
63
|
+
type,
|
|
64
|
+
subType,
|
|
65
|
+
});
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
46
70
|
return await cds.tx(tenantContext, async ({ context }) => {
|
|
47
|
-
return await runEventCombinationForTenant(context, type, subType);
|
|
71
|
+
return await runner.runEventCombinationForTenant(context, type, subType);
|
|
48
72
|
});
|
|
49
73
|
} catch (err) {
|
|
50
74
|
logger.error("could not parse event information", {
|
|
@@ -68,15 +92,17 @@ const broadcastEvent = async (tenantId, type, subType) => {
|
|
|
68
92
|
let context = {};
|
|
69
93
|
if (tenantId) {
|
|
70
94
|
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
95
|
+
const user = new cds.User.Privileged(config.userId);
|
|
71
96
|
context = {
|
|
72
97
|
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
73
98
|
tenant: tenantId,
|
|
99
|
+
user,
|
|
74
100
|
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
75
101
|
};
|
|
76
102
|
}
|
|
77
103
|
|
|
78
104
|
return await cds.tx(context, async ({ context }) => {
|
|
79
|
-
return await runEventCombinationForTenant(context, type, subType);
|
|
105
|
+
return await runner.runEventCombinationForTenant(context, type, subType);
|
|
80
106
|
});
|
|
81
107
|
}
|
|
82
108
|
return;
|
|
@@ -122,4 +148,7 @@ module.exports = {
|
|
|
122
148
|
initEventQueueRedisSubscribe,
|
|
123
149
|
broadcastEvent,
|
|
124
150
|
closeSubscribeClient,
|
|
151
|
+
__: {
|
|
152
|
+
_messageHandlerProcessEvents,
|
|
153
|
+
},
|
|
125
154
|
};
|
package/src/runner.js
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
const { randomUUID } = require("crypto");
|
|
4
4
|
|
|
5
|
+
const cds = require("@sap/cds");
|
|
6
|
+
|
|
5
7
|
const eventQueueConfig = require("./config");
|
|
6
8
|
const { processEventQueue } = require("./processEventQueue");
|
|
7
9
|
const WorkerQueue = require("./shared/WorkerQueue");
|
|
@@ -11,8 +13,9 @@ const SetIntervalDriftSafe = require("./shared/SetIntervalDriftSafe");
|
|
|
11
13
|
const { getSubdomainForTenantId } = require("./shared/cdsHelper");
|
|
12
14
|
const periodicEvents = require("./periodicEvents");
|
|
13
15
|
const { hashStringTo32Bit } = require("./shared/common");
|
|
16
|
+
const config = require("./config");
|
|
14
17
|
|
|
15
|
-
const COMPONENT_NAME = "eventQueue/runner";
|
|
18
|
+
const COMPONENT_NAME = "/eventQueue/runner";
|
|
16
19
|
const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
|
|
17
20
|
const EVENT_QUEUE_RUN_TS = "EVENT_QUEUE_RUN_TS";
|
|
18
21
|
const EVENT_QUEUE_RUN_PERIODIC_EVENT = "EVENT_QUEUE_RUN_PERIODIC_EVENT";
|
|
@@ -107,8 +110,10 @@ const _executeEventsAllTenants = (tenantIds, runId) => {
|
|
|
107
110
|
|
|
108
111
|
return product.map(async ([tenantId, event]) => {
|
|
109
112
|
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
113
|
+
const user = new cds.User.Privileged(config.userId);
|
|
110
114
|
const tenantContext = {
|
|
111
115
|
tenant: tenantId,
|
|
116
|
+
user,
|
|
112
117
|
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
113
118
|
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
114
119
|
};
|
|
@@ -140,8 +145,10 @@ const _executePeriodicEventsAllTenants = (tenantIds, runId) => {
|
|
|
140
145
|
WorkerQueue.instance.addToQueue(1, label, async () => {
|
|
141
146
|
try {
|
|
142
147
|
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
148
|
+
const user = new cds.User.Privileged(config.userId);
|
|
143
149
|
const tenantContext = {
|
|
144
150
|
tenant: tenantId,
|
|
151
|
+
user,
|
|
145
152
|
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
146
153
|
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
147
154
|
};
|
|
@@ -321,7 +328,7 @@ module.exports = {
|
|
|
321
328
|
multiTenancyDb,
|
|
322
329
|
multiTenancyRedis,
|
|
323
330
|
runEventCombinationForTenant,
|
|
324
|
-
|
|
331
|
+
__: {
|
|
325
332
|
_singleTenantDb,
|
|
326
333
|
_multiTenancyRedis,
|
|
327
334
|
_multiTenancyDb,
|
|
@@ -5,7 +5,7 @@ const cds = require("@sap/cds");
|
|
|
5
5
|
const config = require("../config");
|
|
6
6
|
const EventQueueError = require("../EventQueueError");
|
|
7
7
|
|
|
8
|
-
const COMPONENT_NAME = "eventQueue/WorkerQueue";
|
|
8
|
+
const COMPONENT_NAME = "/eventQueue/WorkerQueue";
|
|
9
9
|
const NANO_TO_MS = 1e6;
|
|
10
10
|
const THRESHOLD = {
|
|
11
11
|
INFO: 35 * 1000,
|
package/src/shared/cdsHelper.js
CHANGED
|
@@ -8,7 +8,7 @@ const config = require("../config");
|
|
|
8
8
|
const subdomainCache = {};
|
|
9
9
|
|
|
10
10
|
const VERROR_CLUSTER_NAME = "ExecuteInNewTransactionError";
|
|
11
|
-
const COMPONENT_NAME = "eventQueue/cdsHelper";
|
|
11
|
+
const COMPONENT_NAME = "/eventQueue/cdsHelper";
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* Execute logic in a new managed CDS transaction context, auto-handling commit, rollback and error/exception situations.
|
|
@@ -24,13 +24,14 @@ async function executeInNewTransaction(context = {}, transactionTag, fn, args, {
|
|
|
24
24
|
const parameters = Array.isArray(args) ? args : [args];
|
|
25
25
|
const logger = cds.log(COMPONENT_NAME);
|
|
26
26
|
try {
|
|
27
|
+
const user = new cds.User.Privileged(config.userId);
|
|
27
28
|
if (cds.db.kind === "hana") {
|
|
28
29
|
await cds.tx(
|
|
29
30
|
{
|
|
30
31
|
id: context.id,
|
|
31
32
|
tenant: context.tenant,
|
|
32
33
|
locale: context.locale,
|
|
33
|
-
user
|
|
34
|
+
user,
|
|
34
35
|
headers: context.headers,
|
|
35
36
|
http: context.http,
|
|
36
37
|
},
|
|
@@ -48,13 +49,14 @@ async function executeInNewTransaction(context = {}, transactionTag, fn, args, {
|
|
|
48
49
|
id: context.id,
|
|
49
50
|
tenant: context.tenant,
|
|
50
51
|
locale: context.locale,
|
|
51
|
-
user
|
|
52
|
+
user,
|
|
52
53
|
headers: context.headers,
|
|
53
54
|
http: context.http,
|
|
54
55
|
},
|
|
55
56
|
async (tx) => fn(tx, ...parameters)
|
|
56
57
|
);
|
|
57
58
|
} else {
|
|
59
|
+
contextTx.context.user = user;
|
|
58
60
|
await fn(contextTx, ...parameters);
|
|
59
61
|
}
|
|
60
62
|
}
|
|
@@ -5,7 +5,7 @@ const cds = require("@sap/cds");
|
|
|
5
5
|
const { broadcastEvent } = require("../redisPubSub");
|
|
6
6
|
const config = require("./../config");
|
|
7
7
|
|
|
8
|
-
const COMPONENT_NAME = "eventQueue/shared/eventScheduler";
|
|
8
|
+
const COMPONENT_NAME = "/eventQueue/shared/eventScheduler";
|
|
9
9
|
|
|
10
10
|
let instance;
|
|
11
11
|
class EventScheduler {
|
package/src/shared/redis.js
CHANGED
|
@@ -5,7 +5,7 @@ const redis = require("redis");
|
|
|
5
5
|
const { getEnvInstance } = require("./env");
|
|
6
6
|
const EventQueueError = require("../EventQueueError");
|
|
7
7
|
|
|
8
|
-
const COMPONENT_NAME = "eventQueue/shared/redis";
|
|
8
|
+
const COMPONENT_NAME = "/eventQueue/shared/redis";
|
|
9
9
|
|
|
10
10
|
let mainClientPromise;
|
|
11
11
|
const subscriberChannelClientPromise = {};
|