@cap-js-community/event-queue 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cds-plugin.js +1 -1
- package/package.json +8 -7
- package/src/EventQueueProcessorBase.js +21 -110
- package/src/config.js +9 -9
- package/src/dbHandler.js +24 -0
- package/src/initialize.js +7 -4
- package/src/outbox/EventQueueGenericOutboxHandler.js +1 -0
- package/src/processEventQueue.js +0 -6
- package/src/publishEvent.js +10 -4
- package/src/redis/redisPub.js +2 -5
- package/src/redis/redisSub.js +2 -5
- package/src/runner/openEvents.js +1 -0
- package/src/runner/runner.js +5 -12
- package/src/shared/cdsHelper.js +2 -29
- package/src/shared/common.js +52 -1
- package/src/shared/distributedLock.js +15 -1
package/cds-plugin.js
CHANGED
|
@@ -6,5 +6,5 @@ const eventQueue = require("./src");
|
|
|
6
6
|
const COMPONENT_NAME = "/eventQueue/plugin";
|
|
7
7
|
|
|
8
8
|
if (!cds.build.register && cds.env.eventQueue) {
|
|
9
|
-
eventQueue.initialize().catch((err) => cds.log(COMPONENT_NAME).error(err));
|
|
9
|
+
module.exports = eventQueue.initialize().catch((err) => cds.log(COMPONENT_NAME).error(err));
|
|
10
10
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cap-js-community/event-queue",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.2",
|
|
4
4
|
"description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"files": [
|
|
@@ -42,15 +42,16 @@
|
|
|
42
42
|
"node": ">=18"
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
|
-
"
|
|
46
|
-
"
|
|
47
|
-
"
|
|
45
|
+
"@sap/xssec": "^3.6.1",
|
|
46
|
+
"redis": "^4.6.13",
|
|
47
|
+
"verror": "^1.10.1",
|
|
48
|
+
"yaml": "^2.4.1"
|
|
48
49
|
},
|
|
49
50
|
"devDependencies": {
|
|
50
|
-
"@cap-js/hana": "^0.0
|
|
51
|
+
"@cap-js/hana": "^0.1.0",
|
|
51
52
|
"@cap-js/sqlite": "^1.5.0",
|
|
52
|
-
"@sap/cds": "^7.
|
|
53
|
-
"@sap/cds-dk": "^7.
|
|
53
|
+
"@sap/cds": "^7.8.0",
|
|
54
|
+
"@sap/cds-dk": "^7.8.0",
|
|
54
55
|
"eslint": "^8.56.0",
|
|
55
56
|
"eslint-config-prettier": "^9.1.0",
|
|
56
57
|
"eslint-plugin-jest": "^27.9.0",
|
|
@@ -23,8 +23,6 @@ const TRIES_FOR_EXCEEDED_EVENTS = 3;
|
|
|
23
23
|
const EVENT_START_AFTER_HEADROOM = 3 * 1000;
|
|
24
24
|
const ETAG_CHECK_AFTER_MIN = 10;
|
|
25
25
|
|
|
26
|
-
let serviceBindingCache = null;
|
|
27
|
-
|
|
28
26
|
class EventQueueProcessorBase {
|
|
29
27
|
#eventsWithExceededTries = [];
|
|
30
28
|
#exceededTriesExceeded = [];
|
|
@@ -72,8 +70,6 @@ class EventQueueProcessorBase {
|
|
|
72
70
|
this.__txMap = {};
|
|
73
71
|
this.__txRollback = {};
|
|
74
72
|
this.__queueEntries = [];
|
|
75
|
-
|
|
76
|
-
this.#checkGlobalContextToLocalContext();
|
|
77
73
|
}
|
|
78
74
|
|
|
79
75
|
/**
|
|
@@ -283,22 +279,33 @@ class EventQueueProcessorBase {
|
|
|
283
279
|
eventSubType: this.#eventSubType,
|
|
284
280
|
});
|
|
285
281
|
const statusMap = this.commitOnEventLevel || returnMap ? {} : this.__statusMap;
|
|
286
|
-
|
|
287
|
-
queueEntryProcessingStatusTuple.forEach(([id, processingStatus]) =>
|
|
288
|
-
this.#determineAndAddEventStatusToMap(id, processingStatus, statusMap)
|
|
289
|
-
);
|
|
290
|
-
} catch (error) {
|
|
282
|
+
const errorHandler = (error) => {
|
|
291
283
|
queueEntries.forEach((queueEntry) =>
|
|
292
284
|
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error, statusMap)
|
|
293
285
|
);
|
|
294
286
|
this.logger.error(
|
|
295
287
|
"The supplied status tuple doesn't have the required structure. Setting all entries to error.",
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
288
|
+
...[
|
|
289
|
+
error,
|
|
290
|
+
{
|
|
291
|
+
eventType: this.#eventType,
|
|
292
|
+
eventSubType: this.#eventSubType,
|
|
293
|
+
},
|
|
294
|
+
].filter((a) => a)
|
|
295
|
+
);
|
|
296
|
+
};
|
|
297
|
+
|
|
298
|
+
if (!queueEntryProcessingStatusTuple) {
|
|
299
|
+
errorHandler();
|
|
300
|
+
return statusMap;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
try {
|
|
304
|
+
queueEntryProcessingStatusTuple.forEach(([id, processingStatus]) =>
|
|
305
|
+
this.#determineAndAddEventStatusToMap(id, processingStatus, statusMap)
|
|
301
306
|
);
|
|
307
|
+
} catch (error) {
|
|
308
|
+
errorHandler(error);
|
|
302
309
|
}
|
|
303
310
|
return statusMap;
|
|
304
311
|
}
|
|
@@ -537,10 +544,7 @@ class EventQueueProcessorBase {
|
|
|
537
544
|
async getQueueEntriesAndSetToInProgress() {
|
|
538
545
|
let result = [];
|
|
539
546
|
const refDateStartAfter = new Date(Date.now() + this.#config.runInterval * 1.2);
|
|
540
|
-
this.#checkGlobalContextToLocalContext();
|
|
541
547
|
await executeInNewTransaction(this.__baseContext, "eventQueue-getQueueEntriesAndSetToInProgress", async (tx) => {
|
|
542
|
-
this.#checkGlobalContextToLocalContext();
|
|
543
|
-
await this.checkTxConsistency(tx);
|
|
544
548
|
const entries = await tx.run(
|
|
545
549
|
SELECT.from(this.#config.tableNameEventQueue)
|
|
546
550
|
.forUpdate({ wait: this.#config.forUpdateTimeout })
|
|
@@ -667,57 +671,6 @@ class EventQueueProcessorBase {
|
|
|
667
671
|
return result;
|
|
668
672
|
}
|
|
669
673
|
|
|
670
|
-
async checkTxConsistency(tx) {
|
|
671
|
-
if (!this.#config.enableTxConsistencyCheck) {
|
|
672
|
-
return;
|
|
673
|
-
}
|
|
674
|
-
|
|
675
|
-
const errorHandler = (err) =>
|
|
676
|
-
this.logger.error("tx consistency check failed!", err, {
|
|
677
|
-
type: this.eventType,
|
|
678
|
-
subType: this.eventSubType,
|
|
679
|
-
txTenant: tx.context.tenant,
|
|
680
|
-
globalCdsTenant: cds.context.tenant,
|
|
681
|
-
});
|
|
682
|
-
let txSchema, serviceManagerSchema;
|
|
683
|
-
try {
|
|
684
|
-
const schemaPromise = tx.run("SELECT CURRENT_SCHEMA FROM DUMMY");
|
|
685
|
-
const [schema, serviceManagerBindings] = await Promise.allSettled([schemaPromise, this.#getServiceBindings()]);
|
|
686
|
-
if (schema.reason) {
|
|
687
|
-
errorHandler(schema.reason);
|
|
688
|
-
return;
|
|
689
|
-
}
|
|
690
|
-
if (serviceManagerBindings.reason) {
|
|
691
|
-
errorHandler(serviceManagerBindings.reason);
|
|
692
|
-
return;
|
|
693
|
-
}
|
|
694
|
-
|
|
695
|
-
txSchema = schema.value[0].CURRENT_SCHEMA;
|
|
696
|
-
serviceManagerSchema = serviceManagerBindings.value.find((t) => t.labels.tenant_id[0] === tx.context.tenant)
|
|
697
|
-
.credentials.schema;
|
|
698
|
-
} catch (err) {
|
|
699
|
-
errorHandler(err);
|
|
700
|
-
}
|
|
701
|
-
if (serviceManagerSchema && txSchema !== serviceManagerSchema) {
|
|
702
|
-
const err = EventQueueError.dbClientSchemaMismatch(tx.context.tenant, txSchema, serviceManagerSchema);
|
|
703
|
-
errorHandler(err);
|
|
704
|
-
throw err;
|
|
705
|
-
}
|
|
706
|
-
}
|
|
707
|
-
|
|
708
|
-
async #getServiceBindings() {
|
|
709
|
-
if (!(serviceBindingCache && serviceBindingCache.expireTs >= Date.now())) {
|
|
710
|
-
const mtxServiceManager = require("@sap/cds-mtxs/srv/plugins/hana/srv-mgr");
|
|
711
|
-
serviceBindingCache = {
|
|
712
|
-
expireTs: Date.now() + 10 * 60 * 1000,
|
|
713
|
-
value: mtxServiceManager.getAll().catch(() => {
|
|
714
|
-
serviceBindingCache = null;
|
|
715
|
-
}),
|
|
716
|
-
};
|
|
717
|
-
}
|
|
718
|
-
return await serviceBindingCache.value;
|
|
719
|
-
}
|
|
720
|
-
|
|
721
674
|
async #selectLastSuccessfulPeriodicTimestamp() {
|
|
722
675
|
const entry = await SELECT.one
|
|
723
676
|
.from(this.#config.tableNameEventQueue)
|
|
@@ -730,48 +683,6 @@ class EventQueueProcessorBase {
|
|
|
730
683
|
return entry.lastAttemptsTs;
|
|
731
684
|
}
|
|
732
685
|
|
|
733
|
-
#checkGlobalContextToLocalContext() {
|
|
734
|
-
if (!this.#config.enableTxConsistencyCheck) {
|
|
735
|
-
return;
|
|
736
|
-
}
|
|
737
|
-
if (this.__context.tenant !== cds.context.tenant) {
|
|
738
|
-
throw EventQueueError.globalCdsContextNotMatchingLocal(
|
|
739
|
-
JSON.stringify(
|
|
740
|
-
{
|
|
741
|
-
correlationId: cds.context.id,
|
|
742
|
-
tenantId: cds.context.tenant,
|
|
743
|
-
timestamp: cds.context.timestamp,
|
|
744
|
-
base: JSON.stringify(
|
|
745
|
-
{
|
|
746
|
-
correlationId: cds.context.context?.id,
|
|
747
|
-
tenantId: cds.context.context?.tenant,
|
|
748
|
-
timestamp: cds.context.context?.timestamp,
|
|
749
|
-
},
|
|
750
|
-
null,
|
|
751
|
-
2
|
|
752
|
-
),
|
|
753
|
-
},
|
|
754
|
-
null,
|
|
755
|
-
2
|
|
756
|
-
),
|
|
757
|
-
JSON.stringify(
|
|
758
|
-
{
|
|
759
|
-
correlationId: this.__context.id,
|
|
760
|
-
tenantId: this.__context.tenant,
|
|
761
|
-
timestamp: this.__context.timestamp,
|
|
762
|
-
base: JSON.stringify({
|
|
763
|
-
correlationId: this.__context.context?.id,
|
|
764
|
-
tenantId: this.__context.context?.tenant,
|
|
765
|
-
timestamp: this.__context.context?.timestamp,
|
|
766
|
-
}),
|
|
767
|
-
},
|
|
768
|
-
null,
|
|
769
|
-
2
|
|
770
|
-
)
|
|
771
|
-
);
|
|
772
|
-
}
|
|
773
|
-
}
|
|
774
|
-
|
|
775
686
|
#handleDelayedEvents(delayedEvents) {
|
|
776
687
|
for (const delayedEvent of delayedEvents) {
|
|
777
688
|
this.#eventSchedulerInstance.scheduleEvent(
|
package/src/config.js
CHANGED
|
@@ -64,9 +64,9 @@ class Config {
|
|
|
64
64
|
#thresholdLoggingEventProcessing;
|
|
65
65
|
#useAsCAPOutbox;
|
|
66
66
|
#userId;
|
|
67
|
-
#enableTxConsistencyCheck;
|
|
68
67
|
#cleanupLocksAndEventsForDev;
|
|
69
68
|
#redisOptions;
|
|
69
|
+
#insertEventsBeforeCommit;
|
|
70
70
|
static #instance;
|
|
71
71
|
constructor() {
|
|
72
72
|
this.#logger = cds.log(COMPONENT_NAME);
|
|
@@ -479,14 +479,6 @@ class Config {
|
|
|
479
479
|
return this.#userId;
|
|
480
480
|
}
|
|
481
481
|
|
|
482
|
-
set enableTxConsistencyCheck(value) {
|
|
483
|
-
this.#enableTxConsistencyCheck = value;
|
|
484
|
-
}
|
|
485
|
-
|
|
486
|
-
get enableTxConsistencyCheck() {
|
|
487
|
-
return this.#enableTxConsistencyCheck;
|
|
488
|
-
}
|
|
489
|
-
|
|
490
482
|
set cleanupLocksAndEventsForDev(value) {
|
|
491
483
|
this.#cleanupLocksAndEventsForDev = value;
|
|
492
484
|
}
|
|
@@ -503,6 +495,14 @@ class Config {
|
|
|
503
495
|
return this.#redisOptions;
|
|
504
496
|
}
|
|
505
497
|
|
|
498
|
+
set insertEventsBeforeCommit(value) {
|
|
499
|
+
this.#insertEventsBeforeCommit = value;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
get insertEventsBeforeCommit() {
|
|
503
|
+
return this.#insertEventsBeforeCommit;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
506
|
get isMultiTenancy() {
|
|
507
507
|
return !!cds.requires.multitenancy;
|
|
508
508
|
}
|
package/src/dbHandler.js
CHANGED
|
@@ -6,8 +6,17 @@ const { broadcastEvent } = require("./redis/redisPub");
|
|
|
6
6
|
const config = require("./config");
|
|
7
7
|
|
|
8
8
|
const COMPONENT_NAME = "/eventQueue/dbHandler";
|
|
9
|
+
const registeredHandlers = {
|
|
10
|
+
eventQueueDbHandler: false,
|
|
11
|
+
beforeDbHandler: false,
|
|
12
|
+
};
|
|
9
13
|
|
|
10
14
|
const registerEventQueueDbHandler = (dbService) => {
|
|
15
|
+
if (registeredHandlers.eventQueueDbHandler) {
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
registeredHandlers.eventQueueDbHandler = true;
|
|
11
20
|
const def = dbService.model.definitions[config.tableNameEventQueue];
|
|
12
21
|
dbService.after("CREATE", def, (_, req) => {
|
|
13
22
|
if (req.tx._skipEventQueueBroadcase) {
|
|
@@ -46,6 +55,21 @@ const registerEventQueueDbHandler = (dbService) => {
|
|
|
46
55
|
});
|
|
47
56
|
};
|
|
48
57
|
|
|
58
|
+
const registerBeforeDbHandler = (dbService) => {
|
|
59
|
+
if (!config.insertEventsBeforeCommit || registeredHandlers.beforeDbHandler) {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
registeredHandlers.beforeDbHandler = true;
|
|
64
|
+
dbService.before("COMMIT", async (req) => {
|
|
65
|
+
if (req.context._eventQueueEvents?.length) {
|
|
66
|
+
await cds.tx(req).run(INSERT.into(config.tableNameEventQueue).entries(req.context._eventQueueEvents));
|
|
67
|
+
req.context._eventQueueEvents = null;
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
};
|
|
71
|
+
|
|
49
72
|
module.exports = {
|
|
50
73
|
registerEventQueueDbHandler,
|
|
74
|
+
registerBeforeDbHandler,
|
|
51
75
|
};
|
package/src/initialize.js
CHANGED
|
@@ -15,6 +15,7 @@ const redis = require("./shared/redis");
|
|
|
15
15
|
const eventQueueAsOutbox = require("./outbox/eventQueueAsOutbox");
|
|
16
16
|
const { getAllTenantIds } = require("./shared/cdsHelper");
|
|
17
17
|
const { EventProcessingStatus } = require("./constants");
|
|
18
|
+
const distributedLock = require("./shared/distributedLock");
|
|
18
19
|
|
|
19
20
|
const readFileAsync = promisify(fs.readFile);
|
|
20
21
|
|
|
@@ -32,9 +33,9 @@ const CONFIG_VARS = [
|
|
|
32
33
|
["thresholdLoggingEventProcessing", 50],
|
|
33
34
|
["useAsCAPOutbox", false],
|
|
34
35
|
["userId", null],
|
|
35
|
-
["enableTxConsistencyCheck", false],
|
|
36
36
|
["cleanupLocksAndEventsForDev", false],
|
|
37
37
|
["redisOptions", {}],
|
|
38
|
+
["insertEventsBeforeCommit", false],
|
|
38
39
|
];
|
|
39
40
|
|
|
40
41
|
const initialize = async ({
|
|
@@ -48,9 +49,9 @@ const initialize = async ({
|
|
|
48
49
|
thresholdLoggingEventProcessing,
|
|
49
50
|
useAsCAPOutbox,
|
|
50
51
|
userId,
|
|
51
|
-
enableTxConsistencyCheck,
|
|
52
52
|
cleanupLocksAndEventsForDev,
|
|
53
53
|
redisOptions,
|
|
54
|
+
insertEventsBeforeCommit,
|
|
54
55
|
} = {}) => {
|
|
55
56
|
if (config.initialized) {
|
|
56
57
|
return;
|
|
@@ -68,9 +69,9 @@ const initialize = async ({
|
|
|
68
69
|
thresholdLoggingEventProcessing,
|
|
69
70
|
useAsCAPOutbox,
|
|
70
71
|
userId,
|
|
71
|
-
enableTxConsistencyCheck,
|
|
72
72
|
cleanupLocksAndEventsForDev,
|
|
73
|
-
redisOptions
|
|
73
|
+
redisOptions,
|
|
74
|
+
insertEventsBeforeCommit
|
|
74
75
|
);
|
|
75
76
|
|
|
76
77
|
const logger = cds.log(COMPONENT);
|
|
@@ -80,6 +81,7 @@ const initialize = async ({
|
|
|
80
81
|
cds.on("connect", (service) => {
|
|
81
82
|
if (service.name === "db") {
|
|
82
83
|
config.processEventsAfterPublish && dbHandler.registerEventQueueDbHandler(service);
|
|
84
|
+
config.insertEventsBeforeCommit && dbHandler.registerBeforeDbHandler(service);
|
|
83
85
|
config.cleanupLocksAndEventsForDev && registerCleanupForDevDb().catch(() => {});
|
|
84
86
|
initFinished.then(registerEventProcessors);
|
|
85
87
|
}
|
|
@@ -169,6 +171,7 @@ const mixConfigVarsWithEnv = (...args) => {
|
|
|
169
171
|
|
|
170
172
|
const registerCdsShutdown = () => {
|
|
171
173
|
cds.on("shutdown", async () => {
|
|
174
|
+
await distributedLock.shutdownHandler();
|
|
172
175
|
await Promise.allSettled([redis.closeMainClient(), closeSubscribeClient()]);
|
|
173
176
|
});
|
|
174
177
|
};
|
|
@@ -23,6 +23,7 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
|
|
|
23
23
|
delete msg._fromSend;
|
|
24
24
|
delete msg.contextUser;
|
|
25
25
|
processContext.user = new cds.User.Privileged(userId);
|
|
26
|
+
processContext._eventQueue = { processor: this, key, queueEntries, payload };
|
|
26
27
|
await cds.unboxed(service).tx(processContext)[invocationFn](msg);
|
|
27
28
|
} catch (err) {
|
|
28
29
|
status = EventProcessingStatus.Error;
|
package/src/processEventQueue.js
CHANGED
|
@@ -96,9 +96,6 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
|
|
|
96
96
|
}
|
|
97
97
|
} catch (err) {
|
|
98
98
|
cds.log(COMPONENT_NAME).error("Processing event queue failed with unexpected error.", err, {
|
|
99
|
-
tenantId: context?.tenant,
|
|
100
|
-
tenantIdBase: baseInstance?.context?.tenant,
|
|
101
|
-
globalTenantId: cds.context?.tenant,
|
|
102
99
|
eventType,
|
|
103
100
|
eventSubType,
|
|
104
101
|
});
|
|
@@ -188,9 +185,6 @@ const processPeriodicEvent = async (context, eventTypeInstance) => {
|
|
|
188
185
|
cds.log(COMPONENT_NAME).error("Processing periodic events failed with unexpected error.", err, {
|
|
189
186
|
eventType: eventTypeInstance?.eventType,
|
|
190
187
|
eventSubType: eventTypeInstance?.eventSubType,
|
|
191
|
-
tenantId: context?.tenant,
|
|
192
|
-
tenantIdBase: eventTypeInstance?.context?.tenant,
|
|
193
|
-
globalTenantId: cds.context?.tenant,
|
|
194
188
|
});
|
|
195
189
|
} finally {
|
|
196
190
|
await eventTypeInstance?.handleReleaseLock();
|
package/src/publishEvent.js
CHANGED
|
@@ -45,10 +45,16 @@ const publishEvent = async (tx, events, skipBroadcast = false) => {
|
|
|
45
45
|
throw EventQueueError.manuelPeriodicEventInsert(type, subType);
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
48
|
+
|
|
49
|
+
if (config.insertEventsBeforeCommit) {
|
|
50
|
+
tx.context._eventQueueEvents ??= [];
|
|
51
|
+
tx.context._eventQueueEvents = tx.context._eventQueueEvents.concat(events);
|
|
52
|
+
} else {
|
|
53
|
+
tx._skipEventQueueBroadcase = skipBroadcast;
|
|
54
|
+
const result = await tx.run(INSERT.into(config.tableNameEventQueue).entries(eventsForProcessing));
|
|
55
|
+
tx._skipEventQueueBroadcase = false;
|
|
56
|
+
return result;
|
|
57
|
+
}
|
|
52
58
|
};
|
|
53
59
|
|
|
54
60
|
module.exports = {
|
package/src/redis/redisPub.js
CHANGED
|
@@ -7,7 +7,7 @@ const cds = require("@sap/cds");
|
|
|
7
7
|
const redis = require("../shared/redis");
|
|
8
8
|
const { checkLockExistsAndReturnValue } = require("../shared/distributedLock");
|
|
9
9
|
const config = require("../config");
|
|
10
|
-
const
|
|
10
|
+
const common = require("../shared/common");
|
|
11
11
|
const { runEventCombinationForTenant } = require("../runner/runnerHelper");
|
|
12
12
|
|
|
13
13
|
const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
|
|
@@ -29,13 +29,10 @@ const broadcastEvent = async (tenantId, events) => {
|
|
|
29
29
|
if (config.registerAsEventProcessor) {
|
|
30
30
|
let context = {};
|
|
31
31
|
if (tenantId) {
|
|
32
|
-
const
|
|
33
|
-
const user = new cds.User.Privileged(config.userId);
|
|
32
|
+
const user = new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
|
|
34
33
|
context = {
|
|
35
|
-
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
36
34
|
tenant: tenantId,
|
|
37
35
|
user,
|
|
38
|
-
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
39
36
|
};
|
|
40
37
|
}
|
|
41
38
|
|
package/src/redis/redisSub.js
CHANGED
|
@@ -4,8 +4,8 @@ const cds = require("@sap/cds");
|
|
|
4
4
|
|
|
5
5
|
const redis = require("../shared/redis");
|
|
6
6
|
const config = require("../config");
|
|
7
|
-
const { getSubdomainForTenantId } = require("../shared/cdsHelper");
|
|
8
7
|
const runnerHelper = require("../runner/runnerHelper");
|
|
8
|
+
const common = require("../shared/common");
|
|
9
9
|
|
|
10
10
|
const EVENT_MESSAGE_CHANNEL = "EVENT_QUEUE_MESSAGE_CHANNEL";
|
|
11
11
|
const COMPONENT_NAME = "/eventQueue/redisSub";
|
|
@@ -35,13 +35,10 @@ const _messageHandlerProcessEvents = async (messageData) => {
|
|
|
35
35
|
return;
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
const
|
|
39
|
-
const user = new cds.User.Privileged(config.userId);
|
|
38
|
+
const user = new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
|
|
40
39
|
const tenantContext = {
|
|
41
40
|
tenant: tenantId,
|
|
42
41
|
user,
|
|
43
|
-
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
44
|
-
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
45
42
|
};
|
|
46
43
|
|
|
47
44
|
if (!config.getEventConfig(type, subType)) {
|
package/src/runner/openEvents.js
CHANGED
|
@@ -31,6 +31,7 @@ const getOpenQueueEntries = async (tx) => {
|
|
|
31
31
|
for (const { type, subType } of entries) {
|
|
32
32
|
if (type.startsWith("CAP_OUTBOX")) {
|
|
33
33
|
if (cds.requires[subType]) {
|
|
34
|
+
await cds.connect.to(subType).catch(() => {});
|
|
34
35
|
result.push({ type, subType });
|
|
35
36
|
} else {
|
|
36
37
|
const service = await cds.connect.to(subType).catch(() => {});
|
package/src/runner/runner.js
CHANGED
|
@@ -9,9 +9,8 @@ const WorkerQueue = require("../shared/WorkerQueue");
|
|
|
9
9
|
const cdsHelper = require("../shared/cdsHelper");
|
|
10
10
|
const distributedLock = require("../shared/distributedLock");
|
|
11
11
|
const SetIntervalDriftSafe = require("../shared/SetIntervalDriftSafe");
|
|
12
|
-
const { getSubdomainForTenantId } = require("../shared/cdsHelper");
|
|
13
12
|
const periodicEvents = require("../periodicEvents");
|
|
14
|
-
const
|
|
13
|
+
const common = require("../shared/common");
|
|
15
14
|
const config = require("../config");
|
|
16
15
|
const redisPub = require("../redis/redisPub");
|
|
17
16
|
const openEvents = require("./openEvents");
|
|
@@ -80,7 +79,7 @@ const _multiTenancyRedis = async () => {
|
|
|
80
79
|
};
|
|
81
80
|
|
|
82
81
|
const _checkPeriodicEventUpdate = async (tenantIds) => {
|
|
83
|
-
const hash = hashStringTo32Bit(JSON.stringify(tenantIds));
|
|
82
|
+
const hash = common.hashStringTo32Bit(JSON.stringify(tenantIds));
|
|
84
83
|
if (!tenantIdHash) {
|
|
85
84
|
tenantIdHash = hash;
|
|
86
85
|
return await _multiTenancyPeriodicEvents(tenantIds).catch((err) => {
|
|
@@ -137,13 +136,10 @@ const _executeEventsAllTenants = async (tenantIds, runId) => {
|
|
|
137
136
|
const promises = [];
|
|
138
137
|
|
|
139
138
|
for (const tenantId of tenantIds) {
|
|
140
|
-
const
|
|
141
|
-
const user = new cds.User.Privileged(config.userId);
|
|
139
|
+
const user = new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
|
|
142
140
|
const tenantContext = {
|
|
143
141
|
tenant: tenantId,
|
|
144
142
|
user,
|
|
145
|
-
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
146
|
-
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
147
143
|
};
|
|
148
144
|
const events = await cds.tx(tenantContext, async (tx) => {
|
|
149
145
|
return await openEvents.getOpenQueueEntries(tx);
|
|
@@ -184,13 +180,10 @@ const _executeEventsAllTenants = async (tenantIds, runId) => {
|
|
|
184
180
|
const _executePeriodicEventsAllTenants = async (tenantIds) => {
|
|
185
181
|
for (const tenantId of tenantIds) {
|
|
186
182
|
try {
|
|
187
|
-
const
|
|
188
|
-
const user = new cds.User.Privileged(config.userId);
|
|
183
|
+
const user = new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(tenantId) });
|
|
189
184
|
const tenantContext = {
|
|
190
185
|
tenant: tenantId,
|
|
191
186
|
user,
|
|
192
|
-
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
193
|
-
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
194
187
|
};
|
|
195
188
|
await cds.tx(tenantContext, async ({ context }) => {
|
|
196
189
|
if (!config.redisEnabled) {
|
|
@@ -350,7 +343,7 @@ const _checkPeriodicEventsSingleTenant = async (context) => {
|
|
|
350
343
|
try {
|
|
351
344
|
logger.info("executing updating periodic events", {
|
|
352
345
|
tenantId: context.tenant,
|
|
353
|
-
subdomain: context.
|
|
346
|
+
subdomain: context.user?.authInfo?.getSubdomain(),
|
|
354
347
|
});
|
|
355
348
|
await periodicEvents.checkAndInsertPeriodicEvents(context);
|
|
356
349
|
} catch (err) {
|
package/src/shared/cdsHelper.js
CHANGED
|
@@ -4,8 +4,7 @@ const VError = require("verror");
|
|
|
4
4
|
const cds = require("@sap/cds");
|
|
5
5
|
|
|
6
6
|
const config = require("../config");
|
|
7
|
-
|
|
8
|
-
const subdomainCache = {};
|
|
7
|
+
const common = require("./common");
|
|
9
8
|
|
|
10
9
|
const VERROR_CLUSTER_NAME = "ExecuteInNewTransactionError";
|
|
11
10
|
const COMPONENT_NAME = "/eventQueue/cdsHelper";
|
|
@@ -24,7 +23,7 @@ async function executeInNewTransaction(context = {}, transactionTag, fn, args, {
|
|
|
24
23
|
const parameters = Array.isArray(args) ? args : [args];
|
|
25
24
|
const logger = cds.log(COMPONENT_NAME);
|
|
26
25
|
try {
|
|
27
|
-
const user = new cds.User.Privileged(config.userId);
|
|
26
|
+
const user = new cds.User.Privileged({ id: config.userId, authInfo: await common.getAuthInfo(context.tenant) });
|
|
28
27
|
if (cds.db.kind === "hana") {
|
|
29
28
|
await cds.tx(
|
|
30
29
|
{
|
|
@@ -33,7 +32,6 @@ async function executeInNewTransaction(context = {}, transactionTag, fn, args, {
|
|
|
33
32
|
locale: context.locale,
|
|
34
33
|
user,
|
|
35
34
|
headers: context.headers,
|
|
36
|
-
http: context.http,
|
|
37
35
|
},
|
|
38
36
|
async (tx) => {
|
|
39
37
|
tx.context._ = context._ ?? {};
|
|
@@ -51,7 +49,6 @@ async function executeInNewTransaction(context = {}, transactionTag, fn, args, {
|
|
|
51
49
|
locale: context.locale,
|
|
52
50
|
user,
|
|
53
51
|
headers: context.headers,
|
|
54
|
-
http: context.http,
|
|
55
52
|
},
|
|
56
53
|
async (tx) => fn(tx, ...parameters)
|
|
57
54
|
);
|
|
@@ -102,29 +99,6 @@ class TriggerRollback extends VError {
|
|
|
102
99
|
}
|
|
103
100
|
}
|
|
104
101
|
|
|
105
|
-
const getSubdomainForTenantId = async (tenantId) => {
|
|
106
|
-
if (!config.isMultiTenancy) {
|
|
107
|
-
return null;
|
|
108
|
-
}
|
|
109
|
-
if (subdomainCache[tenantId]) {
|
|
110
|
-
return await subdomainCache[tenantId];
|
|
111
|
-
}
|
|
112
|
-
subdomainCache[tenantId] = new Promise((resolve) => {
|
|
113
|
-
cds.connect
|
|
114
|
-
.to("cds.xt.SaasProvisioningService")
|
|
115
|
-
.then((ssp) => {
|
|
116
|
-
ssp
|
|
117
|
-
.get("/tenant", { subscribedTenantId: tenantId })
|
|
118
|
-
.then((response) => {
|
|
119
|
-
resolve(response.subscribedSubdomain);
|
|
120
|
-
})
|
|
121
|
-
.catch(() => resolve(null));
|
|
122
|
-
})
|
|
123
|
-
.catch(() => resolve(null));
|
|
124
|
-
});
|
|
125
|
-
return await subdomainCache[tenantId];
|
|
126
|
-
};
|
|
127
|
-
|
|
128
102
|
const getAllTenantIds = async () => {
|
|
129
103
|
if (!config.isMultiTenancy) {
|
|
130
104
|
return null;
|
|
@@ -150,6 +124,5 @@ const isFakeTenant = (tenantId) => /00000000-0000-4000-8000-\d{12}/.test(tenantI
|
|
|
150
124
|
module.exports = {
|
|
151
125
|
executeInNewTransaction,
|
|
152
126
|
TriggerRollback,
|
|
153
|
-
getSubdomainForTenantId,
|
|
154
127
|
getAllTenantIds,
|
|
155
128
|
};
|
package/src/shared/common.js
CHANGED
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
const crypto = require("crypto");
|
|
4
|
+
const { promisify } = require("util");
|
|
5
|
+
|
|
6
|
+
const cds = require("@sap/cds");
|
|
7
|
+
const xssec = require("@sap/xssec");
|
|
8
|
+
|
|
9
|
+
const getAuthTokenAsync = promisify(xssec.requests.requestClientCredentialsToken);
|
|
10
|
+
const getCreateSecurityContextAsync = promisify(xssec.createSecurityContext);
|
|
11
|
+
|
|
12
|
+
let authInfoCache = {};
|
|
13
|
+
const MARGIN_AUTH_INFO_EXPIRY = 60 * 1000;
|
|
14
|
+
const COMPONENT_NAME = "/eventQueue/common";
|
|
15
|
+
|
|
4
16
|
const arrayToFlatMap = (array, key = "ID") => {
|
|
5
17
|
return array.reduce((result, element) => {
|
|
6
18
|
result[element[key]] = element;
|
|
@@ -61,4 +73,43 @@ const processChunkedSync = (inputs, chunkSize, chunkHandler) => {
|
|
|
61
73
|
|
|
62
74
|
const hashStringTo32Bit = (value) => crypto.createHash("sha256").update(String(value)).digest("base64").slice(0, 32);
|
|
63
75
|
|
|
64
|
-
|
|
76
|
+
const _getNewAuthInfo = async (tenantId) => {
|
|
77
|
+
try {
|
|
78
|
+
const token = await getAuthTokenAsync(null, cds.requires.auth.credentials, null, tenantId);
|
|
79
|
+
const authInfo = await getCreateSecurityContextAsync(token, cds.requires.auth.credentials);
|
|
80
|
+
authInfoCache[tenantId].expireTs = authInfo.getExpirationDate().getTime() - MARGIN_AUTH_INFO_EXPIRY;
|
|
81
|
+
return authInfo;
|
|
82
|
+
} catch (err) {
|
|
83
|
+
authInfoCache[tenantId] = null;
|
|
84
|
+
cds.log(COMPONENT_NAME).warn("failed to request authInfo", err);
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const getAuthInfo = async (tenantId) => {
|
|
89
|
+
if (!cds.requires?.auth?.credentials) {
|
|
90
|
+
return null; // no credentials not authInfo
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// not existing or existing but expired
|
|
94
|
+
if (
|
|
95
|
+
!authInfoCache[tenantId] ||
|
|
96
|
+
(authInfoCache[tenantId] && authInfoCache[tenantId].expireTs && Date.now() > authInfoCache[tenantId].expireTs)
|
|
97
|
+
) {
|
|
98
|
+
authInfoCache[tenantId] ??= {};
|
|
99
|
+
authInfoCache[tenantId].value = _getNewAuthInfo(tenantId);
|
|
100
|
+
authInfoCache[tenantId].expireTs = null;
|
|
101
|
+
}
|
|
102
|
+
return await authInfoCache[tenantId].value;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
module.exports = {
|
|
106
|
+
arrayToFlatMap,
|
|
107
|
+
limiter,
|
|
108
|
+
isValidDate,
|
|
109
|
+
processChunkedSync,
|
|
110
|
+
hashStringTo32Bit,
|
|
111
|
+
getAuthInfo,
|
|
112
|
+
__: {
|
|
113
|
+
clearAuthInfoCache: () => (authInfoCache = {}),
|
|
114
|
+
},
|
|
115
|
+
};
|
|
@@ -6,6 +6,10 @@ const cdsHelper = require("./cdsHelper");
|
|
|
6
6
|
|
|
7
7
|
const KEY_PREFIX = "EVENT_QUEUE";
|
|
8
8
|
|
|
9
|
+
const existingLocks = {};
|
|
10
|
+
|
|
11
|
+
const REDIS_COMMAND_OK = "OK";
|
|
12
|
+
|
|
9
13
|
const acquireLock = async (context, key, { tenantScoped = true, expiryTime = config.globalTxTimeout } = {}) => {
|
|
10
14
|
const fullKey = _generateKey(context, tenantScoped, key);
|
|
11
15
|
if (config.redisEnabled) {
|
|
@@ -59,7 +63,11 @@ const _acquireLockRedis = async (context, fullKey, expiryTime, { value = "true",
|
|
|
59
63
|
PX: expiryTime,
|
|
60
64
|
...(overrideValue ? null : { NX: true }),
|
|
61
65
|
});
|
|
62
|
-
|
|
66
|
+
const isOk = result === REDIS_COMMAND_OK;
|
|
67
|
+
if (isOk) {
|
|
68
|
+
existingLocks[fullKey] = 1;
|
|
69
|
+
}
|
|
70
|
+
return isOk;
|
|
63
71
|
};
|
|
64
72
|
|
|
65
73
|
const _checkLockExistsRedis = async (context, fullKey) => {
|
|
@@ -78,6 +86,7 @@ const _checkLockExistsDb = async (context, fullKey) => {
|
|
|
78
86
|
const _releaseLockRedis = async (context, fullKey) => {
|
|
79
87
|
const client = await redis.createMainClientAndConnect(config.redisOptions);
|
|
80
88
|
await client.del(fullKey);
|
|
89
|
+
delete existingLocks[fullKey];
|
|
81
90
|
};
|
|
82
91
|
|
|
83
92
|
const _releaseLockDb = async (context, fullKey) => {
|
|
@@ -133,9 +142,14 @@ const _generateKey = (context, tenantScoped, key) => {
|
|
|
133
142
|
return `${KEY_PREFIX}_${keyParts.join("##")}`;
|
|
134
143
|
};
|
|
135
144
|
|
|
145
|
+
const shutdownHandler = async () => {
|
|
146
|
+
await Promise.allSettled(Object.keys(existingLocks).map((key) => _releaseLockRedis(null, key)));
|
|
147
|
+
};
|
|
148
|
+
|
|
136
149
|
module.exports = {
|
|
137
150
|
acquireLock,
|
|
138
151
|
releaseLock,
|
|
139
152
|
checkLockExistsAndReturnValue,
|
|
140
153
|
setValueWithExpire,
|
|
154
|
+
shutdownHandler,
|
|
141
155
|
};
|