@cap-js-community/event-queue 1.11.0-beta.5 → 2.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/Event.cds +1 -0
- package/package.json +18 -12
- package/src/EventQueueProcessorBase.js +18 -3
- package/src/config.js +68 -208
- package/src/dbHandler.js +7 -4
- package/src/index.d.ts +10 -9
- package/src/initialize.js +8 -23
- package/src/outbox/eventQueueAsOutbox.js +19 -9
- package/src/periodicEvents.js +13 -12
- package/src/processEventQueue.js +7 -11
- package/src/publishEvent.js +15 -6
- package/src/redis/redisPub.js +10 -8
- package/src/redis/redisSub.js +16 -5
- package/src/runner/openEvents.js +12 -10
- package/src/runner/runner.js +26 -15
- package/src/runner/runnerHelper.js +10 -4
- package/src/shared/cdsHelper.js +0 -31
- package/src/shared/common.js +7 -2
- package/src/shared/distributedLock.js +23 -13
- package/src/shared/eventScheduler.js +3 -3
- package/src/shared/redis/client.js +46 -0
- package/src/shared/redis/index.js +45 -0
- package/srv/service/admin-service.cds +0 -8
- package/srv/service/admin-service.js +1 -10
- package/src/shared/redis.js +0 -199
package/src/index.d.ts
CHANGED
|
@@ -169,7 +169,12 @@ export function publishEvent(
|
|
|
169
169
|
}
|
|
170
170
|
): Promise<any>;
|
|
171
171
|
|
|
172
|
-
export function processEventQueue(
|
|
172
|
+
export function processEventQueue(
|
|
173
|
+
context: cds.EventContext,
|
|
174
|
+
eventType: string,
|
|
175
|
+
eventSubType: string,
|
|
176
|
+
namespace: string
|
|
177
|
+
): Promise<any>;
|
|
173
178
|
|
|
174
179
|
export function triggerEventProcessingRedis(
|
|
175
180
|
tenantId: string,
|
|
@@ -180,20 +185,16 @@ export function triggerEventProcessingRedis(
|
|
|
180
185
|
declare class Config {
|
|
181
186
|
constructor();
|
|
182
187
|
|
|
183
|
-
getEventConfig(type:
|
|
188
|
+
getEventConfig(type: string, subType: string, namespace: string): any;
|
|
184
189
|
isCapOutboxEvent(type: any): boolean;
|
|
185
|
-
hasEventAfterCommitFlag(type: any, subType: any): any;
|
|
186
|
-
shouldBeProcessedInThisApplication(type: any, subType: any): boolean;
|
|
190
|
+
hasEventAfterCommitFlag(type: any, subType: any, namespace: string): any;
|
|
191
|
+
shouldBeProcessedInThisApplication(type: any, subType: any, namespace: string): boolean;
|
|
187
192
|
checkRedisEnabled(): any;
|
|
188
193
|
attachConfigChangeHandler(): void;
|
|
189
194
|
attachRedisUnsubscribeHandler(): void;
|
|
190
195
|
executeUnsubscribeHandlers(tenantId: any): void;
|
|
191
|
-
handleUnsubscribe(tenantId: any): void;
|
|
192
196
|
attachUnsubscribeHandler(cb: any): void;
|
|
193
197
|
publishConfigChange(key: any, value: any): void;
|
|
194
|
-
blockEvent(type: any, subType: any, isPeriodic: any, tenant?: string): void;
|
|
195
|
-
clearPeriodicEventBlockList(): void;
|
|
196
|
-
unblockEvent(type: any, subType: any, isPeriodic: any, tenant?: string): void;
|
|
197
198
|
addCAPOutboxEventBase(serviceName: any, config: any): void;
|
|
198
199
|
isEventBlocked(type: any, subType: any, isPeriodicEvent: any, tenant: any): any;
|
|
199
200
|
set isEventQueueActive(value: boolean);
|
|
@@ -205,7 +206,7 @@ declare class Config {
|
|
|
205
206
|
isTenantUnsubscribed(tenantId: any): any;
|
|
206
207
|
get events(): any;
|
|
207
208
|
get periodicEvents(): any;
|
|
208
|
-
isPeriodicEvent(type: any, subType: any): any;
|
|
209
|
+
isPeriodicEvent(type: any, subType: any, namespace: string): any;
|
|
209
210
|
get allEvents(): any;
|
|
210
211
|
set forUpdateTimeout(value: number);
|
|
211
212
|
get forUpdateTimeout(): number;
|
package/src/initialize.js
CHANGED
|
@@ -22,7 +22,6 @@ const readFileAsync = promisify(fs.readFile);
|
|
|
22
22
|
|
|
23
23
|
const VERROR_CLUSTER_NAME = "EventQueueInitialization";
|
|
24
24
|
const COMPONENT = "eventQueue/initialize";
|
|
25
|
-
const TIMEOUT_SHUTDOWN = 2500;
|
|
26
25
|
|
|
27
26
|
const CONFIG_VARS = [
|
|
28
27
|
["configFilePath", null],
|
|
@@ -48,6 +47,8 @@ const CONFIG_VARS = [
|
|
|
48
47
|
["crashOnRedisUnavailable", false],
|
|
49
48
|
["enableAdminService", false],
|
|
50
49
|
["disableProcessingOfSuspendedTenants", true],
|
|
50
|
+
["namespace", "default"],
|
|
51
|
+
["processingNamespaces", ["default"]],
|
|
51
52
|
];
|
|
52
53
|
|
|
53
54
|
/**
|
|
@@ -74,6 +75,8 @@ const CONFIG_VARS = [
|
|
|
74
75
|
* @param {string} [options.randomOffsetPeriodicEvents=null] - Default random offset for periodic events.
|
|
75
76
|
* @param {string} [options.publishEventBlockList=true] - If redis is available event blocklist is distributed to all application instances
|
|
76
77
|
* @param {string} [options.crashOnRedisUnavailable=true] - If enabled an error is thrown if the redis connection check is not successful
|
|
78
|
+
* @param {string} [options.namespace=default] - Default namespace in which events are published
|
|
79
|
+
* @param {string} [options.processingNamespaces=[default]] - Namespaces which the application processes
|
|
77
80
|
*/
|
|
78
81
|
const initialize = async (options = {}) => {
|
|
79
82
|
if (config.initialized) {
|
|
@@ -101,7 +104,7 @@ const initialize = async (options = {}) => {
|
|
|
101
104
|
}
|
|
102
105
|
});
|
|
103
106
|
if (redisEnabled) {
|
|
104
|
-
config.redisEnabled = await redis.connectionCheck(
|
|
107
|
+
config.redisEnabled = await redis.connectionCheck();
|
|
105
108
|
if (!config.redisEnabled && config.crashOnRedisUnavailable) {
|
|
106
109
|
throw EventQueueError.redisConnectionFailure();
|
|
107
110
|
}
|
|
@@ -148,7 +151,7 @@ const readConfigFromFile = async (configFilepath) => {
|
|
|
148
151
|
|
|
149
152
|
const registerEventProcessors = () => {
|
|
150
153
|
_registerUnsubscribe();
|
|
151
|
-
config.redisEnabled &&
|
|
154
|
+
config.redisEnabled && redis.attachRedisUnsubscribeHandler();
|
|
152
155
|
|
|
153
156
|
if (!config.registerAsEventProcessor) {
|
|
154
157
|
return;
|
|
@@ -158,7 +161,6 @@ const registerEventProcessors = () => {
|
|
|
158
161
|
|
|
159
162
|
if (config.redisEnabled) {
|
|
160
163
|
redisSub.initEventQueueRedisSubscribe();
|
|
161
|
-
config.attachConfigChangeHandler();
|
|
162
164
|
if (config.isMultiTenancy) {
|
|
163
165
|
runner.multiTenancyRedis().catch(errorHandler);
|
|
164
166
|
} else {
|
|
@@ -203,24 +205,7 @@ const mixConfigVarsWithEnv = (options) => {
|
|
|
203
205
|
};
|
|
204
206
|
|
|
205
207
|
const registerCdsShutdown = () => {
|
|
206
|
-
|
|
207
|
-
return await new Promise((resolve) => {
|
|
208
|
-
let timeoutRef;
|
|
209
|
-
timeoutRef = setTimeout(() => {
|
|
210
|
-
clearTimeout(timeoutRef);
|
|
211
|
-
cds.log(COMPONENT).info("shutdown timeout reached - some locks might not have been released!");
|
|
212
|
-
resolve();
|
|
213
|
-
}, TIMEOUT_SHUTDOWN);
|
|
214
|
-
distributedLock.shutdownHandler().then(() => {
|
|
215
|
-
Promise.allSettled(
|
|
216
|
-
config.redisEnabled ? [redis.closeMainClient(), redis.closeSubscribeClient()] : [Promise.resolve()]
|
|
217
|
-
).then((result) => {
|
|
218
|
-
clearTimeout(timeoutRef);
|
|
219
|
-
resolve(result);
|
|
220
|
-
});
|
|
221
|
-
});
|
|
222
|
-
});
|
|
223
|
-
});
|
|
208
|
+
redis.registerShutdownHandler(distributedLock.shutdownHandler);
|
|
224
209
|
};
|
|
225
210
|
|
|
226
211
|
const registerCleanupForDevDb = async () => {
|
|
@@ -250,7 +235,7 @@ const _registerUnsubscribe = () => {
|
|
|
250
235
|
});
|
|
251
236
|
ds.after("unsubscribe", async (_, req) => {
|
|
252
237
|
const { tenant } = req.data;
|
|
253
|
-
|
|
238
|
+
redis.handleUnsubscribe(tenant);
|
|
254
239
|
});
|
|
255
240
|
})
|
|
256
241
|
.catch(
|
|
@@ -9,7 +9,7 @@ const OUTBOXED = Symbol("outboxed");
|
|
|
9
9
|
const UNBOXED = Symbol("unboxed");
|
|
10
10
|
const CDS_EVENT_TYPE = "CAP_OUTBOX";
|
|
11
11
|
const COMPONENT_NAME = "/eventQueue/eventQueueAsOutbox";
|
|
12
|
-
const EVENT_QUEUE_SPECIFIC_FIELDS = ["startAfter", "referenceEntity", "referenceEntityKey"];
|
|
12
|
+
const EVENT_QUEUE_SPECIFIC_FIELDS = ["startAfter", "referenceEntity", "referenceEntityKey", "namespace"];
|
|
13
13
|
|
|
14
14
|
function outboxed(srv, customOpts) {
|
|
15
15
|
if (!(new.target || customOpts)) {
|
|
@@ -23,7 +23,9 @@ function outboxed(srv, customOpts) {
|
|
|
23
23
|
let outboxOpts = Object.assign(
|
|
24
24
|
{},
|
|
25
25
|
(typeof cds.requires.outbox === "object" && cds.requires.outbox) || {},
|
|
26
|
+
(typeof cds.requires.queue === "object" && cds.requires.queue) || {},
|
|
26
27
|
(typeof srv.options?.outbox === "object" && srv.options.outbox) || {},
|
|
28
|
+
(typeof srv.options?.queued === "object" && srv.options.queued) || {},
|
|
27
29
|
customOpts || {}
|
|
28
30
|
);
|
|
29
31
|
|
|
@@ -43,7 +45,9 @@ function outboxed(srv, customOpts) {
|
|
|
43
45
|
outboxOpts = Object.assign(
|
|
44
46
|
{},
|
|
45
47
|
(typeof cds.requires.outbox === "object" && cds.requires.outbox) || {},
|
|
48
|
+
(typeof cds.requires.queue === "object" && cds.requires.queue) || {},
|
|
46
49
|
(typeof srv.options?.outbox === "object" && srv.options.outbox) || {},
|
|
50
|
+
(typeof srv.options?.queued === "object" && srv.options.queued) || {},
|
|
47
51
|
customOpts || {}
|
|
48
52
|
);
|
|
49
53
|
config.addCAPOutboxEventBase(srv.name, outboxOpts);
|
|
@@ -52,8 +56,9 @@ function outboxed(srv, customOpts) {
|
|
|
52
56
|
outboxOpts = config.addCAPOutboxEventSpecificAction(srv.name, req.event);
|
|
53
57
|
}
|
|
54
58
|
|
|
59
|
+
const namespace = (specificSettings ?? outboxOpts).namespace ?? config.namespace;
|
|
55
60
|
if (["persistent-outbox", "persistent-queue"].includes(outboxOpts.kind)) {
|
|
56
|
-
await _mapToEventAndPublish(context, srv.name, req, !!specificSettings);
|
|
61
|
+
await _mapToEventAndPublish(context, srv.name, req, !!specificSettings, namespace);
|
|
57
62
|
return;
|
|
58
63
|
}
|
|
59
64
|
context.on("succeeded", async () => {
|
|
@@ -79,7 +84,7 @@ function unboxed(srv) {
|
|
|
79
84
|
return srv[UNBOXED] || srv;
|
|
80
85
|
}
|
|
81
86
|
|
|
82
|
-
const _mapToEventAndPublish = async (context, name, req, actionSpecific) => {
|
|
87
|
+
const _mapToEventAndPublish = async (context, name, req, actionSpecific, namespace) => {
|
|
83
88
|
const eventQueueSpecificValues = {};
|
|
84
89
|
for (const header in req.headers ?? {}) {
|
|
85
90
|
for (const field of EVENT_QUEUE_SPECIFIC_FIELDS) {
|
|
@@ -100,12 +105,17 @@ const _mapToEventAndPublish = async (context, name, req, actionSpecific) => {
|
|
|
100
105
|
...(req.query && { query: req.query }),
|
|
101
106
|
};
|
|
102
107
|
|
|
103
|
-
await publishEvent(
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
108
|
+
await publishEvent(
|
|
109
|
+
cds.tx(context),
|
|
110
|
+
{
|
|
111
|
+
type: CDS_EVENT_TYPE,
|
|
112
|
+
subType: actionSpecific ? [name, req.event].join(".") : name,
|
|
113
|
+
payload: JSON.stringify(event),
|
|
114
|
+
namespace: eventQueueSpecificValues.namespace ?? namespace,
|
|
115
|
+
...eventQueueSpecificValues,
|
|
116
|
+
},
|
|
117
|
+
{ allowNotExistingConfiguration: !!eventQueueSpecificValues.namespace }
|
|
118
|
+
);
|
|
109
119
|
};
|
|
110
120
|
|
|
111
121
|
const isUnrecoverable = (service, error) => {
|
package/src/periodicEvents.js
CHANGED
|
@@ -21,11 +21,11 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
21
21
|
const tx = cds.tx(context);
|
|
22
22
|
const baseCqn = SELECT.from(eventConfig.tableNameEventQueue)
|
|
23
23
|
.where([
|
|
24
|
-
{ list: [{ ref: ["type"] }, { ref: ["subType"] }] },
|
|
24
|
+
{ list: [{ ref: ["type"] }, { ref: ["subType"] }, { ref: ["namespace"] }] },
|
|
25
25
|
"IN",
|
|
26
26
|
{
|
|
27
27
|
list: eventConfig.periodicEvents.map((periodicEvent) => ({
|
|
28
|
-
list: [{ val: periodicEvent.type }, { val: periodicEvent.subType }],
|
|
28
|
+
list: [{ val: periodicEvent.type }, { val: periodicEvent.subType }, { val: periodicEvent.namespace }],
|
|
29
29
|
})),
|
|
30
30
|
},
|
|
31
31
|
"AND",
|
|
@@ -35,8 +35,8 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
35
35
|
list: [{ val: EventProcessingStatus.Open }, { val: EventProcessingStatus.InProgress }],
|
|
36
36
|
},
|
|
37
37
|
])
|
|
38
|
-
.groupBy("type", "subType", "createdAt")
|
|
39
|
-
.columns(["type", "subType", "createdAt", "max(startAfter) as startAfter"]);
|
|
38
|
+
.groupBy("type", "subType", "createdAt", "namespace")
|
|
39
|
+
.columns(["type", "subType", "createdAt", "namespace", "max(startAfter) as startAfter"]);
|
|
40
40
|
const currentPeriodEvents = await tx.run(baseCqn);
|
|
41
41
|
currentPeriodEvents.length &&
|
|
42
42
|
(await tx.run(_addWhere(SELECT.from(eventConfig.tableNameEventQueue).columns("ID"), currentPeriodEvents)));
|
|
@@ -56,7 +56,7 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
56
56
|
(result, event) => {
|
|
57
57
|
const existingEvent = exitingEventMap[_generateKey(event)];
|
|
58
58
|
if (existingEvent) {
|
|
59
|
-
const config = eventConfig.getEventConfig(existingEvent.type, existingEvent.subType);
|
|
59
|
+
const config = eventConfig.getEventConfig(existingEvent.type, existingEvent.subType, existingEvent.namespace);
|
|
60
60
|
if (config.cron) {
|
|
61
61
|
result.existingEventsCron.push(exitingEventMap[_generateKey(event)]);
|
|
62
62
|
} else {
|
|
@@ -110,7 +110,7 @@ const _addWhere = (cqnBase, events) => {
|
|
|
110
110
|
|
|
111
111
|
const _determineChangedInterval = (existingEvents, currentDate) => {
|
|
112
112
|
return existingEvents.filter((existingEvent) => {
|
|
113
|
-
const config = eventConfig.getEventConfig(existingEvent.type, existingEvent.subType);
|
|
113
|
+
const config = eventConfig.getEventConfig(existingEvent.type, existingEvent.subType, existingEvent.namespace);
|
|
114
114
|
const eventStartAfter = new Date(existingEvent.startAfter);
|
|
115
115
|
// check if too far in future
|
|
116
116
|
const dueInWithNewInterval = new Date(currentDate.getTime() + config.interval * 1000);
|
|
@@ -120,7 +120,7 @@ const _determineChangedInterval = (existingEvents, currentDate) => {
|
|
|
120
120
|
|
|
121
121
|
const _determineChangedCron = (existingEventsCron) => {
|
|
122
122
|
return existingEventsCron.filter((event) => {
|
|
123
|
-
const config = eventConfig.getEventConfig(event.type, event.subType);
|
|
123
|
+
const config = eventConfig.getEventConfig(event.type, event.subType, event.namespace);
|
|
124
124
|
const eventStartAfter = new Date(event.startAfter);
|
|
125
125
|
const eventCreatedAt = new Date(event.createdAt);
|
|
126
126
|
const randomOffset = config.randomOffset ?? eventConfig.randomOffsetPeriodicEvents ?? 0;
|
|
@@ -141,9 +141,9 @@ const _insertPeriodEvents = async (tx, events, now) => {
|
|
|
141
141
|
const chunks = Math.ceil(events.length / CHUNK_SIZE_INSERT_PERIODIC_EVENTS);
|
|
142
142
|
const logger = cds.log(COMPONENT_NAME);
|
|
143
143
|
const eventsToBeInserted = events.map((event) => {
|
|
144
|
-
const base = { type: event.type, subType: event.subType };
|
|
144
|
+
const base = { type: event.type, subType: event.subType, namespace: event.namespace };
|
|
145
145
|
let startTime = now;
|
|
146
|
-
const config = eventConfig.getEventConfig(event.type, event.subType);
|
|
146
|
+
const config = eventConfig.getEventConfig(event.type, event.subType, event.namespace);
|
|
147
147
|
if (config.cron) {
|
|
148
148
|
startTime = CronExpressionParser.parse(config.cron, {
|
|
149
149
|
currentDate: now,
|
|
@@ -156,11 +156,12 @@ const _insertPeriodEvents = async (tx, events, now) => {
|
|
|
156
156
|
|
|
157
157
|
processChunkedSync(eventsToBeInserted, CHUNK_SIZE_INSERT_PERIODIC_EVENTS, (chunk) => {
|
|
158
158
|
logger.info(`${counter}/${chunks} | inserting chunk of changed or new periodic events`, {
|
|
159
|
-
events: chunk.map(({ type, subType, startAfter }) => {
|
|
160
|
-
const { interval, cron } = eventConfig.getEventConfig(type, subType);
|
|
159
|
+
events: chunk.map(({ namespace, type, subType, startAfter }) => {
|
|
160
|
+
const { interval, cron } = eventConfig.getEventConfig(type, subType, namespace);
|
|
161
161
|
return {
|
|
162
162
|
type,
|
|
163
163
|
subType,
|
|
164
|
+
namespace,
|
|
164
165
|
...(startAfter && { startAfter }),
|
|
165
166
|
...(interval && { interval }),
|
|
166
167
|
...(cron && { cron }),
|
|
@@ -175,7 +176,7 @@ const _insertPeriodEvents = async (tx, events, now) => {
|
|
|
175
176
|
tx._skipEventQueueBroadcase = false;
|
|
176
177
|
};
|
|
177
178
|
|
|
178
|
-
const _generateKey = ({ type, subType }) => [type, subType].join("##");
|
|
179
|
+
const _generateKey = ({ type, subType, namespace }) => [namespace, type, subType].join("##");
|
|
179
180
|
|
|
180
181
|
module.exports = {
|
|
181
182
|
checkAndInsertPeriodicEvents,
|
package/src/processEventQueue.js
CHANGED
|
@@ -13,14 +13,14 @@ const { trace } = require("./shared/openTelemetry");
|
|
|
13
13
|
|
|
14
14
|
const COMPONENT_NAME = "/eventQueue/processEventQueue";
|
|
15
15
|
|
|
16
|
-
const processEventQueue = async (context, eventType, eventSubType) => {
|
|
16
|
+
const processEventQueue = async (context, eventType, eventSubType, namespace = config.namespace) => {
|
|
17
17
|
let iterationCounter = 0;
|
|
18
18
|
let shouldContinue = true;
|
|
19
19
|
let baseInstance;
|
|
20
20
|
let startTime = new Date();
|
|
21
21
|
try {
|
|
22
22
|
let eventTypeInstance;
|
|
23
|
-
const eventConfig = config.getEventConfig(eventType, eventSubType);
|
|
23
|
+
const eventConfig = config.getEventConfig(eventType, eventSubType, namespace);
|
|
24
24
|
const [err, EventTypeClass] = await resilientRequire(eventConfig);
|
|
25
25
|
if (!eventConfig || err || !(typeof EventTypeClass.constructor === "function")) {
|
|
26
26
|
cds.log(COMPONENT_NAME).error("No Implementation found in the provided configuration file.", {
|
|
@@ -287,14 +287,6 @@ const _checkEventIsBlocked = async (baseInstance) => {
|
|
|
287
287
|
subType: baseInstance.eventSubType,
|
|
288
288
|
});
|
|
289
289
|
}
|
|
290
|
-
} else {
|
|
291
|
-
// TODO: we should be able to get rid of baseInstance.isPeriodicEvent with rawEventType
|
|
292
|
-
eventBlocked = config.isEventBlocked(
|
|
293
|
-
baseInstance.eventType,
|
|
294
|
-
baseInstance.eventSubType,
|
|
295
|
-
baseInstance.isPeriodicEvent,
|
|
296
|
-
baseInstance.context.tenant
|
|
297
|
-
);
|
|
298
290
|
}
|
|
299
291
|
|
|
300
292
|
if (!eventBlocked) {
|
|
@@ -310,7 +302,11 @@ const _checkEventIsBlocked = async (baseInstance) => {
|
|
|
310
302
|
}
|
|
311
303
|
|
|
312
304
|
if (!eventBlocked) {
|
|
313
|
-
eventBlocked = !config.shouldBeProcessedInThisApplication(
|
|
305
|
+
eventBlocked = !config.shouldBeProcessedInThisApplication(
|
|
306
|
+
baseInstance.rawEventType,
|
|
307
|
+
baseInstance.eventSubType,
|
|
308
|
+
baseInstance.namespace
|
|
309
|
+
);
|
|
314
310
|
}
|
|
315
311
|
|
|
316
312
|
return eventBlocked;
|
package/src/publishEvent.js
CHANGED
|
@@ -35,23 +35,28 @@ const openTelemetry = require("./shared/openTelemetry");
|
|
|
35
35
|
const publishEvent = async (
|
|
36
36
|
tx,
|
|
37
37
|
events,
|
|
38
|
-
{
|
|
38
|
+
{
|
|
39
|
+
skipBroadcast = false,
|
|
40
|
+
skipInsertEventsBeforeCommit = false,
|
|
41
|
+
addTraceContext = true,
|
|
42
|
+
allowNotExistingConfiguration = false,
|
|
43
|
+
} = {}
|
|
39
44
|
) => {
|
|
40
45
|
if (!config.initialized) {
|
|
41
46
|
throw EventQueueError.notInitialized();
|
|
42
47
|
}
|
|
43
48
|
const eventsForProcessing = Array.isArray(events) ? events : [events];
|
|
44
49
|
for (const event of eventsForProcessing) {
|
|
45
|
-
const { type, subType, startAfter } = event;
|
|
46
|
-
const eventConfig = config.getEventConfig(type, subType);
|
|
47
|
-
if (!eventConfig) {
|
|
50
|
+
const { type, subType, startAfter, namespace } = event;
|
|
51
|
+
const eventConfig = config.getEventConfig(type, subType, namespace);
|
|
52
|
+
if (!eventConfig && !allowNotExistingConfiguration) {
|
|
48
53
|
throw EventQueueError.unknownEventType(type, subType);
|
|
49
54
|
}
|
|
50
55
|
if (startAfter && !common.isValidDate(startAfter)) {
|
|
51
56
|
throw EventQueueError.malformedDate(startAfter);
|
|
52
57
|
}
|
|
53
58
|
|
|
54
|
-
if (eventConfig
|
|
59
|
+
if (eventConfig?.isPeriodic) {
|
|
55
60
|
throw EventQueueError.manuelPeriodicEventInsert(type, subType);
|
|
56
61
|
}
|
|
57
62
|
|
|
@@ -63,9 +68,13 @@ const publishEvent = async (
|
|
|
63
68
|
event.context = JSON.stringify({ traceContext: openTelemetry.getCurrentTraceContext() });
|
|
64
69
|
}
|
|
65
70
|
|
|
66
|
-
if (eventConfig
|
|
71
|
+
if (eventConfig?.timeBucket && !(startAfter in event)) {
|
|
67
72
|
event.startAfter = CronExpressionParser.parse(eventConfig.timeBucket).next().toISOString();
|
|
68
73
|
}
|
|
74
|
+
|
|
75
|
+
if (event.namespace === undefined) {
|
|
76
|
+
event.namespace = config.namespace;
|
|
77
|
+
}
|
|
69
78
|
}
|
|
70
79
|
if (config.insertEventsBeforeCommit && !skipInsertEventsBeforeCommit) {
|
|
71
80
|
_registerHandlerAndAddEvents(tx, events);
|
package/src/redis/redisPub.js
CHANGED
|
@@ -69,15 +69,15 @@ const broadcastEvent = async (tenantId, events, forceBroadcast = false) => {
|
|
|
69
69
|
}
|
|
70
70
|
await cds.tx({ tenant: tenantId }, async ({ context }) => {
|
|
71
71
|
await trace(context, "broadcast-inserted-events", async () => {
|
|
72
|
-
for (const { type, subType } of events) {
|
|
73
|
-
const eventConfig = config.getEventConfig(type, subType);
|
|
72
|
+
for (const { type, subType, namespace } of events) {
|
|
73
|
+
const eventConfig = config.getEventConfig(type, subType, namespace);
|
|
74
74
|
if (!eventConfig) {
|
|
75
75
|
continue;
|
|
76
76
|
}
|
|
77
77
|
for (let i = 0; i < TRIES_FOR_PUBLISH_PERIODIC_EVENT; i++) {
|
|
78
78
|
const result = eventConfig.multiInstanceProcessing
|
|
79
79
|
? false
|
|
80
|
-
: await distributedLock.
|
|
80
|
+
: await distributedLock.checkLockExists(context, [namespace, type, subType].join("##"));
|
|
81
81
|
if (result) {
|
|
82
82
|
logger.debug("skip publish redis event as no lock is available", {
|
|
83
83
|
type,
|
|
@@ -98,9 +98,8 @@ const broadcastEvent = async (tenantId, events, forceBroadcast = false) => {
|
|
|
98
98
|
subType,
|
|
99
99
|
});
|
|
100
100
|
await redis.publishMessage(
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
JSON.stringify({ lockId: cds.utils.uuid(), tenantId, type, subType })
|
|
101
|
+
[namespace, EVENT_MESSAGE_CHANNEL].join("##"),
|
|
102
|
+
JSON.stringify({ lockId: cds.utils.uuid(), tenantId, type, subType, namespace })
|
|
104
103
|
);
|
|
105
104
|
break;
|
|
106
105
|
}
|
|
@@ -128,9 +127,12 @@ const _processLocalWithoutRedis = async (tenantId, events) => {
|
|
|
128
127
|
};
|
|
129
128
|
}
|
|
130
129
|
|
|
131
|
-
for (const { type, subType } of events) {
|
|
130
|
+
for (const { type, subType, namespace } of events) {
|
|
131
|
+
if (!config.shouldProcessNamespace(namespace)) {
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
132
134
|
await cds.tx(context, async ({ context }) => {
|
|
133
|
-
await runEventCombinationForTenant(context, type, subType, { shouldTrace: true });
|
|
135
|
+
await runEventCombinationForTenant(context, type, subType, namespace, { shouldTrace: true });
|
|
134
136
|
});
|
|
135
137
|
}
|
|
136
138
|
}
|
package/src/redis/redisSub.js
CHANGED
|
@@ -16,13 +16,16 @@ const initEventQueueRedisSubscribe = () => {
|
|
|
16
16
|
return;
|
|
17
17
|
}
|
|
18
18
|
initEventQueueRedisSubscribe._initDone = true;
|
|
19
|
-
|
|
19
|
+
|
|
20
|
+
config.processingNamespaces.forEach((namespace) => {
|
|
21
|
+
redis.subscribeRedisChannel([namespace, EVENT_MESSAGE_CHANNEL].join("##"), _messageHandlerProcessEvents);
|
|
22
|
+
});
|
|
20
23
|
};
|
|
21
24
|
|
|
22
25
|
const _messageHandlerProcessEvents = async (messageData) => {
|
|
23
26
|
const logger = cds.log(COMPONENT_NAME);
|
|
24
27
|
try {
|
|
25
|
-
const { lockId, tenantId, type, subType } = JSON.parse(messageData);
|
|
28
|
+
const { lockId, tenantId, type, subType, namespace } = JSON.parse(messageData);
|
|
26
29
|
const tenantShouldBeProcessed = await common.isTenantIdValidCb(TenantIdCheckTypes.eventProcessing, tenantId);
|
|
27
30
|
if (!tenantShouldBeProcessed) {
|
|
28
31
|
return;
|
|
@@ -41,7 +44,7 @@ const _messageHandlerProcessEvents = async (messageData) => {
|
|
|
41
44
|
}
|
|
42
45
|
|
|
43
46
|
const [serviceNameOrSubType, actionName] = subType.split(".");
|
|
44
|
-
if (!config.getEventConfig(type, subType)) {
|
|
47
|
+
if (!config.getEventConfig(type, subType, namespace)) {
|
|
45
48
|
if (config.isCapOutboxEvent(type)) {
|
|
46
49
|
try {
|
|
47
50
|
const service = await cds.connect.to(serviceNameOrSubType);
|
|
@@ -68,7 +71,12 @@ const _messageHandlerProcessEvents = async (messageData) => {
|
|
|
68
71
|
}
|
|
69
72
|
}
|
|
70
73
|
|
|
71
|
-
if (
|
|
74
|
+
if (
|
|
75
|
+
!(
|
|
76
|
+
config.getEventConfig(type, subType, namespace) &&
|
|
77
|
+
config.shouldBeProcessedInThisApplication(type, subType, namespace)
|
|
78
|
+
)
|
|
79
|
+
) {
|
|
72
80
|
logger.debug("event is not configured to be processed on this app-name", {
|
|
73
81
|
tenantId,
|
|
74
82
|
type,
|
|
@@ -87,7 +95,10 @@ const _messageHandlerProcessEvents = async (messageData) => {
|
|
|
87
95
|
};
|
|
88
96
|
|
|
89
97
|
return await cds.tx(tenantContext, async ({ context }) => {
|
|
90
|
-
return await runnerHelper.runEventCombinationForTenant(context, type, subType,
|
|
98
|
+
return await runnerHelper.runEventCombinationForTenant(context, type, subType, namespace, {
|
|
99
|
+
lockId,
|
|
100
|
+
shouldTrace: true,
|
|
101
|
+
});
|
|
91
102
|
});
|
|
92
103
|
} catch (err) {
|
|
93
104
|
logger.error("could not parse event information", {
|
package/src/runner/openEvents.js
CHANGED
|
@@ -14,7 +14,9 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
|
|
|
14
14
|
const entries = await tx.run(
|
|
15
15
|
SELECT.from(eventConfig.tableNameEventQueue)
|
|
16
16
|
.where(
|
|
17
|
-
"
|
|
17
|
+
"namespace IN",
|
|
18
|
+
config.processingNamespaces,
|
|
19
|
+
"AND ( startAfter IS NULL OR startAfter <=",
|
|
18
20
|
refDateStartAfter.toISOString(),
|
|
19
21
|
" ) AND ( status =",
|
|
20
22
|
EventProcessingStatus.Open,
|
|
@@ -30,12 +32,12 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
|
|
|
30
32
|
new Date(startTime.getTime() - 30 * MS_IN_DAYS).toISOString(),
|
|
31
33
|
")"
|
|
32
34
|
)
|
|
33
|
-
.columns("type", "subType")
|
|
34
|
-
.groupBy("type", "subType")
|
|
35
|
+
.columns("type", "subType", "namespace")
|
|
36
|
+
.groupBy("type", "subType", "namespace")
|
|
35
37
|
);
|
|
36
38
|
|
|
37
39
|
const result = [];
|
|
38
|
-
for (const { type, subType } of entries) {
|
|
40
|
+
for (const { type, subType, namespace } of entries) {
|
|
39
41
|
if (eventConfig.isCapOutboxEvent(type)) {
|
|
40
42
|
const [srvName, actionName] = subType.split(".");
|
|
41
43
|
try {
|
|
@@ -48,24 +50,24 @@ const getOpenQueueEntries = async (tx, filterAppSpecificEvents = true) => {
|
|
|
48
50
|
if (actionName) {
|
|
49
51
|
config.addCAPOutboxEventSpecificAction(srvName, actionName);
|
|
50
52
|
}
|
|
51
|
-
if (eventConfig.shouldBeProcessedInThisApplication(type, subType)) {
|
|
52
|
-
result.push({ type, subType });
|
|
53
|
+
if (eventConfig.shouldBeProcessedInThisApplication(type, subType, namespace)) {
|
|
54
|
+
result.push({ namespace, type, subType });
|
|
53
55
|
}
|
|
54
56
|
}
|
|
55
57
|
} catch {
|
|
56
58
|
/* ignore catch */
|
|
57
59
|
} finally {
|
|
58
60
|
if (!filterAppSpecificEvents) {
|
|
59
|
-
result.push({ type, subType });
|
|
61
|
+
result.push({ namespace, type, subType });
|
|
60
62
|
}
|
|
61
63
|
}
|
|
62
64
|
} else {
|
|
63
65
|
if (filterAppSpecificEvents) {
|
|
64
66
|
if (
|
|
65
|
-
eventConfig.getEventConfig(type, subType) &&
|
|
66
|
-
eventConfig.shouldBeProcessedInThisApplication(type, subType)
|
|
67
|
+
eventConfig.getEventConfig(type, subType, namespace) &&
|
|
68
|
+
eventConfig.shouldBeProcessedInThisApplication(type, subType, namespace)
|
|
67
69
|
) {
|
|
68
|
-
result.push({ type, subType });
|
|
70
|
+
result.push({ namespace, type, subType });
|
|
69
71
|
}
|
|
70
72
|
} else {
|
|
71
73
|
result.push({ type, subType });
|
package/src/runner/runner.js
CHANGED
|
@@ -216,8 +216,8 @@ const _executeEventsAllTenants = async (tenantIds, runId) => {
|
|
|
216
216
|
|
|
217
217
|
promises.concat(
|
|
218
218
|
events.map(async (openEvent) => {
|
|
219
|
-
const eventConfig = config.getEventConfig(openEvent.type, openEvent.subType);
|
|
220
|
-
const label =
|
|
219
|
+
const eventConfig = config.getEventConfig(openEvent.type, openEvent.subType, openEvent.namespace);
|
|
220
|
+
const label = [openEvent.namespace, eventConfig.type, eventConfig.subType].join("##");
|
|
221
221
|
return await WorkerQueue.instance.addToQueue(
|
|
222
222
|
eventConfig.load,
|
|
223
223
|
label,
|
|
@@ -237,9 +237,15 @@ const _executeEventsAllTenants = async (tenantIds, runId) => {
|
|
|
237
237
|
if (!couldAcquireLock) {
|
|
238
238
|
return;
|
|
239
239
|
}
|
|
240
|
-
await runEventCombinationForTenant(
|
|
241
|
-
|
|
242
|
-
|
|
240
|
+
await runEventCombinationForTenant(
|
|
241
|
+
context,
|
|
242
|
+
eventConfig.type,
|
|
243
|
+
eventConfig.subType,
|
|
244
|
+
openEvent.namespace,
|
|
245
|
+
{
|
|
246
|
+
skipWorkerPool: true,
|
|
247
|
+
}
|
|
248
|
+
);
|
|
243
249
|
} catch (err) {
|
|
244
250
|
cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
|
|
245
251
|
tenantId,
|
|
@@ -304,8 +310,8 @@ const _singleTenantDb = async () => {
|
|
|
304
310
|
|
|
305
311
|
return await Promise.allSettled(
|
|
306
312
|
events.map(async (openEvent) => {
|
|
307
|
-
const eventConfig = config.getEventConfig(openEvent.type, openEvent.subType);
|
|
308
|
-
const label =
|
|
313
|
+
const eventConfig = config.getEventConfig(openEvent.type, openEvent.subType, openEvent.namespace);
|
|
314
|
+
const label = [openEvent.namespace, eventConfig.type, eventConfig.subType].join("##");
|
|
309
315
|
return await WorkerQueue.instance.addToQueue(
|
|
310
316
|
eventConfig.load,
|
|
311
317
|
label,
|
|
@@ -318,18 +324,23 @@ const _singleTenantDb = async () => {
|
|
|
318
324
|
label,
|
|
319
325
|
async () => {
|
|
320
326
|
try {
|
|
321
|
-
const lockId = `${label}`;
|
|
322
327
|
const couldAcquireLock = eventConfig.multiInstanceProcessing
|
|
323
328
|
? true
|
|
324
|
-
: await distributedLock.acquireLock(context,
|
|
329
|
+
: await distributedLock.acquireLock(context, label, {
|
|
325
330
|
expiryTime: eventQueueConfig.runInterval * 0.95,
|
|
326
331
|
});
|
|
327
332
|
if (!couldAcquireLock) {
|
|
328
333
|
return;
|
|
329
334
|
}
|
|
330
|
-
await runEventCombinationForTenant(
|
|
331
|
-
|
|
332
|
-
|
|
335
|
+
await runEventCombinationForTenant(
|
|
336
|
+
context,
|
|
337
|
+
eventConfig.type,
|
|
338
|
+
eventConfig.subType,
|
|
339
|
+
openEvent.namespace,
|
|
340
|
+
{
|
|
341
|
+
skipWorkerPool: true,
|
|
342
|
+
}
|
|
343
|
+
);
|
|
333
344
|
} catch (err) {
|
|
334
345
|
cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed");
|
|
335
346
|
}
|
|
@@ -405,7 +416,7 @@ const _acquireRunId = async (context) => {
|
|
|
405
416
|
overrideValue: true,
|
|
406
417
|
});
|
|
407
418
|
} else {
|
|
408
|
-
runId = await distributedLock.
|
|
419
|
+
runId = await distributedLock.getValue(context, EVENT_QUEUE_RUN_ID, {
|
|
409
420
|
tenantScoped: false,
|
|
410
421
|
});
|
|
411
422
|
}
|
|
@@ -422,7 +433,7 @@ const _calculateOffsetForFirstRun = async () => {
|
|
|
422
433
|
try {
|
|
423
434
|
await trace(dummyContext, "calculateOffsetForFirstRun", async () => {
|
|
424
435
|
if (eventQueueConfig.redisEnabled) {
|
|
425
|
-
let lastRunTs = await distributedLock.
|
|
436
|
+
let lastRunTs = await distributedLock.getValue(dummyContext, EVENT_QUEUE_RUN_TS, {
|
|
426
437
|
tenantScoped: false,
|
|
427
438
|
});
|
|
428
439
|
if (!lastRunTs) {
|
|
@@ -434,7 +445,7 @@ const _calculateOffsetForFirstRun = async () => {
|
|
|
434
445
|
if (couldSetValue) {
|
|
435
446
|
lastRunTs = ts;
|
|
436
447
|
} else {
|
|
437
|
-
lastRunTs = await distributedLock.
|
|
448
|
+
lastRunTs = await distributedLock.getValue(dummyContext, EVENT_QUEUE_RUN_TS, {
|
|
438
449
|
tenantScoped: false,
|
|
439
450
|
});
|
|
440
451
|
}
|
|
@@ -12,12 +12,18 @@ const { trace } = require("../shared/openTelemetry");
|
|
|
12
12
|
|
|
13
13
|
const COMPONENT_NAME = "/eventQueue/runnerHelper";
|
|
14
14
|
|
|
15
|
-
const runEventCombinationForTenant = async (
|
|
15
|
+
const runEventCombinationForTenant = async (
|
|
16
|
+
context,
|
|
17
|
+
type,
|
|
18
|
+
subType,
|
|
19
|
+
namespace,
|
|
20
|
+
{ skipWorkerPool, lockId, shouldTrace } = {}
|
|
21
|
+
) => {
|
|
16
22
|
try {
|
|
17
23
|
if (skipWorkerPool) {
|
|
18
|
-
return await processEventQueue(context, type, subType);
|
|
24
|
+
return await processEventQueue(context, type, subType, namespace);
|
|
19
25
|
} else {
|
|
20
|
-
const eventConfig = eventQueueConfig.getEventConfig(type, subType);
|
|
26
|
+
const eventConfig = eventQueueConfig.getEventConfig(type, subType, namespace);
|
|
21
27
|
const label = `${type}_${subType}`;
|
|
22
28
|
return await WorkerQueue.instance.addToQueue(
|
|
23
29
|
eventConfig.load,
|
|
@@ -33,7 +39,7 @@ const runEventCombinationForTenant = async (context, type, subType, { skipWorker
|
|
|
33
39
|
}
|
|
34
40
|
}
|
|
35
41
|
|
|
36
|
-
await processEventQueue(context, type, subType);
|
|
42
|
+
await processEventQueue(context, type, subType, namespace);
|
|
37
43
|
};
|
|
38
44
|
if (shouldTrace) {
|
|
39
45
|
return await trace(context, label, _exec, { newRootSpan: true });
|