@cap-js-community/event-queue 1.10.0-beta.0 → 1.10.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/EventQueueProcessorBase.js +1 -1
- package/src/config.js +34 -12
- package/src/initialize.js +1 -0
- package/src/outbox/EventQueueGenericOutboxHandler.js +239 -22
- package/src/outbox/eventQueueAsOutbox.js +2 -1
- package/src/processEventQueue.js +1 -1
- package/src/publishEvent.js +6 -0
- package/src/runner/runner.js +4 -4
- package/src/shared/distributedLock.js +2 -3
- package/src/shared/redis.js +5 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cap-js-community/event-queue",
|
|
3
|
-
"version": "1.10.0-beta.
|
|
3
|
+
"version": "1.10.0-beta.2",
|
|
4
4
|
"description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
@@ -243,7 +243,7 @@ class EventQueueProcessorBase {
|
|
|
243
243
|
* This can be useful for e.g. multiple tasks have been scheduled and always the same user should be informed.
|
|
244
244
|
* In this case the events should be clustered together and only one mail should be sent.
|
|
245
245
|
*/
|
|
246
|
-
clusterQueueEntries(queueEntriesWithPayloadMap) {
|
|
246
|
+
async clusterQueueEntries(queueEntriesWithPayloadMap) {
|
|
247
247
|
Object.entries(queueEntriesWithPayloadMap).forEach(([key, { queueEntry, payload }]) => {
|
|
248
248
|
this.addEntryToProcessingMap(key, queueEntry, payload);
|
|
249
249
|
});
|
package/src/config.js
CHANGED
|
@@ -10,9 +10,12 @@ const { Priorities } = require("./constants");
|
|
|
10
10
|
|
|
11
11
|
const FOR_UPDATE_TIMEOUT = 10;
|
|
12
12
|
const GLOBAL_TX_TIMEOUT = 30 * 60 * 1000;
|
|
13
|
-
const
|
|
13
|
+
const REDIS_PREFIX = "EVENT_QUEUE";
|
|
14
|
+
const REDIS_CONFIG_CHANNEL = "CONFIG_CHANNEL";
|
|
14
15
|
const REDIS_OFFBOARD_TENANT_CHANNEL = "REDIS_OFFBOARD_TENANT_CHANNEL";
|
|
15
|
-
const REDIS_CONFIG_BLOCKLIST_CHANNEL = "
|
|
16
|
+
const REDIS_CONFIG_BLOCKLIST_CHANNEL = "REDIS_CONFIG_BLOCKLIST_CHANNEL";
|
|
17
|
+
const COMMAND_BLOCK = "EVENT_BLOCK";
|
|
18
|
+
const COMMAND_UNBLOCK = "EVENT_UNBLOCK";
|
|
16
19
|
const COMPONENT_NAME = "/eventQueue/config";
|
|
17
20
|
const MIN_INTERVAL_SEC = 10;
|
|
18
21
|
const DEFAULT_LOAD = 1;
|
|
@@ -22,8 +25,6 @@ const DEFAULT_KEEP_ALIVE_INTERVAL = 60;
|
|
|
22
25
|
const DEFAULT_MAX_FACTOR_STUCK_2_KEEP_ALIVE_INTERVAL = 3.5;
|
|
23
26
|
const DEFAULT_INHERIT_TRACE_CONTEXT = true;
|
|
24
27
|
const SUFFIX_PERIODIC = "_PERIODIC";
|
|
25
|
-
const COMMAND_BLOCK = "EVENT_QUEUE_EVENT_BLOCK";
|
|
26
|
-
const COMMAND_UNBLOCK = "EVENT_QUEUE_EVENT_UNBLOCK";
|
|
27
28
|
const CAP_EVENT_TYPE = "CAP_OUTBOX";
|
|
28
29
|
const CAP_PARALLEL_DEFAULT = 5;
|
|
29
30
|
const DELETE_TENANT_BLOCK_AFTER_MS = 5 * 60 * 1000;
|
|
@@ -64,6 +65,7 @@ const ALLOWED_EVENT_OPTIONS_AD_HOC = [
|
|
|
64
65
|
"retryFailedAfter",
|
|
65
66
|
"multiInstanceProcessing",
|
|
66
67
|
"kind",
|
|
68
|
+
"timeBucket",
|
|
67
69
|
];
|
|
68
70
|
|
|
69
71
|
const ALLOWED_EVENT_OPTIONS_PERIODIC_EVENT = [
|
|
@@ -105,6 +107,7 @@ class Config {
|
|
|
105
107
|
#unsubscribeHandlers = [];
|
|
106
108
|
#unsubscribedTenants = {};
|
|
107
109
|
#cronTimezone;
|
|
110
|
+
#redisNamespace;
|
|
108
111
|
#publishEventBlockList;
|
|
109
112
|
#crashOnRedisUnavailable;
|
|
110
113
|
#tenantIdFilterTokenInfoCb;
|
|
@@ -181,7 +184,7 @@ class Config {
|
|
|
181
184
|
|
|
182
185
|
attachConfigChangeHandler() {
|
|
183
186
|
this.#attachBlockListChangeHandler();
|
|
184
|
-
redis.subscribeRedisChannel(this
|
|
187
|
+
redis.subscribeRedisChannel(this.redisOptions, REDIS_CONFIG_CHANNEL, (messageData) => {
|
|
185
188
|
try {
|
|
186
189
|
const { key, value } = JSON.parse(messageData);
|
|
187
190
|
if (this[key] !== value) {
|
|
@@ -198,7 +201,7 @@ class Config {
|
|
|
198
201
|
|
|
199
202
|
attachRedisUnsubscribeHandler() {
|
|
200
203
|
this.#logger.info("attached redis handle for unsubscribe events");
|
|
201
|
-
redis.subscribeRedisChannel(this
|
|
204
|
+
redis.subscribeRedisChannel(this.redisOptions, REDIS_OFFBOARD_TENANT_CHANNEL, (messageData) => {
|
|
202
205
|
try {
|
|
203
206
|
const { tenantId } = JSON.parse(messageData);
|
|
204
207
|
this.#logger.info("received unsubscribe broadcast event", { tenantId });
|
|
@@ -228,7 +231,7 @@ class Config {
|
|
|
228
231
|
handleUnsubscribe(tenantId) {
|
|
229
232
|
if (this.redisEnabled) {
|
|
230
233
|
redis
|
|
231
|
-
.publishMessage(this
|
|
234
|
+
.publishMessage(this.redisOptions, REDIS_OFFBOARD_TENANT_CHANNEL, JSON.stringify({ tenantId }))
|
|
232
235
|
.catch((error) => {
|
|
233
236
|
this.#logger.error(`publishing tenant unsubscribe failed. tenantId: ${tenantId}`, error);
|
|
234
237
|
});
|
|
@@ -246,13 +249,13 @@ class Config {
|
|
|
246
249
|
this.#logger.info("redis not connected, config change won't be published", { key, value });
|
|
247
250
|
return;
|
|
248
251
|
}
|
|
249
|
-
redis.publishMessage(this
|
|
252
|
+
redis.publishMessage(this.redisOptions, REDIS_CONFIG_CHANNEL, JSON.stringify({ key, value })).catch((error) => {
|
|
250
253
|
this.#logger.error(`publishing config change failed key: ${key}, value: ${value}`, error);
|
|
251
254
|
});
|
|
252
255
|
}
|
|
253
256
|
|
|
254
257
|
#attachBlockListChangeHandler() {
|
|
255
|
-
redis.subscribeRedisChannel(this
|
|
258
|
+
redis.subscribeRedisChannel(this.redisOptions, REDIS_CONFIG_BLOCKLIST_CHANNEL, (messageData) => {
|
|
256
259
|
try {
|
|
257
260
|
const { command, key, tenant } = JSON.parse(messageData);
|
|
258
261
|
if (command === COMMAND_BLOCK) {
|
|
@@ -282,7 +285,7 @@ class Config {
|
|
|
282
285
|
|
|
283
286
|
redis
|
|
284
287
|
.publishMessage(
|
|
285
|
-
this
|
|
288
|
+
this.redisOptions,
|
|
286
289
|
REDIS_CONFIG_BLOCKLIST_CHANNEL,
|
|
287
290
|
JSON.stringify({ command: COMMAND_BLOCK, key, tenant })
|
|
288
291
|
)
|
|
@@ -315,7 +318,7 @@ class Config {
|
|
|
315
318
|
|
|
316
319
|
redis
|
|
317
320
|
.publishMessage(
|
|
318
|
-
this
|
|
321
|
+
this.redisOptions,
|
|
319
322
|
REDIS_CONFIG_BLOCKLIST_CHANNEL,
|
|
320
323
|
JSON.stringify({ command: COMMAND_UNBLOCK, key, tenant })
|
|
321
324
|
)
|
|
@@ -630,6 +633,14 @@ class Config {
|
|
|
630
633
|
}
|
|
631
634
|
event.inheritTraceContext = event.inheritTraceContext ?? DEFAULT_INHERIT_TRACE_CONTEXT;
|
|
632
635
|
|
|
636
|
+
if (event.timeBucket) {
|
|
637
|
+
try {
|
|
638
|
+
CronExpressionParser.parse(event.timeBucket);
|
|
639
|
+
} catch {
|
|
640
|
+
throw EventQueueError.cantParseCronExpression(event.type, event.subType, event.timeBucket);
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
|
|
633
644
|
this.#basicEventValidation(event);
|
|
634
645
|
}
|
|
635
646
|
|
|
@@ -865,7 +876,18 @@ class Config {
|
|
|
865
876
|
}
|
|
866
877
|
|
|
867
878
|
get redisOptions() {
|
|
868
|
-
return
|
|
879
|
+
return {
|
|
880
|
+
...this.#redisOptions,
|
|
881
|
+
redisNamespace: `${[REDIS_PREFIX, this.redisNamespace].filter((a) => a).join("_")}`,
|
|
882
|
+
};
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
set redisNamespace(value) {
|
|
886
|
+
this.#redisNamespace = value;
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
get redisNamespace() {
|
|
890
|
+
return this.#redisNamespace;
|
|
869
891
|
}
|
|
870
892
|
|
|
871
893
|
set insertEventsBeforeCommit(value) {
|
package/src/initialize.js
CHANGED
|
@@ -15,34 +15,251 @@ class EventQueueGenericOutboxHandler extends EventQueueBaseClass {
|
|
|
15
15
|
this.logger = cds.log(`${COMPONENT_NAME}/${eventSubType}`);
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
+
async getQueueEntriesAndSetToInProgress() {
|
|
19
|
+
const [serviceName] = this.eventSubType.split(".");
|
|
20
|
+
this.__srv = await cds.connect.to(serviceName);
|
|
21
|
+
this.__srvUnboxed = cds.unboxed(this.__srv);
|
|
22
|
+
const { handlers, clusterRelevant, specificClusterRelevant } = this.__srvUnboxed.handlers.on.reduce(
|
|
23
|
+
(result, handler) => {
|
|
24
|
+
if (handler.on.startsWith("clusterQueueEntries")) {
|
|
25
|
+
if (handler.on.split(".").length === 2) {
|
|
26
|
+
result.specificClusterRelevant = true;
|
|
27
|
+
} else {
|
|
28
|
+
result.clusterRelevant = true;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
result.handlers[handler.on] = handler.on;
|
|
33
|
+
return result;
|
|
34
|
+
},
|
|
35
|
+
{ handlers: {}, clusterRelevant: false, specificClusterRelevant: false }
|
|
36
|
+
);
|
|
37
|
+
this.__onHandlers = handlers;
|
|
38
|
+
this.__genericClusterHandler = clusterRelevant;
|
|
39
|
+
this.__specificClusterHandler = specificClusterRelevant;
|
|
40
|
+
await this.#setContextUser(this.context, config.userId);
|
|
41
|
+
return super.getQueueEntriesAndSetToInProgress();
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// NOTE: issue here: if events are not sorted before it might not be unique here:
|
|
45
|
+
// - we have service events
|
|
46
|
+
// - we have action specific events
|
|
47
|
+
// 1. I need to collect all action names
|
|
48
|
+
// 2. I need to check for which actions a handler exits
|
|
49
|
+
// 3. For actions with specific existing handler --> call specific action
|
|
50
|
+
// 4. For all others call generic handler
|
|
51
|
+
// NOTE: OVERALL idea is that the handler returns the cluster map and MUST not call any baseImpl functions!
|
|
52
|
+
// --> structure is a map of { key: { queueEntries: [], payload: {} }
|
|
53
|
+
// TODO: document that clusterQueueEntries is now async!!!
|
|
54
|
+
// TODO: validate that return structure is as expected
|
|
55
|
+
async clusterQueueEntries(queueEntriesWithPayloadMap) {
|
|
56
|
+
if (!this.__genericClusterRelevantAndAvailable && !this.__specificClusterRelevantAndAvailable) {
|
|
57
|
+
return super.clusterQueueEntries(queueEntriesWithPayloadMap);
|
|
58
|
+
}
|
|
59
|
+
const { genericClusterEvents, specificClusterEvents } = this.#clusterByAction(queueEntriesWithPayloadMap);
|
|
60
|
+
if (Object.keys(genericClusterEvents).length) {
|
|
61
|
+
if (!this.__genericClusterRelevantAndAvailable) {
|
|
62
|
+
await super.clusterQueueEntries(genericClusterEvents);
|
|
63
|
+
} else {
|
|
64
|
+
const msg = new cds.Request({
|
|
65
|
+
event: "clusterQueueEntries",
|
|
66
|
+
eventQueue: {
|
|
67
|
+
processor: this,
|
|
68
|
+
clusterByPayloadProperty: (propertyName, cb) =>
|
|
69
|
+
this.clusterByPayloadProperty(genericClusterEvents, propertyName, cb),
|
|
70
|
+
clusterByEventProperty: (propertyName) => this.clusterByEventProperty(genericClusterEvents, propertyName),
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
|
|
74
|
+
this.#addToProcessingMap(handlerCluster);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
for (const actionName in specificClusterEvents) {
|
|
79
|
+
const msg = new cds.Request({
|
|
80
|
+
event: `clusterQueueEntries.${actionName}`,
|
|
81
|
+
eventQueue: {
|
|
82
|
+
processor: this,
|
|
83
|
+
clusterByPayloadProperty: (propertyName, cb) =>
|
|
84
|
+
this.clusterByPayloadProperty(specificClusterEvents[actionName], propertyName, cb),
|
|
85
|
+
clusterByEventProperty: (propertyName, cb) =>
|
|
86
|
+
this.clusterByEventProperty(specificClusterEvents[actionName], propertyName, cb),
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
const handlerCluster = await this.__srvUnboxed.tx(this.context).send(msg);
|
|
90
|
+
this.#addToProcessingMap(handlerCluster);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
clusterBase(queueEntriesWithPayloadMap, propertyName, refCb, cb) {
|
|
95
|
+
return Object.entries(queueEntriesWithPayloadMap).reduce((result, [, { queueEntry, payload }], index) => {
|
|
96
|
+
const ref = refCb(result, payload, queueEntry);
|
|
97
|
+
ref.queueEntries.push(queueEntry);
|
|
98
|
+
if (cb) {
|
|
99
|
+
const clusterResult = cb(ref.payload.data, queueEntry.payload.data, index);
|
|
100
|
+
ref.payload.data ??= clusterResult;
|
|
101
|
+
}
|
|
102
|
+
return result;
|
|
103
|
+
}, {});
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
clusterByPayloadProperty(queueEntriesWithPayloadMap, propertyName, cb) {
|
|
107
|
+
return this.clusterBase(
|
|
108
|
+
queueEntriesWithPayloadMap,
|
|
109
|
+
propertyName,
|
|
110
|
+
(result, payload) => {
|
|
111
|
+
const parts = propertyName.split(".");
|
|
112
|
+
const data = JSON.parse(JSON.stringify(payload.data));
|
|
113
|
+
let ref = payload;
|
|
114
|
+
for (const part of parts) {
|
|
115
|
+
ref = ref[part];
|
|
116
|
+
}
|
|
117
|
+
result[ref[propertyName]] ??= {
|
|
118
|
+
queueEntries: [],
|
|
119
|
+
payload: { ...payload, data },
|
|
120
|
+
};
|
|
121
|
+
return result[ref[propertyName]];
|
|
122
|
+
},
|
|
123
|
+
cb
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
clusterByEventProperty(queueEntriesWithPayloadMap, propertyName, cb) {
|
|
128
|
+
return this.clusterBase(
|
|
129
|
+
queueEntriesWithPayloadMap,
|
|
130
|
+
propertyName,
|
|
131
|
+
(result, payload, queueEntry) => {
|
|
132
|
+
const parts = propertyName.split(".");
|
|
133
|
+
const payloadCopy = JSON.parse(JSON.stringify(payload));
|
|
134
|
+
let ref = queueEntry;
|
|
135
|
+
for (const part of parts) {
|
|
136
|
+
ref = ref[part];
|
|
137
|
+
}
|
|
138
|
+
result[queueEntry[propertyName]] ??= {
|
|
139
|
+
queueEntries: [],
|
|
140
|
+
payload: payloadCopy,
|
|
141
|
+
};
|
|
142
|
+
return result[queueEntry[propertyName]];
|
|
143
|
+
},
|
|
144
|
+
cb
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
#clusterByAction(queueEntriesWithPayloadMap) {
|
|
149
|
+
return Object.entries(queueEntriesWithPayloadMap).reduce(
|
|
150
|
+
(result, [eventId, clusterData]) => {
|
|
151
|
+
const hasSpecificClusterHandler = this.#hasEventSpecificClusterHandler(clusterData.queueEntry);
|
|
152
|
+
if (hasSpecificClusterHandler && this.__specificClusterRelevantAndAvailable) {
|
|
153
|
+
result.specificClusterEvents[clusterData.payload.event] ??= {};
|
|
154
|
+
result.specificClusterEvents[clusterData.payload.event][eventId] = clusterData;
|
|
155
|
+
} else {
|
|
156
|
+
result.genericClusterEvents[eventId] = clusterData;
|
|
157
|
+
}
|
|
158
|
+
return result;
|
|
159
|
+
},
|
|
160
|
+
{ genericClusterEvents: {}, specificClusterEvents: {} }
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
#addToProcessingMap(handlerCluster) {
|
|
165
|
+
for (const clusterKey in handlerCluster) {
|
|
166
|
+
const { payload, queueEntries } = handlerCluster[clusterKey];
|
|
167
|
+
for (const queueEntry of queueEntries) {
|
|
168
|
+
this.addEntryToProcessingMap(clusterKey, queueEntry, payload);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// NOTE: Currently not exposed to CAP service; I don't see any valid use case at this time
|
|
174
|
+
modifyQueueEntry(queueEntry) {
|
|
175
|
+
super.modifyQueueEntry(queueEntry);
|
|
176
|
+
const hasSpecificClusterHandler = this.#hasEventSpecificClusterHandler(queueEntry);
|
|
177
|
+
if (this.__specificClusterHandler && hasSpecificClusterHandler) {
|
|
178
|
+
this.__specificClusterRelevantAndAvailable = true;
|
|
179
|
+
}
|
|
180
|
+
if (this.__genericClusterHandler && !hasSpecificClusterHandler) {
|
|
181
|
+
this.__genericClusterRelevantAndAvailable = true;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
#hasEventSpecificClusterHandler(queueEntry) {
|
|
186
|
+
return !!this.__onHandlers[["clusterQueueEntries", queueEntry.payload.event].join(".")];
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
async checkEventAndGeneratePayload(queueEntry) {
|
|
190
|
+
const payload = await super.checkEventAndGeneratePayload(queueEntry);
|
|
191
|
+
const { event } = payload;
|
|
192
|
+
const handlerName = this.#checkHandlerExists("checkEventAndGeneratePayload", event);
|
|
193
|
+
if (!handlerName) {
|
|
194
|
+
return payload;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const { msg, userId } = this.#buildDispatchData(this.context, payload, {
|
|
198
|
+
queueEntries: [queueEntry],
|
|
199
|
+
});
|
|
200
|
+
msg.event = handlerName;
|
|
201
|
+
await this.#setContextUser(this.context, userId);
|
|
202
|
+
const data = await this.__srvUnboxed.tx(this.context).send(msg);
|
|
203
|
+
if (data) {
|
|
204
|
+
payload.data = data;
|
|
205
|
+
return payload;
|
|
206
|
+
} else {
|
|
207
|
+
return null;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// simple here as per entry
|
|
212
|
+
async hookForExceededEvents(exceededEvent) {
|
|
213
|
+
return await super.hookForExceededEvents(exceededEvent);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
async beforeProcessingEvents() {
|
|
217
|
+
return await super.beforeProcessingEvents();
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// maybe async getter on req.data // only for periodic events
|
|
221
|
+
// getLastSuccessfulRunTimestamp
|
|
222
|
+
|
|
223
|
+
#checkHandlerExists(eventQueueFn, event) {
|
|
224
|
+
const specificHandler = this.__onHandlers[[eventQueueFn, event].join(".")];
|
|
225
|
+
if (specificHandler) {
|
|
226
|
+
return specificHandler;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const genericHandler = this.__onHandlers[eventQueueFn];
|
|
230
|
+
return genericHandler ?? null;
|
|
231
|
+
}
|
|
232
|
+
|
|
18
233
|
async processPeriodicEvent(processContext, key, queueEntry) {
|
|
19
|
-
const [
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
234
|
+
const [, action] = this.eventSubType.split(".");
|
|
235
|
+
const msg = new cds.Event({ event: action, eventQueue: { processor: this, key, queueEntries: [queueEntry] } });
|
|
236
|
+
await this.#setContextUser(processContext, config.userId);
|
|
237
|
+
await this.__srvUnboxed.tx(processContext).emit(msg);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
#buildDispatchData(context, payload, { key, queueEntries } = {}) {
|
|
241
|
+
const { useEventQueueUser } = this.eventConfig;
|
|
242
|
+
const userId = useEventQueueUser ? config.userId : payload.contextUser;
|
|
243
|
+
const msg = payload._fromSend ? new cds.Request(payload) : new cds.Event(payload);
|
|
244
|
+
const invocationFn = payload._fromSend ? "send" : "emit";
|
|
245
|
+
delete msg._fromSend; // TODO: this changes the source object --> check after multiple invocations
|
|
246
|
+
delete msg.contextUser;
|
|
247
|
+
msg.eventQueue = { processor: this, key, queueEntries, payload };
|
|
248
|
+
return { msg, userId, invocationFn };
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
async #setContextUser(context, userId) {
|
|
252
|
+
context.user = new cds.User.Privileged({
|
|
253
|
+
id: userId,
|
|
254
|
+
authInfo: await common.getTokenInfo(this.baseContext.tenant),
|
|
25
255
|
});
|
|
26
|
-
processContext._eventQueue = { processor: this, key, queueEntries: [queueEntry] };
|
|
27
|
-
await cds.unboxed(service).tx(processContext)["emit"](msg);
|
|
28
256
|
}
|
|
29
257
|
|
|
30
258
|
async processEvent(processContext, key, queueEntries, payload) {
|
|
31
259
|
try {
|
|
32
|
-
const
|
|
33
|
-
|
|
34
|
-
const
|
|
35
|
-
const userId = useEventQueueUser ? config.userId : payload.contextUser;
|
|
36
|
-
const msg = payload._fromSend ? new cds.Request(payload) : new cds.Event(payload);
|
|
37
|
-
const invocationFn = payload._fromSend ? "send" : "emit";
|
|
38
|
-
delete msg._fromSend;
|
|
39
|
-
delete msg.contextUser;
|
|
40
|
-
processContext.user = new cds.User.Privileged({
|
|
41
|
-
id: userId,
|
|
42
|
-
authInfo: await common.getTokenInfo(processContext.tenant),
|
|
43
|
-
});
|
|
44
|
-
processContext._eventQueue = { processor: this, key, queueEntries, payload };
|
|
45
|
-
const result = await cds.unboxed(service).tx(processContext)[invocationFn](msg);
|
|
260
|
+
const { userId, invocationFn, msg } = this.#buildDispatchData(processContext, payload, { key, queueEntries });
|
|
261
|
+
await this.#setContextUser(processContext, userId);
|
|
262
|
+
const result = await this.__srvUnboxed.tx(processContext)[invocationFn](msg);
|
|
46
263
|
return this.#determineResultStatus(result, queueEntries);
|
|
47
264
|
} catch (err) {
|
|
48
265
|
this.logger.error("error processing outboxed service call", err, {
|
|
@@ -47,7 +47,8 @@ function outboxed(srv, customOpts) {
|
|
|
47
47
|
customOpts || {}
|
|
48
48
|
);
|
|
49
49
|
config.addCAPOutboxEventBase(srv.name, outboxOpts);
|
|
50
|
-
|
|
50
|
+
// TODO: check req.event ?? req.method
|
|
51
|
+
const specificSettings = config.getCdsOutboxEventSpecificConfig(srv.name, req.event);
|
|
51
52
|
if (specificSettings) {
|
|
52
53
|
outboxOpts = config.addCAPOutboxEventSpecificAction(srv.name, req.event);
|
|
53
54
|
}
|
package/src/processEventQueue.js
CHANGED
|
@@ -79,7 +79,7 @@ const processEventQueue = async (context, eventType, eventSubType) => {
|
|
|
79
79
|
await executeInNewTransaction(context, `eventQueue-processing-${eventType}##${eventSubType}`, async (tx) => {
|
|
80
80
|
eventTypeInstance.processEventContext = tx.context;
|
|
81
81
|
try {
|
|
82
|
-
eventTypeInstance.clusterQueueEntries(eventTypeInstance.queueEntriesWithPayloadMap);
|
|
82
|
+
await eventTypeInstance.clusterQueueEntries(eventTypeInstance.queueEntriesWithPayloadMap);
|
|
83
83
|
await processEventMap(eventTypeInstance);
|
|
84
84
|
} catch (err) {
|
|
85
85
|
eventTypeInstance.handleErrorDuringClustering(err);
|
package/src/publishEvent.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
+
const { CronExpressionParser } = require("cron-parser");
|
|
4
|
+
|
|
3
5
|
const config = require("./config");
|
|
4
6
|
const common = require("./shared/common");
|
|
5
7
|
const EventQueueError = require("./EventQueueError");
|
|
@@ -60,6 +62,10 @@ const publishEvent = async (
|
|
|
60
62
|
if (addTraceContext) {
|
|
61
63
|
event.context = JSON.stringify({ traceContext: openTelemetry.getCurrentTraceContext() });
|
|
62
64
|
}
|
|
65
|
+
|
|
66
|
+
if (eventConfig.timeBucket) {
|
|
67
|
+
event.startAfter ??= CronExpressionParser.parse(eventConfig.timeBucket).next().toISOString();
|
|
68
|
+
}
|
|
63
69
|
}
|
|
64
70
|
if (config.insertEventsBeforeCommit && !skipInsertEventsBeforeCommit) {
|
|
65
71
|
_registerHandlerAndAddEvents(tx, events);
|
package/src/runner/runner.js
CHANGED
|
@@ -18,10 +18,10 @@ const { runEventCombinationForTenant } = require("./runnerHelper");
|
|
|
18
18
|
const { trace } = require("../shared/openTelemetry");
|
|
19
19
|
|
|
20
20
|
const COMPONENT_NAME = "/eventQueue/runner";
|
|
21
|
-
const EVENT_QUEUE_RUN_ID = "
|
|
22
|
-
const EVENT_QUEUE_RUN_TS = "
|
|
23
|
-
const EVENT_QUEUE_RUN_REDIS_CHECK = "
|
|
24
|
-
const EVENT_QUEUE_UPDATE_PERIODIC_EVENTS = "
|
|
21
|
+
const EVENT_QUEUE_RUN_ID = "RUN_ID";
|
|
22
|
+
const EVENT_QUEUE_RUN_TS = "RUN_TS";
|
|
23
|
+
const EVENT_QUEUE_RUN_REDIS_CHECK = "RUN_REDIS_CHECK";
|
|
24
|
+
const EVENT_QUEUE_UPDATE_PERIODIC_EVENTS = "UPDATE_PERIODIC_EVENTS";
|
|
25
25
|
let OFFSET_FIRST_RUN = 10 * 1000;
|
|
26
26
|
|
|
27
27
|
let tenantIdHash;
|
|
@@ -4,7 +4,6 @@ const redis = require("./redis");
|
|
|
4
4
|
const config = require("../config");
|
|
5
5
|
const cdsHelper = require("./cdsHelper");
|
|
6
6
|
|
|
7
|
-
const KEY_PREFIX = "EVENT_QUEUE";
|
|
8
7
|
const existingLocks = {};
|
|
9
8
|
const REDIS_COMMAND_OK = "OK";
|
|
10
9
|
const COMPONENT_NAME = "/eventQueue/distributedLock";
|
|
@@ -176,10 +175,10 @@ const _acquireLockDB = async (context, fullKey, expiryTime, { value = "true", ov
|
|
|
176
175
|
};
|
|
177
176
|
|
|
178
177
|
const _generateKey = (context, tenantScoped, key) => {
|
|
179
|
-
const keyParts = [];
|
|
178
|
+
const keyParts = [config.redisOptions.redisNamespace];
|
|
180
179
|
tenantScoped && keyParts.push(context.tenant);
|
|
181
180
|
keyParts.push(key);
|
|
182
|
-
return `${
|
|
181
|
+
return `${keyParts.join("##")}`;
|
|
183
182
|
};
|
|
184
183
|
|
|
185
184
|
const shutdownHandler = async () => {
|
package/src/shared/redis.js
CHANGED
|
@@ -46,6 +46,7 @@ const _createClientBase = (redisOptions = {}) => {
|
|
|
46
46
|
password: redisOptions.password ?? options.password ?? credentials.password,
|
|
47
47
|
socket,
|
|
48
48
|
});
|
|
49
|
+
delete socketOptions.redisNamespace;
|
|
49
50
|
if (credentials.cluster_mode) {
|
|
50
51
|
return redis.createCluster({
|
|
51
52
|
rootNodes: [socketOptions],
|
|
@@ -106,9 +107,10 @@ const _subscribeChannels = (options, subscribedChannels, errorHandlerCreateClien
|
|
|
106
107
|
if (client._subscribedChannels[channel]) {
|
|
107
108
|
continue;
|
|
108
109
|
}
|
|
109
|
-
|
|
110
|
+
const prefixedChannelName = [options.redisNamespace, channel].join("_");
|
|
111
|
+
cds.log(COMPONENT_NAME).info("subscribe redis client connected channel", { channel: prefixedChannelName });
|
|
110
112
|
client
|
|
111
|
-
.subscribe(
|
|
113
|
+
.subscribe(prefixedChannelName, fn)
|
|
112
114
|
.then(() => {
|
|
113
115
|
client._subscribedChannels ??= {};
|
|
114
116
|
client._subscribedChannels[channel] = 1;
|
|
@@ -133,7 +135,7 @@ const _subscribeChannels = (options, subscribedChannels, errorHandlerCreateClien
|
|
|
133
135
|
|
|
134
136
|
const publishMessage = async (options, channel, message) => {
|
|
135
137
|
const client = await createMainClientAndConnect(options);
|
|
136
|
-
return await client.publish(channel, message);
|
|
138
|
+
return await client.publish([options.redisNamespace, channel].join("_"), message);
|
|
137
139
|
};
|
|
138
140
|
|
|
139
141
|
const closeMainClient = async () => {
|