@cap-js-community/event-queue 1.2.6 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/EventQueueError.js +19 -4
- package/src/config.js +31 -26
- package/src/constants.js +6 -0
- package/src/processEventQueue.js +39 -25
- package/src/runner.js +15 -12
- package/src/shared/WorkerQueue.js +87 -16
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cap-js-community/event-queue",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "An event queue that enables secure transactional processing of asynchronous and periodic events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"files": [
|
package/src/EventQueueError.js
CHANGED
|
@@ -17,8 +17,9 @@ const ERROR_CODES = {
|
|
|
17
17
|
DUPLICATE_EVENT_REGISTRATION: "DUPLICATE_EVENT_REGISTRATION",
|
|
18
18
|
NO_MANUEL_INSERT_OF_PERIODIC: "NO_MANUEL_INSERT_OF_PERIODIC",
|
|
19
19
|
LOAD_HIGHER_THAN_LIMIT: "LOAD_HIGHER_THAN_LIMIT",
|
|
20
|
+
NOT_ALLOWED_PRIORITY: "NOT_ALLOWED_PRIORITY",
|
|
20
21
|
SCHEMA_TENANT_MISMATCH: "SCHEMA_TENANT_MISMATCH",
|
|
21
|
-
|
|
22
|
+
GLOBAL_CDS_CONTEXT_MISMATCH: "GLOBAL_CDS_CONTEXT_MISMATCH",
|
|
22
23
|
};
|
|
23
24
|
|
|
24
25
|
const ERROR_CODES_META = {
|
|
@@ -65,10 +66,13 @@ const ERROR_CODES_META = {
|
|
|
65
66
|
[ERROR_CODES.LOAD_HIGHER_THAN_LIMIT]: {
|
|
66
67
|
message: "The defined load of an event is higher than the maximum defined limit. Check your configuration!",
|
|
67
68
|
},
|
|
69
|
+
[ERROR_CODES.NOT_ALLOWED_PRIORITY]: {
|
|
70
|
+
message: "The supplied priority is not allowed. Only LOW, MEDIUM, HIGH is allowed!",
|
|
71
|
+
},
|
|
68
72
|
[ERROR_CODES.SCHEMA_TENANT_MISMATCH]: {
|
|
69
73
|
message: "The db client associated to the tenant context does not match! Processing will be skipped.",
|
|
70
74
|
},
|
|
71
|
-
[ERROR_CODES.
|
|
75
|
+
[ERROR_CODES.GLOBAL_CDS_CONTEXT_MISMATCH]: {
|
|
72
76
|
message: "The global cds context does not match the local cds context.",
|
|
73
77
|
},
|
|
74
78
|
};
|
|
@@ -229,6 +233,17 @@ class EventQueueError extends VError {
|
|
|
229
233
|
);
|
|
230
234
|
}
|
|
231
235
|
|
|
236
|
+
static priorityNotAllowed(priority, label) {
|
|
237
|
+
const { message } = ERROR_CODES_META[ERROR_CODES.NOT_ALLOWED_PRIORITY];
|
|
238
|
+
return new EventQueueError(
|
|
239
|
+
{
|
|
240
|
+
name: ERROR_CODES.NOT_ALLOWED_PRIORITY,
|
|
241
|
+
info: { priority, label },
|
|
242
|
+
},
|
|
243
|
+
message
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
|
|
232
247
|
static dbClientSchemaMismatch(tenantId, dbClientSchema, serviceManagerSchema) {
|
|
233
248
|
const { message } = ERROR_CODES_META[ERROR_CODES.SCHEMA_TENANT_MISMATCH];
|
|
234
249
|
return new EventQueueError(
|
|
@@ -241,10 +256,10 @@ class EventQueueError extends VError {
|
|
|
241
256
|
}
|
|
242
257
|
|
|
243
258
|
static globalCdsContextNotMatchingLocal(globalProperties, localProperties) {
|
|
244
|
-
const { message } = ERROR_CODES_META[ERROR_CODES.
|
|
259
|
+
const { message } = ERROR_CODES_META[ERROR_CODES.GLOBAL_CDS_CONTEXT_MISMATCH];
|
|
245
260
|
return new EventQueueError(
|
|
246
261
|
{
|
|
247
|
-
name: ERROR_CODES.
|
|
262
|
+
name: ERROR_CODES.GLOBAL_CDS_CONTEXT_MISMATCH,
|
|
248
263
|
info: { globalProperties, localProperties },
|
|
249
264
|
},
|
|
250
265
|
message
|
package/src/config.js
CHANGED
|
@@ -5,6 +5,7 @@ const cds = require("@sap/cds");
|
|
|
5
5
|
const { getEnvInstance } = require("./shared/env");
|
|
6
6
|
const redis = require("./shared/redis");
|
|
7
7
|
const EventQueueError = require("./EventQueueError");
|
|
8
|
+
const { Priorities } = require("./constants");
|
|
8
9
|
|
|
9
10
|
const FOR_UPDATE_TIMEOUT = 10;
|
|
10
11
|
const GLOBAL_TX_TIMEOUT = 30 * 60 * 1000;
|
|
@@ -13,6 +14,7 @@ const REDIS_CONFIG_BLOCKLIST_CHANNEL = "REDIS_CONFIG_BLOCKLIST_CHANNEL";
|
|
|
13
14
|
const COMPONENT_NAME = "/eventQueue/config";
|
|
14
15
|
const MIN_INTERVAL_SEC = 10;
|
|
15
16
|
const DEFAULT_LOAD = 1;
|
|
17
|
+
const DEFAULT_PRIORITY = Priorities.Medium;
|
|
16
18
|
const SUFFIX_PERIODIC = "_PERIODIC";
|
|
17
19
|
const COMMAND_BLOCK = "EVENT_QUEUE_EVENT_BLOCK";
|
|
18
20
|
const COMMAND_UNBLOCK = "EVENT_QUEUE_EVENT_UNBLOCK";
|
|
@@ -24,6 +26,7 @@ const BASE_PERIODIC_EVENTS = [
|
|
|
24
26
|
{
|
|
25
27
|
type: "EVENT_QUEUE_BASE",
|
|
26
28
|
subType: "DELETE_EVENTS",
|
|
29
|
+
priority: Priorities.Low,
|
|
27
30
|
impl: "./housekeeping/EventQueueDeleteEvents",
|
|
28
31
|
load: 1,
|
|
29
32
|
interval: 86400, // 1 day,
|
|
@@ -51,8 +54,8 @@ class Config {
|
|
|
51
54
|
#env;
|
|
52
55
|
#eventMap;
|
|
53
56
|
#updatePeriodicEvents;
|
|
54
|
-
#
|
|
55
|
-
#
|
|
57
|
+
#blockedEvents;
|
|
58
|
+
#isEventBlockedCb;
|
|
56
59
|
#thresholdLoggingEventProcessing;
|
|
57
60
|
#useAsCAPOutbox;
|
|
58
61
|
#userId;
|
|
@@ -76,7 +79,7 @@ class Config {
|
|
|
76
79
|
this.#skipCsnCheck = null;
|
|
77
80
|
this.#disableRedis = null;
|
|
78
81
|
this.#env = getEnvInstance();
|
|
79
|
-
this.#
|
|
82
|
+
this.#blockedEvents = {};
|
|
80
83
|
}
|
|
81
84
|
|
|
82
85
|
getEventConfig(type, subType) {
|
|
@@ -100,7 +103,7 @@ class Config {
|
|
|
100
103
|
}
|
|
101
104
|
|
|
102
105
|
attachConfigChangeHandler() {
|
|
103
|
-
this.#
|
|
106
|
+
this.#attachBlockListChangeHandler();
|
|
104
107
|
redis.subscribeRedisChannel(REDIS_CONFIG_CHANNEL, (messageData) => {
|
|
105
108
|
try {
|
|
106
109
|
const { key, value } = JSON.parse(messageData);
|
|
@@ -126,14 +129,14 @@ class Config {
|
|
|
126
129
|
});
|
|
127
130
|
}
|
|
128
131
|
|
|
129
|
-
#
|
|
132
|
+
#attachBlockListChangeHandler() {
|
|
130
133
|
redis.subscribeRedisChannel(REDIS_CONFIG_BLOCKLIST_CHANNEL, (messageData) => {
|
|
131
134
|
try {
|
|
132
135
|
const { command, key, tenant } = JSON.parse(messageData);
|
|
133
136
|
if (command === COMMAND_BLOCK) {
|
|
134
|
-
this.#
|
|
137
|
+
this.#blockEventLocalState(key, tenant);
|
|
135
138
|
} else {
|
|
136
|
-
this.#
|
|
139
|
+
this.#unblockEventLocalState(key, tenant);
|
|
137
140
|
}
|
|
138
141
|
} catch (err) {
|
|
139
142
|
this.#logger.error("could not parse event blocklist change", err, {
|
|
@@ -143,14 +146,14 @@ class Config {
|
|
|
143
146
|
});
|
|
144
147
|
}
|
|
145
148
|
|
|
146
|
-
|
|
147
|
-
const typeWithSuffix = `${type}${SUFFIX_PERIODIC}`;
|
|
149
|
+
blockEvent(type, subType, isPeriodic, tenant = "*") {
|
|
150
|
+
const typeWithSuffix = `${type}${isPeriodic ? SUFFIX_PERIODIC : ""}`;
|
|
148
151
|
const config = this.getEventConfig(typeWithSuffix, subType);
|
|
149
152
|
if (!config) {
|
|
150
153
|
return;
|
|
151
154
|
}
|
|
152
155
|
const key = this.generateKey(typeWithSuffix, subType);
|
|
153
|
-
this.#
|
|
156
|
+
this.#blockEventLocalState(key, tenant);
|
|
154
157
|
if (!this.redisEnabled) {
|
|
155
158
|
return;
|
|
156
159
|
}
|
|
@@ -162,24 +165,24 @@ class Config {
|
|
|
162
165
|
});
|
|
163
166
|
}
|
|
164
167
|
|
|
165
|
-
#
|
|
166
|
-
this.#
|
|
167
|
-
this.#
|
|
168
|
+
#blockEventLocalState(key, tenant) {
|
|
169
|
+
this.#blockedEvents[key] ??= {};
|
|
170
|
+
this.#blockedEvents[key][tenant] = true;
|
|
168
171
|
return key;
|
|
169
172
|
}
|
|
170
173
|
|
|
171
174
|
clearPeriodicEventBlockList() {
|
|
172
|
-
this.#
|
|
175
|
+
this.#blockedEvents = {};
|
|
173
176
|
}
|
|
174
177
|
|
|
175
|
-
|
|
176
|
-
const typeWithSuffix = `${type}${SUFFIX_PERIODIC}`;
|
|
178
|
+
unblockEvent(type, subType, isPeriodic, tenant = "*") {
|
|
179
|
+
const typeWithSuffix = `${type}${isPeriodic ? SUFFIX_PERIODIC : ""}`;
|
|
177
180
|
const key = this.generateKey(typeWithSuffix, subType);
|
|
178
181
|
const config = this.getEventConfig(typeWithSuffix, subType);
|
|
179
182
|
if (!config) {
|
|
180
183
|
return;
|
|
181
184
|
}
|
|
182
|
-
this.#
|
|
185
|
+
this.#unblockEventLocalState(key, tenant);
|
|
183
186
|
if (!this.redisEnabled) {
|
|
184
187
|
return;
|
|
185
188
|
}
|
|
@@ -215,17 +218,17 @@ class Config {
|
|
|
215
218
|
this.#eventMap[this.generateKey(CAP_EVENT_TYPE, serviceName)] = eventConfig;
|
|
216
219
|
}
|
|
217
220
|
|
|
218
|
-
#
|
|
219
|
-
const map = this.#
|
|
221
|
+
#unblockEventLocalState(key, tenant) {
|
|
222
|
+
const map = this.#blockedEvents[key];
|
|
220
223
|
if (!map) {
|
|
221
224
|
return;
|
|
222
225
|
}
|
|
223
|
-
this.#
|
|
226
|
+
this.#blockedEvents[key][tenant] = false;
|
|
224
227
|
return key;
|
|
225
228
|
}
|
|
226
229
|
|
|
227
|
-
|
|
228
|
-
const map = this.#
|
|
230
|
+
isEventBlocked(type, subType, tenant) {
|
|
231
|
+
const map = this.#blockedEvents[this.generateKey(type, subType)];
|
|
229
232
|
if (!map) {
|
|
230
233
|
return false;
|
|
231
234
|
}
|
|
@@ -248,12 +251,14 @@ class Config {
|
|
|
248
251
|
config.periodicEvents = (config.periodicEvents ?? []).concat(BASE_PERIODIC_EVENTS.map((event) => ({ ...event })));
|
|
249
252
|
this.#eventMap = config.events.reduce((result, event) => {
|
|
250
253
|
event.load = event.load ?? DEFAULT_LOAD;
|
|
254
|
+
event.priority = event.priority ?? DEFAULT_PRIORITY;
|
|
251
255
|
this.validateAdHocEvents(result, event);
|
|
252
256
|
result[this.generateKey(event.type, event.subType)] = event;
|
|
253
257
|
return result;
|
|
254
258
|
}, {});
|
|
255
259
|
this.#eventMap = config.periodicEvents.reduce((result, event) => {
|
|
256
260
|
event.load = event.load ?? DEFAULT_LOAD;
|
|
261
|
+
event.priority = event.priority ?? DEFAULT_PRIORITY;
|
|
257
262
|
event.type = `${event.type}${SUFFIX_PERIODIC}`;
|
|
258
263
|
event.isPeriodic = true;
|
|
259
264
|
this.validatePeriodicConfig(result, event);
|
|
@@ -371,12 +376,12 @@ class Config {
|
|
|
371
376
|
this.#instanceLoadLimit = value;
|
|
372
377
|
}
|
|
373
378
|
|
|
374
|
-
get
|
|
375
|
-
return this.#
|
|
379
|
+
get isEventBlockedCb() {
|
|
380
|
+
return this.#isEventBlockedCb;
|
|
376
381
|
}
|
|
377
382
|
|
|
378
|
-
set
|
|
379
|
-
this.#
|
|
383
|
+
set isEventBlockedCb(value) {
|
|
384
|
+
this.#isEventBlockedCb = value;
|
|
380
385
|
}
|
|
381
386
|
|
|
382
387
|
get tableNameEventQueue() {
|
package/src/constants.js
CHANGED
package/src/processEventQueue.js
CHANGED
|
@@ -29,6 +29,10 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
|
|
|
29
29
|
return;
|
|
30
30
|
}
|
|
31
31
|
baseInstance = new EventTypeClass(context, eventType, eventSubType, eventConfig);
|
|
32
|
+
if (await _checkEventIsBlocked(baseInstance)) {
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
|
|
32
36
|
const continueProcessing = await baseInstance.acquireDistributedLock();
|
|
33
37
|
if (!continueProcessing) {
|
|
34
38
|
return;
|
|
@@ -119,31 +123,6 @@ const reevaluateShouldContinue = (eventTypeInstance, iterationCounter, startTime
|
|
|
119
123
|
};
|
|
120
124
|
|
|
121
125
|
const processPeriodicEvent = async (context, eventTypeInstance) => {
|
|
122
|
-
const isPeriodicEventBlockedCb = config.isPeriodicEventBlockedCb;
|
|
123
|
-
const params = [eventTypeInstance.eventType, eventTypeInstance.eventSubType, eventTypeInstance.context.tenant];
|
|
124
|
-
let eventBlocked = false;
|
|
125
|
-
if (isPeriodicEventBlockedCb) {
|
|
126
|
-
try {
|
|
127
|
-
eventBlocked = await isPeriodicEventBlockedCb(...params);
|
|
128
|
-
} catch (err) {
|
|
129
|
-
eventBlocked = true;
|
|
130
|
-
eventTypeInstance.logger.error("skipping run because periodic event blocked check failed!", err, {
|
|
131
|
-
type: eventTypeInstance.eventType,
|
|
132
|
-
subType: eventTypeInstance.eventSubType,
|
|
133
|
-
});
|
|
134
|
-
}
|
|
135
|
-
} else {
|
|
136
|
-
eventBlocked = config.isPeriodicEventBlocked(...params);
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
if (eventBlocked) {
|
|
140
|
-
eventTypeInstance.logger.info("skipping run because periodic event is blocked by configuration", {
|
|
141
|
-
type: eventTypeInstance.eventType,
|
|
142
|
-
subType: eventTypeInstance.eventSubType,
|
|
143
|
-
});
|
|
144
|
-
return;
|
|
145
|
-
}
|
|
146
|
-
|
|
147
126
|
try {
|
|
148
127
|
let queueEntry;
|
|
149
128
|
let processNext = true;
|
|
@@ -268,6 +247,41 @@ const processEventMap = async (eventTypeInstance) => {
|
|
|
268
247
|
eventTypeInstance.endPerformanceTracerEvents();
|
|
269
248
|
};
|
|
270
249
|
|
|
250
|
+
const _checkEventIsBlocked = async (baseInstance) => {
|
|
251
|
+
const isEventBlockedCb = config.isEventBlockedCb;
|
|
252
|
+
let eventBlocked;
|
|
253
|
+
if (isEventBlockedCb) {
|
|
254
|
+
try {
|
|
255
|
+
eventBlocked = await isEventBlockedCb(
|
|
256
|
+
baseInstance.eventType,
|
|
257
|
+
baseInstance.eventSubType,
|
|
258
|
+
baseInstance.isPeriodicEvent,
|
|
259
|
+
baseInstance.context.tenant
|
|
260
|
+
);
|
|
261
|
+
} catch (err) {
|
|
262
|
+
eventBlocked = true;
|
|
263
|
+
baseInstance.logger.error("skipping run because periodic event blocked check failed!", err, {
|
|
264
|
+
type: baseInstance.eventType,
|
|
265
|
+
subType: baseInstance.eventSubType,
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
} else {
|
|
269
|
+
eventBlocked = config.isEventBlocked(
|
|
270
|
+
baseInstance.eventType,
|
|
271
|
+
baseInstance.eventSubType,
|
|
272
|
+
baseInstance.context.tenant
|
|
273
|
+
);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
if (eventBlocked) {
|
|
277
|
+
baseInstance.logger.info("skipping run because periodic event is blocked by configuration", {
|
|
278
|
+
type: baseInstance.eventType,
|
|
279
|
+
subType: baseInstance.eventSubType,
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
return eventBlocked;
|
|
283
|
+
};
|
|
284
|
+
|
|
271
285
|
const _processEvent = async (eventTypeInstance, processContext, key, queueEntries, payload) => {
|
|
272
286
|
try {
|
|
273
287
|
const eventOutdated = await eventTypeInstance.isOutdatedAndKeepalive(queueEntries);
|
package/src/runner.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
const { randomUUID } = require("crypto");
|
|
4
|
+
const { AsyncResource } = require("async_hooks");
|
|
4
5
|
|
|
5
6
|
const cds = require("@sap/cds");
|
|
6
7
|
|
|
@@ -14,6 +15,7 @@ const { getSubdomainForTenantId } = require("./shared/cdsHelper");
|
|
|
14
15
|
const periodicEvents = require("./periodicEvents");
|
|
15
16
|
const { hashStringTo32Bit } = require("./shared/common");
|
|
16
17
|
const config = require("./config");
|
|
18
|
+
const { Priorities } = require("./constants");
|
|
17
19
|
|
|
18
20
|
const COMPONENT_NAME = "/eventQueue/runner";
|
|
19
21
|
const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
|
|
@@ -108,7 +110,7 @@ const _executeEventsAllTenants = (tenantIds, runId) => {
|
|
|
108
110
|
}, []);
|
|
109
111
|
|
|
110
112
|
return Promise.allSettled(
|
|
111
|
-
product.map(async ([tenantId,
|
|
113
|
+
product.map(async ([tenantId, eventConfig]) => {
|
|
112
114
|
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
113
115
|
const user = new cds.User.Privileged(config.userId);
|
|
114
116
|
const tenantContext = {
|
|
@@ -117,8 +119,8 @@ const _executeEventsAllTenants = (tenantIds, runId) => {
|
|
|
117
119
|
// NOTE: we need this because of logging otherwise logs would not contain the subdomain
|
|
118
120
|
http: { req: { authInfo: { getSubdomain: () => subdomain } } },
|
|
119
121
|
};
|
|
120
|
-
const label = `${
|
|
121
|
-
return await WorkerQueue.instance.addToQueue(
|
|
122
|
+
const label = `${eventConfig.type}_${eventConfig.subType}`;
|
|
123
|
+
return await WorkerQueue.instance.addToQueue(eventConfig.load, label, eventConfig.priority, async () => {
|
|
122
124
|
return await cds.tx(tenantContext, async ({ context }) => {
|
|
123
125
|
try {
|
|
124
126
|
const lockId = `${runId}_${label}`;
|
|
@@ -128,7 +130,7 @@ const _executeEventsAllTenants = (tenantIds, runId) => {
|
|
|
128
130
|
if (!couldAcquireLock) {
|
|
129
131
|
return;
|
|
130
132
|
}
|
|
131
|
-
await runEventCombinationForTenant(context,
|
|
133
|
+
await runEventCombinationForTenant(context, eventConfig.type, eventConfig.subType, true);
|
|
132
134
|
} catch (err) {
|
|
133
135
|
cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
|
|
134
136
|
tenantId,
|
|
@@ -144,7 +146,7 @@ const _executePeriodicEventsAllTenants = async (tenantIds, runId) => {
|
|
|
144
146
|
return await Promise.allSettled(
|
|
145
147
|
tenantIds.map(async (tenantId) => {
|
|
146
148
|
const label = `UPDATE_PERIODIC_EVENTS_${tenantId}`;
|
|
147
|
-
return await WorkerQueue.instance.addToQueue(1, label, async () => {
|
|
149
|
+
return await WorkerQueue.instance.addToQueue(1, label, Priorities.Low, async () => {
|
|
148
150
|
try {
|
|
149
151
|
const subdomain = await getSubdomainForTenantId(tenantId);
|
|
150
152
|
const user = new cds.User.Privileged(config.userId);
|
|
@@ -176,14 +178,14 @@ const _executePeriodicEventsAllTenants = async (tenantIds, runId) => {
|
|
|
176
178
|
|
|
177
179
|
const _singleTenantDb = async (tenantId) => {
|
|
178
180
|
return Promise.allSettled(
|
|
179
|
-
eventQueueConfig.allEvents.map(async (
|
|
180
|
-
const label = `${
|
|
181
|
+
eventQueueConfig.allEvents.map(async (eventConfig) => {
|
|
182
|
+
const label = `${eventConfig.type}_${eventConfig.subType}`;
|
|
181
183
|
const user = new cds.User.Privileged(config.userId);
|
|
182
184
|
const tenantContext = {
|
|
183
185
|
tenant: tenantId,
|
|
184
186
|
user,
|
|
185
187
|
};
|
|
186
|
-
return await WorkerQueue.instance.addToQueue(
|
|
188
|
+
return await WorkerQueue.instance.addToQueue(eventConfig.load, label, eventConfig.priority, async () => {
|
|
187
189
|
return await cds.tx(tenantContext, async ({ context }) => {
|
|
188
190
|
try {
|
|
189
191
|
const lockId = `${EVENT_QUEUE_RUN_ID}_${label}`;
|
|
@@ -193,7 +195,7 @@ const _singleTenantDb = async (tenantId) => {
|
|
|
193
195
|
if (!couldAcquireLock) {
|
|
194
196
|
return;
|
|
195
197
|
}
|
|
196
|
-
await runEventCombinationForTenant(context,
|
|
198
|
+
await runEventCombinationForTenant(context, eventConfig.type, eventConfig.subType, true);
|
|
197
199
|
} catch (err) {
|
|
198
200
|
cds.log(COMPONENT_NAME).error("executing event-queue run for tenant failed", {
|
|
199
201
|
tenantId,
|
|
@@ -268,12 +270,13 @@ const runEventCombinationForTenant = async (context, type, subType, skipWorkerPo
|
|
|
268
270
|
if (skipWorkerPool) {
|
|
269
271
|
return await processEventQueue(context, type, subType);
|
|
270
272
|
} else {
|
|
271
|
-
const
|
|
273
|
+
const eventConfig = eventQueueConfig.getEventConfig(type, subType);
|
|
272
274
|
const label = `${type}_${subType}`;
|
|
273
275
|
return await WorkerQueue.instance.addToQueue(
|
|
274
|
-
|
|
276
|
+
eventConfig.load,
|
|
275
277
|
label,
|
|
276
|
-
|
|
278
|
+
eventConfig.priority,
|
|
279
|
+
AsyncResource.bind(async () => await processEventQueue(context, type, subType))
|
|
277
280
|
);
|
|
278
281
|
}
|
|
279
282
|
} catch (err) {
|
|
@@ -4,15 +4,30 @@ const cds = require("@sap/cds");
|
|
|
4
4
|
|
|
5
5
|
const config = require("../config");
|
|
6
6
|
const EventQueueError = require("../EventQueueError");
|
|
7
|
+
const { Priorities } = require("../constants");
|
|
8
|
+
const SetIntervalDriftSafe = require("./SetIntervalDriftSafe");
|
|
9
|
+
|
|
10
|
+
const PRIORITIES = Object.values(Priorities).reverse();
|
|
11
|
+
const PRIORITY_MULTIPLICATOR = PRIORITIES.reduce((result, element, index) => {
|
|
12
|
+
result[element] = index + 1;
|
|
13
|
+
return result;
|
|
14
|
+
}, {});
|
|
7
15
|
|
|
8
16
|
const COMPONENT_NAME = "/eventQueue/WorkerQueue";
|
|
9
17
|
const NANO_TO_MS = 1e6;
|
|
18
|
+
const MIN_TO_MS = 60 * 1000;
|
|
19
|
+
const INCREASE_PRIORITY_AFTER = 3;
|
|
20
|
+
|
|
21
|
+
let lastLogTs;
|
|
22
|
+
|
|
10
23
|
const THRESHOLD = {
|
|
11
24
|
INFO: 35 * 1000,
|
|
12
25
|
WARN: 55 * 1000,
|
|
13
26
|
ERROR: 75 * 1000,
|
|
14
27
|
};
|
|
15
28
|
|
|
29
|
+
const CHECK_INTERVAL_QUEUE = 60 * 1000;
|
|
30
|
+
|
|
16
31
|
class WorkerQueue {
|
|
17
32
|
#concurrencyLimit;
|
|
18
33
|
#runningPromises;
|
|
@@ -28,24 +43,53 @@ class WorkerQueue {
|
|
|
28
43
|
}
|
|
29
44
|
this.#runningPromises = [];
|
|
30
45
|
this.#runningLoad = 0;
|
|
31
|
-
this.#queue =
|
|
46
|
+
this.#queue = PRIORITIES.reduce((result, priority) => {
|
|
47
|
+
result[priority] = [];
|
|
48
|
+
return result;
|
|
49
|
+
}, {});
|
|
50
|
+
|
|
51
|
+
const runner = new SetIntervalDriftSafe(CHECK_INTERVAL_QUEUE);
|
|
52
|
+
runner.run(this.#adjustPriority.bind(this));
|
|
32
53
|
}
|
|
33
54
|
|
|
34
|
-
addToQueue(load, label, cb) {
|
|
55
|
+
addToQueue(load, label, priority = Priorities.Medium, cb) {
|
|
35
56
|
if (load > this.#concurrencyLimit) {
|
|
36
57
|
throw EventQueueError.loadHigherThanLimit(load, label);
|
|
37
58
|
}
|
|
38
59
|
|
|
60
|
+
if (!PRIORITIES.includes(priority)) {
|
|
61
|
+
throw EventQueueError.priorityNotAllowed(priority, label);
|
|
62
|
+
}
|
|
63
|
+
|
|
39
64
|
const startTime = process.hrtime.bigint();
|
|
40
65
|
const p = new Promise((resolve, reject) => {
|
|
41
|
-
this.#queue.push([load, label, cb, resolve, reject, startTime]);
|
|
66
|
+
this.#queue[priority].push([load, label, cb, resolve, reject, startTime]);
|
|
42
67
|
});
|
|
43
|
-
this
|
|
68
|
+
this.#checkForNext();
|
|
44
69
|
return p;
|
|
45
70
|
}
|
|
46
71
|
|
|
47
|
-
|
|
48
|
-
|
|
72
|
+
#adjustPriority() {
|
|
73
|
+
const checkTime = process.hrtime.bigint();
|
|
74
|
+
const priorityValues = Object.values(Priorities);
|
|
75
|
+
|
|
76
|
+
for (let i = 0; i < priorityValues.length - 1; i++) {
|
|
77
|
+
const priority = priorityValues[i];
|
|
78
|
+
const nextPriority = priorityValues[i + 1];
|
|
79
|
+
for (let i = 0; i < this.queue[priority].length; i++) {
|
|
80
|
+
const queueEntry = this.queue[priority][i];
|
|
81
|
+
const startTime = queueEntry[6] ?? queueEntry[5];
|
|
82
|
+
if (Math.round(Number(checkTime - startTime) / NANO_TO_MS) > INCREASE_PRIORITY_AFTER * MIN_TO_MS) {
|
|
83
|
+
const [entry] = this.queue[priority].splice(i, 1);
|
|
84
|
+
entry.push(checkTime);
|
|
85
|
+
this.queue[nextPriority].push(entry);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
_executeFunction(load, label, cb, resolve, reject, startTime, priority) {
|
|
92
|
+
this.#checkAndLogWaitingTime(startTime, label, priority);
|
|
49
93
|
const promise = Promise.resolve().then(() => cb());
|
|
50
94
|
this.#runningPromises.push(promise);
|
|
51
95
|
this.#runningLoad = this.#runningLoad + load;
|
|
@@ -53,7 +97,7 @@ class WorkerQueue {
|
|
|
53
97
|
.finally(() => {
|
|
54
98
|
this.#runningLoad = this.#runningLoad - load;
|
|
55
99
|
this.#runningPromises.splice(this.#runningPromises.indexOf(promise), 1);
|
|
56
|
-
this
|
|
100
|
+
this.#checkForNext();
|
|
57
101
|
})
|
|
58
102
|
.then((...results) => {
|
|
59
103
|
resolve(...results);
|
|
@@ -62,15 +106,32 @@ class WorkerQueue {
|
|
|
62
106
|
cds.log(COMPONENT_NAME).error("Error happened in WorkQueue. Errors should be caught before!", err, { label });
|
|
63
107
|
reject(err);
|
|
64
108
|
});
|
|
109
|
+
|
|
110
|
+
if (this.#runningLoad !== this.#concurrencyLimit) {
|
|
111
|
+
this.#checkForNext();
|
|
112
|
+
}
|
|
65
113
|
}
|
|
66
114
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
if (!this.#queue.length || this.#runningLoad + load > this.#concurrencyLimit) {
|
|
115
|
+
#checkForNext() {
|
|
116
|
+
if (!this.#queue.length && this.#runningLoad === this.#concurrencyLimit) {
|
|
70
117
|
return;
|
|
71
118
|
}
|
|
72
|
-
|
|
73
|
-
|
|
119
|
+
|
|
120
|
+
let entryFound = false;
|
|
121
|
+
for (const priority of PRIORITIES) {
|
|
122
|
+
for (let i = 0; i < this.#queue[priority].length; i++) {
|
|
123
|
+
const [load] = this.#queue[priority][i];
|
|
124
|
+
if (this.#runningLoad + load <= this.#concurrencyLimit) {
|
|
125
|
+
const [args] = this.#queue[priority].splice(i, 1);
|
|
126
|
+
this._executeFunction(...args, priority);
|
|
127
|
+
entryFound = true;
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
if (entryFound) {
|
|
132
|
+
break;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
74
135
|
}
|
|
75
136
|
|
|
76
137
|
get runningPromises() {
|
|
@@ -87,14 +148,24 @@ class WorkerQueue {
|
|
|
87
148
|
return WorkerQueue.#instance;
|
|
88
149
|
}
|
|
89
150
|
|
|
90
|
-
|
|
151
|
+
get queue() {
|
|
152
|
+
return this.#queue;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
#checkAndLogWaitingTime(startTime, label, priority) {
|
|
156
|
+
const ts = Date.now();
|
|
157
|
+
if (ts - lastLogTs <= 1000) {
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
lastLogTs = ts;
|
|
91
161
|
const diffMs = Math.round(Number(process.hrtime.bigint() - startTime) / NANO_TO_MS);
|
|
162
|
+
const priorityMultiplication = PRIORITY_MULTIPLICATOR[priority];
|
|
92
163
|
let logLevel;
|
|
93
|
-
if (diffMs >= THRESHOLD.ERROR) {
|
|
164
|
+
if (diffMs >= THRESHOLD.ERROR * priorityMultiplication) {
|
|
94
165
|
logLevel = "error";
|
|
95
|
-
} else if (diffMs >= THRESHOLD.WARN) {
|
|
166
|
+
} else if (diffMs >= THRESHOLD.WARN * priorityMultiplication) {
|
|
96
167
|
logLevel = "warn";
|
|
97
|
-
} else if (diffMs >= THRESHOLD.INFO) {
|
|
168
|
+
} else if (diffMs >= THRESHOLD.INFO * priorityMultiplication) {
|
|
98
169
|
logLevel = "info";
|
|
99
170
|
} else {
|
|
100
171
|
logLevel = "debug";
|