@cap-js-community/event-queue 0.1.50 → 0.1.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cds-plugin.js +1 -1
- package/package.json +9 -2
- package/src/EventQueueError.js +5 -10
- package/src/EventQueueProcessorBase.js +291 -324
- package/src/config.js +1 -4
- package/src/dbHandler.js +1 -4
- package/src/initialize.js +11 -39
- package/src/processEventQueue.js +64 -155
- package/src/publishEvent.js +1 -3
- package/src/redisPubSub.js +6 -20
- package/src/runner.js +40 -78
- package/src/shared/PerformanceTracer.js +1 -3
- package/src/shared/SetIntervalDriftSafe.js +37 -0
- package/src/shared/WorkerQueue.js +3 -14
- package/src/shared/cdsHelper.js +2 -11
- package/src/shared/common.js +2 -7
- package/src/shared/distributedLock.js +41 -85
- package/src/shared/redis.js +1 -3
package/src/config.js
CHANGED
|
@@ -23,7 +23,6 @@ class Config {
|
|
|
23
23
|
this.__tableNameEventLock = null;
|
|
24
24
|
this.__vcapServices = this._parseVcapServices();
|
|
25
25
|
this.__isRunnerDeactivated = false;
|
|
26
|
-
this.__eventsForAutomaticRun = null;
|
|
27
26
|
this.__configFilePath = null;
|
|
28
27
|
this.__processEventsAfterPublish = null;
|
|
29
28
|
this.__skipCsnCheck = null;
|
|
@@ -34,9 +33,7 @@ class Config {
|
|
|
34
33
|
}
|
|
35
34
|
|
|
36
35
|
hasEventAfterCommitFlag(type, subType) {
|
|
37
|
-
return (
|
|
38
|
-
this.__eventMap[[type, subType].join("##")]?.processAfterCommit ?? true
|
|
39
|
-
);
|
|
36
|
+
return this.__eventMap[[type, subType].join("##")]?.processAfterCommit ?? true;
|
|
40
37
|
}
|
|
41
38
|
|
|
42
39
|
_checkRedisIsBound() {
|
package/src/dbHandler.js
CHANGED
|
@@ -14,10 +14,7 @@ const registerEventQueueDbHandler = (dbService) => {
|
|
|
14
14
|
const eventCombinations = Object.keys(
|
|
15
15
|
data.reduce((result, event) => {
|
|
16
16
|
const key = [event.type, event.subType].join("##");
|
|
17
|
-
if (
|
|
18
|
-
!configInstance.hasEventAfterCommitFlag(event.type, event.subType) ||
|
|
19
|
-
eventQueuePublishEvents[key]
|
|
20
|
-
) {
|
|
17
|
+
if (!configInstance.hasEventAfterCommitFlag(event.type, event.subType) || eventQueuePublishEvents[key]) {
|
|
21
18
|
return result;
|
|
22
19
|
}
|
|
23
20
|
eventQueuePublishEvents[key] = true;
|
package/src/initialize.js
CHANGED
|
@@ -56,26 +56,13 @@ const initialize = async ({
|
|
|
56
56
|
);
|
|
57
57
|
|
|
58
58
|
const logger = cds.log(COMPONENT);
|
|
59
|
-
configInstance.fileContent = await readConfigFromFile(
|
|
60
|
-
configInstance.configFilePath
|
|
61
|
-
);
|
|
59
|
+
configInstance.fileContent = await readConfigFromFile(configInstance.configFilePath);
|
|
62
60
|
configInstance.calculateIsRedisEnabled();
|
|
63
61
|
|
|
64
62
|
const dbService = await cds.connect.to("db");
|
|
65
|
-
await (cds.model
|
|
66
|
-
? Promise.resolve()
|
|
67
|
-
: new Promise((resolve) => cds.on("serving", resolve)));
|
|
63
|
+
await (cds.model ? Promise.resolve() : new Promise((resolve) => cds.on("serving", resolve)));
|
|
68
64
|
!configInstance.skipCsnCheck && (await csnCheck());
|
|
69
65
|
if (configInstance.processEventsAfterPublish) {
|
|
70
|
-
// TODO: remove this as soon as CDS fixes the current plugin model issues --> cds 7
|
|
71
|
-
if (BASE_TABLES.EVENT === configInstance.tableNameEventQueue) {
|
|
72
|
-
cds.db.model.definitions[BASE_TABLES.EVENT] =
|
|
73
|
-
cds.model.definitions[BASE_TABLES.EVENT];
|
|
74
|
-
}
|
|
75
|
-
if (BASE_TABLES.LOCK === configInstance.tableNameEventLock) {
|
|
76
|
-
cds.db.model.definitions[BASE_TABLES.LOCK] =
|
|
77
|
-
cds.model.definitions[BASE_TABLES.LOCK];
|
|
78
|
-
}
|
|
79
66
|
dbHandler.registerEventQueueDbHandler(dbService);
|
|
80
67
|
}
|
|
81
68
|
|
|
@@ -161,8 +148,7 @@ const checkCustomTable = (baseCsn, customCsn) => {
|
|
|
161
148
|
|
|
162
149
|
if (
|
|
163
150
|
customCsn.elements[columnName].type !== "cds.Association" &&
|
|
164
|
-
customCsn.elements[columnName].type !==
|
|
165
|
-
baseCsn.elements[columnName].type &&
|
|
151
|
+
customCsn.elements[columnName].type !== baseCsn.elements[columnName].type &&
|
|
166
152
|
columnName === "status" &&
|
|
167
153
|
customCsn.elements[columnName].type !== "cds.Integer"
|
|
168
154
|
) {
|
|
@@ -183,32 +169,18 @@ const mixConfigVarsWithEnv = (
|
|
|
183
169
|
) => {
|
|
184
170
|
const configInstance = getConfigInstance();
|
|
185
171
|
|
|
186
|
-
configInstance.configFilePath =
|
|
187
|
-
configFilePath ?? cds.env.eventQueue?.configFilePath;
|
|
172
|
+
configInstance.configFilePath = configFilePath ?? cds.env.eventQueue?.configFilePath;
|
|
188
173
|
configInstance.registerAsEventProcessor =
|
|
189
|
-
registerAsEventProcessor ??
|
|
190
|
-
cds.env.eventQueue?.registerAsEventProcessor ??
|
|
191
|
-
true;
|
|
174
|
+
registerAsEventProcessor ?? cds.env.eventQueue?.registerAsEventProcessor ?? true;
|
|
192
175
|
configInstance.processEventsAfterPublish =
|
|
193
|
-
processEventsAfterPublish ??
|
|
194
|
-
|
|
195
|
-
true;
|
|
196
|
-
configInstance.runInterval =
|
|
197
|
-
runInterval ?? cds.env.eventQueue?.runInterval ?? 5 * 60 * 1000;
|
|
176
|
+
processEventsAfterPublish ?? cds.env.eventQueue?.processEventsAfterPublish ?? true;
|
|
177
|
+
configInstance.runInterval = runInterval ?? cds.env.eventQueue?.runInterval ?? 5 * 60 * 1000;
|
|
198
178
|
configInstance.parallelTenantProcessing =
|
|
199
|
-
parallelTenantProcessing ??
|
|
200
|
-
cds.env.eventQueue?.parallelTenantProcessing ??
|
|
201
|
-
5;
|
|
179
|
+
parallelTenantProcessing ?? cds.env.eventQueue?.parallelTenantProcessing ?? 5;
|
|
202
180
|
configInstance.tableNameEventQueue =
|
|
203
|
-
tableNameEventQueue ??
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
configInstance.tableNameEventLock =
|
|
207
|
-
tableNameEventLock ??
|
|
208
|
-
cds.env.eventQueue?.tableNameEventLock ??
|
|
209
|
-
BASE_TABLES.LOCK;
|
|
210
|
-
configInstance.skipCsnCheck =
|
|
211
|
-
skipCsnCheck ?? cds.env.eventQueue?.skipCsnCheck ?? false;
|
|
181
|
+
tableNameEventQueue ?? cds.env.eventQueue?.tableNameEventQueue ?? BASE_TABLES.EVENT;
|
|
182
|
+
configInstance.tableNameEventLock = tableNameEventLock ?? cds.env.eventQueue?.tableNameEventLock ?? BASE_TABLES.LOCK;
|
|
183
|
+
configInstance.skipCsnCheck = skipCsnCheck ?? cds.env.eventQueue?.skipCsnCheck ?? false;
|
|
212
184
|
};
|
|
213
185
|
|
|
214
186
|
module.exports = {
|
package/src/processEventQueue.js
CHANGED
|
@@ -9,10 +9,7 @@ const { TransactionMode } = require("./constants");
|
|
|
9
9
|
const { limiter, Funnel } = require("./shared/common");
|
|
10
10
|
|
|
11
11
|
const EventQueueBase = require("./EventQueueProcessorBase");
|
|
12
|
-
const {
|
|
13
|
-
executeInNewTransaction,
|
|
14
|
-
TriggerRollback,
|
|
15
|
-
} = require("./shared/cdsHelper");
|
|
12
|
+
const { executeInNewTransaction, TriggerRollback } = require("./shared/cdsHelper");
|
|
16
13
|
|
|
17
14
|
const COMPONENT_NAME = "eventQueue/processEventQueue";
|
|
18
15
|
const MAX_EXECUTION_TIME = 5 * 60 * 1000;
|
|
@@ -22,47 +19,24 @@ const eventQueueRunner = async (context, events) => {
|
|
|
22
19
|
const funnel = new Funnel();
|
|
23
20
|
await Promise.allSettled(
|
|
24
21
|
events.map((event) =>
|
|
25
|
-
funnel.run(event.load, async () =>
|
|
26
|
-
processEventQueue(context, event.type, event.subType, startTime)
|
|
27
|
-
)
|
|
22
|
+
funnel.run(event.load, async () => processEventQueue(context, event.type, event.subType, startTime))
|
|
28
23
|
)
|
|
29
24
|
);
|
|
30
25
|
};
|
|
31
26
|
|
|
32
|
-
const processEventQueue = async (
|
|
33
|
-
context,
|
|
34
|
-
eventType,
|
|
35
|
-
eventSubType,
|
|
36
|
-
startTime = new Date()
|
|
37
|
-
) => {
|
|
27
|
+
const processEventQueue = async (context, eventType, eventSubType, startTime = new Date()) => {
|
|
38
28
|
let iterationCounter = 0;
|
|
39
29
|
let shouldContinue = true;
|
|
40
30
|
let baseInstance;
|
|
41
31
|
try {
|
|
42
32
|
let eventTypeInstance;
|
|
43
|
-
const eventConfig = getConfigInstance().getEventConfig(
|
|
44
|
-
eventType,
|
|
45
|
-
eventSubType
|
|
46
|
-
);
|
|
33
|
+
const eventConfig = getConfigInstance().getEventConfig(eventType, eventSubType);
|
|
47
34
|
const [err, EventTypeClass] = resilientRequire(eventConfig?.impl);
|
|
48
|
-
if (
|
|
49
|
-
|
|
50
|
-
err ||
|
|
51
|
-
!(typeof EventTypeClass.constructor === "function")
|
|
52
|
-
) {
|
|
53
|
-
await EventQueueBase.handleMissingTypeImplementation(
|
|
54
|
-
context,
|
|
55
|
-
eventType,
|
|
56
|
-
eventSubType
|
|
57
|
-
);
|
|
35
|
+
if (!eventConfig || err || !(typeof EventTypeClass.constructor === "function")) {
|
|
36
|
+
await EventQueueBase.handleMissingTypeImplementation(context, eventType, eventSubType);
|
|
58
37
|
return;
|
|
59
38
|
}
|
|
60
|
-
baseInstance = new EventTypeClass(
|
|
61
|
-
context,
|
|
62
|
-
eventType,
|
|
63
|
-
eventSubType,
|
|
64
|
-
eventConfig
|
|
65
|
-
);
|
|
39
|
+
baseInstance = new EventTypeClass(context, eventType, eventSubType, eventConfig);
|
|
66
40
|
const continueProcessing = await baseInstance.handleDistributedLock();
|
|
67
41
|
if (!continueProcessing) {
|
|
68
42
|
return;
|
|
@@ -70,119 +44,79 @@ const processEventQueue = async (
|
|
|
70
44
|
eventConfig.startTime = startTime;
|
|
71
45
|
while (shouldContinue) {
|
|
72
46
|
iterationCounter++;
|
|
73
|
-
await executeInNewTransaction(
|
|
74
|
-
context,
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
await eventTypeInstance.getQueueEntriesAndSetToInProgress();
|
|
85
|
-
eventTypeInstance.startPerformanceTracerPreprocessing();
|
|
86
|
-
for (const queueEntry of queueEntries) {
|
|
87
|
-
try {
|
|
88
|
-
eventTypeInstance.modifyQueueEntry(queueEntry);
|
|
89
|
-
const payload =
|
|
90
|
-
await eventTypeInstance.checkEventAndGeneratePayload(
|
|
91
|
-
queueEntry
|
|
92
|
-
);
|
|
93
|
-
if (payload === null) {
|
|
94
|
-
eventTypeInstance.setStatusToDone(queueEntry);
|
|
95
|
-
continue;
|
|
96
|
-
}
|
|
97
|
-
if (payload === undefined) {
|
|
98
|
-
eventTypeInstance.handleInvalidPayloadReturned(queueEntry);
|
|
99
|
-
continue;
|
|
100
|
-
}
|
|
101
|
-
eventTypeInstance.addEventWithPayloadForProcessing(
|
|
102
|
-
queueEntry,
|
|
103
|
-
payload
|
|
104
|
-
);
|
|
105
|
-
} catch (err) {
|
|
106
|
-
eventTypeInstance.handleErrorDuringProcessing(err, queueEntry);
|
|
47
|
+
await executeInNewTransaction(context, `eventQueue-pre-processing-${eventType}##${eventSubType}`, async (tx) => {
|
|
48
|
+
eventTypeInstance = new EventTypeClass(tx.context, eventType, eventSubType, eventConfig);
|
|
49
|
+
const queueEntries = await eventTypeInstance.getQueueEntriesAndSetToInProgress();
|
|
50
|
+
eventTypeInstance.startPerformanceTracerPreprocessing();
|
|
51
|
+
for (const queueEntry of queueEntries) {
|
|
52
|
+
try {
|
|
53
|
+
eventTypeInstance.modifyQueueEntry(queueEntry);
|
|
54
|
+
const payload = await eventTypeInstance.checkEventAndGeneratePayload(queueEntry);
|
|
55
|
+
if (payload === null) {
|
|
56
|
+
eventTypeInstance.setStatusToDone(queueEntry);
|
|
57
|
+
continue;
|
|
107
58
|
}
|
|
59
|
+
if (payload === undefined) {
|
|
60
|
+
eventTypeInstance.handleInvalidPayloadReturned(queueEntry);
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
eventTypeInstance.addEventWithPayloadForProcessing(queueEntry, payload);
|
|
64
|
+
} catch (err) {
|
|
65
|
+
eventTypeInstance.handleErrorDuringProcessing(err, queueEntry);
|
|
108
66
|
}
|
|
109
|
-
throw new TriggerRollback();
|
|
110
67
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
context,
|
|
115
|
-
`eventQueue-handleExceededEvents-${eventType}##${eventSubType}`,
|
|
116
|
-
async (tx) => {
|
|
117
|
-
eventTypeInstance.processEventContext = tx.context;
|
|
118
|
-
await eventTypeInstance.handleExceededEvents(
|
|
119
|
-
eventTypeInstance.exceededEvents
|
|
120
|
-
);
|
|
121
|
-
}
|
|
122
|
-
));
|
|
68
|
+
throw new TriggerRollback();
|
|
69
|
+
});
|
|
70
|
+
await eventTypeInstance.handleExceededEvents();
|
|
123
71
|
if (!eventTypeInstance) {
|
|
124
72
|
return;
|
|
125
73
|
}
|
|
126
74
|
eventTypeInstance.endPerformanceTracerPreprocessing();
|
|
127
75
|
if (Object.keys(eventTypeInstance.queueEntriesWithPayloadMap).length) {
|
|
76
|
+
await executeInNewTransaction(context, `eventQueue-processing-${eventType}##${eventSubType}`, async (tx) => {
|
|
77
|
+
eventTypeInstance.processEventContext = tx.context;
|
|
78
|
+
try {
|
|
79
|
+
eventTypeInstance.clusterQueueEntries();
|
|
80
|
+
await processEventMap(eventTypeInstance);
|
|
81
|
+
} catch (err) {
|
|
82
|
+
eventTypeInstance.handleErrorDuringClustering(err);
|
|
83
|
+
}
|
|
84
|
+
if (
|
|
85
|
+
eventTypeInstance.transactionMode !== TransactionMode.alwaysCommit ||
|
|
86
|
+
Object.entries(eventTypeInstance.eventProcessingMap).some(([key]) =>
|
|
87
|
+
eventTypeInstance.shouldRollbackTransaction(key)
|
|
88
|
+
)
|
|
89
|
+
) {
|
|
90
|
+
throw new TriggerRollback();
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
await executeInNewTransaction(context, `eventQueue-persistStatus-${eventType}##${eventSubType}`, async (tx) => {
|
|
95
|
+
await eventTypeInstance.persistEventStatus(tx);
|
|
96
|
+
});
|
|
97
|
+
shouldContinue = reevaluateShouldContinue(eventTypeInstance, iterationCounter, startTime);
|
|
98
|
+
if (!shouldContinue) {
|
|
128
99
|
await executeInNewTransaction(
|
|
129
100
|
context,
|
|
130
|
-
`eventQueue-
|
|
101
|
+
`eventQueue-deleteFinishedEvents-${eventType}##${eventSubType}`,
|
|
131
102
|
async (tx) => {
|
|
132
|
-
eventTypeInstance.
|
|
133
|
-
try {
|
|
134
|
-
eventTypeInstance.clusterQueueEntries();
|
|
135
|
-
await processEventMap(eventTypeInstance);
|
|
136
|
-
} catch (err) {
|
|
137
|
-
eventTypeInstance.handleErrorDuringClustering(err);
|
|
138
|
-
}
|
|
139
|
-
if (
|
|
140
|
-
eventTypeInstance.transactionMode !==
|
|
141
|
-
TransactionMode.alwaysCommit ||
|
|
142
|
-
Object.entries(eventTypeInstance.eventProcessingMap).some(
|
|
143
|
-
([key]) => eventTypeInstance.shouldRollbackTransaction(key)
|
|
144
|
-
)
|
|
145
|
-
) {
|
|
146
|
-
throw new TriggerRollback();
|
|
147
|
-
}
|
|
103
|
+
await eventTypeInstance.deleteFinishedEvents(tx);
|
|
148
104
|
}
|
|
149
105
|
);
|
|
150
106
|
}
|
|
151
|
-
await executeInNewTransaction(
|
|
152
|
-
context,
|
|
153
|
-
`eventQueue-persistStatus-${eventType}##${eventSubType}`,
|
|
154
|
-
async (tx) => {
|
|
155
|
-
await eventTypeInstance.persistEventStatus(tx);
|
|
156
|
-
}
|
|
157
|
-
);
|
|
158
|
-
shouldContinue = reevaluateShouldContinue(
|
|
159
|
-
eventTypeInstance,
|
|
160
|
-
iterationCounter,
|
|
161
|
-
startTime
|
|
162
|
-
);
|
|
163
107
|
}
|
|
164
108
|
} catch (err) {
|
|
165
|
-
cds
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
err,
|
|
170
|
-
{
|
|
171
|
-
eventType,
|
|
172
|
-
eventSubType,
|
|
173
|
-
}
|
|
174
|
-
);
|
|
109
|
+
cds.log(COMPONENT_NAME).error("Processing event queue failed with unexpected error. Error:", err, {
|
|
110
|
+
eventType,
|
|
111
|
+
eventSubType,
|
|
112
|
+
});
|
|
175
113
|
} finally {
|
|
176
114
|
await baseInstance?.handleReleaseLock();
|
|
177
115
|
}
|
|
178
116
|
};
|
|
179
117
|
|
|
180
|
-
const reevaluateShouldContinue = (
|
|
181
|
-
eventTypeInstance
|
|
182
|
-
iterationCounter,
|
|
183
|
-
startTime
|
|
184
|
-
) => {
|
|
185
|
-
if (!eventTypeInstance.getSelectNextChunk()) {
|
|
118
|
+
const reevaluateShouldContinue = (eventTypeInstance, iterationCounter, startTime) => {
|
|
119
|
+
if (!eventTypeInstance.selectNextChunk) {
|
|
186
120
|
return false; // no select next chunk configured for this event
|
|
187
121
|
}
|
|
188
122
|
if (eventTypeInstance.emptyChunkSelected) {
|
|
@@ -211,13 +145,7 @@ const processEventMap = async (eventTypeInstance) => {
|
|
|
211
145
|
eventTypeInstance.baseContext,
|
|
212
146
|
`eventQueue-processEvent-${eventTypeInstance.eventType}##${eventTypeInstance.eventSubType}`,
|
|
213
147
|
async (tx) => {
|
|
214
|
-
statusMap = await _processEvent(
|
|
215
|
-
eventTypeInstance,
|
|
216
|
-
tx.context,
|
|
217
|
-
key,
|
|
218
|
-
queueEntries,
|
|
219
|
-
payload
|
|
220
|
-
);
|
|
148
|
+
statusMap = await _processEvent(eventTypeInstance, tx.context, key, queueEntries, payload);
|
|
221
149
|
if (
|
|
222
150
|
eventTypeInstance.statusMapContainsError(statusMap) ||
|
|
223
151
|
eventTypeInstance.shouldRollbackTransaction(key)
|
|
@@ -238,13 +166,7 @@ const processEventMap = async (eventTypeInstance) => {
|
|
|
238
166
|
}
|
|
239
167
|
);
|
|
240
168
|
} else {
|
|
241
|
-
await _processEvent(
|
|
242
|
-
eventTypeInstance,
|
|
243
|
-
eventTypeInstance.context,
|
|
244
|
-
key,
|
|
245
|
-
queueEntries,
|
|
246
|
-
payload
|
|
247
|
-
);
|
|
169
|
+
await _processEvent(eventTypeInstance, eventTypeInstance.context, key, queueEntries, payload);
|
|
248
170
|
}
|
|
249
171
|
}
|
|
250
172
|
).finally(() => {
|
|
@@ -256,28 +178,15 @@ const processEventMap = async (eventTypeInstance) => {
|
|
|
256
178
|
eventTypeInstance.endPerformanceTracerEvents();
|
|
257
179
|
};
|
|
258
180
|
|
|
259
|
-
const _processEvent = async (
|
|
260
|
-
eventTypeInstance,
|
|
261
|
-
processContext,
|
|
262
|
-
key,
|
|
263
|
-
queueEntries,
|
|
264
|
-
payload
|
|
265
|
-
) => {
|
|
181
|
+
const _processEvent = async (eventTypeInstance, processContext, key, queueEntries, payload) => {
|
|
266
182
|
try {
|
|
267
183
|
eventTypeInstance.logStartMessage(queueEntries);
|
|
268
|
-
const eventOutdated = await eventTypeInstance.isOutdatedAndKeepalive(
|
|
269
|
-
queueEntries
|
|
270
|
-
);
|
|
184
|
+
const eventOutdated = await eventTypeInstance.isOutdatedAndKeepalive(queueEntries);
|
|
271
185
|
if (eventOutdated) {
|
|
272
186
|
return;
|
|
273
187
|
}
|
|
274
188
|
eventTypeInstance.setTxForEventProcessing(key, cds.tx(processContext));
|
|
275
|
-
const statusTuple = await eventTypeInstance.processEvent(
|
|
276
|
-
processContext,
|
|
277
|
-
key,
|
|
278
|
-
queueEntries,
|
|
279
|
-
payload
|
|
280
|
-
);
|
|
189
|
+
const statusTuple = await eventTypeInstance.processEvent(processContext, key, queueEntries, payload);
|
|
281
190
|
return eventTypeInstance.setEventStatus(queueEntries, statusTuple);
|
|
282
191
|
} catch (err) {
|
|
283
192
|
return eventTypeInstance.handleErrorDuringProcessing(err, queueEntries);
|
package/src/publishEvent.js
CHANGED
|
@@ -15,9 +15,7 @@ const publishEvent = async (tx, events) => {
|
|
|
15
15
|
throw EventQueueError.unknownEventType(type, subType);
|
|
16
16
|
}
|
|
17
17
|
}
|
|
18
|
-
return await tx.run(
|
|
19
|
-
INSERT.into(configInstance.tableNameEventQueue).entries(eventsForProcessing)
|
|
20
|
-
);
|
|
18
|
+
return await tx.run(INSERT.into(configInstance.tableNameEventQueue).entries(eventsForProcessing));
|
|
21
19
|
};
|
|
22
20
|
|
|
23
21
|
module.exports = {
|
package/src/redisPubSub.js
CHANGED
|
@@ -22,15 +22,11 @@ const initEventQueueRedisSubscribe = () => {
|
|
|
22
22
|
|
|
23
23
|
const subscribeRedisClient = () => {
|
|
24
24
|
const errorHandlerCreateClient = (err) => {
|
|
25
|
-
cds
|
|
26
|
-
.log(COMPONENT_NAME)
|
|
27
|
-
.error("error from redis client for pub/sub failed", err);
|
|
25
|
+
cds.log(COMPONENT_NAME).error("error from redis client for pub/sub failed", err);
|
|
28
26
|
subscriberClientPromise = null;
|
|
29
27
|
setTimeout(subscribeRedisClient, 5 * 1000).unref();
|
|
30
28
|
};
|
|
31
|
-
subscriberClientPromise = redis.createClientAndConnect(
|
|
32
|
-
errorHandlerCreateClient
|
|
33
|
-
);
|
|
29
|
+
subscriberClientPromise = redis.createClientAndConnect(errorHandlerCreateClient);
|
|
34
30
|
subscriberClientPromise
|
|
35
31
|
.then((client) => {
|
|
36
32
|
cds.log(COMPONENT_NAME).info("subscribe redis client connected");
|
|
@@ -39,10 +35,7 @@ const subscribeRedisClient = () => {
|
|
|
39
35
|
.catch((err) => {
|
|
40
36
|
cds
|
|
41
37
|
.log(COMPONENT_NAME)
|
|
42
|
-
.error(
|
|
43
|
-
"error from redis client for pub/sub failed during startup - trying to reconnect",
|
|
44
|
-
err
|
|
45
|
-
);
|
|
38
|
+
.error("error from redis client for pub/sub failed during startup - trying to reconnect", err);
|
|
46
39
|
});
|
|
47
40
|
};
|
|
48
41
|
|
|
@@ -68,9 +61,7 @@ const messageHandlerProcessEvents = async (messageData) => {
|
|
|
68
61
|
type,
|
|
69
62
|
subType,
|
|
70
63
|
});
|
|
71
|
-
getWorkerPoolInstance().addToQueue(async () =>
|
|
72
|
-
processEventQueue(context, type, subType)
|
|
73
|
-
);
|
|
64
|
+
getWorkerPoolInstance().addToQueue(async () => processEventQueue(context, type, subType));
|
|
74
65
|
};
|
|
75
66
|
|
|
76
67
|
const publishEvent = async (tenantId, type, subType) => {
|
|
@@ -89,9 +80,7 @@ const publishEvent = async (tenantId, type, subType) => {
|
|
|
89
80
|
};
|
|
90
81
|
try {
|
|
91
82
|
if (!publishClient) {
|
|
92
|
-
publishClient = await redis.createClientAndConnect(
|
|
93
|
-
errorHandlerCreateClient
|
|
94
|
-
);
|
|
83
|
+
publishClient = await redis.createClientAndConnect(errorHandlerCreateClient);
|
|
95
84
|
logger.info("publish redis client connected");
|
|
96
85
|
}
|
|
97
86
|
|
|
@@ -108,10 +97,7 @@ const publishEvent = async (tenantId, type, subType) => {
|
|
|
108
97
|
type,
|
|
109
98
|
subType,
|
|
110
99
|
});
|
|
111
|
-
await publishClient.publish(
|
|
112
|
-
MESSAGE_CHANNEL,
|
|
113
|
-
JSON.stringify({ tenantId, type, subType })
|
|
114
|
-
);
|
|
100
|
+
await publishClient.publish(MESSAGE_CHANNEL, JSON.stringify({ tenantId, type, subType }));
|
|
115
101
|
} catch (err) {
|
|
116
102
|
logger.error(`publish event failed with error: ${err.toString()}`, {
|
|
117
103
|
tenantId,
|
package/src/runner.js
CHANGED
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
const uuid = require("uuid");
|
|
4
4
|
|
|
5
5
|
const eventQueueConfig = require("./config");
|
|
6
|
-
const cdsHelper = require("./shared/cdsHelper");
|
|
7
6
|
const { eventQueueRunner } = require("./processEventQueue");
|
|
8
|
-
const distributedLock = require("./shared/distributedLock");
|
|
9
7
|
const { getWorkerPoolInstance } = require("./shared/WorkerQueue");
|
|
10
|
-
const
|
|
8
|
+
const cdsHelper = require("./shared/cdsHelper");
|
|
9
|
+
const distributedLock = require("./shared/distributedLock");
|
|
10
|
+
const SetIntervalDriftSafe = require("./shared/SetIntervalDriftSafe");
|
|
11
11
|
|
|
12
12
|
const COMPONENT_NAME = "eventQueue/runner";
|
|
13
13
|
const EVENT_QUEUE_RUN_ID = "EVENT_QUEUE_RUN_ID";
|
|
@@ -26,17 +26,13 @@ const _scheduleFunction = async (fn) => {
|
|
|
26
26
|
const configInstance = eventQueueConfig.getConfigInstance();
|
|
27
27
|
const eventsForAutomaticRun = configInstance.events;
|
|
28
28
|
if (!eventsForAutomaticRun.length) {
|
|
29
|
-
LOGGER.warn(
|
|
30
|
-
"no events for automatic run are configured - skipping runner registration"
|
|
31
|
-
);
|
|
29
|
+
LOGGER.warn("no events for automatic run are configured - skipping runner registration");
|
|
32
30
|
return;
|
|
33
31
|
}
|
|
34
32
|
|
|
35
33
|
const fnWithRunningCheck = () => {
|
|
36
34
|
if (configInstance.isRunnerDeactivated) {
|
|
37
|
-
LOGGER.info(
|
|
38
|
-
"runner is deactivated via config variable. Skipping this run."
|
|
39
|
-
);
|
|
35
|
+
LOGGER.info("runner is deactivated via config variable. Skipping this run.");
|
|
40
36
|
return;
|
|
41
37
|
}
|
|
42
38
|
return fn();
|
|
@@ -45,14 +41,13 @@ const _scheduleFunction = async (fn) => {
|
|
|
45
41
|
const offsetDependingOnLastRun = await _calculateOffsetForFirstRun();
|
|
46
42
|
|
|
47
43
|
LOGGER.info("first event-queue run scheduled", {
|
|
48
|
-
firstRunScheduledFor: new Date(
|
|
49
|
-
Date.now() + offsetDependingOnLastRun
|
|
50
|
-
).toISOString(),
|
|
44
|
+
firstRunScheduledFor: new Date(Date.now() + offsetDependingOnLastRun).toISOString(),
|
|
51
45
|
});
|
|
52
46
|
|
|
53
47
|
setTimeout(() => {
|
|
54
48
|
fnWithRunningCheck();
|
|
55
|
-
|
|
49
|
+
const intervalRunner = new SetIntervalDriftSafe(configInstance.runInterval);
|
|
50
|
+
intervalRunner.run(fnWithRunningCheck);
|
|
56
51
|
}, offsetDependingOnLastRun).unref();
|
|
57
52
|
};
|
|
58
53
|
|
|
@@ -72,9 +67,7 @@ const _multiTenancyRedis = async () => {
|
|
|
72
67
|
|
|
73
68
|
const _multiTenancyDb = async () => {
|
|
74
69
|
try {
|
|
75
|
-
LOGGER.info(
|
|
76
|
-
"executing event queue run for single instance and multi tenant"
|
|
77
|
-
);
|
|
70
|
+
LOGGER.info("executing event queue run for single instance and multi tenant");
|
|
78
71
|
const tenantIds = await cdsHelper.getAllTenantIds();
|
|
79
72
|
_executeAllTenants(tenantIds, EVENT_QUEUE_RUN_ID);
|
|
80
73
|
} catch (err) {
|
|
@@ -91,13 +84,9 @@ const _executeAllTenants = (tenantIds, runId) => {
|
|
|
91
84
|
workerQueueInstance.addToQueue(async () => {
|
|
92
85
|
try {
|
|
93
86
|
const tenantContext = new cds.EventContext({ tenant: tenantId });
|
|
94
|
-
const couldAcquireLock = await distributedLock.acquireLock(
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
{
|
|
98
|
-
expiryTime: configInstance.runInterval * 0.95,
|
|
99
|
-
}
|
|
100
|
-
);
|
|
87
|
+
const couldAcquireLock = await distributedLock.acquireLock(tenantContext, runId, {
|
|
88
|
+
expiryTime: configInstance.runInterval * 0.95,
|
|
89
|
+
});
|
|
101
90
|
if (!couldAcquireLock) {
|
|
102
91
|
return;
|
|
103
92
|
}
|
|
@@ -129,55 +118,38 @@ const _executeRunForTenant = async (tenantId, runId) => {
|
|
|
129
118
|
});
|
|
130
119
|
await eventQueueRunner(context, eventsForAutomaticRun);
|
|
131
120
|
} catch (err) {
|
|
132
|
-
LOGGER.error(
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
redisEnabled: configInstance.redisEnabled,
|
|
137
|
-
}
|
|
138
|
-
);
|
|
121
|
+
LOGGER.error(`Couldn't process eventQueue for tenant! Next try after defined interval. Error: ${err}`, {
|
|
122
|
+
tenantId,
|
|
123
|
+
redisEnabled: configInstance.redisEnabled,
|
|
124
|
+
});
|
|
139
125
|
}
|
|
140
126
|
};
|
|
141
127
|
|
|
142
128
|
const _acquireRunId = async (context) => {
|
|
143
129
|
const configInstance = eventQueueConfig.getConfigInstance();
|
|
144
130
|
let runId = uuid.v4();
|
|
145
|
-
const couldSetValue = await distributedLock.setValueWithExpire(
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
{
|
|
150
|
-
tenantScoped: false,
|
|
151
|
-
expiryTime: configInstance.runInterval * 0.95,
|
|
152
|
-
}
|
|
153
|
-
);
|
|
131
|
+
const couldSetValue = await distributedLock.setValueWithExpire(context, EVENT_QUEUE_RUN_ID, runId, {
|
|
132
|
+
tenantScoped: false,
|
|
133
|
+
expiryTime: configInstance.runInterval * 0.95,
|
|
134
|
+
});
|
|
154
135
|
|
|
155
136
|
if (couldSetValue) {
|
|
156
|
-
await distributedLock.setValueWithExpire(
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
tenantScoped: false,
|
|
162
|
-
expiryTime: configInstance.runInterval,
|
|
163
|
-
overrideValue: true,
|
|
164
|
-
}
|
|
165
|
-
);
|
|
137
|
+
await distributedLock.setValueWithExpire(context, EVENT_QUEUE_RUN_TS, new Date().toISOString(), {
|
|
138
|
+
tenantScoped: false,
|
|
139
|
+
expiryTime: configInstance.runInterval,
|
|
140
|
+
overrideValue: true,
|
|
141
|
+
});
|
|
166
142
|
} else {
|
|
167
|
-
runId = await distributedLock.checkLockExistsAndReturnValue(
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
{
|
|
171
|
-
tenantScoped: false,
|
|
172
|
-
}
|
|
173
|
-
);
|
|
143
|
+
runId = await distributedLock.checkLockExistsAndReturnValue(context, EVENT_QUEUE_RUN_ID, {
|
|
144
|
+
tenantScoped: false,
|
|
145
|
+
});
|
|
174
146
|
}
|
|
175
147
|
|
|
176
148
|
return runId;
|
|
177
149
|
};
|
|
178
150
|
|
|
179
151
|
const _calculateOffsetForFirstRun = async () => {
|
|
180
|
-
const configInstance = getConfigInstance();
|
|
152
|
+
const configInstance = eventQueueConfig.getConfigInstance();
|
|
181
153
|
let offsetDependingOnLastRun = OFFSET_FIRST_RUN;
|
|
182
154
|
const now = Date.now();
|
|
183
155
|
// NOTE: this is only supported with Redis, because this is a tenant agnostic information
|
|
@@ -185,34 +157,24 @@ const _calculateOffsetForFirstRun = async () => {
|
|
|
185
157
|
try {
|
|
186
158
|
if (configInstance.redisEnabled) {
|
|
187
159
|
const dummyContext = new cds.EventContext({});
|
|
188
|
-
let lastRunTs = await distributedLock.checkLockExistsAndReturnValue(
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
{ tenantScoped: false }
|
|
192
|
-
);
|
|
160
|
+
let lastRunTs = await distributedLock.checkLockExistsAndReturnValue(dummyContext, EVENT_QUEUE_RUN_TS, {
|
|
161
|
+
tenantScoped: false,
|
|
162
|
+
});
|
|
193
163
|
if (!lastRunTs) {
|
|
194
164
|
const ts = new Date(now).toISOString();
|
|
195
|
-
const couldSetValue = await distributedLock.setValueWithExpire(
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
{
|
|
200
|
-
tenantScoped: false,
|
|
201
|
-
expiryTime: configInstance.runInterval,
|
|
202
|
-
}
|
|
203
|
-
);
|
|
165
|
+
const couldSetValue = await distributedLock.setValueWithExpire(dummyContext, EVENT_QUEUE_RUN_TS, ts, {
|
|
166
|
+
tenantScoped: false,
|
|
167
|
+
expiryTime: configInstance.runInterval,
|
|
168
|
+
});
|
|
204
169
|
if (couldSetValue) {
|
|
205
170
|
lastRunTs = ts;
|
|
206
171
|
} else {
|
|
207
|
-
lastRunTs = await distributedLock.checkLockExistsAndReturnValue(
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
{ tenantScoped: false }
|
|
211
|
-
);
|
|
172
|
+
lastRunTs = await distributedLock.checkLockExistsAndReturnValue(dummyContext, EVENT_QUEUE_RUN_TS, {
|
|
173
|
+
tenantScoped: false,
|
|
174
|
+
});
|
|
212
175
|
}
|
|
213
176
|
}
|
|
214
|
-
offsetDependingOnLastRun =
|
|
215
|
-
new Date(lastRunTs).getTime() + configInstance.runInterval - now;
|
|
177
|
+
offsetDependingOnLastRun = new Date(lastRunTs).getTime() + configInstance.runInterval - now;
|
|
216
178
|
}
|
|
217
179
|
} catch (err) {
|
|
218
180
|
LOGGER.error(
|