@cap-js-community/event-queue 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/EventQueueError.js +14 -0
- package/src/EventQueueProcessorBase.js +83 -31
- package/src/config.js +32 -14
- package/src/dbHandler.js +2 -3
- package/src/index.js +1 -1
- package/src/initialize.js +23 -30
- package/src/periodicEvents.js +17 -16
- package/src/processEventQueue.js +59 -61
- package/src/publishEvent.js +3 -4
- package/src/redisPubSub.js +8 -6
- package/src/runner.js +89 -70
- package/src/shared/WorkerQueue.js +69 -28
- package/src/shared/cdsHelper.js +2 -2
- package/src/shared/common.js +1 -72
- package/src/shared/distributedLock.js +15 -21
- package/src/shared/eventScheduler.js +13 -8
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cap-js-community/event-queue",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.4",
|
|
4
4
|
"description": "An event queue that enables secure transactional processing of asynchronous events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"files": [
|
package/src/EventQueueError.js
CHANGED
|
@@ -16,6 +16,7 @@ const ERROR_CODES = {
|
|
|
16
16
|
MISSING_IMPL: "MISSING_IMPL",
|
|
17
17
|
DUPLICATE_EVENT_REGISTRATION: "DUPLICATE_EVENT_REGISTRATION",
|
|
18
18
|
NO_MANUEL_INSERT_OF_PERIODIC: "NO_MANUEL_INSERT_OF_PERIODIC",
|
|
19
|
+
LOAD_HIGHER_THAN_LIMIT: "LOAD_HIGHER_THAN_LIMIT",
|
|
19
20
|
};
|
|
20
21
|
|
|
21
22
|
const ERROR_CODES_META = {
|
|
@@ -59,6 +60,9 @@ const ERROR_CODES_META = {
|
|
|
59
60
|
[ERROR_CODES.NO_MANUEL_INSERT_OF_PERIODIC]: {
|
|
60
61
|
message: "Periodic events are managed by the framework and are not allowed to insert manually.",
|
|
61
62
|
},
|
|
63
|
+
[ERROR_CODES.LOAD_HIGHER_THAN_LIMIT]: {
|
|
64
|
+
message: "The defined load of an event is higher than the maximum defined limit. Check your configuration!",
|
|
65
|
+
},
|
|
62
66
|
};
|
|
63
67
|
|
|
64
68
|
class EventQueueError extends VError {
|
|
@@ -206,6 +210,16 @@ class EventQueueError extends VError {
|
|
|
206
210
|
message
|
|
207
211
|
);
|
|
208
212
|
}
|
|
213
|
+
static loadHigherThanLimit(load) {
|
|
214
|
+
const { message } = ERROR_CODES_META[ERROR_CODES.LOAD_HIGHER_THAN_LIMIT];
|
|
215
|
+
return new EventQueueError(
|
|
216
|
+
{
|
|
217
|
+
name: ERROR_CODES.LOAD_HIGHER_THAN_LIMIT,
|
|
218
|
+
info: { load },
|
|
219
|
+
},
|
|
220
|
+
message
|
|
221
|
+
);
|
|
222
|
+
}
|
|
209
223
|
}
|
|
210
224
|
|
|
211
225
|
module.exports = EventQueueError;
|
|
@@ -8,7 +8,7 @@ const distributedLock = require("./shared/distributedLock");
|
|
|
8
8
|
const EventQueueError = require("./EventQueueError");
|
|
9
9
|
const { arrayToFlatMap } = require("./shared/common");
|
|
10
10
|
const eventScheduler = require("./shared/eventScheduler");
|
|
11
|
-
const
|
|
11
|
+
const eventConfig = require("./config");
|
|
12
12
|
const PerformanceTracer = require("./shared/PerformanceTracer");
|
|
13
13
|
|
|
14
14
|
const IMPLEMENT_ERROR_MESSAGE = "needs to be reimplemented";
|
|
@@ -31,37 +31,43 @@ class EventQueueProcessorBase {
|
|
|
31
31
|
#eventType = null;
|
|
32
32
|
#eventSubType = null;
|
|
33
33
|
#config = null;
|
|
34
|
+
#eventSchedulerInstance = null;
|
|
35
|
+
#eventConfig;
|
|
36
|
+
#isPeriodic;
|
|
34
37
|
|
|
35
38
|
constructor(context, eventType, eventSubType, config) {
|
|
36
39
|
this.__context = context;
|
|
37
40
|
this.__baseContext = context;
|
|
38
41
|
this.__tx = cds.tx(context);
|
|
39
42
|
this.__baseLogger = cds.log(COMPONENT_NAME);
|
|
43
|
+
this.#eventSchedulerInstance = eventScheduler.getInstance();
|
|
44
|
+
this.#config = eventConfig;
|
|
45
|
+
this.#isPeriodic = this.#config.isPeriodicEvent(eventType, eventSubType);
|
|
40
46
|
this.__logger = null;
|
|
41
47
|
this.__eventProcessingMap = {};
|
|
42
48
|
this.__statusMap = {};
|
|
43
49
|
this.__commitedStatusMap = {};
|
|
44
50
|
this.#eventType = eventType;
|
|
45
51
|
this.#eventSubType = eventSubType;
|
|
46
|
-
this
|
|
47
|
-
this.__parallelEventProcessing = this.
|
|
52
|
+
this.#eventConfig = config ?? {};
|
|
53
|
+
this.__parallelEventProcessing = this.#eventConfig.parallelEventProcessing ?? DEFAULT_PARALLEL_EVENT_PROCESSING;
|
|
48
54
|
if (this.__parallelEventProcessing > LIMIT_PARALLEL_EVENT_PROCESSING) {
|
|
49
55
|
this.__parallelEventProcessing = LIMIT_PARALLEL_EVENT_PROCESSING;
|
|
50
56
|
}
|
|
51
57
|
// NOTE: keep the feature, this might be needed again
|
|
52
58
|
this.__concurrentEventProcessing = false;
|
|
53
|
-
this.__startTime = this.
|
|
54
|
-
this.__retryAttempts = this.
|
|
55
|
-
this.__selectMaxChunkSize = this.
|
|
56
|
-
this.__selectNextChunk = !!this.
|
|
59
|
+
this.__startTime = this.#eventConfig.startTime ?? new Date();
|
|
60
|
+
this.__retryAttempts = this.#eventConfig.retryAttempts ?? DEFAULT_RETRY_ATTEMPTS;
|
|
61
|
+
this.__selectMaxChunkSize = this.#eventConfig.selectMaxChunkSize ?? SELECT_LIMIT_EVENTS_PER_TICK;
|
|
62
|
+
this.__selectNextChunk = !!this.#eventConfig.checkForNextChunk;
|
|
57
63
|
this.__keepalivePromises = {};
|
|
58
|
-
this.__outdatedCheckEnabled = this.
|
|
59
|
-
this.__transactionMode = this.
|
|
60
|
-
if (this.
|
|
64
|
+
this.__outdatedCheckEnabled = this.#eventConfig.eventOutdatedCheck ?? true;
|
|
65
|
+
this.__transactionMode = this.#eventConfig.transactionMode ?? TransactionMode.isolated;
|
|
66
|
+
if (this.#eventConfig.deleteFinishedEventsAfterDays) {
|
|
61
67
|
this.__deleteFinishedEventsAfter =
|
|
62
|
-
Number.isInteger(this.
|
|
63
|
-
this.
|
|
64
|
-
? this.
|
|
68
|
+
Number.isInteger(this.#eventConfig.deleteFinishedEventsAfterDays) &&
|
|
69
|
+
this.#eventConfig.deleteFinishedEventsAfterDays > 0
|
|
70
|
+
? this.#eventConfig.deleteFinishedEventsAfterDays
|
|
65
71
|
: DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
|
|
66
72
|
} else {
|
|
67
73
|
this.__deleteFinishedEventsAfter = DEFAULT_DELETE_FINISHED_EVENTS_AFTER;
|
|
@@ -71,7 +77,6 @@ class EventQueueProcessorBase {
|
|
|
71
77
|
this.__txUsageAllowed = true;
|
|
72
78
|
this.__txMap = {};
|
|
73
79
|
this.__txRollback = {};
|
|
74
|
-
this.#config = eventQueueConfig.getConfigInstance();
|
|
75
80
|
this.__queueEntries = [];
|
|
76
81
|
}
|
|
77
82
|
|
|
@@ -93,6 +98,20 @@ class EventQueueProcessorBase {
|
|
|
93
98
|
throw new Error(IMPLEMENT_ERROR_MESSAGE);
|
|
94
99
|
}
|
|
95
100
|
|
|
101
|
+
/**
|
|
102
|
+
* Process one periodic event
|
|
103
|
+
* @param processContext the context valid for the event processing. This context is associated with a valid transaction
|
|
104
|
+
* Access to the context is also possible with this.getContextForEventProcessing(key).
|
|
105
|
+
* The associated tx can be accessed with this.getTxForEventProcessing(key).
|
|
106
|
+
* @param {string} key cluster key generated during the clustering step. By default, this is ID of the event queue entry
|
|
107
|
+
* @param {Object} queueEntry this is the queueEntry which should be processed
|
|
108
|
+
* @returns {Promise<undefined>}
|
|
109
|
+
*/
|
|
110
|
+
// eslint-disable-next-line no-unused-vars
|
|
111
|
+
async processPeriodicEvent(processContext, key, queueEntry) {
|
|
112
|
+
throw new Error(IMPLEMENT_ERROR_MESSAGE);
|
|
113
|
+
}
|
|
114
|
+
|
|
96
115
|
startPerformanceTracerEvents() {
|
|
97
116
|
this.__performanceLoggerEvents = new PerformanceTracer(this.logger, "Processing events");
|
|
98
117
|
}
|
|
@@ -249,7 +268,8 @@ class EventQueueProcessorBase {
|
|
|
249
268
|
this.#determineAndAddEventStatusToMap(queueEntry.ID, EventProcessingStatus.Error, statusMap)
|
|
250
269
|
);
|
|
251
270
|
this.logger.error(
|
|
252
|
-
|
|
271
|
+
"The supplied status tuple doesn't have the required structure. Setting all entries to error.",
|
|
272
|
+
error,
|
|
253
273
|
{
|
|
254
274
|
eventType: this.#eventType,
|
|
255
275
|
eventSubType: this.#eventSubType,
|
|
@@ -290,7 +310,8 @@ class EventQueueProcessorBase {
|
|
|
290
310
|
handleErrorDuringProcessing(error, queueEntries) {
|
|
291
311
|
queueEntries = Array.isArray(queueEntries) ? queueEntries : [queueEntries];
|
|
292
312
|
this.logger.error(
|
|
293
|
-
|
|
313
|
+
"Caught error during event processing - setting queue entry to error. Please catch your promises/exceptions",
|
|
314
|
+
error,
|
|
294
315
|
{
|
|
295
316
|
eventType: this.#eventType,
|
|
296
317
|
eventSubType: this.#eventSubType,
|
|
@@ -304,14 +325,11 @@ class EventQueueProcessorBase {
|
|
|
304
325
|
}
|
|
305
326
|
|
|
306
327
|
handleErrorDuringPeriodicEventProcessing(error, queueEntry) {
|
|
307
|
-
this.logger.error(
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
queueEntryId: queueEntry.ID,
|
|
313
|
-
}
|
|
314
|
-
);
|
|
328
|
+
this.logger.error("Caught error during event periodic processing. Please catch your promises/exceptions.", error, {
|
|
329
|
+
eventType: this.#eventType,
|
|
330
|
+
eventSubType: this.#eventSubType,
|
|
331
|
+
queueEntryId: queueEntry.ID,
|
|
332
|
+
});
|
|
315
333
|
}
|
|
316
334
|
|
|
317
335
|
async setPeriodicEventStatus(queueEntryIds) {
|
|
@@ -486,7 +504,7 @@ class EventQueueProcessorBase {
|
|
|
486
504
|
}
|
|
487
505
|
|
|
488
506
|
handleErrorDuringClustering(error) {
|
|
489
|
-
this.logger.error(
|
|
507
|
+
this.logger.error("Error during clustering of events - setting all queue entries to error.", error, {
|
|
490
508
|
eventType: this.#eventType,
|
|
491
509
|
eventSubType: this.#eventSubType,
|
|
492
510
|
});
|
|
@@ -610,9 +628,8 @@ class EventQueueProcessorBase {
|
|
|
610
628
|
}
|
|
611
629
|
|
|
612
630
|
#handleDelayedEvents(delayedEvents) {
|
|
613
|
-
const eventSchedulerInstance = eventScheduler.getInstance();
|
|
614
631
|
for (const delayedEvent of delayedEvents) {
|
|
615
|
-
eventSchedulerInstance.scheduleEvent(
|
|
632
|
+
this.#eventSchedulerInstance.scheduleEvent(
|
|
616
633
|
this.__context.tenant,
|
|
617
634
|
this.#eventType,
|
|
618
635
|
this.#eventSubType,
|
|
@@ -673,7 +690,8 @@ class EventQueueProcessorBase {
|
|
|
673
690
|
await this.#persistEventQueueStatusForExceeded(this.tx, [exceededEvent], EventProcessingStatus.Exceeded);
|
|
674
691
|
} catch (err) {
|
|
675
692
|
this.logger.error(
|
|
676
|
-
|
|
693
|
+
"Caught error during hook for exceeded events - setting queue entry to error. Please catch your promises/exceptions.",
|
|
694
|
+
err,
|
|
677
695
|
{
|
|
678
696
|
eventType: this.#eventType,
|
|
679
697
|
eventSubType: this.#eventSubType,
|
|
@@ -818,6 +836,10 @@ class EventQueueProcessorBase {
|
|
|
818
836
|
[this.#eventType, this.#eventSubType].join("##")
|
|
819
837
|
);
|
|
820
838
|
if (!lockAcquired) {
|
|
839
|
+
this.logger.debug("no lock available, exit processing", {
|
|
840
|
+
type: this.#eventType,
|
|
841
|
+
subType: this.#eventSubType,
|
|
842
|
+
});
|
|
821
843
|
return false;
|
|
822
844
|
}
|
|
823
845
|
this.__lockAcquired = true;
|
|
@@ -831,22 +853,52 @@ class EventQueueProcessorBase {
|
|
|
831
853
|
try {
|
|
832
854
|
await distributedLock.releaseLock(this.context, [this.#eventType, this.#eventSubType].join("##"));
|
|
833
855
|
} catch (err) {
|
|
834
|
-
this.logger.error("Releasing distributed lock failed.
|
|
856
|
+
this.logger.error("Releasing distributed lock failed.", err);
|
|
835
857
|
}
|
|
836
858
|
}
|
|
837
859
|
|
|
838
860
|
async scheduleNextPeriodEvent(queueEntry) {
|
|
839
|
-
const interval = this.
|
|
861
|
+
const interval = this.#eventConfig.interval;
|
|
840
862
|
const newEvent = {
|
|
841
863
|
type: this.#eventType,
|
|
842
864
|
subType: this.#eventSubType,
|
|
843
865
|
startAfter: new Date(new Date(queueEntry.startAfter).getTime() + interval * 1000),
|
|
844
866
|
};
|
|
867
|
+
const { relative } = this.#eventSchedulerInstance.calculateOffset(
|
|
868
|
+
this.#eventType,
|
|
869
|
+
this.#eventSubType,
|
|
870
|
+
newEvent.startAfter
|
|
871
|
+
);
|
|
872
|
+
|
|
873
|
+
// more than one interval behind - shift tick to keep up
|
|
874
|
+
if (relative < 0 && Math.abs(relative) >= this.#eventConfig.interval * 1000) {
|
|
875
|
+
newEvent.startAfter = new Date(Date.now() + 5 * 1000);
|
|
876
|
+
this.logger.info("interval adjusted because shifted more than one interval", {
|
|
877
|
+
eventType: this.#eventType,
|
|
878
|
+
eventSubType: this.#eventSubType,
|
|
879
|
+
newStartAfter: newEvent.startAfter,
|
|
880
|
+
});
|
|
881
|
+
}
|
|
882
|
+
|
|
845
883
|
this.tx._skipEventQueueBroadcase = true;
|
|
846
884
|
await this.tx.run(INSERT.into(this.#config.tableNameEventQueue).entries({ ...newEvent }));
|
|
847
885
|
this.tx._skipEventQueueBroadcase = false;
|
|
848
886
|
if (interval < this.#config.runInterval) {
|
|
849
887
|
this.#handleDelayedEvents([newEvent]);
|
|
888
|
+
const { relative: relativeAfterSchedule } = this.#eventSchedulerInstance.calculateOffset(
|
|
889
|
+
this.#eventType,
|
|
890
|
+
this.#eventSubType,
|
|
891
|
+
newEvent.startAfter
|
|
892
|
+
);
|
|
893
|
+
// next tick is already behind schedule --> execute direct
|
|
894
|
+
if (relativeAfterSchedule <= 0) {
|
|
895
|
+
this.logger.info("running behind schedule - executing next tick immediately", {
|
|
896
|
+
eventType: this.#eventType,
|
|
897
|
+
eventSubType: this.#eventSubType,
|
|
898
|
+
newStartAfter: newEvent.startAfter,
|
|
899
|
+
});
|
|
900
|
+
return true;
|
|
901
|
+
}
|
|
850
902
|
}
|
|
851
903
|
}
|
|
852
904
|
|
|
@@ -986,7 +1038,7 @@ class EventQueueProcessorBase {
|
|
|
986
1038
|
}
|
|
987
1039
|
|
|
988
1040
|
get isPeriodicEvent() {
|
|
989
|
-
return this.
|
|
1041
|
+
return this.#eventConfig.isPeriodic;
|
|
990
1042
|
}
|
|
991
1043
|
}
|
|
992
1044
|
|
package/src/config.js
CHANGED
|
@@ -6,13 +6,12 @@ const { getEnvInstance } = require("./shared/env");
|
|
|
6
6
|
const redis = require("./shared/redis");
|
|
7
7
|
const EventQueueError = require("./EventQueueError");
|
|
8
8
|
|
|
9
|
-
let instance;
|
|
10
|
-
|
|
11
9
|
const FOR_UPDATE_TIMEOUT = 10;
|
|
12
10
|
const GLOBAL_TX_TIMEOUT = 30 * 60 * 1000;
|
|
13
11
|
const REDIS_CONFIG_CHANNEL = "EVENT_QUEUE_CONFIG_CHANNEL";
|
|
14
12
|
const COMPONENT_NAME = "eventQueue/config";
|
|
15
13
|
const MIN_INTERVAL_SEC = 10;
|
|
14
|
+
const DEFAULT_LOAD = 1;
|
|
16
15
|
|
|
17
16
|
class Config {
|
|
18
17
|
#logger;
|
|
@@ -29,10 +28,12 @@ class Config {
|
|
|
29
28
|
#configFilePath;
|
|
30
29
|
#processEventsAfterPublish;
|
|
31
30
|
#skipCsnCheck;
|
|
31
|
+
#registerAsEventProcessor;
|
|
32
32
|
#disableRedis;
|
|
33
33
|
#env;
|
|
34
34
|
#eventMap;
|
|
35
35
|
#updatePeriodicEvents;
|
|
36
|
+
static #instance;
|
|
36
37
|
constructor() {
|
|
37
38
|
this.#logger = cds.log(COMPONENT_NAME);
|
|
38
39
|
this.#config = null;
|
|
@@ -107,20 +108,25 @@ class Config {
|
|
|
107
108
|
config.events = config.events ?? [];
|
|
108
109
|
config.periodicEvents = config.periodicEvents ?? [];
|
|
109
110
|
this.#eventMap = config.events.reduce((result, event) => {
|
|
111
|
+
event.load = event.load ?? DEFAULT_LOAD;
|
|
110
112
|
this.validateAdHocEvents(result, event);
|
|
111
113
|
result[[event.type, event.subType].join("##")] = event;
|
|
112
114
|
return result;
|
|
113
115
|
}, {});
|
|
114
116
|
this.#eventMap = config.periodicEvents.reduce((result, event) => {
|
|
115
|
-
|
|
117
|
+
event.load = event.load ?? DEFAULT_LOAD;
|
|
118
|
+
const SUFFIX_PERIODIC = "_PERIODIC";
|
|
119
|
+
event.type = `${event.type}${SUFFIX_PERIODIC}`;
|
|
116
120
|
event.isPeriodic = true;
|
|
121
|
+
this.validatePeriodicConfig(result, event);
|
|
117
122
|
result[[event.type, event.subType].join("##")] = event;
|
|
118
123
|
return result;
|
|
119
124
|
}, this.#eventMap);
|
|
120
125
|
}
|
|
121
126
|
|
|
122
127
|
validatePeriodicConfig(eventMap, config) {
|
|
123
|
-
|
|
128
|
+
const key = this.generateKey(config.type, config.subType);
|
|
129
|
+
if (eventMap[key] && eventMap[key].isPeriodic) {
|
|
124
130
|
throw EventQueueError.duplicateEventRegistration(config.type, config.subType);
|
|
125
131
|
}
|
|
126
132
|
|
|
@@ -134,7 +140,8 @@ class Config {
|
|
|
134
140
|
}
|
|
135
141
|
|
|
136
142
|
validateAdHocEvents(eventMap, config) {
|
|
137
|
-
|
|
143
|
+
const key = this.generateKey(config.type, config.subType);
|
|
144
|
+
if (eventMap[key] && !eventMap[key].isPeriodic) {
|
|
138
145
|
throw EventQueueError.duplicateEventRegistration(config.type, config.subType);
|
|
139
146
|
}
|
|
140
147
|
|
|
@@ -271,18 +278,29 @@ class Config {
|
|
|
271
278
|
return this.#updatePeriodicEvents;
|
|
272
279
|
}
|
|
273
280
|
|
|
281
|
+
set registerAsEventProcessor(value) {
|
|
282
|
+
this.#registerAsEventProcessor = value;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
get registerAsEventProcessor() {
|
|
286
|
+
return this.#registerAsEventProcessor;
|
|
287
|
+
}
|
|
288
|
+
|
|
274
289
|
get isMultiTenancy() {
|
|
275
290
|
return !!cds.requires.multitenancy;
|
|
276
291
|
}
|
|
277
|
-
}
|
|
278
292
|
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
293
|
+
/**
|
|
294
|
+
@return { Config }
|
|
295
|
+
**/
|
|
296
|
+
static get instance() {
|
|
297
|
+
if (!Config.#instance) {
|
|
298
|
+
Config.#instance = new Config();
|
|
299
|
+
}
|
|
300
|
+
return Config.#instance;
|
|
282
301
|
}
|
|
283
|
-
|
|
284
|
-
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
const instance = Config.instance;
|
|
285
305
|
|
|
286
|
-
module.exports =
|
|
287
|
-
getConfigInstance,
|
|
288
|
-
};
|
|
306
|
+
module.exports = instance;
|
package/src/dbHandler.js
CHANGED
|
@@ -8,8 +8,7 @@ const config = require("./config");
|
|
|
8
8
|
const COMPONENT_NAME = "eventQueue/dbHandler";
|
|
9
9
|
|
|
10
10
|
const registerEventQueueDbHandler = (dbService) => {
|
|
11
|
-
const
|
|
12
|
-
const def = dbService.model.definitions[configInstance.tableNameEventQueue];
|
|
11
|
+
const def = dbService.model.definitions[config.tableNameEventQueue];
|
|
13
12
|
dbService.after("CREATE", def, (_, req) => {
|
|
14
13
|
if (req.tx._skipEventQueueBroadcase) {
|
|
15
14
|
return;
|
|
@@ -21,7 +20,7 @@ const registerEventQueueDbHandler = (dbService) => {
|
|
|
21
20
|
const eventCombinations = Object.keys(
|
|
22
21
|
data.reduce((result, event) => {
|
|
23
22
|
const key = [event.type, event.subType].join("##");
|
|
24
|
-
if (!
|
|
23
|
+
if (!config.hasEventAfterCommitFlag(event.type, event.subType) || eventQueuePublishEvents[key]) {
|
|
25
24
|
return result;
|
|
26
25
|
}
|
|
27
26
|
eventQueuePublishEvents[key] = true;
|
package/src/index.js
CHANGED
package/src/initialize.js
CHANGED
|
@@ -11,7 +11,7 @@ const VError = require("verror");
|
|
|
11
11
|
const EventQueueError = require("./EventQueueError");
|
|
12
12
|
const runner = require("./runner");
|
|
13
13
|
const dbHandler = require("./dbHandler");
|
|
14
|
-
const
|
|
14
|
+
const config = require("./config");
|
|
15
15
|
const { initEventQueueRedisSubscribe, closeSubscribeClient } = require("./redisPubSub");
|
|
16
16
|
const { closeMainClient } = require("./shared/redis");
|
|
17
17
|
|
|
@@ -54,11 +54,10 @@ const initialize = async ({
|
|
|
54
54
|
// - content of yaml check
|
|
55
55
|
// - betweenRuns and parallelTenantProcessing
|
|
56
56
|
|
|
57
|
-
|
|
58
|
-
if (configInstance.initialized) {
|
|
57
|
+
if (config.initialized) {
|
|
59
58
|
return;
|
|
60
59
|
}
|
|
61
|
-
|
|
60
|
+
config.initialized = true;
|
|
62
61
|
|
|
63
62
|
mixConfigVarsWithEnv(
|
|
64
63
|
configFilePath,
|
|
@@ -75,24 +74,25 @@ const initialize = async ({
|
|
|
75
74
|
);
|
|
76
75
|
|
|
77
76
|
const logger = cds.log(COMPONENT);
|
|
78
|
-
|
|
79
|
-
|
|
77
|
+
config.fileContent = await readConfigFromFile(config.configFilePath);
|
|
78
|
+
config.checkRedisEnabled();
|
|
80
79
|
|
|
81
80
|
const dbService = await cds.connect.to("db");
|
|
82
81
|
await (cds.model ? Promise.resolve() : new Promise((resolve) => cds.on("serving", resolve)));
|
|
83
|
-
!
|
|
84
|
-
if (
|
|
82
|
+
!config.skipCsnCheck && (await csnCheck());
|
|
83
|
+
if (config.processEventsAfterPublish) {
|
|
85
84
|
dbHandler.registerEventQueueDbHandler(dbService);
|
|
86
85
|
}
|
|
87
86
|
|
|
88
87
|
registerEventProcessors();
|
|
89
88
|
registerCdsShutdown();
|
|
90
89
|
logger.info("event queue initialized", {
|
|
91
|
-
registerAsEventProcessor:
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
90
|
+
registerAsEventProcessor: config.registerAsEventProcessor,
|
|
91
|
+
processEventsAfterPublish: config.processEventsAfterPublish,
|
|
92
|
+
multiTenancyEnabled: config.isMultiTenancy,
|
|
93
|
+
redisEnabled: config.redisEnabled,
|
|
94
|
+
runInterval: config.runInterval,
|
|
95
|
+
config: config.parallelTenantProcessing,
|
|
96
96
|
});
|
|
97
97
|
};
|
|
98
98
|
|
|
@@ -114,22 +114,20 @@ const readConfigFromFile = async (configFilepath) => {
|
|
|
114
114
|
};
|
|
115
115
|
|
|
116
116
|
const registerEventProcessors = () => {
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
if (!configInstance.registerAsEventProcessor) {
|
|
117
|
+
if (!config.registerAsEventProcessor) {
|
|
120
118
|
return;
|
|
121
119
|
}
|
|
122
120
|
|
|
123
121
|
const errorHandler = (err) => cds.log(COMPONENT).error("error during init runner", err);
|
|
124
122
|
|
|
125
|
-
if (!
|
|
123
|
+
if (!config.isMultiTenancy) {
|
|
126
124
|
runner.singleTenant().catch(errorHandler);
|
|
127
125
|
return;
|
|
128
126
|
}
|
|
129
127
|
|
|
130
|
-
if (
|
|
128
|
+
if (config.redisEnabled) {
|
|
131
129
|
initEventQueueRedisSubscribe();
|
|
132
|
-
|
|
130
|
+
config.attachConfigChangeHandler();
|
|
133
131
|
runner.multiTenancyRedis().catch(errorHandler);
|
|
134
132
|
} else {
|
|
135
133
|
runner.multiTenancyDb().catch(errorHandler);
|
|
@@ -137,21 +135,17 @@ const registerEventProcessors = () => {
|
|
|
137
135
|
};
|
|
138
136
|
|
|
139
137
|
const csnCheck = async () => {
|
|
140
|
-
const
|
|
141
|
-
const eventCsn = cds.model.definitions[configInstance.tableNameEventQueue];
|
|
138
|
+
const eventCsn = cds.model.definitions[config.tableNameEventQueue];
|
|
142
139
|
if (!eventCsn) {
|
|
143
|
-
throw EventQueueError.missingTableInCsn(
|
|
140
|
+
throw EventQueueError.missingTableInCsn(config.tableNameEventQueue);
|
|
144
141
|
}
|
|
145
142
|
|
|
146
|
-
const lockCsn = cds.model.definitions[
|
|
143
|
+
const lockCsn = cds.model.definitions[config.tableNameEventLock];
|
|
147
144
|
if (!lockCsn) {
|
|
148
|
-
throw EventQueueError.missingTableInCsn(
|
|
145
|
+
throw EventQueueError.missingTableInCsn(config.tableNameEventLock);
|
|
149
146
|
}
|
|
150
147
|
|
|
151
|
-
if (
|
|
152
|
-
configInstance.tableNameEventQueue === BASE_TABLES.EVENT &&
|
|
153
|
-
configInstance.tableNameEventLock === BASE_TABLES.LOCK
|
|
154
|
-
) {
|
|
148
|
+
if (config.tableNameEventQueue === BASE_TABLES.EVENT && config.tableNameEventLock === BASE_TABLES.LOCK) {
|
|
155
149
|
return; // no need to check base tables
|
|
156
150
|
}
|
|
157
151
|
|
|
@@ -181,10 +175,9 @@ const checkCustomTable = (baseCsn, customCsn) => {
|
|
|
181
175
|
};
|
|
182
176
|
|
|
183
177
|
const mixConfigVarsWithEnv = (...args) => {
|
|
184
|
-
const configInstance = getConfigInstance();
|
|
185
178
|
CONFIG_VARS.forEach(([configName, defaultValue], index) => {
|
|
186
179
|
const configValue = args[index];
|
|
187
|
-
|
|
180
|
+
config[configName] = configValue ?? cds.env.eventQueue?.[configName] ?? defaultValue;
|
|
188
181
|
});
|
|
189
182
|
};
|
|
190
183
|
|
package/src/periodicEvents.js
CHANGED
|
@@ -4,19 +4,18 @@ const cds = require("@sap/cds");
|
|
|
4
4
|
|
|
5
5
|
const { EventProcessingStatus } = require("./constants");
|
|
6
6
|
const { processChunkedSync } = require("./shared/common");
|
|
7
|
-
const
|
|
7
|
+
const eventConfig = require("./config");
|
|
8
8
|
|
|
9
9
|
const COMPONENT_NAME = "eventQueue/periodicEvents";
|
|
10
10
|
|
|
11
11
|
const checkAndInsertPeriodicEvents = async (context) => {
|
|
12
12
|
const tx = cds.tx(context);
|
|
13
|
-
const
|
|
14
|
-
const baseCqn = SELECT.from(configInstance.tableNameEventQueue)
|
|
13
|
+
const baseCqn = SELECT.from(eventConfig.tableNameEventQueue)
|
|
15
14
|
.where([
|
|
16
15
|
{ list: [{ ref: ["type"] }, { ref: ["subType"] }] },
|
|
17
16
|
"IN",
|
|
18
17
|
{
|
|
19
|
-
list:
|
|
18
|
+
list: eventConfig.periodicEvents.map((periodicEvent) => ({
|
|
20
19
|
list: [{ val: periodicEvent.type }, { val: periodicEvent.subType }],
|
|
21
20
|
})),
|
|
22
21
|
},
|
|
@@ -30,7 +29,7 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
30
29
|
|
|
31
30
|
if (!currentPeriodEvents.length) {
|
|
32
31
|
// fresh insert all
|
|
33
|
-
return await insertPeriodEvents(tx,
|
|
32
|
+
return await insertPeriodEvents(tx, eventConfig.periodicEvents);
|
|
34
33
|
}
|
|
35
34
|
|
|
36
35
|
const exitingEventMap = currentPeriodEvents.reduce((result, current) => {
|
|
@@ -39,7 +38,7 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
39
38
|
return result;
|
|
40
39
|
}, {});
|
|
41
40
|
|
|
42
|
-
const { newEvents, existingEvents } =
|
|
41
|
+
const { newEvents, existingEvents } = eventConfig.periodicEvents.reduce(
|
|
43
42
|
(result, event) => {
|
|
44
43
|
if (exitingEventMap[_generateKey(event)]) {
|
|
45
44
|
result.existingEvents.push(exitingEventMap[_generateKey(event)]);
|
|
@@ -53,7 +52,7 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
53
52
|
|
|
54
53
|
const currentDate = new Date();
|
|
55
54
|
const exitingWithNotMatchingInterval = existingEvents.filter((existingEvent) => {
|
|
56
|
-
const config =
|
|
55
|
+
const config = eventConfig.getEventConfig(existingEvent.type, existingEvent.subType);
|
|
57
56
|
const eventStartAfter = new Date(existingEvent.startAfter);
|
|
58
57
|
// check if to far in future
|
|
59
58
|
const dueInWithNewInterval = new Date(currentDate.getTime() + config.interval * 1000);
|
|
@@ -64,12 +63,15 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
64
63
|
cds.log(COMPONENT_NAME).info("deleting periodic events because they have changed", {
|
|
65
64
|
changedEvents: exitingWithNotMatchingInterval.map(({ type, subType }) => ({ type, subType })),
|
|
66
65
|
});
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
66
|
+
|
|
67
|
+
if (exitingWithNotMatchingInterval.length) {
|
|
68
|
+
await tx.run(
|
|
69
|
+
DELETE.from(eventConfig.tableNameEventQueue).where(
|
|
70
|
+
"ID IN",
|
|
71
|
+
exitingWithNotMatchingInterval.map(({ ID }) => ID)
|
|
72
|
+
)
|
|
73
|
+
);
|
|
74
|
+
}
|
|
73
75
|
|
|
74
76
|
const newOrChangedEvents = newEvents.concat(exitingWithNotMatchingInterval);
|
|
75
77
|
|
|
@@ -82,11 +84,10 @@ const checkAndInsertPeriodicEvents = async (context) => {
|
|
|
82
84
|
|
|
83
85
|
const insertPeriodEvents = async (tx, events) => {
|
|
84
86
|
const startAfter = new Date();
|
|
85
|
-
const configInstance = getConfigInstance();
|
|
86
87
|
processChunkedSync(events, 4, (chunk) => {
|
|
87
88
|
cds.log(COMPONENT_NAME).info("inserting changed or new periodic events", {
|
|
88
89
|
events: chunk.map(({ type, subType }) => {
|
|
89
|
-
const { interval } =
|
|
90
|
+
const { interval } = eventConfig.getEventConfig(type, subType);
|
|
90
91
|
return { type, subType, interval };
|
|
91
92
|
}),
|
|
92
93
|
});
|
|
@@ -98,7 +99,7 @@ const insertPeriodEvents = async (tx, events) => {
|
|
|
98
99
|
}));
|
|
99
100
|
|
|
100
101
|
tx._skipEventQueueBroadcase = true;
|
|
101
|
-
await tx.run(INSERT.into(
|
|
102
|
+
await tx.run(INSERT.into(eventConfig.tableNameEventQueue).entries(periodEventsInsert));
|
|
102
103
|
tx._skipEventQueueBroadcase = false;
|
|
103
104
|
};
|
|
104
105
|
|