@cap-js-community/event-queue 0.1.55 → 0.1.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/README.md +6 -98
  2. package/package.json +28 -13
  3. package/src/EventQueueProcessorBase.js +5 -5
  4. package/src/processEventQueue.js +1 -1
  5. package/src/redisPubSub.js +4 -2
package/README.md CHANGED
@@ -13,24 +13,8 @@ enhancing the overall performance of your application.
13
13
 
14
14
  ## Getting started
15
15
 
16
- - Run `npm add @cap-community/event-queue` in `@sap/cds` project
17
- - Initialize the event queue for example in the CAP server.js
18
- - Enhance or create `./srv/server.js`:
19
-
20
- ### By Code
21
-
22
- ```js
23
- const cds = require("@sap/cds");
24
- const eventQueue = require("@cap-js-community/event-queue");
25
-
26
- cds.on("bootstrap", () => {
27
- eventQueue.initialize({
28
- configFilePath: "./srv/eventConfig.yml",
29
- });
30
- });
31
-
32
- module.exports = cds.server;
33
- ```
16
+ - Run `npm add @cap-js-community/event-queue` in `@sap/cds` project
17
+ - Activate the cds-plugin in the cds section of the package.sjon
34
18
 
35
19
  ### As cds-plugin
36
20
 
@@ -52,92 +36,16 @@ https://cap.cloud.sap/docs/releases/march23#new-cds-plugin-technique
52
36
  ## Features
53
37
 
54
38
  - load balancing of event processing throughout app instances
39
+ - control concurrency in app instances
55
40
  - managed transactions for event processing
56
41
  - async processing of processing intensive tasks for better UI responsiveness
57
42
  - push/pull mechanism for reducing delay between publish an event and processing
58
43
  - cluster published events during processing (e.g. for combining multiple E-Mail events to one E-Mail)
44
+ - plug and play via cds-plugin
59
45
 
60
- ## Initialize event queue configuration
61
-
62
- | Property | Description |
63
- | ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
64
- | configFilePath | Filepath as a string for the event configuration file. The base path is the root directory of the project. |
65
- | registerAsEventProcessor | Allows enabling/disabling the registration of the app instance as an event processor. The interval is based on the `runInterval` parameter. Based on this interval, all events for all tenants will be processed. The default value is `true`. |
66
- | runInterval | The interval specified in seconds, indicating how often events are processed for all tenants. If `processEventsAfterPublish` is true, in most situations, only erroneous events are processed during this run. All other events are automatically processed after they have been published (see the parameter `processEventsAfterPublish`). |
67
- | processEventsAfterPublish | Allows enabling/disabling the automatic processing of events after they have been published. The behavior depends on whether a Redis service is bound to the app. With Redis, the processing will happen on any available app instance. If Redis is not bound, the processing will happen on the same instance where the event was published. |
68
- | tableNameEventQueue | Allows 'Bring your own table'. This is the name of the event queue table. The required fields for this table can be found in the db-folder. |
69
- | tableNameEventLock | Allows 'Bring your own table'. This is the name of the event lock table. The required fields for this table can be found in the db-folder. |
70
- | skipCsnCheck | Specifies whether to skip the CSN check. This might be useful for testing purposes. |
71
- | parallelTenantProcessing | Specifies the limit as an integer on how many tenants are processed on a given app-instance in parallel. The default is 5. |
72
-
73
- ## Persistence
74
-
75
- This library needs two tables two work as designed. The event tables contains the data and state about the events for
76
- processing.
77
- The second table is for keeping track of semantic locks. This table is used if no redis-instance is available.
78
- There are two options for getting the required tables into the project.
79
-
80
- ### Use provided tables
81
-
82
- Use the tables provided by this library. For that add the following to `package.json` of the project:
83
-
84
- ```json
85
- {
86
- "cds": {
87
- "requires": {
88
- "cds-event-queue": {
89
- "model": "@cap-js-community/event-queue"
90
- }
91
- }
92
- }
93
- }
94
- ```
95
-
96
- ### Bring your own persistence
97
-
98
- The table names can be specified during the initialization of the event queue. It's important that the provided tables
99
- have the same named fields and types as the default tables.
100
-
101
- Own tables can have additional fields but need to meet the minimal requirements. During initialization, the CDS CSN for
102
- the provided table names is checked to ensure they meet the requirements.
103
- If you want to skip this check, you can achieve it by using skipCsnCheck (refer to the "Initialize Event Queue
104
- Configuration" section).
105
-
106
- ```json
107
- {
108
- "cds": {
109
- "eventQueue": {
110
- "plugin": true,
111
- "configFilePath": "./srv/eventQueueConfig.yml",
112
- "tableNameEventQueue": "sap.custom.EventQueue",
113
- "tableNameEventLock": "sap.custom.EventLock"
114
- }
115
- }
116
- }
117
- ```
118
-
119
- ## Configure your events
120
-
121
- Events are configured in a configuration yml file. In this file all relevant information about how events should be
122
- processed can be maintained.
123
-
124
- ```yaml
125
- events:
126
- - type: Notifications
127
- subType: Email
128
- impl: ./test/asset/EventQueueTest
129
- ```
130
-
131
- ## Event Configurations
46
+ ## Documentation
132
47
 
133
- | Property | Description |
134
- | ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
135
- | retryAttempts | For infinite retries, maintain -1 as 'retryAttempts'. Default retry attempts is 3. |
136
- | parallelEventProcessing | How many events of the same type and subType are parallel processed after clustering. Default value is 1 and limit is 10. |
137
- | eventOutdatedCheck | Checks if the db record for the event has been modified since the selection and right before the processing of the event. Default is true. |
138
- | commitOnEventLevel | After processing an event, the associated transaction is committed and the associated status is committed with the same transaction. This should be used if events should be processed atomically. Default is false. |
139
- | selectMaxChunkSize | Number of events which are selected at once. Default is 100. If it should be checked if there are more open events available, set the parameter checkForNextChunk to true. |
140
- | checkForNextChunk | Determines if after processing a chunk (the size depends on the value of selectMaxChunkSize), a next chunk is being processed if there are more open events and the processing time has not already exceeded 5 minutes. Default is false. |
48
+ Head over to our [Documentation](https://cap-js-community.github.io/event-queue/) to learn more.
141
49
 
142
50
  ## Support, Feedback, Contributing
143
51
 
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@cap-js-community/event-queue",
3
- "version": "0.1.55",
4
- "description": "event queue for cds",
3
+ "version": "0.1.57",
4
+ "description": "An event queue that enables secure transactional processing of asynchronous events, featuring instant event processing with Redis Pub/Sub and load distribution across all application instances.",
5
5
  "main": "src/index.js",
6
6
  "files": [
7
7
  "src",
@@ -9,6 +9,16 @@
9
9
  "cds-plugin.js",
10
10
  "index.cds"
11
11
  ],
12
+ "keywords": [
13
+ "CAP",
14
+ "CDS",
15
+ "messaging",
16
+ "async",
17
+ "redis",
18
+ "load-balancing",
19
+ "eventing",
20
+ "multi-tenancy"
21
+ ],
12
22
  "scripts": {
13
23
  "test:unit": "jest --testPathIgnorePatterns=\"/test-integration/\"",
14
24
  "test:integration": "jest --testPathIgnorePatterns=\"/test/\" --runInBand --forceExit",
@@ -31,24 +41,29 @@
31
41
  "node": ">=16"
32
42
  },
33
43
  "dependencies": {
34
- "uuid": "9.0.0",
35
- "redis": "4.6.7",
44
+ "uuid": "9.0.1",
45
+ "redis": "4.6.10",
36
46
  "verror": "1.10.1",
37
- "yaml": "2.3.1"
47
+ "yaml": "2.3.2"
38
48
  },
39
49
  "devDependencies": {
40
50
  "@sap/eslint-plugin-cds": "2.6.3",
41
- "@sap/cds-dk": "7.0.3",
42
- "hdb": "0.19.5",
51
+ "@sap/cds-dk": "7.1.1",
52
+ "hdb": "0.19.6",
43
53
  "eslint-plugin-node": "11.1.0",
44
54
  "sqlite3": "5.1.6",
45
55
  "express": "4.18.2",
46
- "@sap/cds": "7.1.0",
47
- "eslint": "^8.39.0",
48
- "eslint-config-prettier": "^8.8.0",
49
- "eslint-plugin-jest": "^27.2.1",
50
- "jest": "^29.5.0",
51
- "prettier": "^2.8.8"
56
+ "@sap/cds": "7.1.2",
57
+ "eslint": "8.50.0",
58
+ "eslint-config-prettier": "9.0.0",
59
+ "eslint-plugin-jest": "27.4.0",
60
+ "jest": "29.7.0",
61
+ "prettier": "3.0.3"
62
+ },
63
+ "homepage": "https://cap-js-community.github.io/event-queue/",
64
+ "repository": {
65
+ "type": "git",
66
+ "url": "https://github.com/cap-js-community/event-queue.git"
52
67
  },
53
68
  "cds": {
54
69
  "requires": {
package/src/EventQueueProcessorBase.js CHANGED
@@ -25,6 +25,7 @@ class EventQueueProcessorBase {
25
25
  #eventsWithExceededTries = [];
26
26
  #exceededTriesExceeded = [];
27
27
  #selectedEventMap = {};
28
+ #queueEntriesWithPayloadMap = {};
28
29
 
29
30
  constructor(context, eventType, eventSubType, config) {
30
31
  this.__context = context;
@@ -37,7 +38,6 @@ class EventQueueProcessorBase {
37
38
  this.__commitedStatusMap = {};
38
39
  this.__eventType = eventType;
39
40
  this.__eventSubType = eventSubType;
40
- this.__queueEntriesWithPayloadMap = {};
41
41
  this.__config = config ?? {};
42
42
  this.__parallelEventProcessing = this.__config.parallelEventProcessing ?? DEFAULT_PARALLEL_EVENT_PROCESSING;
43
43
  if (this.__parallelEventProcessing > LIMIT_PARALLEL_EVENT_PROCESSING) {
@@ -164,7 +164,7 @@ class EventQueueProcessorBase {
164
164
  );
165
165
  return;
166
166
  }
167
- this.__queueEntriesWithPayloadMap[queueEntry.ID] = {
167
+ this.#queueEntriesWithPayloadMap[queueEntry.ID] = {
168
168
  queueEntry,
169
169
  payload,
170
170
  };
@@ -190,8 +190,8 @@ class EventQueueProcessorBase {
190
190
  * This can be useful for e.g. multiple tasks have been scheduled and always the same user should be informed.
191
191
  * In this case the events should be clustered together and only one mail should be sent.
192
192
  */
193
- clusterQueueEntries() {
194
- Object.entries(this.__queueEntriesWithPayloadMap).forEach(([key, { queueEntry, payload }]) => {
193
+ clusterQueueEntries(queueEntriesWithPayloadMap) {
194
+ Object.entries(queueEntriesWithPayloadMap).forEach(([key, { queueEntry, payload }]) => {
195
195
  this.addEntryToProcessingMap(key, queueEntry, payload);
196
196
  });
197
197
  }
@@ -795,7 +795,7 @@ class EventQueueProcessorBase {
795
795
  }
796
796
 
797
797
  get queueEntriesWithPayloadMap() {
798
- return this.__queueEntriesWithPayloadMap;
798
+ return this.#queueEntriesWithPayloadMap;
799
799
  }
800
800
 
801
801
  get eventProcessingMap() {
package/src/processEventQueue.js CHANGED
@@ -76,7 +76,7 @@ const processEventQueue = async (context, eventType, eventSubType, startTime = n
76
76
  await executeInNewTransaction(context, `eventQueue-processing-${eventType}##${eventSubType}`, async (tx) => {
77
77
  eventTypeInstance.processEventContext = tx.context;
78
78
  try {
79
- eventTypeInstance.clusterQueueEntries();
79
+ eventTypeInstance.clusterQueueEntries(eventTypeInstance.queueEntriesWithPayloadMap);
80
80
  await processEventMap(eventTypeInstance);
81
81
  } catch (err) {
82
82
  eventTypeInstance.handleErrorDuringClustering(err);
package/src/redisPubSub.js CHANGED
@@ -47,7 +47,9 @@ const publishEvent = async (tenantId, type, subType) => {
47
47
  const logger = cds.log(COMPONENT_NAME);
48
48
  const configInstance = config.getConfigInstance();
49
49
  if (!configInstance.redisEnabled) {
50
- await _handleEventInternally(tenantId, type, subType);
50
+ if (configInstance.registerAsEventProcessor) {
51
+ await _handleEventInternally(tenantId, type, subType);
52
+ }
51
53
  return;
52
54
  }
53
55
  try {
@@ -86,7 +88,7 @@ const _handleEventInternally = async (tenantId, type, subType) => {
86
88
  // NOTE: we need this because of logging otherwise logs would not contain the subdomain
87
89
  http: { req: { authInfo: { getSubdomain: () => subdomain } } },
88
90
  });
89
- processEventQueue(context, type, subType);
91
+ getWorkerPoolInstance().addToQueue(async () => processEventQueue(context, type, subType));
90
92
  };
91
93
 
92
94
  module.exports = {