event-storage 0.8.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -540
- package/index.js +5 -6
- package/package.json +28 -31
- package/src/Clock.js +21 -9
- package/src/Consumer.js +6 -7
- package/src/EventStore.js +261 -67
- package/src/EventStream.js +172 -19
- package/src/Index/ReadOnlyIndex.js +3 -3
- package/src/Index/ReadableIndex.js +17 -13
- package/src/Index/WritableIndex.js +17 -11
- package/src/Index.js +7 -5
- package/src/IndexEntry.js +2 -3
- package/src/JoinEventStream.js +34 -32
- package/src/Partition/ReadOnlyPartition.js +3 -3
- package/src/Partition/ReadablePartition.js +110 -57
- package/src/Partition/WritablePartition.js +81 -23
- package/src/Partition.js +7 -4
- package/src/Storage/ReadOnlyStorage.js +4 -4
- package/src/Storage/ReadableStorage.js +144 -22
- package/src/Storage/WritableStorage.js +175 -33
- package/src/Storage.js +9 -4
- package/src/Watcher.js +6 -5
- package/src/WatchesFile.js +8 -7
- package/src/metadataUtil.js +79 -0
- package/src/util.js +74 -73
- package/test/Consumer.spec.js +0 -455
- package/test/EventStore.spec.js +0 -632
- package/test/EventStream.spec.js +0 -120
- package/test/Index.spec.js +0 -591
- package/test/JoinEventStream.spec.js +0 -113
- package/test/Partition.spec.js +0 -488
- package/test/Storage.spec.js +0 -1017
- package/test/Watcher.spec.js +0 -131
package/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "event-storage",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"type": "module",
|
|
4
5
|
"description": "An optimized embedded event store for node.js",
|
|
5
6
|
"keywords": [
|
|
6
7
|
"event-storage",
|
|
@@ -15,31 +16,35 @@
|
|
|
15
16
|
"homepage": "https://github.com/albe/node-event-storage",
|
|
16
17
|
"repository": {
|
|
17
18
|
"type": "git",
|
|
18
|
-
"url": "https://github.com/albe/node-event-storage"
|
|
19
|
+
"url": "git+https://github.com/albe/node-event-storage.git"
|
|
19
20
|
},
|
|
20
21
|
"bugs": {
|
|
21
22
|
"url": "https://github.com/albe/node-event-storage/issues"
|
|
22
23
|
},
|
|
24
|
+
"exports": {
|
|
25
|
+
".": "./index.js"
|
|
26
|
+
},
|
|
23
27
|
"scripts": {
|
|
24
|
-
"test": "
|
|
25
|
-
"coverage": "
|
|
28
|
+
"test": "c8 --reporter=lcov --reporter=text mocha test/*.spec.js",
|
|
29
|
+
"coverage": "c8 report --reporter=text-lcov | coveralls"
|
|
26
30
|
},
|
|
27
31
|
"files": [
|
|
28
|
-
"
|
|
29
|
-
"
|
|
30
|
-
"
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"
|
|
37
|
-
"
|
|
38
|
-
"
|
|
39
|
-
"
|
|
40
|
-
"
|
|
32
|
+
"src/Consumer*.js",
|
|
33
|
+
"src/EventStore*.js",
|
|
34
|
+
"src/EventStream*.js",
|
|
35
|
+
"src/Index*.js",
|
|
36
|
+
"src/IndexEntry*.js",
|
|
37
|
+
"src/JoinEventStream*.js",
|
|
38
|
+
"src/Partition*.js",
|
|
39
|
+
"src/Storage*.js",
|
|
40
|
+
"src/Watcher*.js",
|
|
41
|
+
"src/Clock*.js",
|
|
42
|
+
"src/Index/*.js",
|
|
43
|
+
"src/Partition/*.js",
|
|
44
|
+
"src/Storage/*.js",
|
|
41
45
|
"src/WatchesFile.js",
|
|
42
46
|
"src/util.js",
|
|
47
|
+
"src/metadataUtil.js",
|
|
43
48
|
"index.js"
|
|
44
49
|
],
|
|
45
50
|
"license": "MIT",
|
|
@@ -50,24 +55,16 @@
|
|
|
50
55
|
}
|
|
51
56
|
],
|
|
52
57
|
"engines": {
|
|
53
|
-
"node": ">=
|
|
58
|
+
"node": ">=18.0"
|
|
54
59
|
},
|
|
55
60
|
"dependencies": {
|
|
56
|
-
"mkdirp": "^
|
|
57
|
-
},
|
|
58
|
-
"nyc": {
|
|
59
|
-
"include": [
|
|
60
|
-
"src/**/*.js"
|
|
61
|
-
],
|
|
62
|
-
"exclude": [
|
|
63
|
-
"bench/**/*.js"
|
|
64
|
-
]
|
|
61
|
+
"mkdirp": "^3.0.1"
|
|
65
62
|
},
|
|
66
63
|
"devDependencies": {
|
|
67
|
-
"
|
|
64
|
+
"c8": "^11.0.0",
|
|
65
|
+
"coveralls-next": "^6.0.1",
|
|
68
66
|
"expect.js": "^0.3.1",
|
|
69
|
-
"fs-extra": "^
|
|
70
|
-
"mocha": "^
|
|
71
|
-
"nyc": "^15.0.0"
|
|
67
|
+
"fs-extra": "^11.3.4",
|
|
68
|
+
"mocha": "^11.7.5"
|
|
72
69
|
}
|
|
73
70
|
}
|
package/src/Clock.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const TIME_BASE = process.hrtime();
|
|
2
|
-
const
|
|
3
|
-
const
|
|
1
|
+
const TIME_BASE = process.hrtime.bigint();
|
|
2
|
+
const DATE_FACTOR = 1000000n;
|
|
3
|
+
const DATE_BASE_NS = BigInt(Date.now() + 1) * DATE_FACTOR - 1n;
|
|
4
4
|
const CLOCK_ACCURACY_US = 1; // two process.hrtime() calls take roughly this long, so this is the accuracy we can measure time
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -14,18 +14,30 @@ class Clock {
|
|
|
14
14
|
* @param {Date|number} epoch The epoch to base this clock on, either as a Date or a number of the amount of milliseconds since the unix epoch
|
|
15
15
|
*/
|
|
16
16
|
constructor(epoch) {
|
|
17
|
-
this.epoch =
|
|
17
|
+
this.epoch = BigInt(epoch instanceof Date ? epoch.getTime() : Number(epoch)) * DATE_FACTOR;
|
|
18
|
+
this.lastTime = 0;
|
|
18
19
|
}
|
|
19
20
|
|
|
20
21
|
/**
|
|
21
|
-
* @returns {number} The number of microseconds since the epoch given in the constructor.
|
|
22
|
-
* @note Needs to allow at least
|
|
22
|
+
* @returns {number} The number of microseconds since the epoch given in the constructor.
|
|
23
|
+
* @note Needs to allow at least tenths of ms accuracy, better hundredths of ms
|
|
23
24
|
*/
|
|
24
25
|
time() {
|
|
25
|
-
const delta = process.hrtime(TIME_BASE
|
|
26
|
-
|
|
26
|
+
const delta = process.hrtime.bigint() - TIME_BASE;
|
|
27
|
+
const timeSinceEpoch = Number((DATE_BASE_NS - this.epoch + delta) / 1000n);
|
|
28
|
+
return this.lastTime = Math.max(this.lastTime + 1, timeSinceEpoch);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Return the clock accuracy of the given timestamp. This is only useful for calculating a consistent ordering
|
|
33
|
+
* ala TrueTime for multi-writer scenarios.
|
|
34
|
+
* @param {number} time A timestamp measured by this clock.
|
|
35
|
+
* @returns {number} The amount of µs accuracy this timestamp has.
|
|
36
|
+
*/
|
|
37
|
+
accuracy(time) {
|
|
38
|
+
return CLOCK_ACCURACY_US;
|
|
27
39
|
}
|
|
28
40
|
|
|
29
41
|
}
|
|
30
42
|
|
|
31
|
-
|
|
43
|
+
export default Clock;
|
package/src/Consumer.js
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const Storage = require('./Storage/ReadableStorage');
|
|
1
|
+
import stream from 'stream';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import { assert, ensureDirectory } from './util.js';
|
|
5
|
+
import Storage from './Storage/ReadableStorage.js';
|
|
7
6
|
const MAX_CATCHUP_BATCH = 10;
|
|
8
7
|
|
|
9
8
|
/**
|
|
@@ -299,4 +298,4 @@ class Consumer extends stream.Readable {
|
|
|
299
298
|
}
|
|
300
299
|
}
|
|
301
300
|
|
|
302
|
-
|
|
301
|
+
export default Consumer;
|
package/src/EventStore.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
1
|
+
import EventStream from './EventStream.js';
|
|
2
|
+
import JoinEventStream from './JoinEventStream.js';
|
|
3
|
+
import fs from 'fs';
|
|
4
|
+
import path from 'path';
|
|
5
|
+
import events from 'events';
|
|
6
|
+
import Storage, { ReadOnly as ReadOnlyStorage, LOCK_THROW, LOCK_RECLAIM } from './Storage.js';
|
|
7
|
+
import Index from './Index.js';
|
|
8
|
+
import Consumer from './Consumer.js';
|
|
9
|
+
import { assert, scanForFiles } from './util.js';
|
|
10
10
|
|
|
11
11
|
const ExpectedVersion = {
|
|
12
12
|
Any: -1,
|
|
@@ -15,24 +15,6 @@ const ExpectedVersion = {
|
|
|
15
15
|
|
|
16
16
|
class OptimisticConcurrencyError extends Error {}
|
|
17
17
|
|
|
18
|
-
class EventUnwrapper extends stream.Transform {
|
|
19
|
-
|
|
20
|
-
constructor() {
|
|
21
|
-
super({ objectMode: true });
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
_transform(data, encoding, callback) {
|
|
25
|
-
/* istanbul ignore else */
|
|
26
|
-
if (data.stream && data.payload) {
|
|
27
|
-
this.push(data.payload);
|
|
28
|
-
} else {
|
|
29
|
-
this.push(data);
|
|
30
|
-
}
|
|
31
|
-
callback();
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
}
|
|
35
|
-
|
|
36
18
|
/**
|
|
37
19
|
* An event store optimized for working with many streams.
|
|
38
20
|
* An event stream is implemented as an iterator over an index on the storage, therefore indexes need to be lightweight
|
|
@@ -47,6 +29,10 @@ class EventStore extends events.EventEmitter {
|
|
|
47
29
|
* @param {string} [config.streamsDirectory] The directory where the streams should be stored. Default '{storageDirectory}/streams'.
|
|
48
30
|
* @param {object} [config.storageConfig] Additional config options given to the storage backend. See `Storage`.
|
|
49
31
|
* @param {boolean} [config.readOnly] If the storage should be mounted in read-only mode.
|
|
32
|
+
* @param {object|function(string): object} [config.streamMetadata] A metadata object or a function `(streamName) => object`
|
|
33
|
+
* that is called whenever a new stream partition is created. The returned object is stored once in the partition
|
|
34
|
+
* file header and surfaced to `preCommit` / `preRead` hooks. Takes precedence only when
|
|
35
|
+
* `config.storageConfig.metadata` is not also set.
|
|
50
36
|
*/
|
|
51
37
|
constructor(storeName = 'eventstore', config = {}) {
|
|
52
38
|
super();
|
|
@@ -63,6 +49,17 @@ class EventStore extends events.EventEmitter {
|
|
|
63
49
|
readOnly: config.readOnly || false
|
|
64
50
|
};
|
|
65
51
|
const storageConfig = Object.assign(defaults, config.storageConfig);
|
|
52
|
+
|
|
53
|
+
// Translate the high-level streamMetadata option into the storage-level metadata function,
|
|
54
|
+
// but only when the caller has not already provided a lower-level storageConfig.metadata.
|
|
55
|
+
if (config.streamMetadata !== undefined && storageConfig.metadata === undefined) {
|
|
56
|
+
if (typeof config.streamMetadata === 'function') {
|
|
57
|
+
storageConfig.metadata = config.streamMetadata;
|
|
58
|
+
} else {
|
|
59
|
+
storageConfig.metadata = (streamName) => config.streamMetadata[streamName] || {};
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
66
63
|
this.initialize(storeName, storageConfig);
|
|
67
64
|
}
|
|
68
65
|
|
|
@@ -76,7 +73,7 @@ class EventStore extends events.EventEmitter {
|
|
|
76
73
|
|
|
77
74
|
this.storeName = storeName;
|
|
78
75
|
this.storage = (storageConfig.readOnly === true) ?
|
|
79
|
-
new
|
|
76
|
+
new ReadOnlyStorage(storeName, storageConfig)
|
|
80
77
|
: new Storage(storeName, storageConfig);
|
|
81
78
|
this.storage.open();
|
|
82
79
|
this.streams = Object.create(null);
|
|
@@ -87,10 +84,43 @@ class EventStore extends events.EventEmitter {
|
|
|
87
84
|
this.storage.close();
|
|
88
85
|
throw err;
|
|
89
86
|
}
|
|
87
|
+
this.checkUnfinishedCommits();
|
|
90
88
|
this.emit('ready');
|
|
91
89
|
});
|
|
92
90
|
}
|
|
93
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Check if the last commit in the store was unfinished, which is the case if not all events of the commit have been written.
|
|
94
|
+
* Torn writes are handled at the storage level, so this method only deals with unfinished commits.
|
|
95
|
+
* @private
|
|
96
|
+
*/
|
|
97
|
+
checkUnfinishedCommits() {
|
|
98
|
+
let position = this.storage.length;
|
|
99
|
+
let lastEvent;
|
|
100
|
+
let truncateIndex = false;
|
|
101
|
+
while (position > 0) {
|
|
102
|
+
try {
|
|
103
|
+
lastEvent = this.storage.read(position);
|
|
104
|
+
} catch (e) {
|
|
105
|
+
// A preRead hook may throw (e.g. access control). Stop repair check.
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
if (lastEvent !== false) break;
|
|
109
|
+
truncateIndex = true;
|
|
110
|
+
position--;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (lastEvent && lastEvent.metadata.commitSize && lastEvent.metadata.commitVersion !== lastEvent.metadata.commitSize - 1) {
|
|
114
|
+
this.emit('unfinished-commit', lastEvent);
|
|
115
|
+
// commitId = global sequence number at which the commit started
|
|
116
|
+
this.storage.truncate(lastEvent.metadata.commitId);
|
|
117
|
+
} else if (truncateIndex) {
|
|
118
|
+
// The index contained items that are not in the storage file; truncate everything
|
|
119
|
+
// after `position`, the last sequence number that was successfully read.
|
|
120
|
+
this.storage.truncate(position);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
94
124
|
/**
|
|
95
125
|
* Scan the streams directory for existing streams so they are ready for `getEventStream()`.
|
|
96
126
|
*
|
|
@@ -103,38 +133,44 @@ class EventStore extends events.EventEmitter {
|
|
|
103
133
|
callback = () => {};
|
|
104
134
|
}
|
|
105
135
|
// Find existing streams by scanning dir for filenames starting with 'stream-'
|
|
106
|
-
|
|
107
|
-
if (err) {
|
|
108
|
-
return callback(err);
|
|
109
|
-
}
|
|
110
|
-
let matches;
|
|
111
|
-
for (let file of files) {
|
|
112
|
-
if ((matches = file.match(/(stream-.*)\.index$/)) !== null) {
|
|
113
|
-
this.registerStream(matches[1]);
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
callback();
|
|
117
|
-
});
|
|
136
|
+
scanForFiles(this.streamsDirectory, /(stream-.*)\.index$/, this.registerStream.bind(this), callback);
|
|
118
137
|
this.storage.on('index-created', this.registerStream.bind(this));
|
|
119
138
|
}
|
|
120
139
|
|
|
121
140
|
/**
|
|
122
141
|
* @private
|
|
123
|
-
* @param {string} name The full stream name, including the `stream-` prefix.
|
|
142
|
+
* @param {string} name The full stream name, including the `stream-` prefix (and optional `.closed` suffix).
|
|
124
143
|
*/
|
|
125
144
|
registerStream(name) {
|
|
126
145
|
/* istanbul ignore if */
|
|
127
146
|
if (!name.startsWith('stream-')) {
|
|
128
147
|
return;
|
|
129
148
|
}
|
|
130
|
-
|
|
131
|
-
|
|
149
|
+
let streamName = name.slice(7);
|
|
150
|
+
// Detect the `.closed` suffix — present both in the initial scan and when the directory
|
|
151
|
+
// watcher emits 'index-created' after a writer renames the file (e.g. 'stream-foo-bar.closed').
|
|
152
|
+
let isClosed = false;
|
|
153
|
+
if (streamName.endsWith('.closed')) {
|
|
154
|
+
streamName = streamName.slice(0, -7);
|
|
155
|
+
isClosed = true;
|
|
156
|
+
}
|
|
132
157
|
if (streamName in this.streams) {
|
|
158
|
+
if (isClosed && !this.streams[streamName].closed) {
|
|
159
|
+
// The stream was renamed to .closed while this instance had it open.
|
|
160
|
+
// The old ReadOnlyIndex was already closed via onRename, so we open the new one.
|
|
161
|
+
const closedIndexName = 'stream-' + streamName + '.closed';
|
|
162
|
+
const closedIndex = this.storage.openReadonlyIndex(closedIndexName);
|
|
163
|
+
// deepcode ignore PrototypePollutionFunctionParams: streams is a Map
|
|
164
|
+
this.streams[streamName] = { index: closedIndex, closed: true };
|
|
165
|
+
this.emit('stream-closed', streamName);
|
|
166
|
+
}
|
|
133
167
|
return;
|
|
134
168
|
}
|
|
135
|
-
const index =
|
|
169
|
+
const index = isClosed
|
|
170
|
+
? this.storage.openReadonlyIndex(name)
|
|
171
|
+
: this.storage.openIndex(name);
|
|
136
172
|
// deepcode ignore PrototypePollutionFunctionParams: streams is a Map
|
|
137
|
-
this.streams[streamName] = { index };
|
|
173
|
+
this.streams[streamName] = { index, closed: isClosed };
|
|
138
174
|
this.emit('stream-available', streamName);
|
|
139
175
|
}
|
|
140
176
|
|
|
@@ -147,6 +183,103 @@ class EventStore extends events.EventEmitter {
|
|
|
147
183
|
this.storage.close();
|
|
148
184
|
}
|
|
149
185
|
|
|
186
|
+
/**
|
|
187
|
+
* Override EventEmitter.on() to delegate 'preCommit' and 'preRead' event registrations
|
|
188
|
+
* to the underlying storage, so that `eventstore.on('preCommit', handler)` works naturally.
|
|
189
|
+
* All other events are handled by the default EventEmitter.
|
|
190
|
+
*
|
|
191
|
+
* @param {string} event
|
|
192
|
+
* @param {function} listener
|
|
193
|
+
* @returns {this}
|
|
194
|
+
*/
|
|
195
|
+
on(event, listener) {
|
|
196
|
+
if (event === 'preCommit' || event === 'preRead') {
|
|
197
|
+
if (event === 'preCommit') {
|
|
198
|
+
assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.');
|
|
199
|
+
}
|
|
200
|
+
this.storage.on(event, listener);
|
|
201
|
+
return this;
|
|
202
|
+
}
|
|
203
|
+
return super.on(event, listener);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* @inheritDoc
|
|
208
|
+
*/
|
|
209
|
+
addListener(event, listener) {
|
|
210
|
+
return this.on(event, listener);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Override EventEmitter.once() to delegate 'preCommit' and 'preRead' to the underlying storage.
|
|
215
|
+
*
|
|
216
|
+
* @param {string} event
|
|
217
|
+
* @param {function} listener
|
|
218
|
+
* @returns {this}
|
|
219
|
+
*/
|
|
220
|
+
once(event, listener) {
|
|
221
|
+
if (event === 'preCommit' || event === 'preRead') {
|
|
222
|
+
if (event === 'preCommit') {
|
|
223
|
+
assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.');
|
|
224
|
+
}
|
|
225
|
+
this.storage.once(event, listener);
|
|
226
|
+
return this;
|
|
227
|
+
}
|
|
228
|
+
return super.once(event, listener);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* Override EventEmitter.off() / removeListener() to delegate 'preCommit' and 'preRead'
|
|
233
|
+
* to the underlying storage.
|
|
234
|
+
*
|
|
235
|
+
* @param {string} event
|
|
236
|
+
* @param {function} listener
|
|
237
|
+
* @returns {this}
|
|
238
|
+
*/
|
|
239
|
+
off(event, listener) {
|
|
240
|
+
if (event === 'preCommit' || event === 'preRead') {
|
|
241
|
+
this.storage.off(event, listener);
|
|
242
|
+
return this;
|
|
243
|
+
}
|
|
244
|
+
return super.off(event, listener);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* @inheritDoc
|
|
249
|
+
*/
|
|
250
|
+
removeListener(event, listener) {
|
|
251
|
+
return this.off(event, listener);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Convenience method to register a handler called before an event is committed to storage.
|
|
256
|
+
* Equivalent to `eventstore.on('preCommit', hook)`.
|
|
257
|
+
* The handler receives `(event, partitionMetadata)` and may throw to abort the write.
|
|
258
|
+
* Multiple handlers can be registered; all run on every write in registration order.
|
|
259
|
+
* The handler is invoked on every write, so its logic should be cheap, fast, and synchronous.
|
|
260
|
+
*
|
|
261
|
+
* @api
|
|
262
|
+
* @param {function(object, object): void} hook A function receiving (event, partitionMetadata).
|
|
263
|
+
* @throws {Error} If the storage was opened in read-only mode.
|
|
264
|
+
*/
|
|
265
|
+
preCommit(hook) {
|
|
266
|
+
this.on('preCommit', hook);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Convenience method to register a handler called before an event is read from storage.
|
|
271
|
+
* Equivalent to `eventstore.on('preRead', hook)`.
|
|
272
|
+
* The handler receives `(position, partitionMetadata)` and may throw to abort the read.
|
|
273
|
+
* Multiple handlers can be registered; all run on every read in registration order.
|
|
274
|
+
* The handler is invoked on every read, so its logic should be cheap, fast, and synchronous.
|
|
275
|
+
*
|
|
276
|
+
* @api
|
|
277
|
+
* @param {function(number, object): void} hook A function receiving (position, partitionMetadata).
|
|
278
|
+
*/
|
|
279
|
+
preRead(hook) {
|
|
280
|
+
this.on('preRead', hook);
|
|
281
|
+
}
|
|
282
|
+
|
|
150
283
|
/**
|
|
151
284
|
* Get the number of events stored.
|
|
152
285
|
*
|
|
@@ -204,7 +337,7 @@ class EventStore extends events.EventEmitter {
|
|
|
204
337
|
* @throws {OptimisticConcurrencyError} if the stream is not at the expected version.
|
|
205
338
|
*/
|
|
206
339
|
commit(streamName, events, expectedVersion = ExpectedVersion.Any, metadata = {}, callback = null) {
|
|
207
|
-
assert(!(this.storage instanceof
|
|
340
|
+
assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not commit to it.');
|
|
208
341
|
assert(typeof streamName === 'string' && streamName !== '', 'Must specify a stream name for commit.');
|
|
209
342
|
assert(typeof events !== 'undefined' && events !== null, 'No events specified for commit.');
|
|
210
343
|
|
|
@@ -213,6 +346,7 @@ class EventStore extends events.EventEmitter {
|
|
|
213
346
|
if (!(streamName in this.streams)) {
|
|
214
347
|
this.createEventStream(streamName, { stream: streamName });
|
|
215
348
|
}
|
|
349
|
+
assert(!this.streams[streamName].closed, `Stream "${streamName}" is closed and cannot be written to.`);
|
|
216
350
|
let streamVersion = this.streams[streamName].index.length;
|
|
217
351
|
if (expectedVersion !== ExpectedVersion.Any && streamVersion !== expectedVersion) {
|
|
218
352
|
throw new OptimisticConcurrencyError(`Optimistic Concurrency error. Expected stream "${streamName}" at version ${expectedVersion} but is at version ${streamVersion}.`);
|
|
@@ -265,11 +399,11 @@ class EventStore extends events.EventEmitter {
|
|
|
265
399
|
*
|
|
266
400
|
* @api
|
|
267
401
|
* @param {string} streamName The name of the stream to get.
|
|
268
|
-
* @param {number} [minRevision] The minimum revision to include in the events (inclusive).
|
|
269
|
-
* @param {number} [maxRevision] The maximum revision to include in the events (inclusive).
|
|
402
|
+
* @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
|
|
403
|
+
* @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
|
|
270
404
|
* @returns {EventStream|boolean} The event stream or false if a stream with the name doesn't exist.
|
|
271
405
|
*/
|
|
272
|
-
getEventStream(streamName, minRevision =
|
|
406
|
+
getEventStream(streamName, minRevision = 1, maxRevision = -1) {
|
|
273
407
|
if (!(streamName in this.streams)) {
|
|
274
408
|
return false;
|
|
275
409
|
}
|
|
@@ -281,11 +415,11 @@ class EventStore extends events.EventEmitter {
|
|
|
281
415
|
* This is the same as `getEventStream('_all', ...)`.
|
|
282
416
|
*
|
|
283
417
|
* @api
|
|
284
|
-
* @param {number} [minRevision] The minimum revision to include in the events (inclusive).
|
|
285
|
-
* @param {number} [maxRevision] The maximum revision to include in the events (inclusive).
|
|
418
|
+
* @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
|
|
419
|
+
* @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
|
|
286
420
|
* @returns {EventStream} The event stream.
|
|
287
421
|
*/
|
|
288
|
-
getAllEvents(minRevision =
|
|
422
|
+
getAllEvents(minRevision = 1, maxRevision = -1) {
|
|
289
423
|
return this.getEventStream('_all', minRevision, maxRevision);
|
|
290
424
|
}
|
|
291
425
|
|
|
@@ -294,12 +428,12 @@ class EventStore extends events.EventEmitter {
|
|
|
294
428
|
*
|
|
295
429
|
* @param {string} streamName The (transient) name of the joined stream.
|
|
296
430
|
* @param {Array<string>} streamNames An array of the stream names to join.
|
|
297
|
-
* @param {number} [minRevision] The minimum revision to include in the events (inclusive).
|
|
298
|
-
* @param {number} [maxRevision] The maximum revision to include in the events (inclusive).
|
|
431
|
+
* @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
|
|
432
|
+
* @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
|
|
299
433
|
* @returns {EventStream} The joined event stream.
|
|
300
434
|
* @throws {Error} if any of the streams doesn't exist.
|
|
301
435
|
*/
|
|
302
|
-
fromStreams(streamName, streamNames, minRevision =
|
|
436
|
+
fromStreams(streamName, streamNames, minRevision = 1, maxRevision = -1) {
|
|
303
437
|
assert(streamNames instanceof Array, 'Must specify an array of stream names.');
|
|
304
438
|
|
|
305
439
|
for (let stream of streamNames) {
|
|
@@ -318,12 +452,12 @@ class EventStore extends events.EventEmitter {
|
|
|
318
452
|
*
|
|
319
453
|
* @api
|
|
320
454
|
* @param {string} categoryName The name of the category to get a stream for. A category is a stream name prefix.
|
|
321
|
-
* @param {number} [minRevision] The minimum revision to include in the events (inclusive).
|
|
322
|
-
* @param {number} [maxRevision] The maximum revision to include in the events (inclusive).
|
|
455
|
+
* @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
|
|
456
|
+
* @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
|
|
323
457
|
* @returns {EventStream} The joined event stream for all streams of the given category.
|
|
324
458
|
* @throws {Error} If no stream for this category exists.
|
|
325
459
|
*/
|
|
326
|
-
getEventStreamForCategory(categoryName, minRevision =
|
|
460
|
+
getEventStreamForCategory(categoryName, minRevision = 1, maxRevision = -1) {
|
|
327
461
|
if (categoryName in this.streams) {
|
|
328
462
|
return this.getEventStream(categoryName, minRevision, maxRevision);
|
|
329
463
|
}
|
|
@@ -346,7 +480,7 @@ class EventStore extends events.EventEmitter {
|
|
|
346
480
|
* @throws {Error} If the stream could not be created.
|
|
347
481
|
*/
|
|
348
482
|
createEventStream(streamName, matcher) {
|
|
349
|
-
assert(!(this.storage instanceof
|
|
483
|
+
assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not create new stream on it.');
|
|
350
484
|
assert(!(streamName in this.streams), 'Can not recreate stream!');
|
|
351
485
|
|
|
352
486
|
const streamIndexName = 'stream-' + streamName;
|
|
@@ -370,7 +504,7 @@ class EventStore extends events.EventEmitter {
|
|
|
370
504
|
* @returns void
|
|
371
505
|
*/
|
|
372
506
|
deleteEventStream(streamName) {
|
|
373
|
-
assert(!(this.storage instanceof
|
|
507
|
+
assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not delete a stream on it.');
|
|
374
508
|
|
|
375
509
|
if (!(streamName in this.streams)) {
|
|
376
510
|
return;
|
|
@@ -380,6 +514,46 @@ class EventStore extends events.EventEmitter {
|
|
|
380
514
|
this.emit('stream-deleted', streamName);
|
|
381
515
|
}
|
|
382
516
|
|
|
517
|
+
/**
|
|
518
|
+
* Close a stream so that no new events are indexed into it.
|
|
519
|
+
* The stream will still be readable, but any attempt to write to it will throw an error.
|
|
520
|
+
* A closed stream is persisted by renaming its index file to include a `.closed` marker
|
|
521
|
+
* (e.g. `stream-X.closed.index`), so it will be recognized as closed when the store is reopened.
|
|
522
|
+
*
|
|
523
|
+
* @api
|
|
524
|
+
* @param {string} streamName The name of the stream to close.
|
|
525
|
+
* @returns void
|
|
526
|
+
* @throws {Error} If the storage is read-only.
|
|
527
|
+
* @throws {Error} If the stream does not exist.
|
|
528
|
+
* @throws {Error} If the stream is already closed.
|
|
529
|
+
*/
|
|
530
|
+
closeEventStream(streamName) {
|
|
531
|
+
assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not close a stream on it.');
|
|
532
|
+
assert(streamName in this.streams, `Stream "${streamName}" does not exist.`);
|
|
533
|
+
assert(!this.streams[streamName].closed, `Stream "${streamName}" is already closed.`);
|
|
534
|
+
|
|
535
|
+
const indexName = 'stream-' + streamName;
|
|
536
|
+
const { index } = this.streams[streamName];
|
|
537
|
+
|
|
538
|
+
// Flush and close the index before renaming the file
|
|
539
|
+
index.close();
|
|
540
|
+
|
|
541
|
+
// Rename the index file to mark it as closed (e.g. stream-foo.index -> stream-foo.closed.index)
|
|
542
|
+
const closedFileName = index.fileName.replace(/\.index$/, '.closed.index');
|
|
543
|
+
fs.renameSync(index.fileName, closedFileName);
|
|
544
|
+
|
|
545
|
+
// Remove from secondary indexes so that new writes are no longer indexed into this stream
|
|
546
|
+
delete this.storage.secondaryIndexes[indexName];
|
|
547
|
+
|
|
548
|
+
// Reopen the renamed index for read access, outside the secondary indexes write path
|
|
549
|
+
const closedIndexName = indexName + '.closed';
|
|
550
|
+
const closedIndex = this.storage.openReadonlyIndex(closedIndexName);
|
|
551
|
+
|
|
552
|
+
// deepcode ignore PrototypePollutionFunctionParams: streams is a Map
|
|
553
|
+
this.streams[streamName] = { index: closedIndex, closed: true };
|
|
554
|
+
this.emit('stream-closed', streamName);
|
|
555
|
+
}
|
|
556
|
+
|
|
383
557
|
/**
|
|
384
558
|
* Get a durable consumer for the given stream that will keep receiving events from the last position.
|
|
385
559
|
*
|
|
@@ -390,14 +564,34 @@ class EventStore extends events.EventEmitter {
|
|
|
390
564
|
* @returns {Consumer} A durable consumer for the given stream.
|
|
391
565
|
*/
|
|
392
566
|
getConsumer(streamName, identifier, initialState = {}, since = 0) {
|
|
393
|
-
const consumer = new Consumer(this.storage, 'stream-' + streamName, identifier, initialState, since);
|
|
567
|
+
const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since);
|
|
394
568
|
consumer.streamName = streamName;
|
|
395
|
-
return consumer
|
|
569
|
+
return consumer;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
/**
|
|
573
|
+
* Scan the existing consumers on this EventStore and asynchronously return a list of their names.
|
|
574
|
+
* @param {function(error: Error, consumers: array)} callback A callback that will receive an error as first and the list of consumers as second argument.
|
|
575
|
+
*/
|
|
576
|
+
scanConsumers(callback) {
|
|
577
|
+
const consumersPath = path.join(this.storage.indexDirectory, 'consumers');
|
|
578
|
+
if (!fs.existsSync(consumersPath)) {
|
|
579
|
+
callback(null, []);
|
|
580
|
+
return;
|
|
581
|
+
}
|
|
582
|
+
const regex = new RegExp(`^${this.storage.storageFile}\\.([^.]*\\..*)$`);
|
|
583
|
+
const consumers = [];
|
|
584
|
+
scanForFiles(consumersPath, regex, consumers.push.bind(consumers), /* istanbul ignore next */ (err) => {
|
|
585
|
+
if (err) {
|
|
586
|
+
return callback(err, []);
|
|
587
|
+
}
|
|
588
|
+
callback(null, consumers);
|
|
589
|
+
});
|
|
396
590
|
}
|
|
397
591
|
}
|
|
398
592
|
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
593
|
+
EventStore.Storage = Storage;
|
|
594
|
+
EventStore.Index = Index;
|
|
595
|
+
|
|
596
|
+
export default EventStore;
|
|
597
|
+
export { ExpectedVersion, OptimisticConcurrencyError, LOCK_THROW, LOCK_RECLAIM };
|