event-storage 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/package.json +3 -5
- package/src/Consumer.js +9 -7
- package/src/EventStore.js +165 -83
- package/src/EventStream.js +49 -11
- package/src/Index/ReadableIndex.js +7 -5
- package/src/Index/WritableIndex.js +4 -6
- package/src/IndexMatcher.js +2 -2
- package/src/JoinEventStream.js +30 -46
- package/src/Partition/ReadablePartition.js +153 -85
- package/src/Partition/WritablePartition.js +37 -28
- package/src/Storage/ReadOnlyStorage.js +3 -3
- package/src/Storage/ReadableStorage.js +73 -102
- package/src/Storage/WritableStorage.js +43 -25
- package/src/Watcher.js +1 -1
- package/src/utils/jsonUtil.js +87 -0
- package/src/utils/metadataUtil.js +247 -0
- package/src/{util.js → utils/util.js} +52 -17
- package/src/metadataUtil.js +0 -126
- /package/src/{fsUtil.js → utils/fsUtil.js} +0 -0
package/README.md
CHANGED
|
@@ -75,6 +75,27 @@ eventstore.on('ready', () => {
|
|
|
75
75
|
|
|
76
76
|
---
|
|
77
77
|
|
|
78
|
+
## HTTP API
|
|
79
|
+
|
|
80
|
+
To expose an event store over HTTP, see the companion package **[event-storage-http](https://github.com/albe/node-event-storage-http)**:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
npm install event-storage-http
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
```javascript
|
|
87
|
+
import EventStore from 'event-storage';
|
|
88
|
+
import { createEventStoreHttpServer } from 'event-storage-http';
|
|
89
|
+
|
|
90
|
+
const eventStore = new EventStore('my-store', { storageDirectory: './data' });
|
|
91
|
+
const server = createEventStoreHttpServer(eventStore);
|
|
92
|
+
server.listen(3000);
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
The package exposes NDJSON stream endpoints, durable consumer management, and an `HttpEventStream` client helper for consuming event streams over fetch.
|
|
96
|
+
|
|
97
|
+
---
|
|
98
|
+
|
|
78
99
|
## Documentation
|
|
79
100
|
|
|
80
101
|
The full documentation is hosted at **<https://node-event-storage.readthedocs.io/en/latest/>** and covers:
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "event-storage",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "An optimized embedded event store for node.js",
|
|
6
6
|
"keywords": [
|
|
@@ -43,10 +43,8 @@
|
|
|
43
43
|
"src/Partition/*.js",
|
|
44
44
|
"src/Storage/*.js",
|
|
45
45
|
"src/WatchesFile.js",
|
|
46
|
-
"
|
|
47
|
-
"src/
|
|
48
|
-
"src/metadataUtil.js",
|
|
49
|
-
"index.js"
|
|
46
|
+
"index.js",
|
|
47
|
+
"src/utils/*.js"
|
|
50
48
|
],
|
|
51
49
|
"license": "MIT",
|
|
52
50
|
"maintainers": [
|
package/src/Consumer.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import stream from 'stream';
|
|
2
2
|
import fs from 'fs';
|
|
3
3
|
import path from 'path';
|
|
4
|
-
import { assert } from './util.js';
|
|
5
|
-
import { ensureDirectory } from './fsUtil.js';
|
|
4
|
+
import { assert } from './utils/util.js';
|
|
5
|
+
import { ensureDirectory } from './utils/fsUtil.js';
|
|
6
6
|
import Storage from './Storage/ReadableStorage.js';
|
|
7
7
|
const MAX_CATCHUP_BATCH = 10;
|
|
8
8
|
|
|
@@ -30,8 +30,8 @@ class Consumer extends stream.Readable {
|
|
|
30
30
|
* @param {Storage} storage The storage to create the consumer for.
|
|
31
31
|
* @param {string} indexName The name of the index to consume.
|
|
32
32
|
* @param {string} identifier The unique name to identify this consumer.
|
|
33
|
-
* @param {object} [initialState] The initial state of the consumer.
|
|
34
|
-
* @param {number} [startFrom] The revision to start from within the index to consume.
|
|
33
|
+
* @param {object} [initialState={}] The initial state of the consumer.
|
|
34
|
+
* @param {number} [startFrom=0] The revision to start from within the index to consume.
|
|
35
35
|
*/
|
|
36
36
|
constructor(storage, indexName, identifier, initialState = {}, startFrom = 0) {
|
|
37
37
|
super({ objectMode: true });
|
|
@@ -111,7 +111,7 @@ class Consumer extends stream.Readable {
|
|
|
111
111
|
* May only be called from within the document handling callback.
|
|
112
112
|
*
|
|
113
113
|
* @param {object|function(object):object} newState
|
|
114
|
-
* @param {boolean} [persist] Set to false if this state update should not be persisted yet
|
|
114
|
+
* @param {boolean} [persist=true] Set to false if this state update should not be persisted yet
|
|
115
115
|
* @api
|
|
116
116
|
*/
|
|
117
117
|
setState(newState, persist = true) {
|
|
@@ -151,6 +151,7 @@ class Consumer extends stream.Readable {
|
|
|
151
151
|
if (this.doPersist) {
|
|
152
152
|
this.persist();
|
|
153
153
|
}
|
|
154
|
+
this.emit('progress', this.position, this.state);
|
|
154
155
|
}
|
|
155
156
|
|
|
156
157
|
/**
|
|
@@ -244,6 +245,7 @@ class Consumer extends stream.Readable {
|
|
|
244
245
|
const maxBatchPosition = Math.min(this.position + MAX_CATCHUP_BATCH + 1, this.index.length);
|
|
245
246
|
const documents = this.storage.readRange(this.position + 1, maxBatchPosition, this.index);
|
|
246
247
|
this.consumeDocuments(documents);
|
|
248
|
+
this.emit('progress', this.position, this.state);
|
|
247
249
|
this.once('persisted', () => catchUpBatch());
|
|
248
250
|
this.persist();
|
|
249
251
|
});
|
|
@@ -267,8 +269,8 @@ class Consumer extends stream.Readable {
|
|
|
267
269
|
/**
|
|
268
270
|
* Reset this projection to restart processing all documents again.
|
|
269
271
|
* NOTE: This will overwrite the current state of the projection and hence be destructive.
|
|
270
|
-
* @param {object} [initialState] The initial state of the consumer.
|
|
271
|
-
* @param {number} [startFrom] The revision to start from within the index to consume.
|
|
272
|
+
* @param {object} [initialState={}] The initial state of the consumer.
|
|
273
|
+
* @param {number} [startFrom=0] The revision to start from within the index to consume.
|
|
272
274
|
* @api
|
|
273
275
|
*/
|
|
274
276
|
reset(initialState = {}, startFrom = 0) {
|
package/src/EventStore.js
CHANGED
|
@@ -6,9 +6,9 @@ import events from 'events';
|
|
|
6
6
|
import Storage, { ReadOnly as ReadOnlyStorage, LOCK_THROW, LOCK_RECLAIM } from './Storage.js';
|
|
7
7
|
import Index from './Index.js';
|
|
8
8
|
import Consumer from './Consumer.js';
|
|
9
|
-
import { assert, getPropertyAtPath } from './util.js';
|
|
10
|
-
import { ensureDirectory, scanForFiles } from './fsUtil.js';
|
|
11
|
-
import { buildTypeMatcherFn } from './metadataUtil.js';
|
|
9
|
+
import { assert, getPropertyAtPath } from './utils/util.js';
|
|
10
|
+
import { ensureDirectory, scanForFiles } from './utils/fsUtil.js';
|
|
11
|
+
import { buildTypeMatcherFn } from './utils/metadataUtil.js';
|
|
12
12
|
|
|
13
13
|
const ExpectedVersion = {
|
|
14
14
|
Any: -1,
|
|
@@ -19,6 +19,8 @@ const ExpectedVersion = {
|
|
|
19
19
|
* Default matcher property paths mirroring the Storage default, used for index optimization.
|
|
20
20
|
*/
|
|
21
21
|
const DEFAULT_MATCHER_PROPERTIES = ['stream', 'payload.type'];
|
|
22
|
+
const STREAM_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_]*(?:[\/:@~+=\-#.][A-Za-z0-9_]+)*$/;
|
|
23
|
+
const STORAGE_HOOK_EVENTS = new Set(['preCommit', 'preRead']);
|
|
22
24
|
|
|
23
25
|
class OptimisticConcurrencyError extends Error {}
|
|
24
26
|
|
|
@@ -37,12 +39,14 @@ class OptimisticConcurrencyError extends Error {}
|
|
|
37
39
|
class CommitCondition {
|
|
38
40
|
/**
|
|
39
41
|
* @param {string[]} types
|
|
40
|
-
* @param {function(object, object): boolean|null} [matcher]
|
|
42
|
+
* @param {function(object, object): boolean|object|null} [matcher]
|
|
41
43
|
* @param {number} noneMatchAfter
|
|
44
|
+
* @param {boolean} [raw=false]
|
|
42
45
|
*/
|
|
43
|
-
constructor(types, matcher = null, noneMatchAfter) {
|
|
46
|
+
constructor(types, matcher = null, noneMatchAfter, raw = false) {
|
|
44
47
|
this.types = types;
|
|
45
48
|
this.matcher = matcher;
|
|
49
|
+
this.raw = raw;
|
|
46
50
|
this.noneMatchAfter = noneMatchAfter;
|
|
47
51
|
}
|
|
48
52
|
}
|
|
@@ -132,6 +136,7 @@ class EventStore extends events.EventEmitter {
|
|
|
132
136
|
: new Storage(storeName, storageConfig);
|
|
133
137
|
this.streams = Object.create(null);
|
|
134
138
|
this.streams._all = { index: this.storage.index };
|
|
139
|
+
this.consumers = new Map();
|
|
135
140
|
|
|
136
141
|
this.storage.on('index-created', this.registerStream.bind(this));
|
|
137
142
|
|
|
@@ -214,10 +219,15 @@ class EventStore extends events.EventEmitter {
|
|
|
214
219
|
|
|
215
220
|
/**
|
|
216
221
|
* Close the event store and free up all resources.
|
|
222
|
+
* Stops all registered consumers before closing storage.
|
|
217
223
|
*
|
|
218
224
|
* @api
|
|
219
225
|
*/
|
|
220
226
|
close() {
|
|
227
|
+
for (const consumer of this.consumers.values()) {
|
|
228
|
+
consumer.stop();
|
|
229
|
+
}
|
|
230
|
+
this.consumers.clear();
|
|
221
231
|
this.storage.close();
|
|
222
232
|
}
|
|
223
233
|
|
|
@@ -231,11 +241,8 @@ class EventStore extends events.EventEmitter {
|
|
|
231
241
|
* @returns {this}
|
|
232
242
|
*/
|
|
233
243
|
on(event, listener) {
|
|
234
|
-
if (event
|
|
235
|
-
|
|
236
|
-
assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.');
|
|
237
|
-
}
|
|
238
|
-
this.storage.on(event, listener);
|
|
244
|
+
if (this.isStorageHookEvent(event)) {
|
|
245
|
+
this.delegateStorageHookEvent('on', event, listener);
|
|
239
246
|
return this;
|
|
240
247
|
}
|
|
241
248
|
return super.on(event, listener);
|
|
@@ -256,11 +263,8 @@ class EventStore extends events.EventEmitter {
|
|
|
256
263
|
* @returns {this}
|
|
257
264
|
*/
|
|
258
265
|
once(event, listener) {
|
|
259
|
-
if (event
|
|
260
|
-
|
|
261
|
-
assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.');
|
|
262
|
-
}
|
|
263
|
-
this.storage.once(event, listener);
|
|
266
|
+
if (this.isStorageHookEvent(event)) {
|
|
267
|
+
this.delegateStorageHookEvent('once', event, listener);
|
|
264
268
|
return this;
|
|
265
269
|
}
|
|
266
270
|
return super.once(event, listener);
|
|
@@ -275,13 +279,24 @@ class EventStore extends events.EventEmitter {
|
|
|
275
279
|
* @returns {this}
|
|
276
280
|
*/
|
|
277
281
|
off(event, listener) {
|
|
278
|
-
if (event
|
|
282
|
+
if (this.isStorageHookEvent(event)) {
|
|
279
283
|
this.storage.off(event, listener);
|
|
280
284
|
return this;
|
|
281
285
|
}
|
|
282
286
|
return super.off(event, listener);
|
|
283
287
|
}
|
|
284
288
|
|
|
289
|
+
isStorageHookEvent(event) {
|
|
290
|
+
return STORAGE_HOOK_EVENTS.has(event);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
delegateStorageHookEvent(method, event, listener) {
|
|
294
|
+
if (event === 'preCommit') {
|
|
295
|
+
assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.');
|
|
296
|
+
}
|
|
297
|
+
this.storage[method](event, listener);
|
|
298
|
+
}
|
|
299
|
+
|
|
285
300
|
/**
|
|
286
301
|
* @inheritDoc
|
|
287
302
|
*/
|
|
@@ -378,20 +393,17 @@ class EventStore extends events.EventEmitter {
|
|
|
378
393
|
if (existingTypes.length === 0) return;
|
|
379
394
|
|
|
380
395
|
// Only events after condition.noneMatchAfter can be conflicts.
|
|
396
|
+
// Pass the original matcher and raw flag so the stream filters at the source.
|
|
381
397
|
const stream = this.fromStreams(
|
|
382
398
|
'_check_' + condition.types.join('_'),
|
|
383
399
|
existingTypes,
|
|
384
|
-
condition.noneMatchAfter + 1
|
|
400
|
+
condition.noneMatchAfter + 1,
|
|
401
|
+
-1,
|
|
402
|
+
condition.matcher,
|
|
403
|
+
condition.raw
|
|
385
404
|
);
|
|
386
405
|
|
|
387
|
-
|
|
388
|
-
while ((next = stream.next()) !== false) {
|
|
389
|
-
if (!condition.matcher || condition.matcher(next.payload, next.metadata)) {
|
|
390
|
-
throw new OptimisticConcurrencyError(
|
|
391
|
-
`Optimistic Concurrency error. A conflicting event was committed since the condition was obtained.`
|
|
392
|
-
);
|
|
393
|
-
}
|
|
394
|
-
}
|
|
406
|
+
assert(stream.next() === false, `Optimistic Concurrency error. A conflicting event was committed since the condition was obtained.`, OptimisticConcurrencyError);
|
|
395
407
|
}
|
|
396
408
|
|
|
397
409
|
/**
|
|
@@ -403,7 +415,7 @@ class EventStore extends events.EventEmitter {
|
|
|
403
415
|
ensureTypeStreams(events) {
|
|
404
416
|
if (!this.typeAccessor) return;
|
|
405
417
|
for (const event of events) {
|
|
406
|
-
const type = this.
|
|
418
|
+
const type = this.resolveValidatedTypeStreamName(event);
|
|
407
419
|
if (type && !(type in this.streams)) {
|
|
408
420
|
const matcher = this.typeMatcherFn
|
|
409
421
|
? this.typeMatcherFn(type)
|
|
@@ -413,6 +425,30 @@ class EventStore extends events.EventEmitter {
|
|
|
413
425
|
}
|
|
414
426
|
}
|
|
415
427
|
|
|
428
|
+
resolveValidatedTypeStreamName(event) {
|
|
429
|
+
const type = this.typeAccessor(event);
|
|
430
|
+
if (type === undefined || type === null || type === '') {
|
|
431
|
+
return null;
|
|
432
|
+
}
|
|
433
|
+
assert(typeof type === 'string', 'typeAccessor must return a string.');
|
|
434
|
+
assert(STREAM_NAME_PATTERN.test(type), `typeAccessor must return a valid stream name. Got: "${type}"`);
|
|
435
|
+
return type;
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
getExistingQueryTypes(types) {
|
|
439
|
+
const queryTypes = [];
|
|
440
|
+
for (const type of types) {
|
|
441
|
+
if (type in this.streams) {
|
|
442
|
+
queryTypes.push(type);
|
|
443
|
+
continue;
|
|
444
|
+
}
|
|
445
|
+
if (!this.typeAccessor) {
|
|
446
|
+
throw new Error(`Type stream "${type}" does not exist. Create it with createEventStream() first, or configure typeAccessor to have type streams created automatically on commit.`);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
return queryTypes;
|
|
450
|
+
}
|
|
451
|
+
|
|
416
452
|
/**
|
|
417
453
|
* Commit a list of events for the given stream name, which is expected to be at the given version.
|
|
418
454
|
* Note that the events committed may still appear in other streams too - the given stream name is only
|
|
@@ -450,9 +486,10 @@ class EventStore extends events.EventEmitter {
|
|
|
450
486
|
}
|
|
451
487
|
assert(!this.streams[streamName].closed, `Stream "${streamName}" is closed and cannot be written to.`);
|
|
452
488
|
let streamVersion = this.streams[streamName].index.length;
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
489
|
+
assert(expectedVersion === ExpectedVersion.Any || streamVersion === expectedVersion,
|
|
490
|
+
`Optimistic Concurrency error. Expected stream "${streamName}" at version ${expectedVersion} but is at version ${streamVersion}.`,
|
|
491
|
+
OptimisticConcurrencyError
|
|
492
|
+
);
|
|
456
493
|
|
|
457
494
|
if (events.length > 1) {
|
|
458
495
|
delete metadata.commitVersion;
|
|
@@ -514,35 +551,21 @@ class EventStore extends events.EventEmitter {
|
|
|
514
551
|
*
|
|
515
552
|
* @api
|
|
516
553
|
* @param {string[]} types A non-empty array of event-type names to query.
|
|
517
|
-
* @param {function
|
|
518
|
-
*
|
|
554
|
+
* @param {function|object|null} [matcher] Optional matcher used for stream pre-filtering.
|
|
555
|
+
* In object mode, function predicates receive `(payload, metadata)`.
|
|
519
556
|
* @param {number} [minRevision=1] The 1-based minimum global revision to include in the returned stream (inclusive).
|
|
557
|
+
* @param {boolean} [raw=false] If true, return NDJSON buffers from the query stream.
|
|
520
558
|
* @returns {{ condition: CommitCondition, stream: EventStream }} An object with:
|
|
521
559
|
* - `condition` — the {@link CommitCondition} to pass to {@link EventStore#commit}.
|
|
522
560
|
* - `stream` — a read-only event stream containing all matching events.
|
|
523
561
|
* @throws {Error} if `types` is not a non-empty array.
|
|
524
562
|
* @throws {Error} if `typeAccessor` is not configured and any of the listed type streams do not exist.
|
|
525
563
|
*/
|
|
526
|
-
query(types, matcher = null, minRevision = 1) {
|
|
564
|
+
query(types, matcher = null, minRevision = 1, raw = false) {
|
|
527
565
|
assert(Array.isArray(types) && types.length > 0, 'Must specify a non-empty array of event types for query.');
|
|
528
|
-
|
|
529
|
-
const
|
|
530
|
-
|
|
531
|
-
if (!(type in this.streams)) {
|
|
532
|
-
if (this.typeAccessor) {
|
|
533
|
-
// typeAccessor is configured: type streams are created on commit, so a missing
|
|
534
|
-
// stream simply means no event of this type has been committed yet — treat as empty.
|
|
535
|
-
continue;
|
|
536
|
-
}
|
|
537
|
-
// No typeAccessor: the stream was never created; we cannot know whether events of
|
|
538
|
-
// this type exist in the store, so throw to avoid an unintentional full-store scan.
|
|
539
|
-
throw new Error(`Type stream "${type}" does not exist. Create it with createEventStream() first, or configure typeAccessor to have type streams created automatically on commit.`);
|
|
540
|
-
}
|
|
541
|
-
queryTypes.push(type);
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
const condition = new CommitCondition(types, matcher, this.storage.length);
|
|
545
|
-
const stream = this.fromStreams('_query_' + types.join('_'), queryTypes, minRevision, -1, matcher);
|
|
566
|
+
const queryTypes = this.getExistingQueryTypes(types);
|
|
567
|
+
const condition = new CommitCondition(types, matcher, this.storage.length, raw);
|
|
568
|
+
const stream = this.fromStreams('_query_' + types.join('_'), queryTypes, minRevision, -1, matcher, raw);
|
|
546
569
|
return { stream, condition };
|
|
547
570
|
}
|
|
548
571
|
|
|
@@ -551,15 +574,18 @@ class EventStore extends events.EventEmitter {
|
|
|
551
574
|
*
|
|
552
575
|
* @api
|
|
553
576
|
* @param {string} streamName The name of the stream to get.
|
|
554
|
-
* @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
|
|
555
|
-
* @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
|
|
577
|
+
* @param {number} [minRevision=1] The 1-based minimum revision to include in the events (inclusive).
|
|
578
|
+
* @param {number} [maxRevision=-1] The 1-based maximum revision to include in the events (inclusive).
|
|
579
|
+
* @param {function|object|null} [predicate] Optional matcher (see {@link EventStream}).
|
|
580
|
+
* @param {boolean} [raw=false] If true, return NDJSON buffers.
|
|
556
581
|
* @returns {EventStream|boolean} The event stream or false if a stream with the name doesn't exist.
|
|
557
582
|
*/
|
|
558
|
-
getEventStream(streamName, minRevision = 1, maxRevision = -1) {
|
|
583
|
+
getEventStream(streamName, minRevision = 1, maxRevision = -1, predicate = null, raw = false) {
|
|
584
|
+
({ predicate, raw } = normalizePredicateRaw(predicate, raw));
|
|
559
585
|
if (!(streamName in this.streams)) {
|
|
560
586
|
return false;
|
|
561
587
|
}
|
|
562
|
-
return new EventStream(streamName, this, minRevision, maxRevision);
|
|
588
|
+
return new EventStream(streamName, this, minRevision, maxRevision, predicate, raw);
|
|
563
589
|
}
|
|
564
590
|
|
|
565
591
|
/**
|
|
@@ -567,12 +593,15 @@ class EventStore extends events.EventEmitter {
|
|
|
567
593
|
* This is the same as `getEventStream('_all', ...)`.
|
|
568
594
|
*
|
|
569
595
|
* @api
|
|
570
|
-
* @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
|
|
571
|
-
* @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
|
|
596
|
+
* @param {number} [minRevision=1] The 1-based minimum revision to include in the events (inclusive).
|
|
597
|
+
* @param {number} [maxRevision=-1] The 1-based maximum revision to include in the events (inclusive).
|
|
598
|
+
* @param {function|object|null} [predicate] Optional matcher (see {@link EventStream}).
|
|
599
|
+
* @param {boolean} [raw=false] If true, return NDJSON buffers.
|
|
572
600
|
* @returns {EventStream} The event stream.
|
|
573
601
|
*/
|
|
574
|
-
getAllEvents(minRevision = 1, maxRevision = -1) {
|
|
575
|
-
|
|
602
|
+
getAllEvents(minRevision = 1, maxRevision = -1, predicate = null, raw = false) {
|
|
603
|
+
({ predicate, raw } = normalizePredicateRaw(predicate, raw));
|
|
604
|
+
return this.getEventStream('_all', minRevision, maxRevision, predicate, raw);
|
|
576
605
|
}
|
|
577
606
|
|
|
578
607
|
/**
|
|
@@ -580,14 +609,15 @@ class EventStore extends events.EventEmitter {
|
|
|
580
609
|
*
|
|
581
610
|
* @param {string} streamName The (transient) name of the joined stream.
|
|
582
611
|
* @param {Array<string>} streamNames An array of the stream names to join.
|
|
583
|
-
* @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
|
|
584
|
-
* @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
|
|
585
|
-
* @param {function
|
|
586
|
-
*
|
|
612
|
+
* @param {number} [minRevision=1] The 1-based minimum revision to include in the events (inclusive).
|
|
613
|
+
* @param {number} [maxRevision=-1] The 1-based maximum revision to include in the events (inclusive).
|
|
614
|
+
* @param {function|object|null} [predicate] Optional matcher (see {@link EventStream}).
|
|
615
|
+
* @param {boolean} [raw=false] If true, return NDJSON buffers.
|
|
587
616
|
* @returns {EventStream} The joined event stream.
|
|
588
617
|
* @throws {Error} if any of the streams doesn't exist.
|
|
589
618
|
*/
|
|
590
|
-
fromStreams(streamName, streamNames, minRevision = 1, maxRevision = -1, predicate = null) {
|
|
619
|
+
fromStreams(streamName, streamNames, minRevision = 1, maxRevision = -1, predicate = null, raw = false) {
|
|
620
|
+
({ predicate, raw } = normalizePredicateRaw(predicate, raw));
|
|
591
621
|
assert(streamNames instanceof Array, 'Must specify an array of stream names.');
|
|
592
622
|
|
|
593
623
|
if (streamNames.length === 0) {
|
|
@@ -599,12 +629,12 @@ class EventStore extends events.EventEmitter {
|
|
|
599
629
|
}
|
|
600
630
|
|
|
601
631
|
if (streamNames.length === 1) {
|
|
602
|
-
const stream = new EventStream(streamNames[0], this, minRevision, maxRevision, predicate);
|
|
632
|
+
const stream = new EventStream(streamNames[0], this, minRevision, maxRevision, predicate, raw);
|
|
603
633
|
stream.name = streamName;
|
|
604
634
|
return stream;
|
|
605
635
|
}
|
|
606
636
|
|
|
607
|
-
return new JoinEventStream(streamName, streamNames, this, minRevision, maxRevision, predicate);
|
|
637
|
+
return new JoinEventStream(streamName, streamNames, this, minRevision, maxRevision, predicate, raw);
|
|
608
638
|
}
|
|
609
639
|
|
|
610
640
|
/**
|
|
@@ -618,24 +648,26 @@ class EventStore extends events.EventEmitter {
|
|
|
618
648
|
*
|
|
619
649
|
* @api
|
|
620
650
|
* @param {string} categoryName The name of the category to get a stream for. A category is a stream name prefix.
|
|
621
|
-
* @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
|
|
622
|
-
* @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
|
|
651
|
+
* @param {number} [minRevision=1] The 1-based minimum revision to include in the events (inclusive).
|
|
652
|
+
* @param {number} [maxRevision=-1] The 1-based maximum revision to include in the events (inclusive).
|
|
653
|
+
* @param {function|object|null} [predicate] Optional matcher (see {@link EventStream}).
|
|
654
|
+
* @param {boolean} [raw=false] If true, return NDJSON buffers.
|
|
623
655
|
* @returns {EventStream} The joined event stream for all streams of the given category.
|
|
624
656
|
* @throws {Error} If no stream for this category exists.
|
|
625
657
|
*/
|
|
626
|
-
getEventStreamForCategory(categoryName, minRevision = 1, maxRevision = -1) {
|
|
658
|
+
getEventStreamForCategory(categoryName, minRevision = 1, maxRevision = -1, predicate = null, raw = false) {
|
|
659
|
+
({ predicate, raw } = normalizePredicateRaw(predicate, raw));
|
|
627
660
|
if (categoryName in this.streams) {
|
|
628
|
-
return this.getEventStream(categoryName, minRevision, maxRevision);
|
|
661
|
+
return this.getEventStream(categoryName, minRevision, maxRevision, predicate, raw);
|
|
629
662
|
}
|
|
630
663
|
const categoryStreams = Object.keys(this.streams).filter(streamName =>
|
|
631
664
|
streamName.startsWith(categoryName + '-') ||
|
|
632
665
|
streamName.startsWith(categoryName + '/')
|
|
633
666
|
);
|
|
634
667
|
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
return this.fromStreams(categoryName, categoryStreams, minRevision, maxRevision);
|
|
668
|
+
assert(categoryStreams.length > 0, `No streams for category '${categoryName}' exist.`);
|
|
669
|
+
|
|
670
|
+
return this.fromStreams(categoryName, categoryStreams, minRevision, maxRevision, predicate, raw);
|
|
639
671
|
}
|
|
640
672
|
|
|
641
673
|
/**
|
|
@@ -729,41 +761,91 @@ class EventStore extends events.EventEmitter {
|
|
|
729
761
|
}
|
|
730
762
|
|
|
731
763
|
/**
|
|
732
|
-
* Get a durable consumer for the given stream
|
|
764
|
+
* Get a durable consumer for the given stream, or look up an existing consumer by identifier.
|
|
733
765
|
*
|
|
734
|
-
*
|
|
735
|
-
*
|
|
766
|
+
* When called with a single argument, returns the running consumer registered under that
|
|
767
|
+
* identifier, or `null` if none is found — useful for read endpoints that need the live
|
|
768
|
+
* in-memory instance without creating a new one.
|
|
769
|
+
*
|
|
770
|
+
* When called with two or more arguments, creates (or re-uses) a Consumer for the given
|
|
771
|
+
* stream and identifier, registers it in `this.consumers`, and returns it.
|
|
772
|
+
*
|
|
773
|
+
* @param {string} streamNameOrIdentifier The stream name, or the consumer identifier when used as a registry lookup.
|
|
774
|
+
* @param {string} [identifier] The unique identifying name of this consumer. Omit for registry-only lookup.
|
|
736
775
|
* @param {object} [initialState] The initial state of the consumer.
|
|
737
776
|
* @param {number} [since] The stream revision to start consuming from.
|
|
738
|
-
* @returns {Consumer} A durable consumer
|
|
777
|
+
* @returns {Consumer|null} A durable consumer, or `null` when looking up by identifier and none is registered.
|
|
739
778
|
*/
|
|
740
|
-
getConsumer(
|
|
779
|
+
getConsumer(streamNameOrIdentifier, identifier, initialState = {}, since = 0) {
|
|
780
|
+
if (identifier === undefined) {
|
|
781
|
+
return this.consumers.get(streamNameOrIdentifier) ?? null;
|
|
782
|
+
}
|
|
783
|
+
const streamName = streamNameOrIdentifier;
|
|
784
|
+
if (this.consumers.has(identifier)) {
|
|
785
|
+
return this.consumers.get(identifier);
|
|
786
|
+
}
|
|
741
787
|
const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since);
|
|
742
788
|
consumer.streamName = streamName;
|
|
789
|
+
this.consumers.set(identifier, consumer);
|
|
743
790
|
return consumer;
|
|
744
791
|
}
|
|
745
792
|
|
|
746
793
|
/**
|
|
747
|
-
* Scan the existing consumers on this EventStore and asynchronously
|
|
748
|
-
*
|
|
794
|
+
* Scan the existing consumers on this EventStore and asynchronously invoke a callback with the parsed list.
|
|
795
|
+
*
|
|
796
|
+
* Each consumer entry provides `{ name, stream, identifier }` parsed from the on-disk filename.
|
|
797
|
+
* Pass `autoStart = true` to eagerly open every discovered consumer and register it in
|
|
798
|
+
* `this.consumers` so that it is immediately available for registry lookups.
|
|
799
|
+
*
|
|
800
|
+
* @param {function(error: Error|null, consumers: Array<{name: string, stream: string, identifier: string}>)} callback
|
|
801
|
+
* @param {boolean} [autoStart=false] When true, calls `getConsumer(stream, identifier)` for each discovered consumer.
|
|
749
802
|
*/
|
|
750
|
-
scanConsumers(callback) {
|
|
803
|
+
scanConsumers(callback, autoStart = false) {
|
|
751
804
|
const consumersPath = path.join(this.storage.indexDirectory, 'consumers');
|
|
752
805
|
if (!fs.existsSync(consumersPath)) {
|
|
753
806
|
callback(null, []);
|
|
754
807
|
return;
|
|
755
808
|
}
|
|
756
809
|
const regex = new RegExp(`^${this.storage.storageFile}\\.([^.]*\\..*)$`);
|
|
757
|
-
const
|
|
758
|
-
scanForFiles(consumersPath, regex,
|
|
810
|
+
const consumerNames = [];
|
|
811
|
+
scanForFiles(consumersPath, regex, consumerNames.push.bind(consumerNames), /* istanbul ignore next */ (err) => {
|
|
759
812
|
if (err) {
|
|
760
813
|
return callback(err, []);
|
|
761
814
|
}
|
|
815
|
+
const consumers = consumerNames.map(name => {
|
|
816
|
+
const splitIndex = name.lastIndexOf('.');
|
|
817
|
+
const indexName = name.slice(0, splitIndex);
|
|
818
|
+
const identifier = name.slice(splitIndex + 1);
|
|
819
|
+
const stream = parseStreamFromIndexName(indexName);
|
|
820
|
+
return { name, stream, identifier };
|
|
821
|
+
});
|
|
822
|
+
if (autoStart) {
|
|
823
|
+
for (const { stream, identifier } of consumers) {
|
|
824
|
+
this.getConsumer(stream, identifier);
|
|
825
|
+
}
|
|
826
|
+
}
|
|
762
827
|
callback(null, consumers);
|
|
763
828
|
});
|
|
764
829
|
}
|
|
765
830
|
}
|
|
766
831
|
|
|
832
|
+
function parseStreamFromIndexName(indexName) {
|
|
833
|
+
if (indexName === '_all') {
|
|
834
|
+
return '_all';
|
|
835
|
+
}
|
|
836
|
+
if (indexName.startsWith('stream-')) {
|
|
837
|
+
return indexName.slice(7);
|
|
838
|
+
}
|
|
839
|
+
return indexName;
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
function normalizePredicateRaw(predicate, raw) {
|
|
843
|
+
if (typeof predicate === 'boolean' && raw === false) {
|
|
844
|
+
return { predicate: null, raw: predicate };
|
|
845
|
+
}
|
|
846
|
+
return { predicate, raw };
|
|
847
|
+
}
|
|
848
|
+
|
|
767
849
|
EventStore.Storage = Storage;
|
|
768
850
|
EventStore.Index = Index;
|
|
769
851
|
|