event-storage 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -4
- package/index.js +1 -0
- package/package.json +4 -5
- package/src/Consumer.js +16 -20
- package/src/EventStore.js +176 -118
- package/src/EventStream.js +56 -38
- package/src/Index/ReadOnlyIndex.js +1 -1
- package/src/Index/ReadableIndex.js +9 -9
- package/src/Index/WritableIndex.js +6 -10
- package/src/IndexMatcher.js +2 -2
- package/src/JoinEventStream.js +33 -59
- package/src/Partition/ReadOnlyPartition.js +1 -1
- package/src/Partition/ReadablePartition.js +158 -90
- package/src/Partition/WritablePartition.js +38 -29
- package/src/Storage/ReadOnlyStorage.js +4 -4
- package/src/Storage/ReadableStorage.js +81 -113
- package/src/Storage/WritableStorage.js +52 -37
- package/src/Watcher.js +1 -1
- package/src/utils/apiHelpers.js +123 -0
- package/src/{fsUtil.js → utils/fsUtil.js} +27 -23
- package/src/utils/jsonUtil.js +302 -0
- package/src/utils/metadataUtil.js +517 -0
- package/src/{util.js → utils/util.js} +69 -31
- package/src/metadataUtil.js +0 -126
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import events from 'events';
|
|
4
|
-
import
|
|
5
|
-
import
|
|
6
|
-
import { assert, wrapAndCheck, kWayMerge } from '../util.js';
|
|
7
|
-
import { scanForFiles } from '../fsUtil.js';
|
|
8
|
-
import { createHmac, matches, buildMetadataForMatcher } from '../metadataUtil.js';
|
|
4
|
+
import { ReadOnly as ReadOnlyPartition } from '../Partition.js';
|
|
5
|
+
import { ReadOnly as ReadOnlyIndex } from '../Index.js';
|
|
6
|
+
import { assert, wrapAndCheck, iterate, kWayMerge } from '../utils/util.js';
|
|
7
|
+
import { scanForFiles } from '../utils/fsUtil.js';
|
|
8
|
+
import { createHmac, matches, buildMetadataForMatcher } from '../utils/metadataUtil.js';
|
|
9
|
+
import { normalizeNamedCtorArgs } from '../utils/apiHelpers.js';
|
|
9
10
|
import IndexMatcher from '../IndexMatcher.js';
|
|
10
11
|
import PartitionPool from '../PartitionPool.js';
|
|
11
12
|
|
|
@@ -26,18 +27,6 @@ const DEFAULT_MATCHER_PROPERTIES = ['stream', 'payload.type'];
|
|
|
26
27
|
*/
|
|
27
28
|
const DEFAULT_MAX_OPEN_PARTITIONS = 1024;
|
|
28
29
|
|
|
29
|
-
/**
|
|
30
|
-
* Reverses the items of an iterable
|
|
31
|
-
* @param {Generator|Iterable} iterator
|
|
32
|
-
* @returns {Generator<*>}
|
|
33
|
-
*/
|
|
34
|
-
function *reverse(iterator) {
|
|
35
|
-
const items = Array.from(iterator);
|
|
36
|
-
for (let i = items.length - 1; i >= 0; i--) {
|
|
37
|
-
yield items[i];
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
|
|
41
30
|
/**
|
|
42
31
|
* @typedef {object|function(object):boolean} Matcher
|
|
43
32
|
*/
|
|
@@ -71,10 +60,7 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
71
60
|
*/
|
|
72
61
|
constructor(storageName = 'storage', config = {}) {
|
|
73
62
|
super();
|
|
74
|
-
|
|
75
|
-
config = storageName;
|
|
76
|
-
storageName = undefined;
|
|
77
|
-
}
|
|
63
|
+
({ name: storageName, options: config } = normalizeNamedCtorArgs(storageName, config));
|
|
78
64
|
|
|
79
65
|
this.storageFile = storageName || 'storage';
|
|
80
66
|
const defaults = {
|
|
@@ -169,7 +155,7 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
169
155
|
const partition = this.createPartition(file, this.partitionConfig);
|
|
170
156
|
this.partitions.add(partition.id, partition);
|
|
171
157
|
}, (partErr) => {
|
|
172
|
-
/*
|
|
158
|
+
/* c8 ignore next */
|
|
173
159
|
if (partErr) throw partErr;
|
|
174
160
|
|
|
175
161
|
// Scan was cancelled by close() between the two scan phases.
|
|
@@ -184,7 +170,7 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
184
170
|
this.emit('index-created', name);
|
|
185
171
|
}, (indexErr) => {
|
|
186
172
|
// The directory could disappear between existsSync and readdir (e.g. test cleanup).
|
|
187
|
-
/*
|
|
173
|
+
/* c8 ignore next */
|
|
188
174
|
if (indexErr && indexErr.code !== 'ENOENT') throw indexErr;
|
|
189
175
|
done();
|
|
190
176
|
});
|
|
@@ -284,16 +270,19 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
284
270
|
* @param {number} partitionId The partition to read from.
|
|
285
271
|
* @param {number} position The file position to read from.
|
|
286
272
|
* @param {number} [size] The expected byte size of the document at the given position.
|
|
287
|
-
* @
|
|
273
|
+
* @param {boolean} [raw] Whether to return raw buffers instead of deserialized objects. Default false.
|
|
274
|
+
* @param {boolean} [backwardsHint] If set to true, will optimize buffering for backwards reading.
|
|
275
|
+
* @returns {object|{ buffer: Buffer, time64: number, sequenceNumber: number }} The document stored at the given position.
|
|
288
276
|
* @throws {Error} if the document at the given position can not be deserialized.
|
|
289
277
|
*/
|
|
290
|
-
readFrom(partitionId, position, size) {
|
|
278
|
+
readFrom(partitionId, position, size, raw = false, backwardsHint = false) {
|
|
291
279
|
const partition = this.getPartition(partitionId);
|
|
292
280
|
if (this.listenerCount('preRead') > 0) {
|
|
293
281
|
this.emit('preRead', position, partition.metadata);
|
|
294
282
|
}
|
|
295
|
-
const
|
|
296
|
-
|
|
283
|
+
const headerOut = {};
|
|
284
|
+
const buffer = partition.readFrom(position, size, headerOut, backwardsHint);
|
|
285
|
+
return raw ? { buffer, time64: headerOut.time64, sequenceNumber: headerOut.sequenceNumber } : this.serializer.deserialize(buffer.toString('utf8'));
|
|
297
286
|
}
|
|
298
287
|
|
|
299
288
|
/**
|
|
@@ -306,10 +295,7 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
306
295
|
*/
|
|
307
296
|
read(number, index) {
|
|
308
297
|
index = index || this.index;
|
|
309
|
-
|
|
310
|
-
if (!index.isOpen()) {
|
|
311
|
-
index.open();
|
|
312
|
-
}
|
|
298
|
+
index.open();
|
|
313
299
|
|
|
314
300
|
const entry = index.get(number);
|
|
315
301
|
if (entry === false) {
|
|
@@ -329,30 +315,22 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
329
315
|
* @param {ReadableIndex|false} [index] The index to use for finding the documents in the range.
|
|
330
316
|
* Pass `false` to skip the global index and iterate all partitions directly in sequenceNumber order
|
|
331
317
|
* (useful when the global index is unavailable or corrupted).
|
|
318
|
+
* @param {boolean} [raw] Whether to return raw buffers instead of deserialized objects. Default false.
|
|
332
319
|
* @returns {Generator<object>} A generator that will read each document in the range one by one.
|
|
333
320
|
*/
|
|
334
|
-
*readRange(from, until = -1, index = null) {
|
|
335
|
-
|
|
336
|
-
if (
|
|
337
|
-
|
|
321
|
+
*readRange(from, until = -1, index = null, raw = false) {
|
|
322
|
+
let length = Number.MAX_SAFE_INTEGER;
|
|
323
|
+
if (index !== false) {
|
|
324
|
+
index = index || this.index;
|
|
325
|
+
index.open();
|
|
326
|
+
length = index.length;
|
|
338
327
|
}
|
|
339
328
|
|
|
340
|
-
const readFrom = wrapAndCheck(from,
|
|
341
|
-
const readUntil = wrapAndCheck(until,
|
|
329
|
+
const readFrom = wrapAndCheck(from, length);
|
|
330
|
+
const readUntil = wrapAndCheck(until, length);
|
|
342
331
|
assert(readFrom > 0 && readUntil > 0, `Range scan error for range ${from} - ${until}.`);
|
|
343
332
|
|
|
344
|
-
|
|
345
|
-
const batchSize = 10;
|
|
346
|
-
let batchUntil = readFrom;
|
|
347
|
-
while (batchUntil >= readUntil) {
|
|
348
|
-
const batchFrom = Math.max(readUntil, batchUntil - batchSize);
|
|
349
|
-
yield* reverse(this.iterateRange(batchFrom, batchUntil, index));
|
|
350
|
-
batchUntil = batchFrom - 1;
|
|
351
|
-
}
|
|
352
|
-
return undefined;
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
yield* this.iterateRange(readFrom, readUntil, index);
|
|
333
|
+
yield* this.iterateRange(readFrom, readUntil, index, raw);
|
|
356
334
|
}
|
|
357
335
|
|
|
358
336
|
/**
|
|
@@ -362,23 +340,25 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
362
340
|
* @param {number} from
|
|
363
341
|
* @param {number} until
|
|
364
342
|
* @param {ReadableIndex|false|null} index
|
|
343
|
+
* @param {boolean} [raw] Whether to return raw buffers instead of deserialized objects. Default false.
|
|
365
344
|
* @returns {Generator<object>}
|
|
366
345
|
*/
|
|
367
|
-
*iterateRange(from, until, index) {
|
|
346
|
+
*iterateRange(from, until, index, raw = false) {
|
|
368
347
|
if (index === false) {
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
for (const entry of this.iterateDocumentsNoIndex(from - 1, until - 1)) {
|
|
372
|
-
yield entry.document;
|
|
348
|
+
for (const { document } of this.iterateDocumentsNoIndex(from - 1, until - 1)) {
|
|
349
|
+
yield document;
|
|
373
350
|
}
|
|
374
351
|
return;
|
|
375
352
|
}
|
|
376
353
|
|
|
377
354
|
const idx = index || this.index;
|
|
378
|
-
const
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
355
|
+
const forwards = from <= until;
|
|
356
|
+
const lo = Math.min(from, until);
|
|
357
|
+
const hi = Math.max(from, until);
|
|
358
|
+
const entries = idx.range(lo, hi);
|
|
359
|
+
if (!entries) return;
|
|
360
|
+
for (const entry of iterate(entries, forwards)) {
|
|
361
|
+
yield this.readFrom(entry.partition, entry.position, entry.size, raw, !forwards);
|
|
382
362
|
}
|
|
383
363
|
}
|
|
384
364
|
|
|
@@ -448,76 +428,64 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
448
428
|
}
|
|
449
429
|
}
|
|
450
430
|
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
stream => items.push({
|
|
489
|
-
document: this.serializer.deserialize(stream.data),
|
|
490
|
-
sequenceNumber: stream.headerOut.sequenceNumber,
|
|
491
|
-
partitionName: stream.partitionName,
|
|
492
|
-
position: stream.headerOut.position,
|
|
493
|
-
size: stream.headerOut.dataSize,
|
|
494
|
-
partition: stream.partition,
|
|
495
|
-
})
|
|
496
|
-
);
|
|
497
|
-
|
|
498
|
-
yield* items;
|
|
499
|
-
}
|
|
431
|
+
/**
|
|
432
|
+
* Build the standard document result entry from a readRange yield.
|
|
433
|
+
* @private
|
|
434
|
+
* @param {{ data: Buffer, entry: { number: number, position: number, size: number, partition: number } }} [readItem]
|
|
435
|
+
*/
|
|
436
|
+
buildDocumentEntry(readItem) {
|
|
437
|
+
return {
|
|
438
|
+
document: this.serializer.deserialize(readItem.data.toString('utf8')),
|
|
439
|
+
// Replicate the index entry structure here, so iteration can be used easily to reindex
|
|
440
|
+
entry: readItem.entry
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/**
|
|
445
|
+
* Iterate documents across all partitions in sequenceNumber order using a k-way merge.
|
|
446
|
+
* Opens any closed partition automatically.
|
|
447
|
+
*
|
|
448
|
+
* @protected
|
|
449
|
+
* @param {number} [from=0] The 0-based sequenceNumber to start from (inclusive).
|
|
450
|
+
* @param {number} [until=Number.MAX_SAFE_INTEGER] The 0-based sequenceNumber to read until (inclusive).
|
|
451
|
+
* @returns {Generator<{document: object, entry: { sequenceNumber: number, position: number, size: number, partition: number }}>}
|
|
452
|
+
*/
|
|
453
|
+
*iterateDocumentsNoIndex(from = 0, until = Number.MAX_SAFE_INTEGER) {
|
|
454
|
+
const forwards = from <= until;
|
|
455
|
+
const partitions = [];
|
|
456
|
+
this.forEachPartition(partition => {
|
|
457
|
+
partition.open();
|
|
458
|
+
partitions.push(partition.readRange(from, until));
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
yield* kWayMerge(
|
|
462
|
+
partitions,
|
|
463
|
+
item => item.entry.number,
|
|
464
|
+
forwards,
|
|
465
|
+
item => this.buildDocumentEntry(item)
|
|
466
|
+
);
|
|
467
|
+
}
|
|
500
468
|
|
|
501
469
|
/**
|
|
502
470
|
* Helper method to iterate over all documents, invoking a callback for each one.
|
|
503
471
|
* Pass `noIndex = true` to iterate all partitions directly in sequenceNumber order
|
|
504
472
|
* (useful when the global index is unavailable or corrupted).
|
|
505
473
|
* When `noIndex` is false the second callback argument is the raw index `EntryInterface`.
|
|
506
|
-
* When `noIndex` is true the second callback argument has `{ partition, position, size, sequenceNumber
|
|
474
|
+
* When `noIndex` is true the second callback argument has `{ partition, position, size, sequenceNumber }`.
|
|
507
475
|
*
|
|
508
476
|
* @protected
|
|
509
477
|
* @param {function(object, object): void} iterationHandler
|
|
510
478
|
* @param {boolean} [noIndex=false] When true, bypasses the index and iterates partitions directly.
|
|
511
479
|
*/
|
|
512
480
|
forEachDocument(iterationHandler, noIndex = false) {
|
|
513
|
-
/*
|
|
481
|
+
/* c8 ignore next 3 */
|
|
514
482
|
if (typeof iterationHandler !== 'function') {
|
|
515
483
|
return;
|
|
516
484
|
}
|
|
517
485
|
|
|
518
486
|
if (noIndex) {
|
|
519
|
-
for (const { document,
|
|
520
|
-
iterationHandler(document,
|
|
487
|
+
for (const { document, entry } of this.iterateDocumentsNoIndex()) {
|
|
488
|
+
iterationHandler(document, entry);
|
|
521
489
|
}
|
|
522
490
|
return;
|
|
523
491
|
}
|
|
@@ -541,7 +509,7 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
541
509
|
* @param {object} [matchDocument] If supplied, only indexes the document matches on will be iterated.
|
|
542
510
|
*/
|
|
543
511
|
forEachSecondaryIndex(iterationHandler, matchDocument) {
|
|
544
|
-
/*
|
|
512
|
+
/* c8 ignore next 3 */
|
|
545
513
|
if (typeof iterationHandler !== 'function') {
|
|
546
514
|
return;
|
|
547
515
|
}
|
|
@@ -566,7 +534,7 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
566
534
|
* @param {function(ReadablePartition)} iterationHandler
|
|
567
535
|
*/
|
|
568
536
|
forEachPartition(iterationHandler) {
|
|
569
|
-
/*
|
|
537
|
+
/* c8 ignore next 3 */
|
|
570
538
|
if (typeof iterationHandler !== 'function') {
|
|
571
539
|
return;
|
|
572
540
|
}
|
|
@@ -3,9 +3,10 @@ import path from 'path';
|
|
|
3
3
|
import WritablePartition from '../Partition/WritablePartition.js';
|
|
4
4
|
import WritableIndex, { Entry as WritableIndexEntry } from '../Index/WritableIndex.js';
|
|
5
5
|
import ReadableStorage from './ReadableStorage.js';
|
|
6
|
-
import { assert } from '../util.js';
|
|
7
|
-
import { ensureDirectory } from '../fsUtil.js';
|
|
8
|
-
import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../metadataUtil.js';
|
|
6
|
+
import { assert } from '../utils/util.js';
|
|
7
|
+
import { ensureDirectory } from '../utils/fsUtil.js';
|
|
8
|
+
import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../utils/metadataUtil.js';
|
|
9
|
+
import { normalizeNamedCtorArgs } from '../utils/apiHelpers.js';
|
|
9
10
|
|
|
10
11
|
const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024;
|
|
11
12
|
|
|
@@ -44,10 +45,7 @@ class WritableStorage extends ReadableStorage {
|
|
|
44
45
|
* @param {number} [config.lock] One of LOCK_* constants that defines how an existing lock should be handled.
|
|
45
46
|
*/
|
|
46
47
|
constructor(storageName = 'storage', config = {}) {
|
|
47
|
-
|
|
48
|
-
config = storageName;
|
|
49
|
-
storageName = undefined;
|
|
50
|
-
}
|
|
48
|
+
({ name: storageName, options: config } = normalizeNamedCtorArgs(storageName, config));
|
|
51
49
|
const defaults = {
|
|
52
50
|
partitioner: (document, number) => '',
|
|
53
51
|
writeBufferSize: DEFAULT_WRITE_BUFFER_SIZE,
|
|
@@ -64,6 +62,7 @@ class WritableStorage extends ReadableStorage {
|
|
|
64
62
|
this.lockFile = path.resolve(this.dataDirectory, this.storageFile + '.lock');
|
|
65
63
|
this._lockMode = config.lock;
|
|
66
64
|
this.partitioner = config.partitioner;
|
|
65
|
+
this.partitionIds = {};
|
|
67
66
|
}
|
|
68
67
|
|
|
69
68
|
/**
|
|
@@ -99,7 +98,7 @@ class WritableStorage extends ReadableStorage {
|
|
|
99
98
|
*/
|
|
100
99
|
forEachWritableSecondaryIndex(iterationHandler, matchDocument) {
|
|
101
100
|
this.forEachSecondaryIndex((index, name) => {
|
|
102
|
-
/*
|
|
101
|
+
/* c8 ignore next */
|
|
103
102
|
if (!(index instanceof WritableIndex)) return;
|
|
104
103
|
const wasOpen = index.isOpen();
|
|
105
104
|
if (!wasOpen) index.open();
|
|
@@ -122,7 +121,7 @@ class WritableStorage extends ReadableStorage {
|
|
|
122
121
|
this.forEachPartition(partition => {
|
|
123
122
|
partition.open();
|
|
124
123
|
const last = partition.readLast();
|
|
125
|
-
/*
|
|
124
|
+
/* c8 ignore next */
|
|
126
125
|
if (!last) return;
|
|
127
126
|
const { header: { sequenceNumber, dataSize }, position } = last;
|
|
128
127
|
if (position + partition.documentWriteSize(dataSize) > partition.size) {
|
|
@@ -164,7 +163,7 @@ class WritableStorage extends ReadableStorage {
|
|
|
164
163
|
// Truncate all indexes to the torn-write boundary.
|
|
165
164
|
this.index.open();
|
|
166
165
|
this.index.truncate(lastValidSequenceNumber);
|
|
167
|
-
/*
|
|
166
|
+
/* c8 ignore next */
|
|
168
167
|
this.forEachWritableSecondaryIndex(index => {
|
|
169
168
|
index.truncate(index.find(lastValidSequenceNumber));
|
|
170
169
|
});
|
|
@@ -204,8 +203,8 @@ class WritableStorage extends ReadableStorage {
|
|
|
204
203
|
|
|
205
204
|
// Scan partitions in sequence-number order and rebuild index entries.
|
|
206
205
|
// iterateDocumentsNoIndex opens any closed partitions automatically.
|
|
207
|
-
for (const { document,
|
|
208
|
-
const newEntry = new WritableIndexEntry(this.index.length + 1, position, size, partition);
|
|
206
|
+
for (const { document, entry } of this.iterateDocumentsNoIndex(fromSequenceNumber)) {
|
|
207
|
+
const newEntry = new WritableIndexEntry(this.index.length + 1, entry.position, entry.size, entry.partition);
|
|
209
208
|
this.index.add(newEntry);
|
|
210
209
|
|
|
211
210
|
this.forEachWritableSecondaryIndex((secIndex) => {
|
|
@@ -230,10 +229,9 @@ class WritableStorage extends ReadableStorage {
|
|
|
230
229
|
fs.mkdirSync(this.lockFile);
|
|
231
230
|
this.locked = true;
|
|
232
231
|
} catch (e) {
|
|
233
|
-
/*
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
}
|
|
232
|
+
/* c8 ignore next */
|
|
233
|
+
assert(e.code === 'EEXIST', `Error creating lock for storage ${this.storageFile}: ` + e.message)
|
|
234
|
+
|
|
237
235
|
throw new StorageLockedError(`Storage ${this.storageFile} is locked by another process`);
|
|
238
236
|
}
|
|
239
237
|
return true;
|
|
@@ -290,7 +288,7 @@ class WritableStorage extends ReadableStorage {
|
|
|
290
288
|
const entry = new WritableIndexEntry(this.index.length + 1, position, size, partitionId);
|
|
291
289
|
this.index.add(entry, (indexPosition) => {
|
|
292
290
|
this.emit('wrote', document, entry, indexPosition);
|
|
293
|
-
/*
|
|
291
|
+
/* c8 ignore next 3 */
|
|
294
292
|
if (typeof callback === 'function') {
|
|
295
293
|
return callback(indexPosition);
|
|
296
294
|
}
|
|
@@ -311,6 +309,36 @@ class WritableStorage extends ReadableStorage {
|
|
|
311
309
|
this.on('preCommit', hook);
|
|
312
310
|
}
|
|
313
311
|
|
|
312
|
+
getPartitionIdForName(partitionShortName, partitionName) {
|
|
313
|
+
const partitionId = this.partitionIds[partitionShortName] ?? WritablePartition.idFor(partitionName);
|
|
314
|
+
this.partitionIds[partitionShortName] = partitionId;
|
|
315
|
+
return partitionId;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
buildPartitionConfig(partitionShortName) {
|
|
319
|
+
if (typeof this.partitionConfig.metadata !== 'function') {
|
|
320
|
+
return this.partitionConfig;
|
|
321
|
+
}
|
|
322
|
+
return { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) };
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
ensurePartitionDirectory(partitionName) {
|
|
326
|
+
if (!partitionName.includes('/')) {
|
|
327
|
+
return;
|
|
328
|
+
}
|
|
329
|
+
ensureDirectory(path.join(this.dataDirectory, path.dirname(partitionName)));
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
createNamedPartition(partitionId, partitionName, partitionShortName) {
|
|
333
|
+
if (this.partitions.has(partitionId)) {
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
const partitionConfig = this.buildPartitionConfig(partitionShortName);
|
|
337
|
+
this.ensurePartitionDirectory(partitionName);
|
|
338
|
+
this.partitions.add(partitionId, this.createPartition(partitionName, partitionConfig));
|
|
339
|
+
this.emit('partition-created', partitionId);
|
|
340
|
+
}
|
|
341
|
+
|
|
314
342
|
/**
|
|
315
343
|
* Get a partition either by name or by id.
|
|
316
344
|
* If a partition with the given name does not exist, a new one will be created.
|
|
@@ -327,17 +355,8 @@ class WritableStorage extends ReadableStorage {
|
|
|
327
355
|
if (typeof partitionIdentifier === 'string') {
|
|
328
356
|
const partitionShortName = partitionIdentifier;
|
|
329
357
|
const partitionName = this.storageFile + (partitionIdentifier.length ? '.' + partitionIdentifier : '');
|
|
330
|
-
partitionIdentifier =
|
|
331
|
-
|
|
332
|
-
const partitionConfig = typeof this.partitionConfig.metadata === 'function'
|
|
333
|
-
? { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) }
|
|
334
|
-
: this.partitionConfig;
|
|
335
|
-
if (partitionName.includes('/')) {
|
|
336
|
-
ensureDirectory(path.join(this.dataDirectory, path.dirname(partitionName)));
|
|
337
|
-
}
|
|
338
|
-
this.partitions.add(partitionIdentifier, this.createPartition(partitionName, partitionConfig));
|
|
339
|
-
this.emit('partition-created', partitionIdentifier);
|
|
340
|
-
}
|
|
358
|
+
partitionIdentifier = this.getPartitionIdForName(partitionShortName, partitionName);
|
|
359
|
+
this.createNamedPartition(partitionIdentifier, partitionName, partitionShortName);
|
|
341
360
|
}
|
|
342
361
|
return super.getPartition(partitionIdentifier);
|
|
343
362
|
}
|
|
@@ -354,18 +373,15 @@ class WritableStorage extends ReadableStorage {
|
|
|
354
373
|
|
|
355
374
|
const partitionName = this.partitioner(document, this.index.length + 1);
|
|
356
375
|
const partition = this.getPartition(partitionName);
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
}
|
|
376
|
+
this.emit('preCommit', document, partition.metadata);
|
|
377
|
+
|
|
360
378
|
const position = partition.write(data, this.length, callback);
|
|
361
379
|
|
|
362
380
|
assert(position !== false, 'Error writing document.');
|
|
363
381
|
|
|
364
382
|
const indexEntry = this.addIndex(partition.id, position, dataSize, document);
|
|
365
383
|
this.forEachSecondaryIndex((index, name) => {
|
|
366
|
-
|
|
367
|
-
index.open();
|
|
368
|
-
}
|
|
384
|
+
index.open();
|
|
369
385
|
index.add(indexEntry);
|
|
370
386
|
this.emit('index-add', name, index.length, document);
|
|
371
387
|
}, document);
|
|
@@ -491,9 +507,8 @@ class WritableStorage extends ReadableStorage {
|
|
|
491
507
|
2) truncate all partitions accordingly
|
|
492
508
|
3) truncate/rewrite all indexes
|
|
493
509
|
*/
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
}
|
|
510
|
+
this.index.open();
|
|
511
|
+
|
|
497
512
|
if (after < 0) {
|
|
498
513
|
after += this.index.length;
|
|
499
514
|
}
|
package/src/Watcher.js
CHANGED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Normalize commit overloads into a single argument object.
|
|
3
|
+
*
|
|
4
|
+
* @param {object|object[]} events Event or event list.
|
|
5
|
+
* @param {number|object|function} expectedVersion Expected version, CommitCondition, or already metadata/callback.
|
|
6
|
+
* @param {object|function} metadata Commit metadata or callback.
|
|
7
|
+
* @param {function} callback Completion callback.
|
|
8
|
+
* @param {number} ExpectedVersionAny Fallback value for "any" expectedVersion.
|
|
9
|
+
* @param {Function} CommitConditionClass Class used for CommitCondition checks.
|
|
10
|
+
* @returns {{events: object[], expectedVersion: number|object, metadata: object, callback: function}} Normalized commit arguments.
|
|
11
|
+
*/
|
|
12
|
+
function fixCommitArgumentTypes(events, expectedVersion, metadata, callback, ExpectedVersionAny, CommitConditionClass) {
|
|
13
|
+
if (!(events instanceof Array)) {
|
|
14
|
+
events = [events];
|
|
15
|
+
}
|
|
16
|
+
if (typeof expectedVersion !== 'number' && !(expectedVersion instanceof CommitConditionClass)) {
|
|
17
|
+
callback = metadata;
|
|
18
|
+
metadata = expectedVersion;
|
|
19
|
+
expectedVersion = ExpectedVersionAny;
|
|
20
|
+
}
|
|
21
|
+
if (typeof metadata !== 'object') {
|
|
22
|
+
callback = metadata;
|
|
23
|
+
metadata = {};
|
|
24
|
+
}
|
|
25
|
+
if (typeof callback !== 'function') {
|
|
26
|
+
callback = () => {};
|
|
27
|
+
}
|
|
28
|
+
return { events, expectedVersion, metadata, callback };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Derive the stream name from an index name.
|
|
33
|
+
*
|
|
34
|
+
* @param {string} indexName Index file/index name.
|
|
35
|
+
* @returns {string} Corresponding stream name.
|
|
36
|
+
*/
|
|
37
|
+
function parseStreamFromIndexName(indexName) {
|
|
38
|
+
if (indexName === '_all') {
|
|
39
|
+
return '_all';
|
|
40
|
+
}
|
|
41
|
+
if (indexName.startsWith('stream-')) {
|
|
42
|
+
return indexName.slice(7);
|
|
43
|
+
}
|
|
44
|
+
return indexName;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Support predicate/raw shorthand (`predicate=true`).
|
|
49
|
+
*
|
|
50
|
+
* @param {object|function|boolean|null} predicate Filter predicate or raw shorthand.
|
|
51
|
+
* @param {boolean} raw Raw flag from the call signature.
|
|
52
|
+
* @returns {{predicate: object|function|null, raw: boolean}} Normalized predicate/raw pair.
|
|
53
|
+
*/
|
|
54
|
+
function normalizePredicateRaw(predicate, raw) {
|
|
55
|
+
if (typeof predicate === 'boolean' && raw === false) {
|
|
56
|
+
return { predicate: null, raw: predicate };
|
|
57
|
+
}
|
|
58
|
+
return { predicate, raw };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Support constructor overloads with optional `name`.
|
|
63
|
+
*
|
|
64
|
+
* @param {string|object} name Name or already the options object.
|
|
65
|
+
* @param {object} options Options object when `name` is provided.
|
|
66
|
+
* @param {string|undefined} [fallbackName=undefined] Fallback name when no explicit `name` is passed.
|
|
67
|
+
* @returns {{name: string|undefined, options: object}} Normalized name/options pair.
|
|
68
|
+
*/
|
|
69
|
+
function normalizeNamedCtorArgs(name, options, fallbackName = undefined) {
|
|
70
|
+
if (typeof name !== 'string') {
|
|
71
|
+
return { name: fallbackName, options: name };
|
|
72
|
+
}
|
|
73
|
+
return { name, options };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Normalize negative revisions relative to stream length.
|
|
78
|
+
*
|
|
79
|
+
* @param {number} version Requested revision.
|
|
80
|
+
* @param {number} length Current stream length.
|
|
81
|
+
* @returns {number} Resolved revision.
|
|
82
|
+
*/
|
|
83
|
+
function normalizeRevision(version, length) {
|
|
84
|
+
return version < 0 ? version + length + 1 : version;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Clamp and normalize maxRevision, including negative values.
|
|
89
|
+
*
|
|
90
|
+
* @param {number} length Current stream length.
|
|
91
|
+
* @param {number} maxRevision Requested max revision.
|
|
92
|
+
* @returns {number} Effective max revision in valid range.
|
|
93
|
+
*/
|
|
94
|
+
function normalizeMaxRevision(length, maxRevision) {
|
|
95
|
+
return Math.min(length, maxRevision < 0 ? length + maxRevision + 1 : maxRevision);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Support the consumer overload where the first argument is a numeric start offset.
|
|
100
|
+
*
|
|
101
|
+
* @param {object|number} initialState Initial state or numeric start offset.
|
|
102
|
+
* @param {number} startFrom Start offset from the call signature.
|
|
103
|
+
* @returns {{initialState: object, startFrom: number}} Normalized consumer initialization values.
|
|
104
|
+
*/
|
|
105
|
+
function normalizeConsumerStateArgs(initialState, startFrom) {
|
|
106
|
+
if (typeof initialState === 'number') {
|
|
107
|
+
return { initialState: {}, startFrom: initialState };
|
|
108
|
+
}
|
|
109
|
+
return { initialState, startFrom };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export {
|
|
113
|
+
fixCommitArgumentTypes,
|
|
114
|
+
parseStreamFromIndexName,
|
|
115
|
+
normalizePredicateRaw,
|
|
116
|
+
normalizeNamedCtorArgs,
|
|
117
|
+
normalizeRevision,
|
|
118
|
+
normalizeMaxRevision,
|
|
119
|
+
normalizeConsumerStateArgs
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
|