event-storage 0.8.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -540
- package/index.js +5 -6
- package/package.json +28 -31
- package/src/Clock.js +21 -9
- package/src/Consumer.js +6 -7
- package/src/EventStore.js +261 -67
- package/src/EventStream.js +172 -19
- package/src/Index/ReadOnlyIndex.js +3 -3
- package/src/Index/ReadableIndex.js +17 -13
- package/src/Index/WritableIndex.js +17 -11
- package/src/Index.js +7 -5
- package/src/IndexEntry.js +2 -3
- package/src/JoinEventStream.js +34 -32
- package/src/Partition/ReadOnlyPartition.js +3 -3
- package/src/Partition/ReadablePartition.js +110 -57
- package/src/Partition/WritablePartition.js +81 -23
- package/src/Partition.js +7 -4
- package/src/Storage/ReadOnlyStorage.js +4 -4
- package/src/Storage/ReadableStorage.js +144 -22
- package/src/Storage/WritableStorage.js +175 -33
- package/src/Storage.js +9 -4
- package/src/Watcher.js +6 -5
- package/src/WatchesFile.js +8 -7
- package/src/metadataUtil.js +79 -0
- package/src/util.js +74 -73
- package/test/Consumer.spec.js +0 -455
- package/test/EventStore.spec.js +0 -632
- package/test/EventStream.spec.js +0 -120
- package/test/Index.spec.js +0 -591
- package/test/JoinEventStream.spec.js +0 -113
- package/test/Partition.spec.js +0 -488
- package/test/Storage.spec.js +0 -1017
- package/test/Watcher.spec.js +0 -131
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import events from 'events';
|
|
4
|
+
import Partition, { ReadOnly as ReadOnlyPartition } from '../Partition.js';
|
|
5
|
+
import Index, { ReadOnly as ReadOnlyIndex } from '../Index.js';
|
|
6
|
+
import { assert, wrapAndCheck, kWayMerge } from '../util.js';
|
|
7
|
+
import { createHmac, matches, buildMetadataForMatcher } from '../metadataUtil.js';
|
|
7
8
|
|
|
8
9
|
const DEFAULT_READ_BUFFER_SIZE = 4 * 1024;
|
|
9
10
|
|
|
@@ -66,8 +67,8 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
66
67
|
|
|
67
68
|
this.dataDirectory = path.resolve(config.dataDirectory);
|
|
68
69
|
|
|
69
|
-
this.initializeIndexes(config);
|
|
70
70
|
this.scanPartitions(config);
|
|
71
|
+
this.initializeIndexes(config);
|
|
71
72
|
}
|
|
72
73
|
|
|
73
74
|
/**
|
|
@@ -78,7 +79,7 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
78
79
|
*/
|
|
79
80
|
createIndex(name, options = {}) {
|
|
80
81
|
/** @type ReadableIndex */
|
|
81
|
-
const index = new
|
|
82
|
+
const index = new ReadOnlyIndex(name, options);
|
|
82
83
|
return { index };
|
|
83
84
|
}
|
|
84
85
|
|
|
@@ -89,7 +90,7 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
89
90
|
* @returns {ReadablePartition}
|
|
90
91
|
*/
|
|
91
92
|
createPartition(name, options = {}) {
|
|
92
|
-
return new
|
|
93
|
+
return new ReadOnlyPartition(name, options);
|
|
93
94
|
}
|
|
94
95
|
|
|
95
96
|
/**
|
|
@@ -109,6 +110,7 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
109
110
|
const { index } = this.createIndex(config.indexFile, this.indexOptions);
|
|
110
111
|
this.index = index;
|
|
111
112
|
this.secondaryIndexes = {};
|
|
113
|
+
this.readonlyIndexes = {};
|
|
112
114
|
}
|
|
113
115
|
|
|
114
116
|
/**
|
|
@@ -172,6 +174,9 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
172
174
|
close() {
|
|
173
175
|
this.index.close();
|
|
174
176
|
this.forEachSecondaryIndex(index => index.close());
|
|
177
|
+
for (let index of Object.values(this.readonlyIndexes)) {
|
|
178
|
+
index.close();
|
|
179
|
+
}
|
|
175
180
|
this.forEachPartition(partition => partition.close());
|
|
176
181
|
this.emit('closed');
|
|
177
182
|
}
|
|
@@ -193,6 +198,19 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
193
198
|
return this.partitions[partitionIdentifier];
|
|
194
199
|
}
|
|
195
200
|
|
|
201
|
+
/**
|
|
202
|
+
* Register a handler that is called before a document is read from a partition.
|
|
203
|
+
* The handler receives the position and the partition metadata and may throw to abort the read.
|
|
204
|
+
* Multiple handlers can be registered; all run on every read in registration order.
|
|
205
|
+
* Equivalent to `storage.on('preRead', hook)`.
|
|
206
|
+
*
|
|
207
|
+
* @api
|
|
208
|
+
* @param {function(number, object): void} hook A function receiving (position, partitionMetadata).
|
|
209
|
+
*/
|
|
210
|
+
preRead(hook) {
|
|
211
|
+
this.on('preRead', hook);
|
|
212
|
+
}
|
|
213
|
+
|
|
196
214
|
/**
|
|
197
215
|
* @protected
|
|
198
216
|
* @param {number} partitionId The partition to read from.
|
|
@@ -203,6 +221,9 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
203
221
|
*/
|
|
204
222
|
readFrom(partitionId, position, size) {
|
|
205
223
|
const partition = this.getPartition(partitionId);
|
|
224
|
+
if (this.listenerCount('preRead') > 0) {
|
|
225
|
+
this.emit('preRead', position, partition.metadata);
|
|
226
|
+
}
|
|
206
227
|
const data = partition.readFrom(position, size);
|
|
207
228
|
return this.serializer.deserialize(data);
|
|
208
229
|
}
|
|
@@ -237,21 +258,25 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
237
258
|
* @api
|
|
238
259
|
* @param {number} from The 1-based document number (inclusive) to start reading from.
|
|
239
260
|
* @param {number} [until] The 1-based document number (inclusive) to read until. Defaults to index.length.
|
|
240
|
-
* @param {ReadableIndex} [index] The index to use for finding the documents in the range.
|
|
261
|
+
* @param {ReadableIndex|false} [index] The index to use for finding the documents in the range.
|
|
262
|
+
* Pass `false` to skip the global index and iterate all partitions directly in sequenceNumber order
|
|
263
|
+
* (useful when the global index is unavailable or corrupted).
|
|
241
264
|
* @returns {Generator<object>} A generator that will read each document in the range one by one.
|
|
242
265
|
*/
|
|
243
266
|
*readRange(from, until = -1, index = null) {
|
|
244
|
-
|
|
245
|
-
|
|
267
|
+
const lengthSource = index || this.index;
|
|
268
|
+
if (!lengthSource.isOpen()) {
|
|
269
|
+
lengthSource.open();
|
|
270
|
+
}
|
|
246
271
|
|
|
247
|
-
const readFrom = wrapAndCheck(from,
|
|
248
|
-
const readUntil = wrapAndCheck(until,
|
|
272
|
+
const readFrom = wrapAndCheck(from, lengthSource.length);
|
|
273
|
+
const readUntil = wrapAndCheck(until, lengthSource.length);
|
|
249
274
|
assert(readFrom > 0 && readUntil > 0, `Range scan error for range ${from} - ${until}.`);
|
|
250
275
|
|
|
251
276
|
if (readFrom > readUntil) {
|
|
252
277
|
const batchSize = 10;
|
|
253
278
|
let batchUntil = readFrom;
|
|
254
|
-
while (batchUntil
|
|
279
|
+
while (batchUntil >= readUntil) {
|
|
255
280
|
const batchFrom = Math.max(readUntil, batchUntil - batchSize);
|
|
256
281
|
yield* reverse(this.iterateRange(batchFrom, batchUntil, index));
|
|
257
282
|
batchUntil = batchFrom - 1;
|
|
@@ -264,20 +289,52 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
264
289
|
|
|
265
290
|
/**
|
|
266
291
|
* Iterate all documents in this storage in range from to until inside the index.
|
|
292
|
+
* If index is false, iterates all partitions directly in sequenceNumber order.
|
|
267
293
|
* @private
|
|
268
294
|
* @param {number} from
|
|
269
295
|
* @param {number} until
|
|
270
|
-
* @param {ReadableIndex} index
|
|
296
|
+
* @param {ReadableIndex|false|null} index
|
|
271
297
|
* @returns {Generator<object>}
|
|
272
298
|
*/
|
|
273
299
|
*iterateRange(from, until, index) {
|
|
274
|
-
|
|
300
|
+
if (index === false) {
|
|
301
|
+
// Explicitly disabled index: iterate all partitions and merge by sequenceNumber.
|
|
302
|
+
// Document header sequenceNumber is 0-based; from/until are 1-based index positions.
|
|
303
|
+
for (const entry of this.iterateDocumentsNoIndex(from - 1, until - 1)) {
|
|
304
|
+
yield entry.document;
|
|
305
|
+
}
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const idx = index || this.index;
|
|
310
|
+
const entries = idx.range(from, until);
|
|
275
311
|
for (let entry of entries) {
|
|
276
312
|
const document = this.readFrom(entry.partition, entry.position, entry.size);
|
|
277
313
|
yield document;
|
|
278
314
|
}
|
|
279
315
|
}
|
|
280
316
|
|
|
317
|
+
/**
|
|
318
|
+
* Open an existing readonly index for reading, without registering it in the secondary indexes write path.
|
|
319
|
+
* Use this for indexes whose files carry a status marker (e.g. `stream-foo.closed.index`).
|
|
320
|
+
*
|
|
321
|
+
* @api
|
|
322
|
+
* @param {string} name The readonly index name (e.g. 'stream-foo.closed').
|
|
323
|
+
* @returns {ReadableIndex}
|
|
324
|
+
* @throws {Error} if the readonly index does not exist.
|
|
325
|
+
*/
|
|
326
|
+
openReadonlyIndex(name) {
|
|
327
|
+
if (name in this.readonlyIndexes) {
|
|
328
|
+
return this.readonlyIndexes[name];
|
|
329
|
+
}
|
|
330
|
+
const indexName = this.storageFile + '.' + name + '.index';
|
|
331
|
+
assert(fs.existsSync(path.join(this.indexDirectory, indexName)), `Index "${name}" does not exist.`);
|
|
332
|
+
const { index } = this.createIndex(indexName, Object.assign({}, this.indexOptions));
|
|
333
|
+
index.open();
|
|
334
|
+
this.readonlyIndexes[name] = index;
|
|
335
|
+
return index;
|
|
336
|
+
}
|
|
337
|
+
|
|
281
338
|
/**
|
|
282
339
|
* Open an existing index.
|
|
283
340
|
*
|
|
@@ -289,6 +346,9 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
289
346
|
* @throws {Error} if the HMAC for the matcher does not match.
|
|
290
347
|
*/
|
|
291
348
|
openIndex(name, matcher) {
|
|
349
|
+
if (name === '_all') {
|
|
350
|
+
return this.index;
|
|
351
|
+
}
|
|
292
352
|
if (name in this.secondaryIndexes) {
|
|
293
353
|
return this.secondaryIndexes[name].index;
|
|
294
354
|
}
|
|
@@ -304,17 +364,79 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
304
364
|
}
|
|
305
365
|
|
|
306
366
|
/**
|
|
307
|
-
*
|
|
367
|
+
* Iterate documents across all partitions in sequenceNumber order using a k-way merge.
|
|
368
|
+
* Opens any closed partition automatically.
|
|
369
|
+
*
|
|
370
|
+
* @protected
|
|
371
|
+
* @param {number} [from=0] The 0-based sequenceNumber to start from (inclusive).
|
|
372
|
+
* @param {number} [until=Number.MAX_SAFE_INTEGER] The 0-based sequenceNumber to read until (inclusive).
|
|
373
|
+
* @returns {Generator<{document: object, sequenceNumber: number, partitionName: string, position: number, size: number, partition: number}>}
|
|
374
|
+
*/
|
|
375
|
+
*iterateDocumentsNoIndex(from = 0, until = Number.MAX_SAFE_INTEGER) {
|
|
376
|
+
const streams = [];
|
|
377
|
+
|
|
378
|
+
this.forEachPartition(partition => {
|
|
379
|
+
if (!partition.isOpen()) {
|
|
380
|
+
partition.open();
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
const found = partition.findDocument(from);
|
|
384
|
+
if (found && found.headerOut.sequenceNumber <= until) {
|
|
385
|
+
const nextPosition = found.headerOut.position + partition.documentWriteSize(found.headerOut.dataSize);
|
|
386
|
+
const reader = partition.readAll(nextPosition, found.headerOut);
|
|
387
|
+
streams.push({ ...found, reader, partition: partition.id, partitionName: partition.name });
|
|
388
|
+
}
|
|
389
|
+
});
|
|
390
|
+
|
|
391
|
+
const items = [];
|
|
392
|
+
kWayMerge(
|
|
393
|
+
streams,
|
|
394
|
+
stream => stream.headerOut.sequenceNumber,
|
|
395
|
+
stream => {
|
|
396
|
+
const next = stream.reader.next();
|
|
397
|
+
if (!next.done && stream.headerOut.sequenceNumber <= until) {
|
|
398
|
+
stream.data = next.value;
|
|
399
|
+
return true;
|
|
400
|
+
}
|
|
401
|
+
return false;
|
|
402
|
+
},
|
|
403
|
+
stream => items.push({
|
|
404
|
+
document: this.serializer.deserialize(stream.data),
|
|
405
|
+
sequenceNumber: stream.headerOut.sequenceNumber,
|
|
406
|
+
partitionName: stream.partitionName,
|
|
407
|
+
position: stream.headerOut.position,
|
|
408
|
+
size: stream.headerOut.dataSize,
|
|
409
|
+
partition: stream.partition,
|
|
410
|
+
})
|
|
411
|
+
);
|
|
412
|
+
|
|
413
|
+
yield* items;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* Helper method to iterate over all documents, invoking a callback for each one.
|
|
418
|
+
* Pass `noIndex = true` to iterate all partitions directly in sequenceNumber order
|
|
419
|
+
* (useful when the global index is unavailable or corrupted).
|
|
420
|
+
* When `noIndex` is false the second callback argument is the raw index `EntryInterface`.
|
|
421
|
+
* When `noIndex` is true the second callback argument has `{ partition, position, size, sequenceNumber, partitionName }`.
|
|
308
422
|
*
|
|
309
423
|
* @protected
|
|
310
|
-
* @param {function(object,
|
|
424
|
+
* @param {function(object, object): void} iterationHandler
|
|
425
|
+
* @param {boolean} [noIndex=false] When true, bypasses the index and iterates partitions directly.
|
|
311
426
|
*/
|
|
312
|
-
forEachDocument(iterationHandler) {
|
|
427
|
+
forEachDocument(iterationHandler, noIndex = false) {
|
|
313
428
|
/* istanbul ignore if */
|
|
314
429
|
if (typeof iterationHandler !== 'function') {
|
|
315
430
|
return;
|
|
316
431
|
}
|
|
317
432
|
|
|
433
|
+
if (noIndex) {
|
|
434
|
+
for (const { document, ...entryInfo } of this.iterateDocumentsNoIndex()) {
|
|
435
|
+
iterationHandler(document, entryInfo);
|
|
436
|
+
}
|
|
437
|
+
return;
|
|
438
|
+
}
|
|
439
|
+
|
|
318
440
|
const entries = this.index.all();
|
|
319
441
|
|
|
320
442
|
for (let entry of entries) {
|
|
@@ -362,5 +484,5 @@ class ReadableStorage extends events.EventEmitter {
|
|
|
362
484
|
|
|
363
485
|
}
|
|
364
486
|
|
|
365
|
-
|
|
366
|
-
|
|
487
|
+
export default ReadableStorage;
|
|
488
|
+
export { matches };
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import WritablePartition from '../Partition/WritablePartition.js';
|
|
4
|
+
import WritableIndex, { Entry as WritableIndexEntry } from '../Index/WritableIndex.js';
|
|
5
|
+
import ReadableStorage from './ReadableStorage.js';
|
|
6
|
+
import { assert, ensureDirectory } from '../util.js';
|
|
7
|
+
import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../metadataUtil.js';
|
|
7
8
|
|
|
8
9
|
const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024;
|
|
9
10
|
|
|
@@ -75,29 +76,137 @@ class WritableStorage extends ReadableStorage {
|
|
|
75
76
|
if (!this.lock()) {
|
|
76
77
|
return true;
|
|
77
78
|
}
|
|
78
|
-
|
|
79
|
+
const result = super.open();
|
|
80
|
+
this.emit('ready');
|
|
81
|
+
return result;
|
|
79
82
|
}
|
|
80
83
|
|
|
81
84
|
/**
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
+
* Helper method to iterate over all writable secondary indexes.
|
|
86
|
+
* Opens each index before calling the callback (passing the previous open status),
|
|
87
|
+
* and closes it afterwards if it was not already open.
|
|
88
|
+
*
|
|
89
|
+
* @protected
|
|
90
|
+
* @param {function(WritableIndex, string, boolean)} iterationHandler Called with (index, name, wasOpen).
|
|
91
|
+
* @param {object} [matchDocument] If supplied, only indexes the document matches on will be iterated.
|
|
85
92
|
*/
|
|
86
|
-
|
|
93
|
+
forEachWritableSecondaryIndex(iterationHandler, matchDocument) {
|
|
94
|
+
this.forEachSecondaryIndex((index, name) => {
|
|
95
|
+
/* istanbul ignore if */
|
|
96
|
+
if (!(index instanceof WritableIndex)) return;
|
|
97
|
+
const wasOpen = index.isOpen();
|
|
98
|
+
if (!wasOpen) index.open();
|
|
99
|
+
iterationHandler(index, name, wasOpen);
|
|
100
|
+
if (!wasOpen) index.close();
|
|
101
|
+
}, matchDocument);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Scan every partition's last document to detect torn writes inline.
|
|
106
|
+
* A document is torn when its expected end exceeds the actual file size:
|
|
107
|
+
* position + documentWriteSize(dataSize) > partition.size
|
|
108
|
+
*
|
|
109
|
+
* @private
|
|
110
|
+
* @returns {{ lastValidSequenceNumber: number, maxPartitionSequenceNumber: number }}
|
|
111
|
+
*/
|
|
112
|
+
findTornWriteBoundary() {
|
|
87
113
|
let lastValidSequenceNumber = Number.MAX_SAFE_INTEGER;
|
|
114
|
+
let maxPartitionSequenceNumber = -1;
|
|
88
115
|
this.forEachPartition(partition => {
|
|
89
116
|
partition.open();
|
|
90
|
-
const
|
|
91
|
-
|
|
92
|
-
|
|
117
|
+
const last = partition.readLast();
|
|
118
|
+
/* istanbul ignore if */
|
|
119
|
+
if (!last) return;
|
|
120
|
+
const { header: { sequenceNumber, dataSize }, position } = last;
|
|
121
|
+
if (position + partition.documentWriteSize(dataSize) > partition.size) {
|
|
122
|
+
// Torn write: the document extends beyond the end of the file.
|
|
123
|
+
lastValidSequenceNumber = Math.min(lastValidSequenceNumber, sequenceNumber);
|
|
124
|
+
} else {
|
|
125
|
+
maxPartitionSequenceNumber = Math.max(maxPartitionSequenceNumber, sequenceNumber);
|
|
93
126
|
}
|
|
94
127
|
});
|
|
128
|
+
return { lastValidSequenceNumber, maxPartitionSequenceNumber };
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Check all partitions for torn writes, physically repair each partition, truncate all indexes
|
|
133
|
+
* to the torn-write boundary, and then reindex to rebuild any missing index entries.
|
|
134
|
+
*
|
|
135
|
+
* A document is torn when the partition file ends before the document's expected end position
|
|
136
|
+
* (i.e. position + documentWriteSize(dataSize) > partition.size). Detected inline in
|
|
137
|
+
* findTornWriteBoundary(), without any checkTornWrite() call.
|
|
138
|
+
*
|
|
139
|
+
* Repair flow:
|
|
140
|
+
* 1. findTornWriteBoundary() reads the last document of every partition and finds the global
|
|
141
|
+
* torn-write boundary (minimum torn sequence number across all partitions).
|
|
142
|
+
* 2. If torn writes were found, truncateAfterSequence() removes all documents at or beyond
|
|
143
|
+
* the boundary from each partition.
|
|
144
|
+
* 3. Truncate all indexes to the torn-write boundary, then reindex to fill any lagging entries.
|
|
145
|
+
* 4. If no torn writes were found but the index is lagging, reindex directly.
|
|
146
|
+
*/
|
|
147
|
+
checkTornWrites() {
|
|
148
|
+
const { lastValidSequenceNumber, maxPartitionSequenceNumber } = this.findTornWriteBoundary();
|
|
149
|
+
|
|
95
150
|
if (lastValidSequenceNumber < Number.MAX_SAFE_INTEGER) {
|
|
96
|
-
|
|
151
|
+
// Phase 2: remove all documents at or beyond the torn-write boundary from each partition.
|
|
152
|
+
this.forEachPartition(partition => {
|
|
153
|
+
partition.open();
|
|
154
|
+
partition.truncateAfterSequence(lastValidSequenceNumber - 1);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
// Truncate all indexes to the torn-write boundary.
|
|
158
|
+
this.index.open();
|
|
159
|
+
this.index.truncate(lastValidSequenceNumber);
|
|
160
|
+
/* istanbul ignore next */
|
|
161
|
+
this.forEachWritableSecondaryIndex(index => {
|
|
162
|
+
index.truncate(index.find(lastValidSequenceNumber));
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
// Reindex to fill in any missing complete-document entries.
|
|
166
|
+
this.reindex(this.index.length);
|
|
167
|
+
} else if (maxPartitionSequenceNumber >= 0 && maxPartitionSequenceNumber + 1 > this.index.length) {
|
|
168
|
+
// No torn writes, but the index is lagging — repair it.
|
|
169
|
+
this.reindex(this.index.length);
|
|
97
170
|
}
|
|
171
|
+
|
|
98
172
|
this.forEachPartition(partition => partition.close());
|
|
99
173
|
}
|
|
100
174
|
|
|
175
|
+
/**
|
|
176
|
+
* Rebuild the primary index and all loaded secondary indexes starting from the given sequence
|
|
177
|
+
* number by scanning the partition data directly.
|
|
178
|
+
* This is the building block for both auto-repair (invoked automatically when the primary
|
|
179
|
+
* index is found to be lagging in checkTornWrites()) and for user-driven re-indexing after
|
|
180
|
+
* index corruption.
|
|
181
|
+
*
|
|
182
|
+
* @api
|
|
183
|
+
* @param {number} [fromSequenceNumber=0] The number of primary index entries to keep intact.
|
|
184
|
+
* All index entries beyond this position will be removed and rebuilt from partition data.
|
|
185
|
+
* Defaults to 0, which rebuilds all indexes from scratch.
|
|
186
|
+
*/
|
|
187
|
+
reindex(fromSequenceNumber = 0) {
|
|
188
|
+
this.index.truncate(fromSequenceNumber);
|
|
189
|
+
|
|
190
|
+
// Truncate all loaded secondary indexes to match the new primary length.
|
|
191
|
+
this.forEachWritableSecondaryIndex(index => {
|
|
192
|
+
// find(0) returns 0, so truncate(0) will remove all entries when fromSequenceNumber===0
|
|
193
|
+
index.truncate(fromSequenceNumber === 0 ? 0 : index.find(fromSequenceNumber));
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
// Scan partitions in sequence-number order and rebuild index entries.
|
|
197
|
+
// iterateDocumentsNoIndex opens any closed partitions automatically.
|
|
198
|
+
for (const { document, partition, position, size } of this.iterateDocumentsNoIndex(fromSequenceNumber, Number.MAX_SAFE_INTEGER)) {
|
|
199
|
+
const newEntry = new WritableIndexEntry(this.index.length + 1, position, size, partition);
|
|
200
|
+
this.index.add(newEntry);
|
|
201
|
+
|
|
202
|
+
this.forEachWritableSecondaryIndex((secIndex) => {
|
|
203
|
+
secIndex.add(newEntry);
|
|
204
|
+
}, document);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
this.flush();
|
|
208
|
+
}
|
|
209
|
+
|
|
101
210
|
/**
|
|
102
211
|
* Attempt to lock this storage by means of a lock directory.
|
|
103
212
|
* @returns {boolean} True if the lock was created or false if the lock is already in place.
|
|
@@ -167,7 +276,7 @@ class WritableStorage extends ReadableStorage {
|
|
|
167
276
|
throw new Error('Corrupted index, needs to be rebuilt!');
|
|
168
277
|
}*/
|
|
169
278
|
|
|
170
|
-
const entry = new
|
|
279
|
+
const entry = new WritableIndexEntry(this.index.length + 1, position, size, partitionId);
|
|
171
280
|
this.index.add(entry, (indexPosition) => {
|
|
172
281
|
this.emit('wrote', document, entry, indexPosition);
|
|
173
282
|
/* istanbul ignore if */
|
|
@@ -178,6 +287,19 @@ class WritableStorage extends ReadableStorage {
|
|
|
178
287
|
return entry;
|
|
179
288
|
}
|
|
180
289
|
|
|
290
|
+
/**
|
|
291
|
+
* Register a handler that is called before a document is written to storage.
|
|
292
|
+
* The handler receives the document and the partition metadata and may throw to abort the write.
|
|
293
|
+
* Multiple handlers can be registered; all run on every write in registration order.
|
|
294
|
+
* Equivalent to `storage.on('preCommit', hook)`.
|
|
295
|
+
*
|
|
296
|
+
* @api
|
|
297
|
+
* @param {function(object, object): void} hook A function receiving (document, partitionMetadata).
|
|
298
|
+
*/
|
|
299
|
+
preCommit(hook) {
|
|
300
|
+
this.on('preCommit', hook);
|
|
301
|
+
}
|
|
302
|
+
|
|
181
303
|
/**
|
|
182
304
|
* Get a partition either by name or by id.
|
|
183
305
|
* If a partition with the given name does not exist, a new one will be created.
|
|
@@ -190,10 +312,14 @@ class WritableStorage extends ReadableStorage {
|
|
|
190
312
|
*/
|
|
191
313
|
getPartition(partitionIdentifier) {
|
|
192
314
|
if (typeof partitionIdentifier === 'string') {
|
|
315
|
+
const partitionShortName = partitionIdentifier;
|
|
193
316
|
const partitionName = this.storageFile + (partitionIdentifier.length ? '.' + partitionIdentifier : '');
|
|
194
317
|
partitionIdentifier = WritablePartition.idFor(partitionName);
|
|
195
318
|
if (!this.partitions[partitionIdentifier]) {
|
|
196
|
-
|
|
319
|
+
const partitionConfig = typeof this.partitionConfig.metadata === 'function'
|
|
320
|
+
? { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) }
|
|
321
|
+
: this.partitionConfig;
|
|
322
|
+
this.partitions[partitionIdentifier] = this.createPartition(partitionName, partitionConfig);
|
|
197
323
|
this.emit('partition-created', partitionIdentifier);
|
|
198
324
|
}
|
|
199
325
|
this.partitions[partitionIdentifier].open();
|
|
@@ -214,6 +340,9 @@ class WritableStorage extends ReadableStorage {
|
|
|
214
340
|
|
|
215
341
|
const partitionName = this.partitioner(document, this.index.length + 1);
|
|
216
342
|
const partition = this.getPartition(partitionName);
|
|
343
|
+
if (this.listenerCount('preCommit') > 0) {
|
|
344
|
+
this.emit('preCommit', document, partition.metadata);
|
|
345
|
+
}
|
|
217
346
|
const position = partition.write(data, this.length, callback);
|
|
218
347
|
|
|
219
348
|
assert(position !== false, 'Error writing document.');
|
|
@@ -241,6 +370,9 @@ class WritableStorage extends ReadableStorage {
|
|
|
241
370
|
* @throws {Error} if the index doesn't exist yet and no matcher was specified.
|
|
242
371
|
*/
|
|
243
372
|
ensureIndex(name, matcher) {
|
|
373
|
+
if (name === '_all') {
|
|
374
|
+
return this.index;
|
|
375
|
+
}
|
|
244
376
|
if (name in this.secondaryIndexes) {
|
|
245
377
|
return this.secondaryIndexes[name].index;
|
|
246
378
|
}
|
|
@@ -307,6 +439,10 @@ class WritableStorage extends ReadableStorage {
|
|
|
307
439
|
/**
|
|
308
440
|
* Truncate all partitions after the given (global) sequence number.
|
|
309
441
|
*
|
|
442
|
+
* Assumes the primary index is fully consistent with the partition data. Looks up the first
|
|
443
|
+
* index entry after `after` for each affected partition and truncates the partition file
|
|
444
|
+
* at that entry's byte position.
|
|
445
|
+
*
|
|
310
446
|
* @private
|
|
311
447
|
* @param {number} after The document sequence number to truncate after.
|
|
312
448
|
*/
|
|
@@ -340,26 +476,34 @@ class WritableStorage extends ReadableStorage {
|
|
|
340
476
|
if (!this.index.isOpen()) {
|
|
341
477
|
this.index.open();
|
|
342
478
|
}
|
|
479
|
+
if (after < 0) {
|
|
480
|
+
after += this.index.length;
|
|
481
|
+
}
|
|
343
482
|
|
|
344
483
|
this.truncatePartitions(after);
|
|
345
484
|
|
|
346
485
|
this.index.truncate(after);
|
|
347
|
-
this.
|
|
348
|
-
if (!(index instanceof WritableIndex)) {
|
|
349
|
-
return;
|
|
350
|
-
}
|
|
351
|
-
let closeIndex = false;
|
|
352
|
-
if (!index.isOpen()) {
|
|
353
|
-
index.open();
|
|
354
|
-
closeIndex = true;
|
|
355
|
-
}
|
|
486
|
+
this.forEachWritableSecondaryIndex(index => {
|
|
356
487
|
index.truncate(index.find(after));
|
|
357
|
-
if (closeIndex) {
|
|
358
|
-
index.close();
|
|
359
|
-
}
|
|
360
488
|
});
|
|
361
489
|
}
|
|
362
490
|
|
|
491
|
+
/**
|
|
492
|
+
* @inheritDoc
|
|
493
|
+
* Open an existing secondary index and repair any stale entries beyond the current primary
|
|
494
|
+
* index length. Stale entries can be present when checkTornWrites() truncated the primary
|
|
495
|
+
* index before this secondary index was loaded into memory.
|
|
496
|
+
*/
|
|
497
|
+
openIndex(name, matcher) {
|
|
498
|
+
const index = super.openIndex(name, matcher);
|
|
499
|
+
const lastEntry = index.lastEntry;
|
|
500
|
+
if (lastEntry !== false && lastEntry.number > this.index.length) {
|
|
501
|
+
// Secondary index is ahead of primary: truncate stale entries.
|
|
502
|
+
index.truncate(index.find(this.index.length));
|
|
503
|
+
}
|
|
504
|
+
return index;
|
|
505
|
+
}
|
|
506
|
+
|
|
363
507
|
/**
|
|
364
508
|
* @protected
|
|
365
509
|
* @param {string} name
|
|
@@ -396,7 +540,5 @@ class WritableStorage extends ReadableStorage {
|
|
|
396
540
|
|
|
397
541
|
}
|
|
398
542
|
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
module.exports.LOCK_THROW = LOCK_THROW;
|
|
402
|
-
module.exports.LOCK_RECLAIM = LOCK_RECLAIM;
|
|
543
|
+
export default WritableStorage;
|
|
544
|
+
export { StorageLockedError, LOCK_THROW, LOCK_RECLAIM };
|
package/src/Storage.js
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
import WritableStorage, { StorageLockedError, LOCK_THROW, LOCK_RECLAIM } from './Storage/WritableStorage.js';
|
|
2
|
+
import ReadOnlyStorage from './Storage/ReadOnlyStorage.js';
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
WritableStorage.ReadOnly = ReadOnlyStorage;
|
|
5
|
+
WritableStorage.StorageLockedError = StorageLockedError;
|
|
6
|
+
WritableStorage.LOCK_THROW = LOCK_THROW;
|
|
7
|
+
WritableStorage.LOCK_RECLAIM = LOCK_RECLAIM;
|
|
8
|
+
|
|
9
|
+
export default WritableStorage;
|
|
10
|
+
export { ReadOnlyStorage as ReadOnly, StorageLockedError, LOCK_THROW, LOCK_RECLAIM };
|
package/src/Watcher.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import events from 'events';
|
|
4
|
+
import { assert } from './util.js';
|
|
5
5
|
|
|
6
6
|
/** @type {Map<string, DirectoryWatcher>} */
|
|
7
7
|
const directoryWatchers = new Map();
|
|
@@ -28,6 +28,7 @@ class DirectoryWatcher extends events.EventEmitter {
|
|
|
28
28
|
assert(fs.existsSync(directory), `Can not watch a non-existing directory "${directory}".`);
|
|
29
29
|
assert(fs.statSync(directory).isDirectory(), `Can only watch directories, but "${directory}" is none.`);
|
|
30
30
|
super();
|
|
31
|
+
this.setMaxListeners(1000);
|
|
31
32
|
directoryWatchers.set(directory, this);
|
|
32
33
|
this.directory = directory;
|
|
33
34
|
this.watcher = fs.watch(directory, Object.assign({ persistent: false }, options), this.emit.bind(this));
|
|
@@ -146,4 +147,4 @@ class Watcher {
|
|
|
146
147
|
|
|
147
148
|
}
|
|
148
149
|
|
|
149
|
-
|
|
150
|
+
export default Watcher;
|
package/src/WatchesFile.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
import Watcher from './Watcher.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* A mixin that provides a file watcher for this.fileName which triggers a method `onChange` on the class, that needs to be implemented.
|
|
@@ -34,12 +34,13 @@ const WatchesFile = Base => class extends Base {
|
|
|
34
34
|
* @returns {boolean}
|
|
35
35
|
*/
|
|
36
36
|
open() {
|
|
37
|
-
if (
|
|
38
|
-
|
|
37
|
+
if (super.open()) {
|
|
38
|
+
if (!this.watcher) {
|
|
39
|
+
this.watchFile();
|
|
40
|
+
}
|
|
41
|
+
return true;
|
|
39
42
|
}
|
|
40
|
-
|
|
41
|
-
this.watchFile();
|
|
42
|
-
return super.open();
|
|
43
|
+
return false;
|
|
43
44
|
}
|
|
44
45
|
|
|
45
46
|
/**
|
|
@@ -52,4 +53,4 @@ const WatchesFile = Base => class extends Base {
|
|
|
52
53
|
|
|
53
54
|
};
|
|
54
55
|
|
|
55
|
-
|
|
56
|
+
export default WatchesFile;
|