event-storage 0.8.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,10 @@
1
- const fs = require('fs');
2
- const path = require('path');
3
- const events = require('events');
4
- const Partition = require('../Partition');
5
- const Index = require('../Index');
6
- const { assert, createHmac, matches, wrapAndCheck, buildMetadataForMatcher } = require('../util');
1
+ import fs from 'fs';
2
+ import path from 'path';
3
+ import events from 'events';
4
+ import Partition, { ReadOnly as ReadOnlyPartition } from '../Partition.js';
5
+ import Index, { ReadOnly as ReadOnlyIndex } from '../Index.js';
6
+ import { assert, wrapAndCheck, kWayMerge } from '../util.js';
7
+ import { createHmac, matches, buildMetadataForMatcher } from '../metadataUtil.js';
7
8
 
8
9
  const DEFAULT_READ_BUFFER_SIZE = 4 * 1024;
9
10
 
@@ -66,8 +67,8 @@ class ReadableStorage extends events.EventEmitter {
66
67
 
67
68
  this.dataDirectory = path.resolve(config.dataDirectory);
68
69
 
69
- this.initializeIndexes(config);
70
70
  this.scanPartitions(config);
71
+ this.initializeIndexes(config);
71
72
  }
72
73
 
73
74
  /**
@@ -78,7 +79,7 @@ class ReadableStorage extends events.EventEmitter {
78
79
  */
79
80
  createIndex(name, options = {}) {
80
81
  /** @type ReadableIndex */
81
- const index = new Index.ReadOnly(name, options);
82
+ const index = new ReadOnlyIndex(name, options);
82
83
  return { index };
83
84
  }
84
85
 
@@ -89,7 +90,7 @@ class ReadableStorage extends events.EventEmitter {
89
90
  * @returns {ReadablePartition}
90
91
  */
91
92
  createPartition(name, options = {}) {
92
- return new Partition.ReadOnly(name, options);
93
+ return new ReadOnlyPartition(name, options);
93
94
  }
94
95
 
95
96
  /**
@@ -109,6 +110,7 @@ class ReadableStorage extends events.EventEmitter {
109
110
  const { index } = this.createIndex(config.indexFile, this.indexOptions);
110
111
  this.index = index;
111
112
  this.secondaryIndexes = {};
113
+ this.readonlyIndexes = {};
112
114
  }
113
115
 
114
116
  /**
@@ -172,6 +174,9 @@ class ReadableStorage extends events.EventEmitter {
172
174
  close() {
173
175
  this.index.close();
174
176
  this.forEachSecondaryIndex(index => index.close());
177
+ for (let index of Object.values(this.readonlyIndexes)) {
178
+ index.close();
179
+ }
175
180
  this.forEachPartition(partition => partition.close());
176
181
  this.emit('closed');
177
182
  }
@@ -193,6 +198,19 @@ class ReadableStorage extends events.EventEmitter {
193
198
  return this.partitions[partitionIdentifier];
194
199
  }
195
200
 
201
+ /**
202
+ * Register a handler that is called before a document is read from a partition.
203
+ * The handler receives the position and the partition metadata and may throw to abort the read.
204
+ * Multiple handlers can be registered; all run on every read in registration order.
205
+ * Equivalent to `storage.on('preRead', hook)`.
206
+ *
207
+ * @api
208
+ * @param {function(number, object): void} hook A function receiving (position, partitionMetadata).
209
+ */
210
+ preRead(hook) {
211
+ this.on('preRead', hook);
212
+ }
213
+
196
214
  /**
197
215
  * @protected
198
216
  * @param {number} partitionId The partition to read from.
@@ -203,6 +221,9 @@ class ReadableStorage extends events.EventEmitter {
203
221
  */
204
222
  readFrom(partitionId, position, size) {
205
223
  const partition = this.getPartition(partitionId);
224
+ if (this.listenerCount('preRead') > 0) {
225
+ this.emit('preRead', position, partition.metadata);
226
+ }
206
227
  const data = partition.readFrom(position, size);
207
228
  return this.serializer.deserialize(data);
208
229
  }
@@ -237,21 +258,25 @@ class ReadableStorage extends events.EventEmitter {
237
258
  * @api
238
259
  * @param {number} from The 1-based document number (inclusive) to start reading from.
239
260
  * @param {number} [until] The 1-based document number (inclusive) to read until. Defaults to index.length.
240
- * @param {ReadableIndex} [index] The index to use for finding the documents in the range.
261
+ * @param {ReadableIndex|false} [index] The index to use for finding the documents in the range.
262
+ * Pass `false` to skip the global index and iterate all partitions directly in sequenceNumber order
263
+ * (useful when the global index is unavailable or corrupted).
241
264
  * @returns {Generator<object>} A generator that will read each document in the range one by one.
242
265
  */
243
266
  *readRange(from, until = -1, index = null) {
244
- index = index || this.index;
245
- index.open();
267
+ const lengthSource = index || this.index;
268
+ if (!lengthSource.isOpen()) {
269
+ lengthSource.open();
270
+ }
246
271
 
247
- const readFrom = wrapAndCheck(from, index.length);
248
- const readUntil = wrapAndCheck(until, index.length);
272
+ const readFrom = wrapAndCheck(from, lengthSource.length);
273
+ const readUntil = wrapAndCheck(until, lengthSource.length);
249
274
  assert(readFrom > 0 && readUntil > 0, `Range scan error for range ${from} - ${until}.`);
250
275
 
251
276
  if (readFrom > readUntil) {
252
277
  const batchSize = 10;
253
278
  let batchUntil = readFrom;
254
- while (batchUntil > readUntil) {
279
+ while (batchUntil >= readUntil) {
255
280
  const batchFrom = Math.max(readUntil, batchUntil - batchSize);
256
281
  yield* reverse(this.iterateRange(batchFrom, batchUntil, index));
257
282
  batchUntil = batchFrom - 1;
@@ -264,20 +289,52 @@ class ReadableStorage extends events.EventEmitter {
264
289
 
265
290
  /**
266
291
  * Iterate all documents in this storage in range from to until inside the index.
292
+ * If index is false, iterates all partitions directly in sequenceNumber order.
267
293
  * @private
268
294
  * @param {number} from
269
295
  * @param {number} until
270
- * @param {ReadableIndex} index
296
+ * @param {ReadableIndex|false|null} index
271
297
  * @returns {Generator<object>}
272
298
  */
273
299
  *iterateRange(from, until, index) {
274
- const entries = index.range(from, until);
300
+ if (index === false) {
301
+ // Explicitly disabled index: iterate all partitions and merge by sequenceNumber.
302
+ // Document header sequenceNumber is 0-based; from/until are 1-based index positions.
303
+ for (const entry of this.iterateDocumentsNoIndex(from - 1, until - 1)) {
304
+ yield entry.document;
305
+ }
306
+ return;
307
+ }
308
+
309
+ const idx = index || this.index;
310
+ const entries = idx.range(from, until);
275
311
  for (let entry of entries) {
276
312
  const document = this.readFrom(entry.partition, entry.position, entry.size);
277
313
  yield document;
278
314
  }
279
315
  }
280
316
 
317
+ /**
318
+ * Open an existing readonly index for reading, without registering it in the secondary indexes write path.
319
+ * Use this for indexes whose files carry a status marker (e.g. `stream-foo.closed.index`).
320
+ *
321
+ * @api
322
+ * @param {string} name The readonly index name (e.g. 'stream-foo.closed').
323
+ * @returns {ReadableIndex}
324
+ * @throws {Error} if the readonly index does not exist.
325
+ */
326
+ openReadonlyIndex(name) {
327
+ if (name in this.readonlyIndexes) {
328
+ return this.readonlyIndexes[name];
329
+ }
330
+ const indexName = this.storageFile + '.' + name + '.index';
331
+ assert(fs.existsSync(path.join(this.indexDirectory, indexName)), `Index "${name}" does not exist.`);
332
+ const { index } = this.createIndex(indexName, Object.assign({}, this.indexOptions));
333
+ index.open();
334
+ this.readonlyIndexes[name] = index;
335
+ return index;
336
+ }
337
+
281
338
  /**
282
339
  * Open an existing index.
283
340
  *
@@ -289,6 +346,9 @@ class ReadableStorage extends events.EventEmitter {
289
346
  * @throws {Error} if the HMAC for the matcher does not match.
290
347
  */
291
348
  openIndex(name, matcher) {
349
+ if (name === '_all') {
350
+ return this.index;
351
+ }
292
352
  if (name in this.secondaryIndexes) {
293
353
  return this.secondaryIndexes[name].index;
294
354
  }
@@ -304,17 +364,79 @@ class ReadableStorage extends events.EventEmitter {
304
364
  }
305
365
 
306
366
  /**
307
- * Helper method to iterate over all documents.
367
+ * Iterate documents across all partitions in sequenceNumber order using a k-way merge.
368
+ * Opens any closed partition automatically.
369
+ *
370
+ * @protected
371
+ * @param {number} [from=0] The 0-based sequenceNumber to start from (inclusive).
372
+ * @param {number} [until=Number.MAX_SAFE_INTEGER] The 0-based sequenceNumber to read until (inclusive).
373
+ * @returns {Generator<{document: object, sequenceNumber: number, partitionName: string, position: number, size: number, partition: number}>}
374
+ */
375
+ *iterateDocumentsNoIndex(from = 0, until = Number.MAX_SAFE_INTEGER) {
376
+ const streams = [];
377
+
378
+ this.forEachPartition(partition => {
379
+ if (!partition.isOpen()) {
380
+ partition.open();
381
+ }
382
+
383
+ const found = partition.findDocument(from);
384
+ if (found && found.headerOut.sequenceNumber <= until) {
385
+ const nextPosition = found.headerOut.position + partition.documentWriteSize(found.headerOut.dataSize);
386
+ const reader = partition.readAll(nextPosition, found.headerOut);
387
+ streams.push({ ...found, reader, partition: partition.id, partitionName: partition.name });
388
+ }
389
+ });
390
+
391
+ const items = [];
392
+ kWayMerge(
393
+ streams,
394
+ stream => stream.headerOut.sequenceNumber,
395
+ stream => {
396
+ const next = stream.reader.next();
397
+ if (!next.done && stream.headerOut.sequenceNumber <= until) {
398
+ stream.data = next.value;
399
+ return true;
400
+ }
401
+ return false;
402
+ },
403
+ stream => items.push({
404
+ document: this.serializer.deserialize(stream.data),
405
+ sequenceNumber: stream.headerOut.sequenceNumber,
406
+ partitionName: stream.partitionName,
407
+ position: stream.headerOut.position,
408
+ size: stream.headerOut.dataSize,
409
+ partition: stream.partition,
410
+ })
411
+ );
412
+
413
+ yield* items;
414
+ }
415
+
416
+ /**
417
+ * Helper method to iterate over all documents, invoking a callback for each one.
418
+ * Pass `noIndex = true` to iterate all partitions directly in sequenceNumber order
419
+ * (useful when the global index is unavailable or corrupted).
420
+ * When `noIndex` is false the second callback argument is the raw index `EntryInterface`.
421
+ * When `noIndex` is true the second callback argument has `{ partition, position, size, sequenceNumber, partitionName }`.
308
422
  *
309
423
  * @protected
310
- * @param {function(object, EntryInterface)} iterationHandler
424
+ * @param {function(object, object): void} iterationHandler
425
+ * @param {boolean} [noIndex=false] When true, bypasses the index and iterates partitions directly.
311
426
  */
312
- forEachDocument(iterationHandler) {
427
+ forEachDocument(iterationHandler, noIndex = false) {
313
428
  /* istanbul ignore if */
314
429
  if (typeof iterationHandler !== 'function') {
315
430
  return;
316
431
  }
317
432
 
433
+ if (noIndex) {
434
+ for (const { document, ...entryInfo } of this.iterateDocumentsNoIndex()) {
435
+ iterationHandler(document, entryInfo);
436
+ }
437
+ return;
438
+ }
439
+
318
440
  const entries = this.index.all();
319
441
 
320
442
  for (let entry of entries) {
@@ -362,5 +484,5 @@ class ReadableStorage extends events.EventEmitter {
362
484
 
363
485
  }
364
486
 
365
- module.exports = ReadableStorage;
366
- module.exports.matches = matches;
487
+ export default ReadableStorage;
488
+ export { matches };
@@ -1,9 +1,10 @@
1
- const fs = require('fs');
2
- const path = require('path');
3
- const WritablePartition = require('../Partition/WritablePartition');
4
- const WritableIndex = require('../Index/WritableIndex');
5
- const ReadableStorage = require('./ReadableStorage');
6
- const { assert, matches, buildMetadataForMatcher, buildMatcherFromMetadata, ensureDirectory } = require('../util');
1
+ import fs from 'fs';
2
+ import path from 'path';
3
+ import WritablePartition from '../Partition/WritablePartition.js';
4
+ import WritableIndex, { Entry as WritableIndexEntry } from '../Index/WritableIndex.js';
5
+ import ReadableStorage from './ReadableStorage.js';
6
+ import { assert, ensureDirectory } from '../util.js';
7
+ import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../metadataUtil.js';
7
8
 
8
9
  const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024;
9
10
 
@@ -75,29 +76,137 @@ class WritableStorage extends ReadableStorage {
75
76
  if (!this.lock()) {
76
77
  return true;
77
78
  }
78
- return super.open();
79
+ const result = super.open();
80
+ this.emit('ready');
81
+ return result;
79
82
  }
80
83
 
81
84
  /**
82
- * Check all partitions torn writes and truncate the storage to the position before the first torn write.
83
- * This might delete correctly written events in partitions, if their sequence number is higher than the
84
- * torn write in another partition.
85
+ * Helper method to iterate over all writable secondary indexes.
86
+ * Opens each index before calling the callback (passing the previous open status),
87
+ * and closes it afterwards if it was not already open.
88
+ *
89
+ * @protected
90
+ * @param {function(WritableIndex, string, boolean)} iterationHandler Called with (index, name, wasOpen).
91
+ * @param {object} [matchDocument] If supplied, only indexes the document matches on will be iterated.
85
92
  */
86
- checkTornWrites() {
93
+ forEachWritableSecondaryIndex(iterationHandler, matchDocument) {
94
+ this.forEachSecondaryIndex((index, name) => {
95
+ /* istanbul ignore if */
96
+ if (!(index instanceof WritableIndex)) return;
97
+ const wasOpen = index.isOpen();
98
+ if (!wasOpen) index.open();
99
+ iterationHandler(index, name, wasOpen);
100
+ if (!wasOpen) index.close();
101
+ }, matchDocument);
102
+ }
103
+
104
+ /**
105
+ * Scan every partition's last document to detect torn writes inline.
106
+ * A document is torn when its expected end exceeds the actual file size:
107
+ * position + documentWriteSize(dataSize) > partition.size
108
+ *
109
+ * @private
110
+ * @returns {{ lastValidSequenceNumber: number, maxPartitionSequenceNumber: number }}
111
+ */
112
+ findTornWriteBoundary() {
87
113
  let lastValidSequenceNumber = Number.MAX_SAFE_INTEGER;
114
+ let maxPartitionSequenceNumber = -1;
88
115
  this.forEachPartition(partition => {
89
116
  partition.open();
90
- const tornSequenceNumber = partition.checkTornWrite();
91
- if (tornSequenceNumber >= 0) {
92
- lastValidSequenceNumber = Math.min(lastValidSequenceNumber, tornSequenceNumber);
117
+ const last = partition.readLast();
118
+ /* istanbul ignore if */
119
+ if (!last) return;
120
+ const { header: { sequenceNumber, dataSize }, position } = last;
121
+ if (position + partition.documentWriteSize(dataSize) > partition.size) {
122
+ // Torn write: the document extends beyond the end of the file.
123
+ lastValidSequenceNumber = Math.min(lastValidSequenceNumber, sequenceNumber);
124
+ } else {
125
+ maxPartitionSequenceNumber = Math.max(maxPartitionSequenceNumber, sequenceNumber);
93
126
  }
94
127
  });
128
+ return { lastValidSequenceNumber, maxPartitionSequenceNumber };
129
+ }
130
+
131
+ /**
132
+ * Check all partitions for torn writes, physically repair each partition, truncate all indexes
133
+ * to the torn-write boundary, and then reindex to rebuild any missing index entries.
134
+ *
135
+ * A document is torn when the partition file ends before the document's expected end position
136
+ * (i.e. position + documentWriteSize(dataSize) > partition.size). Detected inline in
137
+ * findTornWriteBoundary(), without any checkTornWrite() call.
138
+ *
139
+ * Repair flow:
140
+ * 1. findTornWriteBoundary() reads the last document of every partition and finds the global
141
+ * torn-write boundary (minimum torn sequence number across all partitions).
142
+ * 2. If torn writes were found, truncateAfterSequence() removes all documents at or beyond
143
+ * the boundary from each partition.
144
+ * 3. Truncate all indexes to the torn-write boundary, then reindex to fill any lagging entries.
145
+ * 4. If no torn writes were found but the index is lagging, reindex directly.
146
+ */
147
+ checkTornWrites() {
148
+ const { lastValidSequenceNumber, maxPartitionSequenceNumber } = this.findTornWriteBoundary();
149
+
95
150
  if (lastValidSequenceNumber < Number.MAX_SAFE_INTEGER) {
96
- this.truncate(lastValidSequenceNumber);
151
+ // Phase 2: remove all documents at or beyond the torn-write boundary from each partition.
152
+ this.forEachPartition(partition => {
153
+ partition.open();
154
+ partition.truncateAfterSequence(lastValidSequenceNumber - 1);
155
+ });
156
+
157
+ // Truncate all indexes to the torn-write boundary.
158
+ this.index.open();
159
+ this.index.truncate(lastValidSequenceNumber);
160
+ /* istanbul ignore next */
161
+ this.forEachWritableSecondaryIndex(index => {
162
+ index.truncate(index.find(lastValidSequenceNumber));
163
+ });
164
+
165
+ // Reindex to fill in any missing complete-document entries.
166
+ this.reindex(this.index.length);
167
+ } else if (maxPartitionSequenceNumber >= 0 && maxPartitionSequenceNumber + 1 > this.index.length) {
168
+ // No torn writes, but the index is lagging — repair it.
169
+ this.reindex(this.index.length);
97
170
  }
171
+
98
172
  this.forEachPartition(partition => partition.close());
99
173
  }
100
174
 
175
+ /**
176
+ * Rebuild the primary index and all loaded secondary indexes starting from the given sequence
177
+ * number by scanning the partition data directly.
178
+ * This is the building block for both auto-repair (invoked automatically when the primary
179
+ * index is found to be lagging in checkTornWrites()) and for user-driven re-indexing after
180
+ * index corruption.
181
+ *
182
+ * @api
183
+ * @param {number} [fromSequenceNumber=0] The number of primary index entries to keep intact.
184
+ * All index entries beyond this position will be removed and rebuilt from partition data.
185
+ * Defaults to 0, which rebuilds all indexes from scratch.
186
+ */
187
+ reindex(fromSequenceNumber = 0) {
188
+ this.index.truncate(fromSequenceNumber);
189
+
190
+ // Truncate all loaded secondary indexes to match the new primary length.
191
+ this.forEachWritableSecondaryIndex(index => {
192
+ // find(0) returns 0, so truncate(0) will remove all entries when fromSequenceNumber===0
193
+ index.truncate(fromSequenceNumber === 0 ? 0 : index.find(fromSequenceNumber));
194
+ });
195
+
196
+ // Scan partitions in sequence-number order and rebuild index entries.
197
+ // iterateDocumentsNoIndex opens any closed partitions automatically.
198
+ for (const { document, partition, position, size } of this.iterateDocumentsNoIndex(fromSequenceNumber, Number.MAX_SAFE_INTEGER)) {
199
+ const newEntry = new WritableIndexEntry(this.index.length + 1, position, size, partition);
200
+ this.index.add(newEntry);
201
+
202
+ this.forEachWritableSecondaryIndex((secIndex) => {
203
+ secIndex.add(newEntry);
204
+ }, document);
205
+ }
206
+
207
+ this.flush();
208
+ }
209
+
101
210
  /**
102
211
  * Attempt to lock this storage by means of a lock directory.
103
212
  * @returns {boolean} True if the lock was created or false if the lock is already in place.
@@ -167,7 +276,7 @@ class WritableStorage extends ReadableStorage {
167
276
  throw new Error('Corrupted index, needs to be rebuilt!');
168
277
  }*/
169
278
 
170
- const entry = new WritableIndex.Entry(this.index.length + 1, position, size, partitionId);
279
+ const entry = new WritableIndexEntry(this.index.length + 1, position, size, partitionId);
171
280
  this.index.add(entry, (indexPosition) => {
172
281
  this.emit('wrote', document, entry, indexPosition);
173
282
  /* istanbul ignore if */
@@ -178,6 +287,19 @@ class WritableStorage extends ReadableStorage {
178
287
  return entry;
179
288
  }
180
289
 
290
+ /**
291
+ * Register a handler that is called before a document is written to storage.
292
+ * The handler receives the document and the partition metadata and may throw to abort the write.
293
+ * Multiple handlers can be registered; all run on every write in registration order.
294
+ * Equivalent to `storage.on('preCommit', hook)`.
295
+ *
296
+ * @api
297
+ * @param {function(object, object): void} hook A function receiving (document, partitionMetadata).
298
+ */
299
+ preCommit(hook) {
300
+ this.on('preCommit', hook);
301
+ }
302
+
181
303
  /**
182
304
  * Get a partition either by name or by id.
183
305
  * If a partition with the given name does not exist, a new one will be created.
@@ -190,10 +312,14 @@ class WritableStorage extends ReadableStorage {
190
312
  */
191
313
  getPartition(partitionIdentifier) {
192
314
  if (typeof partitionIdentifier === 'string') {
315
+ const partitionShortName = partitionIdentifier;
193
316
  const partitionName = this.storageFile + (partitionIdentifier.length ? '.' + partitionIdentifier : '');
194
317
  partitionIdentifier = WritablePartition.idFor(partitionName);
195
318
  if (!this.partitions[partitionIdentifier]) {
196
- this.partitions[partitionIdentifier] = this.createPartition(partitionName, this.partitionConfig);
319
+ const partitionConfig = typeof this.partitionConfig.metadata === 'function'
320
+ ? { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) }
321
+ : this.partitionConfig;
322
+ this.partitions[partitionIdentifier] = this.createPartition(partitionName, partitionConfig);
197
323
  this.emit('partition-created', partitionIdentifier);
198
324
  }
199
325
  this.partitions[partitionIdentifier].open();
@@ -214,6 +340,9 @@ class WritableStorage extends ReadableStorage {
214
340
 
215
341
  const partitionName = this.partitioner(document, this.index.length + 1);
216
342
  const partition = this.getPartition(partitionName);
343
+ if (this.listenerCount('preCommit') > 0) {
344
+ this.emit('preCommit', document, partition.metadata);
345
+ }
217
346
  const position = partition.write(data, this.length, callback);
218
347
 
219
348
  assert(position !== false, 'Error writing document.');
@@ -241,6 +370,9 @@ class WritableStorage extends ReadableStorage {
241
370
  * @throws {Error} if the index doesn't exist yet and no matcher was specified.
242
371
  */
243
372
  ensureIndex(name, matcher) {
373
+ if (name === '_all') {
374
+ return this.index;
375
+ }
244
376
  if (name in this.secondaryIndexes) {
245
377
  return this.secondaryIndexes[name].index;
246
378
  }
@@ -307,6 +439,10 @@ class WritableStorage extends ReadableStorage {
307
439
  /**
308
440
  * Truncate all partitions after the given (global) sequence number.
309
441
  *
442
+ * Assumes the primary index is fully consistent with the partition data. Looks up the first
443
+ * index entry after `after` for each affected partition and truncates the partition file
444
+ * at that entry's byte position.
445
+ *
310
446
  * @private
311
447
  * @param {number} after The document sequence number to truncate after.
312
448
  */
@@ -340,26 +476,34 @@ class WritableStorage extends ReadableStorage {
340
476
  if (!this.index.isOpen()) {
341
477
  this.index.open();
342
478
  }
479
+ if (after < 0) {
480
+ after += this.index.length;
481
+ }
343
482
 
344
483
  this.truncatePartitions(after);
345
484
 
346
485
  this.index.truncate(after);
347
- this.forEachSecondaryIndex(index => {
348
- if (!(index instanceof WritableIndex)) {
349
- return;
350
- }
351
- let closeIndex = false;
352
- if (!index.isOpen()) {
353
- index.open();
354
- closeIndex = true;
355
- }
486
+ this.forEachWritableSecondaryIndex(index => {
356
487
  index.truncate(index.find(after));
357
- if (closeIndex) {
358
- index.close();
359
- }
360
488
  });
361
489
  }
362
490
 
491
+ /**
492
+ * @inheritDoc
493
+ * Open an existing secondary index and repair any stale entries beyond the current primary
494
+ * index length. Stale entries can be present when checkTornWrites() truncated the primary
495
+ * index before this secondary index was loaded into memory.
496
+ */
497
+ openIndex(name, matcher) {
498
+ const index = super.openIndex(name, matcher);
499
+ const lastEntry = index.lastEntry;
500
+ if (lastEntry !== false && lastEntry.number > this.index.length) {
501
+ // Secondary index is ahead of primary: truncate stale entries.
502
+ index.truncate(index.find(this.index.length));
503
+ }
504
+ return index;
505
+ }
506
+
363
507
  /**
364
508
  * @protected
365
509
  * @param {string} name
@@ -396,7 +540,5 @@ class WritableStorage extends ReadableStorage {
396
540
 
397
541
  }
398
542
 
399
- module.exports = WritableStorage;
400
- module.exports.StorageLockedError = StorageLockedError;
401
- module.exports.LOCK_THROW = LOCK_THROW;
402
- module.exports.LOCK_RECLAIM = LOCK_RECLAIM;
543
+ export default WritableStorage;
544
+ export { StorageLockedError, LOCK_THROW, LOCK_RECLAIM };
package/src/Storage.js CHANGED
@@ -1,5 +1,10 @@
1
- const WritableStorage = require('./Storage/WritableStorage');
2
- const ReadOnlyStorage = require('./Storage/ReadOnlyStorage');
1
+ import WritableStorage, { StorageLockedError, LOCK_THROW, LOCK_RECLAIM } from './Storage/WritableStorage.js';
2
+ import ReadOnlyStorage from './Storage/ReadOnlyStorage.js';
3
3
 
4
- module.exports = WritableStorage;
5
- module.exports.ReadOnly = ReadOnlyStorage;
4
+ WritableStorage.ReadOnly = ReadOnlyStorage;
5
+ WritableStorage.StorageLockedError = StorageLockedError;
6
+ WritableStorage.LOCK_THROW = LOCK_THROW;
7
+ WritableStorage.LOCK_RECLAIM = LOCK_RECLAIM;
8
+
9
+ export default WritableStorage;
10
+ export { ReadOnlyStorage as ReadOnly, StorageLockedError, LOCK_THROW, LOCK_RECLAIM };
package/src/Watcher.js CHANGED
@@ -1,7 +1,7 @@
1
- const fs = require('fs');
2
- const path = require('path');
3
- const events = require('events');
4
- const { assert } = require('./util');
1
+ import fs from 'fs';
2
+ import path from 'path';
3
+ import events from 'events';
4
+ import { assert } from './util.js';
5
5
 
6
6
  /** @type {Map<string, DirectoryWatcher>} */
7
7
  const directoryWatchers = new Map();
@@ -28,6 +28,7 @@ class DirectoryWatcher extends events.EventEmitter {
28
28
  assert(fs.existsSync(directory), `Can not watch a non-existing directory "${directory}".`);
29
29
  assert(fs.statSync(directory).isDirectory(), `Can only watch directories, but "${directory}" is none.`);
30
30
  super();
31
+ this.setMaxListeners(1000);
31
32
  directoryWatchers.set(directory, this);
32
33
  this.directory = directory;
33
34
  this.watcher = fs.watch(directory, Object.assign({ persistent: false }, options), this.emit.bind(this));
@@ -146,4 +147,4 @@ class Watcher {
146
147
 
147
148
  }
148
149
 
149
- module.exports = Watcher;
150
+ export default Watcher;
@@ -1,4 +1,4 @@
1
- const Watcher = require('./Watcher');
1
+ import Watcher from './Watcher.js';
2
2
 
3
3
  /**
4
4
  * A mixin that provides a file watcher for this.fileName which triggers a method `onChange` on the class, that needs to be implemented.
@@ -34,12 +34,13 @@ const WatchesFile = Base => class extends Base {
34
34
  * @returns {boolean}
35
35
  */
36
36
  open() {
37
- if (this.fd) {
38
- return false;
37
+ if (super.open()) {
38
+ if (!this.watcher) {
39
+ this.watchFile();
40
+ }
41
+ return true;
39
42
  }
40
-
41
- this.watchFile();
42
- return super.open();
43
+ return false;
43
44
  }
44
45
 
45
46
  /**
@@ -52,4 +53,4 @@ const WatchesFile = Base => class extends Base {
52
53
 
53
54
  };
54
55
 
55
- module.exports = WatchesFile;
56
+ export default WatchesFile;