event-storage 0.8.0 → 0.9.1

package/src/Partition/WritablePartition.js

@@ -77,6 +77,8 @@ class WritablePartition extends ReadablePartition {
         // How many documents are currently in the write buffer
         this.writeBufferDocuments = 0;
         this.flushCallbacks = [];
+        // Pre-allocated buffer for document header (16 bytes) + size footer (4 bytes) used in writeUnbuffered
+        this.writeMetaBuffer = Buffer.allocUnsafe(DOCUMENT_HEADER_SIZE + 4);
 
         this.clock = new this.ClockConstructor(this.metadata.epoch);
 
@@ -97,6 +99,7 @@ class WritablePartition extends ReadablePartition {
             this.writeBuffer = null;
             this.writeBufferCursor = 0;
             this.writeBufferDocuments = 0;
+            this.writeMetaBuffer = null;
         }
         super.close();
     }
@@ -134,8 +137,9 @@ class WritablePartition extends ReadablePartition {
 
         this.writeBufferCursor = 0;
         this.writeBufferDocuments = 0;
-        this.flushCallbacks.forEach(callback => callback());
+        const callbacks = this.flushCallbacks;
         this.flushCallbacks = [];
+        for (let i = 0; i < callbacks.length; i++) callbacks[i]();
 
         return true;
     }
@@ -199,17 +203,15 @@ class WritablePartition extends ReadablePartition {
      */
     writeUnbuffered(data, dataSize, sequenceNumber, callback) {
         this.flush();
-        const dataHeader = Buffer.alloc(DOCUMENT_HEADER_SIZE);
-        this.writeDocumentHeader(dataHeader, 0, dataSize, sequenceNumber);
+        this.writeDocumentHeader(this.writeMetaBuffer, 0, dataSize, sequenceNumber);
 
         let bytesWritten = 0;
-        bytesWritten += fs.writeSync(this.fd, dataHeader);
+        bytesWritten += fs.writeSync(this.fd, this.writeMetaBuffer, 0, DOCUMENT_HEADER_SIZE);
         bytesWritten += fs.writeSync(this.fd, data);
         const padSize = alignTo(dataSize + DOCUMENT_FOOTER_SIZE, DOCUMENT_ALIGNMENT);
         bytesWritten += fs.writeSync(this.fd, DOCUMENT_PAD.substr(0, padSize));
-        const dataSizeBuffer = Buffer.alloc(4);
-        dataSizeBuffer.writeUInt32BE(dataSize, 0);
-        bytesWritten += fs.writeSync(this.fd, dataSizeBuffer);
+        this.writeMetaBuffer.writeUInt32BE(dataSize, 0);
+        bytesWritten += fs.writeSync(this.fd, this.writeMetaBuffer, 0, 4);
         bytesWritten += fs.writeSync(this.fd, DOCUMENT_SEPARATOR);
         if (typeof callback === 'function') {
             process.nextTick(callback);
@@ -227,7 +229,7 @@ class WritablePartition extends ReadablePartition {
      * @returns {number} Number of bytes written.
      */
     writeBuffered(data, dataSize, sequenceNumber, callback) {
-        const bytesToWrite = this.documentWriteSize(Buffer.byteLength(data, 'utf8'));
+        const bytesToWrite = this.documentWriteSize(dataSize);
         this.flushIfWriteBufferTooSmall(bytesToWrite);
 
         let bytesWritten = 0;
@@ -293,7 +295,40 @@ class WritablePartition extends ReadablePartition {
     }
 
     /**
-     * Truncate the internal read buffer after the given position.
+     * Prepare the read buffer for reading *before* the specified position. Don't try to read *after* the returned cursor.
+     *
+     * @protected
+     * @param {number} position The position in the file to prepare the read buffer for reading before.
+     * @returns {object} A reader object with properties `buffer`, `cursor` and `length`.
+     */
+    prepareReadBufferBackwards(position) {
+        const bufferPos = this.size - this.writeBufferCursor;
+        // Handle the case when data that is still in write buffer is supposed to be read backwards
+        if (this.dirtyReads && this.writeBufferCursor > 0 && position > bufferPos) {
+            return { buffer: this.writeBuffer, cursor: position - bufferPos, length: this.writeBufferCursor };
+        }
+        return super.prepareReadBufferBackwards(position);
+    }
+
+    /**
+     * Read all documents in reverse write order, ignoring any unflushed write-buffer data when
+     * dirty reads are disabled.
+     *
+     * @api
+     * @param {number} [before] The document position to start reading backward from.
+     * @returns {Generator<string>} A generator that returns all documents in this partition in reverse order.
+     */
+    *readAllBackwards(before = -1) {
+        if (!this.dirtyReads && this.writeBufferCursor > 0) {
+            const flushedSize = this.size - this.writeBufferCursor;
+            const clampedBefore = before < 0 ? flushedSize : Math.min(before, flushedSize);
+            yield* super.readAllBackwards(clampedBefore);
+            return;
+        }
+        yield* super.readAllBackwards(before);
+    }
+
+    /**
      *
      * @internal
      * @param {number} after The byte position to truncate the read buffer after.
@@ -307,6 +342,31 @@ class WritablePartition extends ReadablePartition {
         }
     }
 
+    /**
+     * Truncate the partition, removing all documents with sequenceNumber > after.
+     * Scans the partition file backwards to find the cutoff position.
+     *
+     * @api
+     * @param {number} after Keep all documents with sequenceNumber <= after. Documents with
+     *   sequenceNumber > after (and any torn write at the end) are removed.
+     */
+    truncateAfterSequence(after) {
+        let position = this.size;
+        let truncateAt = this.size; // default: nothing to truncate
+        while ((position = this.findDocumentPositionBefore(position)) !== false) {
+            const reader = this.prepareReadBufferBackwards(position);
+            if (!reader.buffer) break;
+            const { sequenceNumber } = this.readDocumentHeader(reader.buffer, reader.cursor, position);
+            if (sequenceNumber > after) {
+                // This document must be removed; record its start as the new cutoff.
+                truncateAt = position;
+            } else {
+                break;
+            }
+        }
+        this.truncate(truncateAt);
+    }
+
     /**
      * Truncate the partition storage at the given position.
      *
@@ -314,13 +374,16 @@ class WritablePartition extends ReadablePartition {
      * @param {number} after The file position after which to truncate the partition.
      */
     truncate(after) {
-        if (after > this.size) {
+        if (after >= this.size) {
             return;
         }
         this.open();
         after = Math.max(0, after);
         this.flush();
 
+        // Always save the truncated part for manual recovery, even if it contains corrupted data
+        this.branchOff('truncated-' + Date.now(), after);
+
         try {
             this.readFrom(after);
         } catch (e) {
@@ -329,10 +392,6 @@ class WritablePartition extends ReadablePartition {
             }
         }
 
-        // copy all truncated documents to some delete log
-        const backupName = (new Date()).toISOString().substring(0,10);
-        this.branchOff(backupName, after);
-
         fs.truncateSync(this.fileName, this.headerSize + after);
         this.truncateReadBuffer(after);
         this.size = after;

package/src/Storage/ReadableStorage.js

@@ -3,7 +3,8 @@ const path = require('path');
 const events = require('events');
 const Partition = require('../Partition');
 const Index = require('../Index');
-const { assert, createHmac, matches, wrapAndCheck, buildMetadataForMatcher } = require('../util');
+const { assert, wrapAndCheck, kWayMerge } = require('../util');
+const { createHmac, matches, buildMetadataForMatcher } = require('../metadataUtil');
 
 const DEFAULT_READ_BUFFER_SIZE = 4 * 1024;
 
@@ -66,8 +67,8 @@ class ReadableStorage extends events.EventEmitter {
 
         this.dataDirectory = path.resolve(config.dataDirectory);
 
-        this.initializeIndexes(config);
         this.scanPartitions(config);
+        this.initializeIndexes(config);
     }
 
     /**
@@ -109,6 +110,7 @@ class ReadableStorage extends events.EventEmitter {
         const { index } = this.createIndex(config.indexFile, this.indexOptions);
         this.index = index;
         this.secondaryIndexes = {};
+        this.readonlyIndexes = {};
     }
 
     /**
@@ -172,6 +174,9 @@ class ReadableStorage extends events.EventEmitter {
     close() {
         this.index.close();
         this.forEachSecondaryIndex(index => index.close());
+        for (let index of Object.values(this.readonlyIndexes)) {
+            index.close();
+        }
         this.forEachPartition(partition => partition.close());
         this.emit('closed');
     }
@@ -193,6 +198,19 @@ class ReadableStorage extends events.EventEmitter {
         return this.partitions[partitionIdentifier];
     }
 
+    /**
+     * Register a handler that is called before a document is read from a partition.
+     * The handler receives the position and the partition metadata and may throw to abort the read.
+     * Multiple handlers can be registered; all run on every read in registration order.
+     * Equivalent to `storage.on('preRead', hook)`.
+     *
+     * @api
+     * @param {function(number, object): void} hook A function receiving (position, partitionMetadata).
+     */
+    preRead(hook) {
+        this.on('preRead', hook);
+    }
+
     /**
      * @protected
      * @param {number} partitionId The partition to read from.
@@ -203,6 +221,9 @@ class ReadableStorage extends events.EventEmitter {
      */
     readFrom(partitionId, position, size) {
         const partition = this.getPartition(partitionId);
+        if (this.listenerCount('preRead') > 0) {
+            this.emit('preRead', position, partition.metadata);
+        }
         const data = partition.readFrom(position, size);
         return this.serializer.deserialize(data);
     }
@@ -237,21 +258,25 @@ class ReadableStorage extends events.EventEmitter {
      * @api
      * @param {number} from The 1-based document number (inclusive) to start reading from.
      * @param {number} [until] The 1-based document number (inclusive) to read until. Defaults to index.length.
-     * @param {ReadableIndex} [index] The index to use for finding the documents in the range.
+     * @param {ReadableIndex|false} [index] The index to use for finding the documents in the range.
+     *   Pass `false` to skip the global index and iterate all partitions directly in sequenceNumber order
+     *   (useful when the global index is unavailable or corrupted).
      * @returns {Generator<object>} A generator that will read each document in the range one by one.
      */
     *readRange(from, until = -1, index = null) {
-        index = index || this.index;
-        index.open();
+        const lengthSource = index || this.index;
+        if (!lengthSource.isOpen()) {
+            lengthSource.open();
+        }
 
-        const readFrom = wrapAndCheck(from, index.length);
-        const readUntil = wrapAndCheck(until, index.length);
+        const readFrom = wrapAndCheck(from, lengthSource.length);
+        const readUntil = wrapAndCheck(until, lengthSource.length);
         assert(readFrom > 0 && readUntil > 0, `Range scan error for range ${from} - ${until}.`);
 
         if (readFrom > readUntil) {
             const batchSize = 10;
             let batchUntil = readFrom;
-            while (batchUntil > readUntil) {
+            while (batchUntil >= readUntil) {
                 const batchFrom = Math.max(readUntil, batchUntil - batchSize);
                 yield* reverse(this.iterateRange(batchFrom, batchUntil, index));
                 batchUntil = batchFrom - 1;
@@ -264,20 +289,52 @@ class ReadableStorage extends events.EventEmitter {
 
     /**
      * Iterate all documents in this storage in range from to until inside the index.
+     * If index is false, iterates all partitions directly in sequenceNumber order.
      * @private
      * @param {number} from
      * @param {number} until
-     * @param {ReadableIndex} index
+     * @param {ReadableIndex|false|null} index
      * @returns {Generator<object>}
      */
     *iterateRange(from, until, index) {
-        const entries = index.range(from, until);
+        if (index === false) {
+            // Explicitly disabled index: iterate all partitions and merge by sequenceNumber.
+            // Document header sequenceNumber is 0-based; from/until are 1-based index positions.
+            for (const entry of this.iterateDocumentsNoIndex(from - 1, until - 1)) {
+                yield entry.document;
+            }
+            return;
+        }
+
+        const idx = index || this.index;
+        const entries = idx.range(from, until);
         for (let entry of entries) {
             const document = this.readFrom(entry.partition, entry.position, entry.size);
             yield document;
         }
     }
 
+    /**
+     * Open an existing readonly index for reading, without registering it in the secondary indexes write path.
+     * Use this for indexes whose files carry a status marker (e.g. `stream-foo.closed.index`).
+     *
+     * @api
+     * @param {string} name The readonly index name (e.g. 'stream-foo.closed').
+     * @returns {ReadableIndex}
+     * @throws {Error} if the readonly index does not exist.
+     */
+    openReadonlyIndex(name) {
+        if (name in this.readonlyIndexes) {
+            return this.readonlyIndexes[name];
+        }
+        const indexName = this.storageFile + '.' + name + '.index';
+        assert(fs.existsSync(path.join(this.indexDirectory, indexName)), `Index "${name}" does not exist.`);
+        const { index } = this.createIndex(indexName, Object.assign({}, this.indexOptions));
+        index.open();
+        this.readonlyIndexes[name] = index;
+        return index;
+    }
+
     /**
      * Open an existing index.
      *
@@ -289,6 +346,9 @@ class ReadableStorage extends events.EventEmitter {
      * @throws {Error} if the HMAC for the matcher does not match.
      */
     openIndex(name, matcher) {
+        if (name === '_all') {
+            return this.index;
+        }
         if (name in this.secondaryIndexes) {
             return this.secondaryIndexes[name].index;
         }
@@ -304,17 +364,79 @@ class ReadableStorage extends events.EventEmitter {
     }
 
     /**
-     * Helper method to iterate over all documents.
+     * Iterate documents across all partitions in sequenceNumber order using a k-way merge.
+     * Opens any closed partition automatically.
+     *
+     * @protected
+     * @param {number} [from=0] The 0-based sequenceNumber to start from (inclusive).
+     * @param {number} [until=Number.MAX_SAFE_INTEGER] The 0-based sequenceNumber to read until (inclusive).
+     * @returns {Generator<{document: object, sequenceNumber: number, partitionName: string, position: number, size: number, partition: number}>}
+     */
+    *iterateDocumentsNoIndex(from = 0, until = Number.MAX_SAFE_INTEGER) {
+        const streams = [];
+
+        this.forEachPartition(partition => {
+            if (!partition.isOpen()) {
+                partition.open();
+            }
+
+            const found = partition.findDocument(from);
+            if (found && found.headerOut.sequenceNumber <= until) {
+                const nextPosition = found.headerOut.position + partition.documentWriteSize(found.headerOut.dataSize);
+                const reader = partition.readAll(nextPosition, found.headerOut);
+                streams.push({ ...found, reader, partition: partition.id, partitionName: partition.name });
+            }
+        });
+
+        const items = [];
+        kWayMerge(
+            streams,
+            stream => stream.headerOut.sequenceNumber,
+            stream => {
+                const next = stream.reader.next();
+                if (!next.done && stream.headerOut.sequenceNumber <= until) {
+                    stream.data = next.value;
+                    return true;
+                }
+                return false;
+            },
+            stream => items.push({
+                document: this.serializer.deserialize(stream.data),
+                sequenceNumber: stream.headerOut.sequenceNumber,
+                partitionName: stream.partitionName,
+                position: stream.headerOut.position,
+                size: stream.headerOut.dataSize,
+                partition: stream.partition,
+            })
+        );
+
+        yield* items;
+    }
+
+    /**
+     * Helper method to iterate over all documents, invoking a callback for each one.
+     * Pass `noIndex = true` to iterate all partitions directly in sequenceNumber order
+     * (useful when the global index is unavailable or corrupted).
+     * When `noIndex` is false the second callback argument is the raw index `EntryInterface`.
+     * When `noIndex` is true the second callback argument has `{ partition, position, size, sequenceNumber, partitionName }`.
      *
      * @protected
-     * @param {function(object, EntryInterface)} iterationHandler
+     * @param {function(object, object): void} iterationHandler
+     * @param {boolean} [noIndex=false] When true, bypasses the index and iterates partitions directly.
      */
-    forEachDocument(iterationHandler) {
+    forEachDocument(iterationHandler, noIndex = false) {
         /* istanbul ignore if  */
         if (typeof iterationHandler !== 'function') {
             return;
         }
 
+        if (noIndex) {
+            for (const { document, ...entryInfo } of this.iterateDocumentsNoIndex()) {
+                iterationHandler(document, entryInfo);
+            }
+            return;
+        }
+
         const entries = this.index.all();
 
         for (let entry of entries) {
@@ -364,3 +486,4 @@ class ReadableStorage extends events.EventEmitter {
 
 module.exports = ReadableStorage;
 module.exports.matches = matches;
+module.exports.CorruptFileError = Partition.CorruptFileError;

package/src/Storage/WritableStorage.js

@@ -3,7 +3,8 @@ const path = require('path');
 const WritablePartition = require('../Partition/WritablePartition');
 const WritableIndex = require('../Index/WritableIndex');
 const ReadableStorage = require('./ReadableStorage');
-const { assert, matches, buildMetadataForMatcher, buildMatcherFromMetadata, ensureDirectory } = require('../util');
+const { assert, ensureDirectory } = require('../util');
+const { matches, buildMetadataForMatcher, buildMatcherFromMetadata } = require('../metadataUtil');
 
 const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024;
 
@@ -75,29 +76,137 @@ class WritableStorage extends ReadableStorage {
         if (!this.lock()) {
             return true;
         }
-        return super.open();
+        const result = super.open();
+        this.emit('ready');
+        return result;
     }
 
     /**
-     * Check all partitions torn writes and truncate the storage to the position before the first torn write.
-     * This might delete correctly written events in partitions, if their sequence number is higher than the
-     * torn write in another partition.
+     * Helper method to iterate over all writable secondary indexes.
+     * Opens each index before calling the callback (passing the previous open status),
+     * and closes it afterwards if it was not already open.
+     *
+     * @protected
+     * @param {function(WritableIndex, string, boolean)} iterationHandler Called with (index, name, wasOpen).
+     * @param {object} [matchDocument] If supplied, only indexes the document matches on will be iterated.
      */
-    checkTornWrites() {
+    forEachWritableSecondaryIndex(iterationHandler, matchDocument) {
+        this.forEachSecondaryIndex((index, name) => {
+            /* istanbul ignore if */
+            if (!(index instanceof WritableIndex)) return;
+            const wasOpen = index.isOpen();
+            if (!wasOpen) index.open();
+            iterationHandler(index, name, wasOpen);
+            if (!wasOpen) index.close();
+        }, matchDocument);
+    }
+
+    /**
+     * Scan every partition's last document to detect torn writes inline.
+     * A document is torn when its expected end exceeds the actual file size:
+     *   position + documentWriteSize(dataSize) > partition.size
+     *
+     * @private
+     * @returns {{ lastValidSequenceNumber: number, maxPartitionSequenceNumber: number }}
+     */
+    findTornWriteBoundary() {
         let lastValidSequenceNumber = Number.MAX_SAFE_INTEGER;
+        let maxPartitionSequenceNumber = -1;
         this.forEachPartition(partition => {
             partition.open();
-            const tornSequenceNumber = partition.checkTornWrite();
-            if (tornSequenceNumber >= 0) {
-                lastValidSequenceNumber = Math.min(lastValidSequenceNumber, tornSequenceNumber);
+            const last = partition.readLast();
+            /* istanbul ignore if */
+            if (!last) return;
+            const { header: { sequenceNumber, dataSize }, position } = last;
+            if (position + partition.documentWriteSize(dataSize) > partition.size) {
+                // Torn write: the document extends beyond the end of the file.
+                lastValidSequenceNumber = Math.min(lastValidSequenceNumber, sequenceNumber);
+            } else {
+                maxPartitionSequenceNumber = Math.max(maxPartitionSequenceNumber, sequenceNumber);
             }
         });
+        return { lastValidSequenceNumber, maxPartitionSequenceNumber };
+    }
+
+    /**
+     * Check all partitions for torn writes, physically repair each partition, truncate all indexes
+     * to the torn-write boundary, and then reindex to rebuild any missing index entries.
+     *
+     * A document is torn when the partition file ends before the document's expected end position
+     * (i.e. position + documentWriteSize(dataSize) > partition.size).  Detected inline in
+     * findTornWriteBoundary(), without any checkTornWrite() call.
+     *
+     * Repair flow:
+     * 1. findTornWriteBoundary() reads the last document of every partition and finds the global
+     *    torn-write boundary (minimum torn sequence number across all partitions).
+     * 2. If torn writes were found, truncateAfterSequence() removes all documents at or beyond
+     *    the boundary from each partition.
+     * 3. Truncate all indexes to the torn-write boundary, then reindex to fill any lagging entries.
+     * 4. If no torn writes were found but the index is lagging, reindex directly.
+     */
+    checkTornWrites() {
+        const { lastValidSequenceNumber, maxPartitionSequenceNumber } = this.findTornWriteBoundary();
+
         if (lastValidSequenceNumber < Number.MAX_SAFE_INTEGER) {
-            this.truncate(lastValidSequenceNumber);
+            // Phase 2: remove all documents at or beyond the torn-write boundary from each partition.
+            this.forEachPartition(partition => {
+                partition.open();
+                partition.truncateAfterSequence(lastValidSequenceNumber - 1);
+            });
+
+            // Truncate all indexes to the torn-write boundary.
+            this.index.open();
+            this.index.truncate(lastValidSequenceNumber);
+            /* istanbul ignore next */
+            this.forEachWritableSecondaryIndex(index => {
+                index.truncate(index.find(lastValidSequenceNumber));
+            });
+
+            // Reindex to fill in any missing complete-document entries.
+            this.reindex(this.index.length);
+        } else if (maxPartitionSequenceNumber >= 0 && maxPartitionSequenceNumber + 1 > this.index.length) {
+            // No torn writes, but the index is lagging — repair it.
+            this.reindex(this.index.length);
         }
+
         this.forEachPartition(partition => partition.close());
     }
 
+    /**
+     * Rebuild the primary index and all loaded secondary indexes starting from the given sequence
+     * number by scanning the partition data directly.
+     * This is the building block for both auto-repair (invoked automatically when the primary
+     * index is found to be lagging in checkTornWrites()) and for user-driven re-indexing after
+     * index corruption.
+     *
+     * @api
+     * @param {number} [fromSequenceNumber=0] The number of primary index entries to keep intact.
+     *   All index entries beyond this position will be removed and rebuilt from partition data.
+     *   Defaults to 0, which rebuilds all indexes from scratch.
+     */
+    reindex(fromSequenceNumber = 0) {
+        this.index.truncate(fromSequenceNumber);
+
+        // Truncate all loaded secondary indexes to match the new primary length.
+        this.forEachWritableSecondaryIndex(index => {
+            // find(0) returns 0, so truncate(0) will remove all entries when fromSequenceNumber===0
+            index.truncate(fromSequenceNumber === 0 ? 0 : index.find(fromSequenceNumber));
+        });
+
+        // Scan partitions in sequence-number order and rebuild index entries.
+        // iterateDocumentsNoIndex opens any closed partitions automatically.
+        for (const { document, partition, position, size } of this.iterateDocumentsNoIndex(fromSequenceNumber, Number.MAX_SAFE_INTEGER)) {
+            const newEntry = new WritableIndex.Entry(this.index.length + 1, position, size, partition);
+            this.index.add(newEntry);
+
+            this.forEachWritableSecondaryIndex((secIndex) => {
+                secIndex.add(newEntry);
+            }, document);
+        }
+
+        this.flush();
+    }
+
     /**
      * Attempt to lock this storage by means of a lock directory.
      * @returns {boolean} True if the lock was created or false if the lock is already in place.
@@ -178,6 +287,19 @@ class WritableStorage extends ReadableStorage {
         return entry;
     }
 
+    /**
+     * Register a handler that is called before a document is written to storage.
+     * The handler receives the document and the partition metadata and may throw to abort the write.
+     * Multiple handlers can be registered; all run on every write in registration order.
+     * Equivalent to `storage.on('preCommit', hook)`.
+     *
+     * @api
+     * @param {function(object, object): void} hook A function receiving (document, partitionMetadata).
+     */
+    preCommit(hook) {
+        this.on('preCommit', hook);
+    }
+
     /**
      * Get a partition either by name or by id.
      * If a partition with the given name does not exist, a new one will be created.
@@ -190,10 +312,14 @@ class WritableStorage extends ReadableStorage {
      */
     getPartition(partitionIdentifier) {
         if (typeof partitionIdentifier === 'string') {
+            const partitionShortName = partitionIdentifier;
             const partitionName = this.storageFile + (partitionIdentifier.length ? '.' + partitionIdentifier : '');
             partitionIdentifier = WritablePartition.idFor(partitionName);
             if (!this.partitions[partitionIdentifier]) {
-                this.partitions[partitionIdentifier] = this.createPartition(partitionName, this.partitionConfig);
+                const partitionConfig = typeof this.partitionConfig.metadata === 'function'
+                    ? { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) }
+                    : this.partitionConfig;
+                this.partitions[partitionIdentifier] = this.createPartition(partitionName, partitionConfig);
                 this.emit('partition-created', partitionIdentifier);
             }
             this.partitions[partitionIdentifier].open();
@@ -214,6 +340,9 @@ class WritableStorage extends ReadableStorage {
 
         const partitionName = this.partitioner(document, this.index.length + 1);
         const partition = this.getPartition(partitionName);
+        if (this.listenerCount('preCommit') > 0) {
+            this.emit('preCommit', document, partition.metadata);
+        }
         const position = partition.write(data, this.length, callback);
 
         assert(position !== false, 'Error writing document.');
@@ -241,6 +370,9 @@ class WritableStorage extends ReadableStorage {
      * @throws {Error} if the index doesn't exist yet and no matcher was specified.
      */
     ensureIndex(name, matcher) {
+        if (name === '_all') {
+            return this.index;
+        }
         if (name in this.secondaryIndexes) {
             return this.secondaryIndexes[name].index;
         }
@@ -307,6 +439,10 @@ class WritableStorage extends ReadableStorage {
     /**
      * Truncate all partitions after the given (global) sequence number.
      *
+     * Assumes the primary index is fully consistent with the partition data. Looks up the first
+     * index entry after `after` for each affected partition and truncates the partition file
+     * at that entry's byte position.
+     *
      * @private
      * @param {number} after The document sequence number to truncate after.
      */
@@ -340,26 +476,34 @@ class WritableStorage extends ReadableStorage {
         if (!this.index.isOpen()) {
             this.index.open();
         }
+        if (after < 0) {
+            after += this.index.length;
+        }
 
         this.truncatePartitions(after);
 
         this.index.truncate(after);
-        this.forEachSecondaryIndex(index => {
-            if (!(index instanceof WritableIndex)) {
-                return;
-            }
-            let closeIndex = false;
-            if (!index.isOpen()) {
-                index.open();
-                closeIndex = true;
-            }
+        this.forEachWritableSecondaryIndex(index => {
             index.truncate(index.find(after));
-            if (closeIndex) {
-                index.close();
-            }
         });
     }
 
+    /**
+     * @inheritDoc
+     * Open an existing secondary index and repair any stale entries beyond the current primary
+     * index length. Stale entries can be present when checkTornWrites() truncated the primary
+     * index before this secondary index was loaded into memory.
+     */
+    openIndex(name, matcher) {
+        const index = super.openIndex(name, matcher);
+        const lastEntry = index.lastEntry;
+        if (lastEntry !== false && lastEntry.number > this.index.length) {
+            // Secondary index is ahead of primary: truncate stale entries.
+            index.truncate(index.find(this.index.length));
+        }
+        return index;
+    }
+
     /**
      * @protected
      * @param {string} name
@@ -398,5 +542,6 @@ class WritableStorage extends ReadableStorage {
 
 module.exports = WritableStorage;
 module.exports.StorageLockedError = StorageLockedError;
+module.exports.CorruptFileError = ReadableStorage.CorruptFileError;
 module.exports.LOCK_THROW = LOCK_THROW;
-module.exports.LOCK_RECLAIM = LOCK_RECLAIM;
+module.exports.LOCK_RECLAIM = LOCK_RECLAIM;