event-storage 0.7.2 → 0.9.1

package/src/Partition/ReadablePartition.js

@@ -1,46 +1,27 @@
 const fs = require('fs');
 const path = require('path');
-const EventEmitter = require('events');
-const { assert } = require('../util');
+const events = require('events');
+const { assert, alignTo, hash, binarySearch } = require('../util');
 
 const DEFAULT_READ_BUFFER_SIZE = 64 * 1024;
 const DOCUMENT_HEADER_SIZE = 16;
 const DOCUMENT_ALIGNMENT = 4;
+const DOCUMENT_SEPARATOR = "\x00\x00\x1E\n";
+const DOCUMENT_FOOTER_SIZE = 4 /* additional data size footer */ + DOCUMENT_SEPARATOR.length;
 
-// node-event-store partition V02
-const HEADER_MAGIC = "nesprt02";
+// node-event-store partition V03
+const HEADER_MAGIC = "nesprt03";
+
+const NES_EPOCH = new Date('2020-01-01T00:00:00');
 
 class CorruptFileError extends Error {}
 class InvalidDataSizeError extends Error {}
 
-/**
- * Method for hashing a string (partition name) to a 32-bit unsigned integer.
- *
- * @param {string} str
- * @returns {number}
- */
-function hash(str) {
-    if (str.length === 0) {
-        return 0;
-    }
-    let hash = 5381,
-        i    = str.length;
-
-    while(i) {
-        hash = ((hash << 5) + hash) ^ str.charCodeAt(--i); // jshint ignore:line
-    }
-
-    /* JavaScript does bitwise operations (like XOR, above) on 32-bit signed
-     * integers. Since we want the results to be always positive, convert the
-     * signed int to an unsigned by doing an unsigned bitshift. */
-    return hash >>> 0; // jshint ignore:line
-}
-
 /**
  * A partition is a single file where the storage will write documents to depending on some partitioning rules.
  * In the case of an event store, this is most likely the (write) streams.
  */
-class ReadablePartition extends EventEmitter {
+class ReadablePartition extends events.EventEmitter {
 
     /**
      * Get the id for a specific partition name.
@@ -109,6 +90,7 @@ class ReadablePartition extends EventEmitter {
         this.headerSize = 0;
         this.size = this.readFileSize();
         if (this.size <= 0) {
+            this.close();
             return false;
         }
 
@@ -146,6 +128,7 @@ class ReadablePartition extends EventEmitter {
         const metadata = metadataBuffer.toString('utf8').trim();
         try {
             this.metadata = JSON.parse(metadata);
+            this.metadata.epoch = this.metadata.epoch /* istanbul ignore next */|| NES_EPOCH.getTime();
         } catch (e) {
             throw new Error('Invalid metadata.');
         }
@@ -160,12 +143,12 @@ class ReadablePartition extends EventEmitter {
      * @returns {number} The size of the data including header, padded to 16 bytes alignment and ended with a line break.
      */
     documentWriteSize(dataSize) {
-        const padSize = (DOCUMENT_ALIGNMENT - ((dataSize + 1) % DOCUMENT_ALIGNMENT)) % DOCUMENT_ALIGNMENT;
-        return dataSize + 1 + padSize + DOCUMENT_HEADER_SIZE;
+        const padSize = alignTo(dataSize + DOCUMENT_FOOTER_SIZE, DOCUMENT_ALIGNMENT);
+        return DOCUMENT_HEADER_SIZE + dataSize + padSize + DOCUMENT_FOOTER_SIZE;
     }
 
     /**
-     * @private
+     * @protected
      * @returns {number} The file size not including the file header.
      */
     readFileSize() {
@@ -208,7 +191,7 @@ class ReadablePartition extends EventEmitter {
      * @param {number} offset The position inside the buffer to start reading from.
      * @param {number} position The file position to start reading from.
      * @param {number} [size] The expected byte size of the document at the given position.
-     * @returns {{ dataSize: number, sequenceNumber, number, time64: number }} The metadata fields of the document
+     * @returns {{ dataSize: number, sequenceNumber: number, time64: number }} The metadata fields of the document
      * @throws {Error} if the storage entry at the given position is corrupted.
      * @throws {InvalidDataSizeError} if the document size at the given position does not match the provided size.
      * @throws {CorruptFileError} if the document at the given position can not be read completely.
@@ -221,10 +204,6 @@ class ReadablePartition extends EventEmitter {
             throw new InvalidDataSizeError(`Invalid document size ${dataSize} at position ${position}, expected ${size}.`);
         }
 
-        if (position + dataSize + DOCUMENT_HEADER_SIZE > this.size) {
-            throw new CorruptFileError(`Invalid document at position ${position}. This may be caused by an unfinished write.`);
-        }
-
         const sequenceNumber = buffer.readUInt32BE(offset + 4);
         const time64 = buffer.readDoubleBE(offset + 8);
         return ({ dataSize, sequenceNumber, time64 });
@@ -249,23 +228,41 @@ class ReadablePartition extends EventEmitter {
         return ({ buffer: this.readBuffer, cursor: bufferCursor, length: this.readBufferLength });
     }
 
+    /**
+     * Prepare the read buffer for reading *before* the specified position. Don't try to reader *after* the returned cursor.
+     *
+     * @protected
+     * @param {number} position The position in the file to prepare the read buffer for reading before.
+     * @returns {{ buffer: Buffer|null, cursor: number, length: number }} A reader object with properties `buffer`, `cursor` and `length`.
+     */
+    prepareReadBufferBackwards(position) {
+        if (position < 0) {
+            return ({ buffer: null, cursor: 0, length: 0 });
+        }
+        let bufferCursor = position - this.readBufferPos;
+        if (this.readBufferPos < 0 || (this.readBufferPos > 0 && bufferCursor < DOCUMENT_FOOTER_SIZE) || bufferCursor > this.readBufferLength) {
+            this.fillBuffer(Math.max(position - this.readBuffer.byteLength, 0));
+            bufferCursor = position - this.readBufferPos;
+        }
+        return ({ buffer: this.readBuffer, cursor: bufferCursor, length: this.readBufferLength });
+    }
+
     /**
      * Read the data from the given position.
      *
      * @api
      * @param {number} position The file position to read from.
      * @param {number} [size] The expected byte size of the document at the given position.
+     * @param {object|null} [headerOut] Optional object to populate with the document header fields
+     *   (`dataSize`, `sequenceNumber`, `time64`). Pass an existing object to avoid extra allocation.
      * @returns {string|boolean} The data stored at the given position or false if no data could be read.
      * @throws {Error} if the storage entry at the given position is corrupted.
      * @throws {InvalidDataSizeError} if the document size at the given position does not match the provided size.
      * @throws {CorruptFileError} if the document at the given position can not be read completely.
      */
-    readFrom(position, size = 0) {
-        if (!this.fd) {
-            return false;
-        }
-
-        assert((position % DOCUMENT_ALIGNMENT) === 0, `Invalid read position. Needs to be a multiple of ${DOCUMENT_ALIGNMENT}.`);
+    readFrom(position, size = 0, headerOut = null) {
+        assert(this.fd, 'Partition is not opened.');
+        assert((position % DOCUMENT_ALIGNMENT) === 0, `Invalid read position ${position}. Needs to be a multiple of ${DOCUMENT_ALIGNMENT}.`);
 
         const reader = this.prepareReadBuffer(position);
         if (reader.length < size + DOCUMENT_HEADER_SIZE) {
@@ -273,7 +270,18 @@ class ReadablePartition extends EventEmitter {
         }
 
         let dataPosition = reader.cursor + DOCUMENT_HEADER_SIZE;
-        const { dataSize } = this.readDocumentHeader(reader.buffer, reader.cursor, position, size);
+        const { dataSize, sequenceNumber, time64 } = this.readDocumentHeader(reader.buffer, reader.cursor, position, size);
+        if (headerOut !== null) {
+            headerOut.dataSize = dataSize;
+            headerOut.sequenceNumber = sequenceNumber;
+            headerOut.time64 = time64;
+        }
+
+        // TODO: This should only be checked on opening
+        const writeSize = this.documentWriteSize(dataSize);
+        if (position + writeSize > this.size) {
+            throw new CorruptFileError(`Invalid document at position ${position}. This may be caused by an unfinished write.`);
+        }
 
         if (dataSize + DOCUMENT_HEADER_SIZE > reader.buffer.byteLength) {
             //console.log('sync read for large document size', dataLength, 'at position', position);
@@ -291,21 +299,165 @@ class ReadablePartition extends EventEmitter {
     }
 
     /**
+     * Find the start position of the document that precedes the given position.
+     *
+     * @protected
+     * @param {number} position The file position to read backwards from.
+     * @returns {number|boolean} The start position of the first document before the given position or false if no header could be found.
+     */
+    findDocumentPositionBefore(position) {
+        assert(this.fd, 'Partition is not opened.');
+        position -= (position % DOCUMENT_ALIGNMENT);
+        if (position <= 0) {
+            return false;
+        }
+
+        const separatorSize = DOCUMENT_SEPARATOR.length;
+        // Optimization if we are at an exact document boundary, where we can just read the document size
+        let reader = this.prepareReadBufferBackwards(position);
+        const block = reader.buffer.toString('ascii', reader.cursor - separatorSize, reader.cursor);
+        if (block === DOCUMENT_SEPARATOR) {
+            const dataSize = reader.buffer.readUInt32BE(reader.cursor - separatorSize - 4);
+            return position - this.documentWriteSize(dataSize);
+        }
+
+        do {
+            reader = this.prepareReadBufferBackwards(position - separatorSize);
+
+            const bufferSeparatorPosition = reader.buffer.lastIndexOf(DOCUMENT_SEPARATOR, reader.cursor - separatorSize, 'ascii');
+            if (bufferSeparatorPosition >= 0) {
+                position = this.readBufferPos + bufferSeparatorPosition + separatorSize;
+                break;
+            }
+            position -= this.readBufferLength;
+        } while (position > 0);
+        return Math.max(0, position);
+    }
+
+    /**
+     * Find the document that starts immediately before `position`, fill the read buffer
+     * centered around that document's start, and return its file position and parsed header.
+     * The buffer is centered by calling `prepareReadBufferBackwards` with a position
+     * half a buffer-length ahead of the document start, clamped to file size, so the
+     * document start lands near the middle of the buffer.
+     *
+     * @private
+     * @param {number} position The file position to search before.
+     * @returns {{ header: {dataSize: number, sequenceNumber: number, time64: number}, position: number }|null}
+     *   The document header and file position, or null if no document could be found.
+     */
+    readDocumentBefore(position) {
+        const docPos = this.findDocumentPositionBefore(position);
+        /* istanbul ignore if */
+        if (docPos === false || docPos < 0) return null;
+        const reader = this.prepareReadBufferBackwards(Math.min(docPos + (this.readBuffer.byteLength >> 1), this.size));
+        /* istanbul ignore if */
+        if (!reader.buffer) return null;
+        const cursor = docPos - this.readBufferPos;
+        /* istanbul ignore if */
+        if (cursor < 0 || cursor + DOCUMENT_HEADER_SIZE > reader.length) return null;
+        const header = this.readDocumentHeader(reader.buffer, cursor, docPos);
+        return { header, position: docPos };
+    }
+
+    /**
+     * Read the header and file position of the last document in this partition.
+     *
      * @api
-     * @returns {Generator} A generator that returns all documents in this partition.
+     * @returns {{ header: {dataSize: number, sequenceNumber: number, time64: number}, position: number } | null}
+     *   The last document's header and its file position, or null if the partition is empty or unreadable.
      */
-    *readAll() {
-        let position = 0;
+    readLast() {
+        if (this.size === 0) return null;
+        return this.readDocumentBefore(this.size);
+    }
+
+    /**
+     * Find the first document whose sequenceNumber is >= the given value.
+     * Uses readLast() to short-circuit when the partition contains no such document.
+     * Uses a binary search over file positions via readDocumentBefore() to locate the
+     * document. The search tracks both the lower bound (position just after the last
+     * confirmed "< sequenceNumber" doc) and the upper bound (minimum position of any
+     * probed doc with sequenceNumber >= target). The upper bound, when available, is
+     * the exact target document, so no further linear scan is needed.
+     *
+     * @api
+     * @param {number} sequenceNumber The 0-based sequence number to search for.
+     * @returns {{ reader: Generator<string>, headerOut: object, data: string }|null}
+     *   The matched document with its reader and shared headerOut, or null if no such document exists.
+     */
+    findDocument(sequenceNumber) {
+        const last = this.readLast();
+        if (!last || last.header.sequenceNumber < sequenceNumber) {
+            return null;
+        }
+
+        let startPosition = this.size;
+        binarySearch(
+            sequenceNumber,
+            this.size,
+            (pos) => {
+                const doc = this.readDocumentBefore(pos);
+                if (!doc) return sequenceNumber;
+                if (doc.header.sequenceNumber < sequenceNumber) {
+                    startPosition = Math.max(startPosition, doc.position + this.documentWriteSize(doc.header.dataSize));
+                } else {
+                    startPosition = Math.min(startPosition, doc.position);
+                }
+                return doc.header.sequenceNumber;
+            }
+        );
+
+        const headerOut = {};
+        const data = this.readFrom(startPosition, 0, headerOut);
+        /* istanbul ignore if */
+        if (data === false) {
+            return null;
+        }
+        headerOut.position = startPosition;
+        return { headerOut, data };
+    }
+
+    /**
+     * @api
+     * @param {number} [after] The document position to start reading from.
+     * @param {object|null} [headerOut] Optional object to populate with document header fields
+     *   (`dataSize`, `sequenceNumber`, `time64`, `position`) on each yield. Pass an existing object
+     *   to avoid extra allocation. The object is mutated in place before each yield.
+     * @returns {Generator<string>} A generator that returns all documents in this partition.
+     */
+    *readAll(after = 0, headerOut = null) {
+        let position = after < 0 ? this.size + after + 1 : after;
+        const internalHeader = headerOut !== null ? headerOut : {};
         let data;
-        while ((data = this.readFrom(position)) !== false) {
+        while ((data = this.readFrom(position, 0, internalHeader)) !== false) {
+            if (headerOut !== null) {
+                headerOut.position = position;
+            }
             yield data;
-            position += this.documentWriteSize(Buffer.byteLength(data, 'utf8'));
+            position += this.documentWriteSize(internalHeader.dataSize);
         }
     }
 
+    /**
+     * @api
+     * @param {number} [before] The document position to start reading backward from.
+     * @returns {Generator<string>} A generator that returns all documents in this partition in reverse order.
+     */
+    *readAllBackwards(before = -1) {
+        let position = before < 0 ? this.size + before + 1 : before;
+        while ((position = this.findDocumentPositionBefore(position)) !== false) {
+            const data = this.readFrom(position);
+            yield data;
+        }
+    }
 }
 
 module.exports = ReadablePartition;
 module.exports.CorruptFileError = CorruptFileError;
 module.exports.InvalidDataSizeError = InvalidDataSizeError;
-module.exports.HEADER_MAGIC = HEADER_MAGIC;
+module.exports.HEADER_MAGIC = HEADER_MAGIC;
+module.exports.DOCUMENT_SEPARATOR = DOCUMENT_SEPARATOR;
+module.exports.DOCUMENT_ALIGNMENT = DOCUMENT_ALIGNMENT;
+module.exports.DOCUMENT_HEADER_SIZE = DOCUMENT_HEADER_SIZE;
+module.exports.DOCUMENT_FOOTER_SIZE = DOCUMENT_FOOTER_SIZE;

package/src/Partition/WritablePartition.js

@@ -1,24 +1,11 @@
 const fs = require('fs');
-const mkdirpSync = require('mkdirp').sync;
 const ReadablePartition = require('./ReadablePartition');
-const { assert, buildMetadataHeader } = require('../util');
+const { assert, buildMetadataHeader, alignTo, ensureDirectory } = require('../util');
 const Clock = require('../Clock');
 
 const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024;
-const DOCUMENT_HEADER_SIZE = 16;
-const DOCUMENT_ALIGNMENT = 4;
-const DOCUMENT_PAD = ' '.repeat(15) + "\n";
-
-const NES_EPOCH = new Date('2020-01-01T00:00:00');
-
-/**
- * @param {number} dataSize
- * @returns {string} The data padded to 16 bytes alignment and ended with a line break.
- */
-function padData(dataSize) {
-    const padSize = (DOCUMENT_ALIGNMENT - ((dataSize + 1) % DOCUMENT_ALIGNMENT)) % DOCUMENT_ALIGNMENT;
-    return DOCUMENT_PAD.substr(-padSize - 1);
-}
+const { DOCUMENT_ALIGNMENT, DOCUMENT_SEPARATOR, DOCUMENT_HEADER_SIZE, DOCUMENT_FOOTER_SIZE } = ReadablePartition;
+const DOCUMENT_PAD = ' '.repeat(DOCUMENT_ALIGNMENT);
 
 /**
  * A partition is a single file where the storage will write documents to depending on some partitioning rules.
@@ -49,11 +36,10 @@ class WritablePartition extends ReadablePartition {
             },
             clock: Clock
         };
+        config.metadata = Object.assign(defaults.metadata, config.metadata);
         config = Object.assign(defaults, config);
         super(name, config);
-        if (!fs.existsSync(this.dataDirectory)) {
-            mkdirpSync(this.dataDirectory);
-        }
+        ensureDirectory(this.dataDirectory);
         this.fileMode = 'a+';
         this.writeBufferSize = config.writeBufferSize >>> 0; // jshint ignore:line
         this.maxWriteBufferDocuments = config.maxWriteBufferDocuments >>> 0; // jshint ignore:line
@@ -75,25 +61,28 @@ class WritablePartition extends ReadablePartition {
             return true;
         }
 
-        this.writeBuffer = Buffer.allocUnsafeSlow(this.writeBufferSize);
+        let success = super.open();
+        if (!success) {
+            if (this.size + this.headerSize !== 0) {
+                // If file is not empty, we can not open and initialize it
+                return false;
+            }
+            this.writeMetadata();
+            success = super.open();
+        }
+
+        this.writeBuffer = success && Buffer.allocUnsafeSlow(this.writeBufferSize);
         // Where inside the write buffer the next write is added
         this.writeBufferCursor = 0;
         // How many documents are currently in the write buffer
         this.writeBufferDocuments = 0;
         this.flushCallbacks = [];
+        // Pre-allocated buffer for document header (16 bytes) + size footer (4 bytes) used in writeUnbuffered
+        this.writeMetaBuffer = Buffer.allocUnsafe(DOCUMENT_HEADER_SIZE + 4);
 
-        if (super.open() === false) {
-            const stat = fs.statSync(this.fileName);
-            if (stat.size !== 0) {
-                return false;
-            }
-            this.metadata.epoch = Date.now();
-            this.writeMetadata();
-            this.size = 0;
-        }
-        this.clock = new this.ClockConstructor(this.metadata.epoch || NES_EPOCH.getTime());
+        this.clock = new this.ClockConstructor(this.metadata.epoch);
 
-        return true;
+        return success;
     }
 
     /**
@@ -103,14 +92,14 @@ class WritablePartition extends ReadablePartition {
      * @returns void
      */
     close() {
-        if (this.fd) {
+        if (this.fd && this.writeBuffer) {
             this.flush();
             fs.fsyncSync(this.fd);
-        }
-        if (this.writeBuffer) {
+
             this.writeBuffer = null;
             this.writeBufferCursor = 0;
             this.writeBufferDocuments = 0;
+            this.writeMetaBuffer = null;
         }
         super.close();
     }
@@ -123,7 +112,7 @@ class WritablePartition extends ReadablePartition {
      */
     writeMetadata() {
         const metadataBuffer = buildMetadataHeader(ReadablePartition.HEADER_MAGIC, this.metadata);
-        fs.writeSync(this.fd, metadataBuffer, 0, metadataBuffer.byteLength, 0);
+        fs.writeFileSync(this.fileName, metadataBuffer);
         this.headerSize = metadataBuffer.byteLength;
     }
 
@@ -148,8 +137,9 @@ class WritablePartition extends ReadablePartition {
 
         this.writeBufferCursor = 0;
         this.writeBufferDocuments = 0;
-        this.flushCallbacks.forEach(callback => callback());
+        const callbacks = this.flushCallbacks;
         this.flushCallbacks = [];
+        for (let i = 0; i < callbacks.length; i++) callbacks[i]();
 
         return true;
     }
@@ -192,6 +182,7 @@ class WritablePartition extends ReadablePartition {
         if (time64 === null) {
             time64 = this.clock.time();
         }
+        /* istanbul ignore if */
         if (time64 < 0) {
             throw new Error('Time may not be negative!');
         }
@@ -212,12 +203,16 @@ class WritablePartition extends ReadablePartition {
      */
     writeUnbuffered(data, dataSize, sequenceNumber, callback) {
         this.flush();
-        const dataHeader = Buffer.alloc(DOCUMENT_HEADER_SIZE);
-        this.writeDocumentHeader(dataHeader, 0, dataSize, sequenceNumber);
+        this.writeDocumentHeader(this.writeMetaBuffer, 0, dataSize, sequenceNumber);
 
         let bytesWritten = 0;
-        bytesWritten += fs.writeSync(this.fd, dataHeader);
+        bytesWritten += fs.writeSync(this.fd, this.writeMetaBuffer, 0, DOCUMENT_HEADER_SIZE);
         bytesWritten += fs.writeSync(this.fd, data);
+        const padSize = alignTo(dataSize + DOCUMENT_FOOTER_SIZE, DOCUMENT_ALIGNMENT);
+        bytesWritten += fs.writeSync(this.fd, DOCUMENT_PAD.substr(0, padSize));
+        this.writeMetaBuffer.writeUInt32BE(dataSize, 0);
+        bytesWritten += fs.writeSync(this.fd, this.writeMetaBuffer, 0, 4);
+        bytesWritten += fs.writeSync(this.fd, DOCUMENT_SEPARATOR);
         if (typeof callback === 'function') {
             process.nextTick(callback);
         }
@@ -234,12 +229,17 @@ class WritablePartition extends ReadablePartition {
      * @returns {number} Number of bytes written.
      */
     writeBuffered(data, dataSize, sequenceNumber, callback) {
-        const bytesToWrite = Buffer.byteLength(data, 'utf8') + DOCUMENT_HEADER_SIZE;
+        const bytesToWrite = this.documentWriteSize(dataSize);
         this.flushIfWriteBufferTooSmall(bytesToWrite);
 
         let bytesWritten = 0;
         bytesWritten += this.writeDocumentHeader(this.writeBuffer, this.writeBufferCursor, dataSize, sequenceNumber);
         bytesWritten += this.writeBuffer.write(data, this.writeBufferCursor + bytesWritten, 'utf8');
+        const padSize = alignTo(dataSize + DOCUMENT_FOOTER_SIZE, DOCUMENT_ALIGNMENT);
+        bytesWritten += this.writeBuffer.write(DOCUMENT_PAD.substr(0, padSize), this.writeBufferCursor + bytesWritten, 'utf8');
+        this.writeBuffer.writeUInt32BE(dataSize, this.writeBufferCursor + bytesWritten);
+        bytesWritten += 4;
+        bytesWritten += this.writeBuffer.write(DOCUMENT_SEPARATOR, this.writeBufferCursor + bytesWritten, 'utf8');
         this.writeBufferCursor += bytesWritten;
         this.writeBufferDocuments++;
         if (typeof callback === 'function') {
@@ -257,9 +257,7 @@ class WritablePartition extends ReadablePartition {
      * @returns {number|boolean} The file position at which the data was written or false on error.
      */
     write(data, sequenceNumber, callback) {
-        if (!this.fd) {
-            return false;
-        }
+        assert(this.fd, 'Partition is not opened.');
         if (typeof sequenceNumber === 'function') {
             callback = sequenceNumber;
             sequenceNumber = null;
@@ -267,7 +265,6 @@ class WritablePartition extends ReadablePartition {
         const dataSize = Buffer.byteLength(data, 'utf8');
         assert(dataSize <= 64 * 1024 * 1024, 'Document is too large! Maximum is 64 MB');
 
-        data += padData(dataSize);
         const dataPosition = this.size;
         if (dataSize + DOCUMENT_HEADER_SIZE >= this.writeBuffer.byteLength * 4 / 5) {
             this.size += this.writeUnbuffered(data, dataSize, sequenceNumber, callback);
@@ -298,7 +295,42 @@ class WritablePartition extends ReadablePartition {
     }
 
     /**
-     * Truncate the internal read buffer after the given position.
+     * Prepare the read buffer for reading *before* the specified position. Don't try to read *after* the returned cursor.
+     *
+     * @protected
+     * @param {number} position The position in the file to prepare the read buffer for reading before.
+     * @returns {object} A reader object with properties `buffer`, `cursor` and `length`.
+     */
+    prepareReadBufferBackwards(position) {
+        const bufferPos = this.size - this.writeBufferCursor;
+        // Handle the case when data that is still in write buffer is supposed to be read backwards
+        if (this.dirtyReads && this.writeBufferCursor > 0 && position > bufferPos) {
+            return { buffer: this.writeBuffer, cursor: position - bufferPos, length: this.writeBufferCursor };
+        }
+        return super.prepareReadBufferBackwards(position);
+    }
+
+    /**
+     * Read all documents in reverse write order, ignoring any unflushed write-buffer data when
+     * dirty reads are disabled.
+     *
+     * @api
+     * @param {number} [before] The document position to start reading backward from.
+     * @returns {Generator<string>} A generator that returns all documents in this partition in reverse order.
+     */
+    *readAllBackwards(before = -1) {
+        if (!this.dirtyReads && this.writeBufferCursor > 0) {
+            const flushedSize = this.size - this.writeBufferCursor;
+            const clampedBefore = before < 0 ? flushedSize : Math.min(before, flushedSize);
+            yield* super.readAllBackwards(clampedBefore);
+            return;
+        }
+        yield* super.readAllBackwards(before);
+    }
+
+    /**
+     *
+     * @internal
      * @param {number} after The byte position to truncate the read buffer after.
      */
     truncateReadBuffer(after) {
@@ -310,39 +342,80 @@ class WritablePartition extends ReadablePartition {
         }
     }
 
+    /**
+     * Truncate the partition, removing all documents with sequenceNumber > after.
+     * Scans the partition file backwards to find the cutoff position.
+     *
+     * @api
+     * @param {number} after Keep all documents with sequenceNumber <= after. Documents with
+     *   sequenceNumber > after (and any torn write at the end) are removed.
+     */
+    truncateAfterSequence(after) {
+        let position = this.size;
+        let truncateAt = this.size; // default: nothing to truncate
+        while ((position = this.findDocumentPositionBefore(position)) !== false) {
+            const reader = this.prepareReadBufferBackwards(position);
+            if (!reader.buffer) break;
+            const { sequenceNumber } = this.readDocumentHeader(reader.buffer, reader.cursor, position);
+            if (sequenceNumber > after) {
+                // This document must be removed; record its start as the new cutoff.
+                truncateAt = position;
+            } else {
+                break;
+            }
+        }
+        this.truncate(truncateAt);
+    }
+
     /**
      * Truncate the partition storage at the given position.
      *
+     * @api
      * @param {number} after The file position after which to truncate the partition.
      */
     truncate(after) {
-        if (after > this.size) {
+        if (after >= this.size) {
             return;
         }
-        if (after < 0) {
-            after = 0;
-        }
+        this.open();
+        after = Math.max(0, after);
         this.flush();
 
-        let position = after, data;
+        // Always save the truncated part for manual recovery, even if it contains corrupted data
+        this.branchOff('truncated-' + Date.now(), after);
+
         try {
-            data = this.readFrom(position);
+            this.readFrom(after);
         } catch (e) {
-            throw new Error('Can only truncate on valid document boundaries.');
-        }
-        // copy all truncated documents to some delete log
-        const deletedBranch = new WritablePartition(this.name + '-' + after + '.branch', { dataDirectory: this.dataDirectory });
-        deletedBranch.open();
-        while (data) {
-            deletedBranch.write(data);
-            position += this.documentWriteSize(Buffer.byteLength(data, 'utf8'));
-            data = this.readFrom(position);
+            if (!(e instanceof ReadablePartition.CorruptFileError)) {
+                throw new Error('Can only truncate on valid document boundaries.');
+            }
         }
-        deletedBranch.close();
 
         fs.truncateSync(this.fileName, this.headerSize + after);
         this.truncateReadBuffer(after);
         this.size = after;
+        this.emit('truncated', after);
+    }
+
+    /**
+     * Create a branch of this partition starting from the given position.
+     *
+     * @internal
+     * @param {string} branchName The name that identifies the branch (will be prefixed with this partition name and suffixed with the position)
+     * @param {number} position The file position from where to branch off
+     * @returns {WritablePartition} The branched off partition
+     */
+    branchOff(branchName, position) {
+        const deletedBranch = new WritablePartition(this.name + '-' + branchName + '-' + position + '.branch', { dataDirectory: this.dataDirectory, metadata: { epoch: this.metadata.epoch } });
+        deletedBranch.open();
+        do {
+            const reader = this.prepareReadBuffer(position);
+            fs.writeSync(deletedBranch.fd, reader.buffer, reader.cursor, reader.length);
+            position += reader.length;
+        } while (position < this.size);
+        deletedBranch.close();
+        return deletedBranch;
     }
 }