event-storage 0.8.0 → 0.9.1

package/src/EventStream.js

@@ -2,16 +2,14 @@ const stream = require('stream');
 const { assert } = require('./util');
 
 /**
- * Adjusts a revision number from the EventStore/EventStream interface range into the underlying storage item number.
+ * Calculate the actual version number from a possibly relative (negative) version number.
  *
- * @param {number} rev A zero-based revision number, or a negative number to denote a "from the end" position
- * @returns {number} A one-based storage item number
+ * @param {number} version The version to normalize.
+ * @param {number} length The maximum version number
+ * @returns {number} The absolute version number.
  */
-function adjustedRevision(rev) {
-    if (rev >= 0) {
-        return rev + 1;
-    }
-    return rev;
+function normalizeVersion(version, length) {
+    return version < 0 ? version + length + 1 : version;
 }
 
 /**
@@ -36,22 +34,163 @@ class EventStream extends stream.Readable {
      * @param {number} [minRevision] The minimum revision to include in the events (inclusive).
      * @param {number} [maxRevision] The maximum revision to include in the events (inclusive).
      */
-    constructor(name, eventStore, minRevision = 0, maxRevision = -1) {
+    constructor(name, eventStore, minRevision = 1, maxRevision = -1) {
         super({ objectMode: true });
         assert(typeof name === 'string' && name !== '', 'Need to specify a stream name.');
         assert(typeof eventStore === 'object' && eventStore !== null, `Need to provide EventStore instance to create EventStream ${name}.`);
 
         this.name = name;
         if (eventStore.streams[name]) {
-            const streamIndex = eventStore.streams[name].index;
-            this.version = minVersion(streamIndex.length, maxRevision);
-            minRevision = adjustedRevision(minRevision);
-            maxRevision = adjustedRevision(maxRevision);
-            this.iterator = eventStore.storage.readRange(minRevision, maxRevision, streamIndex);
+            this.streamIndex = eventStore.streams[name].index;
+            this.minRevision = normalizeVersion(minRevision, this.streamIndex.length);
+            this.maxRevision = normalizeVersion(maxRevision, this.streamIndex.length);
+            this.version = minVersion(this.streamIndex.length, maxRevision);
+            this._iterator = null;
+            this.fetch = function() {
+                return eventStore.storage.readRange(this.minRevision, this.maxRevision, this.streamIndex);
+            }
         } else {
+            this.streamIndex = { length: 0 };
             this.version = -1;
-            this.iterator = { next() { return { done: true }; } };
+            this._iterator = { next() { return { done: true }; } };
+        }
+    }
+
+    /**
+     * @api
+     * @param {number} revision The event revision to start reading from (inclusive).
+     * @returns {EventStream}
+     */
+    from(revision) {
+        this.minRevision = normalizeVersion(revision, this.streamIndex.length);
+        return this;
+    }
+
+    /**
+     * @api
+     * @param {number} revision The event revision to read until (inclusive).
+     * @returns {EventStream}
+     */
+    until(revision) {
+        this.maxRevision = normalizeVersion(revision, this.streamIndex.length);
+        this.version = minVersion(this.streamIndex.length, this.maxRevision);
+        return this;
+    }
+
+    /**
+     * @api
+     * @param {number} amount The amount of events at the start of the stream to return in chronological order.
+     * @returns {EventStream}
+     */
+    first(amount) {
+        return this.fromStart().following(amount);
+    }
+
+    /**
+     * @api
+     * @param {number} amount The amount of events at the end of the stream to return in chronological order.
+     * @returns {EventStream}
+     */
+    last(amount) {
+        return this.fromEnd().previous(amount).forwards();
+    }
+
+    /**
+     * @api
+     * @returns {EventStream}
+     */
+    fromStart() {
+        this.minRevision = 1;
+        return this;
+    }
+
+    /**
+     * @api
+     * @returns {EventStream}
+     */
+    fromEnd() {
+        this.minRevision = this.streamIndex.length;
+        return this;
+    }
+
+    /**
+     * @param {number} amount The amount of events to return in reverse chronological order.
+     * @returns {EventStream}
+     */
+    previous(amount) {
+        this.maxRevision = Math.max(1, this.minRevision - amount + 1);
+        return this;
+    }
+
+    /**
+     * @param {number} amount The amount of events to return in chronological order.
+     * @returns {EventStream}
+     */
+    following(amount) {
+        this.maxRevision = Math.min(this.streamIndex.length, this.minRevision + amount - 1);
+        return this;
+    }
+
+    /**
+     * @api
+     * @returns {EventStream}
+     */
+    toEnd() {
+        this.maxRevision = this.version = this.streamIndex.length;
+        return this;
+    }
+
+    /**
+     * @api
+     * @returns {EventStream}
+     */
+    toStart() {
+        this.maxRevision = 1;
+        return this;
+    }
+
+    /**
+     * Reverse the current range of events, no matter which direction it currently has.
+     * @returns {EventStream}
+     */
+    reverse() {
+        let tmp = this.maxRevision;
+        this.maxRevision = this.minRevision;
+        this.minRevision = tmp;
+        this.version = minVersion(this.streamIndex.length, this.maxRevision);
+        return this;
+    }
+
+    /**
+     * Make the current range of events read in forward chronological order.
+     * @api
+     * @param {number} [amount] Amount of events to read forward. If not specified, will read forward until the previously set limit.
+     * @returns {EventStream}
+     */
+    forwards(amount = 0) {
+        if (amount > 0) {
+            this.following(amount);
+        }
+        if (this.maxRevision < this.minRevision) {
+            this.reverse();
         }
+        return this;
+    }
+
+    /**
+     * Make the current range of events read in backward chronological order.
+     * @api
+     * @param {number} [amount] Amount of events to read backward. If not specified, will read backward until the previously set limit.
+     * @returns {EventStream}
+     */
+    backwards(amount = 0) {
+        if (amount > 0) {
+            this.previous(amount);
+        }
+        if (this.maxRevision > this.minRevision) {
+            this.reverse();
+        }
+        return this;
     }
 
     /**
@@ -75,6 +214,7 @@ class EventStream extends stream.Readable {
      * Iterate over the events in this stream with a callback.
      * This method is useful to gain access to the event metadata.
      *
+     * @api
      * @param {function(object, object, string)} callback A callback function that will receive the event, the storage metadata and the original stream name for every event in this stream.
      */
     forEach(callback) {
@@ -94,13 +234,26 @@ class EventStream extends stream.Readable {
         }
     }
 
+    /**
+     * Reset this stream to the start so it can be iterated again.
+     * @returns {EventStream}
+     */
+    reset() {
+        this._iterator = null;
+        this._events = null;
+        return this;
+    }
+
     /**
      * @returns {object|boolean} The next event or false if no more events in the stream.
      */
     next() {
+        if (!this._iterator) {
+            this._iterator = this.fetch();
+        }
         let next;
         try {
-            next = this.iterator.next();
+            next = this._iterator.next();
         } catch(e) {
             return false;
         }

package/src/Index/ReadableIndex.js

@@ -72,7 +72,7 @@ class ReadableIndex extends events.EventEmitter {
         this.EntryClass = options.EntryClass;
         this.dataDirectory = options.dataDirectory;
         this.fileName = path.resolve(options.dataDirectory, this.name);
-        this.readBuffer = Buffer.allocUnsafe(options.EntryClass.size);
+        this.readBuffer = Buffer.allocUnsafe(Math.max(options.EntryClass.size, options.writeBufferSize > 0 ? options.writeBufferSize : 4096));
 
         if (options.metadata) {
             this.metadata = Object.assign({entryClass: options.EntryClass.name, entrySize: options.EntryClass.size}, options.metadata);
@@ -281,8 +281,11 @@ class ReadableIndex extends events.EventEmitter {
         const readFrom = Math.max(this.readUntil + 1, from);
         const amount = (until - readFrom + 1);
 
-        const readBuffer = Buffer.allocUnsafe(amount * this.EntryClass.size);
-        let readSize = fs.readSync(this.fd, readBuffer, 0, readBuffer.byteLength, this.headerSize + readFrom * this.EntryClass.size);
+        const bufferSize = amount * this.EntryClass.size;
+        const readBuffer = bufferSize <= this.readBuffer.byteLength
+            ? this.readBuffer
+            : Buffer.allocUnsafe(bufferSize);
+        let readSize = fs.readSync(this.fd, readBuffer, 0, bufferSize, this.headerSize + readFrom * this.EntryClass.size);
         let index = 0;
         while (index < amount && readSize > 0) {
             this.data[index + readFrom] = this.EntryClass.fromBuffer(readBuffer, index * this.EntryClass.size);
@@ -388,6 +391,9 @@ class ReadableIndex extends events.EventEmitter {
      * @returns {number} The last index entry position that is lower than or equal to the `number`. Returns 0 if no index matches.
      */
     find(number, min = false) {
+        if (this.length < 1) {
+            return 0;
+        }
         // We only need to search until the searched number because entry.number is always >= position
         const [low, high] = binarySearch(number, Math.min(this.length, number), index => this.get(index).number);
         return min ? low : high;

package/src/Index/WritableIndex.js

@@ -156,8 +156,9 @@ class WritableIndex extends ReadableIndex {
         }
 
         this.writeBufferCursor = 0;
-        this.flushCallbacks.forEach(callback => callback());
+        const callbacks = this.flushCallbacks;
         this.flushCallbacks = [];
+        for (let i = 0; i < callbacks.length; i += 2) callbacks[i](callbacks[i + 1]);
         return true;
     }
 
@@ -172,7 +173,7 @@ class WritableIndex extends ReadableIndex {
         if (typeof callback !== 'function') {
             return;
         }
-        this.flushCallbacks.push(() => callback(position));
+        this.flushCallbacks.push(callback, position);
     }
 
     /**
@@ -187,10 +188,15 @@ class WritableIndex extends ReadableIndex {
         assertEqual(entry.constructor.name, this.EntryClass.name, `Wrong entry object.`);
         assertEqual(entry.constructor.size, this.EntryClass.size, `Invalid entry size.`);
 
-        if (this.readUntil === this.data.length - 1) {
+        const dataLen = this.data.length;
+        if (dataLen > 0 && this.data[dataLen - 1].number >= entry.number) {
+            throw new Error('Consistency error. Tried to add an index that should come before existing last entry.');
+        }
+
+        if (this.readUntil === dataLen - 1) {
             this.readUntil++;
         }
-        this.data[this.data.length] = entry;
+        this.data[dataLen] = entry;
 
         if (this.writeBufferCursor === 0) {
             this.flushTimeout = setTimeout(() => this.flush(), this.flushDelay);

package/src/JoinEventStream.js

@@ -1,18 +1,15 @@
 const EventStream = require('./EventStream');
+const { wrapAndCheck } = require('./util');
 
 /**
- * Translates an EventStore revision number to an index sequence number and wraps it around the given length, if it's < 0
+ * Calculate the actual version number from a possibly relative (negative) version number.
  *
- * @param {number} rev The zero-based EventStore revision
- * @param {number} length The length of the store
- * @returns {number} The 1-based index sequence number
+ * @param {number} version The version to normalize.
+ * @param {number} length The maximum version number
+ * @returns {number} The absolute version number.
  */
-function wrapRevision(rev, length) {
-    rev++;
-    if (rev <= 0) {
-        rev += length;
-    }
-    return rev;
+function normalizeVersion(version, length) {
+    return version < 0 ? version + length + 1 : version;
 }
 
 /**
@@ -25,30 +22,32 @@ class JoinEventStream extends EventStream {
      * @param {string} name The name of the stream.
      * @param {Array<string>} streams The name of the streams to join together.
      * @param {EventStore} eventStore The event store to get the stream from.
-     * @param {number} [minRevision] The minimum revision to include in the events (inclusive).
-     * @param {number} [maxRevision] The maximum revision to include in the events (inclusive).
+     * @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
+     * @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
      */
-    constructor(name, streams, eventStore, minRevision = 0, maxRevision = -1) {
+    constructor(name, streams, eventStore, minRevision = 1, maxRevision = -1) {
         super(name, eventStore, minRevision, maxRevision);
         if (!(streams instanceof Array) || streams.length === 0) {
             throw new Error(`Invalid list of streams supplied to JoinStream ${name}.`);
         }
-        this._next = new Array(streams.length).fill(undefined);
 
+        this.streamIndex = eventStore.storage.index;
         // Translate revisions to index numbers (1-based) and wrap around negatives
-        minRevision = wrapRevision(minRevision, eventStore.length);
-        maxRevision = wrapRevision(maxRevision, eventStore.length);
-
-        this.reverse = minRevision > maxRevision;
-        this.iterator = streams.map(streamName => {
-            if (!eventStore.streams[streamName]) {
-                return { next() { return { done: true }; } };
-            }
-            const streamIndex = eventStore.streams[streamName].index;
-            const from = streamIndex.find(minRevision, !this.reverse);
-            const until = streamIndex.find(maxRevision, this.reverse);
-            return eventStore.storage.readRange(from || 1, until, streamIndex);
-        });
+        this.minRevision = normalizeVersion(minRevision, eventStore.length);
+        this.maxRevision = normalizeVersion(maxRevision, eventStore.length);
+        this.fetch = function() {
+            this._next = new Array(streams.length).fill(undefined);
+            return streams.map(streamName => {
+                if (!eventStore.streams[streamName]) {
+                    return { next() { return { done: true }; } };
+                }
+                const streamIndex = eventStore.streams[streamName].index;
+                const from = streamIndex.find(this.minRevision, this.minRevision <= this.maxRevision);
+                const until = streamIndex.find(this.maxRevision, this.minRevision > this.maxRevision);
+                return eventStore.storage.readRange(from, until, streamIndex);
+            });
+        }
+        this._iterator = null;
     }
 
     /**
@@ -57,7 +56,7 @@ class JoinEventStream extends EventStream {
      * @returns {*}
      */
     getValue(index) {
-        const next = this.iterator[index].next();
+        const next = this._iterator[index].next();
         return next.done ? false : next.value;
     }
 
@@ -65,10 +64,10 @@ class JoinEventStream extends EventStream {
      * @private
      * @param {number} first
      * @param {number} second
-     * @returns {boolean} If the first item follows after the second in the given read order determined by this.reverse flag.
+     * @returns {boolean} If the first item follows after the second in the given read order determined by this.minRevision and this.maxRevision.
      */
     follows(first, second) {
-        return (this.reverse ? first < second : first > second);
+        return (this.minRevision > this.maxRevision ? first < second : first > second);
     }
 
     /**
@@ -76,6 +75,9 @@ class JoinEventStream extends EventStream {
      * @returns {object|boolean} The next event or false if no more events in the stream.
      */
     next() {
+        if (!this._iterator) {
+            this._iterator = this.fetch();
+        }
         let nextIndex = -1;
         this._next.forEach((value, index) => {
             if (typeof value === 'undefined') {

package/src/Partition/ReadablePartition.js

@@ -1,7 +1,7 @@
 const fs = require('fs');
 const path = require('path');
 const events = require('events');
-const { assert, alignTo } = require('../util');
+const { assert, alignTo, hash, binarySearch } = require('../util');
 
 const DEFAULT_READ_BUFFER_SIZE = 64 * 1024;
 const DOCUMENT_HEADER_SIZE = 16;
@@ -17,30 +17,6 @@ const NES_EPOCH = new Date('2020-01-01T00:00:00');
 class CorruptFileError extends Error {}
 class InvalidDataSizeError extends Error {}
 
-/**
- * Method for hashing a string (partition name) to a 32-bit unsigned integer.
- *
- * @param {string} str
- * @returns {number}
- */
-function hash(str) {
-    /* istanbul ignore if */
-    if (str.length === 0) {
-        return 0;
-    }
-    let hash = 5381,
-        i    = str.length;
-
-    while(i) {
-        hash = ((hash << 5) + hash) ^ str.charCodeAt(--i); // jshint ignore:line
-    }
-
-    /* JavaScript does bitwise operations (like XOR, above) on 32-bit signed
-     * integers. Since we want the results to be always positive, convert the
-     * signed int to an unsigned by doing an unsigned bitshift. */
-    return hash >>> 0; // jshint ignore:line
-}
-
 /**
  * A partition is a single file where the storage will write documents to depending on some partitioning rules.
  * In the case of an event store, this is most likely the (write) streams.
@@ -123,21 +99,6 @@ class ReadablePartition extends events.EventEmitter {
         return true;
     }
 
-    /**
-     * @returns {number} -1 if the partition is ok and the sequence number of the broken document if a torn write was detected.
-     */
-    checkTornWrite() {
-        const reader = this.prepareReadBufferBackwards(this.size);
-        const separator = reader.buffer.toString('ascii', reader.cursor - DOCUMENT_SEPARATOR.length, reader.cursor);
-        if (separator !== DOCUMENT_SEPARATOR) {
-            const position = this.findDocumentPositionBefore(this.size);
-            const reader = this.prepareReadBuffer(position);
-            const { sequenceNumber } = this.readDocumentHeader(reader.buffer, reader.cursor, position);
-            return sequenceNumber;
-        }
-        return -1;
-    }
-
     /**
      * Read the partition metadata from the file.
      *
@@ -279,7 +240,7 @@ class ReadablePartition extends events.EventEmitter {
             return ({ buffer: null, cursor: 0, length: 0 });
         }
         let bufferCursor = position - this.readBufferPos;
-        if (this.readBufferPos < 0 || (this.readBufferPos > 0 && bufferCursor < DOCUMENT_FOOTER_SIZE)) {
+        if (this.readBufferPos < 0 || (this.readBufferPos > 0 && bufferCursor < DOCUMENT_FOOTER_SIZE) || bufferCursor > this.readBufferLength) {
             this.fillBuffer(Math.max(position - this.readBuffer.byteLength, 0));
             bufferCursor = position - this.readBufferPos;
         }
@@ -292,12 +253,14 @@ class ReadablePartition extends events.EventEmitter {
      * @api
      * @param {number} position The file position to read from.
      * @param {number} [size] The expected byte size of the document at the given position.
+     * @param {object|null} [headerOut] Optional object to populate with the document header fields
+     *   (`dataSize`, `sequenceNumber`, `time64`). Pass an existing object to avoid extra allocation.
      * @returns {string|boolean} The data stored at the given position or false if no data could be read.
      * @throws {Error} if the storage entry at the given position is corrupted.
      * @throws {InvalidDataSizeError} if the document size at the given position does not match the provided size.
      * @throws {CorruptFileError} if the document at the given position can not be read completely.
      */
-    readFrom(position, size = 0) {
+    readFrom(position, size = 0, headerOut = null) {
         assert(this.fd, 'Partition is not opened.');
         assert((position % DOCUMENT_ALIGNMENT) === 0, `Invalid read position ${position}. Needs to be a multiple of ${DOCUMENT_ALIGNMENT}.`);
 
@@ -307,7 +270,12 @@ class ReadablePartition extends events.EventEmitter {
         }
 
         let dataPosition = reader.cursor + DOCUMENT_HEADER_SIZE;
-        const { dataSize } = this.readDocumentHeader(reader.buffer, reader.cursor, position, size);
+        const { dataSize, sequenceNumber, time64 } = this.readDocumentHeader(reader.buffer, reader.cursor, position, size);
+        if (headerOut !== null) {
+            headerOut.dataSize = dataSize;
+            headerOut.sequenceNumber = sequenceNumber;
+            headerOut.time64 = time64;
+        }
 
         // TODO: This should only be checked on opening
         const writeSize = this.documentWriteSize(dataSize);
@@ -366,17 +334,108 @@ class ReadablePartition extends events.EventEmitter {
         return Math.max(0, position);
     }
 
+    /**
+     * Find the document that starts immediately before `position`, fill the read buffer
+     * centered around that document's start, and return its file position and parsed header.
+     * The buffer is centered by calling `prepareReadBufferBackwards` with a position
+     * half a buffer-length ahead of the document start, clamped to file size, so the
+     * document start lands near the middle of the buffer.
+     *
+     * @private
+     * @param {number} position The file position to search before.
+     * @returns {{ header: {dataSize: number, sequenceNumber: number, time64: number}, position: number }|null}
+     *   The document header and file position, or null if no document could be found.
+     */
+    readDocumentBefore(position) {
+        const docPos = this.findDocumentPositionBefore(position);
+        /* istanbul ignore if */
+        if (docPos === false || docPos < 0) return null;
+        const reader = this.prepareReadBufferBackwards(Math.min(docPos + (this.readBuffer.byteLength >> 1), this.size));
+        /* istanbul ignore if */
+        if (!reader.buffer) return null;
+        const cursor = docPos - this.readBufferPos;
+        /* istanbul ignore if */
+        if (cursor < 0 || cursor + DOCUMENT_HEADER_SIZE > reader.length) return null;
+        const header = this.readDocumentHeader(reader.buffer, cursor, docPos);
+        return { header, position: docPos };
+    }
+
+    /**
+     * Read the header and file position of the last document in this partition.
+     *
+     * @api
+     * @returns {{ header: {dataSize: number, sequenceNumber: number, time64: number}, position: number } | null}
+     *   The last document's header and its file position, or null if the partition is empty or unreadable.
+     */
+    readLast() {
+        if (this.size === 0) return null;
+        return this.readDocumentBefore(this.size);
+    }
+
+    /**
+     * Find the first document whose sequenceNumber is >= the given value.
+     * Uses readLast() to short-circuit when the partition contains no such document.
+     * Uses a binary search over file positions via readDocumentBefore() to locate the
+     * document. The search tracks both the lower bound (position just after the last
+     * confirmed "< sequenceNumber" doc) and the upper bound (minimum position of any
+     * probed doc with sequenceNumber >= target). The upper bound, when available, is
+     * the exact target document, so no further linear scan is needed.
+     *
+     * @api
+     * @param {number} sequenceNumber The 0-based sequence number to search for.
+     * @returns {{ reader: Generator<string>, headerOut: object, data: string }|null}
+     *   The matched document with its reader and shared headerOut, or null if no such document exists.
+     */
+    findDocument(sequenceNumber) {
+        const last = this.readLast();
+        if (!last || last.header.sequenceNumber < sequenceNumber) {
+            return null;
+        }
+
+        let startPosition = this.size;
+        binarySearch(
+            sequenceNumber,
+            this.size,
+            (pos) => {
+                const doc = this.readDocumentBefore(pos);
+                if (!doc) return sequenceNumber;
+                if (doc.header.sequenceNumber < sequenceNumber) {
+                    startPosition = Math.max(startPosition, doc.position + this.documentWriteSize(doc.header.dataSize));
+                } else {
+                    startPosition = Math.min(startPosition, doc.position);
+                }
+                return doc.header.sequenceNumber;
+            }
+        );
+
+        const headerOut = {};
+        const data = this.readFrom(startPosition, 0, headerOut);
+        /* istanbul ignore if */
+        if (data === false) {
+            return null;
+        }
+        headerOut.position = startPosition;
+        return { headerOut, data };
+    }
+
     /**
      * @api
      * @param {number} [after] The document position to start reading from.
+     * @param {object|null} [headerOut] Optional object to populate with document header fields
+     *   (`dataSize`, `sequenceNumber`, `time64`, `position`) on each yield. Pass an existing object
+     *   to avoid extra allocation. The object is mutated in place before each yield.
      * @returns {Generator<string>} A generator that returns all documents in this partition.
      */
-    *readAll(after = 0) {
+    *readAll(after = 0, headerOut = null) {
         let position = after < 0 ? this.size + after + 1 : after;
+        const internalHeader = headerOut !== null ? headerOut : {};
         let data;
-        while ((data = this.readFrom(position)) !== false) {
+        while ((data = this.readFrom(position, 0, internalHeader)) !== false) {
+            if (headerOut !== null) {
+                headerOut.position = position;
+            }
             yield data;
-            position += this.documentWriteSize(Buffer.byteLength(data, 'utf8'));
+            position += this.documentWriteSize(internalHeader.dataSize);
         }
     }