event-storage 1.1.0 → 1.2.0

package/src/EventStream.js

@@ -1,5 +1,8 @@
 import stream from 'stream';
-import { assert } from './util.js';
+import { assert } from './utils/util.js';
+import { buildRawBufferMatcher, matches } from './utils/metadataUtil.js';
+
+const NDJSON_NEWLINE = Buffer.from('\n');
 
 /**
  * Calculate the actual version number from a possibly relative (negative) version number.
@@ -33,25 +36,31 @@ class EventStream extends stream.Readable {
      * @param {EventStore} eventStore The event store to get the stream from.
      * @param {number} [minRevision] The minimum revision to include in the events (inclusive).
      * @param {number} [maxRevision] The maximum revision to include in the events (inclusive).
-     * @param {function(object, object): boolean|null} [predicate] An optional filter function
-     *   `(payload, metadata) => boolean`.  Only events for which this returns truthy are yielded.
+     * @param {function|object|null} [predicate] Optional matcher:
+     *   - object mode: function `(payload, metadata) => boolean` or object matcher against `{ stream, payload, metadata }`
+     *   - raw mode: function `(buffer) => boolean` or object matcher against compact NDJSON bytes.
+     * @param {boolean} [raw=false] If true, emit NDJSON Buffers instead of event payload objects.
      */
-    constructor(name, eventStore, minRevision = 1, maxRevision = -1, predicate = null) {
-        super({ objectMode: true });
+    constructor(name, eventStore, minRevision = 1, maxRevision = -1, predicate = null, raw = false) {
+        if (typeof predicate === 'boolean' && raw === false) {
+            raw = predicate;
+            predicate = null;
+        }
+        super({ objectMode: !raw });
         assert(typeof name === 'string' && name !== '', 'Need to specify a stream name.');
         assert(typeof eventStore === 'object' && eventStore !== null, `Need to provide EventStore instance to create EventStream ${name}.`);
 
         this.name = name;
+        this.raw = raw;
         this.predicate = predicate || null;
+        this.rawMatcher = null;
         if (eventStore.streams[name]) {
             this.streamIndex = eventStore.streams[name].index;
             this.minRevision = normalizeVersion(minRevision, this.streamIndex.length);
             this.maxRevision = normalizeVersion(maxRevision, this.streamIndex.length);
             this.version = minVersion(this.streamIndex.length, maxRevision);
             this._iterator = null;
-            this.fetch = function() {
-                return eventStore.storage.readRange(this.minRevision, this.maxRevision, this.streamIndex);
-            }
+            this.fetch = () => eventStore.storage.readRange(this.minRevision, this.maxRevision, this.streamIndex, raw);
         } else {
             this.streamIndex = { length: 0 };
             this.version = -1;
@@ -233,10 +242,18 @@ class EventStream extends stream.Readable {
     *[Symbol.iterator]() {
         let next;
         while ((next = this.next()) !== false) {
-            yield next.payload;
+            yield this.raw ? this.toRawBuffer(next) : next.payload;
         }
     }
 
+    /**
+     * @param {{ buffer: Buffer }} entry
+     * @returns {Buffer}
+     */
+    toRawBuffer(entry) {
+        return Buffer.concat([entry.buffer, NDJSON_NEWLINE]);
+    }
+
     /**
      * Reset this stream to the start so it can be iterated again.
      * @returns {EventStream}
@@ -259,11 +276,32 @@ class EventStream extends stream.Readable {
      */
     filter(predicate) {
         this.predicate = predicate || null;
+        this.rawMatcher = null;
         this._iterator = null;
         this._events = null;
         return this;
     }
 
+    matchesPredicate(entry) {
+        if (!this.predicate) {
+            return true;
+        }
+        if (this.raw) {
+            if (typeof this.predicate === 'function') {
+                return this.predicate(entry.buffer);
+            }
+            if (!this.rawMatcher) {
+                this.rawMatcher = buildRawBufferMatcher(this.predicate);
+            }
+            return this.rawMatcher(entry.buffer);
+        }
+
+        if (typeof this.predicate === 'function') {
+            return this.predicate(entry.payload, entry.metadata);
+        }
+        return matches(entry, this.predicate);
+    }
+
     /**
      * @returns {object|boolean} The next event or false if no more events in the stream.
      */
@@ -275,7 +313,7 @@ class EventStream extends stream.Readable {
             while (true) {
                 const result = this._iterator.next();
                 if (result.done) return false;
-                if (!this.predicate || this.predicate(result.value.payload, result.value.metadata)) {
+                if (this.matchesPredicate(result.value)) {
                     return result.value;
                 }
             }
@@ -291,7 +329,7 @@ class EventStream extends stream.Readable {
      */
     _read() {
         const next = this.next();
-        this.push(next ? next.payload : null);
+        this.push(next ? (this.raw ? this.toRawBuffer(next) : next.payload) : null);
     }
 
 }

package/src/Index/ReadableIndex.js

@@ -2,7 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import events from 'events';
 import Entry, { assertValidEntryClass } from '../IndexEntry.js';
-import { assert, wrapAndCheck, binarySearch } from '../util.js';
+import { assert, wrapAndCheck, binarySearch } from '../utils/util.js';
 
 // node-event-store-index V01
 const HEADER_MAGIC = "nesidx01";
@@ -77,6 +77,7 @@ class ReadableIndex extends events.EventEmitter {
         if (options.metadata) {
             this.metadata = Object.assign({entryClass: options.EntryClass.name, entrySize: options.EntryClass.size}, options.metadata);
         }
+        this.headerSize = 0;
     }
 
     /**
@@ -208,12 +209,13 @@ class ReadableIndex extends events.EventEmitter {
      * @throws {Error} if the metadata size in the header is invalid.
      */
     readMetadata() {
+        if (this.headerSize > 0) return this.headerSize;
         const headerBuffer = Buffer.allocUnsafe(8 + 4);
         fs.readSync(this.fd, headerBuffer, 0, 8 + 4, 0);
         const headerMagic = headerBuffer.toString('utf8', 0, 8);
 
-        assert(headerMagic.substr(0, 6) === HEADER_MAGIC.substr(0, 6), 'Invalid file header.');
-        assert(headerMagic === HEADER_MAGIC, `Invalid file version. The index ${this.fileName} was created with a different library version (${headerMagic.substr(6)}).`);
+        assert(headerMagic.substring(0, 6) === HEADER_MAGIC.substring(0, 6), 'Invalid file header.');
+        assert(headerMagic === HEADER_MAGIC, `Invalid file version. The index ${this.fileName} was created with a different library version (${headerMagic.substring(6)}).`);
 
         const metadataSize = headerBuffer.readUInt32BE(8);
         assert(metadataSize >= 3, 'Invalid metadata size.');
@@ -353,7 +355,7 @@ class ReadableIndex extends events.EventEmitter {
      *
      * @api
      * @param {number} from The 1-based index position from where to get entries from (inclusive). If < 0 will start at that position from end.
-     * @param {number} [until] The 1-based index position until where to get entries to (inclusive). If < 0 will get until that position from the end. Defaults to this.length.
+     * @param {number} [until=-1] The 1-based index position until where to get entries to (inclusive). If < 0 will get until that position from the end. Defaults to this.length.
      * @returns {Array<Entry>|boolean} An array of entries for the given range or false on error.
      */
     range(from, until = -1) {
@@ -387,7 +389,7 @@ class ReadableIndex extends events.EventEmitter {
      *
      * @api
      * @param {number} number The sequence number to search for.
-     * @param {boolean} [min] If set to true, will return the first entry that has a sequence number greater than or equal to `number`.
+     * @param {boolean} [min=false] If set to true, will return the first entry that has a sequence number greater than or equal to `number`.
      * @returns {number} The last index entry position that is lower than or equal to the `number`. Returns 0 if no index matches.
      */
     find(number, min = false) {

package/src/Index/WritableIndex.js

@@ -1,8 +1,8 @@
 import fs from 'fs';
 import ReadableIndex, { Entry, CorruptedIndexError, HEADER_MAGIC } from './ReadableIndex.js';
-import { assertEqual } from '../util.js';
-import { buildMetadataHeader } from '../metadataUtil.js';
-import { ensureDirectory } from '../fsUtil.js';
+import { assert, assertEqual } from '../utils/util.js';
+import { buildMetadataHeader } from '../utils/metadataUtil.js';
+import { ensureDirectory } from '../utils/fsUtil.js';
 
 /**
  * An index is a simple append-only file that stores an ordered list of entry elements pointing to the actual file position
@@ -191,9 +191,7 @@ class WritableIndex extends ReadableIndex {
         assertEqual(entry.constructor.size, this.EntryClass.size, `Invalid entry size.`);
 
         const dataLen = this.data.length;
-        if (dataLen > 0 && this.data[dataLen - 1].number >= entry.number) {
-            throw new Error('Consistency error. Tried to add an index that should come before existing last entry.');
-        }
+        assert(dataLen === 0 || this.data.at(-1).number < entry.number, 'Consistency error. Tried to add an index that should come before existing last entry.');
 
         if (this.readUntil === dataLen - 1) {
             this.readUntil++;

package/src/IndexMatcher.js

@@ -1,5 +1,5 @@
-import { getPropertyAtPath } from './util.js';
-import { matches } from './metadataUtil.js';
+import { getPropertyAtPath } from './utils/util.js';
+import { matches } from './utils/metadataUtil.js';
 
 /**
  * @typedef {object|function(object):boolean} Matcher

package/src/JoinEventStream.js

@@ -1,5 +1,5 @@
 import EventStream from './EventStream.js';
-import { wrapAndCheck } from './util.js';
+import { assert, kWayMerge } from './utils/util.js';
 
 /** Reusable sentinel used for missing or empty per-stream iterators. */
 const emptyIterator = Object.freeze({ next() { return { done: true }; } });
@@ -27,21 +27,18 @@ class JoinEventStream extends EventStream {
      * @param {EventStore} eventStore The event store to get the stream from.
      * @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
      * @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
-     * @param {function(object, object): boolean|null} [predicate] An optional filter function
-     *   `(payload, metadata) => boolean`.  Only events for which this returns truthy are yielded.
+     * @param {function|object|null} [predicate] Optional matcher (see {@link EventStream}).
+     * @param {boolean} [raw=false] If true, emit NDJSON Buffers.
      */
-    constructor(name, streams, eventStore, minRevision = 1, maxRevision = -1, predicate = null) {
-        super(name, eventStore, minRevision, maxRevision, predicate);
-        if (!(streams instanceof Array) || streams.length === 0) {
-            throw new Error(`Invalid list of streams supplied to JoinStream ${name}.`);
-        }
+    constructor(name, streams, eventStore, minRevision = 1, maxRevision = -1, predicate = null, raw = false) {
+        super(name, eventStore, minRevision, maxRevision, predicate, raw);
+        assert(streams instanceof Array && streams.length > 0, `Invalid list of streams supplied to JoinStream ${name}.`);
 
         this.streamIndex = eventStore.storage.index;
         // Translate revisions to index numbers (1-based) and wrap around negatives
         this.minRevision = normalizeVersion(minRevision, eventStore.length);
         this.maxRevision = normalizeVersion(maxRevision, eventStore.length);
         this.fetch = function() {
-            this._next = new Array(streams.length).fill(undefined);
             return streams.map(streamName => {
                 const streamIndex = eventStore.streams[streamName]?.index;
                 if (!streamIndex || streamIndex.length === 0) {
@@ -54,60 +51,47 @@ class JoinEventStream extends EventStream {
                     // (e.g. minRevision > all entries, or maxRevision < all entries).
                     return emptyIterator;
                 }
-                return eventStore.storage.readRange(from, until, streamIndex);
+                // Raw mode: get { buffer, time64, sequenceNumber } for binary-header ordering.
+                // Object mode: storage deserializes for us and we order by metadata.commitId.
+                return eventStore.storage.readRange(from, until, streamIndex, this.raw);
             });
         }
         this._iterator = null;
     }
 
     /**
-     * Returns the value of the iterator at position `index`
-     * @param {number} index The iterator position for which to return the next value
-     * @returns {*}
-     */
-    getValue(index) {
-        const next = this._iterator[index].next();
-        return next.done ? false : next.value;
-    }
-
-    /**
-     * @private
-     * @param {number} first
-     * @param {number} second
-     * @returns {boolean} If the first item follows after the second in the given read order determined by this.minRevision and this.maxRevision.
+     * @returns {Generator<object>}
      */
-    follows(first, second) {
-        return (this.minRevision > this.maxRevision ? first < second : first > second);
+    createMergedIterator() {
+        const ascending = this.minRevision <= this.maxRevision;
+        const raw = this.raw;
+        return kWayMerge(
+            this.fetch(),
+            entry => raw ? entry.sequenceNumber : entry.metadata.commitId,
+            ascending
+        );
     }
 
     /**
-     * @private
-     * @returns {object|boolean} The next event or false if no more events in the stream.
+     * Returns the next event in merge order.
+     *
+     * In raw mode: returns `{ buffer, time64, sequenceNumber }` from the binary header — no JSON
+     * deserialization. In object mode: returns a deserialized `{ stream, payload, metadata }` document
+     * produced by the storage layer.
+     * @returns {object|false} The next event, or `false` when the stream is exhausted.
      */
     next() {
         if (!this._iterator) {
-            this._iterator = this.fetch();
+            this._iterator = this.createMergedIterator();
         }
         while (true) {
-            let nextIndex = -1;
-            this._next.forEach((value, index) => {
-                if (typeof value === 'undefined') {
-                    value = this._next[index] = this.getValue(index);
-                }
-                if (value === false) {
-                    return;
-                }
-                if (nextIndex === -1 || this.follows(this._next[nextIndex].metadata.commitId, value.metadata.commitId)) {
-                    nextIndex = index;
-                }
-            });
-
-            if (nextIndex === -1) {
+            const step = this._iterator.next();
+            if (step.done) {
                 return false;
             }
-            const next = this._next[nextIndex];
-            this._next[nextIndex] = undefined;
-            if (!this.predicate || this.predicate(next.payload, next.metadata)) {
+            const next = step.value;
+
+            if (this.matchesPredicate(next)) {
                 return next;
             }
         }