event-storage 1.0.0 → 1.2.0

package/src/EventStream.js

@@ -1,5 +1,8 @@
 import stream from 'stream';
-import { assert } from './util.js';
+import { assert } from './utils/util.js';
+import { buildRawBufferMatcher, matches } from './utils/metadataUtil.js';
+
+const NDJSON_NEWLINE = Buffer.from('\n');
 
 /**
  * Calculate the actual version number from a possibly relative (negative) version number.
@@ -33,22 +36,31 @@ class EventStream extends stream.Readable {
      * @param {EventStore} eventStore The event store to get the stream from.
      * @param {number} [minRevision] The minimum revision to include in the events (inclusive).
      * @param {number} [maxRevision] The maximum revision to include in the events (inclusive).
+     * @param {function|object|null} [predicate] Optional matcher:
+     *   - object mode: function `(payload, metadata) => boolean` or object matcher against `{ stream, payload, metadata }`
+     *   - raw mode: function `(buffer) => boolean` or object matcher against compact NDJSON bytes.
+     * @param {boolean} [raw=false] If true, emit NDJSON Buffers instead of event payload objects.
      */
-    constructor(name, eventStore, minRevision = 1, maxRevision = -1) {
-        super({ objectMode: true });
+    constructor(name, eventStore, minRevision = 1, maxRevision = -1, predicate = null, raw = false) {
+        if (typeof predicate === 'boolean' && raw === false) {
+            raw = predicate;
+            predicate = null;
+        }
+        super({ objectMode: !raw });
         assert(typeof name === 'string' && name !== '', 'Need to specify a stream name.');
         assert(typeof eventStore === 'object' && eventStore !== null, `Need to provide EventStore instance to create EventStream ${name}.`);
 
         this.name = name;
+        this.raw = raw;
+        this.predicate = predicate || null;
+        this.rawMatcher = null;
         if (eventStore.streams[name]) {
             this.streamIndex = eventStore.streams[name].index;
             this.minRevision = normalizeVersion(minRevision, this.streamIndex.length);
             this.maxRevision = normalizeVersion(maxRevision, this.streamIndex.length);
             this.version = minVersion(this.streamIndex.length, maxRevision);
             this._iterator = null;
-            this.fetch = function() {
-                return eventStore.storage.readRange(this.minRevision, this.maxRevision, this.streamIndex);
-            }
+            this.fetch = () => eventStore.storage.readRange(this.minRevision, this.maxRevision, this.streamIndex, raw);
         } else {
             this.streamIndex = { length: 0 };
             this.version = -1;
@@ -230,10 +242,18 @@ class EventStream extends stream.Readable {
     *[Symbol.iterator]() {
         let next;
         while ((next = this.next()) !== false) {
-            yield next.payload;
+            yield this.raw ? this.toRawBuffer(next) : next.payload;
         }
     }
 
+    /**
+     * @param {{ buffer: Buffer }} entry
+     * @returns {Buffer}
+     */
+    toRawBuffer(entry) {
+        return Buffer.concat([entry.buffer, NDJSON_NEWLINE]);
+    }
+
     /**
      * Reset this stream to the start so it can be iterated again.
      * @returns {EventStream}
@@ -244,6 +264,44 @@ class EventStream extends stream.Readable {
         return this;
     }
 
+    /**
+     * Apply a filter predicate to this stream.  Only events for which `predicate(payload, metadata)`
+     * returns a truthy value will be yielded.  The predicate is stored as a first-class property
+     * of the stream and applied in {@link EventStream#next}.
+     *
+     * @api
+     * @param {function(object, object): boolean} predicate A function receiving `(payload, metadata)`.
+     *   Events for which the predicate returns falsy are skipped.
+     * @returns {EventStream} `this`
+     */
+    filter(predicate) {
+        this.predicate = predicate || null;
+        this.rawMatcher = null;
+        this._iterator = null;
+        this._events = null;
+        return this;
+    }
+
+    matchesPredicate(entry) {
+        if (!this.predicate) {
+            return true;
+        }
+        if (this.raw) {
+            if (typeof this.predicate === 'function') {
+                return this.predicate(entry.buffer);
+            }
+            if (!this.rawMatcher) {
+                this.rawMatcher = buildRawBufferMatcher(this.predicate);
+            }
+            return this.rawMatcher(entry.buffer);
+        }
+
+        if (typeof this.predicate === 'function') {
+            return this.predicate(entry.payload, entry.metadata);
+        }
+        return matches(entry, this.predicate);
+    }
+
     /**
      * @returns {object|boolean} The next event or false if no more events in the stream.
      */
@@ -251,13 +309,17 @@ class EventStream extends stream.Readable {
         if (!this._iterator) {
             this._iterator = this.fetch();
         }
-        let next;
         try {
-            next = this._iterator.next();
+            while (true) {
+                const result = this._iterator.next();
+                if (result.done) return false;
+                if (this.matchesPredicate(result.value)) {
+                    return result.value;
+                }
+            }
         } catch(e) {
             return false;
         }
-        return next.done ? false : next.value;
     }
 
     // noinspection JSUnusedGlobalSymbols
@@ -267,7 +329,7 @@ class EventStream extends stream.Readable {
      */
     _read() {
         const next = this.next();
-        this.push(next ? next.payload : null);
+        this.push(next ? (this.raw ? this.toRawBuffer(next) : next.payload) : null);
     }
 
 }

package/src/Index/ReadableIndex.js

@@ -2,7 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import events from 'events';
 import Entry, { assertValidEntryClass } from '../IndexEntry.js';
-import { assert, wrapAndCheck, binarySearch } from '../util.js';
+import { assert, wrapAndCheck, binarySearch } from '../utils/util.js';
 
 // node-event-store-index V01
 const HEADER_MAGIC = "nesidx01";
@@ -77,6 +77,7 @@ class ReadableIndex extends events.EventEmitter {
         if (options.metadata) {
             this.metadata = Object.assign({entryClass: options.EntryClass.name, entrySize: options.EntryClass.size}, options.metadata);
         }
+        this.headerSize = 0;
     }
 
     /**
@@ -208,12 +209,13 @@ class ReadableIndex extends events.EventEmitter {
      * @throws {Error} if the metadata size in the header is invalid.
      */
     readMetadata() {
+        if (this.headerSize > 0) return this.headerSize;
         const headerBuffer = Buffer.allocUnsafe(8 + 4);
         fs.readSync(this.fd, headerBuffer, 0, 8 + 4, 0);
         const headerMagic = headerBuffer.toString('utf8', 0, 8);
 
-        assert(headerMagic.substr(0, 6) === HEADER_MAGIC.substr(0, 6), 'Invalid file header.');
-        assert(headerMagic === HEADER_MAGIC, `Invalid file version. The index ${this.fileName} was created with a different library version (${headerMagic.substr(6)}).`);
+        assert(headerMagic.substring(0, 6) === HEADER_MAGIC.substring(0, 6), 'Invalid file header.');
+        assert(headerMagic === HEADER_MAGIC, `Invalid file version. The index ${this.fileName} was created with a different library version (${headerMagic.substring(6)}).`);
 
         const metadataSize = headerBuffer.readUInt32BE(8);
         assert(metadataSize >= 3, 'Invalid metadata size.');
@@ -353,7 +355,7 @@ class ReadableIndex extends events.EventEmitter {
      *
      * @api
      * @param {number} from The 1-based index position from where to get entries from (inclusive). If < 0 will start at that position from end.
-     * @param {number} [until] The 1-based index position until where to get entries to (inclusive). If < 0 will get until that position from the end. Defaults to this.length.
+     * @param {number} [until=-1] The 1-based index position until where to get entries to (inclusive). If < 0 will get until that position from the end. Defaults to this.length.
      * @returns {Array<Entry>|boolean} An array of entries for the given range or false on error.
      */
     range(from, until = -1) {
@@ -387,7 +389,7 @@ class ReadableIndex extends events.EventEmitter {
      *
      * @api
      * @param {number} number The sequence number to search for.
-     * @param {boolean} [min] If set to true, will return the first entry that has a sequence number greater than or equal to `number`.
+     * @param {boolean} [min=false] If set to true, will return the first entry that has a sequence number greater than or equal to `number`.
      * @returns {number} The last index entry position that is lower than or equal to the `number`. Returns 0 if no index matches.
      */
     find(number, min = false) {

package/src/Index/WritableIndex.js

@@ -1,6 +1,8 @@
 import fs from 'fs';
 import ReadableIndex, { Entry, CorruptedIndexError, HEADER_MAGIC } from './ReadableIndex.js';
-import { assertEqual, buildMetadataHeader, ensureDirectory } from '../util.js';
+import { assert, assertEqual } from '../utils/util.js';
+import { buildMetadataHeader } from '../utils/metadataUtil.js';
+import { ensureDirectory } from '../utils/fsUtil.js';
 
 /**
  * An index is a simple append-only file that stores an ordered list of entry elements pointing to the actual file position
@@ -189,9 +191,7 @@ class WritableIndex extends ReadableIndex {
         assertEqual(entry.constructor.size, this.EntryClass.size, `Invalid entry size.`);
 
         const dataLen = this.data.length;
-        if (dataLen > 0 && this.data[dataLen - 1].number >= entry.number) {
-            throw new Error('Consistency error. Tried to add an index that should come before existing last entry.');
-        }
+        assert(dataLen === 0 || this.data.at(-1).number < entry.number, 'Consistency error. Tried to add an index that should come before existing last entry.');
 
         if (this.readUntil === dataLen - 1) {
             this.readUntil++;

package/src/IndexMatcher.js

@@ -0,0 +1,205 @@
+import { getPropertyAtPath } from './utils/util.js';
+import { matches } from './utils/metadataUtil.js';
+
+/**
+ * @typedef {object|function(object):boolean} Matcher
+ */
+
+/**
+ * Classifies secondary-index matchers into a fast lookup table keyed on a
+ * configurable ordered list of "discriminant" property paths.  This enables O(1)
+ * candidate resolution on write instead of evaluating every registered matcher.
+ *
+ * Object matchers that contain at least one of the discriminant property paths
+ * (in priority order) are stored in a nested Map keyed first by property path and
+ * then by the scalar value found at that path in the matcher.  Function matchers
+ * and object matchers whose discriminant properties all resolve to undefined/object
+ * are kept in separate fallback sets and are always evaluated in full.
+ *
+ * When the discriminant property list is empty, `forEachMatch()` falls back to a
+ * full O(N) scan over all registered indexes.
+ */
+class IndexMatcher {
+
+    /**
+     * @param {string[]} [properties] Ordered list of document property paths (dot-notation)
+     *   used as discriminant keys.  The first path that resolves to a non-null scalar inside
+     *   a given object matcher is used as the key; remaining paths are ignored for that matcher.
+     *   Pass an empty array (the default) to disable the fast path entirely.
+     */
+    constructor(properties = []) {
+        this.properties = properties;
+        /** Map<indexName, Matcher> — stores every registered matcher for full match verification. */
+        this.matchers = new Map();
+        /**
+         * Nested lookup table: Map<propPath, Map<discriminantValue, Set<indexName>>>.
+         * Populated only for object matchers that contain at least one discriminant property.
+         */
+        this.table = new Map();
+        /** Set of index names whose matchers are functions (always evaluated in full). */
+        this.functionMatchers = new Set();
+        /**
+         * Set of index names whose object matchers contain none of the configured
+         * discriminant properties (evaluated in full against every incoming document).
+         */
+        this.unclassifiedMatchers = new Set();
+    }
+
+    /**
+     * Register an index name and its matcher in the lookup structures.
+     *
+     * @param {string} indexName
+     * @param {Matcher} matcher
+     */
+    add(indexName, matcher) {
+        this.matchers.set(indexName, matcher);
+        if (typeof matcher === 'function') {
+            this.functionMatchers.add(indexName);
+            return;
+        }
+        if (!matcher || typeof matcher !== 'object') {
+            this.unclassifiedMatchers.add(indexName);
+            return;
+        }
+
+        const discriminant = this.findDiscriminant(matcher);
+        if (!discriminant) {
+            this.unclassifiedMatchers.add(indexName);
+            return;
+        }
+
+        let propMap = this.table.get(discriminant.propPath);
+        if (!propMap) {
+            propMap = new Map();
+            this.table.set(discriminant.propPath, propMap);
+        }
+        for (const v of discriminant.values) {
+            let indexSet = propMap.get(v);
+            if (!indexSet) {
+                indexSet = new Set();
+                propMap.set(v, indexSet);
+            }
+            indexSet.add(indexName);
+        }
+    }
+
+    /**
+     * Remove an index name from the lookup structures.
+     * The matcher is retrieved from the internal registry, so only the index name
+     * is required.
+     *
+     * @param {string} indexName
+     */
+    remove(indexName) {
+        if (!this.matchers.has(indexName)) {
+            return;
+        }
+        const matcher = this.matchers.get(indexName);
+        this.matchers.delete(indexName);
+        if (typeof matcher === 'function') {
+            this.functionMatchers.delete(indexName);
+            return;
+        }
+        if (!matcher || typeof matcher !== 'object') {
+            this.unclassifiedMatchers.delete(indexName);
+            return;
+        }
+
+        const discriminant = this.findDiscriminant(matcher);
+        if (!discriminant) {
+            this.unclassifiedMatchers.delete(indexName);
+            return;
+        }
+
+        for (const v of discriminant.values) {
+            this.table.get(discriminant.propPath)?.get(v)?.delete(indexName);
+        }
+    }
+
+    /**
+     * Iterate over every registered index whose matcher matches `document`, calling
+     * `iterationHandler` with the index name for each match.
+     *
+     * When `this.properties` is non-empty, an O(1) discriminant lookup narrows the
+     * candidate set before the full `matches()` check is applied.  Each candidate's
+     * matcher is still verified with `matches()` to handle multi-property matchers
+     * where only the first property was used as the discriminant.
+     *
+     * When `this.properties` is empty the method falls back to a full O(N) scan.
+     *
+     * @param {object} document
+     * @param {function(string): void} iterationHandler Called with the index name for each match.
+     */
+    forEachMatch(document, iterationHandler) {
+        if (this.properties.length === 0) {
+            // Fast path disabled: full O(N) scan.
+            for (const [indexName, matcher] of this.matchers) {
+                if (matches(document, matcher)) {
+                    iterationHandler(indexName);
+                }
+            }
+            return;
+        }
+
+        for (const propPath of this.properties) {
+            const docValue = getPropertyAtPath(document, propPath);
+            if (docValue === undefined || docValue === null || typeof docValue === 'object') {
+                continue;
+            }
+
+            const indexSet = this.table.get(propPath)?.get(String(docValue));
+            if (!indexSet) {
+                continue;
+            }
+
+            for (const indexName of indexSet) {
+                if (matches(document, this.matchers.get(indexName))) {
+                    iterationHandler(indexName);
+                }
+            }
+        }
+
+        for (const indexName of this.unclassifiedMatchers) {
+            if (matches(document, this.matchers.get(indexName))) {
+                iterationHandler(indexName);
+            }
+        }
+
+        for (const indexName of this.functionMatchers) {
+            if (matches(document, this.matchers.get(indexName))) {
+                iterationHandler(indexName);
+            }
+        }
+    }
+
+    /**
+     * Find the first usable discriminant for an object matcher.
+     * Returns `{ propPath, values }` for the first entry in `this.properties` that
+     * resolves to a non-null, non-object scalar (or a non-empty array of scalars) inside
+     * `matcher`, or `null` if none.
+     *
+     * For a scalar discriminant `values` contains exactly one element.
+     * For an array-valued discriminant `values` contains all elements of the array.
+     *
+     * @param {object} matcher
+     * @returns {{ propPath: string, values: string[] }|null}
+     */
+    findDiscriminant(matcher) {
+        for (const propPath of this.properties) {
+            const value = getPropertyAtPath(matcher, propPath);
+            if (value !== undefined && value !== null) {
+                if (typeof value !== 'object') {
+                    return { propPath, values: [String(value)] };
+                }
+                if (Array.isArray(value) && value.length > 0 &&
+                    value.every(v => v !== null && v !== undefined && typeof v !== 'object')) {
+                    return { propPath, values: value.map(String) };
+                }
+            }
+        }
+        return null;
+    }
+
+}
+
+export default IndexMatcher;

package/src/JoinEventStream.js

@@ -1,5 +1,8 @@
 import EventStream from './EventStream.js';
-import { wrapAndCheck } from './util.js';
+import { assert, kWayMerge } from './utils/util.js';
+
+/** Reusable sentinel used for missing or empty per-stream iterators. */
+const emptyIterator = Object.freeze({ next() { return { done: true }; } });
 
 /**
  * Calculate the actual version number from a possibly relative (negative) version number.
@@ -24,79 +27,74 @@ class JoinEventStream extends EventStream {
      * @param {EventStore} eventStore The event store to get the stream from.
      * @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
      * @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
+     * @param {function|object|null} [predicate] Optional matcher (see {@link EventStream}).
+     * @param {boolean} [raw=false] If true, emit NDJSON Buffers.
      */
-    constructor(name, streams, eventStore, minRevision = 1, maxRevision = -1) {
-        super(name, eventStore, minRevision, maxRevision);
-        if (!(streams instanceof Array) || streams.length === 0) {
-            throw new Error(`Invalid list of streams supplied to JoinStream ${name}.`);
-        }
+    constructor(name, streams, eventStore, minRevision = 1, maxRevision = -1, predicate = null, raw = false) {
+        super(name, eventStore, minRevision, maxRevision, predicate, raw);
+        assert(streams instanceof Array && streams.length > 0, `Invalid list of streams supplied to JoinStream ${name}.`);
 
         this.streamIndex = eventStore.storage.index;
         // Translate revisions to index numbers (1-based) and wrap around negatives
         this.minRevision = normalizeVersion(minRevision, eventStore.length);
         this.maxRevision = normalizeVersion(maxRevision, eventStore.length);
         this.fetch = function() {
-            this._next = new Array(streams.length).fill(undefined);
             return streams.map(streamName => {
-                if (!eventStore.streams[streamName]) {
-                    return { next() { return { done: true }; } };
+                const streamIndex = eventStore.streams[streamName]?.index;
+                if (!streamIndex || streamIndex.length === 0) {
+                    return emptyIterator;
                 }
-                const streamIndex = eventStore.streams[streamName].index;
                 const from = streamIndex.find(this.minRevision, this.minRevision <= this.maxRevision);
                 const until = streamIndex.find(this.maxRevision, this.minRevision > this.maxRevision);
-                return eventStore.storage.readRange(from, until, streamIndex);
+                if (from === 0 || until === 0) {
+                    // find() returns 0 when the requested revision is outside the stream's range
+                    // (e.g. minRevision > all entries, or maxRevision < all entries).
+                    return emptyIterator;
+                }
+                // Raw mode: get { buffer, time64, sequenceNumber } for binary-header ordering.
+                // Object mode: storage deserializes for us and we order by metadata.commitId.
+                return eventStore.storage.readRange(from, until, streamIndex, this.raw);
             });
         }
         this._iterator = null;
     }
 
     /**
-     * Returns the value of the iterator at position `index`
-     * @param {number} index The iterator position for which to return the next value
-     * @returns {*}
-     */
-    getValue(index) {
-        const next = this._iterator[index].next();
-        return next.done ? false : next.value;
-    }
-
-    /**
-     * @private
-     * @param {number} first
-     * @param {number} second
-     * @returns {boolean} If the first item follows after the second in the given read order determined by this.minRevision and this.maxRevision.
+     * @returns {Generator<object>}
      */
-    follows(first, second) {
-        return (this.minRevision > this.maxRevision ? first < second : first > second);
+    createMergedIterator() {
+        const ascending = this.minRevision <= this.maxRevision;
+        const raw = this.raw;
+        return kWayMerge(
+            this.fetch(),
+            entry => raw ? entry.sequenceNumber : entry.metadata.commitId,
+            ascending
+        );
     }
 
     /**
-     * @private
-     * @returns {object|boolean} The next event or false if no more events in the stream.
+     * Returns the next event in merge order.
+     *
+     * In raw mode: returns `{ buffer, time64, sequenceNumber }` from the binary header — no JSON
+     * deserialization. In object mode: returns a deserialized `{ stream, payload, metadata }` document
+     * produced by the storage layer.
+     * @returns {object|false} The next event, or `false` when the stream is exhausted.
      */
     next() {
         if (!this._iterator) {
-            this._iterator = this.fetch();
+            this._iterator = this.createMergedIterator();
         }
-        let nextIndex = -1;
-        this._next.forEach((value, index) => {
-            if (typeof value === 'undefined') {
-                value = this._next[index] = this.getValue(index);
-            }
-            if (value === false) {
-                return;
+        while (true) {
+            const step = this._iterator.next();
+            if (step.done) {
+                return false;
             }
-            if (nextIndex === -1 || this.follows(this._next[nextIndex].metadata.commitId, value.metadata.commitId)) {
-                nextIndex = index;
-            }
-        });
+            const next = step.value;
 
-        if (nextIndex === -1) {
-            return false;
+            if (this.matchesPredicate(next)) {
+                return next;
+            }
         }
-        const next = this._next[nextIndex];
-        this._next[nextIndex] = undefined;
-        return next;
     }
 
 }