event-storage 1.0.0 → 1.1.0

package/src/IndexMatcher.js

@@ -0,0 +1,205 @@
+import { getPropertyAtPath } from './util.js';
+import { matches } from './metadataUtil.js';
+
+/**
+ * @typedef {object|function(object):boolean} Matcher
+ */
+
+/**
+ * Classifies secondary-index matchers into a fast lookup table keyed on a
+ * configurable ordered list of "discriminant" property paths.  This enables O(1)
+ * candidate resolution on write instead of evaluating every registered matcher.
+ *
+ * Object matchers that contain at least one of the discriminant property paths
+ * (in priority order) are stored in a nested Map keyed first by property path and
+ * then by the scalar value found at that path in the matcher.  Function matchers
+ * and object matchers whose discriminant properties all resolve to undefined/object
+ * are kept in separate fallback sets and are always evaluated in full.
+ *
+ * When the discriminant property list is empty, `forEachMatch()` falls back to a
+ * full O(N) scan over all registered indexes.
+ */
+class IndexMatcher {
+
+    /**
+     * @param {string[]} [properties] Ordered list of document property paths (dot-notation)
+     *   used as discriminant keys.  The first path that resolves to a non-null scalar inside
+     *   a given object matcher is used as the key; remaining paths are ignored for that matcher.
+     *   Pass an empty array (the default) to disable the fast path entirely.
+     */
+    constructor(properties = []) {
+        this.properties = properties;
+        /** Map<indexName, Matcher> — stores every registered matcher for full match verification. */
+        this.matchers = new Map();
+        /**
+         * Nested lookup table: Map<propPath, Map<discriminantValue, Set<indexName>>>.
+         * Populated only for object matchers that contain at least one discriminant property.
+         */
+        this.table = new Map();
+        /** Set of index names whose matchers are functions (always evaluated in full). */
+        this.functionMatchers = new Set();
+        /**
+         * Set of index names whose object matchers contain none of the configured
+         * discriminant properties (evaluated in full against every incoming document).
+         */
+        this.unclassifiedMatchers = new Set();
+    }
+
+    /**
+     * Register an index name and its matcher in the lookup structures.
+     *
+     * @param {string} indexName
+     * @param {Matcher} matcher
+     */
+    add(indexName, matcher) {
+        this.matchers.set(indexName, matcher);
+        if (typeof matcher === 'function') {
+            this.functionMatchers.add(indexName);
+            return;
+        }
+        if (!matcher || typeof matcher !== 'object') {
+            this.unclassifiedMatchers.add(indexName);
+            return;
+        }
+
+        const discriminant = this.findDiscriminant(matcher);
+        if (!discriminant) {
+            this.unclassifiedMatchers.add(indexName);
+            return;
+        }
+
+        let propMap = this.table.get(discriminant.propPath);
+        if (!propMap) {
+            propMap = new Map();
+            this.table.set(discriminant.propPath, propMap);
+        }
+        for (const v of discriminant.values) {
+            let indexSet = propMap.get(v);
+            if (!indexSet) {
+                indexSet = new Set();
+                propMap.set(v, indexSet);
+            }
+            indexSet.add(indexName);
+        }
+    }
+
+    /**
+     * Remove an index name from the lookup structures.
+     * The matcher is retrieved from the internal registry, so only the index name
+     * is required.
+     *
+     * @param {string} indexName
+     */
+    remove(indexName) {
+        if (!this.matchers.has(indexName)) {
+            return;
+        }
+        const matcher = this.matchers.get(indexName);
+        this.matchers.delete(indexName);
+        if (typeof matcher === 'function') {
+            this.functionMatchers.delete(indexName);
+            return;
+        }
+        if (!matcher || typeof matcher !== 'object') {
+            this.unclassifiedMatchers.delete(indexName);
+            return;
+        }
+
+        const discriminant = this.findDiscriminant(matcher);
+        if (!discriminant) {
+            this.unclassifiedMatchers.delete(indexName);
+            return;
+        }
+
+        for (const v of discriminant.values) {
+            this.table.get(discriminant.propPath)?.get(v)?.delete(indexName);
+        }
+    }
+
+    /**
+     * Iterate over every registered index whose matcher matches `document`, calling
+     * `iterationHandler` with the index name for each match.
+     *
+     * When `this.properties` is non-empty, an O(1) discriminant lookup narrows the
+     * candidate set before the full `matches()` check is applied.  Each candidate's
+     * matcher is still verified with `matches()` to handle multi-property matchers
+     * where only the first property was used as the discriminant.
+     *
+     * When `this.properties` is empty the method falls back to a full O(N) scan.
+     *
+     * @param {object} document
+     * @param {function(string): void} iterationHandler Called with the index name for each match.
+     */
+    forEachMatch(document, iterationHandler) {
+        if (this.properties.length === 0) {
+            // Fast path disabled: full O(N) scan.
+            for (const [indexName, matcher] of this.matchers) {
+                if (matches(document, matcher)) {
+                    iterationHandler(indexName);
+                }
+            }
+            return;
+        }
+
+        for (const propPath of this.properties) {
+            const docValue = getPropertyAtPath(document, propPath);
+            if (docValue === undefined || docValue === null || typeof docValue === 'object') {
+                continue;
+            }
+
+            const indexSet = this.table.get(propPath)?.get(String(docValue));
+            if (!indexSet) {
+                continue;
+            }
+
+            for (const indexName of indexSet) {
+                if (matches(document, this.matchers.get(indexName))) {
+                    iterationHandler(indexName);
+                }
+            }
+        }
+
+        for (const indexName of this.unclassifiedMatchers) {
+            if (matches(document, this.matchers.get(indexName))) {
+                iterationHandler(indexName);
+            }
+        }
+
+        for (const indexName of this.functionMatchers) {
+            if (matches(document, this.matchers.get(indexName))) {
+                iterationHandler(indexName);
+            }
+        }
+    }
+
+    /**
+     * Find the first usable discriminant for an object matcher.
+     * Returns `{ propPath, values }` for the first entry in `this.properties` that
+     * resolves to a non-null, non-object scalar (or a non-empty array of scalars) inside
+     * `matcher`, or `null` if none.
+     *
+     * For a scalar discriminant `values` contains exactly one element.
+     * For an array-valued discriminant `values` contains all elements of the array.
+     *
+     * @param {object} matcher
+     * @returns {{ propPath: string, values: string[] }|null}
+     */
+    findDiscriminant(matcher) {
+        for (const propPath of this.properties) {
+            const value = getPropertyAtPath(matcher, propPath);
+            if (value !== undefined && value !== null) {
+                if (typeof value !== 'object') {
+                    return { propPath, values: [String(value)] };
+                }
+                if (Array.isArray(value) && value.length > 0 &&
+                    value.every(v => v !== null && v !== undefined && typeof v !== 'object')) {
+                    return { propPath, values: value.map(String) };
+                }
+            }
+        }
+        return null;
+    }
+
+}
+
+export default IndexMatcher;

package/src/JoinEventStream.js

@@ -1,6 +1,9 @@
 import EventStream from './EventStream.js';
 import { wrapAndCheck } from './util.js';
 
+/** Reusable sentinel used for missing or empty per-stream iterators. */
+const emptyIterator = Object.freeze({ next() { return { done: true }; } });
+
 /**
  * Calculate the actual version number from a possibly relative (negative) version number.
  *
@@ -24,9 +27,11 @@ class JoinEventStream extends EventStream {
      * @param {EventStore} eventStore The event store to get the stream from.
      * @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
      * @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
+     * @param {function(object, object): boolean|null} [predicate] An optional filter function
+     *   `(payload, metadata) => boolean`.  Only events for which this returns truthy are yielded.
      */
-    constructor(name, streams, eventStore, minRevision = 1, maxRevision = -1) {
-        super(name, eventStore, minRevision, maxRevision);
+    constructor(name, streams, eventStore, minRevision = 1, maxRevision = -1, predicate = null) {
+        super(name, eventStore, minRevision, maxRevision, predicate);
         if (!(streams instanceof Array) || streams.length === 0) {
             throw new Error(`Invalid list of streams supplied to JoinStream ${name}.`);
         }
@@ -38,12 +43,17 @@ class JoinEventStream extends EventStream {
         this.fetch = function() {
             this._next = new Array(streams.length).fill(undefined);
             return streams.map(streamName => {
-                if (!eventStore.streams[streamName]) {
-                    return { next() { return { done: true }; } };
+                const streamIndex = eventStore.streams[streamName]?.index;
+                if (!streamIndex || streamIndex.length === 0) {
+                    return emptyIterator;
                 }
-                const streamIndex = eventStore.streams[streamName].index;
                 const from = streamIndex.find(this.minRevision, this.minRevision <= this.maxRevision);
                 const until = streamIndex.find(this.maxRevision, this.minRevision > this.maxRevision);
+                if (from === 0 || until === 0) {
+                    // find() returns 0 when the requested revision is outside the stream's range
+                    // (e.g. minRevision > all entries, or maxRevision < all entries).
+                    return emptyIterator;
+                }
                 return eventStore.storage.readRange(from, until, streamIndex);
             });
         }
@@ -78,25 +88,29 @@ class JoinEventStream extends EventStream {
         if (!this._iterator) {
             this._iterator = this.fetch();
         }
-        let nextIndex = -1;
-        this._next.forEach((value, index) => {
-            if (typeof value === 'undefined') {
-                value = this._next[index] = this.getValue(index);
-            }
-            if (value === false) {
-                return;
+        while (true) {
+            let nextIndex = -1;
+            this._next.forEach((value, index) => {
+                if (typeof value === 'undefined') {
+                    value = this._next[index] = this.getValue(index);
+                }
+                if (value === false) {
+                    return;
+                }
+                if (nextIndex === -1 || this.follows(this._next[nextIndex].metadata.commitId, value.metadata.commitId)) {
+                    nextIndex = index;
+                }
+            });
+
+            if (nextIndex === -1) {
+                return false;
             }
-            if (nextIndex === -1 || this.follows(this._next[nextIndex].metadata.commitId, value.metadata.commitId)) {
-                nextIndex = index;
+            const next = this._next[nextIndex];
+            this._next[nextIndex] = undefined;
+            if (!this.predicate || this.predicate(next.payload, next.metadata)) {
+                return next;
             }
-        });
-
-        if (nextIndex === -1) {
-            return false;
         }
-        const next = this._next[nextIndex];
-        this._next[nextIndex] = undefined;
-        return next;
     }
 
 }

package/src/Partition/WritablePartition.js

@@ -1,6 +1,8 @@
 import fs from 'fs';
 import ReadablePartition, { CorruptFileError, HEADER_MAGIC, DOCUMENT_ALIGNMENT, DOCUMENT_SEPARATOR, DOCUMENT_HEADER_SIZE, DOCUMENT_FOOTER_SIZE } from './ReadablePartition.js';
-import { assert, buildMetadataHeader, alignTo, ensureDirectory } from '../util.js';
+import { assert, alignTo } from '../util.js';
+import { buildMetadataHeader } from '../metadataUtil.js';
+import { ensureDirectory } from '../fsUtil.js';
 import Clock from '../Clock.js';
 
 const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024;

package/src/PartitionPool.js

@@ -0,0 +1,149 @@
+/**
+ * A fixed-capacity registry of partitions with LRU eviction of open file handles.
+ *
+ * All partitions are stored by their numeric id and may be queried at any time.
+ * The pool additionally tracks which partitions currently have an open file descriptor
+ * in LRU (least-recently-used) order.  When the pool is asked to open a partition and
+ * doing so would exceed the configured cap, the least-recently-used open partition is
+ * closed first to stay within the limit.
+ *
+ * Setting the cap to 0 disables eviction: all partitions are allowed to remain open
+ * simultaneously, which matches the uncapped behaviour of the original implementation.
+ */
+class PartitionPool {
+
+    /**
+     * @param {number} [maxOpen=0] Maximum number of simultaneously open partition file
+     *   handles.  0 disables the limit (no eviction).
+     */
+    constructor(maxOpen = 0) {
+        this.maxOpen = maxOpen;
+        /** Registry of all known partitions keyed by id. */
+        this.registry = Object.create(null);
+        /**
+         * Insertion-order map used for LRU tracking of open file handles.
+         * Key = partition id, value = true.
+         * Oldest (least-recently-used) entry is first; newest (most-recently-used) is last.
+         */
+        this.handles = new Map();
+    }
+
+    /**
+     * Register a partition under the given id.
+     *
+     * @param {number|string} id
+     * @param {object} partition
+     */
+    add(id, partition) {
+        this.registry[id] = partition;
+    }
+
+    /**
+     * Retrieve a registered partition without opening it.
+     *
+     * @param {number|string} id
+     * @returns {object|undefined}
+     */
+    get(id) {
+        return this.registry[id];
+    }
+
+    /**
+     * Check whether a partition with the given id is registered in the pool.
+     *
+     * @param {number|string} id
+     * @returns {boolean}
+     */
+    has(id) {
+        return id in this.registry;
+    }
+
+    /**
+     * Open the partition with the given id, applying LRU eviction if necessary.
+     *
+     * If the partition is not yet open and adding it would exceed `maxOpen`, the
+     * least-recently-used open partition is closed first.  Stale entries (partitions
+     * that were closed externally) are discarded from the LRU map as they are
+     * encountered; if all tracked entries turn out to be stale the loop exits without
+     * closing any partition — the handle count stays temporarily inflated (bounded by
+     * the number of external closes since the last `open()` call) but correctness is
+     * preserved.
+     *
+     * @param {number|string} id
+     * @returns {object} The opened partition.
+     */
+    open(id) {
+        const partition = this.registry[id];
+
+        if (this.maxOpen > 0) {
+            // Remove id first — this may already bring the handle count below the cap.
+            this.handles.delete(id);
+            if (this.handles.size >= this.maxOpen) {
+                for (const [lruId] of this.handles) {
+                    this.handles.delete(lruId);
+                    const lruPartition = this.registry[lruId];
+                    if (lruPartition && lruPartition.isOpen()) {
+                        lruPartition.close();
+                        break;
+                    }
+                }
+            }
+            // (Re-)add id at the MRU end of the map.
+            this.handles.set(id, true);
+        }
+
+        partition.open();
+        return partition;
+    }
+
+    /**
+     * Invoke `callback` for every registered partition.
+     *
+     * @param {function(object): void} callback
+     */
+    forEach(callback) {
+        for (const id of Object.keys(this.registry)) {
+            callback(this.registry[id]);
+        }
+    }
+
+    /**
+     * Yield every registered partition object.
+     *
+     * @returns {Generator<object>}
+     */
+    *values() {
+        for (const id of Object.keys(this.registry)) {
+            yield this.registry[id];
+        }
+    }
+
+    /**
+     * The total number of registered partitions.
+     * @returns {number}
+     */
+    get count() {
+        return Object.keys(this.registry).length;
+    }
+
+    /**
+     * The number of open partition file handles currently tracked by the pool.
+     * @returns {number}
+     */
+    get openCount() {
+        return this.handles.size;
+    }
+
+    /**
+     * Reset the open-handle tracking without closing any partitions.
+     *
+     * Call this after externally closing all partitions (e.g. after
+     * `checkTornWrites`) to keep the pool's LRU state consistent with reality.
+     */
+    clearOpenHandles() {
+        this.handles.clear();
+    }
+
+}
+
+export default PartitionPool;

package/src/Storage/ReadOnlyStorage.js

@@ -54,9 +54,9 @@ class ReadOnlyStorage extends ReadableStorage {
         }
 
         const partitionId = ReadablePartition.idFor(filename);
-        if (!this.partitions[partitionId]) {
+        if (!this.partitions.has(partitionId)) {
             const partition = this.createPartition(filename, this.partitionConfig);
-            this.partitions[partition.id] = partition;
+            this.partitions.add(partition.id, partition);
             this.emit('partition-created', partition.id);
         }
     }