event-storage 1.0.0 → 1.2.0

package/src/EventStore.js

@@ -6,15 +6,51 @@ import events from 'events';
 import Storage, { ReadOnly as ReadOnlyStorage, LOCK_THROW, LOCK_RECLAIM } from './Storage.js';
 import Index from './Index.js';
 import Consumer from './Consumer.js';
-import { assert, scanForFiles } from './util.js';
+import { assert, getPropertyAtPath } from './utils/util.js';
+import { ensureDirectory, scanForFiles } from './utils/fsUtil.js';
+import { buildTypeMatcherFn } from './utils/metadataUtil.js';
 
 const ExpectedVersion = {
     Any: -1,
     EmptyStream: 0
 };
 
+/**
+ * Default matcher property paths mirroring the Storage default, used for index optimization.
+ */
+const DEFAULT_MATCHER_PROPERTIES = ['stream', 'payload.type'];
+const STREAM_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_]*(?:[\/:@~+=\-#.][A-Za-z0-9_]+)*$/;
+const STORAGE_HOOK_EVENTS = new Set(['preCommit', 'preRead']);
+
 class OptimisticConcurrencyError extends Error {}
 
+/**
+ * An accept condition that captures the global event-log position at the time a {@link EventStore#query}
+ * call was made.  Pass it as the `expectedVersion` argument to {@link EventStore#commit} to enforce
+ * DCB-style (Dynamic Consistency Boundary) optimistic concurrency: the commit is rejected only when
+ * one or more events that match the original query (types + optional matcher) have been appended to
+ * the store between the `query` call and the `commit` call.
+ *
+ * @property {string[]} types   The event types included in the query.
+ * @property {function(object, object): boolean|null} matcher An optional function `(payload, metadata) => boolean`
+ *   used to narrow the conflict check.  When `null`, any new event of a listed type causes a conflict.
+ * @property {number}   noneMatchAfter The global store length (total event count) at the time the query was made.
+ */
+class CommitCondition {
+    /**
+     * @param {string[]} types
+     * @param {function(object, object): boolean|object|null} [matcher]
+     * @param {number}   noneMatchAfter
+     * @param {boolean}  [raw=false]
+     */
+    constructor(types, matcher = null, noneMatchAfter, raw = false) {
+        this.types = types;
+        this.matcher = matcher;
+        this.raw = raw;
+        this.noneMatchAfter = noneMatchAfter;
+    }
+}
+
 /**
  * An event store optimized for working with many streams.
  * An event stream is implemented as an iterator over an index on the storage, therefore indexes need to be lightweight
@@ -29,6 +65,9 @@ class EventStore extends events.EventEmitter {
      * @param {string} [config.streamsDirectory] The directory where the streams should be stored. Default '{storageDirectory}/streams'.
      * @param {object} [config.storageConfig] Additional config options given to the storage backend. See `Storage`.
      * @param {boolean} [config.readOnly] If the storage should be mounted in read-only mode.
+     * @param {string|function(object): string} [config.typeAccessor] Dot-notation path (e.g. `'type'`) or
+     *   function `(event) => string` identifying the event type. Enables type-based queries via
+     *   {@link EventStore#query} and ensures proper index routing for those queries.
      * @param {object|function(string): object} [config.streamMetadata] A metadata object or a function `(streamName) => object`
      *   that is called whenever a new stream partition is created. The returned object is stored once in the partition
      *   file header and surfaced to `preCommit` / `preRead` hooks. Takes precedence only when
@@ -41,6 +80,15 @@ class EventStore extends events.EventEmitter {
             storeName = 'eventstore';
         }
 
+        if (typeof config.typeAccessor === 'string' && config.typeAccessor) {
+            const accessorPath = config.typeAccessor;
+            this.typeAccessor = (event) => getPropertyAtPath(event, accessorPath);
+            this.typeMatcherFn = buildTypeMatcherFn(accessorPath);
+        } else {
+            this.typeAccessor = typeof config.typeAccessor === 'function' ? config.typeAccessor : null;
+            this.typeMatcherFn = null;
+        }
+
         this.storageDirectory = path.resolve(config.storageDirectory || /* istanbul ignore next */ './data');
         let defaults = {
             dataDirectory: this.storageDirectory,
@@ -50,6 +98,17 @@ class EventStore extends events.EventEmitter {
         };
         const storageConfig = Object.assign(defaults, config.storageConfig);
 
+        // When typeAccessor is a string path, ensure the corresponding full document path
+        // (payload.<path>) is present in matcherProperties so the IndexMatcher discriminant
+        // table can route type-stream lookups in O(1) on every write.
+        if (this.typeMatcherFn) {
+            const fullPath = `payload.${config.typeAccessor}`;
+            const currentProps = storageConfig.matcherProperties || DEFAULT_MATCHER_PROPERTIES;
+            if (!currentProps.includes(fullPath)) {
+                storageConfig.matcherProperties = [...currentProps, fullPath];
+            }
+        }
+
         // Translate the high-level streamMetadata option into the storage-level metadata function,
         // but only when the caller has not already provided a lower-level storageConfig.metadata.
         if (config.streamMetadata !== undefined && storageConfig.metadata === undefined) {
@@ -75,18 +134,18 @@ class EventStore extends events.EventEmitter {
         this.storage = (storageConfig.readOnly === true) ?
                         new ReadOnlyStorage(storeName, storageConfig)
                         : new Storage(storeName, storageConfig);
-        this.storage.open();
         this.streams = Object.create(null);
         this.streams._all = { index: this.storage.index };
+        this.consumers = new Map();
 
-        this.scanStreams((err) => {
-            if (err) {
-                this.storage.close();
-                throw err;
-            }
+        this.storage.on('index-created', this.registerStream.bind(this));
+
+        this.storage.on('opened', () => {
             this.checkUnfinishedCommits();
             this.emit('ready');
         });
+
+        this.storage.open();
     }
 
     /**
@@ -121,22 +180,6 @@ class EventStore extends events.EventEmitter {
         }
     }
 
-    /**
-     * Scan the streams directory for existing streams so they are ready for `getEventStream()`.
-     *
-     * @private
-     * @param {function} callback A callback that will be called when all existing streams are found.
-     */
-    scanStreams(callback) {
-        /* istanbul ignore if */
-        if (typeof callback !== 'function') {
-            callback = () => {};
-        }
-        // Find existing streams by scanning dir for filenames starting with 'stream-'
-        scanForFiles(this.streamsDirectory, /(stream-.*)\.index$/, this.registerStream.bind(this), callback);
-        this.storage.on('index-created', this.registerStream.bind(this));
-    }
-
     /**
      * @private
      * @param {string} name The full stream name, including the `stream-` prefix (and optional `.closed` suffix).
@@ -176,10 +219,15 @@ class EventStore extends events.EventEmitter {
 
     /**
      * Close the event store and free up all resources.
+     * Stops all registered consumers before closing storage.
      *
      * @api
      */
     close() {
+        for (const consumer of this.consumers.values()) {
+            consumer.stop();
+        }
+        this.consumers.clear();
         this.storage.close();
     }
 
@@ -193,11 +241,8 @@ class EventStore extends events.EventEmitter {
      * @returns {this}
      */
     on(event, listener) {
-        if (event === 'preCommit' || event === 'preRead') {
-            if (event === 'preCommit') {
-                assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.');
-            }
-            this.storage.on(event, listener);
+        if (this.isStorageHookEvent(event)) {
+            this.delegateStorageHookEvent('on', event, listener);
             return this;
         }
         return super.on(event, listener);
@@ -218,11 +263,8 @@ class EventStore extends events.EventEmitter {
      * @returns {this}
      */
     once(event, listener) {
-        if (event === 'preCommit' || event === 'preRead') {
-            if (event === 'preCommit') {
-                assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.');
-            }
-            this.storage.once(event, listener);
+        if (this.isStorageHookEvent(event)) {
+            this.delegateStorageHookEvent('once', event, listener);
             return this;
         }
         return super.once(event, listener);
@@ -237,13 +279,24 @@ class EventStore extends events.EventEmitter {
      * @returns {this}
      */
     off(event, listener) {
-        if (event === 'preCommit' || event === 'preRead') {
+        if (this.isStorageHookEvent(event)) {
             this.storage.off(event, listener);
             return this;
         }
         return super.off(event, listener);
     }
 
+    isStorageHookEvent(event) {
+        return STORAGE_HOOK_EVENTS.has(event);
+    }
+
+    delegateStorageHookEvent(method, event, listener) {
+        if (event === 'preCommit') {
+            assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.');
+        }
+        this.storage[method](event, listener);
+    }
+
     /**
      * @inheritDoc
      */
@@ -299,16 +352,16 @@ class EventStore extends events.EventEmitter {
      *
      * @private
      * @param {Array<object>|object} events
-     * @param {number} [expectedVersion]
+     * @param {number|CommitCondition} [expectedVersion]
      * @param {object|function} [metadata]
      * @param {function} [callback]
-     * @returns {{events: Array<object>, metadata: object, callback: function, expectedVersion: number}}
+     * @returns {{events: Array<object>, metadata: object, callback: function, expectedVersion: number|CommitCondition}}
      */
     static fixArgumentTypes(events, expectedVersion, metadata, callback) {
         if (!(events instanceof Array)) {
             events = [events];
         }
-        if (typeof expectedVersion !== 'number') {
+        if (typeof expectedVersion !== 'number' && !(expectedVersion instanceof CommitCondition)) {
             callback = metadata;
             metadata = expectedVersion;
             expectedVersion = ExpectedVersion.Any;
@@ -323,6 +376,79 @@ class EventStore extends events.EventEmitter {
         return { events, expectedVersion, metadata, callback };
     }
 
+    /**
+     * Check a {@link CommitCondition} against the current state of the store.
+     * Iterates a join stream over all condition type streams starting from
+     * `condition.noneMatchAfter` (the global position captured at query time), and throws an
+     * {@link OptimisticConcurrencyError} when a new event of a listed type satisfies
+     * `condition.matcher(payload, metadata)` (or any such event when no matcher is provided).
+     *
+     * @param {CommitCondition} condition
+     * @throws {OptimisticConcurrencyError}
+     */
+    checkCondition(condition) {
+        if (this.storage.length <= condition.noneMatchAfter) return; // no new events since condition was obtained
+
+        const existingTypes = condition.types.filter(t => t in this.streams);
+        if (existingTypes.length === 0) return;
+
+        // Only events after condition.noneMatchAfter can be conflicts.
+        // Pass the original matcher and raw flag so the stream filters at the source.
+        const stream = this.fromStreams(
+            '_check_' + condition.types.join('_'),
+            existingTypes,
+            condition.noneMatchAfter + 1,
+            -1,
+            condition.matcher,
+            condition.raw
+        );
+
+        assert(stream.next() === false, `Optimistic Concurrency error. A conflicting event was committed since the condition was obtained.`, OptimisticConcurrencyError);
+    }
+
+    /**
+     * Ensure a dedicated type stream exists for each event's type, creating it if needed.
+     * Must be called before the entity stream is created to guarantee correct index routing.
+     *
+     * @param {Array<object>} events The events to process.
+     */
+    ensureTypeStreams(events) {
+        if (!this.typeAccessor) return;
+        for (const event of events) {
+            const type = this.resolveValidatedTypeStreamName(event);
+            if (type && !(type in this.streams)) {
+                const matcher = this.typeMatcherFn
+                    ? this.typeMatcherFn(type)
+                    : (doc) => this.typeAccessor(doc.payload) === type;
+                this.createEventStream(type, matcher, false);
+            }
+        }
+    }
+
+    resolveValidatedTypeStreamName(event) {
+        const type = this.typeAccessor(event);
+        if (type === undefined || type === null || type === '') {
+            return null;
+        }
+        assert(typeof type === 'string', 'typeAccessor must return a string.');
+        assert(STREAM_NAME_PATTERN.test(type), `typeAccessor must return a valid stream name. Got: "${type}"`);
+        return type;
+    }
+
+    getExistingQueryTypes(types) {
+        const queryTypes = [];
+        for (const type of types) {
+            if (type in this.streams) {
+                queryTypes.push(type);
+                continue;
+            }
+            if (!this.typeAccessor) {
+                throw new Error(`Type stream "${type}" does not exist. Create it with createEventStream() first, or configure typeAccessor to have type streams created automatically on commit.`);
+            }
+        }
+        return queryTypes;
+    }
+
     /**
      * Commit a list of events for the given stream name, which is expected to be at the given version.
      * Note that the events committed may still appear in other streams too - the given stream name is only
@@ -331,10 +457,12 @@ class EventStore extends events.EventEmitter {
      * @api
      * @param {string} streamName The name of the stream to commit the events to.
      * @param {Array<object>|object} events The events to commit or a single event.
-     * @param {number} [expectedVersion] One of ExpectedVersion constants or a positive version number that the stream is supposed to be at before commit.
+     * @param {number|CommitCondition} [expectedVersion] One of the `ExpectedVersion` constants, a positive
+     *   stream version number, or a {@link CommitCondition} obtained from {@link EventStore#query}.
      * @param {object} [metadata] The commit metadata to use as base. Useful for replication and adding storage metadata.
      * @param {function} [callback] A function that will be executed when all events have been committed.
-     * @throws {OptimisticConcurrencyError} if the stream is not at the expected version.
+     * @throws {OptimisticConcurrencyError} if the stream is not at the expected version, or if a
+     *   {@link CommitCondition} was provided and conflicting events have been committed since it was obtained.
      */
     commit(streamName, events, expectedVersion = ExpectedVersion.Any, metadata = {}, callback = null) {
         assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not commit to it.');
@@ -343,14 +471,25 @@ class EventStore extends events.EventEmitter {
 
         ({ events, expectedVersion, metadata, callback } = EventStore.fixArgumentTypes(events, expectedVersion, metadata, callback));
 
+        // Perform DCB-style concurrency check when a CommitCondition is provided.
+        if (expectedVersion instanceof CommitCondition) {
+            this.checkCondition(expectedVersion);
+            expectedVersion = ExpectedVersion.Any;
+        }
+
+        // When typeAccessor is configured, ensure a dedicated type stream exists for each event
+        // before the entity stream write so the type stream index is never incomplete.
+        this.ensureTypeStreams(events);
+
         if (!(streamName in this.streams)) {
-            this.createEventStream(streamName, { stream: streamName });
+            this.createEventStream(streamName, { stream: streamName }, false);
         }
         assert(!this.streams[streamName].closed, `Stream "${streamName}" is closed and cannot be written to.`);
         let streamVersion = this.streams[streamName].index.length;
-        if (expectedVersion !== ExpectedVersion.Any && streamVersion !== expectedVersion) {
-            throw new OptimisticConcurrencyError(`Optimistic Concurrency error. Expected stream "${streamName}" at version ${expectedVersion} but is at version ${streamVersion}.`);
-        }
+        assert(expectedVersion === ExpectedVersion.Any || streamVersion === expectedVersion,
+            `Optimistic Concurrency error. Expected stream "${streamName}" at version ${expectedVersion} but is at version ${streamVersion}.`,
+            OptimisticConcurrencyError
+        );
 
         if (events.length > 1) {
             delete metadata.commitVersion;
@@ -394,20 +533,59 @@ class EventStore extends events.EventEmitter {
         return this.streams[streamName].index.length;
     }
 
+    /**
+     * Query the event store for events matching a set of event types and an optional filter function.
+     * Returns a pre-filtered event stream and a {@link CommitCondition} that can be passed to
+     * {@link EventStore#commit} to enforce optimistic concurrency.
+     *
+     * A conflict occurs when at least one event appended between the `query` call and the `commit` call
+     * belongs to one of the listed types and (when `matcher` is provided) also satisfies
+     * `matcher(payload, metadata)`.  Events written before the `query` call are never treated as conflicts.
+     *
+     * **Behaviour when a type stream does not exist:**
+     * - Without `typeAccessor` configured: throws an error, because the store cannot guarantee that no
+     *   events of that type exist (the stream was never created).  Create the stream explicitly first,
+     *   or configure `typeAccessor` to have streams created automatically on commit.
+     * - With `typeAccessor` configured: treats the missing stream as empty (0-length).  The stream will
+     *   be created automatically the first time an event of that type is committed.
+     *
+     * @api
+     * @param {string[]} types A non-empty array of event-type names to query.
+     * @param {function|object|null} [matcher] Optional matcher used for stream pre-filtering.
+     *   In object mode, function predicates receive `(payload, metadata)`.
+     * @param {number} [minRevision=1] The 1-based minimum global revision to include in the returned stream (inclusive).
+     * @param {boolean} [raw=false] If true, return NDJSON buffers from the query stream.
+     * @returns {{ condition: CommitCondition, stream: EventStream }} An object with:
+     *   - `condition` — the {@link CommitCondition} to pass to {@link EventStore#commit}.
+     *   - `stream` — a read-only event stream containing all matching events.
+     * @throws {Error} if `types` is not a non-empty array.
+     * @throws {Error} if `typeAccessor` is not configured and any of the listed type streams do not exist.
+     */
+    query(types, matcher = null, minRevision = 1, raw = false) {
+        assert(Array.isArray(types) && types.length > 0, 'Must specify a non-empty array of event types for query.');
+        const queryTypes = this.getExistingQueryTypes(types);
+        const condition = new CommitCondition(types, matcher, this.storage.length, raw);
+        const stream = this.fromStreams('_query_' + types.join('_'), queryTypes, minRevision, -1, matcher, raw);
+        return { stream, condition };
+    }
+
     /**
      * Get an event stream for the given stream name within the revision boundaries.
      *
      * @api
      * @param {string} streamName The name of the stream to get.
-     * @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
-     * @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
+     * @param {number} [minRevision=1] The 1-based minimum revision to include in the events (inclusive).
+     * @param {number} [maxRevision=-1] The 1-based maximum revision to include in the events (inclusive).
+     * @param {function|object|null} [predicate] Optional matcher (see {@link EventStream}).
+     * @param {boolean} [raw=false] If true, return NDJSON buffers.
      * @returns {EventStream|boolean} The event stream or false if a stream with the name doesn't exist.
      */
-    getEventStream(streamName, minRevision = 1, maxRevision = -1) {
+    getEventStream(streamName, minRevision = 1, maxRevision = -1, predicate = null, raw = false) {
+        ({ predicate, raw } = normalizePredicateRaw(predicate, raw));
         if (!(streamName in this.streams)) {
             return false;
         }
-        return new EventStream(streamName, this, minRevision, maxRevision);
+        return new EventStream(streamName, this, minRevision, maxRevision, predicate, raw);
     }
 
     /**
@@ -415,12 +593,15 @@ class EventStore extends events.EventEmitter {
      * This is the same as `getEventStream('_all', ...)`.
      *
      * @api
-     * @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
-     * @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
+     * @param {number} [minRevision=1] The 1-based minimum revision to include in the events (inclusive).
+     * @param {number} [maxRevision=-1] The 1-based maximum revision to include in the events (inclusive).
+     * @param {function|object|null} [predicate] Optional matcher (see {@link EventStream}).
+     * @param {boolean} [raw=false] If true, return NDJSON buffers.
      * @returns {EventStream} The event stream.
      */
-    getAllEvents(minRevision = 1, maxRevision = -1) {
-        return this.getEventStream('_all', minRevision, maxRevision);
+    getAllEvents(minRevision = 1, maxRevision = -1, predicate = null, raw = false) {
+        ({ predicate, raw } = normalizePredicateRaw(predicate, raw));
+        return this.getEventStream('_all', minRevision, maxRevision, predicate, raw);
     }
 
     /**
@@ -428,45 +609,65 @@ class EventStore extends events.EventEmitter {
      *
      * @param {string} streamName The (transient) name of the joined stream.
      * @param {Array<string>} streamNames An array of the stream names to join.
-     * @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
-     * @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
+     * @param {number} [minRevision=1] The 1-based minimum revision to include in the events (inclusive).
+     * @param {number} [maxRevision=-1] The 1-based maximum revision to include in the events (inclusive).
+     * @param {function|object|null} [predicate] Optional matcher (see {@link EventStream}).
+     * @param {boolean} [raw=false] If true, return NDJSON buffers.
      * @returns {EventStream} The joined event stream.
      * @throws {Error} if any of the streams doesn't exist.
      */
-    fromStreams(streamName, streamNames, minRevision = 1, maxRevision = -1) {
+    fromStreams(streamName, streamNames, minRevision = 1, maxRevision = -1, predicate = null, raw = false) {
+        ({ predicate, raw } = normalizePredicateRaw(predicate, raw));
         assert(streamNames instanceof Array, 'Must specify an array of stream names.');
 
+        if (streamNames.length === 0) {
+            return new EventStream(streamName, this);
+        }
+
         for (let stream of streamNames) {
             assert(stream in this.streams, `Stream "${stream}" does not exist.`);
         }
-        return new JoinEventStream(streamName, streamNames, this, minRevision, maxRevision);
+
+        if (streamNames.length === 1) {
+            const stream = new EventStream(streamNames[0], this, minRevision, maxRevision, predicate, raw);
+            stream.name = streamName;
+            return stream;
+        }
+
+        return new JoinEventStream(streamName, streamNames, this, minRevision, maxRevision, predicate, raw);
     }
 
     /**
      * Get a stream for a category of streams. This will effectively return a joined stream of all streams that start
-     * with the given `categoryName` followed by a dash.
+     * with the given `categoryName` followed by a dash (flat layout, e.g. `users-123`) or a slash (hierarchical
+     * layout, e.g. `users/123`).
      * If you frequently use this for a category consisting of a lot of streams (e.g. `users`), consider creating a
      * dedicated physical stream for the category:
      *
-     *    `eventstore.createEventStream('users', e => e.stream.startsWith('users-'))`
+     *    `eventstore.createEventStream('users', e => e.stream.startsWith('users-') || e.stream.startsWith('users/'))`
      *
      * @api
      * @param {string} categoryName The name of the category to get a stream for. A category is a stream name prefix.
-     * @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
-     * @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
+     * @param {number} [minRevision=1] The 1-based minimum revision to include in the events (inclusive).
+     * @param {number} [maxRevision=-1] The 1-based maximum revision to include in the events (inclusive).
+     * @param {function|object|null} [predicate] Optional matcher (see {@link EventStream}).
+     * @param {boolean} [raw=false] If true, return NDJSON buffers.
      * @returns {EventStream} The joined event stream for all streams of the given category.
      * @throws {Error} If no stream for this category exists.
      */
-    getEventStreamForCategory(categoryName, minRevision = 1, maxRevision = -1) {
+    getEventStreamForCategory(categoryName, minRevision = 1, maxRevision = -1, predicate = null, raw = false) {
+        ({ predicate, raw } = normalizePredicateRaw(predicate, raw));
         if (categoryName in this.streams) {
-            return this.getEventStream(categoryName, minRevision, maxRevision);
+            return this.getEventStream(categoryName, minRevision, maxRevision, predicate, raw);
         }
-        const categoryStreams = Object.keys(this.streams).filter(streamName => streamName.startsWith(categoryName + '-'));
+        const categoryStreams = Object.keys(this.streams).filter(streamName =>
+            streamName.startsWith(categoryName + '-') ||
+            streamName.startsWith(categoryName + '/')
+        );
 
-        if (categoryStreams.length === 0) {
-            throw new Error(`No streams for category '${categoryName}' exist.`);
-        }
-        return this.fromStreams(categoryName, categoryStreams, minRevision, maxRevision);
+        assert(categoryStreams.length > 0, `No streams for category '${categoryName}' exist.`);
+
+        return this.fromStreams(categoryName, categoryStreams, minRevision, maxRevision, predicate, raw);
     }
 
     /**
@@ -475,16 +676,21 @@ class EventStore extends events.EventEmitter {
      * @api
      * @param {string} streamName The name of the stream to create.
      * @param {object|function(event)} matcher A matcher object, denoting the properties that need to match on an event a function that takes the event and returns true if the event should be added.
+     * @param {boolean} [reindex=true] Whether to scan existing documents and populate the new index. Set to false when it is known that no existing documents can match the matcher (e.g. when creating a brand-new write stream).
      * @returns {EventStream} The EventStream with all existing events matching the matcher.
      * @throws {Error} If a stream with that name already exists.
      * @throws {Error} If the stream could not be created.
      */
-    createEventStream(streamName, matcher) {
+    createEventStream(streamName, matcher, reindex = true) {
         assert(!(this.storage instanceof ReadOnlyStorage), 'The storage was opened in read-only mode. Can not create new stream on it.');
         assert(!(streamName in this.streams), 'Can not recreate stream!');
 
         const streamIndexName = 'stream-' + streamName;
-        const index = this.storage.ensureIndex(streamIndexName, matcher);
+        if (streamName.includes('/')) {
+            const subDir = path.join(this.streamsDirectory, this.storeName + '.stream-' + path.dirname(streamName));
+            ensureDirectory(subDir);
+        }
+        const index = this.storage.ensureIndex(streamIndexName, matcher, reindex);
         assert(index !== null, `Error creating stream index ${streamName}.`);
 
         // deepcode ignore PrototypePollutionFunctionParams: streams is a Map
@@ -543,7 +749,7 @@ class EventStore extends events.EventEmitter {
         fs.renameSync(index.fileName, closedFileName);
 
         // Remove from secondary indexes so that new writes are no longer indexed into this stream
-        delete this.storage.secondaryIndexes[indexName];
+        this.storage.removeSecondaryIndex(indexName);
 
         // Reopen the renamed index for read access, outside the secondary indexes write path
         const closedIndexName = indexName + '.closed';
@@ -555,43 +761,93 @@ class EventStore extends events.EventEmitter {
     }
 
     /**
-     * Get a durable consumer for the given stream that will keep receiving events from the last position.
+     * Get a durable consumer for the given stream, or look up an existing consumer by identifier.
+     *
+     * When called with a single argument, returns the running consumer registered under that
+     * identifier, or `null` if none is found — useful for read endpoints that need the live
+     * in-memory instance without creating a new one.
+     *
+     * When called with two or more arguments, creates (or re-uses) a Consumer for the given
+     * stream and identifier, registers it in `this.consumers`, and returns it.
      *
-     * @param {string} streamName The name of the stream to consume.
-     * @param {string} identifier The unique identifying name of this consumer.
+     * @param {string} streamNameOrIdentifier The stream name, or the consumer identifier when used as a registry lookup.
+     * @param {string} [identifier] The unique identifying name of this consumer. Omit for registry-only lookup.
      * @param {object} [initialState] The initial state of the consumer.
      * @param {number} [since] The stream revision to start consuming from.
-     * @returns {Consumer} A durable consumer for the given stream.
+     * @returns {Consumer|null} A durable consumer, or `null` when looking up by identifier and none is registered.
      */
-    getConsumer(streamName, identifier, initialState = {}, since = 0) {
+    getConsumer(streamNameOrIdentifier, identifier, initialState = {}, since = 0) {
+        if (identifier === undefined) {
+            return this.consumers.get(streamNameOrIdentifier) ?? null;
+        }
+        const streamName = streamNameOrIdentifier;
+        if (this.consumers.has(identifier)) {
+            return this.consumers.get(identifier);
+        }
         const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since);
         consumer.streamName = streamName;
+        this.consumers.set(identifier, consumer);
         return consumer;
     }
 
     /**
-     * Scan the existing consumers on this EventStore and asynchronously return a list of their names.
-     * @param {function(error: Error, consumers: array)} callback A callback that will receive an error as first and the list of consumers as second argument.
+     * Scan the existing consumers on this EventStore and asynchronously invoke a callback with the parsed list.
+     *
+     * Each consumer entry provides `{ name, stream, identifier }` parsed from the on-disk filename.
+     * Pass `autoStart = true` to eagerly open every discovered consumer and register it in
+     * `this.consumers` so that it is immediately available for registry lookups.
+     *
+     * @param {function(error: Error|null, consumers: Array<{name: string, stream: string, identifier: string}>)} callback
+     * @param {boolean} [autoStart=false] When true, calls `getConsumer(stream, identifier)` for each discovered consumer.
      */
-    scanConsumers(callback) {
+    scanConsumers(callback, autoStart = false) {
         const consumersPath = path.join(this.storage.indexDirectory, 'consumers');
         if (!fs.existsSync(consumersPath)) {
             callback(null, []);
             return;
         }
         const regex = new RegExp(`^${this.storage.storageFile}\\.([^.]*\\..*)$`);
-        const consumers = [];
-        scanForFiles(consumersPath, regex, consumers.push.bind(consumers), /* istanbul ignore next */ (err) => {
+        const consumerNames = [];
+        scanForFiles(consumersPath, regex, consumerNames.push.bind(consumerNames), /* istanbul ignore next */ (err) => {
             if (err) {
                 return callback(err, []);
             }
+            const consumers = consumerNames.map(name => {
+                const splitIndex = name.lastIndexOf('.');
+                const indexName = name.slice(0, splitIndex);
+                const identifier = name.slice(splitIndex + 1);
+                const stream = parseStreamFromIndexName(indexName);
+                return { name, stream, identifier };
+            });
+            if (autoStart) {
+                for (const { stream, identifier } of consumers) {
+                    this.getConsumer(stream, identifier);
+                }
+            }
             callback(null, consumers);
         });
     }
 }
 
+function parseStreamFromIndexName(indexName) {
+    if (indexName === '_all') {
+        return '_all';
+    }
+    if (indexName.startsWith('stream-')) {
+        return indexName.slice(7);
+    }
+    return indexName;
+}
+
+function normalizePredicateRaw(predicate, raw) {
+    if (typeof predicate === 'boolean' && raw === false) {
+        return { predicate: null, raw: predicate };
+    }
+    return { predicate, raw };
+}
+
 EventStore.Storage = Storage;
 EventStore.Index = Index;
 
 export default EventStore;
-export { ExpectedVersion, OptimisticConcurrencyError, LOCK_THROW, LOCK_RECLAIM };
+export { ExpectedVersion, OptimisticConcurrencyError, CommitCondition, LOCK_THROW, LOCK_RECLAIM };