event-storage 1.0.0 → 1.2.0

package/src/Storage/ReadOnlyStorage.js

@@ -23,7 +23,7 @@ class ReadOnlyStorage extends ReadableStorage {
      * @returns {boolean}
      */
     storageFilesFilter(filename) {
-        return filename.substr(-7) !== '.branch' && filename.substr(0, this.storageFile.length) === this.storageFile;
+        return !filename.endsWith('.branch') && filename.substring(0, this.storageFile.length) === this.storageFile;
     }
 
     /**
@@ -46,17 +46,17 @@ class ReadOnlyStorage extends ReadableStorage {
      * @param {string} filename
      */
     onStorageFileChanged(filename) {
-        if (filename.substr(-6) === '.index') {
-            const indexName = filename.substr(this.storageFile.length + 1, filename.length - this.storageFile.length - 7);
+        if (filename.endsWith('.index')) {
+            const indexName = filename.substring(this.storageFile.length + 1, filename.length - 6);
             // New indexes are not automatically opened in the reader
             this.emit('index-created', indexName);
             return;
         }
 
         const partitionId = ReadablePartition.idFor(filename);
-        if (!this.partitions[partitionId]) {
+        if (!this.partitions.has(partitionId)) {
             const partition = this.createPartition(filename, this.partitionConfig);
-            this.partitions[partition.id] = partition;
+            this.partitions.add(partition.id, partition);
             this.emit('partition-created', partition.id);
         }
     }

package/src/Storage/ReadableStorage.js

@@ -3,22 +3,29 @@ import path from 'path';
 import events from 'events';
 import Partition, { ReadOnly as ReadOnlyPartition } from '../Partition.js';
 import Index, { ReadOnly as ReadOnlyIndex } from '../Index.js';
-import { assert, wrapAndCheck, kWayMerge } from '../util.js';
-import { createHmac, matches, buildMetadataForMatcher } from '../metadataUtil.js';
+import { assert, wrapAndCheck, iterate, kWayMerge } from '../utils/util.js';
+import { scanForFiles } from '../utils/fsUtil.js';
+import { createHmac, matches, buildMetadataForMatcher } from '../utils/metadataUtil.js';
+import IndexMatcher from '../IndexMatcher.js';
+import PartitionPool from '../PartitionPool.js';
 
 const DEFAULT_READ_BUFFER_SIZE = 4 * 1024;
+const NDJSON_NEWLINE = Buffer.from('\n');
 
 /**
- * Reverses the items of an iterable
- * @param {Generator|Iterable} iterator
- * @returns {Generator<*>}
+ * Default ordered list of document property paths used as discriminant keys when
+ * classifying object matchers into the fast-lookup table.  Each path may use
+ * dot-notation for nested access (e.g. `'payload.type'`).  The first path that
+ * resolves to a scalar value in a given matcher wins; remaining paths are not
+ * examined for that matcher.
  */
-function *reverse(iterator) {
-    const items = Array.from(iterator);
-    for (let i = items.length - 1; i >= 0; i--) {
-        yield items[i];
-    }
-}
+const DEFAULT_MATCHER_PROPERTIES = ['stream', 'payload.type'];
+
+/**
+ * Default maximum number of partition file descriptors kept open simultaneously.
+ * Partitions beyond this limit are evicted using LRU order. 0 disables the limit.
+ */
+const DEFAULT_MAX_OPEN_PARTITIONS = 1024;
 
 /**
  * @typedef {object|function(object):boolean} Matcher
@@ -43,6 +50,13 @@ class ReadableStorage extends events.EventEmitter {
      * @param {object} [config.indexOptions] An options object that should be passed to all indexes on construction.
      * @param {string} [config.hmacSecret] A private key that is used to verify matchers retrieved from indexes.
      * @param {object} [config.metadata] A metadata object to be stored in all partitions belonging to this storage.
+     * @param {string[]} [config.matcherProperties] Ordered list of document property paths (dot-notation) used as
+     *   discriminant keys for the fast secondary-index lookup table. Only the first property that resolves to a scalar
+     *   value inside a given object matcher is used; the rest are checked via the full `matches()` fallback.
+     *   Default: `['stream', 'payload.type']`.
+     * @param {number} [config.maxOpenPartitions] Maximum number of partition file descriptors kept open at one time.
+     *   When the limit is reached the least-recently-used partition is closed to make room. 0 disables the limit.
+     *   Default: 1024.
      */
     constructor(storageName = 'storage', config = {}) {
         super();
@@ -58,7 +72,9 @@ class ReadableStorage extends events.EventEmitter {
             indexFile: this.storageFile + '.index',
             indexOptions: {},
             hmacSecret: '',
-            metadata: {}
+            metadata: {},
+            matcherProperties: DEFAULT_MATCHER_PROPERTIES,
+            maxOpenPartitions: DEFAULT_MAX_OPEN_PARTITIONS
         };
         config = Object.assign(defaults, config);
         this.serializer = config.serializer;
@@ -67,7 +83,13 @@ class ReadableStorage extends events.EventEmitter {
 
         this.dataDirectory = path.resolve(config.dataDirectory);
 
-        this.scanPartitions(config);
+        const partitionDefaults = { readBufferSize: DEFAULT_READ_BUFFER_SIZE };
+        this.partitionConfig = Object.assign(partitionDefaults, config);
+        this.partitions = new PartitionPool(config.maxOpenPartitions);
+
+        // initialized: null = not started (or scan cancelled), false = in progress, true = done
+        this.initialized = null;
+
         this.initializeIndexes(config);
     }
 
@@ -111,6 +133,9 @@ class ReadableStorage extends events.EventEmitter {
         this.index = index;
         this.secondaryIndexes = {};
         this.readonlyIndexes = {};
+
+        /** Fast secondary-index lookup — classifies matchers for O(1) candidate resolution on write. */
+        this.indexMatcher = new IndexMatcher(config.matcherProperties);
     }
 
     /**
@@ -122,56 +147,91 @@ class ReadableStorage extends events.EventEmitter {
     }
 
     /**
-     * Scan the data directory for all existing partitions.
-     * Every file beginning with the storageFile name is considered a partition.
-     *
-     * @private
-     * @param {object} config The configuration object containing options for the partitions.
-     * @returns void
+     * Scan partitions and secondary index files; emit 'index-created' for each found index.
+     * @param {function} done Called when both scans finish.
      */
-    scanPartitions(config) {
-        const defaults = {
-            readBufferSize: DEFAULT_READ_BUFFER_SIZE
-        };
-        this.partitionConfig = Object.assign(defaults, config);
-        this.partitions = Object.create(null);
+    scanFiles(done) {
+        const escaped = this.storageFile.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+        const partitionPattern = new RegExp(`^(${escaped}.*)$`);
+        scanForFiles(this.dataDirectory, partitionPattern, (file) => {
+            if (file.endsWith('.index') || file.endsWith('.branch') || file.endsWith('.lock')) return;
+            const partition = this.createPartition(file, this.partitionConfig);
+            this.partitions.add(partition.id, partition);
+        }, (partErr) => {
+            /* istanbul ignore if */
+            if (partErr) throw partErr;
 
-        const files = fs.readdirSync(this.dataDirectory);
-        for (let file of files) {
-            if (file.substr(-6) === '.index') continue;
-            if (file.substr(-7) === '.branch') continue;
-            if (file.substr(-5) === '.lock') continue;
-            if (file.substr(0, this.storageFile.length) !== this.storageFile) continue;
+            // Scan was cancelled by close() between the two scan phases.
+            if (this.initialized === null) return;
 
-            const partition = this.createPartition(file, this.partitionConfig);
-            this.partitions[partition.id] = partition;
-        }
+            // No secondary indexes exist yet — nothing to scan.
+            if (!fs.existsSync(this.indexDirectory)) {
+                return done();
+            }
+            const indexPattern = new RegExp(`^${escaped}\\.(.+)\\.index$`);
+            scanForFiles(this.indexDirectory, indexPattern, (name) => {
+                this.emit('index-created', name);
+            }, (indexErr) => {
+                // The directory could disappear between existsSync and readdir (e.g. test cleanup).
+                /* istanbul ignore if */
+                if (indexErr && indexErr.code !== 'ENOENT') throw indexErr;
+                done();
+            });
+        });
     }
 
     /**
-     * Open the storage and indexes and create read and write buffers eagerly.
-     * Will emit an 'opened' event if finished.
+     * Only the primary index is opened eagerly; secondary indexes open on demand.
      *
-     * @api
-     * @returns {boolean}
+     * @protected
      */
-    open() {
+    openIndexes() {
         this.index.open();
+    }
 
-        this.forEachSecondaryIndex(index => index.open());
-
-        this.emit('opened');
+    /**
+     * Open the storage; scans existing partitions and indexes asynchronously on first open.
+     * Re-opens after `close()` are synchronous.
+     * Will emit an `'opened'` event when finished.
+     *
+     * @api
+     * @param {function(): void} [callback] Called after indexes open, before `'opened'` is emitted.
+     *   Can be used as a synchronous alternative to listening to the `'opened'` event.
+     * @returns {boolean}
+     */
+    open(callback) {
+        if (this.initialized === true) {
+            this.openIndexes();
+            callback?.();
+            this.emit('opened');
+            return true;
+        }
+        if (this.initialized === false) {
+            return true;
+        }
+        this.initialized = false;
+        this.scanFiles(() => {
+            // Guard: close() while scanning resets initialized to null.
+            if (this.initialized === null) return;
+            this.initialized = true;
+            this.openIndexes();
+            callback?.();
+            this.emit('opened');
+        });
         return true;
     }
 
     /**
-     * Close the storage and frees up all resources.
+     * Close the storage and free up all resources.
      * Will emit a 'closed' event when finished.
      *
      * @api
-     * @returns void
      */
     close() {
+        // Cancel in-progress scan so the callback does not re-open after an explicit close.
+        if (this.initialized === false) {
+            this.initialized = null;
+        }
         this.index.close();
         this.forEachSecondaryIndex(index => index.close());
         for (let index of Object.values(this.readonlyIndexes)) {
@@ -182,20 +242,17 @@ class ReadableStorage extends events.EventEmitter {
     }
 
     /**
-     * Get a partition either by name or by id.
-     * If a partition with the given name does not exist, a new one will be created.
+     * Get a partition by its id.
      * If a partition with the given id does not exist, an error is thrown.
      *
      * @protected
-     * @param {string|number} partitionIdentifier The partition name or the partition Id
+     * @param {number|string} partitionIdentifier The partition Id
      * @returns {ReadablePartition}
-     * @throws {Error} If an id is given and no such partition exists.
+     * @throws {Error} If no such partition exists.
      */
     getPartition(partitionIdentifier) {
-        assert(partitionIdentifier in this.partitions, `Partition #${partitionIdentifier} does not exist.`);
-
-        this.partitions[partitionIdentifier].open();
-        return this.partitions[partitionIdentifier];
+        assert(this.partitions.has(partitionIdentifier), `Partition #${partitionIdentifier} does not exist.`);
+        return this.partitions.open(partitionIdentifier);
     }
 
     /**
@@ -216,16 +273,19 @@ class ReadableStorage extends events.EventEmitter {
      * @param {number} partitionId The partition to read from.
      * @param {number} position The file position to read from.
      * @param {number} [size] The expected byte size of the document at the given position.
-     * @returns {object} The document stored at the given position.
+     * @param {boolean} [raw] Whether to return raw buffers instead of deserialized objects. Default false.
+     * @param {boolean} [backwardsHint] If set to true, will optimize buffering for backwards reading.
+     * @returns {object|{ buffer: Buffer, time64: number, sequenceNumber: number }} The document stored at the given position.
      * @throws {Error} if the document at the given position can not be deserialized.
      */
-    readFrom(partitionId, position, size) {
+    readFrom(partitionId, position, size, raw = false, backwardsHint = false) {
         const partition = this.getPartition(partitionId);
         if (this.listenerCount('preRead') > 0) {
             this.emit('preRead', position, partition.metadata);
         }
-        const data = partition.readFrom(position, size);
-        return this.serializer.deserialize(data);
+        const headerOut = {};
+        const buffer = partition.readFrom(position, size, headerOut, backwardsHint);
+        return raw ? { buffer, time64: headerOut.time64, sequenceNumber: headerOut.sequenceNumber } : this.serializer.deserialize(buffer.toString('utf8'));
     }
 
     /**
@@ -238,10 +298,7 @@ class ReadableStorage extends events.EventEmitter {
      */
     read(number, index) {
         index = index || this.index;
-
-        if (!index.isOpen()) {
-            index.open();
-        }
+        index.open();
 
         const entry = index.get(number);
         if (entry === false) {
@@ -261,30 +318,22 @@ class ReadableStorage extends events.EventEmitter {
      * @param {ReadableIndex|false} [index] The index to use for finding the documents in the range.
      *   Pass `false` to skip the global index and iterate all partitions directly in sequenceNumber order
      *   (useful when the global index is unavailable or corrupted).
+     * @param {boolean} [raw] Whether to return raw buffers instead of deserialized objects. Default false.
      * @returns {Generator<object>} A generator that will read each document in the range one by one.
      */
-    *readRange(from, until = -1, index = null) {
-        const lengthSource = index || this.index;
-        if (!lengthSource.isOpen()) {
-            lengthSource.open();
+    *readRange(from, until = -1, index = null, raw = false) {
+        let length = Number.MAX_SAFE_INTEGER;
+        if (index !== false) {
+            index = index || this.index;
+            index.open();
+            length = index.length;
         }
 
-        const readFrom = wrapAndCheck(from, lengthSource.length);
-        const readUntil = wrapAndCheck(until, lengthSource.length);
+        const readFrom = wrapAndCheck(from, length);
+        const readUntil = wrapAndCheck(until, length);
         assert(readFrom > 0 && readUntil > 0, `Range scan error for range ${from} - ${until}.`);
 
-        if (readFrom > readUntil) {
-            const batchSize = 10;
-            let batchUntil = readFrom;
-            while (batchUntil >= readUntil) {
-                const batchFrom = Math.max(readUntil, batchUntil - batchSize);
-                yield* reverse(this.iterateRange(batchFrom, batchUntil, index));
-                batchUntil = batchFrom - 1;
-            }
-            return undefined;
-        }
-
-        yield* this.iterateRange(readFrom, readUntil, index);
+        yield* this.iterateRange(readFrom, readUntil, index, raw);
     }
 
     /**
@@ -294,23 +343,25 @@ class ReadableStorage extends events.EventEmitter {
      * @param {number} from
      * @param {number} until
      * @param {ReadableIndex|false|null} index
+     * @param {boolean} [raw] Whether to return raw buffers instead of deserialized objects. Default false.
      * @returns {Generator<object>}
      */
-    *iterateRange(from, until, index) {
+    *iterateRange(from, until, index, raw = false) {
         if (index === false) {
-            // Explicitly disabled index: iterate all partitions and merge by sequenceNumber.
-            // Document header sequenceNumber is 0-based; from/until are 1-based index positions.
-            for (const entry of this.iterateDocumentsNoIndex(from - 1, until - 1)) {
-                yield entry.document;
+            for (const { document } of this.iterateDocumentsNoIndex(from - 1, until - 1)) {
+                yield document;
             }
             return;
         }
 
         const idx = index || this.index;
-        const entries = idx.range(from, until);
-        for (let entry of entries) {
-            const document = this.readFrom(entry.partition, entry.position, entry.size);
-            yield document;
+        const forwards = from <= until;
+        const lo = Math.min(from, until);
+        const hi = Math.max(from, until);
+        const entries = idx.range(lo, hi);
+        if (!entries) return;
+        for (const entry of iterate(entries, forwards)) {
+            yield this.readFrom(entry.partition, entry.position, entry.size, raw, !forwards);
         }
     }
 
@@ -359,66 +410,71 @@ class ReadableStorage extends events.EventEmitter {
         const metadata = buildMetadataForMatcher(matcher, this.hmac);
         let { index } = this.secondaryIndexes[name] = this.createIndex(indexName, Object.assign({}, this.indexOptions, { metadata }));
 
+        // Register the actual stored matcher (may have been reconstructed from metadata by WritableStorage.createIndex).
+        this.indexMatcher.add(name, this.secondaryIndexes[name].matcher);
+
         index.open();
         return index;
     }
 
     /**
-     * Iterate documents across all partitions in sequenceNumber order using a k-way merge.
-     * Opens any closed partition automatically.
+     * Remove a secondary index from the write path and the matcher lookup table.
      *
-     * @protected
-     * @param {number} [from=0] The 0-based sequenceNumber to start from (inclusive).
-     * @param {number} [until=Number.MAX_SAFE_INTEGER] The 0-based sequenceNumber to read until (inclusive).
-     * @returns {Generator<{document: object, sequenceNumber: number, partitionName: string, position: number, size: number, partition: number}>}
+     * @api
+     * @param {string} name The secondary index name to remove.
      */
-    *iterateDocumentsNoIndex(from = 0, until = Number.MAX_SAFE_INTEGER) {
-        const streams = [];
-
-        this.forEachPartition(partition => {
-            if (!partition.isOpen()) {
-                partition.open();
-            }
-
-            const found = partition.findDocument(from);
-            if (found && found.headerOut.sequenceNumber <= until) {
-                const nextPosition = found.headerOut.position + partition.documentWriteSize(found.headerOut.dataSize);
-                const reader = partition.readAll(nextPosition, found.headerOut);
-                streams.push({ ...found, reader, partition: partition.id, partitionName: partition.name });
-            }
-        });
-
-        const items = [];
-        kWayMerge(
-            streams,
-            stream => stream.headerOut.sequenceNumber,
-            stream => {
-                const next = stream.reader.next();
-                if (!next.done && stream.headerOut.sequenceNumber <= until) {
-                    stream.data = next.value;
-                    return true;
-                }
-                return false;
-            },
-            stream => items.push({
-                document: this.serializer.deserialize(stream.data),
-                sequenceNumber: stream.headerOut.sequenceNumber,
-                partitionName: stream.partitionName,
-                position: stream.headerOut.position,
-                size: stream.headerOut.dataSize,
-                partition: stream.partition,
-            })
-        );
-
-        yield* items;
+    removeSecondaryIndex(name) {
+        const entry = this.secondaryIndexes[name];
+        if (entry) {
+            this.indexMatcher.remove(name);
+            delete this.secondaryIndexes[name];
+        }
     }
 
+     /**
+      * Build the standard document result entry from a readRange yield.
+      * @private
+      * @param {{ data: Buffer, entry: { number: number, position: number, size: number, partition: number } }} [readItem]
+      */
+     buildDocumentEntry(readItem) {
+         return {
+             document: this.serializer.deserialize(readItem.data.toString('utf8')),
+             // Replicate the index entry structure here, so iteration can be used easily to reindex
+             entry: readItem.entry
+         };
+     }
+
+     /**
+      * Iterate documents across all partitions in sequenceNumber order using a k-way merge.
+      * Opens any closed partition automatically.
+      *
+      * @protected
+      * @param {number} [from=0] The 0-based sequenceNumber to start from (inclusive).
+      * @param {number} [until=Number.MAX_SAFE_INTEGER] The 0-based sequenceNumber to read until (inclusive).
+      * @returns {Generator<{document: object, entry: { sequenceNumber: number, position: number, size: number, partition: number }}>}
+      */
+     *iterateDocumentsNoIndex(from = 0, until = Number.MAX_SAFE_INTEGER) {
+         const forwards = from <= until;
+         const partitions = [];
+         this.forEachPartition(partition => {
+             partition.open();
+             partitions.push(partition.readRange(from, until));
+         });
+
+         yield* kWayMerge(
+             partitions,
+             item => item.entry.number,
+             forwards,
+             item => this.buildDocumentEntry(item)
+         );
+     }
+
     /**
      * Helper method to iterate over all documents, invoking a callback for each one.
      * Pass `noIndex = true` to iterate all partitions directly in sequenceNumber order
      * (useful when the global index is unavailable or corrupted).
      * When `noIndex` is false the second callback argument is the raw index `EntryInterface`.
-     * When `noIndex` is true the second callback argument has `{ partition, position, size, sequenceNumber, partitionName }`.
+     * When `noIndex` is true the second callback argument has `{ partition, position, size, sequenceNumber }`.
      *
      * @protected
      * @param {function(object, object): void} iterationHandler
@@ -431,8 +487,8 @@ class ReadableStorage extends events.EventEmitter {
         }
 
         if (noIndex) {
-            for (const { document, ...entryInfo } of this.iterateDocumentsNoIndex()) {
-                iterationHandler(document, entryInfo);
+            for (const { document, entry } of this.iterateDocumentsNoIndex()) {
+                iterationHandler(document, entry);
             }
             return;
         }
@@ -448,6 +504,9 @@ class ReadableStorage extends events.EventEmitter {
     /**
      * Helper method to iterate over all secondary indexes.
      *
+     * When `matchDocument` is provided, `this.indexMatcher.forEachMatch()` is used to
+     * efficiently find only the matching indexes via the discriminant lookup table.
+     *
      * @protected
      * @param {function(ReadableIndex, string)} iterationHandler
      * @param {object} [matchDocument] If supplied, only indexes the document matches on will be iterated.
@@ -458,11 +517,17 @@ class ReadableStorage extends events.EventEmitter {
             return;
         }
 
-        for (let indexName of Object.keys(this.secondaryIndexes)) {
-            if (!matchDocument || matches(matchDocument, this.secondaryIndexes[indexName].matcher)) {
+        if (!matchDocument) {
+            // No document filter: iterate all secondary indexes unconditionally.
+            for (const indexName of Object.keys(this.secondaryIndexes)) {
                 iterationHandler(this.secondaryIndexes[indexName].index, indexName);
             }
+            return;
         }
+
+        this.indexMatcher.forEachMatch(matchDocument, indexName => {
+            iterationHandler(this.secondaryIndexes[indexName].index, indexName);
+        });
     }
 
     /**
@@ -477,9 +542,7 @@ class ReadableStorage extends events.EventEmitter {
             return;
         }
 
-        for (let partition of Object.keys(this.partitions)) {
-            iterationHandler(this.partitions[partition]);
-        }
+        this.partitions.forEach(iterationHandler);
     }
 
 }