event-storage 0.9.1 → 1.1.0

package/src/PartitionPool.js

@@ -0,0 +1,149 @@
+/**
+ * A fixed-capacity registry of partitions with LRU eviction of open file handles.
+ *
+ * All partitions are stored by their numeric id and may be queried at any time.
+ * The pool additionally tracks which partitions currently have an open file descriptor
+ * in LRU (least-recently-used) order.  When the pool is asked to open a partition and
+ * doing so would exceed the configured cap, the least-recently-used open partition is
+ * closed first to stay within the limit.
+ *
+ * Setting the cap to 0 disables eviction: all partitions are allowed to remain open
+ * simultaneously, which matches the uncapped behaviour of the original implementation.
+ */
+class PartitionPool {
+
+    /**
+     * @param {number} [maxOpen=0] Maximum number of simultaneously open partition file
+     *   handles.  0 disables the limit (no eviction).
+     */
+    constructor(maxOpen = 0) {
+        this.maxOpen = maxOpen;
+        /** Registry of all known partitions keyed by id. */
+        this.registry = Object.create(null);
+        /**
+         * Insertion-order map used for LRU tracking of open file handles.
+         * Key = partition id, value = true.
+         * Oldest (least-recently-used) entry is first; newest (most-recently-used) is last.
+         */
+        this.handles = new Map();
+    }
+
+    /**
+     * Register a partition under the given id.
+     *
+     * @param {number|string} id
+     * @param {object} partition
+     */
+    add(id, partition) {
+        this.registry[id] = partition;
+    }
+
+    /**
+     * Retrieve a registered partition without opening it.
+     *
+     * @param {number|string} id
+     * @returns {object|undefined}
+     */
+    get(id) {
+        return this.registry[id];
+    }
+
+    /**
+     * Check whether a partition with the given id is registered in the pool.
+     *
+     * @param {number|string} id
+     * @returns {boolean}
+     */
+    has(id) {
+        return id in this.registry;
+    }
+
+    /**
+     * Open the partition with the given id, applying LRU eviction if necessary.
+     *
+     * If the partition is not yet open and adding it would exceed `maxOpen`, the
+     * least-recently-used open partition is closed first.  Stale entries (partitions
+     * that were closed externally) are discarded from the LRU map as they are
+     * encountered; if all tracked entries turn out to be stale the loop exits without
+     * closing any partition — the handle count stays temporarily inflated (bounded by
+     * the number of external closes since the last `open()` call) but correctness is
+     * preserved.
+     *
+     * @param {number|string} id
+     * @returns {object} The opened partition.
+     */
+    open(id) {
+        const partition = this.registry[id];
+
+        if (this.maxOpen > 0) {
+            // Remove id first — this may already bring the handle count below the cap.
+            this.handles.delete(id);
+            if (this.handles.size >= this.maxOpen) {
+                for (const [lruId] of this.handles) {
+                    this.handles.delete(lruId);
+                    const lruPartition = this.registry[lruId];
+                    if (lruPartition && lruPartition.isOpen()) {
+                        lruPartition.close();
+                        break;
+                    }
+                }
+            }
+            // (Re-)add id at the MRU end of the map.
+            this.handles.set(id, true);
+        }
+
+        partition.open();
+        return partition;
+    }
+
+    /**
+     * Invoke `callback` for every registered partition.
+     *
+     * @param {function(object): void} callback
+     */
+    forEach(callback) {
+        for (const id of Object.keys(this.registry)) {
+            callback(this.registry[id]);
+        }
+    }
+
+    /**
+     * Yield every registered partition object.
+     *
+     * @returns {Generator<object>}
+     */
+    *values() {
+        for (const id of Object.keys(this.registry)) {
+            yield this.registry[id];
+        }
+    }
+
+    /**
+     * The total number of registered partitions.
+     * @returns {number}
+     */
+    get count() {
+        return Object.keys(this.registry).length;
+    }
+
+    /**
+     * The number of open partition file handles currently tracked by the pool.
+     * @returns {number}
+     */
+    get openCount() {
+        return this.handles.size;
+    }
+
+    /**
+     * Reset the open-handle tracking without closing any partitions.
+     *
+     * Call this after externally closing all partitions (e.g. after
+     * `checkTornWrites`) to keep the pool's LRU state consistent with reality.
+     */
+    clearOpenHandles() {
+        this.handles.clear();
+    }
+
+}
+
+export default PartitionPool;

package/src/Storage/ReadOnlyStorage.js

@@ -1,6 +1,6 @@
-const ReadableStorage = require('./ReadableStorage');
-const ReadablePartition = require('../Partition/ReadablePartition');
-const Watcher = require('../Watcher');
+import ReadableStorage from './ReadableStorage.js';
+import ReadablePartition from '../Partition/ReadablePartition.js';
+import Watcher from '../Watcher.js';
 
 /**
  * An append-only storage with highly performant positional range scans.
@@ -54,9 +54,9 @@ class ReadOnlyStorage extends ReadableStorage {
         }
 
         const partitionId = ReadablePartition.idFor(filename);
-        if (!this.partitions[partitionId]) {
+        if (!this.partitions.has(partitionId)) {
             const partition = this.createPartition(filename, this.partitionConfig);
-            this.partitions[partition.id] = partition;
+            this.partitions.add(partition.id, partition);
             this.emit('partition-created', partition.id);
         }
     }
@@ -113,4 +113,4 @@ class ReadOnlyStorage extends ReadableStorage {
     }
 }
 
-module.exports = ReadOnlyStorage;
+export default ReadOnlyStorage;

package/src/Storage/ReadableStorage.js

@@ -1,13 +1,31 @@
-const fs = require('fs');
-const path = require('path');
-const events = require('events');
-const Partition = require('../Partition');
-const Index = require('../Index');
-const { assert, wrapAndCheck, kWayMerge } = require('../util');
-const { createHmac, matches, buildMetadataForMatcher } = require('../metadataUtil');
+import fs from 'fs';
+import path from 'path';
+import events from 'events';
+import Partition, { ReadOnly as ReadOnlyPartition } from '../Partition.js';
+import Index, { ReadOnly as ReadOnlyIndex } from '../Index.js';
+import { assert, wrapAndCheck, kWayMerge } from '../util.js';
+import { scanForFiles } from '../fsUtil.js';
+import { createHmac, matches, buildMetadataForMatcher } from '../metadataUtil.js';
+import IndexMatcher from '../IndexMatcher.js';
+import PartitionPool from '../PartitionPool.js';
 
 const DEFAULT_READ_BUFFER_SIZE = 4 * 1024;
 
+/**
+ * Default ordered list of document property paths used as discriminant keys when
+ * classifying object matchers into the fast-lookup table.  Each path may use
+ * dot-notation for nested access (e.g. `'payload.type'`).  The first path that
+ * resolves to a scalar value in a given matcher wins; remaining paths are not
+ * examined for that matcher.
+ */
+const DEFAULT_MATCHER_PROPERTIES = ['stream', 'payload.type'];
+
+/**
+ * Default maximum number of partition file descriptors kept open simultaneously.
+ * Partitions beyond this limit are evicted using LRU order. 0 disables the limit.
+ */
+const DEFAULT_MAX_OPEN_PARTITIONS = 1024;
+
 /**
  * Reverses the items of an iterable
  * @param {Generator|Iterable} iterator
@@ -43,6 +61,13 @@ class ReadableStorage extends events.EventEmitter {
      * @param {object} [config.indexOptions] An options object that should be passed to all indexes on construction.
      * @param {string} [config.hmacSecret] A private key that is used to verify matchers retrieved from indexes.
      * @param {object} [config.metadata] A metadata object to be stored in all partitions belonging to this storage.
+     * @param {string[]} [config.matcherProperties] Ordered list of document property paths (dot-notation) used as
+     *   discriminant keys for the fast secondary-index lookup table. Only the first property that resolves to a scalar
+     *   value inside a given object matcher is used; the rest are checked via the full `matches()` fallback.
+     *   Default: `['stream', 'payload.type']`.
+     * @param {number} [config.maxOpenPartitions] Maximum number of partition file descriptors kept open at one time.
+     *   When the limit is reached the least-recently-used partition is closed to make room. 0 disables the limit.
+     *   Default: 1024.
      */
     constructor(storageName = 'storage', config = {}) {
         super();
@@ -58,7 +83,9 @@ class ReadableStorage extends events.EventEmitter {
             indexFile: this.storageFile + '.index',
             indexOptions: {},
             hmacSecret: '',
-            metadata: {}
+            metadata: {},
+            matcherProperties: DEFAULT_MATCHER_PROPERTIES,
+            maxOpenPartitions: DEFAULT_MAX_OPEN_PARTITIONS
         };
         config = Object.assign(defaults, config);
         this.serializer = config.serializer;
@@ -67,7 +94,13 @@ class ReadableStorage extends events.EventEmitter {
 
         this.dataDirectory = path.resolve(config.dataDirectory);
 
-        this.scanPartitions(config);
+        const partitionDefaults = { readBufferSize: DEFAULT_READ_BUFFER_SIZE };
+        this.partitionConfig = Object.assign(partitionDefaults, config);
+        this.partitions = new PartitionPool(config.maxOpenPartitions);
+
+        // initialized: null = not started (or scan cancelled), false = in progress, true = done
+        this.initialized = null;
+
         this.initializeIndexes(config);
     }
 
@@ -79,7 +112,7 @@ class ReadableStorage extends events.EventEmitter {
      */
     createIndex(name, options = {}) {
         /** @type ReadableIndex */
-        const index = new Index.ReadOnly(name, options);
+        const index = new ReadOnlyIndex(name, options);
         return { index };
     }
 
@@ -90,7 +123,7 @@ class ReadableStorage extends events.EventEmitter {
      * @returns {ReadablePartition}
      */
     createPartition(name, options = {}) {
-        return new Partition.ReadOnly(name, options);
+        return new ReadOnlyPartition(name, options);
     }
 
     /**
@@ -111,6 +144,9 @@ class ReadableStorage extends events.EventEmitter {
         this.index = index;
         this.secondaryIndexes = {};
         this.readonlyIndexes = {};
+
+        /** Fast secondary-index lookup — classifies matchers for O(1) candidate resolution on write. */
+        this.indexMatcher = new IndexMatcher(config.matcherProperties);
     }
 
     /**
@@ -122,56 +158,91 @@ class ReadableStorage extends events.EventEmitter {
     }
 
     /**
-     * Scan the data directory for all existing partitions.
-     * Every file beginning with the storageFile name is considered a partition.
-     *
-     * @private
-     * @param {object} config The configuration object containing options for the partitions.
-     * @returns void
+     * Scan partitions and secondary index files; emit 'index-created' for each found index.
+     * @param {function} done Called when both scans finish.
      */
-    scanPartitions(config) {
-        const defaults = {
-            readBufferSize: DEFAULT_READ_BUFFER_SIZE
-        };
-        this.partitionConfig = Object.assign(defaults, config);
-        this.partitions = Object.create(null);
+    scanFiles(done) {
+        const escaped = this.storageFile.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+        const partitionPattern = new RegExp(`^(${escaped}.*)$`);
+        scanForFiles(this.dataDirectory, partitionPattern, (file) => {
+            if (file.endsWith('.index') || file.endsWith('.branch') || file.endsWith('.lock')) return;
+            const partition = this.createPartition(file, this.partitionConfig);
+            this.partitions.add(partition.id, partition);
+        }, (partErr) => {
+            /* istanbul ignore if */
+            if (partErr) throw partErr;
 
-        const files = fs.readdirSync(this.dataDirectory);
-        for (let file of files) {
-            if (file.substr(-6) === '.index') continue;
-            if (file.substr(-7) === '.branch') continue;
-            if (file.substr(-5) === '.lock') continue;
-            if (file.substr(0, this.storageFile.length) !== this.storageFile) continue;
+            // Scan was cancelled by close() between the two scan phases.
+            if (this.initialized === null) return;
 
-            const partition = this.createPartition(file, this.partitionConfig);
-            this.partitions[partition.id] = partition;
-        }
+            // No secondary indexes exist yet — nothing to scan.
+            if (!fs.existsSync(this.indexDirectory)) {
+                return done();
+            }
+            const indexPattern = new RegExp(`^${escaped}\\.(.+)\\.index$`);
+            scanForFiles(this.indexDirectory, indexPattern, (name) => {
+                this.emit('index-created', name);
+            }, (indexErr) => {
+                // The directory could disappear between existsSync and readdir (e.g. test cleanup).
+                /* istanbul ignore if */
+                if (indexErr && indexErr.code !== 'ENOENT') throw indexErr;
+                done();
+            });
+        });
     }
 
     /**
-     * Open the storage and indexes and create read and write buffers eagerly.
-     * Will emit an 'opened' event if finished.
+     * Only the primary index is opened eagerly; secondary indexes open on demand.
      *
-     * @api
-     * @returns {boolean}
+     * @protected
      */
-    open() {
+    openIndexes() {
         this.index.open();
+    }
 
-        this.forEachSecondaryIndex(index => index.open());
-
-        this.emit('opened');
+    /**
+     * Open the storage; scans existing partitions and indexes asynchronously on first open.
+     * Re-opens after `close()` are synchronous.
+     * Will emit an `'opened'` event when finished.
+     *
+     * @api
+     * @param {function(): void} [callback] Called after indexes open, before `'opened'` is emitted.
+     *   Can be used as a synchronous alternative to listening to the `'opened'` event.
+     * @returns {boolean}
+     */
+    open(callback) {
+        if (this.initialized === true) {
+            this.openIndexes();
+            callback?.();
+            this.emit('opened');
+            return true;
+        }
+        if (this.initialized === false) {
+            return true;
+        }
+        this.initialized = false;
+        this.scanFiles(() => {
+            // Guard: close() while scanning resets initialized to null.
+            if (this.initialized === null) return;
+            this.initialized = true;
+            this.openIndexes();
+            callback?.();
+            this.emit('opened');
+        });
         return true;
     }
 
     /**
-     * Close the storage and frees up all resources.
+     * Close the storage and free up all resources.
      * Will emit a 'closed' event when finished.
      *
      * @api
-     * @returns void
      */
     close() {
+        // Cancel in-progress scan so the callback does not re-open after an explicit close.
+        if (this.initialized === false) {
+            this.initialized = null;
+        }
         this.index.close();
         this.forEachSecondaryIndex(index => index.close());
         for (let index of Object.values(this.readonlyIndexes)) {
@@ -182,20 +253,17 @@ class ReadableStorage extends events.EventEmitter {
     }
 
     /**
-     * Get a partition either by name or by id.
-     * If a partition with the given name does not exist, a new one will be created.
+     * Get a partition by its id.
      * If a partition with the given id does not exist, an error is thrown.
      *
      * @protected
-     * @param {string|number} partitionIdentifier The partition name or the partition Id
+     * @param {number|string} partitionIdentifier The partition Id
      * @returns {ReadablePartition}
-     * @throws {Error} If an id is given and no such partition exists.
+     * @throws {Error} If no such partition exists.
      */
     getPartition(partitionIdentifier) {
-        assert(partitionIdentifier in this.partitions, `Partition #${partitionIdentifier} does not exist.`);
-
-        this.partitions[partitionIdentifier].open();
-        return this.partitions[partitionIdentifier];
+        assert(this.partitions.has(partitionIdentifier), `Partition #${partitionIdentifier} does not exist.`);
+        return this.partitions.open(partitionIdentifier);
     }
 
     /**
@@ -359,10 +427,27 @@ class ReadableStorage extends events.EventEmitter {
         const metadata = buildMetadataForMatcher(matcher, this.hmac);
         let { index } = this.secondaryIndexes[name] = this.createIndex(indexName, Object.assign({}, this.indexOptions, { metadata }));
 
+        // Register the actual stored matcher (may have been reconstructed from metadata by WritableStorage.createIndex).
+        this.indexMatcher.add(name, this.secondaryIndexes[name].matcher);
+
         index.open();
         return index;
     }
 
+    /**
+     * Remove a secondary index from the write path and the matcher lookup table.
+     *
+     * @api
+     * @param {string} name The secondary index name to remove.
+     */
+    removeSecondaryIndex(name) {
+        const entry = this.secondaryIndexes[name];
+        if (entry) {
+            this.indexMatcher.remove(name);
+            delete this.secondaryIndexes[name];
+        }
+    }
+
     /**
      * Iterate documents across all partitions in sequenceNumber order using a k-way merge.
      * Opens any closed partition automatically.
@@ -448,6 +533,9 @@ class ReadableStorage extends events.EventEmitter {
     /**
      * Helper method to iterate over all secondary indexes.
      *
+     * When `matchDocument` is provided, `this.indexMatcher.forEachMatch()` is used to
+     * efficiently find only the matching indexes via the discriminant lookup table.
+     *
      * @protected
      * @param {function(ReadableIndex, string)} iterationHandler
      * @param {object} [matchDocument] If supplied, only indexes the document matches on will be iterated.
@@ -458,11 +546,17 @@ class ReadableStorage extends events.EventEmitter {
             return;
         }
 
-        for (let indexName of Object.keys(this.secondaryIndexes)) {
-            if (!matchDocument || matches(matchDocument, this.secondaryIndexes[indexName].matcher)) {
+        if (!matchDocument) {
+            // No document filter: iterate all secondary indexes unconditionally.
+            for (const indexName of Object.keys(this.secondaryIndexes)) {
                 iterationHandler(this.secondaryIndexes[indexName].index, indexName);
             }
+            return;
         }
+
+        this.indexMatcher.forEachMatch(matchDocument, indexName => {
+            iterationHandler(this.secondaryIndexes[indexName].index, indexName);
+        });
     }
 
     /**
@@ -477,13 +571,10 @@ class ReadableStorage extends events.EventEmitter {
             return;
         }
 
-        for (let partition of Object.keys(this.partitions)) {
-            iterationHandler(this.partitions[partition]);
-        }
+        this.partitions.forEach(iterationHandler);
     }
 
 }
 
-module.exports = ReadableStorage;
-module.exports.matches = matches;
-module.exports.CorruptFileError = Partition.CorruptFileError;
+export default ReadableStorage;
+export { matches };