event-storage 1.0.0 → 1.2.0

package/src/Storage/WritableStorage.js

@@ -3,8 +3,9 @@ import path from 'path';
 import WritablePartition from '../Partition/WritablePartition.js';
 import WritableIndex, { Entry as WritableIndexEntry } from '../Index/WritableIndex.js';
 import ReadableStorage from './ReadableStorage.js';
-import { assert, ensureDirectory } from '../util.js';
-import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../metadataUtil.js';
+import { assert } from '../utils/util.js';
+import { ensureDirectory } from '../utils/fsUtil.js';
+import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../utils/metadataUtil.js';
 
 const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024;
 
@@ -61,24 +62,31 @@ class WritableStorage extends ReadableStorage {
         super(storageName, config);
 
         this.lockFile = path.resolve(this.dataDirectory, this.storageFile + '.lock');
-        if (config.lock === LOCK_RECLAIM) {
-            this.unlock();
-        }
+        this._lockMode = config.lock;
         this.partitioner = config.partitioner;
+        this.partitionIds = {};
     }
 
     /**
      * @inheritDoc
+     * Acquires the write lock synchronously.
+     * For LOCK_RECLAIM, removes any orphaned lock before trying to acquire our own; torn-write
+     * repair runs after the primary index is open, before `'opened'` is emitted.
+     *
      * @returns {boolean}
      * @throws {StorageLockedError} If this storage is locked by another process.
      */
-    open() {
+    open(callback) {
+        const needsRepair = this._lockMode === LOCK_RECLAIM && this.unlock();
+
         if (!this.lock()) {
             return true;
         }
-        const result = super.open();
-        this.emit('ready');
-        return result;
+
+        const onOpen = needsRepair
+            ? () => { this.checkTornWrites(); callback?.(); }
+            : callback;
+        return super.open(onOpen);
     }
 
     /**
@@ -170,6 +178,8 @@ class WritableStorage extends ReadableStorage {
         }
 
         this.forEachPartition(partition => partition.close());
+        // Partitions were closed directly (bypassing the pool), so reset the open-handle tracking.
+        this.partitions.clearOpenHandles();
     }
 
     /**
@@ -195,8 +205,8 @@ class WritableStorage extends ReadableStorage {
 
         // Scan partitions in sequence-number order and rebuild index entries.
         // iterateDocumentsNoIndex opens any closed partitions automatically.
-        for (const { document, partition, position, size } of this.iterateDocumentsNoIndex(fromSequenceNumber, Number.MAX_SAFE_INTEGER)) {
-            const newEntry = new WritableIndexEntry(this.index.length + 1, position, size, partition);
+        for (const { document, entry } of this.iterateDocumentsNoIndex(fromSequenceNumber)) {
+            const newEntry = new WritableIndexEntry(this.index.length + 1, entry.position, entry.size, entry.partition);
             this.index.add(newEntry);
 
             this.forEachWritableSecondaryIndex((secIndex) => {
@@ -222,9 +232,8 @@ class WritableStorage extends ReadableStorage {
             this.locked = true;
         } catch (e) {
             /* istanbul ignore if */
-            if (e.code !== 'EEXIST') {
-                throw new Error(`Error creating lock for storage ${this.storageFile}: ` + e.message);
-            }
+            assert(e.code === 'EEXIST', `Error creating lock for storage ${this.storageFile}: ` + e.message)
+
             throw new StorageLockedError(`Storage ${this.storageFile} is locked by another process`);
         }
         return true;
@@ -234,19 +243,21 @@ class WritableStorage extends ReadableStorage {
      * Unlock this storage, no matter if it was previously locked by this writer.
      * Only use this if you are sure there is no other process still having a writer open.
      * Current implementation just deletes a lock file that is named like the storage.
+     * @returns {boolean} True if an orphaned lock from another process was removed.
      */
     unlock() {
-        if (fs.existsSync(this.lockFile)) {
-            if (!this.locked) {
-                this.checkTornWrites();
-            }
+        const lockExists = fs.existsSync(this.lockFile);
+        const orphaned = lockExists && !this.locked;
+        if (lockExists) {
             fs.rmdirSync(this.lockFile);
         }
         this.locked = false;
+        return orphaned;
     }
 
     /**
      * @inheritDoc
+     * Unlocks the storage, then delegates to the parent close().
      */
     close() {
         if (this.locked) {
@@ -300,11 +311,43 @@ class WritableStorage extends ReadableStorage {
         this.on('preCommit', hook);
     }
 
+    getPartitionIdForName(partitionShortName, partitionName) {
+        const partitionId = this.partitionIds[partitionShortName] ?? WritablePartition.idFor(partitionName);
+        this.partitionIds[partitionShortName] = partitionId;
+        return partitionId;
+    }
+
+    buildPartitionConfig(partitionShortName) {
+        if (typeof this.partitionConfig.metadata !== 'function') {
+            return this.partitionConfig;
+        }
+        return { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) };
+    }
+
+    ensurePartitionDirectory(partitionName) {
+        if (!partitionName.includes('/')) {
+            return;
+        }
+        ensureDirectory(path.join(this.dataDirectory, path.dirname(partitionName)));
+    }
+
+    createNamedPartition(partitionId, partitionName, partitionShortName) {
+        if (this.partitions.has(partitionId)) {
+            return;
+        }
+        const partitionConfig = this.buildPartitionConfig(partitionShortName);
+        this.ensurePartitionDirectory(partitionName);
+        this.partitions.add(partitionId, this.createPartition(partitionName, partitionConfig));
+        this.emit('partition-created', partitionId);
+    }
+
     /**
      * Get a partition either by name or by id.
      * If a partition with the given name does not exist, a new one will be created.
      * If a partition with the given id does not exist, an error is thrown.
      *
+     * Partition opening and LRU tracking are delegated to `super.getPartition()`.
+     *
      * @protected
      * @param {string|number} partitionIdentifier The partition name or the partition Id
      * @returns {ReadablePartition}
@@ -314,16 +357,8 @@ class WritableStorage extends ReadableStorage {
         if (typeof partitionIdentifier === 'string') {
             const partitionShortName = partitionIdentifier;
             const partitionName = this.storageFile + (partitionIdentifier.length ? '.' + partitionIdentifier : '');
-            partitionIdentifier = WritablePartition.idFor(partitionName);
-            if (!this.partitions[partitionIdentifier]) {
-                const partitionConfig = typeof this.partitionConfig.metadata === 'function'
-                    ? { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) }
-                    : this.partitionConfig;
-                this.partitions[partitionIdentifier] = this.createPartition(partitionName, partitionConfig);
-                this.emit('partition-created', partitionIdentifier);
-            }
-            this.partitions[partitionIdentifier].open();
-            return this.partitions[partitionIdentifier];
+            partitionIdentifier = this.getPartitionIdForName(partitionShortName, partitionName);
+            this.createNamedPartition(partitionIdentifier, partitionName, partitionShortName);
         }
         return super.getPartition(partitionIdentifier);
     }
@@ -340,18 +375,15 @@ class WritableStorage extends ReadableStorage {
 
         const partitionName = this.partitioner(document, this.index.length + 1);
         const partition = this.getPartition(partitionName);
-        if (this.listenerCount('preCommit') > 0) {
-            this.emit('preCommit', document, partition.metadata);
-        }
+        this.emit('preCommit', document, partition.metadata);
+
         const position = partition.write(data, this.length, callback);
 
         assert(position !== false, 'Error writing document.');
 
         const indexEntry = this.addIndex(partition.id, position, dataSize, document);
         this.forEachSecondaryIndex((index, name) => {
-            if (!index.isOpen()) {
-                index.open();
-            }
+            index.open();
             index.add(indexEntry);
             this.emit('index-add', name, index.length, document);
         }, document);
@@ -366,10 +398,11 @@ class WritableStorage extends ReadableStorage {
      * @api
      * @param {string} name The index name.
      * @param {Matcher} [matcher] An object that describes the document properties that need to match to add it this index or a function that receives a document and returns true if the document should be indexed.
+     * @param {boolean} [reindex=true] Whether to scan existing documents and populate the new index. Set to false when it is known that no existing documents can match the matcher.
      * @returns {ReadableIndex} The index containing all documents that match the query.
      * @throws {Error} if the index doesn't exist yet and no matcher was specified.
      */
-    ensureIndex(name, matcher) {
+    ensureIndex(name, matcher, reindex = true) {
         if (name === '_all') {
             return this.index;
         }
@@ -386,18 +419,21 @@ class WritableStorage extends ReadableStorage {
 
         const metadata = buildMetadataForMatcher(matcher, this.hmac);
         const { index } = this.createIndex(indexName, Object.assign({}, this.indexOptions, { metadata }));
-        try {
-            this.forEachDocument((document, indexEntry) => {
-                if (matches(document, matcher)) {
-                    index.add(indexEntry);
-                }
-            });
-        } catch (e) {
-            index.destroy();
-            throw e;
+        if (reindex) {
+            try {
+                this.forEachDocument((document, indexEntry) => {
+                    if (matches(document, matcher)) {
+                        index.add(indexEntry);
+                    }
+                });
+            } catch (e) {
+                index.destroy();
+                throw e;
+            }
         }
 
         this.secondaryIndexes[name] = { index, matcher };
+        this.indexMatcher.add(name, matcher);
         this.emit('index-created', name);
         return index;
     }
@@ -423,7 +459,7 @@ class WritableStorage extends ReadableStorage {
      */
     forEachDistinctPartitionOf(entries, iterationHandler) {
         const partitions = [];
-        const numPartitions = Object.keys(this.partitions).length;
+        const numPartitions = this.partitions.count;
         for (let entry of entries) {
             if (partitions.indexOf(entry.partition) >= 0) {
                 continue;

package/src/Watcher.js

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import path from 'path';
 import events from 'events';
-import { assert } from './util.js';
+import { assert } from './utils/util.js';
 
 /** @type {Map<string, DirectoryWatcher>} */
 const directoryWatchers = new Map();

package/src/utils/fsUtil.js

@@ -0,0 +1,123 @@
+import fs from 'fs';
+import path from 'path';
+import { mkdirpSync } from 'mkdirp';
+
+/**
+ * Ensure that the given directory exists.
+ * @param {string} dirName
+ * @return {boolean} true if the directory existed already
+ */
+function ensureDirectory(dirName) {
+    if (!fs.existsSync(dirName)) {
+        try {
+            mkdirpSync(dirName);
+        } catch (e) {
+        }
+        return false;
+    }
+    return true;
+}
+
+/**
+ * Invoke `onEach` if `relativePath` matches `regexPattern`, passing the first capture group or the full match.
+ *
+ * @param {string} relativePath
+ * @param {RegExp} regexPattern
+ * @param {function(string)} onEach
+ */
+function visitMatchingPath(relativePath, regexPattern, onEach) {
+    const match = relativePath.match(regexPattern);
+    if (match !== null) {
+        onEach(match[1] !== undefined ? match[1] : match[0]);
+    }
+}
+
+/**
+ * Classify `entries` into matching files (visited via `onEach`) and subdirectory names (returned).
+ *
+ * @param {fs.Dirent[]} entries
+ * @param {string} relativePrefix
+ * @param {RegExp} regexPattern
+ * @param {function(string)} onEach
+ * @returns {string[]} names of subdirectory entries
+ */
+function classifyEntries(entries, relativePrefix, regexPattern, onEach) {
+    const subdirs = [];
+    for (let entry of entries) {
+        if (entry.isDirectory()) {
+            subdirs.push(entry.name);
+        } else {
+            visitMatchingPath(relativePrefix + entry.name, regexPattern, onEach);
+        }
+    }
+    return subdirs;
+}
+
+/**
+ * Sequentially scan each name in `subdirs`, calling `done` when all are complete or on first error.
+ *
+ * @param {string[]} subdirs
+ * @param {string} dir
+ * @param {string} relativePrefix
+ * @param {RegExp} regexPattern
+ * @param {function(string)} onEach
+ * @param {function(Error?)} done
+ */
+function scanSubdirs(subdirs, dir, relativePrefix, regexPattern, onEach, done) {
+    let i = 0;
+    function next() {
+        if (i >= subdirs.length) return done(null);
+        const name = subdirs[i++];
+        scanDir(path.join(dir, name), relativePrefix + name + '/', false, regexPattern, onEach, (err) => {
+            if (err) return done(err);
+            next();
+        });
+    }
+    next();
+}
+
+/**
+ * Asynchronously scan one directory level, then recurse into subdirectories sequentially.
+ *
+ * @param {string} dir
+ * @param {string} relativePrefix
+ * @param {boolean} isRoot
+ * @param {RegExp} regexPattern
+ * @param {function(string)} onEach
+ * @param {function(Error?)} done
+ */
+function scanDir(dir, relativePrefix, isRoot, regexPattern, onEach, done) {
+    fs.readdir(dir, { withFileTypes: true }, (err, entries) => {
+        if (err) {
+            /* istanbul ignore next */
+            if (!isRoot && err.code === 'ENOENT') return done(null);
+            return done(err);
+        }
+        const subdirs = classifyEntries(entries, relativePrefix, regexPattern, onEach);
+        scanSubdirs(subdirs, dir, relativePrefix, regexPattern, onEach, done);
+    });
+}
+
+/**
+ * Scan a directory (and its subdirectories) for files whose relative paths match a regex pattern,
+ * calling a callback for each match.
+ *
+ * The regex is matched against the **relative path from `directory`** (e.g. `eventstore.stream-x/foo.index`),
+ * so patterns that capture a path prefix work transparently for both flat and nested layouts.
+ *
+ * The `onEach` callback receives the first capturing group of the match (`match[1]`), or the full
+ * match (`match[0]`) when no capturing group is defined in the pattern.
+ *
+ * @param {string} directory The root directory to scan.
+ * @param {RegExp} regexPattern The pattern to match relative file paths against.
+ * @param {function(string)} onEach Called with the first capturing group (or full match) for each matching path.
+ * @param {function(Error?)} onDone Called when the scan is complete, or with an error if one occurred.
+ */
+function scanForFiles(directory, regexPattern, onEach, onDone) {
+    scanDir(directory, '', true, regexPattern, onEach, onDone);
+}
+
+export {
+    ensureDirectory,
+    scanForFiles,
+};

package/src/utils/jsonUtil.js

@@ -0,0 +1,87 @@
+const BYTE_QUOTE = 0x22;
+const BYTE_ESCAPE = 0x5c;
+const BYTE_OPEN_OBJECT = 0x7b;
+const BYTE_CLOSE_OBJECT = 0x7d;
+const BYTE_OPEN_ARRAY = 0x5b;
+const BYTE_CLOSE_ARRAY = 0x5d;
+const BYTE_COMMA = 0x2c;
+
+/**
+ * Find the position of `pattern` within `buffer` at depth 0 (the top-level object), starting
+ * from `startOffset`.  It scans character-by-character tracking JSON nesting depth and string
+ * quoting.  If `matchPosition` arrives at depth > 0 it means the pattern is inside a nested
+ * object/array, so the scan continues searching for the next candidate at depth 0.  Returns -1
+ * when no such position exists before the end of the buffer or when a closing brace reduces depth
+ * below zero (the top-level object has ended).
+ */
+function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition) {
+    /* c8 ignore start */
+    // Defensive fallback: public call path precomputes an initial candidate in preCheck.
+    if (matchPosition === undefined) {
+        matchPosition = buffer.indexOf(pattern, startOffset);
+    }
+    if (matchPosition === -1) {
+        return -1;
+    }
+    /* c8 ignore stop */
+
+    let depth = 0;
+    let inString = false;
+    let i = startOffset;
+
+    while (i < buffer.length) {
+        if (inString) {
+            if (buffer[i] === BYTE_ESCAPE) { // '\\'
+                i += 2;
+                continue;
+            }
+            if (buffer[i] === BYTE_QUOTE) { // '"'
+                inString = false;
+            }
+            i++;
+            continue;
+        }
+
+        const ch = buffer[i];
+        if (ch === BYTE_OPEN_OBJECT || ch === BYTE_OPEN_ARRAY) { // '{' or '['
+            depth++;
+            i++;
+            continue;
+        } else if (ch === BYTE_CLOSE_OBJECT || ch === BYTE_CLOSE_ARRAY) { // '}' or ']'
+            depth--;
+
+            if (depth < 0) {
+                return -1;
+            }
+
+            i++;
+            continue;
+        } else if (ch === BYTE_QUOTE) { // '"'
+            inString = true;
+        }
+
+        if (i >= matchPosition) {
+            if (i === matchPosition && ch === BYTE_QUOTE && depth === 0) { // '"'
+                const end = i + pattern.length;
+                if (pattern[pattern.length - 1] === BYTE_OPEN_OBJECT) { // '{'
+                    return i;
+                }
+                if (buffer[end] === BYTE_COMMA || buffer[end] === BYTE_CLOSE_OBJECT || buffer[end] === BYTE_CLOSE_ARRAY) { // ',' or '}' or ']'
+                    return i;
+                }
+            }
+
+            matchPosition = buffer.indexOf(pattern, matchPosition + 1);
+            if (matchPosition < 0) {
+                return -1;
+            }
+        }
+
+        i++;
+    }
+
+    /* c8 ignore next */
+    return -1;
+}
+
+export { BYTE_OPEN_OBJECT, indexOfSameLevel };