event-storage 0.9.1 → 1.1.0

package/src/Storage/WritableStorage.js

@@ -1,10 +1,11 @@
-const fs = require('fs');
-const path = require('path');
-const WritablePartition = require('../Partition/WritablePartition');
-const WritableIndex = require('../Index/WritableIndex');
-const ReadableStorage = require('./ReadableStorage');
-const { assert, ensureDirectory } = require('../util');
-const { matches, buildMetadataForMatcher, buildMatcherFromMetadata } = require('../metadataUtil');
+import fs from 'fs';
+import path from 'path';
+import WritablePartition from '../Partition/WritablePartition.js';
+import WritableIndex, { Entry as WritableIndexEntry } from '../Index/WritableIndex.js';
+import ReadableStorage from './ReadableStorage.js';
+import { assert } from '../util.js';
+import { ensureDirectory } from '../fsUtil.js';
+import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../metadataUtil.js';
 
 const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024;
 
@@ -61,24 +62,30 @@ class WritableStorage extends ReadableStorage {
         super(storageName, config);
 
         this.lockFile = path.resolve(this.dataDirectory, this.storageFile + '.lock');
-        if (config.lock === LOCK_RECLAIM) {
-            this.unlock();
-        }
+        this._lockMode = config.lock;
         this.partitioner = config.partitioner;
     }
 
     /**
      * @inheritDoc
+     * Acquires the write lock synchronously.
+     * For LOCK_RECLAIM, removes any orphaned lock before trying to acquire our own; torn-write
+     * repair runs after the primary index is open, before `'opened'` is emitted.
+     *
      * @returns {boolean}
      * @throws {StorageLockedError} If this storage is locked by another process.
      */
-    open() {
+    open(callback) {
+        const needsRepair = this._lockMode === LOCK_RECLAIM && this.unlock();
+
         if (!this.lock()) {
             return true;
         }
-        const result = super.open();
-        this.emit('ready');
-        return result;
+
+        const onOpen = needsRepair
+            ? () => { this.checkTornWrites(); callback?.(); }
+            : callback;
+        return super.open(onOpen);
     }
 
     /**
@@ -170,6 +177,8 @@ class WritableStorage extends ReadableStorage {
         }
 
         this.forEachPartition(partition => partition.close());
+        // Partitions were closed directly (bypassing the pool), so reset the open-handle tracking.
+        this.partitions.clearOpenHandles();
     }
 
     /**
@@ -196,7 +205,7 @@ class WritableStorage extends ReadableStorage {
         // Scan partitions in sequence-number order and rebuild index entries.
         // iterateDocumentsNoIndex opens any closed partitions automatically.
         for (const { document, partition, position, size } of this.iterateDocumentsNoIndex(fromSequenceNumber, Number.MAX_SAFE_INTEGER)) {
-            const newEntry = new WritableIndex.Entry(this.index.length + 1, position, size, partition);
+            const newEntry = new WritableIndexEntry(this.index.length + 1, position, size, partition);
             this.index.add(newEntry);
 
             this.forEachWritableSecondaryIndex((secIndex) => {
@@ -234,19 +243,21 @@ class WritableStorage extends ReadableStorage {
      * Unlock this storage, no matter if it was previously locked by this writer.
      * Only use this if you are sure there is no other process still having a writer open.
      * Current implementation just deletes a lock file that is named like the storage.
+     * @returns {boolean} True if an orphaned lock from another process was removed.
      */
     unlock() {
-        if (fs.existsSync(this.lockFile)) {
-            if (!this.locked) {
-                this.checkTornWrites();
-            }
+        const lockExists = fs.existsSync(this.lockFile);
+        const orphaned = lockExists && !this.locked;
+        if (lockExists) {
             fs.rmdirSync(this.lockFile);
         }
         this.locked = false;
+        return orphaned;
     }
 
     /**
      * @inheritDoc
+     * Unlocks the storage, then delegates to the parent close().
      */
     close() {
         if (this.locked) {
@@ -276,7 +287,7 @@ class WritableStorage extends ReadableStorage {
          throw new Error('Corrupted index, needs to be rebuilt!');
          }*/
 
-        const entry = new WritableIndex.Entry(this.index.length + 1, position, size, partitionId);
+        const entry = new WritableIndexEntry(this.index.length + 1, position, size, partitionId);
         this.index.add(entry, (indexPosition) => {
             this.emit('wrote', document, entry, indexPosition);
             /* istanbul ignore if  */
@@ -305,6 +316,8 @@ class WritableStorage extends ReadableStorage {
      * If a partition with the given name does not exist, a new one will be created.
      * If a partition with the given id does not exist, an error is thrown.
      *
+     * Partition opening and LRU tracking are delegated to `super.getPartition()`.
+     *
      * @protected
      * @param {string|number} partitionIdentifier The partition name or the partition Id
      * @returns {ReadablePartition}
@@ -315,15 +328,16 @@ class WritableStorage extends ReadableStorage {
             const partitionShortName = partitionIdentifier;
             const partitionName = this.storageFile + (partitionIdentifier.length ? '.' + partitionIdentifier : '');
             partitionIdentifier = WritablePartition.idFor(partitionName);
-            if (!this.partitions[partitionIdentifier]) {
+            if (!this.partitions.has(partitionIdentifier)) {
                 const partitionConfig = typeof this.partitionConfig.metadata === 'function'
                     ? { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) }
                     : this.partitionConfig;
-                this.partitions[partitionIdentifier] = this.createPartition(partitionName, partitionConfig);
+                if (partitionName.includes('/')) {
+                    ensureDirectory(path.join(this.dataDirectory, path.dirname(partitionName)));
+                }
+                this.partitions.add(partitionIdentifier, this.createPartition(partitionName, partitionConfig));
                 this.emit('partition-created', partitionIdentifier);
             }
-            this.partitions[partitionIdentifier].open();
-            return this.partitions[partitionIdentifier];
         }
         return super.getPartition(partitionIdentifier);
     }
@@ -366,10 +380,11 @@ class WritableStorage extends ReadableStorage {
      * @api
      * @param {string} name The index name.
      * @param {Matcher} [matcher] An object that describes the document properties that need to match to add it this index or a function that receives a document and returns true if the document should be indexed.
+     * @param {boolean} [reindex=true] Whether to scan existing documents and populate the new index. Set to false when it is known that no existing documents can match the matcher.
      * @returns {ReadableIndex} The index containing all documents that match the query.
      * @throws {Error} if the index doesn't exist yet and no matcher was specified.
      */
-    ensureIndex(name, matcher) {
+    ensureIndex(name, matcher, reindex = true) {
         if (name === '_all') {
             return this.index;
         }
@@ -386,18 +401,21 @@ class WritableStorage extends ReadableStorage {
 
         const metadata = buildMetadataForMatcher(matcher, this.hmac);
         const { index } = this.createIndex(indexName, Object.assign({}, this.indexOptions, { metadata }));
-        try {
-            this.forEachDocument((document, indexEntry) => {
-                if (matches(document, matcher)) {
-                    index.add(indexEntry);
-                }
-            });
-        } catch (e) {
-            index.destroy();
-            throw e;
+        if (reindex) {
+            try {
+                this.forEachDocument((document, indexEntry) => {
+                    if (matches(document, matcher)) {
+                        index.add(indexEntry);
+                    }
+                });
+            } catch (e) {
+                index.destroy();
+                throw e;
+            }
         }
 
         this.secondaryIndexes[name] = { index, matcher };
+        this.indexMatcher.add(name, matcher);
         this.emit('index-created', name);
         return index;
     }
@@ -423,7 +441,7 @@ class WritableStorage extends ReadableStorage {
      */
     forEachDistinctPartitionOf(entries, iterationHandler) {
         const partitions = [];
-        const numPartitions = Object.keys(this.partitions).length;
+        const numPartitions = this.partitions.count;
         for (let entry of entries) {
             if (partitions.indexOf(entry.partition) >= 0) {
                 continue;
@@ -540,8 +558,5 @@ class WritableStorage extends ReadableStorage {
 
 }
 
-module.exports = WritableStorage;
-module.exports.StorageLockedError = StorageLockedError;
-module.exports.CorruptFileError = ReadableStorage.CorruptFileError;
-module.exports.LOCK_THROW = LOCK_THROW;
-module.exports.LOCK_RECLAIM = LOCK_RECLAIM;
+export default WritableStorage;
+export { StorageLockedError, LOCK_THROW, LOCK_RECLAIM };

package/src/Storage.js

@@ -1,5 +1,10 @@
-const WritableStorage = require('./Storage/WritableStorage');
-const ReadOnlyStorage = require('./Storage/ReadOnlyStorage');
+import WritableStorage, { StorageLockedError, LOCK_THROW, LOCK_RECLAIM } from './Storage/WritableStorage.js';
+import ReadOnlyStorage from './Storage/ReadOnlyStorage.js';
 
-module.exports = WritableStorage;
-module.exports.ReadOnly = ReadOnlyStorage;
+WritableStorage.ReadOnly = ReadOnlyStorage;
+WritableStorage.StorageLockedError = StorageLockedError;
+WritableStorage.LOCK_THROW = LOCK_THROW;
+WritableStorage.LOCK_RECLAIM = LOCK_RECLAIM;
+
+export default WritableStorage;
+export { ReadOnlyStorage as ReadOnly, StorageLockedError, LOCK_THROW, LOCK_RECLAIM };

package/src/Watcher.js

@@ -1,7 +1,7 @@
-const fs = require('fs');
-const path = require('path');
-const events = require('events');
-const { assert } = require('./util');
+import fs from 'fs';
+import path from 'path';
+import events from 'events';
+import { assert } from './util.js';
 
 /** @type {Map<string, DirectoryWatcher>} */
 const directoryWatchers = new Map();
@@ -147,4 +147,4 @@ class Watcher {
 
 }
 
-module.exports = Watcher;
+export default Watcher;

package/src/WatchesFile.js

@@ -1,4 +1,4 @@
-const Watcher = require('./Watcher');
+import Watcher from './Watcher.js';
 
 /**
  * A mixin that provides a file watcher for this.fileName which triggers a method `onChange` on the class, that needs to be implemented.
@@ -53,4 +53,4 @@ const WatchesFile = Base => class extends Base {
 
 };
 
-module.exports = WatchesFile;
+export default WatchesFile;

package/src/fsUtil.js

@@ -0,0 +1,123 @@
+import fs from 'fs';
+import path from 'path';
+import { mkdirpSync } from 'mkdirp';
+
+/**
+ * Ensure that the given directory exists.
+ * @param {string} dirName
+ * @return {boolean} true if the directory existed already
+ */
+function ensureDirectory(dirName) {
+    if (!fs.existsSync(dirName)) {
+        try {
+            mkdirpSync(dirName);
+        } catch (e) {
+        }
+        return false;
+    }
+    return true;
+}
+
+/**
+ * Invoke `onEach` if `relativePath` matches `regexPattern`, passing the first capture group or the full match.
+ *
+ * @param {string} relativePath
+ * @param {RegExp} regexPattern
+ * @param {function(string)} onEach
+ */
+function visitMatchingPath(relativePath, regexPattern, onEach) {
+    const match = relativePath.match(regexPattern);
+    if (match !== null) {
+        onEach(match[1] !== undefined ? match[1] : match[0]);
+    }
+}
+
+/**
+ * Classify `entries` into matching files (visited via `onEach`) and subdirectory names (returned).
+ *
+ * @param {fs.Dirent[]} entries
+ * @param {string} relativePrefix
+ * @param {RegExp} regexPattern
+ * @param {function(string)} onEach
+ * @returns {string[]} names of subdirectory entries
+ */
+function classifyEntries(entries, relativePrefix, regexPattern, onEach) {
+    const subdirs = [];
+    for (let entry of entries) {
+        if (entry.isDirectory()) {
+            subdirs.push(entry.name);
+        } else {
+            visitMatchingPath(relativePrefix + entry.name, regexPattern, onEach);
+        }
+    }
+    return subdirs;
+}
+
+/**
+ * Sequentially scan each name in `subdirs`, calling `done` when all are complete or on first error.
+ *
+ * @param {string[]} subdirs
+ * @param {string} dir
+ * @param {string} relativePrefix
+ * @param {RegExp} regexPattern
+ * @param {function(string)} onEach
+ * @param {function(Error?)} done
+ */
+function scanSubdirs(subdirs, dir, relativePrefix, regexPattern, onEach, done) {
+    let i = 0;
+    function next() {
+        if (i >= subdirs.length) return done(null);
+        const name = subdirs[i++];
+        scanDir(path.join(dir, name), relativePrefix + name + '/', false, regexPattern, onEach, (err) => {
+            if (err) return done(err);
+            next();
+        });
+    }
+    next();
+}
+
+/**
+ * Asynchronously scan one directory level, then recurse into subdirectories sequentially.
+ *
+ * @param {string} dir
+ * @param {string} relativePrefix
+ * @param {boolean} isRoot
+ * @param {RegExp} regexPattern
+ * @param {function(string)} onEach
+ * @param {function(Error?)} done
+ */
+function scanDir(dir, relativePrefix, isRoot, regexPattern, onEach, done) {
+    fs.readdir(dir, { withFileTypes: true }, (err, entries) => {
+        if (err) {
+            /* istanbul ignore next */
+            if (!isRoot && err.code === 'ENOENT') return done(null);
+            return done(err);
+        }
+        const subdirs = classifyEntries(entries, relativePrefix, regexPattern, onEach);
+        scanSubdirs(subdirs, dir, relativePrefix, regexPattern, onEach, done);
+    });
+}
+
+/**
+ * Scan a directory (and its subdirectories) for files whose relative paths match a regex pattern,
+ * calling a callback for each match.
+ *
+ * The regex is matched against the **relative path from `directory`** (e.g. `eventstore.stream-x/foo.index`),
+ * so patterns that capture a path prefix work transparently for both flat and nested layouts.
+ *
+ * The `onEach` callback receives the first capturing group of the match (`match[1]`), or the full
+ * match (`match[0]`) when no capturing group is defined in the pattern.
+ *
+ * @param {string} directory The root directory to scan.
+ * @param {RegExp} regexPattern The pattern to match relative file paths against.
+ * @param {function(string)} onEach Called with the first capturing group (or full match) for each matching path.
+ * @param {function(Error?)} onDone Called when the scan is complete, or with an error if one occurred.
+ */
+function scanForFiles(directory, regexPattern, onEach, onDone) {
+    scanDir(directory, '', true, regexPattern, onEach, onDone);
+}
+
+export {
+    ensureDirectory,
+    scanForFiles,
+};

package/src/metadataUtil.js

@@ -1,4 +1,27 @@
-const crypto = require('crypto');
+import crypto from 'crypto';
+import { assertEqual } from './util.js';
+
+/**
+ * Build a buffer containing the file magic header and a JSON stringified metadata block, padded to be a multiple of 16 bytes long.
+ *
+ * @param {string} magic
+ * @param {object} metadata
+ * @returns {Buffer} A buffer containing the header data
+ */
+function buildMetadataHeader(magic, metadata) {
+    assertEqual(magic.length, 8, 'The header magic bytes length is wrong.');
+    let metadataString = JSON.stringify(metadata);
+    let metadataSize = Buffer.byteLength(metadataString, 'utf8');
+    // 8 byte MAGIC, 4 byte metadata size, 1 byte line break
+    const pad = (16 - ((8 + 4 + metadataSize + 1) % 16)) % 16;
+    metadataString += ' '.repeat(pad) + "\n";
+    metadataSize += pad + 1;
+    const metadataBuffer = Buffer.allocUnsafe(8 + 4 + metadataSize);
+    metadataBuffer.write(magic, 0, 8, 'utf8');
+    metadataBuffer.writeUInt32BE(metadataSize, 8);
+    metadataBuffer.write(metadataString, 8 + 4, metadataSize, 'utf8');
+    return metadataBuffer;
+}
 
 /**
  * @param {string} secret The secret to use for calculating further HMACs
@@ -26,7 +49,11 @@ function matches(document, matcher) {
     if (typeof matcher === 'function') return matcher(document);
 
     for (let prop of Object.getOwnPropertyNames(matcher)) {
-        if (typeof matcher[prop] === 'object') {
+        if (Array.isArray(matcher[prop])) {
+            if (!matcher[prop].includes(document[prop])) {
+                return false;
+            }
+        } else if (typeof matcher[prop] === 'object') {
             if (!matches(document[prop], matcher[prop])) {
                 return false;
             }
@@ -71,9 +98,29 @@ function buildMatcherFromMetadata(matcherMetadata, hmac) {
     return matcher;
 }
 
-module.exports = {
+/**
+ * Builds a factory function that, given a type string, returns an object matcher for
+ * documents whose payload contains that type at the given dot-notation path.
+ *
+ * @param {string} payloadPath Dot-notation path relative to the event payload (e.g. `'type'`, `'meta.kind'`).
+ * @returns {function(string): object} A function `(typeValue) => objectMatcher`.
+ */
+function buildTypeMatcherFn(payloadPath) {
+    const parts = payloadPath.split('.');
+    return function(typeValue) {
+        let obj = typeValue;
+        for (let i = parts.length - 1; i >= 0; i--) {
+            obj = { [parts[i]]: obj };
+        }
+        return { payload: obj };
+    };
+}
+
+export {
     createHmac,
     matches,
+    buildMetadataHeader,
     buildMetadataForMatcher,
-    buildMatcherFromMetadata
+    buildMatcherFromMetadata,
+    buildTypeMatcherFn
 };

package/src/util.js

@@ -1,6 +1,3 @@
-const fs = require('fs');
-const mkdirpSync = require('mkdirp').sync;
-
 /**
  * Assert that actual and expected match or throw an Error with the given message appended by information about expected and actual value.
  *
@@ -62,28 +59,6 @@ function hash(str) {
     return hash >>> 0; // jshint ignore:line
 }
 
-/**
- * Build a buffer containing the file magic header and a JSON stringified metadata block, padded to be a multiple of 16 bytes long.
- *
- * @param {string} magic
- * @param {object} metadata
- * @returns {Buffer} A buffer containing the header data
- */
-function buildMetadataHeader(magic, metadata) {
-    assertEqual(magic.length, 8, 'The header magic bytes length is wrong.');
-    let metadataString = JSON.stringify(metadata);
-    let metadataSize = Buffer.byteLength(metadataString, 'utf8');
-    // 8 byte MAGIC, 4 byte metadata size, 1 byte line break
-    const pad = (16 - ((8 + 4 + metadataSize + 1) % 16)) % 16;
-    metadataString += ' '.repeat(pad) + "\n";
-    metadataSize += pad + 1;
-    const metadataBuffer = Buffer.allocUnsafe(8 + 4 + metadataSize);
-    metadataBuffer.write(magic, 0, 8, 'utf8');
-    metadataBuffer.writeUInt32BE(metadataSize, 8);
-    metadataBuffer.write(metadataString, 8 + 4, metadataSize, 'utf8');
-    return metadataBuffer;
-}
-
 /**
  * Do a binary search for number in the range 1-length with values retrieved via a provided getter.
  *
@@ -137,22 +112,6 @@ function wrapAndCheck(index, length) {
     return index;
 }
 
-/**
- * Ensure that the given directory exists.
- * @param {string} dirName
- * @return {boolean} true if the directory existed already
- */
-function ensureDirectory(dirName) {
-    if (!fs.existsSync(dirName)) {
-        try {
-            mkdirpSync(dirName);
-        } catch (e) {
-        }
-        return false;
-    }
-    return true;
-}
-
 /**
  * Perform a k-way merge over multiple streams, invoking a callback for each item in ascending key order.
  * Each stream object is mutated in place by the `advance` function.
@@ -179,40 +138,30 @@ function kWayMerge(streams, getKey, advance, visit) {
 }
 
 /**
- * Scan a directory for files whose names match a regex pattern, calling a callback for each match.
- * The `onEach` callback receives the first capturing group of the match (`match[1]`), or the full
- * match (`match[0]`) when no capturing group is defined in the pattern.
+ * Read a scalar value at a dot-notation path from an object.
+ * Returns `undefined` if any path segment is absent or an intermediate value is not an object.
  *
- * @param {string} directory The directory to scan.
- * @param {RegExp} regexPattern The pattern to match file names against.
- * @param {function(string)} onEach Called with the first capturing group (or full match) for each matching file name.
- * @param {function(Error?)} onDone Called when the scan is complete, or with an error if one occurred.
+ * @param {object} obj
+ * @param {string} dotPath Dot-separated property path, e.g. `'payload.type'`.
+ * @returns {*}
  */
-function scanForFiles(directory, regexPattern, onEach, onDone) {
-    fs.readdir(directory, (err, files) => {
-        if (err) {
-            return onDone(err);
-        }
-        let match;
-        for (let file of files) {
-            if ((match = file.match(regexPattern)) !== null) {
-                onEach(match[1] !== undefined ? match[1] : match[0]);
-            }
-        }
-        onDone(null);
-    });
+function getPropertyAtPath(obj, dotPath) {
+    let current = obj;
+    const parts = dotPath.split('.');
+    for (const part of parts) {
+        if (current == null || typeof current !== 'object') return undefined;
+        current = current[part];
+    }
+    return current;
 }
 
-
-module.exports = {
+export {
     assert,
     assertEqual,
     hash,
     wrapAndCheck,
     binarySearch,
-    buildMetadataHeader,
     alignTo,
-    ensureDirectory,
-    scanForFiles,
-    kWayMerge
+    kWayMerge,
+    getPropertyAtPath
 };