event-storage 0.8.0 → 0.9.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "event-storage",
-  "version": "0.8.0",
+  "version": "0.9.1",
   "description": "An optimized embedded event store for node.js",
   "keywords": [
     "event-storage",
@@ -15,7 +15,7 @@
   "homepage": "https://github.com/albe/node-event-storage",
   "repository": {
     "type": "git",
-    "url": "https://github.com/albe/node-event-storage"
+    "url": "git+https://github.com/albe/node-event-storage.git"
   },
   "bugs": {
     "url": "https://github.com/albe/node-event-storage/issues"
@@ -25,21 +25,22 @@
     "coverage": "nyc report --reporter=text-lcov | coveralls"
   },
   "files": [
-    "*/Consumer*.js",
-    "*/EventStore*.js",
-    "*/EventStream*.js",
-    "*/Index*.js",
-    "*/IndexEntry*.js",
-    "*/JoinEventStream*.js",
-    "*/Partition*.js",
-    "*/Storage*.js",
-    "*/Watcher*.js",
-    "*/Clock*.js",
-    "*/Index/*.js",
-    "*/Partition/*.js",
-    "*/Storage/*.js",
+    "src/Consumer*.js",
+    "src/EventStore*.js",
+    "src/EventStream*.js",
+    "src/Index*.js",
+    "src/IndexEntry*.js",
+    "src/JoinEventStream*.js",
+    "src/Partition*.js",
+    "src/Storage*.js",
+    "src/Watcher*.js",
+    "src/Clock*.js",
+    "src/Index/*.js",
+    "src/Partition/*.js",
+    "src/Storage/*.js",
     "src/WatchesFile.js",
     "src/util.js",
+    "src/metadataUtil.js",
     "index.js"
   ],
   "license": "MIT",
@@ -50,10 +51,10 @@
     }
   ],
   "engines": {
-    "node": ">=10.0"
+    "node": ">=18.0"
   },
   "dependencies": {
-    "mkdirp": "^1.0.3"
+    "mkdirp": "^3.0.1"
   },
   "nyc": {
     "include": [
@@ -64,10 +65,10 @@
     ]
   },
   "devDependencies": {
-    "coveralls": "^3.0.2",
+    "coveralls-next": "^6.0.1",
     "expect.js": "^0.3.1",
-    "fs-extra": "^9.0.1",
-    "mocha": "^8.0.1",
-    "nyc": "^15.0.0"
+    "fs-extra": "^11.3.4",
+    "mocha": "^11.7.5",
+    "nyc": "^18.0.0"
   }
 }

package/src/Clock.js

@@ -1,6 +1,6 @@
-const TIME_BASE = process.hrtime();
-const DATE_BASE_US = Date.now() * 1000.0 + 999.0; // DATE_BASE_US should match the TIME_BASE with lower resolution, but never be less
-const US_PER_SEC = 1.0e6;
+const TIME_BASE = process.hrtime.bigint();
+const DATE_FACTOR = 1000000n;
+const DATE_BASE_NS = BigInt(Date.now() + 1) * DATE_FACTOR - 1n;
 const CLOCK_ACCURACY_US = 1; // two process.hrtime() calls take roughly this long, so this is the accuracy we can measure time
 
 /**
@@ -14,16 +14,28 @@ class Clock {
      * @param {Date|number} epoch The epoch to base this clock on, either as a Date or a number of the amount of milliseconds since the unix epoch
      */
     constructor(epoch) {
-        this.epoch = Math.floor((epoch instanceof Date ? epoch.getTime() : Number(epoch)) * 1000.0);
+        this.epoch = BigInt(epoch instanceof Date ? epoch.getTime() : Number(epoch)) * DATE_FACTOR;
+        this.lastTime = 0;
     }
 
     /**
-     * @returns {number} The number of microseconds since the epoch given in the constructor. The decimal part denotes the accuracy of the clock in milliseconds.
-     * @note Needs to allow at least tens of ms accuracy, better hundreds of ms
+     * @returns {number} The number of microseconds since the epoch given in the constructor.
+     * @note Needs to allow at least tenths of ms accuracy, better hundredths of ms
      */
     time() {
-        const delta = process.hrtime(TIME_BASE);
-        return DATE_BASE_US - this.epoch + (delta[0] * US_PER_SEC + Math.floor(delta[1] / 1000)) + (CLOCK_ACCURACY_US / 1000.0);
+        const delta = process.hrtime.bigint() - TIME_BASE;
+        const timeSinceEpoch = Number((DATE_BASE_NS - this.epoch + delta) / 1000n);
+        return this.lastTime = Math.max(this.lastTime + 1, timeSinceEpoch);
+    }
+
+    /**
+     * Return the clock accuracy of the given timestamp. This is only useful for calculating a consistent ordering
+     * ala TrueTime for multi-writer scenarios.
+     * @param {number} time A timestamp measured by this clock.
+     * @returns {number} The amount of µs accuracy this timestamp has.
+     */
+    accuracy(time) {
+        return CLOCK_ACCURACY_US;
     }
 
 }

package/src/EventStore.js

@@ -5,8 +5,7 @@ const path = require('path');
 const events = require('events');
 const Storage = require('./Storage');
 const Consumer = require('./Consumer');
-const stream = require('stream');
-const { assert } = require('./util');
+const { assert, scanForFiles } = require('./util');
 
 const ExpectedVersion = {
     Any: -1,
@@ -15,24 +14,6 @@ const ExpectedVersion = {
 
 class OptimisticConcurrencyError extends Error {}
 
-class EventUnwrapper extends stream.Transform {
-
-    constructor() {
-        super({ objectMode: true });
-    }
-
-    _transform(data, encoding, callback) {
-        /* istanbul ignore else */
-        if (data.stream && data.payload) {
-            this.push(data.payload);
-        } else {
-            this.push(data);
-        }
-        callback();
-    }
-
-}
-
 /**
  * An event store optimized for working with many streams.
  * An event stream is implemented as an iterator over an index on the storage, therefore indexes need to be lightweight
@@ -47,6 +28,10 @@ class EventStore extends events.EventEmitter {
      * @param {string} [config.streamsDirectory] The directory where the streams should be stored. Default '{storageDirectory}/streams'.
      * @param {object} [config.storageConfig] Additional config options given to the storage backend. See `Storage`.
      * @param {boolean} [config.readOnly] If the storage should be mounted in read-only mode.
+     * @param {object|function(string): object} [config.streamMetadata] A metadata object or a function `(streamName) => object`
+     *   that is called whenever a new stream partition is created. The returned object is stored once in the partition
+     *   file header and surfaced to `preCommit` / `preRead` hooks. Takes precedence only when
+     *   `config.storageConfig.metadata` is not also set.
      */
     constructor(storeName = 'eventstore', config = {}) {
         super();
@@ -63,6 +48,17 @@ class EventStore extends events.EventEmitter {
             readOnly: config.readOnly || false
         };
         const storageConfig = Object.assign(defaults, config.storageConfig);
+
+        // Translate the high-level streamMetadata option into the storage-level metadata function,
+        // but only when the caller has not already provided a lower-level storageConfig.metadata.
+        if (config.streamMetadata !== undefined && storageConfig.metadata === undefined) {
+            if (typeof config.streamMetadata === 'function') {
+                storageConfig.metadata = config.streamMetadata;
+            } else {
+                storageConfig.metadata = (streamName) => config.streamMetadata[streamName] || {};
+            }
+        }
+
         this.initialize(storeName, storageConfig);
     }
 
@@ -87,10 +83,43 @@ class EventStore extends events.EventEmitter {
                 this.storage.close();
                 throw err;
             }
+            this.checkUnfinishedCommits();
             this.emit('ready');
         });
     }
 
+    /**
+     * Check if the last commit in the store was unfinished, which is the case if not all events of the commit have been written.
+     * Torn writes are handled at the storage level, so this method only deals with unfinished commits.
+     * @private
+     */
+    checkUnfinishedCommits() {
+        let position = this.storage.length;
+        let lastEvent;
+        let truncateIndex = false;
+        while (position > 0) {
+            try {
+                lastEvent = this.storage.read(position);
+            } catch (e) {
+                // A preRead hook may throw (e.g. access control). Stop repair check.
+                return;
+            }
+            if (lastEvent !== false) break;
+            truncateIndex = true;
+            position--;
+        }
+
+        if (lastEvent && lastEvent.metadata.commitSize && lastEvent.metadata.commitVersion !== lastEvent.metadata.commitSize - 1) {
+            this.emit('unfinished-commit', lastEvent);
+            // commitId = global sequence number at which the commit started
+            this.storage.truncate(lastEvent.metadata.commitId);
+        } else if (truncateIndex) {
+            // The index contained items that are not in the storage file; truncate everything
+            // after `position`, the last sequence number that was successfully read.
+            this.storage.truncate(position);
+        }
+    }
+
     /**
      * Scan the streams directory for existing streams so they are ready for `getEventStream()`.
      *
@@ -103,38 +132,44 @@ class EventStore extends events.EventEmitter {
             callback = () => {};
         }
         // Find existing streams by scanning dir for filenames starting with 'stream-'
-        fs.readdir(this.streamsDirectory, (err, files) => {
-            if (err) {
-                return callback(err);
-            }
-            let matches;
-            for (let file of files) {
-                if ((matches = file.match(/(stream-.*)\.index$/)) !== null) {
-                    this.registerStream(matches[1]);
-                }
-            }
-            callback();
-        });
+        scanForFiles(this.streamsDirectory, /(stream-.*)\.index$/, this.registerStream.bind(this), callback);
         this.storage.on('index-created', this.registerStream.bind(this));
     }
 
     /**
      * @private
-     * @param {string} name The full stream name, including the `stream-` prefix.
+     * @param {string} name The full stream name, including the `stream-` prefix (and optional `.closed` suffix).
      */
     registerStream(name) {
         /* istanbul ignore if */
         if (!name.startsWith('stream-')) {
             return;
         }
-        const streamName = name.substr(7, name.length - 7);
-        /* istanbul ignore if */
+        let streamName = name.slice(7);
+        // Detect the `.closed` suffix — present both in the initial scan and when the directory
+        // watcher emits 'index-created' after a writer renames the file (e.g. 'stream-foo-bar.closed').
+        let isClosed = false;
+        if (streamName.endsWith('.closed')) {
+            streamName = streamName.slice(0, -7);
+            isClosed = true;
+        }
         if (streamName in this.streams) {
+            if (isClosed && !this.streams[streamName].closed) {
+                // The stream was renamed to .closed while this instance had it open.
+                // The old ReadOnlyIndex was already closed via onRename, so we open the new one.
+                const closedIndexName = 'stream-' + streamName + '.closed';
+                const closedIndex = this.storage.openReadonlyIndex(closedIndexName);
+                // deepcode ignore PrototypePollutionFunctionParams: streams is a Map
+                this.streams[streamName] = { index: closedIndex, closed: true };
+                this.emit('stream-closed', streamName);
+            }
             return;
         }
-        const index = this.storage.openIndex('stream-'+streamName);
+        const index = isClosed
+            ? this.storage.openReadonlyIndex(name)
+            : this.storage.openIndex(name);
         // deepcode ignore PrototypePollutionFunctionParams: streams is a Map
-        this.streams[streamName] = { index };
+        this.streams[streamName] = { index, closed: isClosed };
         this.emit('stream-available', streamName);
     }
 
@@ -147,6 +182,103 @@ class EventStore extends events.EventEmitter {
         this.storage.close();
     }
 
+    /**
+     * Override EventEmitter.on() to delegate 'preCommit' and 'preRead' event registrations
+     * to the underlying storage, so that `eventstore.on('preCommit', handler)` works naturally.
+     * All other events are handled by the default EventEmitter.
+     *
+     * @param {string} event
+     * @param {function} listener
+     * @returns {this}
+     */
+    on(event, listener) {
+        if (event === 'preCommit' || event === 'preRead') {
+            if (event === 'preCommit') {
+                assert(!(this.storage instanceof Storage.ReadOnly), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.');
+            }
+            this.storage.on(event, listener);
+            return this;
+        }
+        return super.on(event, listener);
+    }
+
+    /**
+     * @inheritDoc
+     */
+    addListener(event, listener) {
+        return this.on(event, listener);
+    }
+
+    /**
+     * Override EventEmitter.once() to delegate 'preCommit' and 'preRead' to the underlying storage.
+     *
+     * @param {string} event
+     * @param {function} listener
+     * @returns {this}
+     */
+    once(event, listener) {
+        if (event === 'preCommit' || event === 'preRead') {
+            if (event === 'preCommit') {
+                assert(!(this.storage instanceof Storage.ReadOnly), 'The storage was opened in read-only mode. Can not register a preCommit handler on it.');
+            }
+            this.storage.once(event, listener);
+            return this;
+        }
+        return super.once(event, listener);
+    }
+
+    /**
+     * Override EventEmitter.off() / removeListener() to delegate 'preCommit' and 'preRead'
+     * to the underlying storage.
+     *
+     * @param {string} event
+     * @param {function} listener
+     * @returns {this}
+     */
+    off(event, listener) {
+        if (event === 'preCommit' || event === 'preRead') {
+            this.storage.off(event, listener);
+            return this;
+        }
+        return super.off(event, listener);
+    }
+
+    /**
+     * @inheritDoc
+     */
+    removeListener(event, listener) {
+        return this.off(event, listener);
+    }
+
+    /**
+     * Convenience method to register a handler called before an event is committed to storage.
+     * Equivalent to `eventstore.on('preCommit', hook)`.
+     * The handler receives `(event, partitionMetadata)` and may throw to abort the write.
+     * Multiple handlers can be registered; all run on every write in registration order.
+     * The handler is invoked on every write, so its logic should be cheap, fast, and synchronous.
+     *
+     * @api
+     * @param {function(object, object): void} hook A function receiving (event, partitionMetadata).
+     * @throws {Error} If the storage was opened in read-only mode.
+     */
+    preCommit(hook) {
+        this.on('preCommit', hook);
+    }
+
+    /**
+     * Convenience method to register a handler called before an event is read from storage.
+     * Equivalent to `eventstore.on('preRead', hook)`.
+     * The handler receives `(position, partitionMetadata)` and may throw to abort the read.
+     * Multiple handlers can be registered; all run on every read in registration order.
+     * The handler is invoked on every read, so its logic should be cheap, fast, and synchronous.
+     *
+     * @api
+     * @param {function(number, object): void} hook A function receiving (position, partitionMetadata).
+     */
+    preRead(hook) {
+        this.on('preRead', hook);
+    }
+
     /**
      * Get the number of events stored.
      *
@@ -213,6 +345,7 @@ class EventStore extends events.EventEmitter {
         if (!(streamName in this.streams)) {
             this.createEventStream(streamName, { stream: streamName });
         }
+        assert(!this.streams[streamName].closed, `Stream "${streamName}" is closed and cannot be written to.`);
         let streamVersion = this.streams[streamName].index.length;
         if (expectedVersion !== ExpectedVersion.Any && streamVersion !== expectedVersion) {
             throw new OptimisticConcurrencyError(`Optimistic Concurrency error. Expected stream "${streamName}" at version ${expectedVersion} but is at version ${streamVersion}.`);
@@ -265,11 +398,11 @@ class EventStore extends events.EventEmitter {
      *
      * @api
      * @param {string} streamName The name of the stream to get.
-     * @param {number} [minRevision] The minimum revision to include in the events (inclusive).
-     * @param {number} [maxRevision] The maximum revision to include in the events (inclusive).
+     * @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
+     * @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
      * @returns {EventStream|boolean} The event stream or false if a stream with the name doesn't exist.
      */
-    getEventStream(streamName, minRevision = 0, maxRevision = -1) {
+    getEventStream(streamName, minRevision = 1, maxRevision = -1) {
         if (!(streamName in this.streams)) {
             return false;
         }
@@ -281,11 +414,11 @@ class EventStore extends events.EventEmitter {
      * This is the same as `getEventStream('_all', ...)`.
      *
      * @api
-     * @param {number} [minRevision] The minimum revision to include in the events (inclusive).
-     * @param {number} [maxRevision] The maximum revision to include in the events (inclusive).
+     * @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
+     * @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
      * @returns {EventStream} The event stream.
      */
-    getAllEvents(minRevision = 0, maxRevision = -1) {
+    getAllEvents(minRevision = 1, maxRevision = -1) {
         return this.getEventStream('_all', minRevision, maxRevision);
     }
 
@@ -294,12 +427,12 @@ class EventStore extends events.EventEmitter {
      *
      * @param {string} streamName The (transient) name of the joined stream.
      * @param {Array<string>} streamNames An array of the stream names to join.
-     * @param {number} [minRevision] The minimum revision to include in the events (inclusive).
-     * @param {number} [maxRevision] The maximum revision to include in the events (inclusive).
+     * @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
+     * @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
      * @returns {EventStream} The joined event stream.
      * @throws {Error} if any of the streams doesn't exist.
      */
-    fromStreams(streamName, streamNames, minRevision = 0, maxRevision = -1) {
+    fromStreams(streamName, streamNames, minRevision = 1, maxRevision = -1) {
         assert(streamNames instanceof Array, 'Must specify an array of stream names.');
 
         for (let stream of streamNames) {
@@ -318,12 +451,12 @@ class EventStore extends events.EventEmitter {
      *
      * @api
      * @param {string} categoryName The name of the category to get a stream for. A category is a stream name prefix.
-     * @param {number} [minRevision] The minimum revision to include in the events (inclusive).
-     * @param {number} [maxRevision] The maximum revision to include in the events (inclusive).
+     * @param {number} [minRevision] The 1-based minimum revision to include in the events (inclusive).
+     * @param {number} [maxRevision] The 1-based maximum revision to include in the events (inclusive).
      * @returns {EventStream} The joined event stream for all streams of the given category.
      * @throws {Error} If no stream for this category exists.
      */
-    getEventStreamForCategory(categoryName, minRevision = 0, maxRevision = -1) {
+    getEventStreamForCategory(categoryName, minRevision = 1, maxRevision = -1) {
         if (categoryName in this.streams) {
             return this.getEventStream(categoryName, minRevision, maxRevision);
         }
@@ -380,6 +513,46 @@ class EventStore extends events.EventEmitter {
         this.emit('stream-deleted', streamName);
     }
 
+    /**
+     * Close a stream so that no new events are indexed into it.
+     * The stream will still be readable, but any attempt to write to it will throw an error.
+     * A closed stream is persisted by renaming its index file to include a `.closed` marker
+     * (e.g. `stream-X.closed.index`), so it will be recognized as closed when the store is reopened.
+     *
+     * @api
+     * @param {string} streamName The name of the stream to close.
+     * @returns void
+     * @throws {Error} If the storage is read-only.
+     * @throws {Error} If the stream does not exist.
+     * @throws {Error} If the stream is already closed.
+     */
+    closeEventStream(streamName) {
+        assert(!(this.storage instanceof Storage.ReadOnly), 'The storage was opened in read-only mode. Can not close a stream on it.');
+        assert(streamName in this.streams, `Stream "${streamName}" does not exist.`);
+        assert(!this.streams[streamName].closed, `Stream "${streamName}" is already closed.`);
+
+        const indexName = 'stream-' + streamName;
+        const { index } = this.streams[streamName];
+
+        // Flush and close the index before renaming the file
+        index.close();
+
+        // Rename the index file to mark it as closed (e.g. stream-foo.index -> stream-foo.closed.index)
+        const closedFileName = index.fileName.replace(/\.index$/, '.closed.index');
+        fs.renameSync(index.fileName, closedFileName);
+
+        // Remove from secondary indexes so that new writes are no longer indexed into this stream
+        delete this.storage.secondaryIndexes[indexName];
+
+        // Reopen the renamed index for read access, outside the secondary indexes write path
+        const closedIndexName = indexName + '.closed';
+        const closedIndex = this.storage.openReadonlyIndex(closedIndexName);
+
+        // deepcode ignore PrototypePollutionFunctionParams: streams is a Map
+        this.streams[streamName] = { index: closedIndex, closed: true };
+        this.emit('stream-closed', streamName);
+    }
+
     /**
      * Get a durable consumer for the given stream that will keep receiving events from the last position.
      *
@@ -390,9 +563,29 @@ class EventStore extends events.EventEmitter {
      * @returns {Consumer} A durable consumer for the given stream.
      */
     getConsumer(streamName, identifier, initialState = {}, since = 0) {
-        const consumer = new Consumer(this.storage, 'stream-' + streamName, identifier, initialState, since);
+        const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since);
         consumer.streamName = streamName;
-        return consumer.pipe(new EventUnwrapper());
+        return consumer;
+    }
+
+    /**
+     * Scan the existing consumers on this EventStore and asynchronously return a list of their names.
+     * @param {function(error: Error, consumers: array)} callback A callback that will receive an error as first and the list of consumers as second argument.
+     */
+    scanConsumers(callback) {
+        const consumersPath = path.join(this.storage.indexDirectory, 'consumers');
+        if (!fs.existsSync(consumersPath)) {
+            callback(null, []);
+            return;
+        }
+        const regex = new RegExp(`^${this.storage.storageFile}\\.([^.]*\\..*)$`);
+        const consumers = [];
+        scanForFiles(consumersPath, regex, consumers.push.bind(consumers), /* istanbul ignore next */ (err) => {
+            if (err) {
+                return callback(err, []);
+            }
+            callback(null, consumers);
+        });
     }
 }