event-storage 1.1.0 → 1.2.0

package/src/Storage/ReadableStorage.js

@@ -3,13 +3,14 @@ import path from 'path';
 import events from 'events';
 import Partition, { ReadOnly as ReadOnlyPartition } from '../Partition.js';
 import Index, { ReadOnly as ReadOnlyIndex } from '../Index.js';
-import { assert, wrapAndCheck, kWayMerge } from '../util.js';
-import { scanForFiles } from '../fsUtil.js';
-import { createHmac, matches, buildMetadataForMatcher } from '../metadataUtil.js';
+import { assert, wrapAndCheck, iterate, kWayMerge } from '../utils/util.js';
+import { scanForFiles } from '../utils/fsUtil.js';
+import { createHmac, matches, buildMetadataForMatcher } from '../utils/metadataUtil.js';
 import IndexMatcher from '../IndexMatcher.js';
 import PartitionPool from '../PartitionPool.js';
 
 const DEFAULT_READ_BUFFER_SIZE = 4 * 1024;
+const NDJSON_NEWLINE = Buffer.from('\n');
 
 /**
  * Default ordered list of document property paths used as discriminant keys when
@@ -26,18 +27,6 @@ const DEFAULT_MATCHER_PROPERTIES = ['stream', 'payload.type'];
  */
 const DEFAULT_MAX_OPEN_PARTITIONS = 1024;
 
-/**
- * Reverses the items of an iterable
- * @param {Generator|Iterable} iterator
- * @returns {Generator<*>}
- */
-function *reverse(iterator) {
-    const items = Array.from(iterator);
-    for (let i = items.length - 1; i >= 0; i--) {
-        yield items[i];
-    }
-}
-
 /**
  * @typedef {object|function(object):boolean} Matcher
  */
@@ -284,16 +273,19 @@ class ReadableStorage extends events.EventEmitter {
      * @param {number} partitionId The partition to read from.
      * @param {number} position The file position to read from.
      * @param {number} [size] The expected byte size of the document at the given position.
-     * @returns {object} The document stored at the given position.
+     * @param {boolean} [raw] Whether to return raw buffers instead of deserialized objects. Default false.
+     * @param {boolean} [backwardsHint] If set to true, will optimize buffering for backwards reading.
+     * @returns {object|{ buffer: Buffer, time64: number, sequenceNumber: number }} The document stored at the given position.
      * @throws {Error} if the document at the given position can not be deserialized.
      */
-    readFrom(partitionId, position, size) {
+    readFrom(partitionId, position, size, raw = false, backwardsHint = false) {
         const partition = this.getPartition(partitionId);
         if (this.listenerCount('preRead') > 0) {
             this.emit('preRead', position, partition.metadata);
         }
-        const data = partition.readFrom(position, size);
-        return this.serializer.deserialize(data);
+        const headerOut = {};
+        const buffer = partition.readFrom(position, size, headerOut, backwardsHint);
+        return raw ? { buffer, time64: headerOut.time64, sequenceNumber: headerOut.sequenceNumber } : this.serializer.deserialize(buffer.toString('utf8'));
     }
 
     /**
@@ -306,10 +298,7 @@ class ReadableStorage extends events.EventEmitter {
      */
     read(number, index) {
         index = index || this.index;
-
-        if (!index.isOpen()) {
-            index.open();
-        }
+        index.open();
 
         const entry = index.get(number);
         if (entry === false) {
@@ -329,30 +318,22 @@ class ReadableStorage extends events.EventEmitter {
      * @param {ReadableIndex|false} [index] The index to use for finding the documents in the range.
      *   Pass `false` to skip the global index and iterate all partitions directly in sequenceNumber order
      *   (useful when the global index is unavailable or corrupted).
+     * @param {boolean} [raw] Whether to return raw buffers instead of deserialized objects. Default false.
      * @returns {Generator<object>} A generator that will read each document in the range one by one.
      */
-    *readRange(from, until = -1, index = null) {
-        const lengthSource = index || this.index;
-        if (!lengthSource.isOpen()) {
-            lengthSource.open();
+    *readRange(from, until = -1, index = null, raw = false) {
+        let length = Number.MAX_SAFE_INTEGER;
+        if (index !== false) {
+            index = index || this.index;
+            index.open();
+            length = index.length;
         }
 
-        const readFrom = wrapAndCheck(from, lengthSource.length);
-        const readUntil = wrapAndCheck(until, lengthSource.length);
+        const readFrom = wrapAndCheck(from, length);
+        const readUntil = wrapAndCheck(until, length);
         assert(readFrom > 0 && readUntil > 0, `Range scan error for range ${from} - ${until}.`);
 
-        if (readFrom > readUntil) {
-            const batchSize = 10;
-            let batchUntil = readFrom;
-            while (batchUntil >= readUntil) {
-                const batchFrom = Math.max(readUntil, batchUntil - batchSize);
-                yield* reverse(this.iterateRange(batchFrom, batchUntil, index));
-                batchUntil = batchFrom - 1;
-            }
-            return undefined;
-        }
-
-        yield* this.iterateRange(readFrom, readUntil, index);
+        yield* this.iterateRange(readFrom, readUntil, index, raw);
     }
 
     /**
@@ -362,23 +343,25 @@ class ReadableStorage extends events.EventEmitter {
      * @param {number} from
      * @param {number} until
      * @param {ReadableIndex|false|null} index
+     * @param {boolean} [raw] Whether to return raw buffers instead of deserialized objects. Default false.
      * @returns {Generator<object>}
      */
-    *iterateRange(from, until, index) {
+    *iterateRange(from, until, index, raw = false) {
         if (index === false) {
-            // Explicitly disabled index: iterate all partitions and merge by sequenceNumber.
-            // Document header sequenceNumber is 0-based; from/until are 1-based index positions.
-            for (const entry of this.iterateDocumentsNoIndex(from - 1, until - 1)) {
-                yield entry.document;
+            for (const { document } of this.iterateDocumentsNoIndex(from - 1, until - 1)) {
+                yield document;
             }
             return;
         }
 
         const idx = index || this.index;
-        const entries = idx.range(from, until);
-        for (let entry of entries) {
-            const document = this.readFrom(entry.partition, entry.position, entry.size);
-            yield document;
+        const forwards = from <= until;
+        const lo = Math.min(from, until);
+        const hi = Math.max(from, until);
+        const entries = idx.range(lo, hi);
+        if (!entries) return;
+        for (const entry of iterate(entries, forwards)) {
+            yield this.readFrom(entry.partition, entry.position, entry.size, raw, !forwards);
         }
     }
 
@@ -448,62 +431,50 @@ class ReadableStorage extends events.EventEmitter {
         }
     }
 
-    /**
-     * Iterate documents across all partitions in sequenceNumber order using a k-way merge.
-     * Opens any closed partition automatically.
-     *
-     * @protected
-     * @param {number} [from=0] The 0-based sequenceNumber to start from (inclusive).
-     * @param {number} [until=Number.MAX_SAFE_INTEGER] The 0-based sequenceNumber to read until (inclusive).
-     * @returns {Generator<{document: object, sequenceNumber: number, partitionName: string, position: number, size: number, partition: number}>}
-     */
-    *iterateDocumentsNoIndex(from = 0, until = Number.MAX_SAFE_INTEGER) {
-        const streams = [];
-
-        this.forEachPartition(partition => {
-            if (!partition.isOpen()) {
-                partition.open();
-            }
-
-            const found = partition.findDocument(from);
-            if (found && found.headerOut.sequenceNumber <= until) {
-                const nextPosition = found.headerOut.position + partition.documentWriteSize(found.headerOut.dataSize);
-                const reader = partition.readAll(nextPosition, found.headerOut);
-                streams.push({ ...found, reader, partition: partition.id, partitionName: partition.name });
-            }
-        });
-
-        const items = [];
-        kWayMerge(
-            streams,
-            stream => stream.headerOut.sequenceNumber,
-            stream => {
-                const next = stream.reader.next();
-                if (!next.done && stream.headerOut.sequenceNumber <= until) {
-                    stream.data = next.value;
-                    return true;
-                }
-                return false;
-            },
-            stream => items.push({
-                document: this.serializer.deserialize(stream.data),
-                sequenceNumber: stream.headerOut.sequenceNumber,
-                partitionName: stream.partitionName,
-                position: stream.headerOut.position,
-                size: stream.headerOut.dataSize,
-                partition: stream.partition,
-            })
-        );
-
-        yield* items;
-    }
+     /**
+      * Build the standard document result entry from a readRange yield.
+      * @private
+      * @param {{ data: Buffer, entry: { number: number, position: number, size: number, partition: number } }} [readItem]
+      */
+     buildDocumentEntry(readItem) {
+         return {
+             document: this.serializer.deserialize(readItem.data.toString('utf8')),
+             // Replicate the index entry structure here, so iteration can be used easily to reindex
+             entry: readItem.entry
+         };
+     }
+
+     /**
+      * Iterate documents across all partitions in sequenceNumber order using a k-way merge.
+      * Opens any closed partition automatically.
+      *
+      * @protected
+      * @param {number} [from=0] The 0-based sequenceNumber to start from (inclusive).
+      * @param {number} [until=Number.MAX_SAFE_INTEGER] The 0-based sequenceNumber to read until (inclusive).
+      * @returns {Generator<{document: object, entry: { sequenceNumber: number, position: number, size: number, partition: number }}>}
+      */
+     *iterateDocumentsNoIndex(from = 0, until = Number.MAX_SAFE_INTEGER) {
+         const forwards = from <= until;
+         const partitions = [];
+         this.forEachPartition(partition => {
+             partition.open();
+             partitions.push(partition.readRange(from, until));
+         });
+
+         yield* kWayMerge(
+             partitions,
+             item => item.entry.number,
+             forwards,
+             item => this.buildDocumentEntry(item)
+         );
+     }
 
     /**
      * Helper method to iterate over all documents, invoking a callback for each one.
      * Pass `noIndex = true` to iterate all partitions directly in sequenceNumber order
      * (useful when the global index is unavailable or corrupted).
      * When `noIndex` is false the second callback argument is the raw index `EntryInterface`.
-     * When `noIndex` is true the second callback argument has `{ partition, position, size, sequenceNumber, partitionName }`.
+     * When `noIndex` is true the second callback argument has `{ partition, position, size, sequenceNumber }`.
      *
      * @protected
      * @param {function(object, object): void} iterationHandler
@@ -516,8 +487,8 @@ class ReadableStorage extends events.EventEmitter {
         }
 
         if (noIndex) {
-            for (const { document, ...entryInfo } of this.iterateDocumentsNoIndex()) {
-                iterationHandler(document, entryInfo);
+            for (const { document, entry } of this.iterateDocumentsNoIndex()) {
+                iterationHandler(document, entry);
             }
             return;
         }

package/src/Storage/WritableStorage.js

@@ -3,9 +3,9 @@ import path from 'path';
 import WritablePartition from '../Partition/WritablePartition.js';
 import WritableIndex, { Entry as WritableIndexEntry } from '../Index/WritableIndex.js';
 import ReadableStorage from './ReadableStorage.js';
-import { assert } from '../util.js';
-import { ensureDirectory } from '../fsUtil.js';
-import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../metadataUtil.js';
+import { assert } from '../utils/util.js';
+import { ensureDirectory } from '../utils/fsUtil.js';
+import { matches, buildMetadataForMatcher, buildMatcherFromMetadata } from '../utils/metadataUtil.js';
 
 const DEFAULT_WRITE_BUFFER_SIZE = 16 * 1024;
 
@@ -64,6 +64,7 @@ class WritableStorage extends ReadableStorage {
         this.lockFile = path.resolve(this.dataDirectory, this.storageFile + '.lock');
         this._lockMode = config.lock;
         this.partitioner = config.partitioner;
+        this.partitionIds = {};
     }
 
     /**
@@ -204,8 +205,8 @@ class WritableStorage extends ReadableStorage {
 
         // Scan partitions in sequence-number order and rebuild index entries.
         // iterateDocumentsNoIndex opens any closed partitions automatically.
-        for (const { document, partition, position, size } of this.iterateDocumentsNoIndex(fromSequenceNumber, Number.MAX_SAFE_INTEGER)) {
-            const newEntry = new WritableIndexEntry(this.index.length + 1, position, size, partition);
+        for (const { document, entry } of this.iterateDocumentsNoIndex(fromSequenceNumber)) {
+            const newEntry = new WritableIndexEntry(this.index.length + 1, entry.position, entry.size, entry.partition);
             this.index.add(newEntry);
 
             this.forEachWritableSecondaryIndex((secIndex) => {
@@ -231,9 +232,8 @@ class WritableStorage extends ReadableStorage {
             this.locked = true;
         } catch (e) {
             /* istanbul ignore if */
-            if (e.code !== 'EEXIST') {
-                throw new Error(`Error creating lock for storage ${this.storageFile}: ` + e.message);
-            }
+            assert(e.code === 'EEXIST', `Error creating lock for storage ${this.storageFile}: ` + e.message)
+
             throw new StorageLockedError(`Storage ${this.storageFile} is locked by another process`);
         }
         return true;
@@ -311,6 +311,36 @@ class WritableStorage extends ReadableStorage {
         this.on('preCommit', hook);
     }
 
+    getPartitionIdForName(partitionShortName, partitionName) {
+        const partitionId = this.partitionIds[partitionShortName] ?? WritablePartition.idFor(partitionName);
+        this.partitionIds[partitionShortName] = partitionId;
+        return partitionId;
+    }
+
+    buildPartitionConfig(partitionShortName) {
+        if (typeof this.partitionConfig.metadata !== 'function') {
+            return this.partitionConfig;
+        }
+        return { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) };
+    }
+
+    ensurePartitionDirectory(partitionName) {
+        if (!partitionName.includes('/')) {
+            return;
+        }
+        ensureDirectory(path.join(this.dataDirectory, path.dirname(partitionName)));
+    }
+
+    createNamedPartition(partitionId, partitionName, partitionShortName) {
+        if (this.partitions.has(partitionId)) {
+            return;
+        }
+        const partitionConfig = this.buildPartitionConfig(partitionShortName);
+        this.ensurePartitionDirectory(partitionName);
+        this.partitions.add(partitionId, this.createPartition(partitionName, partitionConfig));
+        this.emit('partition-created', partitionId);
+    }
+
     /**
      * Get a partition either by name or by id.
      * If a partition with the given name does not exist, a new one will be created.
@@ -327,17 +357,8 @@ class WritableStorage extends ReadableStorage {
         if (typeof partitionIdentifier === 'string') {
             const partitionShortName = partitionIdentifier;
             const partitionName = this.storageFile + (partitionIdentifier.length ? '.' + partitionIdentifier : '');
-            partitionIdentifier = WritablePartition.idFor(partitionName);
-            if (!this.partitions.has(partitionIdentifier)) {
-                const partitionConfig = typeof this.partitionConfig.metadata === 'function'
-                    ? { ...this.partitionConfig, metadata: this.partitionConfig.metadata(partitionShortName) }
-                    : this.partitionConfig;
-                if (partitionName.includes('/')) {
-                    ensureDirectory(path.join(this.dataDirectory, path.dirname(partitionName)));
-                }
-                this.partitions.add(partitionIdentifier, this.createPartition(partitionName, partitionConfig));
-                this.emit('partition-created', partitionIdentifier);
-            }
+            partitionIdentifier = this.getPartitionIdForName(partitionShortName, partitionName);
+            this.createNamedPartition(partitionIdentifier, partitionName, partitionShortName);
         }
         return super.getPartition(partitionIdentifier);
     }
@@ -354,18 +375,15 @@ class WritableStorage extends ReadableStorage {
 
         const partitionName = this.partitioner(document, this.index.length + 1);
         const partition = this.getPartition(partitionName);
-        if (this.listenerCount('preCommit') > 0) {
-            this.emit('preCommit', document, partition.metadata);
-        }
+        this.emit('preCommit', document, partition.metadata);
+
         const position = partition.write(data, this.length, callback);
 
         assert(position !== false, 'Error writing document.');
 
         const indexEntry = this.addIndex(partition.id, position, dataSize, document);
         this.forEachSecondaryIndex((index, name) => {
-            if (!index.isOpen()) {
-                index.open();
-            }
+            index.open();
             index.add(indexEntry);
             this.emit('index-add', name, index.length, document);
         }, document);

package/src/Watcher.js

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import path from 'path';
 import events from 'events';
-import { assert } from './util.js';
+import { assert } from './utils/util.js';
 
 /** @type {Map<string, DirectoryWatcher>} */
 const directoryWatchers = new Map();

package/src/utils/jsonUtil.js

@@ -0,0 +1,87 @@
+const BYTE_QUOTE = 0x22;
+const BYTE_ESCAPE = 0x5c;
+const BYTE_OPEN_OBJECT = 0x7b;
+const BYTE_CLOSE_OBJECT = 0x7d;
+const BYTE_OPEN_ARRAY = 0x5b;
+const BYTE_CLOSE_ARRAY = 0x5d;
+const BYTE_COMMA = 0x2c;
+
+/**
+ * Find the position of `pattern` within `buffer` at depth 0 (the top-level object), starting
+ * from `startOffset`.  It scans character-by-character tracking JSON nesting depth and string
+ * quoting.  If `matchPosition` arrives at depth > 0 it means the pattern is inside a nested
+ * object/array, so the scan continues searching for the next candidate at depth 0.  Returns -1
+ * when no such position exists before the end of the buffer or when a closing brace reduces depth
+ * below zero (the top-level object has ended).
+ */
+function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition) {
+    /* c8 ignore start */
+    // Defensive fallback: public call path precomputes an initial candidate in preCheck.
+    if (matchPosition === undefined) {
+        matchPosition = buffer.indexOf(pattern, startOffset);
+    }
+    if (matchPosition === -1) {
+        return -1;
+    }
+    /* c8 ignore stop */
+
+    let depth = 0;
+    let inString = false;
+    let i = startOffset;
+
+    while (i < buffer.length) {
+        if (inString) {
+            if (buffer[i] === BYTE_ESCAPE) { // '\\'
+                i += 2;
+                continue;
+            }
+            if (buffer[i] === BYTE_QUOTE) { // '"'
+                inString = false;
+            }
+            i++;
+            continue;
+        }
+
+        const ch = buffer[i];
+        if (ch === BYTE_OPEN_OBJECT || ch === BYTE_OPEN_ARRAY) { // '{' or '['
+            depth++;
+            i++;
+            continue;
+        } else if (ch === BYTE_CLOSE_OBJECT || ch === BYTE_CLOSE_ARRAY) { // '}' or ']'
+            depth--;
+
+            if (depth < 0) {
+                return -1;
+            }
+
+            i++;
+            continue;
+        } else if (ch === BYTE_QUOTE) { // '"'
+            inString = true;
+        }
+
+        if (i >= matchPosition) {
+            if (i === matchPosition && ch === BYTE_QUOTE && depth === 0) { // '"'
+                const end = i + pattern.length;
+                if (pattern[pattern.length - 1] === BYTE_OPEN_OBJECT) { // '{'
+                    return i;
+                }
+                if (buffer[end] === BYTE_COMMA || buffer[end] === BYTE_CLOSE_OBJECT || buffer[end] === BYTE_CLOSE_ARRAY) { // ',' or '}' or ']'
+                    return i;
+                }
+            }
+
+            matchPosition = buffer.indexOf(pattern, matchPosition + 1);
+            if (matchPosition < 0) {
+                return -1;
+            }
+        }
+
+        i++;
+    }
+
+    /* c8 ignore next */
+    return -1;
+}
+
+export { BYTE_OPEN_OBJECT, indexOfSameLevel };