@lakutata/orm 3.0.0 → 3.1.1

package/dist/vendor/typeorm/driver/mongo-embedded/EmbeddedCollectionStore.js

@@ -0,0 +1,1017 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EmbeddedCollectionStore = exports.SCAN_TIMEOUT = exports.EmbeddedBulkWriteError = exports.EmbeddedKeyTooLongError = exports.EmbeddedDuplicateKeyError = void 0;
+const tslib_1 = require("tslib");
+const crypto_1 = require("crypto");
+const fs = tslib_1.__importStar(require("fs"));
+const zlib = tslib_1.__importStar(require("zlib"));
+const TypeORMError_1 = require("../../error/TypeORMError");
+const EmbeddedBTree_1 = require("./EmbeddedBTree");
+const EmbeddedCache_1 = require("./EmbeddedCache");
+const EmbeddedKeyEncoding_1 = require("./EmbeddedKeyEncoding");
+const EmbeddedKeydir_1 = require("./EmbeddedKeydir");
+const EmbeddedLog_1 = require("./EmbeddedLog");
+/**
+ * lakutata: the error of a duplicate key, with the code and the message of the one of MongoDB.
+ */
+class EmbeddedDuplicateKeyError extends TypeORMError_1.TypeORMError {
+    constructor(collection, index, keyValue) {
+        super(`E11000 duplicate key error collection: ${collection} index: ${index} dup key: ${JSON.stringify(keyValue)}`);
+        this.collection = collection;
+        this.index = index;
+        this.keyValue = keyValue;
+        this.code = 11000;
+    }
+}
+exports.EmbeddedDuplicateKeyError = EmbeddedDuplicateKeyError;
+/**
+ * lakutata: the error of a key too large for an index (its values encoded, with the _id, about 1000 bytes at most),
+ * with the code of the one of MongoDB before 4.2 (KeyTooLong). The write fails before it is written.
+ */
+class EmbeddedKeyTooLongError extends TypeORMError_1.TypeORMError {
+    constructor(collection, index, size, keyValue) {
+        super(`Key too large to index: collection: ${collection} index: ${index} key size: ${size} bytes`);
+        this.collection = collection;
+        this.index = index;
+        this.size = size;
+        this.keyValue = keyValue;
+        this.code = 17280;
+    }
+}
+exports.EmbeddedKeyTooLongError = EmbeddedKeyTooLongError;
+/**
+ * lakutata: the error of an insertMany of which documents were not inserted, with the fields of the MongoBulkWriteError
+ * of MongoDB: the code and the message of its first error, its errors (writeErrors), and the number and the _id of the
+ * documents inserted (insertedCount, insertedIds by their indexes). Ordered, the documents after the first one failing
+ * are not inserted; unordered, the other ones are.
+ */
+class EmbeddedBulkWriteError extends TypeORMError_1.TypeORMError {
+    constructor(writeErrors, insertedCount, insertedIds) {
+        super(writeErrors[0].errmsg);
+        this.writeErrors = writeErrors;
+        this.insertedCount = insertedCount;
+        this.insertedIds = insertedIds;
+        this.code = writeErrors[0].code;
+    }
+}
+exports.EmbeddedBulkWriteError = EmbeddedBulkWriteError;
+/**
+ * lakutata: the time after which a scan not read any more (a cursor not read nor closed) lets its collection be
+ * compacted, the scan closed (10 minutes, the timeout of the cursors of MongoDB).
+ */
+exports.SCAN_TIMEOUT = 10 * 60 * 1000;
+/**
+ * lakutata: the first bytes of a snapshot of a collection. A snapshot holds: these 8 bytes; the CRC-32 of the bytes
+ * after it (4 bytes, little endian); the length of its header (4 bytes, little endian); its header (BSON, its format
+ * 3, not the one of the files: the version of the keys of the indexes); then the buffers of the keydir and of the trees of the indexes, each one from a
+ * multiple of 8 bytes of the file (zero padding before it).
+ */
+const SNAPSHOT_MAGIC = Buffer.from("LKEMBSN1");
+/**
+ * lakutata: a commit record (all of them are these bytes).
+ */
+const COMMIT_RECORD = (0, EmbeddedLog_1.encodeRecord)(EmbeddedLog_1.RecordType.Commit, Buffer.alloc(0));
+/**
+ * lakutata: the bytes read or written by a call of the system (a call reads or writes up to 2^31 - 1 bytes).
+ */
+const IO_CHUNK = 1024 * 1024 * 1024;
+/**
+ * lakutata: the documents read by the build of an index between two yields of the event loop.
+ */
+const BUILD_BATCH = 1000;
+/**
+ * lakutata: a number rounded up to a multiple of 8 (the alignment of the arrays of 8 bytes).
+ *
+ * @param value
+ */
+function align(value) {
+    return Math.ceil(value / 8) * 8;
+}
+/**
+ * lakutata: the documents of a collection: written to its file (records appended, an operation in one write), read by
+ * the offsets of their records kept in its keydir, outside the heap of V8.
+ */
+class EmbeddedCollectionStore {
+    constructor(name, file, values, cache) {
+        this.name = name;
+        this.file = file;
+        this.values = values;
+        this.cache = cache;
+        this.keydir = new EmbeddedKeydir_1.EmbeddedKeydir();
+        /**
+         * Whether it changed since its snapshot was written or read.
+         */
+        this.changed = false;
+        /**
+         * How it was opened: from its snapshot (and the records written after it), or from all the records of its file.
+         */
+        this.openedFrom = "records";
+        /**
+         * The number of its file in the slots of its keydir (0 or 1): a compaction writes a new file, of the other number.
+         */
+        this.generation = 0;
+        /**
+         * Whether its files are closed.
+         */
+        this.closed = false;
+        /**
+         * The scans of its files running: a compaction does not start while one of them read in the last SCAN_TIMEOUT.
+         */
+        this.scans = new Set();
+        /**
+         * The offset of its file its snapshot is the state of (0 without a snapshot).
+         */
+        this.snapshotOffset = 0;
+        /**
+         * Whether a snapshot of it is to be written (at the next turn of the event loop).
+         */
+        this.snapshotScheduled = false;
+        this.indexes = [];
+        /**
+         * The bytes of the records of the live documents (the rest of the file is the records replaced, deleted, and the
+         * records of the operations).
+         */
+        this.liveBytes = 0;
+        /**
+         * The entries of the indexes (the keys of the values of the documents, then the keys of their _id), by their names.
+         */
+        this.trees = new Map();
+        /**
+         * The indexes a document of which has several keys (the items of an array): their ranges of two bounds are not the
+         * ones of the queries, and their order is not the one of the sorts.
+         */
+        this.multikey = new Set();
+        this.encoding = new EmbeddedKeyEncoding_1.EmbeddedKeyEncoding(values);
+    }
+    /**
+     * Creates the file of a collection: its header record.
+     *
+     * @param name
+     * @param file
+     * @param values
+     * @param cache
+     */
+    static create(name, file, values, cache) {
+        const id = (0, crypto_1.randomBytes)(8).toString("hex");
+        file.append(Buffer.concat([
+            (0, EmbeddedLog_1.encodeRecord)(EmbeddedLog_1.RecordType.Header, values.bson.BSON.serialize({
+                name: name,
+                format: 1,
+                id: id,
+            })),
+            COMMIT_RECORD,
+        ]));
+        const store = new EmbeddedCollectionStore(name, file, values, cache);
+        store.logId = id;
+        store.changed = true;
+        return store;
+    }
+    /**
+     * Opens the file of a collection: its snapshot read when it is the one of the file, then the records written after
+     * it applied; else all its records (their CRC-32 checked) applied an operation at a time, its indexes built from
+     * its documents. The records after its last operation (cut by a crash) are cut off. A record whose CRC-32 does not
+     * match, followed by a whole operation, is the corruption of the file: it fails. Undefined for a file without its
+     * header record (cut when it was created).
+     *
+     * @param file
+     * @param values
+     * @param cache
+     * @param path the path of the file, for the errors
+     * @param snapshot the path of its snapshot
+     */
+    static open(file, values, cache, path, snapshot) {
+        const first = new EmbeddedLog_1.EmbeddedRecordReader(file).next(true);
+        if (typeof first === "string")
+            return undefined;
+        if (first.type !== EmbeddedLog_1.RecordType.Header)
+            throw new TypeORMError_1.TypeORMError(`The file ${path} is not a file of mongo-embedded`);
+        const header = values.deserialize(first.payload);
+        const store = new EmbeddedCollectionStore(header.name, file, values, cache);
+        store.logId = header.id;
+        const start = snapshot ? store.readSnapshot(snapshot) : undefined;
+        const replaying = start !== undefined;
+        const reader = new EmbeddedLog_1.EmbeddedRecordReader(file, start ?? 0);
+        const created = [];
+        let pending = [];
+        let committed = start ?? 0;
+        for (;;) {
+            const record = reader.next(true);
+            if (record === "end" || record === "torn")
+                break;
+            if (record === "checksum") {
+                if (EmbeddedCollectionStore.isFollowedByCommit(file, reader))
+                    throw new TypeORMError_1.TypeORMError(`The file ${path} of mongo-embedded is corrupted at its offset ${reader.position}`);
+                break;
+            }
+            if (record.type === EmbeddedLog_1.RecordType.Commit) {
+                for (const done of pending)
+                    if (replaying)
+                        store.replay(done, created);
+                    else
+                        store.recover(done);
+                pending = [];
+                committed = record.offset + record.length;
+                continue;
+            }
+            const keyed = record.type === EmbeddedLog_1.RecordType.Put ||
+                record.type === EmbeddedLog_1.RecordType.Delete;
+            pending.push({
+                type: record.type,
+                offset: record.offset,
+                length: record.length,
+                key: keyed ? values.idKeyOf(record.payload) : undefined,
+                // The documents replayed after a snapshot update the indexes; read from the start, the indexes are
+                // built once the records are read (from the documents read again)
+                payload: (replaying && record.type === EmbeddedLog_1.RecordType.Put) || !keyed
+                    ? values.deserialize(record.payload)
+                    : undefined,
+            });
+        }
+        if (committed < file.size) {
+            file.truncate(committed);
+            file.sync();
+        }
+        if (replaying) {
+            store.openedFrom = "snapshot";
+            store.snapshotOffset = start;
+            store.buildIndexes(created.filter((index) => store.indexes.includes(index)));
+            store.changed = committed > start;
+        }
+        else {
+            store.buildIndexes(store.indexes);
+            store.changed = true;
+        }
+        return store;
+    }
+    /**
+     * Writes its snapshot: its keydir and the trees of its indexes as they are in memory, and the offset of its file
+     * they are the state of (beside the file, then renamed over it). It is not written to the disk at once (synced): a
+     * snapshot cut by a crash is refused when it is read (its CRC-32 and its size), as the one of another file (its
+     * id, the commit record at its offset), the file then read from its start.
+     *
+     * @param path
+     */
+    writeSnapshot(path) {
+        const keydir = this.keydir.export();
+        const trees = this.indexes
+            .filter((index) => this.trees.has(index.name))
+            .map((index) => ({
+            name: index.name,
+            ...this.trees.get(index.name).export(),
+        }));
+        const header = this.values.bson.BSON.serialize({
+            format: 3,
+            generation: this.generation,
+            logId: this.logId,
+            logOffset: this.file.size,
+            liveBytes: this.liveBytes,
+            indexes: this.indexes,
+            multikey: [...this.multikey],
+            keydir: {
+                state: keydir.state,
+                sizes: keydir.buffers.map((buffer) => buffer.byteLength),
+            },
+            trees: trees.map((tree) => ({
+                name: tree.name,
+                state: tree.state,
+                sizes: tree.buffers.map((buffer) => buffer.byteLength),
+            })),
+        });
+        const length = Buffer.allocUnsafe(4);
+        length.writeUInt32LE(header.length);
+        const parts = [length, header];
+        // The magic, the CRC-32 and the length before the header
+        let size = 16 + header.length;
+        const pad = () => {
+            if (size % 8)
+                parts.push(Buffer.alloc(align(size) - size));
+            size = align(size);
+        };
+        pad();
+        for (const buffer of [
+            ...keydir.buffers,
+            ...trees.flatMap((tree) => tree.buffers),
+        ]) {
+            parts.push(buffer);
+            size += buffer.byteLength;
+            pad();
+        }
+        let checksum = 0;
+        for (const part of parts)
+            checksum = zlib.crc32(part, checksum);
+        const crc = Buffer.allocUnsafe(4);
+        crc.writeUInt32LE(checksum);
+        const temporary = `${path}.tmp`;
+        const descriptor = fs.openSync(temporary, "w");
+        try {
+            for (const part of [SNAPSHOT_MAGIC, crc, ...parts]) {
+                let written = 0;
+                while (written < part.byteLength)
+                    written += fs.writeSync(descriptor, part, written, Math.min(part.byteLength - written, IO_CHUNK));
+            }
+        }
+        finally {
+            fs.closeSync(descriptor);
+        }
+        fs.renameSync(temporary, path);
+        this.changed = false;
+        this.snapshotOffset = this.file.size;
+    }
+    /**
+     * Reads its snapshot, when it is whole and the one of its file (its id, an offset of it ending an operation): the
+     * offset its records are applied from; undefined otherwise.
+     *
+     * @param path
+     */
+    readSnapshot(path) {
+        let descriptor;
+        try {
+            descriptor = fs.openSync(path, "r");
+        }
+        catch {
+            return undefined;
+        }
+        try {
+            // Read a part at a time, each one its own buffer (the buffers of Bun hold up to 4 GiB)
+            let position = 0;
+            let checksum = 0;
+            const read = (size, checked = true) => {
+                const bytes = Buffer.allocUnsafe(size);
+                let done = 0;
+                while (done < size) {
+                    const count = fs.readSync(descriptor, bytes, done, Math.min(size - done, IO_CHUNK), position + done);
+                    if (count === 0)
+                        throw new Error("The snapshot is cut");
+                    done += count;
+                }
+                position += size;
+                if (checked)
+                    checksum = zlib.crc32(bytes, checksum);
+                return bytes;
+            };
+            const skipPadding = () => {
+                if (position % 8)
+                    read(align(position) - position);
+            };
+            if (!read(8, false).equals(SNAPSHOT_MAGIC))
+                return undefined;
+            const expected = read(4, false).readUInt32LE(0);
+            const length = read(4).readUInt32LE(0);
+            const header = this.values.deserialize(read(length));
+            if (header.format !== 3 ||
+                header.logId !== this.logId ||
+                header.logOffset > this.file.size)
+                return undefined;
+            const end = Buffer.allocUnsafe(EmbeddedLog_1.RECORD_HEADER);
+            this.file.readInto(end, 0, EmbeddedLog_1.RECORD_HEADER, header.logOffset - EmbeddedLog_1.RECORD_HEADER);
+            if (!end.equals(COMMIT_RECORD))
+                return undefined;
+            const take = (size) => {
+                skipPadding();
+                return read(size);
+            };
+            const keydir = header.keydir.sizes.map(take);
+            const trees = header.trees.map((tree) => tree.sizes.map(take));
+            skipPadding();
+            if (checksum !== expected ||
+                position !== fs.fstatSync(descriptor).size)
+                return undefined;
+            this.keydir = EmbeddedKeydir_1.EmbeddedKeydir.import(header.keydir.state, keydir);
+            header.trees.forEach((tree, index) => this.trees.set(tree.name, EmbeddedBTree_1.EmbeddedBTree.import(tree.state, trees[index])));
+            this.indexes.push(...header.indexes);
+            for (const name of header.multikey)
+                this.multikey.add(name);
+            this.liveBytes = header.liveBytes;
+            this.generation = header.generation;
+            return header.logOffset;
+        }
+        catch {
+            this.keydir = new EmbeddedKeydir_1.EmbeddedKeydir();
+            this.trees.clear();
+            this.indexes.length = 0;
+            this.multikey.clear();
+            this.liveBytes = 0;
+            return undefined;
+        }
+        finally {
+            fs.closeSync(descriptor);
+        }
+    }
+    /**
+     * Whether whole operations follow a record whose CRC-32 does not match (its length read as it is).
+     *
+     * @param file
+     * @param reader the reader stopped at the record
+     */
+    static isFollowedByCommit(file, reader) {
+        const header = Buffer.allocUnsafe(EmbeddedLog_1.RECORD_HEADER);
+        if (file.readInto(header, 0, EmbeddedLog_1.RECORD_HEADER, reader.position) <
+            EmbeddedLog_1.RECORD_HEADER)
+            return false;
+        const following = new EmbeddedLog_1.EmbeddedRecordReader(file, reader.position + EmbeddedLog_1.RECORD_HEADER + header.readUInt32LE(0));
+        for (;;) {
+            const record = following.next(true);
+            if (typeof record === "string")
+                return false;
+            if (record.type === EmbeddedLog_1.RecordType.Commit)
+                return true;
+        }
+    }
+    /**
+     * Whether a document of a key is held.
+     *
+     * @param key
+     */
+    has(key) {
+        return this.keydir.has(key);
+    }
+    /**
+     * The document of a key, cached or read from the file (and cached). It is shared: it must not be changed.
+     *
+     * @param key
+     */
+    get(key) {
+        const text = key.toString("latin1");
+        const cached = this.cache.get(text);
+        if (cached)
+            return cached;
+        const slot = this.keydir.find(key);
+        if (slot < 0)
+            return undefined;
+        const length = this.keydir.lengthAt(slot);
+        const offset = this.keydir.offsetAt(slot);
+        const record = Buffer.allocUnsafe(length);
+        this.fileOf(this.keydir.fileAt(slot)).readInto(record, 0, length, offset);
+        const payload = record.subarray(EmbeddedLog_1.RECORD_HEADER);
+        if (zlib.crc32(payload, zlib.crc32(record.subarray(8, 9))) !==
+            record.readUInt32LE(4))
+            throw this.corrupted(offset);
+        const document = this.values.deserialize(payload);
+        this.cache.set(text, document, (0, EmbeddedCache_1.cachedSizeOf)(length));
+        return document;
+    }
+    /**
+     * The documents of the collection: the ones cached when they all are (as they are when it is called), else the
+     * ones of its files read in their order as it is iterated (cached while the cache has room): a document written
+     * meanwhile may be missed (updated or deleted before it is read, inserted) or, while the collection is compacted,
+     * read twice. They are shared: they must not be changed.
+     */
+    *documents() {
+        if (this.cache.size === this.keydir.size) {
+            yield* this.cache.documents();
+            return;
+        }
+        // During a compaction, the documents not copied yet are in the previous file
+        for (const [file, generation] of this.previous
+            ? [
+                [this.previous, this.generation ^ 1],
+                [this.file, this.generation],
+            ]
+            : [[this.file, this.generation]])
+            yield* this.liveDocuments(file, generation);
+    }
+    /**
+     * The documents of a file its keydir holds (cached while the cache has room).
+     *
+     * @param file
+     * @param generation
+     */
+    *liveDocuments(file, generation) {
+        const reader = new EmbeddedLog_1.EmbeddedRecordReader(file);
+        for (;;) {
+            const record = reader.next(true);
+            if (record === "end")
+                return;
+            if (typeof record === "string")
+                throw this.corrupted(reader.position);
+            if (record.type !== EmbeddedLog_1.RecordType.Put)
+                continue;
+            const key = this.values.idKeyOf(record.payload);
+            const slot = this.keydir.find(key);
+            if (slot < 0 ||
+                this.keydir.fileAt(slot) !== generation ||
+                this.keydir.offsetAt(slot) !== record.offset)
+                continue;
+            const text = key.toString("latin1");
+            let document = this.cache.get(text);
+            if (!document) {
+                document = this.values.deserialize(record.payload);
+                this.cache.offer(text, document, (0, EmbeddedCache_1.cachedSizeOf)(record.length));
+            }
+            yield document;
+        }
+    }
+    /**
+     * Writes changes of documents as one operation: all of them, or none when one of them has a duplicate key.
+     *
+     * @param changes
+     */
+    write(changes) {
+        if (!changes.length)
+            return;
+        const entries = this.indexChanges(changes);
+        this.changed = true;
+        const records = changes.map((change) => change.next
+            ? (0, EmbeddedLog_1.encodeRecord)(EmbeddedLog_1.RecordType.Put, change.data ?? this.values.serialize(change.next))
+            : (0, EmbeddedLog_1.encodeRecord)(EmbeddedLog_1.RecordType.Delete, this.values.serialize({ _id: change.previous._id })));
+        let offset = this.file.append(Buffer.concat([
+            ...records,
+            (0, EmbeddedLog_1.encodeRecord)(EmbeddedLog_1.RecordType.Commit, Buffer.alloc(0)),
+        ]));
+        changes.forEach((change, index) => {
+            const text = change.key.toString("latin1");
+            if (change.next) {
+                this.liveBytes +=
+                    records[index].length -
+                        this.keydir.set(change.key, offset, records[index].length, this.generation);
+                this.cache.set(text, change.next, (0, EmbeddedCache_1.cachedSizeOf)(records[index].length));
+            }
+            else {
+                this.liveBytes -= this.keydir.delete(change.key);
+                this.cache.delete(text);
+            }
+            offset += records[index].length;
+        });
+        for (const { index, tree, removed, added, id, keys } of entries) {
+            for (const key of removed)
+                tree.delete(key, id);
+            for (const key of added)
+                tree.insert(key, id);
+            if (keys > 1)
+                this.multikey.add(index);
+        }
+    }
+    /**
+     * The keys of the _id of the documents of the entries of ranges of an index, in its order (or reversed): a
+     * document is given once for each of its keys in the ranges.
+     *
+     * @param index
+     * @param ranges
+     * @param reverse
+     */
+    *idsIn(index, ranges, reverse = false) {
+        const tree = this.trees.get(index);
+        if (!tree)
+            return;
+        for (const range of reverse ? [...ranges].reverse() : ranges)
+            for (const [, id] of tree.range(range.lower, range.upper, reverse))
+                yield id;
+    }
+    /**
+     * Adds an index, unless one of its name exists: built from the documents (read from the two files of a compaction
+     * whose end it holds), in batches with the event loop yielded between them, while the writes of the collection wait
+     * (building) and no compaction starts; it fails on a key of a document too large (EmbeddedKeyTooLongError), and a
+     * unique one on the duplicate keys of the documents, the index not added. The queries use it once it is built.
+     *
+     * @param index
+     */
+    async createIndex(index) {
+        while (this.building)
+            await this.building;
+        if (this.indexes.some((existing) => existing.name === index.name))
+            return;
+        let built;
+        this.building = new Promise((resolve) => (built = resolve));
+        try {
+            // No compaction copying its documents meanwhile (a compaction copied waits for the build to end, and none
+            // starts while it runs)
+            while (this.compacting)
+                await this.compacting;
+            const tree = new EmbeddedBTree_1.EmbeddedBTree();
+            let multikey = false;
+            let read = 0;
+            for (const document of this.documents()) {
+                const id = this.values.idKey(document._id);
+                const keys = this.keysOf(index, document);
+                if (keys.length > 1)
+                    multikey = true;
+                for (const key of keys) {
+                    if (!EmbeddedBTree_1.EmbeddedBTree.fits(key, id))
+                        throw new EmbeddedKeyTooLongError(this.name, index.name, key.length, this.keyValue(index, document));
+                    if (index.unique && this.ownerOf(tree, key))
+                        throw new EmbeddedDuplicateKeyError(this.name, index.name, this.keyValue(index, document));
+                    tree.insert(key, id);
+                }
+                if (++read % BUILD_BATCH === 0)
+                    await new Promise((resolve) => setImmediate(resolve));
+            }
+            this.trees.set(index.name, tree);
+            if (multikey)
+                this.multikey.add(index.name);
+            this.changed = true;
+            this.file.append(Buffer.concat([
+                (0, EmbeddedLog_1.encodeRecord)(EmbeddedLog_1.RecordType.CreateIndex, this.values.bson.BSON.serialize(index)),
+                (0, EmbeddedLog_1.encodeRecord)(EmbeddedLog_1.RecordType.Commit, Buffer.alloc(0)),
+            ]));
+            this.indexes.push(index);
+        }
+        finally {
+            this.building = undefined;
+            built();
+        }
+    }
+    /**
+     * Removes an index; whether it existed.
+     *
+     * @param name
+     */
+    dropIndex(name) {
+        const position = this.indexes.findIndex((index) => index.name === name);
+        if (position < 0)
+            return false;
+        this.changed = true;
+        this.file.append(Buffer.concat([
+            (0, EmbeddedLog_1.encodeRecord)(EmbeddedLog_1.RecordType.DropIndex, this.values.bson.BSON.serialize({ name: name })),
+            (0, EmbeddedLog_1.encodeRecord)(EmbeddedLog_1.RecordType.Commit, Buffer.alloc(0)),
+        ]));
+        this.indexes.splice(position, 1);
+        this.trees.delete(name);
+        this.multikey.delete(name);
+        return true;
+    }
+    /**
+     * Whether a scan of its file read in the last SCAN_TIMEOUT; else its scans are closed (their next read fails).
+     *
+     * @param now
+     */
+    isScanned(now = Date.now()) {
+        for (const scan of this.scans)
+            if (now - scan.read < exports.SCAN_TIMEOUT)
+                return true;
+        for (const scan of this.scans)
+            scan.closed = true;
+        this.scans.clear();
+        return false;
+    }
+    /**
+     * Whether its file is worth compacting: larger than a size, and a ratio of the bytes of its live documents.
+     *
+     * @param minimumSize
+     * @param ratio
+     */
+    isWorthCompacting(minimumSize, ratio) {
+        return (!this.previous &&
+            this.file.size >= minimumSize &&
+            this.file.size > this.liveBytes * ratio);
+    }
+    /**
+     * Starts a compaction: a new file (its header record and the records of its indexes) is written from now on, the
+     * live documents of the previous one copied to it then.
+     *
+     * @param next
+     */
+    beginCompaction(next) {
+        const id = (0, crypto_1.randomBytes)(8).toString("hex");
+        next.append(Buffer.concat([
+            (0, EmbeddedLog_1.encodeRecord)(EmbeddedLog_1.RecordType.Header, this.values.bson.BSON.serialize({
+                name: this.name,
+                format: 1,
+                id: id,
+            })),
+            ...this.indexes.map((index) => (0, EmbeddedLog_1.encodeRecord)(EmbeddedLog_1.RecordType.CreateIndex, this.values.bson.BSON.serialize(index))),
+            COMMIT_RECORD,
+        ]));
+        this.previous = this.file;
+        this.file = next;
+        this.generation ^= 1;
+        this.logId = id;
+        this.changed = true;
+    }
+    /**
+     * Copies the documents of the previous file its keydir still holds there to the current one, a batch (one operation)
+     * at a time, yielding between the batches (the writes made meanwhile are written to the current file).
+     */
+    *copyLiveDocuments() {
+        const previous = this.previous;
+        const old = this.generation ^ 1;
+        const reader = new EmbeddedLog_1.EmbeddedRecordReader(previous);
+        let records = [];
+        let keys = [];
+        const flush = () => {
+            if (!records.length)
+                return;
+            let offset = this.file.append(Buffer.concat([...records, COMMIT_RECORD]));
+            records.forEach((record, index) => {
+                this.keydir.set(keys[index], offset, record.length, this.generation);
+                offset += record.length;
+            });
+            records = [];
+            keys = [];
+        };
+        for (;;) {
+            const record = reader.next(true);
+            if (record === "end")
+                break;
+            if (typeof record === "string")
+                throw this.corrupted(reader.position);
+            if (record.type !== EmbeddedLog_1.RecordType.Put)
+                continue;
+            const key = this.values.idKeyOf(record.payload);
+            const slot = this.keydir.find(key);
+            if (slot < 0 ||
+                this.keydir.fileAt(slot) !== old ||
+                this.keydir.offsetAt(slot) !== record.offset)
+                continue;
+            // Copied as it is: the same bytes, its live bytes unchanged. Its keydir slot checked above and set by
+            // the flush in one synchronous span (the flush before the yield): a write made meanwhile is not
+            // overwritten by its stale copy
+            records.push((0, EmbeddedLog_1.encodeRecord)(EmbeddedLog_1.RecordType.Put, record.payload));
+            keys.push(key);
+            if (records.length < 1000)
+                continue;
+            flush();
+            yield;
+        }
+        flush();
+    }
+    /**
+     * Ends a compaction: the previous file closed (its documents all copied).
+     */
+    endCompaction() {
+        this.previous.close();
+        this.previous = undefined;
+    }
+    /**
+     * Applies the records of the file of a compaction cut by a crash (written after the ones of its file), then runs
+     * the compaction to its end: its documents not copied yet copied.
+     *
+     * @param next
+     * @param path the path of the file, for the errors
+     */
+    resumeCompaction(next, path) {
+        const first = new EmbeddedLog_1.EmbeddedRecordReader(next).next(true);
+        if (typeof first === "string" || first.type !== EmbeddedLog_1.RecordType.Header)
+            throw new TypeORMError_1.TypeORMError(`The file ${path} is not a file of mongo-embedded`);
+        this.previous = this.file;
+        this.file = next;
+        this.generation ^= 1;
+        this.logId = this.values.deserialize(first.payload).id;
+        const created = [];
+        const reader = new EmbeddedLog_1.EmbeddedRecordReader(next, first.offset + first.length);
+        let pending = [];
+        let committed = first.offset + first.length;
+        for (;;) {
+            const record = reader.next(true);
+            if (record === "end" || record === "torn")
+                break;
+            if (record === "checksum") {
+                if (EmbeddedCollectionStore.isFollowedByCommit(next, reader))
+                    throw this.corrupted(reader.position);
+                break;
+            }
+            if (record.type === EmbeddedLog_1.RecordType.Commit) {
+                for (const done of pending)
+                    this.replay(done, created);
+                pending = [];
+                committed = record.offset + record.length;
+                continue;
+            }
+            const keyed = record.type === EmbeddedLog_1.RecordType.Put ||
+                record.type === EmbeddedLog_1.RecordType.Delete;
+            pending.push({
+                type: record.type,
+                offset: record.offset,
+                length: record.length,
+                key: keyed ? this.values.idKeyOf(record.payload) : undefined,
+                payload: record.type !== EmbeddedLog_1.RecordType.Delete
+                    ? this.values.deserialize(record.payload)
+                    : undefined,
+            });
+        }
+        if (committed < next.size) {
+            next.truncate(committed);
+            next.sync();
+        }
+        this.buildIndexes(created.filter((index) => this.indexes.includes(index)));
+        for (const _ of this.copyLiveDocuments())
+            continue;
+        this.changed = true;
+    }
+    /**
+     * Closes its files (the database closed, or the collection dropped): a compaction running stops, their uses after
+     * throw.
+     */
+    close() {
+        this.closed = true;
+        this.cache.close();
+        this.previous?.close();
+        this.file.close();
+    }
+    /**
+     * The file of a number of the slots of the keydir.
+     *
+     * @param generation
+     */
+    fileOf(generation) {
+        return generation === this.generation ? this.file : this.previous;
+    }
+    /**
+     * The error of a record of its file whose CRC-32 does not match.
+     *
+     * @param offset
+     */
+    corrupted(offset) {
+        return new TypeORMError_1.TypeORMError(`The file ${this.file.path ?? this.name} of mongo-embedded is corrupted at its offset ${offset}`);
+    }
+    /**
+     * Applies a record of an operation recovered.
+     *
+     * @param record
+     */
+    recover(record) {
+        switch (record.type) {
+            case EmbeddedLog_1.RecordType.Put:
+                this.liveBytes +=
+                    record.length -
+                        this.keydir.set(record.key, record.offset, record.length, this.generation);
+                break;
+            case EmbeddedLog_1.RecordType.Delete:
+                this.liveBytes -= this.keydir.delete(record.key);
+                break;
+            case EmbeddedLog_1.RecordType.CreateIndex:
+                if (!this.indexes.some((index) => index.name === record.payload.name))
+                    this.indexes.push(record.payload);
+                break;
+            case EmbeddedLog_1.RecordType.DropIndex: {
+                const position = this.indexes.findIndex((index) => index.name === record.payload.name);
+                if (position >= 0)
+                    this.indexes.splice(position, 1);
+                break;
+            }
+        }
+    }
+    /**
+     * Applies a record written after the snapshot read: its document in the keydir and the indexes (the indexes created
+     * are built once the records are applied).
+     *
+     * @param record
+     * @param created the indexes created by the records
+     */
+    replay(record, created) {
+        switch (record.type) {
+            case EmbeddedLog_1.RecordType.Put:
+            case EmbeddedLog_1.RecordType.Delete: {
+                const previous = this.get(record.key);
+                const next = record.type === EmbeddedLog_1.RecordType.Put ? record.payload : undefined;
+                const id = record.key;
+                for (const index of this.indexes) {
+                    const tree = this.trees.get(index.name);
+                    if (!tree)
+                        continue;
+                    const before = previous ? this.keysOf(index, previous) : [];
+                    const after = next ? this.keysOf(index, next) : [];
+                    for (const key of before)
+                        tree.delete(key, id);
+                    for (const key of after)
+                        tree.insert(key, id);
+                    if (after.length > 1)
+                        this.multikey.add(index.name);
+                }
+                this.cache.delete(id.toString("latin1"));
+                if (next)
+                    this.liveBytes +=
+                        record.length -
+                            this.keydir.set(id, record.offset, record.length, this.generation);
+                else
+                    this.liveBytes -= this.keydir.delete(id);
+                break;
+            }
+            case EmbeddedLog_1.RecordType.CreateIndex:
+                if (!this.indexes.some((index) => index.name === record.payload.name)) {
+                    const index = record.payload;
+                    this.indexes.push(index);
+                    created.push(index);
+                }
+                break;
+            case EmbeddedLog_1.RecordType.DropIndex: {
+                const position = this.indexes.findIndex((index) => index.name === record.payload.name);
+                if (position >= 0)
+                    this.indexes.splice(position, 1);
+                this.trees.delete(record.payload.name);
+                this.multikey.delete(record.payload.name);
+                break;
+            }
+        }
+    }
+    /**
+     * Builds indexes from the documents, read once; it fails on the duplicate keys of a unique one.
+     *
+     * @param indexes
+     */
+    buildIndexes(indexes) {
+        if (!indexes.length)
+            return;
+        const trees = indexes.map(() => new EmbeddedBTree_1.EmbeddedBTree());
+        for (const document of this.documents()) {
+            const id = this.values.idKey(document._id);
+            indexes.forEach((index, position) => {
+                const keys = this.keysOf(index, document);
+                if (keys.length > 1)
+                    this.multikey.add(index.name);
+                for (const key of keys) {
+                    if (index.unique && this.ownerOf(trees[position], key))
+                        throw new EmbeddedDuplicateKeyError(this.name, index.name, this.keyValue(index, document));
+                    trees[position].insert(key, id);
+                }
+            });
+        }
+        indexes.forEach((index, position) => this.trees.set(index.name, trees[position]));
+    }
+    /**
+     * The changes of the entries of the indexes of a write, its unique keys checked (also between its changes).
+     *
+     * @param changes
+     */
+    indexChanges(changes) {
+        const entries = [];
+        const inserted = new Set();
+        // The keys of the unique indexes written: their owners (the keys of their _id), null for the ones removed
+        const pending = new Map();
+        for (const change of changes) {
+            const owner = change.key.toString("latin1");
+            if (!change.previous && change.next) {
+                if (this.keydir.has(change.key) || inserted.has(owner))
+                    throw new EmbeddedDuplicateKeyError(this.name, "_id_", {
+                        _id: change.next._id,
+                    });
+                inserted.add(owner);
+            }
+            for (const index of this.indexes) {
+                const tree = this.trees.get(index.name);
+                const before = change.previous
+                    ? this.keysOf(index, change.previous)
+                    : [];
+                const after = change.next ? this.keysOf(index, change.next) : [];
+                const kept = new Set(before.map((key) => key.toString("latin1")));
+                const now = new Set(after.map((key) => key.toString("latin1")));
+                const removed = before.filter((key) => !now.has(key.toString("latin1")));
+                const added = after.filter((key) => !kept.has(key.toString("latin1")));
+                // Checked before the write is written: an entry not fitting a tree would throw once it is
+                for (const key of added)
+                    if (!EmbeddedBTree_1.EmbeddedBTree.fits(key, change.key))
+                        throw new EmbeddedKeyTooLongError(this.name, index.name, key.length, this.keyValue(index, change.next));
+                if (index.unique) {
+                    let written = pending.get(index.name);
+                    if (!written)
+                        pending.set(index.name, (written = new Map()));
+                    for (const key of removed)
+                        written.set(key.toString("latin1"), null);
+                    for (const key of added) {
+                        const text = key.toString("latin1");
+                        const existing = written.has(text)
+                            ? written.get(text)
+                            : this.ownerOf(tree, key)?.toString("latin1");
+                        if (existing !== undefined &&
+                            existing !== null &&
+                            existing !== owner)
+                            throw new EmbeddedDuplicateKeyError(this.name, index.name, this.keyValue(index, change.next));
+                        written.set(text, owner);
+                    }
+                }
+                entries.push({
+                    index: index.name,
+                    tree: tree,
+                    removed: removed,
+                    added: added,
+                    id: change.key,
+                    keys: after.length,
+                });
+            }
+        }
+        return entries;
+    }
+    /**
+     * The key of the _id of the document holding a key in an index, if any.
+     *
+     * @param tree
+     * @param key
+     */
+    ownerOf(tree, key) {
+        for (const [, id] of tree.range({ key: key, inclusive: true }, { key: key, inclusive: true }))
+            return id;
+        return undefined;
+    }
+    /**
+     * The keys of a document in an index.
+     *
+     * @param index
+     * @param document
+     */
+    keysOf(index, document) {
+        return this.encoding.keysOf(index.key, index.sparse, document);
+    }
+    /**
+     * The fields of a document in an index, for the message of a duplicate key.
+     *
+     * @param index
+     * @param document
+     */
+    keyValue(index, document) {
+        const value = {};
+        for (const path of Object.keys(index.key))
+            value[path] = this.values.valueAt(document, path) ?? null;
+        return value;
+    }
+}
+exports.EmbeddedCollectionStore = EmbeddedCollectionStore;
+//# sourceMappingURL=EmbeddedCollectionStore.js.map