@harperfast/harper 5.0.0-alpha.10

package/dist/resources/blob.js

@@ -0,0 +1,1273 @@
+"use strict";
+/**
+ * This module provides a Blob class that can be used to store binary data in the database, and can be used to store large binary data in a file
+ * on the server. The Blob class is a subclass of the global Blob class, and can be used in the same way.
+ * The Blob-backed files begin with an 8-byte header:
+ * - The first 2 bytes indicate the type of storage:
+ * 		- 0: Uncompressed
+ * 		- 1: Compressed with deflate
+ * 	  - 0xff: Error state (followed by error message). A record can be saved prior to an error in saving a blob, so we must be capable of tracking and even replicating that state
+ * - The next 6 bytes are the size of the content
+ *   - While the file is being written, 0xffffffffffff is used as a placeholder to indicate that the file is not finished being written (this nicely matches the logic that if the written content size is less than the indicated content size, it is not finished)
+ *   - Note that for compressed data, the size is the uncompressed size, and the compressed size in the file
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.databasePaths = exports.blobsWereEncoded = exports.Blob = void 0;
+exports.deleteBlob = deleteBlob;
+exports.setDeletionDelay = setDeletionDelay;
+exports.saveBlob = saveBlob;
+exports.getFileId = getFileId;
+exports.isSaving = isSaving;
+exports.getFilePathForBlob = getFilePathForBlob;
+exports.getRootBlobPathsForDB = getRootBlobPathsForDB;
+exports.deleteRootBlobPathsForDB = deleteRootBlobPathsForDB;
+exports.encodeBlobsWithFilePath = encodeBlobsWithFilePath;
+exports.encodeBlobsAsBuffers = encodeBlobsAsBuffers;
+exports.decodeBlobsWithWrites = decodeBlobsWithWrites;
+exports.decodeWithBlobCallback = decodeWithBlobCallback;
+exports.decodeFromDatabase = decodeFromDatabase;
+exports.deleteBlobsInObject = deleteBlobsInObject;
+exports.findBlobsInObject = findBlobsInObject;
+exports.startPreCommitBlobsForRecord = startPreCommitBlobsForRecord;
+exports.cleanupOrphans = cleanupOrphans;
+const msgpackr_1 = require("msgpackr");
+const promises_1 = require("node:fs/promises");
+const node_fs_1 = require("node:fs");
+const node_zlib_1 = require("node:zlib");
+const node_stream_1 = require("node:stream");
+const fs_extra_1 = require("fs-extra");
+const environmentManager_js_1 = require("../utility/environment/environmentManager.js");
+const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
+const path_1 = require("path");
+const logger_ts_1 = require("../utility/logging/logger.js");
+const contentTypes_ts_1 = require("../server/serverHelpers/contentTypes.js");
+const auditStore_ts_1 = require("./auditStore.js");
+const node_v8_1 = require("node:v8");
+const promises_2 = require("node:timers/promises");
+const FILE_STORAGE_THRESHOLD = 8192; // if the file is below this size, we will store it in memory, or within the record itself, otherwise we will store it in a file
+// We want to keep the file path private (but accessible to the extension)
+const HEADER_SIZE = 8;
+const UNCOMPRESSED_TYPE = 0;
+const DEFLATE_TYPE = 1;
+const ERROR_TYPE = 0xff;
+const DEFAULT_HEADER = new Uint8Array([0, UNCOMPRESSED_TYPE, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]);
+const COMPRESS_HEADER = new Uint8Array([0, DEFLATE_TYPE, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]);
+const UNKNOWN_SIZE = 0xffffffffffff;
+const storageInfoForBlob = new WeakMap();
+let currentBlobCallback;
+exports.Blob = global.Blob || polyfillBlob(); // use the global Blob class if it exists (it doesn't on Node v16)
+let encodeForStorageForRecordId = undefined; // only enable encoding of the file path if we are saving to the DB, not for serialization to external clients, and only for one record
+let promisedWrites;
+let currentStore; // the root store of the database we are currently encoding for
+exports.blobsWereEncoded = false; // keep track of whether blobs were encoded with file paths
+// the header is 8 bytes
+// this is a reusable buffer for reading and writing to the header (without having to create new allocations)
+const HEADER = new Uint8Array(8);
+const headerView = new DataView(HEADER.buffer);
+const FILE_READ_TIMEOUT = 60000;
+// We want FileBackedBlob instances to be an instanceof Blob, but we don't want to actually extend the class and call Blob's constructor, which is quite expensive because it has to set it up as a transferrable.
+function InstanceOfBlobWithNoConstructor() { }
+InstanceOfBlobWithNoConstructor.prototype = exports.Blob.prototype;
+// @ts-ignore
+/**
+ * A blob that is backed by a file, and can be saved to the database as a reference
+ * Note that this is used instead of the native Blob class for a few reasons:
+ * 1. This has the built-in functionality for reading from the file-based storage
+ * 2. This support for streams and asynchronous access to data that may not have a known size ahead of time
+ * 3. This also avoids the Blob constructor which is expensive due to the transferred setup
+ * Harper still supports saving native Blobs, but when they blobs are retrieved from storage, they always use this class.
+ */
+class FileBackedBlob extends InstanceOfBlobWithNoConstructor {
+    type = '';
+    size;
+    #onError;
+    #onSize;
+    constructor(options) {
+        super();
+        if (options?.type)
+            this.type = options.type;
+        if (options?.size != undefined)
+            this.size = options.size;
+        if (options?.saveBeforeCommit != undefined)
+            this.saveBeforeCommit = options.saveBeforeCommit;
+    }
+    on(type, callback) {
+        if (type === 'error') {
+            this.#onError ??= [];
+            this.#onError.push(callback);
+        }
+        else if (type === 'size') {
+            this.#onSize ??= [];
+            this.#onSize.push(callback);
+        }
+        else
+            throw new Error("Only 'error' and 'size' events are supported");
+    }
+    toJSON() {
+        if (this.type?.startsWith('text')) {
+            const storageInfo = storageInfoForBlob.get(this);
+            let { start, end, contentBuffer, asString } = storageInfo;
+            if (asString) {
+                return asString;
+            }
+            if (contentBuffer && (end !== undefined || start !== undefined)) {
+                contentBuffer = contentBuffer.subarray(start ?? 0, end ?? storageInfo.contentBuffer.length);
+            }
+            // if we have a content buffer we can return
+            if (contentBuffer) {
+                storageInfo.asString = contentBuffer.toString();
+                return storageInfo.asString;
+            }
+            if ((0, contentTypes_ts_1.hasAsyncSerialization)())
+                (0, contentTypes_ts_1.asyncSerialization)(this.bytes().then((buffer) => (storageInfo.contentBuffer = buffer)));
+            return `[blob: ${this.type}, ${this.size} bytes]`;
+        }
+        return {
+            description: 'Blobs that are not of type text/* can not be directly serialized as JSON, use as the body of a response or convert to another type',
+        };
+    }
+    async text() {
+        return (await this.bytes()).toString();
+    }
+    bytes() {
+        const storageInfo = storageInfoForBlob.get(this);
+        let { start, end, contentBuffer } = storageInfo;
+        if (contentBuffer) {
+            if (end !== undefined || start !== undefined) {
+                contentBuffer = contentBuffer.subarray(start ?? 0, end ?? storageInfo.contentBuffer.length);
+            }
+            return Promise.resolve(contentBuffer);
+        }
+        const filePath = getFilePath(storageInfo);
+        let writeFinished;
+        const readContents = async () => {
+            let rawBytes;
+            let size = HEADER_SIZE;
+            try {
+                rawBytes = await (0, promises_1.readFile)(filePath);
+                if (rawBytes.length >= HEADER_SIZE) {
+                    rawBytes.copy(HEADER, 0, 0, HEADER_SIZE);
+                    const headerValue = headerView.getBigUint64(0);
+                    if (Number(headerValue >> 48n) === ERROR_TYPE) {
+                        throw new Error('Error in blob: ' + rawBytes.subarray(HEADER_SIZE));
+                    }
+                    size = Number(headerValue & 0xffffffffffffn);
+                    if (size < end)
+                        size = end;
+                    if (size < UNKNOWN_SIZE) {
+                        this.size = size;
+                        if (this.#onSize) {
+                            for (const callback of this.#onSize)
+                                callback(size);
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                if (error.code !== 'ENOENT')
+                    throw error;
+                rawBytes = Buffer.alloc(0);
+            }
+            function checkCompletion(rawBytes) {
+                if (size > rawBytes.length) {
+                    // the file is not finished being written, wait for the write lock to complete
+                    const store = storageInfo.store;
+                    const lockKey = storageInfo.fileId + ':blob';
+                    if (writeFinished) {
+                        throw new Error(`Incomplete blob for ${filePath}`);
+                    }
+                    return new Promise((resolve) => {
+                        const callback = () => {
+                            writeFinished = true;
+                            return resolve(readContents());
+                        };
+                        const lockAcquired = store.tryLock(lockKey, callback);
+                        if (lockAcquired) {
+                            writeFinished = true;
+                            store.unlock(lockKey);
+                            return resolve(readContents());
+                        }
+                    });
+                }
+                if (end != undefined || start != undefined) {
+                    rawBytes = rawBytes.subarray(start ?? 0, end ?? rawBytes.length);
+                }
+                return rawBytes;
+            }
+            if (rawBytes[1] === DEFLATE_TYPE) {
+                return new Promise((resolve, reject) => {
+                    (0, node_zlib_1.deflate)(rawBytes.subarray(HEADER_SIZE), (error, result) => {
+                        if (error)
+                            reject(error);
+                        else
+                            resolve(checkCompletion(result));
+                    });
+                });
+            }
+            return checkCompletion(rawBytes.subarray(HEADER_SIZE));
+        };
+        return readContents();
+    }
+    async arrayBuffer() {
+        const bytes = await this.bytes();
+        const arrayBuffer = new ArrayBuffer(bytes.length);
+        const bufferUint8 = new Uint8Array(arrayBuffer);
+        bufferUint8.set(bytes);
+        return arrayBuffer;
+    }
+    stream() {
+        const storageInfo = storageInfoForBlob.get(this);
+        let { contentBuffer, start, end } = storageInfo;
+        if (contentBuffer) {
+            if (end != undefined || start != undefined) {
+                contentBuffer = contentBuffer.subarray(start ?? 0, end ?? storageInfo.contentBuffer.length);
+            }
+            return new ReadableStream({
+                pull(controller) {
+                    controller.enqueue(contentBuffer);
+                    controller.close();
+                },
+            });
+        }
+        const filePath = getFilePath(storageInfo);
+        let fd;
+        let position = 0;
+        let totalContentRead = 0;
+        let watcher;
+        let timer;
+        let isBeingWritten;
+        let previouslyFinishedWriting = false;
+        const blob = this;
+        return new ReadableStream({
+            start() {
+                let retries = 1000;
+                const openFile = (resolve, reject) => {
+                    (0, node_fs_1.open)(filePath, 'r', (error, openedFd) => {
+                        if (error) {
+                            if (error.code === 'ENOENT' && isBeingWritten !== false) {
+                                logger_ts_1.logger.debug?.('File does not exist yet, waiting for it to be created', filePath, retries);
+                                // the file doesn't exist, so we need to wait for it to be created
+                                if (retries-- > 0)
+                                    return setTimeout(() => {
+                                        checkIfIsBeingWritten();
+                                        openFile(resolve, reject);
+                                    }, 20).unref();
+                            }
+                            reject(error);
+                            blob.#onError?.forEach((callback) => callback(error));
+                        }
+                        else {
+                            fd = openedFd;
+                            resolve(openedFd);
+                        }
+                    });
+                };
+                return new Promise(openFile);
+            },
+            pull: (controller) => {
+                let size = 0;
+                let retries = 100;
+                return new Promise(function readMore(resolve, reject) {
+                    function onError(error) {
+                        (0, node_fs_1.close)(fd);
+                        clearTimeout(timer);
+                        if (watcher)
+                            watcher.close();
+                        reject(error);
+                        blob.#onError?.forEach((callback) => callback(error));
+                    }
+                    // allocate a buffer for reading. Note that we could do a stat to get the size, but that is a little more complicated, and might be a little extra overhead
+                    const buffer = Buffer.allocUnsafe(0x40000);
+                    (0, node_fs_1.read)(fd, buffer, 0, buffer.length, position, (error, bytesRead, buffer) => {
+                        // TODO: Implement support for decompression
+                        totalContentRead += bytesRead;
+                        if (error)
+                            return onError(error);
+                        if (position === 0) {
+                            // for the first read, we need to read the header and skip it for the data
+                            // but first check to see if we read anything
+                            if (bytesRead < HEADER_SIZE) {
+                                // didn't read any bytes, have to try again
+                                if (retries-- > 0 && isBeingWritten !== false) {
+                                    checkIfIsBeingWritten();
+                                    logger_ts_1.logger.debug?.('File was empty, waiting for data to be written', filePath, retries);
+                                    setTimeout(() => readMore(resolve, reject), 20).unref();
+                                }
+                                else {
+                                    logger_ts_1.logger.debug?.('File was empty, throwing error', filePath, retries);
+                                    onError(new Error(`Blob ${storageInfo.fileId} was empty`));
+                                }
+                                // else throw new Error();
+                                return;
+                            }
+                            buffer.copy(HEADER, 0, 0, HEADER_SIZE);
+                            const headerValue = headerView.getBigUint64(0);
+                            if (Number(headerValue >> 48n) === ERROR_TYPE) {
+                                return onError(new Error('Error in blob: ' + buffer.subarray(HEADER_SIZE, bytesRead)));
+                            }
+                            size = Number(headerValue & 0xffffffffffffn);
+                            if (size < UNKNOWN_SIZE && blob.size !== size) {
+                                blob.size = size;
+                                if (blob.#onSize) {
+                                    for (const callback of blob.#onSize)
+                                        callback(size);
+                                }
+                            }
+                            buffer = buffer.subarray(HEADER_SIZE, bytesRead);
+                            totalContentRead -= HEADER_SIZE;
+                        }
+                        else if (bytesRead === 0) {
+                            const buffer = Buffer.allocUnsafe(8);
+                            return (0, node_fs_1.read)(fd, buffer, 0, HEADER_SIZE, 0, (error) => {
+                                if (error)
+                                    return onError(error);
+                                HEADER.set(buffer);
+                                size = Number(headerView.getBigUint64(0) & 0xffffffffffffn);
+                                if (size > totalContentRead) {
+                                    if (checkIfIsBeingWritten()) {
+                                        // the file is not finished being written, watch the file for changes to resume reading
+                                        // set up a watcher to be notified of file changes
+                                        watcher = (0, node_fs_1.watch)(filePath, { persistent: false }, () => {
+                                            watcher.close();
+                                            watcher = null;
+                                            clearTimeout(timer); // clear it
+                                            readMore(resolve, reject);
+                                        });
+                                        // immediately try to read again in case there was a change before we started watching,
+                                        // readSync should be fine here, the data should be in memory
+                                        if ((0, node_fs_1.readSync)(fd, buffer, 0, buffer.length, position) > 0) {
+                                            // never mind with the watcher, let's read more data
+                                            watcher.close();
+                                            watcher = null;
+                                            readMore(resolve, reject);
+                                        }
+                                        else {
+                                            // set a timer for the watcher too
+                                            timer = setTimeout(() => {
+                                                onError(new Error(`File read timed out reading from ${filePath}`));
+                                            }, FILE_READ_TIMEOUT).unref();
+                                        }
+                                    }
+                                    else {
+                                        if (previouslyFinishedWriting) {
+                                            // we verified that the blob was finished writing before the last read, we can confidently say it is incomplete
+                                            onError(new Error('Blob is incomplete'));
+                                        }
+                                        else {
+                                            previouslyFinishedWriting = true;
+                                            readMore(resolve, reject); // try again (possibly for the last time) now that we know the status of the file writing
+                                        }
+                                        // do NOT close the controller, or the error won't propagate to the stream
+                                    }
+                                    return;
+                                }
+                                (0, node_fs_1.close)(fd);
+                                controller.close();
+                                resolve();
+                            });
+                        }
+                        else {
+                            buffer = buffer.subarray(0, bytesRead);
+                        }
+                        if (start !== undefined || end !== undefined) {
+                            if (start && totalContentRead < start) {
+                                // we are before the start of the slice, so we need to read more
+                                position += bytesRead;
+                                return readMore(resolve, reject);
+                            }
+                            if (end && totalContentRead >= end) {
+                                // we are past or reached the end of the slice, so we have reached the end, indicate
+                                if (totalContentRead > end)
+                                    buffer = buffer.subarray(0, end - position);
+                                totalContentRead = size = end;
+                            }
+                            if (start && start > position) {
+                                // we need to skip ahead to the start of the slice
+                                buffer = buffer.subarray(start - position);
+                            }
+                        }
+                        position += bytesRead;
+                        try {
+                            controller.enqueue(buffer);
+                        }
+                        catch (error) {
+                            // we need to catch the error here, because if the controller is closed, it will throw an error
+                            // but we still want to resolve the promise
+                            logger_ts_1.logger.debug?.('Error enqueuing chunk', error);
+                            return resolve();
+                        }
+                        if (totalContentRead === size) {
+                            (0, node_fs_1.close)(fd);
+                            controller.close();
+                        }
+                        resolve();
+                    });
+                });
+            },
+            cancel() {
+                (0, node_fs_1.close)(fd);
+                clearTimeout(timer);
+                if (watcher)
+                    watcher.close();
+            },
+        });
+        function checkIfIsBeingWritten() {
+            if (isBeingWritten === undefined) {
+                const store = storageInfo.store;
+                const lockKey = storageInfo.fileId + ':blob';
+                isBeingWritten = !store.tryLock(lockKey, () => {
+                    isBeingWritten = false;
+                });
+                if (!isBeingWritten)
+                    store.unlock(lockKey);
+            }
+            return isBeingWritten;
+        }
+    }
+    slice(start, end, type) {
+        const sourceStorageInfo = storageInfoForBlob.get(this);
+        const slicedBlob = new FileBackedBlob(type && { type });
+        if (sourceStorageInfo?.fileId) {
+            const slicedStorageInfo = {
+                ...sourceStorageInfo,
+                start,
+                end,
+            };
+            storageInfoForBlob.set(slicedBlob, slicedStorageInfo);
+            if (this.size != undefined)
+                slicedBlob.size = (end == undefined ? this.size : Math.min(end, this.size)) - (start ?? 0);
+        }
+        else if (sourceStorageInfo?.contentBuffer && !sourceStorageInfo.storageBuffer) {
+            const slicedStorageInfo = {
+                ...sourceStorageInfo,
+                contentBuffer: sourceStorageInfo.contentBuffer.subarray(start, end),
+            };
+            storageInfoForBlob.set(slicedBlob, slicedStorageInfo);
+            slicedBlob.size = (end ?? this.size) - start;
+        }
+        else {
+            // TODO: Implement this
+            throw new Error('Can not slice a streaming blob that is not backed by a file');
+        }
+        return slicedBlob;
+    }
+    get written() {
+        return storageInfoForBlob.get(this)?.saving ?? Promise.resolve();
+    }
+}
+let deletionDelay = 500;
+/**
+ * Delete the file for the blob
+ * @param blob
+ */
+function deleteBlob(blob) {
+    // do we even need to check for completion here?
+    const filePath = getFilePathForBlob(blob);
+    if (!filePath) {
+        return;
+    }
+    setTimeout(() => {
+        // TODO: we need to determine when any read transaction are done with the file, and then delete it, this is a hack to just give it some time for that
+        (0, node_fs_1.unlink)(filePath, (error) => {
+            if (error)
+                logger_ts_1.logger.debug?.('Error trying to remove blob file', error);
+        });
+    }, deletionDelay);
+}
+function setDeletionDelay(delay) {
+    deletionDelay = delay;
+}
+/**
+ * Create a blob from a readable stream or a buffer by creating a file in the blob storage path with a new unique internal id, that
+ * can be saved/stored.
+ * @param source
+ */
+global.createBlob = function (source, options) {
+    const blob = new FileBackedBlob(options);
+    const storageInfo = {
+        storageIndex: 0,
+        fileId: null,
+        flush: options?.flush,
+        compress: options?.compress,
+    };
+    storageInfoForBlob.set(blob, storageInfo);
+    if (source instanceof Uint8Array) {
+        blob.size = source.length;
+        storageInfo.contentBuffer = source;
+    }
+    else if (source instanceof node_stream_1.Readable) {
+        storageInfo.source = source;
+    }
+    else if (typeof source === 'string')
+        storageInfo.contentBuffer = Buffer.from(source);
+    else if (source?.[Symbol.asyncIterator] || source?.[Symbol.iterator])
+        storageInfo.source = node_stream_1.Readable.from(source);
+    else
+        throw new Error('Invalid source type');
+    return blob;
+};
+function saveBlob(blob, deleteOnFailure = false) {
+    let storageInfo = storageInfoForBlob.get(blob);
+    if (!storageInfo) {
+        storageInfo = { storageIndex: 0, fileId: null, store: currentStore };
+        storageInfoForBlob.set(blob, storageInfo);
+    }
+    else {
+        if (storageInfo.fileId)
+            return storageInfo; // if there is any file id, we are already saving and can return the info
+        storageInfo.store = currentStore;
+    }
+    storageInfo.deleteOnFailure = deleteOnFailure;
+    if (blob.saveInRecord) {
+        if (!storageInfo.contentBuffer) {
+            // TODO: Add support for this: https://github.com/HarperFast/harper/issues/141
+            throw new Error('Cannot publish a message with a streamed blob');
+        }
+        return storageInfo; // nothing more to do if it supposed to be saved in the record
+    }
+    generateFilePath(storageInfo);
+    if (storageInfo.source)
+        writeBlobWithStream(blob, storageInfo.source, storageInfo);
+    else if (storageInfo.contentBuffer)
+        writeBlobWithBuffer(blob, storageInfo);
+    else {
+        // for native blobs, we have to read them from the stream
+        writeBlobWithStream(blob, node_stream_1.Readable.from(blob.stream()), storageInfo);
+    }
+    return storageInfo;
+}
+/**
+ * Create a blob from a readable stream
+ */
+function writeBlobWithStream(blob, stream, storageInfo) {
+    const { filePath, fileId, store, compress, flush } = storageInfo;
+    storageInfo.saving = new Promise((resolve, reject) => {
+        // pipe the stream to the file
+        const lockKey = fileId + ':blob';
+        if (!store.tryLock(lockKey)) {
+            throw new Error(`Unable to get lock for blob file ${fileId}`);
+        }
+        const writeStream = (0, node_fs_1.createWriteStream)(filePath, { autoClose: false, flags: 'w' });
+        let wroteSize = false;
+        if (blob.size !== undefined) {
+            // if we know the size, we can write the header immediately
+            writeStream.write(createHeader(blob.size)); // write the default header
+            wroteSize = true;
+        }
+        let compressedStream;
+        if (compress) {
+            if (!wroteSize)
+                writeStream.write(COMPRESS_HEADER); // write the default header to the file
+            compressedStream = (0, node_zlib_1.createDeflate)();
+            (0, node_stream_1.pipeline)(stream, compressedStream, writeStream, finished);
+        }
+        else {
+            if (!wroteSize)
+                writeStream.write(DEFAULT_HEADER); // write the default header to the file
+            (0, node_stream_1.pipeline)(stream, writeStream, finished);
+        }
+        function createHeader(size) {
+            let headerValue = BigInt(size);
+            const header = new Uint8Array(HEADER_SIZE);
+            const headerView = new DataView(header.buffer);
+            headerValue |= BigInt(compress ? DEFLATE_TYPE : UNCOMPRESSED_TYPE) << 48n;
+            headerView.setBigInt64(0, headerValue);
+            return header;
+        }
+        // when the stream is finished, we may need to flush, and then close the handle and resolve the promise
+        function finished(error) {
+            const fd = writeStream.fd;
+            if (error) {
+                store.unlock(lockKey);
+                if (fd) {
+                    (0, node_fs_1.close)(fd);
+                    writeStream.fd = null; // do not close the same fd twice, that is very dangerous because it might represent a new fd
+                }
+                if (storageInfo.deleteOnFailure) {
+                    (0, node_fs_1.unlink)(filePath, (error) => {
+                        if (error)
+                            logger_ts_1.logger.debug?.('Error while deleting aborted blob file', error);
+                    });
+                }
+                else {
+                    try {
+                        if ((0, node_fs_1.statSync)(filePath).size === 0) {
+                            // if there was an error in the stream, nothing may have been written, so we can write the error message instead
+                            const errorBuffer = Buffer.from(error.toString());
+                            (0, node_fs_1.writeFile)(filePath, Buffer.concat([createHeader(BigInt(errorBuffer.length) + 0xff000000000000n), errorBuffer]), (error) => {
+                                if (error)
+                                    logger_ts_1.logger.debug?.('Error write error message to blob file', error);
+                            });
+                        }
+                    }
+                    catch (error) {
+                        logger_ts_1.logger.debug?.('Error checking blob file after abort', error);
+                    }
+                }
+                reject(error);
+            }
+            else {
+                if (!wroteSize) {
+                    wroteSize = true;
+                    const size = compressedStream ? compressedStream.bytesWritten : writeStream.bytesWritten - HEADER_SIZE;
+                    blob.size = size;
+                    (0, node_fs_1.write)(fd, createHeader(size), 0, HEADER_SIZE, 0, finished);
+                    return; // not finished yet, wait for this write and then we are finished
+                }
+                store.unlock(lockKey);
+                if (flush) {
+                    // we just use fdatasync because we really aren't that concerned with flushing file metadata
+                    (0, node_fs_1.fdatasync)(fd, (error) => {
+                        if (error)
+                            reject(error);
+                        resolve();
+                        (0, node_fs_1.close)(fd);
+                        writeStream.fd = null; // do not close the same fd twice, that is very dangerous because it might represent a new fd
+                    });
+                }
+                else {
+                    resolve();
+                    (0, node_fs_1.close)(fd);
+                    writeStream.fd = null; // do not close the same fd twice, that is very dangerous because it might represent a new fd
+                }
+            }
+        }
+    });
+    return blob;
+}
+function getFileId(blob) {
+    return storageInfoForBlob.get(blob)?.fileId;
+}
+function isSaving(blob) {
+    return storageInfoForBlob.get(blob)?.saving;
+}
+function getFilePathForBlob(blob) {
+    const storageInfo = storageInfoForBlob.get(blob);
+    return storageInfo?.fileId && getFilePath(storageInfo);
+}
+exports.databasePaths = new Map();
+function getRootBlobPathsForDB(store) {
+    if (!store) {
+        throw new Error('No store specified, can not determine blob storage path');
+    }
+    let paths = exports.databasePaths.get(store);
+    if (!paths) {
+        if (!store.databaseName) {
+            logger_ts_1.logger.warn?.('No database name specified, can not determine blob storage path');
+            return [];
+        }
+        const blobPaths = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_BLOBPATHS);
+        if (blobPaths) {
+            paths = blobPaths.map((path) => (0, path_1.join)(path, store.databaseName));
+        }
+        else {
+            paths = [(0, path_1.join)((0, environmentManager_js_1.getHdbBasePath)(), 'blobs', store.databaseName)];
+        }
+        exports.databasePaths.set(store, paths);
+    }
+    return paths;
+}
+async function deleteRootBlobPathsForDB(store) {
+    const paths = getRootBlobPathsForDB(store);
+    if (paths) {
+        await Promise.all(paths.map((path) => rimrafSteadily(path)));
+    }
+}
+/**
+ * recursively delete a directory and all of its contents, but do it one at a time, so that we don't run out of memory and hog resources
+ * @param path
+ */
+async function rimrafSteadily(path) {
+    if (!(0, node_fs_1.existsSync)(path))
+        return;
+    for (const entry of await (0, promises_1.readdir)(path, { withFileTypes: true })) {
+        if (entry.isDirectory()) {
+            await rimrafSteadily((0, path_1.join)(path, entry.name));
+        }
+        else {
+            try {
+                await (0, promises_1.unlink)((0, path_1.join)(path, entry.name));
+            }
+            catch (error) {
+                logger_ts_1.logger.warn?.('Error deleting file', error);
+            }
+        }
+    }
+    try {
+        await (0, promises_1.rmdir)(path);
+    }
+    catch (error) {
+        logger_ts_1.logger.warn?.('Error deleting directory', error);
+    }
+}
+function getFilePath({ storageIndex, fileId, store }) {
+    const blobStoragePaths = getRootBlobPathsForDB(store);
+    return (0, path_1.join)(
+    // Use a hierarchy of directories to store the file by id, to avoid to many entries in a single directory. This uses 4096 files or directories per parent directory
+    blobStoragePaths[storageIndex], fileId.slice(-9, -6) || '0', fileId.slice(-6, -3) || '0', fileId.length <= 9 ? fileId.slice(-3) : fileId.slice(0, -9) + fileId.slice(-3) // after 68 billion entries, we effectively wrap around and start reusing directories again, assuming the most the entries have been deleted
+    );
+}
+/**
+ * Create a blob from a buffer
+ * @param buffer
+ */
+function writeBlobWithBuffer(blob, storageInfo) {
+    // we know the size, so we can create the header immediately
+    const buffer = storageInfo.contentBuffer;
+    const size = buffer.length;
+    if (size < FILE_STORAGE_THRESHOLD) {
+        // if the buffer is small enough, just store it in memory
+        return;
+    }
+    blob.size = size;
+    return writeBlobWithStream(blob, node_stream_1.Readable.from([buffer]), storageInfo);
+}
+/**
+ * Create a blob that is backed by a *new* file with a new unique internal id, so it can be filled with data and saved to the database
+ */
+function generateFilePath(storageInfo) {
+    const blobStoragePaths = getRootBlobPathsForDB(storageInfo.store);
+    const id = getNextFileId();
+    // get the storage index, which is the index of the blob storage path to use, distributed round-robin based on the id
+    const storageIndex = blobStoragePaths?.length > 1 ? getNextStorageIndex(blobStoragePaths, id) : 0;
+    const fileId = id.toString(16); // get the next file id
+    storageInfo.storageIndex = storageIndex;
+    storageInfo.fileId = fileId;
+    const filePath = getFilePath(storageInfo);
+    const fileDir = (0, path_1.dirname)(filePath);
+    // ensure the directory structure exists
+    if (!(0, node_fs_1.existsSync)(fileDir))
+        (0, fs_extra_1.ensureDirSync)(fileDir);
+    storageInfo.filePath = filePath;
+}
+const idIncrementers = new Map();
+function getNextFileId() {
+    // all threads will use a shared buffer to atomically increment the id
+    // first, we create our proposed incrementer buffer that will be used if we are the first thread to get here
+    // and initialize it with the starting id
+    let idIncrementer = idIncrementers.get(currentStore);
+    if (!idIncrementer) {
+        // get the last id by checking the highest id in all the blob storage paths
+        let highestId = 0;
+        const blobStoragePaths = getRootBlobPathsForDB(currentStore);
+        for (let path of blobStoragePaths) {
+            let id = 0;
+            // we need to get the highest id in the directory structure, so we need to iterate through all the directories to find the highest byte sequence
+            for (let i = 0; i < 3; i++) {
+                id = id * 0x1000;
+                let highest = 0;
+                if ((0, node_fs_1.existsSync)(path)) {
+                    for (const entry of (0, node_fs_1.readdirSync)(path)) {
+                        let n = parseInt(entry, 16);
+                        if (i === 2 && entry.length > 3) {
+                            // the last iteration is filenames, and if they are longer than 3 characters then the last 3 characters of the id, and the preceding characters are the highest value
+                            n = parseInt(entry.slice(-3), 16);
+                            n += parseInt(entry.slice(0, -3), 16) * 0x1000000000;
+                        }
+                        if (n > highest) {
+                            highest = n;
+                        }
+                    }
+                }
+                id += highest;
+                path = (0, path_1.join)(path, highest.toString(16));
+            }
+            highestId = Math.max(highestId, id);
+        }
+        idIncrementer = new BigInt64Array([BigInt(highestId) + 1n]);
+        // now get the selected incrementer buffer, this is the shared buffer was first registered and that all threads will use
+        idIncrementer = new BigInt64Array(currentStore.getUserSharedBuffer('blob-file-id', idIncrementer.buffer));
+        idIncrementers.set(currentStore, idIncrementer);
+    }
+    return Number(Atomics.add(idIncrementer, 0, 1n));
+}
+const FREQUENCY_TABLE_SIZE = 128;
+/**
+ * Select the next index from the storage paths, where the frequency of selecting each storage path is (mostly) proportional to the available space (which is occasionally updated)
+ * @param blobStoragePaths
+ */
+function getNextStorageIndex(blobStoragePaths, fileId) {
+    const now = Date.now();
+    if (!blobStoragePaths.frequencyTable) {
+        blobStoragePaths.lastUpdated = 0;
+        // setup default frequency table with even distribution
+        const frequencyTable = new Array(FREQUENCY_TABLE_SIZE);
+        for (let i = 0; i < frequencyTable.length; i++) {
+            frequencyTable[i] = i % blobStoragePaths.length;
+        }
+        blobStoragePaths.frequencyTable = frequencyTable;
+    }
+    if ((blobStoragePaths.lastUpdated ?? 0) + 60000 < now) {
+        blobStoragePaths.lastUpdated = now;
+        // create a new frequency table based on the available space
+        createFrequencyTableForStoragePaths(blobStoragePaths);
+    }
+    const nextIndex = blobStoragePaths.frequencyTable[fileId % FREQUENCY_TABLE_SIZE];
+    return nextIndex;
+}
+/**
+ * Create a frequency table for the storage paths, based on the available space, that allocates storage paths with more space more often
+ * and can be assigned quickly and consistently across threads (all threads will usually incrementally assign ids to the same alternating set of storage paths)
+ * @param blobStoragePaths
+ */
+async function createFrequencyTableForStoragePaths(blobStoragePaths) {
+    if (!promises_1.statfs)
+        return; // statfs is not available on all older node versions
+    const availableSpaces = await Promise.all(blobStoragePaths.map(async (path) => {
+        let stats;
+        try {
+            stats = await (0, promises_1.statfs)(path);
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT')
+                throw error;
+            // if the path doesn't exist, go ahead and create it
+            (0, fs_extra_1.ensureDirSync)(path);
+            // try again after the path is created
+            stats = await (0, promises_1.statfs)(path);
+        }
+        const availableSpace = stats.bavail * stats.bsize;
+        return Math.pow(availableSpace, 0.8); // we don't want this to be quite linear, so we use a power function to reduce the impact of large differences in available space
+    }));
+    const frequencyTable = new Array(FREQUENCY_TABLE_SIZE);
+    const pathPeriods = availableSpaces.map((space) => 1 / space);
+    for (let i = 0; i < FREQUENCY_TABLE_SIZE; i++) {
+        let nextScore = Infinity;
+        let nextIndex = 0;
+        // find the next storage path to use, based on the lowest remaining period for each path
+        for (let i = 0; i < pathPeriods.length; i++) {
+            if (pathPeriods[i] < nextScore) {
+                nextIndex = i;
+                nextScore = pathPeriods[i];
+            }
+        }
+        // increment the period that we used, inversely proportional to the available space
+        pathPeriods[nextIndex] += 1 / availableSpaces[nextIndex];
+        frequencyTable[i] = nextIndex;
+    }
+    blobStoragePaths.frequencyTable = frequencyTable;
+}
+/**
+ * Encode blobs with file paths, so that they can be saved to the database
+ * @param callback
+ * @param encodingId
+ * @param objectToClear
+ */
+function encodeBlobsWithFilePath(callback, encodingId, store) {
+    encodeForStorageForRecordId = encodingId;
+    currentStore = store;
+    exports.blobsWereEncoded = false;
+    try {
+        return callback();
+    }
+    finally {
+        encodeForStorageForRecordId = undefined;
+        currentStore = undefined;
+    }
+}
+/**
+ * Encode blobs as buffers, so they can be transferred remotely
+ * @param callback
+ * @param encodingId
+ * @param objectToClear
+ */
+function encodeBlobsAsBuffers(callback) {
+    promisedWrites = [];
+    let result;
+    try {
+        result = callback();
+    }
+    catch (error) {
+        // if anything throws, we want to make sure we clear the promise aggregator
+        promisedWrites = undefined;
+        throw error;
+    }
+    const finished = promisedWrites.length < 2 ? promisedWrites[0] : Promise.all(promisedWrites);
+    promisedWrites = undefined;
+    return finished ? finished.then(() => callback()) : result;
+}
+/**
+ * Decode blobs, creating local storage to hold the blogs and returning a promise that resolves when all the blobs are written to disk
+ * @param callback
+ */
+function decodeBlobsWithWrites(callback, store, blobCallback) {
+    try {
+        promisedWrites = [];
+        currentBlobCallback = blobCallback;
+        currentStore = store;
+        callback();
+    }
+    catch (error) {
+        // if anything throws, we want to make sure we clear the promise aggregator
+        currentBlobCallback = undefined;
+        promisedWrites = undefined;
+        throw error;
+    }
+    currentBlobCallback = undefined;
+    const finished = promisedWrites.length < 2 ? promisedWrites[0] : Promise.all(promisedWrites);
+    promisedWrites = undefined;
+    return finished;
+}
+/**
+ * Decode with a callback for when blobs are encountered, allowing for detecting of blobs
+ * @param callback
+ */
+function decodeWithBlobCallback(callback, blobCallback, rootStore) {
+    currentStore = rootStore;
+    try {
+        currentBlobCallback = blobCallback;
+        return callback();
+    }
+    finally {
+        currentBlobCallback = undefined;
+    }
+}
+/**
+ * Decode with a callback for when blobs are encountered, allowing for detecting of blobs
+ * @param callback
+ */
+function decodeFromDatabase(callback, rootStore) {
+    // note that this is actually called recursively (but always the same root store), so we don't clear afterwards
+    currentStore = rootStore;
+    return callback();
+}
+/**
+ * Delete blobs in an object, recursively searching for blobs
+ * @param object
+ */
+function deleteBlobsInObject(object) {
+    findBlobsInObject(object, (object) => {
+        deleteBlob(object);
+    });
+}
+/**
+ * Find all blobs in an object, recursively searching for Blob instances
+ * @param object
+ * @param callback
+ */
+function findBlobsInObject(object, callback) {
+    if (object instanceof exports.Blob) {
+        // eslint-disable-next-line
+        // @ts-ignore
+        callback(object);
+    }
+    else if (Array.isArray(object)) {
+        for (const value of object) {
+            if (typeof value === 'object' && value)
+                findBlobsInObject(value, callback);
+        }
+    }
+    else if (object && typeof object === 'object' && !object[Symbol.iterator]) {
+        // try to find plain objects, excluding things like buffers and typed arrays
+        for (const key in object) {
+            const value = object[key];
+            if (typeof value === 'object' && value)
+                findBlobsInObject(object[key], callback);
+        }
+    }
+}
+/**
+ * Do a shallow/fast search for blobs on the record and start saving them if they are supposed to be saved before a commit
+ * @param record
+ * @param store
+ */
+function startPreCommitBlobsForRecord(record, store, saveInRecord) {
+    let blobsNeedingSaving = [];
+    for (const key in record) {
+        const value = record[key];
+        if (value instanceof FileBackedBlob && (saveInRecord || value.saveBeforeCommit)) {
+            currentStore = store;
+            if (saveInRecord) {
+                value.saveInRecord = true;
+            }
+            blobsNeedingSaving.push(value);
+        }
+    }
+    if (blobsNeedingSaving.length > 0) {
+        // we do have blobs, start saving once the returned function is called
+        return () => {
+            currentStore = store;
+            return Promise.all(blobsNeedingSaving.map((blob) => {
+                return saveBlob(blob, true).saving ?? Promise.resolve();
+            }));
+        };
+    }
+}
+const copyingUnpacker = new msgpackr_1.Packr({ copyBuffers: true, mapsAsObjects: true });
+(0, msgpackr_1.addExtension)({
+    Class: exports.Blob,
+    type: 11,
+    unpack: function (buffer) {
+        const blobInfo = copyingUnpacker.unpack(buffer);
+        const blob = new FileBackedBlob();
+        Object.assign(blob, blobInfo[0]); // copy any properties
+        if (typeof blobInfo[1] !== 'object') {
+            // this is a reference, not followed by any buffer
+            storageInfoForBlob.set(blob, {
+                storageIndex: blobInfo[1],
+                fileId: blobInfo[2],
+                store: currentStore,
+            });
+            if (currentBlobCallback)
+                return currentBlobCallback(blob) ?? blob;
+            if (!currentStore) {
+                throw new Error('No store specified, cannot load blob from storage');
+            }
+        }
+        else {
+            storageInfoForBlob.set(blob, {
+                storageIndex: 0,
+                fileId: null,
+                storageBuffer: buffer,
+                contentBuffer: blobInfo[1],
+            });
+            blob.size = blobInfo[1]?.length;
+        }
+        return blob;
+    },
+    pack: function (blob) {
+        let storageInfo = storageInfoForBlob.get(blob);
+        if (encodeForStorageForRecordId !== undefined) {
+            exports.blobsWereEncoded = true;
+            if (storageInfo?.recordId !== undefined && storageInfo.recordId !== encodeForStorageForRecordId) {
+                throw new Error('Cannot use the same blob in two different records');
+            }
+        }
+        const options = { ...blob };
+        if (blob.type)
+            options.type = blob.type;
+        if (blob.size !== undefined)
+            options.size = blob.size;
+        if (storageInfo) {
+            if (storageInfo.storageBuffer) {
+                return storageInfo.storageBuffer;
+            }
+            if (storageInfo.contentBuffer &&
+                (storageInfo.contentBuffer?.length < FILE_STORAGE_THRESHOLD || blob.saveInRecord)) {
+                options.size = storageInfo.contentBuffer.length;
+                return (0, msgpackr_1.pack)([options, storageInfo.contentBuffer]);
+            }
+        }
+        if (encodeForStorageForRecordId !== undefined) {
+            storageInfo = saveBlob(blob);
+            if (!storageInfo.fileId) {
+                throw new Error('Unable to save blob without file id');
+            }
+            storageInfo.recordId = encodeForStorageForRecordId;
+            return (0, msgpackr_1.pack)([options, storageInfo.storageIndex, storageInfo.fileId]);
+        }
+        if (storageInfo) {
+            if (currentBlobCallback) {
+                currentBlobCallback(blob);
+                return (0, msgpackr_1.pack)([options, storageInfo.storageIndex, storageInfo.fileId]);
+            }
+            // if we want to encode as binary (necessary for replication), we need to encode as a buffer, not sure if we should always do that
+            // also, for replication, we would presume that this is most likely in OS cache, and sync will be fast. For other situations, a large sync call could be
+            // unpleasant
+            // we include the headers, as the receiving end will need them, and this differentiates from a reference
+            try {
+                const buffer = (0, node_fs_1.readFileSync)(getFilePath(storageInfo));
+                if (buffer.length >= HEADER_SIZE) {
+                    buffer.copy(HEADER, 0, 0, HEADER_SIZE);
+                    const size = Number(headerView.getBigUint64(0) & 0xffffffffffffn);
+                    if (size === buffer.length - HEADER_SIZE) {
+                        // the file is there and complete, we can return the encoding
+                        return Buffer.concat([(0, msgpackr_1.pack)([options]), buffer]);
+                    }
+                }
+                // declare it as being fulfilled
+                if (promisedWrites)
+                    promisedWrites.push(blob.bytes());
+                else {
+                    throw new Error('Incomplete blob');
+                }
+                return Buffer.alloc(0);
+            }
+            catch (error) {
+                if (error.code === 'ENOENT' && promisedWrites) {
+                    promisedWrites.push(blob.bytes());
+                    return Buffer.alloc(0);
+                }
+                else
+                    throw error;
+            }
+        }
+        else {
+            throw new Error('Blob has no storage info or buffer attached to it');
+        }
+    },
+});
+// with Blobs, it is easy to forget to await the creation, make sure that the blob is created before continuing
+(0, msgpackr_1.addExtension)({
+    Class: Promise,
+    type: 12, // not actually used, but we need to define a type
+    pack() {
+        throw new Error('Cannot encode a promise');
+    },
+});
+function polyfillBlob() {
+    // polyfill Blob for older Node, it has just enough to handle a single Buffer
+    return class Blob {
+        content;
+        constructor(contents) {
+            this.content = contents[0];
+        }
+        stream() {
+            return new ReadableStream({
+                start(controller) {
+                    controller.enqueue(this.content);
+                    controller.close();
+                },
+            });
+        }
+        text() {
+            return Promise.resolve(this.content.toString());
+        }
+        arrayBuffer() {
+            return Promise.resolve(this.content.buffer);
+        }
+        get size() {
+            return this.content.length;
+        }
+        slice() {
+            throw new Error('Not implemented');
+        }
+        bytes() {
+            return Promise.resolve(this.content);
+        }
+        get type() {
+            return '';
+        }
+    };
+}
+/**
+ * Scans for blobs on the file system and then checks to verify they are referenced
+ * from the database, and if not, deletes them
+ * @param database
+ */
+async function cleanupOrphans(database, databaseName) {
+    let store;
+    let auditStore;
+    let orphansDeleted = 0;
+    for (const tableName in database) {
+        const table = database[tableName];
+        store = table.primaryStore.rootStore;
+        auditStore = table.auditStore;
+        if (auditStore)
+            break;
+    }
+    const pathsToCheck = new Set();
+    const rootPaths = getRootBlobPathsForDB(store);
+    if (rootPaths) {
+        // search all the root paths
+        for (const rootPath of rootPaths) {
+            await searchPath(rootPath);
+        }
+    }
+    // remove all remaining paths are not referenced
+    await removePathsThatAreNotReferenced();
+    logger_ts_1.logger.warn?.(`Cleaned Orphan Blobs from ${databaseName ?? 'database'}, deleted ${orphansDeleted} blobs)`);
+    return orphansDeleted;
+    async function searchPath(path) {
+        try {
+            if (!(0, node_fs_1.existsSync)(path))
+                return;
+            for (const entry of await (0, promises_1.readdir)(path, { withFileTypes: true })) {
+                const entryPath = (0, path_1.join)(path, entry.name);
+                if (entry.isDirectory()) {
+                    // keep recursively searching
+                    await searchPath(entryPath);
+                }
+                else {
+                    if (pathsToCheck.size % 1_000_000 === 0)
+                        logger_ts_1.logger.info?.('Finding all blobs for orphan check, paths accumulated', pathsToCheck.size);
+                    pathsToCheck.add(entryPath);
+                    if (pathsToCheck.size % 2000 === 0) {
+                        // this might be a bit expensive, so only check occasionally
+                        const stats = (0, node_v8_1.getHeapStatistics)();
+                        // The maximum size of a Set is 16,777,216, so we limit the size of the set and we try to limit memory usage (starting at 80%, but gradually going down as the Set gets bigger)
+                        if (stats.used_heap_size > stats.heap_size_limit * (0.8 - pathsToCheck.size / 16_000_000)) {
+                            // if our array gets too big and we are running out of space, we can start the db search for references before running out of memory
+                            await removePathsThatAreNotReferenced();
+                        }
+                    }
+                }
+            }
+        }
+        catch (error) {
+            logger_ts_1.logger.error?.('Error searching path for blobs', path, error);
+        }
+    }
+    async function removePathsThatAreNotReferenced() {
+        let i = 0;
+        const perMS = Math.floor(((0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_BLOBCLEANUPSPEED) ?? 10000) / 1000 + 1);
+        // search all the tables for references
+        for (const tableName in database) {
+            logger_ts_1.logger.warn?.('Checking for references to potential orphaned blobs in table', tableName);
+            const table = database[tableName];
+            for (const entry of table.primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) {
+                try {
+                    if (entry.metadataFlags & auditStore_ts_1.HAS_BLOBS && entry.value) {
+                        checkObjectForReferences(entry.value);
+                    }
+                    // slow this down a bit to reduce excessive load, this runs approximately at 10k per second
+                    if (i++ % perMS === 0)
+                        await (0, promises_2.setTimeout)(1); // one millisecond delay to avoid overloading the system
+                    else
+                        await (0, promises_2.setImmediate)();
+                }
+                catch (error) {
+                    logger_ts_1.logger.error?.('Error searching table', tableName, ' for references to potential orphaned blobs failed', error);
+                }
+            }
+        }
+        logger_ts_1.logger.warn?.('Checking for references to potential orphaned blobs in the audit log');
+        // search the audit store for references
+        for (const auditRecord of auditStore.getRange({ start: 1, snapshot: false, lazy: true })) {
+            try {
+                const primaryStore = auditStore.tableStores[auditRecord.tableId];
+                if (!primaryStore)
+                    continue;
+                const entry = primaryStore?.getEntry(auditRecord.recordId);
+                if (!entry || entry.version !== auditRecord.version || !entry.value) {
+                    checkObjectForReferences(auditRecord.getValue(primaryStore));
+                }
+                // slow this down a bit to reduce excessive load, this runs approximately at 10k per second
+                if (i++ % perMS === 0)
+                    await (0, promises_2.setTimeout)(1); // one millisecond delay to avoid overloading the system
+                else
+                    await (0, promises_2.setImmediate)();
+            }
+            catch (error) {
+                logger_ts_1.logger.error?.('Error searching audit log for references to potential orphaned blobs failed', error);
+            }
+        }
+        logger_ts_1.logger.warn?.('Deleting', pathsToCheck.size, 'orphaned blobs');
+        orphansDeleted += pathsToCheck.size;
+        for (const path of pathsToCheck) {
+            try {
+                await (0, promises_1.unlink)(path);
+            }
+            catch (error) {
+                logger_ts_1.logger.debug?.('Error deleting file', error);
+            }
+        }
+        logger_ts_1.logger.warn?.('Finished deleting', pathsToCheck.size, 'orphaned blobs');
+        pathsToCheck.clear();
+    }
+    // check each object for any blob references and removes from the paths to check if found
+    function checkObjectForReferences(value) {
+        findBlobsInObject(value, (blob) => {
+            if (blob instanceof FileBackedBlob) {
+                const storageInfo = storageInfoForBlob.get(blob);
+                if (storageInfo.fileId != null) {
+                    const path = getFilePath(storageInfo);
+                    if (pathsToCheck.has(path)) {
+                        pathsToCheck.delete(path);
+                    }
+                }
+            }
+        });
+    }
+}
+//# sourceMappingURL=blob.js.map