@bopen-io/wallet-toolbox 1.7.18 → 1.7.20-idb-fix.1

package/out/src/services/chaintracker/chaintracks/util/BulkFileDataManager.js

@@ -0,0 +1,848 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BulkFileDataReader = exports.BulkFileDataManager = void 0;
+exports.selectBulkHeaderFiles = selectBulkHeaderFiles;
+const sdk_1 = require("../../../../sdk");
+const sdk_2 = require("@bsv/sdk");
+const utilityHelpers_noBuffer_1 = require("../../../../utility/utilityHelpers.noBuffer");
+const validBulkHeaderFilesByFileHash_1 = require("./validBulkHeaderFilesByFileHash");
+const HeightRange_1 = require("./HeightRange");
+const blockHeaderUtilities_1 = require("./blockHeaderUtilities");
+const ChaintracksFetch_1 = require("./ChaintracksFetch");
+const SingleWriterMultiReaderLock_1 = require("./SingleWriterMultiReaderLock");
+/**
+ * Manages bulk file data (typically 8MB chunks of 100,000 headers each).
+ *
+ * If not cached in memory,
+ * optionally fetches data by `sourceUrl` from CDN on demand,
+ * optionally finds data by `fileId` in a database on demand,
+ * and retains a limited number of files in memory,
+ * subject to the optional `maxRetained` limit.
+ */
+class BulkFileDataManager {
+    static createDefaultOptions(chain) {
+        return {
+            chain,
+            maxPerFile: 100000,
+            maxRetained: 2,
+            fetch: new ChaintracksFetch_1.ChaintracksFetch(),
+            fromKnownSourceUrl: 'https://cdn.projectbabbage.com/blockheaders'
+        };
+    }
+    constructor(options) {
+        this.log = () => { };
+        this.bfds = [];
+        this.fileHashToIndex = {};
+        this.lock = new SingleWriterMultiReaderLock_1.SingleWriterMultiReaderLock();
+        if (typeof options === 'object')
+            options = options;
+        else
+            options = BulkFileDataManager.createDefaultOptions(options);
+        this.chain = options.chain;
+        this.maxPerFile = options.maxPerFile;
+        this.maxRetained = options.maxRetained;
+        this.fromKnownSourceUrl = options.fromKnownSourceUrl;
+        this.fetch = options.fetch;
+        this.deleteBulkFilesNoLock();
+    }
+    async deleteBulkFiles() {
+        return this.lock.withWriteLock(async () => this.deleteBulkFilesNoLock());
+    }
+    deleteBulkFilesNoLock() {
+        this.bfds = [];
+        this.fileHashToIndex = {};
+        if (this.fromKnownSourceUrl) {
+            const vbhfs = validBulkHeaderFilesByFileHash_1.validBulkHeaderFiles;
+            const filtered = vbhfs.filter(f => f.sourceUrl === this.fromKnownSourceUrl);
+            const files = selectBulkHeaderFiles(filtered, this.chain, this.maxPerFile);
+            for (const file of files) {
+                this.add({ ...file, fileHash: file.fileHash, mru: Date.now() });
+            }
+        }
+    }
+    /**
+     * If `bfds` are going to be backed by persistent storage,
+     * must be called before making storage available.
+     *
+     * Synchronizes bfds and storage files, after which this manager maintains sync.
+     * There should be no changes to bulk files by direct access to storage bulk file methods.
+     */
+    async setStorage(storage, log) {
+        return this.lock.withWriteLock(async () => this.setStorageNoLock(storage, log));
+    }
+    async setStorageNoLock(storage, log) {
+        this.storage = storage;
+        this.log = log;
+        // Get files currently in persistent storage.
+        let sfs = await this.storage.getBulkFiles();
+        // Sync bfds with storage. Two scenarios supported:
+        const bfdsRanges = this.heightRangesFromBulkFiles(this.bfds);
+        const sfsRanges = this.heightRangesFromBulkFiles(sfs);
+        if (sfsRanges.cdn.length >= bfdsRanges.cdn.length) {
+            // Storage win if it has greater or equal CDN coverage
+            // Replace all bfds with sfs
+            this.bfds = [];
+            for (const file of sfs) {
+                const vbf = await this.validateFileInfo(file);
+                this.bfds.push(vbf);
+            }
+        }
+        else {
+            // Bfds win if they have greater CDN coverage
+            // Replace all sfs with bfds
+            for (const s of sfs.reverse())
+                await this.storage.deleteBulkFile(s.fileId);
+            for (const bfd of this.bfds) {
+                await this.ensureData(bfd);
+                bfd.fileId = await this.storage.insertBulkFile(bfdToInfo(bfd, true));
+            }
+        }
+    }
+    heightRangesFromBulkFiles(files) {
+        const ranges = { all: new HeightRange_1.HeightRange(0, -1), cdn: new HeightRange_1.HeightRange(0, -1), incremental: new HeightRange_1.HeightRange(0, -1) };
+        for (const file of files) {
+            const range = new HeightRange_1.HeightRange(file.firstHeight, file.firstHeight + file.count - 1);
+            ranges.all = ranges.all.union(range);
+            if (isBdfCdn(file))
+                ranges.cdn = ranges.cdn.union(range);
+            if (isBdfIncremental(file))
+                ranges.incremental = ranges.incremental.union(range);
+        }
+        return ranges;
+    }
+    async createReader(range, maxBufferSize) {
+        range = range || (await this.getHeightRange());
+        maxBufferSize = maxBufferSize || 1000000 * 80; // 100,000 headers, 8MB
+        return new BulkFileDataReader(this, range, maxBufferSize);
+    }
+    async updateFromUrl(cdnUrl) {
+        if (!this.fetch)
+            throw new sdk_1.WERR_INVALID_OPERATION('fetch is not defined in the BulkFileDataManager.');
+        const toUrl = (file) => this.fetch.pathJoin(cdnUrl, file);
+        const url = toUrl(`${this.chain}NetBlockHeaders.json`);
+        const availableBulkFiles = (await this.fetch.fetchJson(url));
+        if (!availableBulkFiles)
+            throw new sdk_1.WERR_INVALID_PARAMETER(`cdnUrl`, `a valid BulkHeaderFilesInfo JSON resource available from ${url}`);
+        const selectedFiles = selectBulkHeaderFiles(availableBulkFiles.files, this.chain, this.maxPerFile || availableBulkFiles.headersPerFile);
+        for (const bf of selectedFiles) {
+            if (!bf.fileHash) {
+                throw new sdk_1.WERR_INVALID_PARAMETER(`fileHash`, `valid for all files in json downloaded from ${url}`);
+            }
+            if (!bf.chain || bf.chain !== this.chain) {
+                throw new sdk_1.WERR_INVALID_PARAMETER(`chain`, `"${this.chain}" for all files in json downloaded from ${url}`);
+            }
+            if (!bf.sourceUrl || bf.sourceUrl !== cdnUrl)
+                bf.sourceUrl = cdnUrl;
+        }
+        const rangeBefore = await this.getHeightRange();
+        const r = await this.merge(selectedFiles);
+        const rangeAfter = await this.getHeightRange();
+        let log = 'BulkDataFileManager.updateFromUrl\n';
+        log += `  url: ${url}\n`;
+        log += `  bulk range before: ${rangeBefore}\n`;
+        log += `  bulk range after:  ${rangeAfter}\n`;
+        this.log(log);
+    }
+    async merge(files) {
+        return this.lock.withWriteLock(async () => this.mergeNoLock(files));
+    }
+    async mergeNoLock(files) {
+        const r = { inserted: [], updated: [], unchanged: [], dropped: [] };
+        for (const file of files) {
+            const hbf = this.getBfdForHeight(file.firstHeight);
+            if (hbf && file.fileId)
+                hbf.fileId = file.fileId; // Always update fileId if provided
+            const lbf = this.getLastBfd();
+            if (hbf &&
+                hbf.fileHash === file.fileHash &&
+                hbf.count === file.count &&
+                hbf.lastHash === file.lastHash &&
+                hbf.lastChainWork === file.lastChainWork) {
+                // We already have an identical matching file...
+                r.unchanged.push(bfdToInfo(hbf));
+                continue;
+            }
+            const vbf = await this.validateFileInfo(file);
+            if (hbf) {
+                // We have a matching file by firstHeight but count and fileHash differ
+                await this.update(vbf, hbf, r);
+            }
+            else if (isBdfIncremental(vbf) && lbf && isBdfIncremental(lbf)) {
+                await this.mergeIncremental(lbf, vbf, r);
+            }
+            else {
+                const added = await this.add(vbf);
+                r.inserted.push(added);
+            }
+        }
+        this.log(`BulkFileDataManager.merge:\n${this.toLogString(r)}\n`);
+        return r;
+    }
+    async mergeIncremental(lbf, vbf, r) {
+        lbf.count += vbf.count;
+        lbf.lastHash = vbf.lastHash;
+        lbf.lastChainWork = vbf.lastChainWork;
+        await this.ensureData(lbf);
+        const newData = new Uint8Array(lbf.data.length + vbf.data.length);
+        newData.set(lbf.data);
+        newData.set(vbf.data, lbf.data.length);
+        lbf.data = newData;
+        delete this.fileHashToIndex[lbf.fileHash];
+        lbf.fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(newData)), 'base64');
+        this.fileHashToIndex[lbf.fileHash] = this.bfds.length - 1;
+        lbf.mru = Date.now();
+        const lbfInfo = bfdToInfo(lbf, true);
+        r.updated.push(lbfInfo);
+        if (this.storage && lbf.fileId) {
+            await this.storage.updateBulkFile(lbf.fileId, lbfInfo);
+        }
+    }
+    toLogString(what) {
+        let log = '';
+        if (!what) {
+            log += this.toLogString(this.bfds);
+        }
+        else if (what['updated']) {
+            what = what;
+            for (const { category, bfds } of [
+                { category: 'unchanged', bfds: what.unchanged },
+                { category: 'dropped', bfds: what.dropped },
+                { category: 'updated', bfds: what.updated },
+                { category: 'inserted', bfds: what.inserted }
+            ]) {
+                if (bfds.length > 0) {
+                    log += `  ${category}:\n`;
+                    log += this.toLogString(bfds);
+                }
+            }
+        }
+        else if (Array.isArray(what)) {
+            what = what;
+            let i = -1;
+            for (const bfd of what) {
+                i++;
+                log += `  ${i}: ${bfd.fileName} fileId=${bfd.fileId} ${bfd.firstHeight}-${bfd.firstHeight + bfd.count - 1}\n`;
+            }
+        }
+        return log;
+    }
+    async mergeIncrementalBlockHeaders(newBulkHeaders, incrementalChainWork) {
+        if (newBulkHeaders.length === 0)
+            return;
+        return this.lock.withWriteLock(async () => {
+            const lbf = this.getLastFileNoLock();
+            const nextHeight = lbf ? lbf.firstHeight + lbf.count : 0;
+            if (nextHeight > 0 && newBulkHeaders.length > 0 && newBulkHeaders[0].height < nextHeight) {
+                // Don't modify the incoming array...
+                newBulkHeaders = [...newBulkHeaders];
+                // If we have more headers than we need, drop the incoming headers.
+                while (newBulkHeaders.length > 0 && newBulkHeaders[0].height < nextHeight) {
+                    const h = newBulkHeaders.shift();
+                    if (h && incrementalChainWork) {
+                        incrementalChainWork = (0, blockHeaderUtilities_1.subWork)(incrementalChainWork, (0, blockHeaderUtilities_1.convertBitsToWork)(h.bits));
+                    }
+                }
+            }
+            if (newBulkHeaders.length === 0)
+                return;
+            if (!lbf || nextHeight !== newBulkHeaders[0].height)
+                throw new sdk_1.WERR_INVALID_PARAMETER('newBulkHeaders', 'an extension of existing bulk headers');
+            if (!lbf.lastHash)
+                throw new sdk_1.WERR_INTERNAL(`lastHash is not defined for the last bulk file ${lbf.fileName}`);
+            const fbh = newBulkHeaders[0];
+            const lbh = newBulkHeaders.slice(-1)[0];
+            let lastChainWork = lbf.lastChainWork;
+            if (incrementalChainWork) {
+                lastChainWork = (0, blockHeaderUtilities_1.addWork)(incrementalChainWork, lastChainWork);
+            }
+            else {
+                // If lastChainWork is not provided, calculate it from the last file with basic validation.
+                let lastHeight = lbf.firstHeight + lbf.count - 1;
+                let lastHash = lbf.lastHash;
+                for (const h of newBulkHeaders) {
+                    if (h.height !== lastHeight + 1 || h.previousHash !== lastHash) {
+                        throw new sdk_1.WERR_INVALID_PARAMETER('headers', `an extension of existing bulk headers, header with height ${h.height} is non-sequential`);
+                    }
+                    lastChainWork = (0, blockHeaderUtilities_1.addWork)(lastChainWork, (0, blockHeaderUtilities_1.convertBitsToWork)(h.bits));
+                    lastHeight = h.height;
+                    lastHash = h.hash;
+                }
+            }
+            const data = (0, blockHeaderUtilities_1.serializeBaseBlockHeaders)(newBulkHeaders);
+            const fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(data)), 'base64');
+            const bf = {
+                fileId: undefined,
+                chain: this.chain,
+                sourceUrl: undefined,
+                fileName: 'incremental',
+                firstHeight: fbh.height,
+                count: newBulkHeaders.length,
+                prevChainWork: lbf.lastChainWork,
+                lastChainWork,
+                prevHash: lbf.lastHash,
+                lastHash: lbh.hash,
+                fileHash,
+                data
+            };
+            await this.mergeNoLock([bf]);
+        });
+    }
+    async getBulkFiles(keepData) {
+        return this.lock.withReadLock(async () => {
+            return this.bfds.map(bfd => bfdToInfo(bfd, keepData));
+        });
+    }
+    async getHeightRange() {
+        return this.lock.withReadLock(async () => {
+            if (this.bfds.length === 0)
+                return HeightRange_1.HeightRange.empty;
+            const first = this.bfds[0];
+            const last = this.bfds[this.bfds.length - 1];
+            return new HeightRange_1.HeightRange(first.firstHeight, last.firstHeight + last.count - 1);
+        });
+    }
+    async getDataFromFile(file, offset, length) {
+        const bfd = await this.getBfdForHeight(file.firstHeight);
+        if (!bfd || bfd.count < file.count)
+            throw new sdk_1.WERR_INVALID_PARAMETER('file', `a match for ${file.firstHeight}, ${file.count} in the BulkFileDataManager.`);
+        return this.lock.withReadLock(async () => this.getDataFromFileNoLock(bfd, offset, length));
+    }
+    async getDataFromFileNoLock(bfd, offset, length) {
+        const fileLength = bfd.count * 80;
+        offset = offset || 0;
+        if (offset > fileLength - 1)
+            return undefined;
+        length = length || bfd.count * 80 - offset;
+        length = Math.min(length, fileLength - offset);
+        let data;
+        if (bfd.data) {
+            data = bfd.data.slice(offset, offset + length);
+        }
+        else if (bfd.fileId && this.storage) {
+            data = await this.storage.getBulkFileData(bfd.fileId, offset, length);
+        }
+        if (!data) {
+            await this.ensureData(bfd);
+            if (bfd.data)
+                data = bfd.data.slice(offset, offset + length);
+        }
+        if (!data)
+            return undefined;
+        return data;
+    }
+    async findHeaderForHeightOrUndefined(height) {
+        return this.lock.withReadLock(async () => {
+            if (!Number.isInteger(height) || height < 0)
+                throw new sdk_1.WERR_INVALID_PARAMETER('height', `a non-negative integer (${height}).`);
+            const file = this.bfds.find(f => f.firstHeight <= height && f.firstHeight + f.count > height);
+            if (!file)
+                return undefined;
+            const offset = (height - file.firstHeight) * 80;
+            const data = await this.getDataFromFileNoLock(file, offset, 80);
+            if (!data)
+                return undefined;
+            const header = (0, blockHeaderUtilities_1.deserializeBlockHeader)(data, 0, height);
+            return header;
+        });
+    }
+    async getFileForHeight(height) {
+        return this.lock.withReadLock(async () => {
+            const bfd = this.getBfdForHeight(height);
+            if (!bfd)
+                return undefined;
+            return bfdToInfo(bfd);
+        });
+    }
+    getBfdForHeight(height) {
+        if (!Number.isInteger(height) || height < 0)
+            throw new sdk_1.WERR_INVALID_PARAMETER('height', `a non-negative integer (${height}).`);
+        const file = this.bfds.find(f => f.firstHeight <= height && f.firstHeight + f.count > height);
+        return file;
+    }
+    getLastBfd(fromEnd = 1) {
+        if (this.bfds.length < fromEnd)
+            return undefined;
+        const bfd = this.bfds[this.bfds.length - fromEnd];
+        return bfd;
+    }
+    async getLastFile(fromEnd = 1) {
+        return this.lock.withReadLock(async () => this.getLastFileNoLock(fromEnd));
+    }
+    getLastFileNoLock(fromEnd = 1) {
+        const bfd = this.getLastBfd(fromEnd);
+        if (!bfd)
+            return undefined;
+        return bfdToInfo(bfd);
+    }
+    async getDataByFileHash(fileHash) {
+        const index = this.fileHashToIndex[fileHash];
+        if (index === undefined)
+            throw new sdk_1.WERR_INVALID_PARAMETER('fileHash', `known to the BulkFileDataManager. ${fileHash} is unknown.`);
+        const bfd = this.bfds[index];
+        const data = await this.ensureData(bfd);
+        return data;
+    }
+    async getDataByFileId(fileId) {
+        const bfd = this.bfds.find(f => f.fileId === fileId);
+        if (bfd === undefined)
+            throw new sdk_1.WERR_INVALID_PARAMETER('fileId', `known to the BulkFileDataManager. ${fileId} is unknown.`);
+        const data = await this.ensureData(bfd);
+        return data;
+    }
+    async validateFileInfo(file) {
+        var _a;
+        if (file.chain !== this.chain)
+            throw new sdk_1.WERR_INVALID_PARAMETER('chain', `${this.chain}`);
+        if (file.count <= 0)
+            throw new sdk_1.WERR_INVALID_PARAMETER('bf.count', `expected count to be greater than 0, but got ${file.count}`);
+        if (file.count > this.maxPerFile && file.fileName !== 'incremental')
+            throw new sdk_1.WERR_INVALID_PARAMETER('count', `less than or equal to maxPerFile ${this.maxPerFile}`);
+        if (!file.fileHash)
+            throw new sdk_1.WERR_INVALID_PARAMETER('fileHash', `defined`);
+        if (!file.sourceUrl && !file.fileId && !file.data)
+            throw new sdk_1.WERR_INVALID_PARAMETER('data', `defined when sourceUrl and fileId are undefined`);
+        let bfd = {
+            ...file,
+            fileHash: file.fileHash,
+            mru: Date.now()
+        };
+        if (!bfd.validated) {
+            await this.ensureData(bfd);
+            if (!bfd.data || bfd.data.length !== bfd.count * 80)
+                throw new sdk_1.WERR_INVALID_PARAMETER('file.data', `bulk file ${bfd.fileName} data length ${(_a = bfd.data) === null || _a === void 0 ? void 0 : _a.length} does not match expected count ${bfd.count}`);
+            bfd.fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(bfd.data)), 'base64');
+            if (file.fileHash && file.fileHash !== bfd.fileHash)
+                throw new sdk_1.WERR_INVALID_PARAMETER('file.fileHash', `expected ${file.fileHash} but got ${bfd.fileHash}`);
+            if (!(0, validBulkHeaderFilesByFileHash_1.isKnownValidBulkHeaderFile)(bfd)) {
+                const pbf = bfd.firstHeight > 0 ? this.getBfdForHeight(bfd.firstHeight - 1) : undefined;
+                const prevHash = pbf ? pbf.lastHash : '00'.repeat(32);
+                const prevChainWork = pbf ? pbf.lastChainWork : '00'.repeat(32);
+                const { lastHeaderHash, lastChainWork } = (0, blockHeaderUtilities_1.validateBufferOfHeaders)(bfd.data, prevHash, 0, undefined, prevChainWork);
+                if (bfd.lastHash && bfd.lastHash !== lastHeaderHash)
+                    throw new sdk_1.WERR_INVALID_PARAMETER('file.lastHash', `expected ${bfd.lastHash} but got ${lastHeaderHash}`);
+                if (bfd.lastChainWork && bfd.lastChainWork !== lastChainWork)
+                    throw new sdk_1.WERR_INVALID_PARAMETER('file.lastChainWork', `expected ${bfd.lastChainWork} but got ${lastChainWork}`);
+                bfd.lastHash = lastHeaderHash;
+                bfd.lastChainWork = lastChainWork;
+                if (bfd.firstHeight === 0) {
+                    (0, blockHeaderUtilities_1.validateGenesisHeader)(bfd.data, bfd.chain);
+                }
+            }
+            bfd.validated = true;
+        }
+        return bfd;
+    }
+    async ReValidate() {
+        return this.lock.withReadLock(async () => this.ReValidateNoLock());
+    }
+    async ReValidateNoLock() {
+        for (const file of this.bfds) {
+            await this.ensureData(file);
+            file.validated = false; // Reset validation to re-validate on next access
+            const bfd = await this.validateFileInfo(file);
+            if (!bfd.validated)
+                throw new sdk_1.WERR_INTERNAL(`BulkFileDataManager.ReValidate failed for file ${bfd.fileName}`);
+            file.validated = true;
+        }
+    }
+    validateBfdForAdd(bfd) {
+        if (this.bfds.length === 0 && bfd.firstHeight !== 0)
+            throw new sdk_1.WERR_INVALID_PARAMETER('firstHeight', `0 for the first file`);
+        if (this.bfds.length > 0) {
+            const last = this.bfds[this.bfds.length - 1];
+            if (bfd.firstHeight !== last.firstHeight + last.count)
+                throw new sdk_1.WERR_INVALID_PARAMETER('firstHeight', `the last file's firstHeight + count`);
+            if (bfd.prevHash !== last.lastHash || bfd.prevChainWork !== last.lastChainWork)
+                throw new sdk_1.WERR_INVALID_PARAMETER('prevHash/prevChainWork', `the last file's lastHash/lastChainWork`);
+        }
+    }
+    async add(bfd) {
+        this.validateBfdForAdd(bfd);
+        const index = this.bfds.length;
+        this.bfds.push(bfd);
+        this.fileHashToIndex[bfd.fileHash] = index;
+        this.ensureMaxRetained();
+        const info = bfdToInfo(bfd, true);
+        if (this.storage) {
+            info.fileId = bfd.fileId = await this.storage.insertBulkFile(info);
+        }
+        return info;
+    }
+    replaceBfdAtIndex(index, update) {
+        const oldBfd = this.bfds[index];
+        delete this.fileHashToIndex[oldBfd.fileHash];
+        this.bfds[index] = update;
+        this.fileHashToIndex[update.fileHash] = index;
+    }
+    /**
+     * Updating an existing file occurs in two specific contexts:
+     *
+     * 1. CDN Update: CDN files of a specific `maxPerFile` series typically ends in a partial file
+     * which may periodically add more headers until the next file is started.
+     * If the CDN update is the second to last file (followed by an incremental file),
+     * then the incremental file is updated or deleted and also returned as the result (with a count of zero if deleted).
+     *
+     * 2. Incremental Update: The last bulk file is almost always an "incremental" file
+     * which is not limited by "maxPerFile" and holds all non-CDN bulk headers.
+     * If is updated with new bulk headers which come either from non CDN ingestors or from live header migration to bulk.
+     *
+     * Updating preserves the following properties:
+     *
+     * - Any existing headers following this update are preserved and must form an unbroken chain.
+     * - There can be at most one incremental file and it must be the last file.
+     * - The update start conditions (height, prevHash, prevChainWork) must match an existing file which may be either CDN or internal.
+     * - The update fileId must match, it may be undefind.
+     * - The fileName does not need to match.
+     * - The incremental file must always have fileName "incremental" and sourceUrl must be undefined.
+     * - The update count must be greater than 0.
+     * - The update count must be greater than current count for CDN to CDN update.
+     *
+     * @param update new validated BulkFileData to update.
+     * @param hbf corresponding existing BulkFileData to update.
+     */
+    async update(update, hbf, r) {
+        if (!hbf ||
+            hbf.firstHeight !== update.firstHeight ||
+            hbf.prevChainWork !== update.prevChainWork ||
+            hbf.prevHash !== update.prevHash)
+            throw new sdk_1.WERR_INVALID_PARAMETER('file', `an existing file by height, prevChainWork and prevHash`);
+        if (isBdfCdn(update) === isBdfCdn(hbf) && update.count <= hbf.count)
+            throw new sdk_1.WERR_INVALID_PARAMETER('file.count', `greater than the current count ${hbf.count}`);
+        const lbf = this.getLastBfd();
+        let index = this.bfds.length - 1;
+        let truncate = undefined;
+        let replaced = undefined;
+        let drop = undefined;
+        if (hbf.firstHeight === lbf.firstHeight) {
+            // If the update is for the last file, there are three cases:
+            if (isBdfIncremental(update)) {
+                // 1. Incremental file may only be extended with more incremental headers.
+                if (!isBdfIncremental(lbf))
+                    throw new sdk_1.WERR_INVALID_PARAMETER('file', `an incremental file to update an existing incremental file`);
+            }
+            else {
+                // The update is a CDN bulk file.
+                if (isBdfCdn(lbf)) {
+                    // 2. An updated CDN file replaces a partial CDN file.
+                    if (update.count <= lbf.count)
+                        throw new sdk_1.WERR_INVALID_PARAMETER('update.count', `CDN update must have more headers. ${update.count} <= ${lbf.count}`);
+                }
+                else {
+                    // 3. A new CDN file replaces some or all of current incremental file.
+                    // Retain extra incremental headers if any.
+                    if (update.count < lbf.count) {
+                        // The new CDN partially replaces the last incremental file, prepare to shift work and re-add it.
+                        await this.ensureData(lbf);
+                        truncate = lbf;
+                    }
+                }
+            }
+        }
+        else {
+            // If the update is NOT for the last file, then it MUST be for the second to last file which MUST be a CDN file:
+            // - it must be a CDN file update with more headers than the current CDN file.
+            // - the last file must be an incremental file which is updated or deleted. The updated (or deleted) last file is returned.
+            const lbf2 = this.getLastBfd(2);
+            if (!lbf2 || hbf.firstHeight !== lbf2.firstHeight)
+                throw new sdk_1.WERR_INVALID_PARAMETER('file', `an update to last or second to last file`);
+            if (!isBdfCdn(update) || !isBdfCdn(lbf2) || update.count <= lbf2.count)
+                throw new sdk_1.WERR_INVALID_PARAMETER('file', `a CDN file update with more headers than the current CDN file`);
+            if (!isBdfIncremental(lbf))
+                throw new sdk_1.WERR_INVALID_PARAMETER('file', `a CDN file update followed by an incremental file`);
+            if (!update.fileId)
+                update.fileId = lbf2.fileId; // Update fileId if not provided
+            if (update.count >= lbf2.count + lbf.count) {
+                // The current last file is fully replaced by the CDN update.
+                drop = lbf;
+            }
+            else {
+                // If the update doesn't fully replace the last incremental file, make sure data is available to be truncated.
+                await this.ensureData(lbf);
+                truncate = lbf;
+                // The existing second to last file is fully replaced by the update.
+                replaced = lbf2;
+            }
+            index = index - 1; // The update replaces the second to last file.
+        }
+        // In all cases the bulk file at the current fileId if any is updated.
+        this.replaceBfdAtIndex(index, update);
+        if (truncate) {
+            // If there is a bulk file to be truncated, it becomes the new (reduced) last file.
+            await this.shiftWork(update, truncate, replaced);
+        }
+        if (drop) {
+            this.dropLastBulkFile(drop);
+        }
+        const updateInfo = bfdToInfo(update, true);
+        const truncateInfo = truncate ? bfdToInfo(truncate, true) : undefined;
+        if (this.storage) {
+            // Keep storage in sync.
+            if (update.fileId) {
+                await this.storage.updateBulkFile(update.fileId, updateInfo);
+            }
+            if (truncate && truncateInfo) {
+                if (replaced) {
+                    await this.storage.updateBulkFile(truncate.fileId, truncateInfo);
+                }
+                else {
+                    truncateInfo.fileId = undefined; // Make sure truncate is a new file.
+                    truncate.fileId = await this.storage.insertBulkFile(truncateInfo);
+                }
+            }
+            if (drop && drop.fileId) {
+                await this.storage.deleteBulkFile(drop.fileId);
+            }
+        }
+        if (r) {
+            // Update results for logging...
+            r.updated.push(updateInfo);
+            if (truncateInfo) {
+                if (replaced) {
+                    r.updated.push(truncateInfo);
+                }
+                else {
+                    r.inserted.push(truncateInfo);
+                }
+            }
+            if (drop) {
+                r.dropped.push(bfdToInfo(drop));
+            }
+        }
+        this.ensureMaxRetained();
+    }
+    dropLastBulkFile(lbf) {
+        delete this.fileHashToIndex[lbf.fileHash];
+        const index = this.bfds.indexOf(lbf);
+        if (index !== this.bfds.length - 1)
+            throw new sdk_1.WERR_INTERNAL(`dropLastBulkFile requires lbf is the current last file.`);
+        this.bfds.pop();
+    }
+    /**
+     * Remove work (and headers) from `truncate` that now exists in `update`.
+     * There are two scenarios:
+     * 1. `replaced` is undefined: update is a CDN file that splits an incremental file that must be truncated.
+     * 2. `replaced` is valid: update is a CDN update that replaced an existing CDN file and splits an incremental file that must be truncated.
+     * @param update the new CDN update file.
+     * @param truncate the incremental file to be truncated (losing work which now exists in `update`).
+     * @param replaced the existing CDN file that was replaced by `update` (if any).
+     */
+    async shiftWork(update, truncate, replaced) {
+        var _a;
+        const updateIndex = this.fileHashToIndex[update.fileHash];
+        // replaced will be valid if the update replaced it and it must become the new last file.
+        // truncateIndex will be updateIndex + 1 if the existing last file is being truncated and update is second to last.
+        const truncateIndex = this.fileHashToIndex[truncate.fileHash];
+        if (truncateIndex !== undefined && truncateIndex !== updateIndex + 1)
+            throw new sdk_1.WERR_INTERNAL(`shiftWork requires update to have replaced truncate or truncate to follow update`);
+        if (truncateIndex !== undefined && !replaced)
+            throw new sdk_1.WERR_INTERNAL(`shiftWork requires valid replaced when update hasn't replaced truncate`);
+        truncate.prevHash = update.lastHash;
+        truncate.prevChainWork = update.lastChainWork;
+        // truncate.lastChainWork, truncate.lastHash remain unchanged
+        let count = update.count;
+        if (replaced) {
+            count -= replaced.count;
+        }
+        else {
+            // The truncated file is itself being replaced by the update and must be inserted as a new file.
+            truncate.fileId = undefined;
+            this.bfds.push(truncate); // Add the truncated file as a new entry.
+        }
+        truncate.count -= count;
+        truncate.firstHeight += count;
+        truncate.data = (_a = truncate.data) === null || _a === void 0 ? void 0 : _a.slice(count * 80);
+        delete this.fileHashToIndex[truncate.fileHash];
+        truncate.fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(truncate.data)), 'base64');
+        this.fileHashToIndex[truncate.fileHash] = updateIndex + 1;
+    }
+    /**
+     *
+     * @param bfd
+     * @returns
+     */
+    async ensureData(bfd) {
+        if (bfd.data)
+            return bfd.data;
+        if (this.storage && bfd.fileId) {
+            bfd.data = await this.storage.getBulkFileData(bfd.fileId);
+            if (!bfd.data)
+                throw new sdk_1.WERR_INVALID_PARAMETER('fileId', `valid, data not found for fileId ${bfd.fileId}`);
+        }
+        if (!bfd.data && this.fetch && bfd.sourceUrl) {
+            // TODO - restore this change
+            const url = this.fetch.pathJoin(bfd.sourceUrl, bfd.fileName);
+            //const url = this.fetch.pathJoin('http://localhost:8842/blockheaders', bfd.fileName)
+            try {
+                bfd.data = await this.fetch.download(url);
+            }
+            catch (err) {
+                bfd.data = await this.fetch.download(url);
+            }
+            if (!bfd.data)
+                throw new sdk_1.WERR_INVALID_PARAMETER('sourceUrl', `data not found for sourceUrl ${url}`);
+        }
+        if (!bfd.data)
+            throw new sdk_1.WERR_INVALID_PARAMETER('data', `defined. Unable to retrieve data for ${bfd.fileName}`);
+        bfd.mru = Date.now();
+        // Validate retrieved data.
+        const fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(bfd.data)), 'base64');
+        if (fileHash !== bfd.fileHash)
+            throw new sdk_1.WERR_INVALID_PARAMETER('fileHash', `a match for retrieved data for ${bfd.fileName}`);
+        this.ensureMaxRetained();
+        return bfd.data;
+    }
+    ensureMaxRetained() {
+        if (this.maxRetained === undefined)
+            return;
+        let withData = this.bfds.filter(bfd => bfd.data && (bfd.fileId || bfd.sourceUrl));
+        let countToRelease = withData.length - this.maxRetained;
+        if (countToRelease <= 0)
+            return;
+        const sorted = withData.sort((a, b) => a.mru - b.mru);
+        while (countToRelease-- > 0 && sorted.length > 0) {
+            const oldest = sorted.shift();
+            // Release the least recently used data
+            oldest.data = undefined; // Release the data
+        }
+    }
+    async exportHeadersToFs(toFs, toHeadersPerFile, toFolder, sourceUrl, maxHeight) {
+        const chain = this.chain;
+        const toFileName = (i) => `${chain}Net_${i}.headers`;
+        const toPath = (i) => toFs.pathJoin(toFolder, toFileName(i));
+        const toJsonPath = () => toFs.pathJoin(toFolder, `${chain}NetBlockHeaders.json`);
+        const toBulkFiles = {
+            rootFolder: sourceUrl || toFolder,
+            jsonFilename: `${chain}NetBlockHeaders.json`,
+            headersPerFile: toHeadersPerFile,
+            files: []
+        };
+        let range = await this.getHeightRange();
+        if (maxHeight)
+            range = range.intersect(new HeightRange_1.HeightRange(0, maxHeight));
+        const reader = await this.createReader(range, toHeadersPerFile * 80);
+        let firstHeight = 0;
+        let lastHeaderHash = '00'.repeat(32);
+        let lastChainWork = '00'.repeat(32);
+        let i = -1;
+        for (;;) {
+            i++;
+            const data = await reader.read();
+            if (!data || data.length === 0) {
+                break;
+            }
+            const last = (0, blockHeaderUtilities_1.validateBufferOfHeaders)(data, lastHeaderHash, 0, undefined, lastChainWork);
+            await toFs.writeFile(toPath(i), data);
+            const fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(data)), 'base64');
+            const file = {
+                chain,
+                count: data.length / 80,
+                fileHash,
+                fileName: toFileName(i),
+                firstHeight,
+                lastChainWork: last.lastChainWork,
+                lastHash: last.lastHeaderHash,
+                prevChainWork: lastChainWork,
+                prevHash: lastHeaderHash,
+                sourceUrl
+            };
+            toBulkFiles.files.push(file);
+            firstHeight += file.count;
+            lastHeaderHash = file.lastHash;
+            lastChainWork = file.lastChainWork;
+        }
+        await toFs.writeFile(toJsonPath(), (0, utilityHelpers_noBuffer_1.asUint8Array)(JSON.stringify(toBulkFiles), 'utf8'));
+    }
+}
+exports.BulkFileDataManager = BulkFileDataManager;
+function selectBulkHeaderFiles(files, chain, maxPerFile) {
+    const r = [];
+    let height = 0;
+    for (;;) {
+        const choices = files.filter(f => f.firstHeight === height && f.count <= maxPerFile && f.chain === chain);
+        // Pick the file with the maximum count
+        const choice = choices.reduce((a, b) => (a.count > b.count ? a : b), choices[0]);
+        if (!choice)
+            break; // no more files to select
+        r.push(choice);
+        height += choice.count;
+    }
+    return r;
+}
+function isBdfIncremental(bfd) {
+    return bfd.fileName === 'incremental' && !bfd.sourceUrl;
+}
+function isBdfCdn(bfd) {
+    return !isBdfIncremental(bfd);
+}
+function bfdToInfo(bfd, keepData) {
+    return {
+        chain: bfd.chain,
+        fileHash: bfd.fileHash,
+        fileName: bfd.fileName,
+        sourceUrl: bfd.sourceUrl,
+        fileId: bfd.fileId,
+        count: bfd.count,
+        prevChainWork: bfd.prevChainWork,
+        lastChainWork: bfd.lastChainWork,
+        firstHeight: bfd.firstHeight,
+        prevHash: bfd.prevHash,
+        lastHash: bfd.lastHash,
+        validated: bfd.validated || false,
+        data: keepData ? bfd.data : undefined
+    };
+}
+class BulkFileDataReader {
+    constructor(manager, range, maxBufferSize) {
+        this.manager = manager;
+        this.range = range;
+        this.maxBufferSize = maxBufferSize;
+        this.nextHeight = range.minHeight;
+    }
+    /**
+     * Returns the Buffer of block headers from the given `file` for the given `range`.
+     * If `range` is undefined, the file's full height range is read.
+     * The returned Buffer will only contain headers in `file` and in `range`
+     * @param file
+     * @param range
+     */
+    async readBufferFromFile(file, range) {
+        // Constrain the range to the file's contents...
+        let fileRange = new HeightRange_1.HeightRange(file.firstHeight, file.firstHeight + file.count - 1);
+        if (range)
+            fileRange = fileRange.intersect(range);
+        if (fileRange.isEmpty)
+            return undefined;
+        const offset = (fileRange.minHeight - file.firstHeight) * 80;
+        const length = fileRange.length * 80;
+        return await this.manager.getDataFromFile(file, offset, length);
+    }
+    /**
+     * @returns an array containing the next `maxBufferSize` bytes of headers from the files.
+     */
+    async read() {
+        if (this.nextHeight === undefined || !this.range || this.range.isEmpty || this.nextHeight > this.range.maxHeight)
+            return undefined;
+        let lastHeight = this.nextHeight + this.maxBufferSize / 80 - 1;
+        lastHeight = Math.min(lastHeight, this.range.maxHeight);
+        let file = await this.manager.getFileForHeight(this.nextHeight);
+        if (!file)
+            throw new sdk_1.WERR_INTERNAL(`logic error`);
+        const readRange = new HeightRange_1.HeightRange(this.nextHeight, lastHeight);
+        let buffers = new Uint8Array(readRange.length * 80);
+        let offset = 0;
+        while (file) {
+            const buffer = await this.readBufferFromFile(file, readRange);
+            if (!buffer)
+                break;
+            buffers.set(buffer, offset);
+            offset += buffer.length;
+            file = await this.manager.getFileForHeight(file.firstHeight + file.count);
+        }
+        if (!buffers.length || offset !== readRange.length * 80)
+            return undefined;
+        this.nextHeight = lastHeight + 1;
+        return buffers;
+    }
+}
+exports.BulkFileDataReader = BulkFileDataReader;
+//# sourceMappingURL=BulkFileDataManager.js.map