@bsv/wallet-toolbox-client 2.1.24 → 3.0.0-alpha.0

package/out/src/services/chaintracker/chaintracks/util/BulkFileDataManager.js

@@ -35,19 +35,16 @@ class BulkFileDataManager {
         this.bfds = [];
         this.fileHashToIndex = {};
         this.lock = new SingleWriterMultiReaderLock_1.SingleWriterMultiReaderLock();
-        if (typeof options === 'object')
-            options = options;
-        else
-            options = BulkFileDataManager.createDefaultOptions(options);
-        this.chain = options.chain;
-        this.maxPerFile = options.maxPerFile;
-        this.maxRetained = options.maxRetained;
-        this.fromKnownSourceUrl = options.fromKnownSourceUrl;
-        this.fetch = options.fetch;
+        const resolvedOptions = typeof options === 'object' ? options : BulkFileDataManager.createDefaultOptions(options);
+        this.chain = resolvedOptions.chain;
+        this.maxPerFile = resolvedOptions.maxPerFile;
+        this.maxRetained = resolvedOptions.maxRetained;
+        this.fromKnownSourceUrl = resolvedOptions.fromKnownSourceUrl;
+        this.fetch = resolvedOptions.fetch;
         this.deleteBulkFilesNoLock();
     }
     async deleteBulkFiles() {
-        return this.lock.withWriteLock(async () => this.deleteBulkFilesNoLock());
+        return await this.lock.withWriteLock(async () => this.deleteBulkFilesNoLock());
     }
     deleteBulkFilesNoLock() {
         this.bfds = [];
@@ -69,13 +66,13 @@ class BulkFileDataManager {
      * There should be no changes to bulk files by direct access to storage bulk file methods.
      */
     async setStorage(storage, log) {
-        return this.lock.withWriteLock(async () => this.setStorageNoLock(storage, log));
+        return await this.lock.withWriteLock(async () => await this.setStorageNoLock(storage, log));
     }
     async setStorageNoLock(storage, log) {
         this.storage = storage;
         this.log = log;
         // Get files currently in persistent storage.
-        let sfs = await this.storage.getBulkFiles();
+        const sfs = await this.storage.getBulkFiles();
         // Sync bfds with storage. Two scenarios supported:
         const bfdsRanges = this.heightRangesFromBulkFiles(this.bfds);
         const sfsRanges = this.heightRangesFromBulkFiles(sfs);
@@ -91,7 +88,9 @@ class BulkFileDataManager {
         else {
             // Bfds win if they have greater CDN coverage
             // Replace all sfs with bfds
-            for (const s of sfs.reverse())
+            const reversedFiles = [...sfs];
+            reversedFiles.reverse();
+            for (const s of reversedFiles)
                 await this.storage.deleteBulkFile(s.fileId);
             for (const bfd of this.bfds) {
                 await this.ensureData(bfd);
@@ -117,26 +116,27 @@ class BulkFileDataManager {
         return new BulkFileDataReader_1.BulkFileDataReader(this, range, maxBufferSize);
     }
     async updateFromUrl(cdnUrl) {
-        if (!this.fetch)
+        if (this.fetch == null)
             throw new sdk_1.WERR_INVALID_OPERATION('fetch is not defined in the BulkFileDataManager.');
         const toUrl = (file) => this.fetch.pathJoin(cdnUrl, file);
         const url = toUrl(`${this.chain}NetBlockHeaders.json`);
-        const availableBulkFiles = (await this.fetch.fetchJson(url));
-        if (!availableBulkFiles)
-            throw new sdk_1.WERR_INVALID_PARAMETER(`cdnUrl`, `a valid BulkHeaderFilesInfo JSON resource available from ${url}`);
+        const availableBulkFiles = await this.fetch.fetchJson(url);
+        if (!availableBulkFiles) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('cdnUrl', `a valid BulkHeaderFilesInfo JSON resource available from ${url}`);
+        }
         const selectedFiles = selectBulkHeaderFiles(availableBulkFiles.files, this.chain, this.maxPerFile || availableBulkFiles.headersPerFile);
         for (const bf of selectedFiles) {
             if (!bf.fileHash) {
-                throw new sdk_1.WERR_INVALID_PARAMETER(`fileHash`, `valid for all files in json downloaded from ${url}`);
+                throw new sdk_1.WERR_INVALID_PARAMETER('fileHash', `valid for all files in json downloaded from ${url}`);
             }
             if (!bf.chain || bf.chain !== this.chain) {
-                throw new sdk_1.WERR_INVALID_PARAMETER(`chain`, `"${this.chain}" for all files in json downloaded from ${url}`);
+                throw new sdk_1.WERR_INVALID_PARAMETER('chain', `"${this.chain}" for all files in json downloaded from ${url}`);
             }
             if (!bf.sourceUrl || bf.sourceUrl !== cdnUrl)
                 bf.sourceUrl = cdnUrl;
         }
         const rangeBefore = await this.getHeightRange();
-        const r = await this.merge(selectedFiles);
+        await this.merge(selectedFiles);
         const rangeAfter = await this.getHeightRange();
         let log = 'BulkDataFileManager.updateFromUrl\n';
         log += `  url: ${url}\n`;
@@ -145,17 +145,16 @@ class BulkFileDataManager {
         this.log(log);
     }
     async merge(files) {
-        return this.lock.withWriteLock(async () => this.mergeNoLock(files));
+        return await this.lock.withWriteLock(async () => await this.mergeNoLock(files));
     }
     async mergeNoLock(files) {
         const r = { inserted: [], updated: [], unchanged: [], dropped: [] };
         for (const file of files) {
             const hbf = this.getBfdForHeight(file.firstHeight);
-            if (hbf && file.fileId)
+            if ((hbf != null) && file.fileId)
                 hbf.fileId = file.fileId; // Always update fileId if provided
             const lbf = this.getLastBfd();
-            if (hbf &&
-                hbf.fileHash === file.fileHash &&
+            if ((hbf === null || hbf === void 0 ? void 0 : hbf.fileHash) === file.fileHash &&
                 hbf.count === file.count &&
                 hbf.lastHash === file.lastHash &&
                 hbf.lastChainWork === file.lastChainWork) {
@@ -164,11 +163,11 @@ class BulkFileDataManager {
                 continue;
             }
             const vbf = await this.validateFileInfo(file);
-            if (hbf) {
+            if (hbf != null) {
                 // We have a matching file by firstHeight but count and fileHash differ
                 await this.update(vbf, hbf, r);
             }
-            else if (isBdfIncremental(vbf) && lbf && isBdfIncremental(lbf)) {
+            else if (isBdfIncremental(vbf) && (lbf != null) && isBdfIncremental(lbf)) {
                 await this.mergeIncremental(lbf, vbf, r);
             }
             else {
@@ -194,13 +193,13 @@ class BulkFileDataManager {
         lbf.mru = Date.now();
         const lbfInfo = bfdToInfo(lbf, true);
         r.updated.push(lbfInfo);
-        if (this.storage && lbf.fileId) {
+        if ((this.storage != null) && lbf.fileId) {
             await this.storage.updateBulkFile(lbf.fileId, lbfInfo);
         }
     }
     toLogString(what) {
         let log = '';
-        if (!what) {
+        if (what == null) {
             log += this.toLogString(this.bfds);
         }
         else if (what['updated']) {
@@ -230,45 +229,20 @@ class BulkFileDataManager {
     async mergeIncrementalBlockHeaders(newBulkHeaders, incrementalChainWork) {
         if (newBulkHeaders.length === 0)
             return;
-        return this.lock.withWriteLock(async () => {
+        return await this.lock.withWriteLock(async () => {
             const lbf = this.getLastFileNoLock();
-            const nextHeight = lbf ? lbf.firstHeight + lbf.count : 0;
-            if (nextHeight > 0 && newBulkHeaders.length > 0 && newBulkHeaders[0].height < nextHeight) {
-                // Don't modify the incoming array...
-                newBulkHeaders = [...newBulkHeaders];
-                // If we have more headers than we need, drop the incoming headers.
-                while (newBulkHeaders.length > 0 && newBulkHeaders[0].height < nextHeight) {
-                    const h = newBulkHeaders.shift();
-                    if (h && incrementalChainWork) {
-                        incrementalChainWork = (0, blockHeaderUtilities_1.subWork)(incrementalChainWork, (0, blockHeaderUtilities_1.convertBitsToWork)(h.bits));
-                    }
-                }
-            }
+            const nextHeight = lbf != null ? lbf.firstHeight + lbf.count : 0;
+            ({ headers: newBulkHeaders, incrementalChainWork } = trimAlreadyStoredHeaders(newBulkHeaders, nextHeight, incrementalChainWork));
             if (newBulkHeaders.length === 0)
                 return;
-            if (!lbf || nextHeight !== newBulkHeaders[0].height)
+            if ((lbf == null) || nextHeight !== newBulkHeaders[0].height) {
                 throw new sdk_1.WERR_INVALID_PARAMETER('newBulkHeaders', 'an extension of existing bulk headers');
+            }
             if (!lbf.lastHash)
                 throw new sdk_1.WERR_INTERNAL(`lastHash is not defined for the last bulk file ${lbf.fileName}`);
-            const fbh = newBulkHeaders[0];
-            const lbh = newBulkHeaders.slice(-1)[0];
-            let lastChainWork = lbf.lastChainWork;
-            if (incrementalChainWork) {
-                lastChainWork = (0, blockHeaderUtilities_1.addWork)(incrementalChainWork, lastChainWork);
-            }
-            else {
-                // If lastChainWork is not provided, calculate it from the last file with basic validation.
-                let lastHeight = lbf.firstHeight + lbf.count - 1;
-                let lastHash = lbf.lastHash;
-                for (const h of newBulkHeaders) {
-                    if (h.height !== lastHeight + 1 || h.previousHash !== lastHash) {
-                        throw new sdk_1.WERR_INVALID_PARAMETER('headers', `an extension of existing bulk headers, header with height ${h.height} is non-sequential`);
-                    }
-                    lastChainWork = (0, blockHeaderUtilities_1.addWork)(lastChainWork, (0, blockHeaderUtilities_1.convertBitsToWork)(h.bits));
-                    lastHeight = h.height;
-                    lastHash = h.hash;
-                }
-            }
+            const lastChainWork = incrementalChainWork
+                ? (0, blockHeaderUtilities_1.addWork)(incrementalChainWork, lbf.lastChainWork)
+                : computeChainWorkFromHeaders(newBulkHeaders, lbf);
             const data = (0, blockHeaderUtilities_1.serializeBaseBlockHeaders)(newBulkHeaders);
             const fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(data)), 'base64');
             const bf = {
@@ -276,12 +250,12 @@ class BulkFileDataManager {
                 chain: this.chain,
                 sourceUrl: undefined,
                 fileName: 'incremental',
-                firstHeight: fbh.height,
+                firstHeight: newBulkHeaders[0].height,
                 count: newBulkHeaders.length,
                 prevChainWork: lbf.lastChainWork,
                 lastChainWork,
                 prevHash: lbf.lastHash,
-                lastHash: lbh.hash,
+                lastHash: newBulkHeaders.at(-1).hash,
                 fileHash,
                 data
             };
@@ -289,24 +263,25 @@ class BulkFileDataManager {
         });
     }
     async getBulkFiles(keepData) {
-        return this.lock.withReadLock(async () => {
+        return await this.lock.withReadLock(async () => {
             return this.bfds.map(bfd => bfdToInfo(bfd, keepData));
         });
     }
     async getHeightRange() {
-        return this.lock.withReadLock(async () => {
+        return await this.lock.withReadLock(async () => {
             if (this.bfds.length === 0)
                 return HeightRange_1.HeightRange.empty;
             const first = this.bfds[0];
-            const last = this.bfds[this.bfds.length - 1];
+            const last = this.bfds.at(-1);
             return new HeightRange_1.HeightRange(first.firstHeight, last.firstHeight + last.count - 1);
         });
     }
     async getDataFromFile(file, offset, length) {
-        const bfd = await this.getBfdForHeight(file.firstHeight);
-        if (!bfd || bfd.count < file.count)
+        const bfd = this.getBfdForHeight(file.firstHeight);
+        if ((bfd == null) || bfd.count < file.count) {
             throw new sdk_1.WERR_INVALID_PARAMETER('file', `a match for ${file.firstHeight}, ${file.count} in the BulkFileDataManager.`);
-        return this.lock.withReadLock(async () => this.getDataFromFileNoLock(bfd, offset, length));
+        }
+        return await this.lock.withReadLock(async () => await this.getDataFromFileNoLock(bfd, offset, length));
     }
     async getDataFromFileNoLock(bfd, offset, length) {
         const fileLength = bfd.count * 80;
@@ -316,47 +291,49 @@ class BulkFileDataManager {
         length = length || bfd.count * 80 - offset;
         length = Math.min(length, fileLength - offset);
         let data;
-        if (bfd.data) {
+        if (bfd.data != null) {
             data = bfd.data.slice(offset, offset + length);
         }
-        else if (bfd.fileId && this.storage) {
+        else if (bfd.fileId && (this.storage != null)) {
             data = await this.storage.getBulkFileData(bfd.fileId, offset, length);
         }
-        if (!data) {
+        if (data == null) {
             await this.ensureData(bfd);
-            if (bfd.data)
+            if (bfd.data != null)
                 data = bfd.data.slice(offset, offset + length);
         }
-        if (!data)
+        if (data == null)
             return undefined;
         return data;
     }
     async findHeaderForHeightOrUndefined(height) {
-        return this.lock.withReadLock(async () => {
-            if (!Number.isInteger(height) || height < 0)
+        return await this.lock.withReadLock(async () => {
+            if (!Number.isInteger(height) || height < 0) {
                 throw new sdk_1.WERR_INVALID_PARAMETER('height', `a non-negative integer (${height}).`);
+            }
             const file = this.bfds.find(f => f.firstHeight <= height && f.firstHeight + f.count > height);
-            if (!file)
+            if (file == null)
                 return undefined;
             const offset = (height - file.firstHeight) * 80;
             const data = await this.getDataFromFileNoLock(file, offset, 80);
-            if (!data)
+            if (data == null)
                 return undefined;
-            const header = (0, blockHeaderUtilities_1.deserializeBlockHeader)(data, 0, height);
+            const header = (0, blockHeaderUtilities_1.deserializeBlockHeader)(data, height, 0);
             return header;
         });
     }
     async getFileForHeight(height) {
-        return this.lock.withReadLock(async () => {
+        return await this.lock.withReadLock(async () => {
             const bfd = this.getBfdForHeight(height);
-            if (!bfd)
+            if (bfd == null)
                 return undefined;
             return bfdToInfo(bfd);
         });
     }
     getBfdForHeight(height) {
-        if (!Number.isInteger(height) || height < 0)
+        if (!Number.isInteger(height) || height < 0) {
             throw new sdk_1.WERR_INVALID_PARAMETER('height', `a non-negative integer (${height}).`);
+        }
         const file = this.bfds.find(f => f.firstHeight <= height && f.firstHeight + f.count > height);
         return file;
     }
@@ -367,74 +344,85 @@ class BulkFileDataManager {
         return bfd;
     }
     async getLastFile(fromEnd = 1) {
-        return this.lock.withReadLock(async () => this.getLastFileNoLock(fromEnd));
+        return await this.lock.withReadLock(async () => this.getLastFileNoLock(fromEnd));
     }
     getLastFileNoLock(fromEnd = 1) {
         const bfd = this.getLastBfd(fromEnd);
-        if (!bfd)
+        if (bfd == null)
             return undefined;
         return bfdToInfo(bfd);
     }
     async getDataByFileHash(fileHash) {
         const index = this.fileHashToIndex[fileHash];
-        if (index === undefined)
+        if (index === undefined) {
             throw new sdk_1.WERR_INVALID_PARAMETER('fileHash', `known to the BulkFileDataManager. ${fileHash} is unknown.`);
+        }
         const bfd = this.bfds[index];
         const data = await this.ensureData(bfd);
         return data;
     }
     async getDataByFileId(fileId) {
         const bfd = this.bfds.find(f => f.fileId === fileId);
-        if (bfd === undefined)
+        if (bfd === undefined) {
             throw new sdk_1.WERR_INVALID_PARAMETER('fileId', `known to the BulkFileDataManager. ${fileId} is unknown.`);
+        }
         const data = await this.ensureData(bfd);
         return data;
     }
     async validateFileInfo(file) {
-        var _a;
         if (file.chain !== this.chain)
             throw new sdk_1.WERR_INVALID_PARAMETER('chain', `${this.chain}`);
-        if (file.count <= 0)
+        if (file.count <= 0) {
             throw new sdk_1.WERR_INVALID_PARAMETER('bf.count', `expected count to be greater than 0, but got ${file.count}`);
-        if (file.count > this.maxPerFile && file.fileName !== 'incremental')
+        }
+        if (file.count > this.maxPerFile && file.fileName !== 'incremental') {
             throw new sdk_1.WERR_INVALID_PARAMETER('count', `less than or equal to maxPerFile ${this.maxPerFile}`);
+        }
         if (!file.fileHash)
-            throw new sdk_1.WERR_INVALID_PARAMETER('fileHash', `defined`);
-        if (!file.sourceUrl && !file.fileId && !file.data)
-            throw new sdk_1.WERR_INVALID_PARAMETER('data', `defined when sourceUrl and fileId are undefined`);
-        let bfd = {
-            ...file,
-            fileHash: file.fileHash,
-            mru: Date.now()
-        };
+            throw new sdk_1.WERR_INVALID_PARAMETER('fileHash', 'defined');
+        if (!file.sourceUrl && !file.fileId && (file.data == null)) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('data', 'defined when sourceUrl and fileId are undefined');
+        }
+        const bfd = { ...file, fileHash: file.fileHash, mru: Date.now() };
         if (!bfd.validated) {
-            await this.ensureData(bfd);
-            if (!bfd.data || bfd.data.length !== bfd.count * 80)
-                throw new sdk_1.WERR_INVALID_PARAMETER('file.data', `bulk file ${bfd.fileName} data length ${(_a = bfd.data) === null || _a === void 0 ? void 0 : _a.length} does not match expected count ${bfd.count}`);
-            bfd.fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(bfd.data)), 'base64');
-            if (file.fileHash && file.fileHash !== bfd.fileHash)
-                throw new sdk_1.WERR_INVALID_PARAMETER('file.fileHash', `expected ${file.fileHash} but got ${bfd.fileHash}`);
-            if (!(0, validBulkHeaderFilesByFileHash_1.isKnownValidBulkHeaderFile)(bfd)) {
-                const pbf = bfd.firstHeight > 0 ? this.getBfdForHeight(bfd.firstHeight - 1) : undefined;
-                const prevHash = pbf ? pbf.lastHash : '00'.repeat(32);
-                const prevChainWork = pbf ? pbf.lastChainWork : '00'.repeat(32);
-                const { lastHeaderHash, lastChainWork } = (0, blockHeaderUtilities_1.validateBufferOfHeaders)(bfd.data, prevHash, 0, undefined, prevChainWork);
-                if (bfd.lastHash && bfd.lastHash !== lastHeaderHash)
-                    throw new sdk_1.WERR_INVALID_PARAMETER('file.lastHash', `expected ${bfd.lastHash} but got ${lastHeaderHash}`);
-                if (bfd.lastChainWork && bfd.lastChainWork !== lastChainWork)
-                    throw new sdk_1.WERR_INVALID_PARAMETER('file.lastChainWork', `expected ${bfd.lastChainWork} but got ${lastChainWork}`);
-                bfd.lastHash = lastHeaderHash;
-                bfd.lastChainWork = lastChainWork;
-                if (bfd.firstHeight === 0) {
-                    (0, blockHeaderUtilities_1.validateGenesisHeader)(bfd.data, bfd.chain);
-                }
-            }
+            await this.validateBfdData(bfd, file.fileHash);
             bfd.validated = true;
         }
         return bfd;
     }
+    async validateBfdData(bfd, expectedFileHash) {
+        var _a;
+        await this.ensureData(bfd);
+        if ((bfd.data == null) || bfd.data.length !== bfd.count * 80) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('file.data', `bulk file ${bfd.fileName} data length ${(_a = bfd.data) === null || _a === void 0 ? void 0 : _a.length} does not match expected count ${bfd.count}`);
+        }
+        bfd.fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(bfd.data)), 'base64');
+        if (expectedFileHash && expectedFileHash !== bfd.fileHash) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('file.fileHash', `expected ${expectedFileHash} but got ${bfd.fileHash}`);
+        }
+        if (!(0, validBulkHeaderFilesByFileHash_1.isKnownValidBulkHeaderFile)(bfd)) {
+            this.validateBfdHeaders(bfd);
+        }
+    }
+    validateBfdHeaders(bfd) {
+        var _a, _b;
+        const pbf = bfd.firstHeight > 0 ? this.getBfdForHeight(bfd.firstHeight - 1) : undefined;
+        const prevHash = (_a = pbf === null || pbf === void 0 ? void 0 : pbf.lastHash) !== null && _a !== void 0 ? _a : '00'.repeat(32);
+        const prevChainWork = (_b = pbf === null || pbf === void 0 ? void 0 : pbf.lastChainWork) !== null && _b !== void 0 ? _b : '00'.repeat(32);
+        const { lastHeaderHash, lastChainWork } = (0, blockHeaderUtilities_1.validateBufferOfHeaders)(bfd.data, prevHash, 0, undefined, prevChainWork);
+        if (bfd.lastHash && bfd.lastHash !== lastHeaderHash) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('file.lastHash', `expected ${bfd.lastHash} but got ${lastHeaderHash}`);
+        }
+        if (bfd.lastChainWork && bfd.lastChainWork !== lastChainWork) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('file.lastChainWork', `expected ${bfd.lastChainWork} but got ${lastChainWork}`);
+        }
+        bfd.lastHash = lastHeaderHash;
+        bfd.lastChainWork = lastChainWork;
+        if (bfd.firstHeight === 0)
+            (0, blockHeaderUtilities_1.validateGenesisHeader)(bfd.data, bfd.chain);
+    }
     async ReValidate() {
-        return this.lock.withReadLock(async () => this.ReValidateNoLock());
+        return await this.lock.withReadLock(async () => await this.ReValidateNoLock());
     }
     async ReValidateNoLock() {
         for (const file of this.bfds) {
@@ -447,14 +435,17 @@ class BulkFileDataManager {
         }
     }
     validateBfdForAdd(bfd) {
-        if (this.bfds.length === 0 && bfd.firstHeight !== 0)
-            throw new sdk_1.WERR_INVALID_PARAMETER('firstHeight', `0 for the first file`);
+        if (this.bfds.length === 0 && bfd.firstHeight !== 0) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('firstHeight', '0 for the first file');
+        }
         if (this.bfds.length > 0) {
-            const last = this.bfds[this.bfds.length - 1];
-            if (bfd.firstHeight !== last.firstHeight + last.count)
-                throw new sdk_1.WERR_INVALID_PARAMETER('firstHeight', `the last file's firstHeight + count`);
-            if (bfd.prevHash !== last.lastHash || bfd.prevChainWork !== last.lastChainWork)
-                throw new sdk_1.WERR_INVALID_PARAMETER('prevHash/prevChainWork', `the last file's lastHash/lastChainWork`);
+            const last = this.bfds.at(-1);
+            if (bfd.firstHeight !== last.firstHeight + last.count) {
+                throw new sdk_1.WERR_INVALID_PARAMETER('firstHeight', 'the last file\'s firstHeight + count');
+            }
+            if (bfd.prevHash !== last.lastHash || bfd.prevChainWork !== last.lastChainWork) {
+                throw new sdk_1.WERR_INVALID_PARAMETER('prevHash/prevChainWork', 'the last file\'s lastHash/lastChainWork');
+            }
         }
     }
     async add(bfd) {
@@ -464,7 +455,7 @@ class BulkFileDataManager {
         this.fileHashToIndex[bfd.fileHash] = index;
         this.ensureMaxRetained();
         const info = bfdToInfo(bfd, true);
-        if (this.storage) {
+        if (this.storage != null) {
             info.fileId = bfd.fileId = await this.storage.insertBulkFile(info);
         }
         return info;
@@ -502,120 +493,109 @@ class BulkFileDataManager {
      * @param hbf corresponding existing BulkFileData to update.
      */
     async update(update, hbf, r) {
-        if (!hbf ||
-            hbf.firstHeight !== update.firstHeight ||
-            hbf.prevChainWork !== update.prevChainWork ||
-            hbf.prevHash !== update.prevHash)
-            throw new sdk_1.WERR_INVALID_PARAMETER('file', `an existing file by height, prevChainWork and prevHash`);
-        if (isBdfCdn(update) === isBdfCdn(hbf) && update.count <= hbf.count)
+        if ((hbf === null || hbf === void 0 ? void 0 : hbf.firstHeight) !== update.firstHeight ||
+            (hbf === null || hbf === void 0 ? void 0 : hbf.prevChainWork) !== update.prevChainWork ||
+            (hbf === null || hbf === void 0 ? void 0 : hbf.prevHash) !== update.prevHash) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('file', 'an existing file by height, prevChainWork and prevHash');
+        }
+        if (isBdfCdn(update) === isBdfCdn(hbf) && update.count <= hbf.count) {
             throw new sdk_1.WERR_INVALID_PARAMETER('file.count', `greater than the current count ${hbf.count}`);
+        }
+        const { index, truncate, replaced, drop } = await this.resolveUpdatePlan(update, hbf);
+        this.replaceBfdAtIndex(index, update);
+        if (truncate != null)
+            await this.shiftWork(update, truncate, replaced);
+        if (drop != null)
+            this.dropLastBulkFile(drop);
+        await this.persistUpdate(update, truncate, replaced, drop);
+        this.recordUpdateResults(r, update, truncate, replaced, drop);
+        this.ensureMaxRetained();
+    }
+    async resolveUpdatePlan(update, hbf) {
         const lbf = this.getLastBfd();
         let index = this.bfds.length - 1;
-        let truncate = undefined;
-        let replaced = undefined;
-        let drop = undefined;
+        let truncate;
+        let replaced;
+        let drop;
         if (hbf.firstHeight === lbf.firstHeight) {
-            // If the update is for the last file, there are three cases:
+            // Update targets the last file — three cases:
             if (isBdfIncremental(update)) {
-                // 1. Incremental file may only be extended with more incremental headers.
+                // 1. Incremental → incremental extension.
                 if (!isBdfIncremental(lbf))
-                    throw new sdk_1.WERR_INVALID_PARAMETER('file', `an incremental file to update an existing incremental file`);
+                    throw new sdk_1.WERR_INVALID_PARAMETER('file', 'an incremental file to update an existing incremental file');
             }
-            else {
-                // The update is a CDN bulk file.
-                if (isBdfCdn(lbf)) {
-                    // 2. An updated CDN file replaces a partial CDN file.
-                    if (update.count <= lbf.count)
-                        throw new sdk_1.WERR_INVALID_PARAMETER('update.count', `CDN update must have more headers. ${update.count} <= ${lbf.count}`);
-                }
-                else {
-                    // 3. A new CDN file replaces some or all of current incremental file.
-                    // Retain extra incremental headers if any.
-                    if (update.count < lbf.count) {
-                        // The new CDN partially replaces the last incremental file, prepare to shift work and re-add it.
-                        await this.ensureData(lbf);
-                        truncate = lbf;
-                    }
-                }
+            else if (isBdfCdn(lbf)) {
+                // 2. CDN → CDN replacement (must grow).
+                if (update.count <= lbf.count)
+                    throw new sdk_1.WERR_INVALID_PARAMETER('update.count', `CDN update must have more headers. ${update.count} <= ${lbf.count}`);
+            }
+            else if (update.count < lbf.count) {
+                // 3. New CDN partially replaces incremental tail — retain excess incremental headers.
+                await this.ensureData(lbf);
+                truncate = lbf;
             }
         }
         else {
-            // If the update is NOT for the last file, then it MUST be for the second to last file which MUST be a CDN file:
-            // - it must be a CDN file update with more headers than the current CDN file.
-            // - the last file must be an incremental file which is updated or deleted. The updated (or deleted) last file is returned.
+            // Update targets the second-to-last file — must be CDN replacing CDN, last must be incremental.
             const lbf2 = this.getLastBfd(2);
-            if (!lbf2 || hbf.firstHeight !== lbf2.firstHeight)
-                throw new sdk_1.WERR_INVALID_PARAMETER('file', `an update to last or second to last file`);
+            if ((lbf2 == null) || hbf.firstHeight !== lbf2.firstHeight)
+                throw new sdk_1.WERR_INVALID_PARAMETER('file', 'an update to last or second to last file');
             if (!isBdfCdn(update) || !isBdfCdn(lbf2) || update.count <= lbf2.count)
-                throw new sdk_1.WERR_INVALID_PARAMETER('file', `a CDN file update with more headers than the current CDN file`);
+                throw new sdk_1.WERR_INVALID_PARAMETER('file', 'a CDN file update with more headers than the current CDN file');
             if (!isBdfIncremental(lbf))
-                throw new sdk_1.WERR_INVALID_PARAMETER('file', `a CDN file update followed by an incremental file`);
+                throw new sdk_1.WERR_INVALID_PARAMETER('file', 'a CDN file update followed by an incremental file');
             if (!update.fileId)
-                update.fileId = lbf2.fileId; // Update fileId if not provided
+                update.fileId = lbf2.fileId;
             if (update.count >= lbf2.count + lbf.count) {
-                // The current last file is fully replaced by the CDN update.
                 drop = lbf;
             }
             else {
-                // If the update doesn't fully replace the last incremental file, make sure data is available to be truncated.
                 await this.ensureData(lbf);
                 truncate = lbf;
-                // The existing second to last file is fully replaced by the update.
                 replaced = lbf2;
             }
-            index = index - 1; // The update replaces the second to last file.
-        }
-        // In all cases the bulk file at the current fileId if any is updated.
-        this.replaceBfdAtIndex(index, update);
-        if (truncate) {
-            // If there is a bulk file to be truncated, it becomes the new (reduced) last file.
-            await this.shiftWork(update, truncate, replaced);
-        }
-        if (drop) {
-            this.dropLastBulkFile(drop);
+            index = index - 1;
         }
-        const updateInfo = bfdToInfo(update, true);
-        const truncateInfo = truncate ? bfdToInfo(truncate, true) : undefined;
-        if (this.storage) {
-            // Keep storage in sync.
-            if (update.fileId) {
-                await this.storage.updateBulkFile(update.fileId, updateInfo);
-            }
-            if (truncate && truncateInfo) {
-                if (replaced) {
-                    await this.storage.updateBulkFile(truncate.fileId, truncateInfo);
-                }
-                else {
-                    truncateInfo.fileId = undefined; // Make sure truncate is a new file.
-                    truncate.fileId = await this.storage.insertBulkFile(truncateInfo);
-                }
+        return { index, truncate, replaced, drop };
+    }
+    async persistUpdate(update, truncate, replaced, drop) {
+        if (this.storage == null)
+            return;
+        if (update.fileId)
+            await this.storage.updateBulkFile(update.fileId, bfdToInfo(update, true));
+        if ((truncate != null)) {
+            const truncateInfo = bfdToInfo(truncate, true);
+            if (replaced != null) {
+                await this.storage.updateBulkFile(truncate.fileId, truncateInfo);
             }
-            if (drop && drop.fileId) {
-                await this.storage.deleteBulkFile(drop.fileId);
+            else {
+                truncateInfo.fileId = undefined;
+                truncate.fileId = await this.storage.insertBulkFile(truncateInfo);
             }
         }
-        if (r) {
-            // Update results for logging...
-            r.updated.push(updateInfo);
-            if (truncateInfo) {
-                if (replaced) {
-                    r.updated.push(truncateInfo);
-                }
-                else {
-                    r.inserted.push(truncateInfo);
-                }
-            }
-            if (drop) {
-                r.dropped.push(bfdToInfo(drop));
-            }
+        if (drop === null || drop === void 0 ? void 0 : drop.fileId)
+            await this.storage.deleteBulkFile(drop.fileId);
+    }
+    recordUpdateResults(r, update, truncate, replaced, drop) {
+        if (!r)
+            return;
+        r.updated.push(bfdToInfo(update, true));
+        if (truncate != null) {
+            const truncateInfo = bfdToInfo(truncate, true);
+            if (replaced != null)
+                r.updated.push(truncateInfo);
+            else
+                r.inserted.push(truncateInfo);
         }
-        this.ensureMaxRetained();
+        if (drop != null)
+            r.dropped.push(bfdToInfo(drop));
     }
     dropLastBulkFile(lbf) {
         delete this.fileHashToIndex[lbf.fileHash];
         const index = this.bfds.indexOf(lbf);
-        if (index !== this.bfds.length - 1)
-            throw new sdk_1.WERR_INTERNAL(`dropLastBulkFile requires lbf is the current last file.`);
+        if (index !== this.bfds.length - 1) {
+            throw new sdk_1.WERR_INTERNAL('dropLastBulkFile requires lbf is the current last file.');
+        }
         this.bfds.pop();
     }
     /**
@@ -633,15 +613,17 @@ class BulkFileDataManager {
         // replaced will be valid if the update replaced it and it must become the new last file.
         // truncateIndex will be updateIndex + 1 if the existing last file is being truncated and update is second to last.
         const truncateIndex = this.fileHashToIndex[truncate.fileHash];
-        if (truncateIndex !== undefined && truncateIndex !== updateIndex + 1)
-            throw new sdk_1.WERR_INTERNAL(`shiftWork requires update to have replaced truncate or truncate to follow update`);
-        if (truncateIndex !== undefined && !replaced)
-            throw new sdk_1.WERR_INTERNAL(`shiftWork requires valid replaced when update hasn't replaced truncate`);
+        if (truncateIndex !== undefined && truncateIndex !== updateIndex + 1) {
+            throw new sdk_1.WERR_INTERNAL('shiftWork requires update to have replaced truncate or truncate to follow update');
+        }
+        if (truncateIndex !== undefined && (replaced == null)) {
+            throw new sdk_1.WERR_INTERNAL('shiftWork requires valid replaced when update hasn\'t replaced truncate');
+        }
         truncate.prevHash = update.lastHash;
         truncate.prevChainWork = update.lastChainWork;
         // truncate.lastChainWork, truncate.lastHash remain unchanged
         let count = update.count;
-        if (replaced) {
+        if (replaced != null) {
             count -= replaced.count;
         }
         else {
@@ -662,44 +644,46 @@ class BulkFileDataManager {
      * @returns
      */
     async ensureData(bfd) {
-        if (bfd.data)
+        if (bfd.data != null)
             return bfd.data;
-        if (this.storage && bfd.fileId) {
+        if ((this.storage != null) && bfd.fileId) {
             bfd.data = await this.storage.getBulkFileData(bfd.fileId);
-            if (!bfd.data)
+            if (bfd.data == null)
                 throw new sdk_1.WERR_INVALID_PARAMETER('fileId', `valid, data not found for fileId ${bfd.fileId}`);
         }
-        if (!bfd.data && this.fetch && bfd.sourceUrl) {
-            // TODO - restore this change
+        if ((bfd.data == null) && (this.fetch != null) && bfd.sourceUrl) {
             const url = this.fetch.pathJoin(bfd.sourceUrl, bfd.fileName);
-            //const url = this.fetch.pathJoin('http://localhost:8842/blockheaders', bfd.fileName)
             try {
                 bfd.data = await this.fetch.download(url);
             }
-            catch (err) {
+            catch (firstAttemptErr) {
+                // First download attempt failed (e.g. transient network error); retry once.
+                console.debug(`BulkFileDataManager: first download attempt failed for ${url}, retrying`, firstAttemptErr);
                 bfd.data = await this.fetch.download(url);
             }
             if (!bfd.data)
                 throw new sdk_1.WERR_INVALID_PARAMETER('sourceUrl', `data not found for sourceUrl ${url}`);
         }
-        if (!bfd.data)
+        if (bfd.data == null)
             throw new sdk_1.WERR_INVALID_PARAMETER('data', `defined. Unable to retrieve data for ${bfd.fileName}`);
         bfd.mru = Date.now();
         // Validate retrieved data.
         const fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(bfd.data)), 'base64');
-        if (fileHash !== bfd.fileHash)
+        if (fileHash !== bfd.fileHash) {
             throw new sdk_1.WERR_INVALID_PARAMETER('fileHash', `a match for retrieved data for ${bfd.fileName}`);
+        }
         this.ensureMaxRetained();
         return bfd.data;
     }
     ensureMaxRetained() {
         if (this.maxRetained === undefined)
             return;
-        let withData = this.bfds.filter(bfd => bfd.data && (bfd.fileId || bfd.sourceUrl));
+        const withData = this.bfds.filter(bfd => (bfd.data != null) && (bfd.fileId || bfd.sourceUrl));
         let countToRelease = withData.length - this.maxRetained;
         if (countToRelease <= 0)
             return;
-        const sorted = withData.sort((a, b) => a.mru - b.mru);
+        const sorted = [...withData];
+        sorted.sort((a, b) => a.mru - b.mru);
         while (countToRelease-- > 0 && sorted.length > 0) {
             const oldest = sorted.shift();
             // Release the least recently used data
@@ -728,7 +712,7 @@ class BulkFileDataManager {
         for (;;) {
             i++;
             const data = await reader.read();
-            if (!data || data.length === 0) {
+            if ((data == null) || data.length === 0) {
                 break;
             }
             const last = (0, blockHeaderUtilities_1.validateBufferOfHeaders)(data, lastHeaderHash, 0, undefined, lastChainWork);
@@ -792,6 +776,42 @@ function bfdToInfo(bfd, keepData) {
         data: keepData ? bfd.data : undefined
     };
 }
+/**
+ * Drops any headers whose height is already covered by bulk storage,
+ * reducing `incrementalChainWork` accordingly (when provided).
+ */
+function trimAlreadyStoredHeaders(headers, nextHeight, incrementalChainWork) {
+    if (nextHeight <= 0 || headers.length === 0 || headers[0].height >= nextHeight) {
+        return { headers, incrementalChainWork };
+    }
+    // Avoid mutating the caller's array.
+    headers = [...headers];
+    while (headers.length > 0 && headers[0].height < nextHeight) {
+        const h = headers.shift();
+        if ((h != null) && incrementalChainWork) {
+            incrementalChainWork = (0, blockHeaderUtilities_1.subWork)(incrementalChainWork, (0, blockHeaderUtilities_1.convertBitsToWork)(h.bits));
+        }
+    }
+    return { headers, incrementalChainWork };
+}
+/**
+ * Computes `lastChainWork` for a sequence of new bulk headers extending `lbf`,
+ * validating that the sequence is contiguous.
+ */
+function computeChainWorkFromHeaders(headers, lbf) {
+    let lastHeight = lbf.firstHeight + lbf.count - 1;
+    let lastHash = lbf.lastHash;
+    let lastChainWork = lbf.lastChainWork;
+    for (const h of headers) {
+        if (h.height !== lastHeight + 1 || h.previousHash !== lastHash) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('headers', `an extension of existing bulk headers, header with height ${h.height} is non-sequential`);
+        }
+        lastChainWork = (0, blockHeaderUtilities_1.addWork)(lastChainWork, (0, blockHeaderUtilities_1.convertBitsToWork)(h.bits));
+        lastHeight = h.height;
+        lastHash = h.hash;
+    }
+    return lastChainWork;
+}
 // Re-export BulkFileDataReader for backward compatibility
 var BulkFileDataReader_2 = require("./BulkFileDataReader");
 Object.defineProperty(exports, "BulkFileDataReader", { enumerable: true, get: function () { return BulkFileDataReader_2.BulkFileDataReader; } });