@bopen-io/wallet-toolbox 2.6.2-brc153.5 → 2.14.6

package/out/src/services/chaintracker/chaintracks/util/BulkFileDataManager.js

@@ -1,16 +1,202 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BulkFileDataReader = exports.BulkFileDataManager = void 0;
+exports.normalizeBulkHeaderFileInfo = normalizeBulkHeaderFileInfo;
+exports.normalizeBulkHeaderFilesInfo = normalizeBulkHeaderFilesInfo;
+exports.normalizeBulkHeaderFileSequence = normalizeBulkHeaderFileSequence;
 exports.selectBulkHeaderFiles = selectBulkHeaderFiles;
 const BulkFileDataReader_1 = require("./BulkFileDataReader");
 const sdk_1 = require("../../../../sdk");
-const sdk_2 = require("@bsv/sdk");
+const Hash_1 = require("@bsv/sdk/primitives/Hash");
 const utilityHelpers_noBuffer_1 = require("../../../../utility/utilityHelpers.noBuffer");
 const validBulkHeaderFilesByFileHash_1 = require("./validBulkHeaderFilesByFileHash");
 const HeightRange_1 = require("./HeightRange");
 const blockHeaderUtilities_1 = require("./blockHeaderUtilities");
 const ChaintracksFetch_1 = require("./ChaintracksFetch");
 const SingleWriterMultiReaderLock_1 = require("./SingleWriterMultiReaderLock");
+const BulkFileDataValidatorApi_1 = require("../Api/BulkFileDataValidatorApi");
+const InlineBulkFileDataValidator_1 = require("./InlineBulkFileDataValidator");
+const MAX_BULK_FILE_HEADERS = 100_000;
+const MAX_BULK_MANIFEST_FILES = 10_000;
+const MAX_BULK_FILE_NAME_BYTES = 255;
+const HEX_32_BYTES = /^[0-9a-f]{64}$/;
+const SHA256_BASE64 = /^[A-Za-z0-9+/]{43}=$/;
+const SAFE_BULK_FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._~-]*$/;
+const SUPPORTED_CHAINS = new Set(['main', 'test', 'stn', 'ttn', 'tstn', 'mock']);
+function ownDataValue(value, property, required = true) {
+    if (value == null || typeof value !== 'object') {
+        throw new sdk_1.WERR_INVALID_PARAMETER('bulk file metadata', 'an object containing plain data properties');
+    }
+    const descriptor = Object.getOwnPropertyDescriptor(value, property);
+    if (descriptor == null) {
+        if (!required)
+            return undefined;
+        throw new sdk_1.WERR_INVALID_PARAMETER(`bulk file ${property}`, 'defined');
+    }
+    if (!('value' in descriptor)) {
+        throw new sdk_1.WERR_INVALID_PARAMETER(`bulk file ${property}`, 'an own data property');
+    }
+    return descriptor.value;
+}
+function optionalString(value, name, maximum) {
+    if (value === undefined)
+        return undefined;
+    if (typeof value !== 'string' || value.length > maximum || /[\r\n]/.test(value) || value.includes('\u0000')) {
+        throw new sdk_1.WERR_INVALID_PARAMETER(name, `a string no longer than ${maximum} characters without control lines`);
+    }
+    return value;
+}
+function normalizeBulkHeaderFileInfo(value, allowStoredBoolean = false) {
+    const chain = ownDataValue(value, 'chain');
+    const count = ownDataValue(value, 'count');
+    const firstHeight = ownDataValue(value, 'firstHeight');
+    const fileName = ownDataValue(value, 'fileName');
+    const fileHash = ownDataValue(value, 'fileHash');
+    const prevHash = ownDataValue(value, 'prevHash');
+    const lastHash = ownDataValue(value, 'lastHash');
+    const prevChainWork = ownDataValue(value, 'prevChainWork');
+    const lastChainWork = ownDataValue(value, 'lastChainWork');
+    if (!SUPPORTED_CHAINS.has(chain)) {
+        throw new sdk_1.WERR_INVALID_PARAMETER('bulk file chain', 'a supported Chaintracks network');
+    }
+    if (!Number.isSafeInteger(count) || count < 1 || count > MAX_BULK_FILE_HEADERS) {
+        throw new sdk_1.WERR_INVALID_PARAMETER('bulk file count', `an integer from 1 through ${MAX_BULK_FILE_HEADERS}`);
+    }
+    if (!Number.isSafeInteger(firstHeight) ||
+        firstHeight < 0 ||
+        firstHeight > 0x7fffffff ||
+        firstHeight + count - 1 > 0x7fffffff) {
+        throw new sdk_1.WERR_INVALID_PARAMETER('bulk file firstHeight', 'a supported non-negative block-header range');
+    }
+    if (typeof fileName !== 'string' ||
+        fileName.length === 0 ||
+        fileName.length > MAX_BULK_FILE_NAME_BYTES ||
+        !SAFE_BULK_FILE_NAME.test(fileName) ||
+        fileName === '.' ||
+        fileName === '..') {
+        throw new sdk_1.WERR_INVALID_PARAMETER('bulk file fileName', 'a safe path-free ASCII file name');
+    }
+    let canonicalFileHash = false;
+    if (typeof fileHash === 'string' && SHA256_BASE64.test(fileHash)) {
+        try {
+            const digest = (0, utilityHelpers_noBuffer_1.asUint8Array)(fileHash, 'base64');
+            canonicalFileHash = digest.length === 32 && (0, utilityHelpers_noBuffer_1.asString)(digest, 'base64') === fileHash;
+        }
+        catch {
+            canonicalFileHash = false;
+        }
+    }
+    if (!canonicalFileHash) {
+        throw new sdk_1.WERR_INVALID_PARAMETER('bulk file fileHash', 'a canonical base64 SHA-256 digest');
+    }
+    for (const [name, candidate] of [
+        ['prevHash', prevHash],
+        ['lastHash', lastHash],
+        ['prevChainWork', prevChainWork],
+        ['lastChainWork', lastChainWork]
+    ]) {
+        if (typeof candidate !== 'string' || !HEX_32_BYTES.test(candidate)) {
+            throw new sdk_1.WERR_INVALID_PARAMETER(`bulk file ${name}`, 'exactly 32 lowercase hexadecimal bytes');
+        }
+    }
+    const storedSourceUrl = ownDataValue(value, 'sourceUrl', false);
+    const sourceUrl = optionalString(allowStoredBoolean && storedSourceUrl === null ? undefined : storedSourceUrl, 'bulk file sourceUrl', 2048);
+    const fileIdValue = ownDataValue(value, 'fileId', false);
+    if (fileIdValue !== undefined && (!Number.isSafeInteger(fileIdValue) || fileIdValue < 1)) {
+        throw new sdk_1.WERR_INVALID_PARAMETER('bulk file fileId', 'a positive safe integer');
+    }
+    const validatedValue = ownDataValue(value, 'validated', false);
+    if (validatedValue !== undefined &&
+        typeof validatedValue !== 'boolean' &&
+        !(allowStoredBoolean && (validatedValue === 0 || validatedValue === 1))) {
+        throw new sdk_1.WERR_INVALID_PARAMETER('bulk file validated', 'a boolean when defined');
+    }
+    const dataValue = ownDataValue(value, 'data', false);
+    if (dataValue !== undefined && !(dataValue instanceof Uint8Array)) {
+        throw new sdk_1.WERR_INVALID_PARAMETER('bulk file data', 'a Uint8Array when defined');
+    }
+    return {
+        chain: chain,
+        count: count,
+        firstHeight: firstHeight,
+        fileName: fileName,
+        fileHash: fileHash,
+        prevHash: prevHash,
+        lastHash: lastHash,
+        prevChainWork: prevChainWork,
+        lastChainWork: lastChainWork,
+        sourceUrl,
+        fileId: fileIdValue,
+        validated: validatedValue === undefined ? undefined : validatedValue === true || validatedValue === 1,
+        data: dataValue === undefined ? undefined : new Uint8Array(dataValue)
+    };
+}
+function normalizeBulkHeaderFilesInfo(value) {
+    const filesValue = ownDataValue(value, 'files');
+    const headersPerFile = ownDataValue(value, 'headersPerFile');
+    if (!Array.isArray(filesValue) || filesValue.length > MAX_BULK_MANIFEST_FILES) {
+        throw new sdk_1.WERR_INVALID_PARAMETER('bulk manifest files', `a dense array of no more than ${MAX_BULK_MANIFEST_FILES} entries`);
+    }
+    if (!Number.isSafeInteger(headersPerFile) ||
+        headersPerFile < 1 ||
+        headersPerFile > MAX_BULK_FILE_HEADERS) {
+        throw new sdk_1.WERR_INVALID_PARAMETER('bulk manifest headersPerFile', `an integer from 1 through ${MAX_BULK_FILE_HEADERS}`);
+    }
+    const files = [];
+    for (let index = 0; index < filesValue.length; index++) {
+        const descriptor = Object.getOwnPropertyDescriptor(filesValue, String(index));
+        if (descriptor == null || !('value' in descriptor)) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('bulk manifest files', 'a dense array of plain data entries');
+        }
+        files.push(normalizeBulkHeaderFileInfo(descriptor.value));
+    }
+    return {
+        rootFolder: optionalString(ownDataValue(value, 'rootFolder', false), 'bulk manifest rootFolder', 2048) ?? '',
+        jsonFilename: optionalString(ownDataValue(value, 'jsonFilename', false), 'bulk manifest jsonFilename', 255) ?? '',
+        headersPerFile: headersPerFile,
+        files
+    };
+}
+function normalizeBulkHeaderFileSequence(files, allowStoredBoolean = false) {
+    if (!Array.isArray(files) || files.length > MAX_BULK_MANIFEST_FILES) {
+        throw new sdk_1.WERR_INVALID_PARAMETER('files', `a dense array of no more than ${MAX_BULK_MANIFEST_FILES} bulk files`);
+    }
+    const canonical = [];
+    const hashes = new Set();
+    const ids = new Set();
+    for (let index = 0; index < files.length; index++) {
+        const descriptor = Object.getOwnPropertyDescriptor(files, String(index));
+        if (descriptor == null || !('value' in descriptor)) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('files', 'a dense array of plain data entries');
+        }
+        const file = normalizeBulkHeaderFileInfo(descriptor.value, allowStoredBoolean);
+        const fileHash = file.fileHash;
+        if (hashes.has(fileHash))
+            throw new sdk_1.WERR_INVALID_PARAMETER('files', 'unique file hashes');
+        hashes.add(fileHash);
+        if (file.fileId !== undefined) {
+            if (ids.has(file.fileId))
+                throw new sdk_1.WERR_INVALID_PARAMETER('files', 'unique file ids');
+            ids.add(file.fileId);
+        }
+        const previous = canonical.at(-1);
+        if (previous == null) {
+            if (file.firstHeight !== 0 || file.prevHash !== '00'.repeat(32) || file.prevChainWork !== '00'.repeat(32)) {
+                throw new sdk_1.WERR_INVALID_PARAMETER('files', 'a genesis-anchored first bulk file');
+            }
+        }
+        else if (file.firstHeight !== previous.firstHeight + previous.count ||
+            file.prevHash !== previous.lastHash ||
+            file.prevChainWork !== previous.lastChainWork) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('files', 'one contiguous chain of bulk files');
+        }
+        if (previous != null && isBdfIncremental(previous)) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('files', 'an incremental bulk file only in the final position');
+        }
+        canonical.push(file);
+    }
+    return canonical;
+}
 /**
  * Manages bulk file data (typically 8MB chunks of 100,000 headers each).
  *
@@ -32,35 +218,93 @@ class BulkFileDataManager {
     }
     log = () => { };
     bfds = [];
-    fileHashToIndex = {};
+    fileHashToIndex = Object.create(null);
     lock = new SingleWriterMultiReaderLock_1.SingleWriterMultiReaderLock();
+    inFlightLoads = new Map();
+    failedLoads = new Map();
     storage;
+    stats = {
+        memoryHits: 0,
+        storageHits: 0,
+        persistentCacheHits: 0,
+        persistentCacheMisses: 0,
+        persistentCacheRejects: 0,
+        coalescedLoads: 0,
+        downloads: 0,
+        downloadedBytes: 0,
+        loadBackoffs: 0
+    };
     chain;
     maxPerFile;
     fetch;
     maxRetained;
     fromKnownSourceUrl;
+    cache;
+    downloadBudget;
+    validator;
+    failedLoadRetryMsecs;
     constructor(options) {
+        if (options == null)
+            throw new sdk_1.WERR_INVALID_PARAMETER('options', 'a Chain or BulkFileDataManagerOptions object');
         const resolvedOptions = typeof options === 'object' ? options : BulkFileDataManager.createDefaultOptions(options);
         this.chain = resolvedOptions.chain;
         this.maxPerFile = resolvedOptions.maxPerFile;
         this.maxRetained = resolvedOptions.maxRetained;
         this.fromKnownSourceUrl = resolvedOptions.fromKnownSourceUrl;
         this.fetch = resolvedOptions.fetch;
+        this.cache = resolvedOptions.cache;
+        this.downloadBudget = resolvedOptions.downloadBudget;
+        this.validator = resolvedOptions.validator ?? new InlineBulkFileDataValidator_1.InlineBulkFileDataValidator();
+        this.failedLoadRetryMsecs = resolvedOptions.failedLoadRetryMsecs ?? 30 * 1000;
+        if (!SUPPORTED_CHAINS.has(this.chain)) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('chain', 'a supported Chaintracks network');
+        }
+        if (!Number.isSafeInteger(this.maxPerFile) || this.maxPerFile < 1 || this.maxPerFile > MAX_BULK_FILE_HEADERS) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('maxPerFile', `an integer from 1 through ${MAX_BULK_FILE_HEADERS}`);
+        }
+        if (this.maxRetained !== undefined &&
+            (!Number.isSafeInteger(this.maxRetained) || this.maxRetained < 0 || this.maxRetained > 1000)) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('maxRetained', 'an integer from 0 through 1000 when defined');
+        }
+        if (!Number.isSafeInteger(this.failedLoadRetryMsecs) || this.failedLoadRetryMsecs < 0) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('failedLoadRetryMsecs', 'a non-negative safe integer');
+        }
+        if (this.fromKnownSourceUrl !== undefined &&
+            (typeof this.fromKnownSourceUrl !== 'string' ||
+                this.fromKnownSourceUrl.length > 2048 ||
+                /[\r\n]/.test(this.fromKnownSourceUrl) ||
+                this.fromKnownSourceUrl.includes('\u0000'))) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('fromKnownSourceUrl', 'a bounded string without control lines when defined');
+        }
+        if (typeof this.validator?.validate !== 'function') {
+            throw new sdk_1.WERR_INVALID_PARAMETER('validator', 'an object with a validate function');
+        }
+        if (this.cache != null && (typeof this.cache.get !== 'function' || typeof this.cache.set !== 'function')) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('cache', 'an object with get and set functions when defined');
+        }
+        if (this.downloadBudget != null && typeof this.downloadBudget.consume !== 'function') {
+            throw new sdk_1.WERR_INVALID_PARAMETER('downloadBudget', 'an object with a consume function when defined');
+        }
         this.deleteBulkFilesNoLock();
     }
+    getStats() {
+        return {
+            ...this.stats,
+            validation: this.validator.getStats?.(),
+            downloadBudget: this.downloadBudget?.snapshot?.()
+        };
+    }
     async deleteBulkFiles() {
         return await this.lock.withWriteLock(async () => this.deleteBulkFilesNoLock());
     }
     deleteBulkFilesNoLock() {
-        this.bfds = [];
-        this.fileHashToIndex = {};
+        this.setBulkFilesState([]);
         if (this.fromKnownSourceUrl) {
             const vbhfs = validBulkHeaderFilesByFileHash_1.validBulkHeaderFiles;
             const filtered = vbhfs.filter(f => f.sourceUrl === this.fromKnownSourceUrl);
             const files = selectBulkHeaderFiles(filtered, this.chain, this.maxPerFile);
             for (const file of files) {
-                this.add({ ...file, fileHash: file.fileHash, mru: Date.now() });
+                this.add({ ...file, fileHash: file.fileHash, validated: false, mru: Date.now() });
             }
         }
     }
@@ -75,33 +319,62 @@ class BulkFileDataManager {
         return await this.lock.withWriteLock(async () => await this.setStorageNoLock(storage, log));
     }
     async setStorageNoLock(storage, log) {
-        this.storage = storage;
-        this.log = log;
+        if (storage == null ||
+            typeof storage !== 'object' ||
+            typeof storage.getBulkFiles !== 'function' ||
+            typeof storage.getBulkFileData !== 'function' ||
+            typeof storage.insertBulkFile !== 'function' ||
+            typeof storage.updateBulkFile !== 'function' ||
+            typeof storage.deleteBulkFile !== 'function') {
+            throw new sdk_1.WERR_INVALID_PARAMETER('storage', 'a complete ChaintracksStorageBulkFileApi implementation');
+        }
+        if (typeof log !== 'function')
+            throw new sdk_1.WERR_INVALID_PARAMETER('log', 'a function');
         // Get files currently in persistent storage.
-        const sfs = await this.storage.getBulkFiles();
+        const sfs = await storage.getBulkFiles();
         // Sync bfds with storage. Two scenarios supported:
         const bfdsRanges = this.heightRangesFromBulkFiles(this.bfds);
         const sfsRanges = this.heightRangesFromBulkFiles(sfs);
         if (sfsRanges.cdn.length >= bfdsRanges.cdn.length) {
             // Storage win if it has greater or equal CDN coverage
-            // Replace all bfds with sfs
-            this.bfds = [];
-            for (const file of sfs) {
-                const vbf = await this.validateFileInfo(file);
-                this.bfds.push(vbf);
+            // Validate the entire durable sequence before making it authoritative.
+            const priorStorage = this.storage;
+            const priorLog = this.log;
+            const priorFiles = this.bfds;
+            this.storage = storage;
+            this.log = log;
+            this.setBulkFilesState([]);
+            try {
+                for (const file of sfs) {
+                    const vbf = await this.validateFileInfo(file);
+                    this.validateBfdForAdd(vbf);
+                    this.bfds.push(vbf);
+                    this.fileHashToIndex[vbf.fileHash] = this.bfds.length - 1;
+                }
+            }
+            catch (error) {
+                this.storage = priorStorage;
+                this.log = priorLog;
+                this.setBulkFilesState(priorFiles);
+                throw error;
             }
         }
         else {
             // Bfds win if they have greater CDN coverage
-            // Replace all sfs with bfds
-            const reversedFiles = [...sfs];
-            reversedFiles.reverse();
-            for (const s of reversedFiles)
-                await this.storage.deleteBulkFile(s.fileId);
+            // Replace durable state as one transaction. Sequential deletion and
+            // insertion can expose a valid-looking but incomplete header chain.
+            if (typeof storage.replaceBulkFiles !== 'function') {
+                throw new sdk_1.WERR_INVALID_OPERATION('storage must implement atomic replaceBulkFiles before replacing an existing bulk-header set');
+            }
+            const desired = [];
             for (const bfd of this.bfds) {
-                await this.ensureData(bfd);
-                bfd.fileId = await this.storage.insertBulkFile(bfdToInfo(bfd, true));
+                const data = await this.ensureData(bfd);
+                desired.push({ ...bfdToInfo(bfd, true), data: new Uint8Array(data) });
             }
+            const committed = await storage.replaceBulkFiles(desired);
+            this.applyCommittedBulkFiles(committed);
+            this.storage = storage;
+            this.log = log;
         }
     }
     heightRangesFromBulkFiles(files) {
@@ -117,8 +390,8 @@ class BulkFileDataManager {
         return ranges;
     }
     async createReader(range, maxBufferSize) {
-        range = range || (await this.getHeightRange());
-        maxBufferSize = maxBufferSize || 1000000 * 80; // 100,000 headers, 8MB
+        range ??= await this.getHeightRange();
+        maxBufferSize ??= MAX_BULK_FILE_HEADERS * 80;
         return new BulkFileDataReader_1.BulkFileDataReader(this, range, maxBufferSize);
     }
     async updateFromUrl(cdnUrl) {
@@ -126,10 +399,11 @@ class BulkFileDataManager {
             throw new sdk_1.WERR_INVALID_OPERATION('fetch is not defined in the BulkFileDataManager.');
         const toUrl = (file) => this.fetch.pathJoin(cdnUrl, file);
         const url = toUrl(`${this.chain}NetBlockHeaders.json`);
-        const availableBulkFiles = await this.fetch.fetchJson(url);
-        if (!availableBulkFiles) {
+        const response = await this.fetch.fetchJson(url);
+        if (!response) {
             throw new sdk_1.WERR_INVALID_PARAMETER('cdnUrl', `a valid BulkHeaderFilesInfo JSON resource available from ${url}`);
         }
+        const availableBulkFiles = normalizeBulkHeaderFilesInfo(response);
         const selectedFiles = selectBulkHeaderFiles(availableBulkFiles.files, this.chain, this.maxPerFile || availableBulkFiles.headersPerFile);
         for (const bf of selectedFiles) {
             if (!bf.fileHash) {
@@ -138,6 +412,8 @@ class BulkFileDataManager {
             if (!bf.chain || bf.chain !== this.chain) {
                 throw new sdk_1.WERR_INVALID_PARAMETER('chain', `"${this.chain}" for all files in json downloaded from ${url}`);
             }
+            // A remote manifest never has authority over local storage identities.
+            bf.fileId = undefined;
             if (!bf.sourceUrl || bf.sourceUrl !== cdnUrl)
                 bf.sourceUrl = cdnUrl;
         }
@@ -154,11 +430,20 @@ class BulkFileDataManager {
         return await this.lock.withWriteLock(async () => await this.mergeNoLock(files));
     }
     async mergeNoLock(files) {
+        if (!Array.isArray(files) || files.length > MAX_BULK_MANIFEST_FILES) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('files', `an array of no more than ${MAX_BULK_MANIFEST_FILES} entries`);
+        }
         const r = { inserted: [], updated: [], unchanged: [], dropped: [] };
-        for (const file of files) {
+        for (let index = 0; index < files.length; index++) {
+            const descriptor = Object.getOwnPropertyDescriptor(files, String(index));
+            if (descriptor == null || !('value' in descriptor)) {
+                throw new sdk_1.WERR_INVALID_PARAMETER('files', 'a dense array of plain data entries');
+            }
+            const file = normalizeBulkHeaderFileInfo(descriptor.value);
             const hbf = this.getBfdForHeight(file.firstHeight);
-            if (hbf != null && file.fileId)
-                hbf.fileId = file.fileId; // Always update fileId if provided
+            if (this.storage != null && hbf == null && file.fileId !== undefined) {
+                throw new sdk_1.WERR_INVALID_PARAMETER('file.fileId', 'undefined for a new bulk file');
+            }
             const lbf = this.getLastBfd();
             if (hbf?.fileHash === file.fileHash &&
                 hbf.count === file.count &&
@@ -171,12 +456,18 @@ class BulkFileDataManager {
             const vbf = await this.validateFileInfo(file);
             if (hbf != null) {
                 // We have a matching file by firstHeight but count and fileHash differ
+                await this.ensureData(vbf);
                 await this.update(vbf, hbf, r);
             }
             else if (isBdfIncremental(vbf) && lbf != null && isBdfIncremental(lbf)) {
                 await this.mergeIncremental(lbf, vbf, r);
             }
             else {
+                // A storage-backed record must never be created with metadata only:
+                // subsequent reads prefer its fileId and correctly fail closed when
+                // data is absent. Validate and materialize it before insertion.
+                if (this.storage != null)
+                    await this.ensureData(vbf);
                 const added = await this.add(vbf);
                 r.inserted.push(added);
             }
@@ -185,23 +476,32 @@ class BulkFileDataManager {
         return r;
     }
     async mergeIncremental(lbf, vbf, r) {
-        lbf.count += vbf.count;
-        lbf.lastHash = vbf.lastHash;
-        lbf.lastChainWork = vbf.lastChainWork;
+        if (lbf.count + vbf.count > MAX_BULK_FILE_HEADERS) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('incremental headers', `no more than ${MAX_BULK_FILE_HEADERS} per file`);
+        }
         await this.ensureData(lbf);
         const newData = new Uint8Array(lbf.data.length + vbf.data.length);
         newData.set(lbf.data);
         newData.set(vbf.data, lbf.data.length);
-        lbf.data = newData;
-        delete this.fileHashToIndex[lbf.fileHash];
-        lbf.fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(newData)), 'base64');
-        this.fileHashToIndex[lbf.fileHash] = this.bfds.length - 1;
-        lbf.mru = Date.now();
-        const lbfInfo = bfdToInfo(lbf, true);
-        r.updated.push(lbfInfo);
-        if (this.storage != null && lbf.fileId) {
-            await this.storage.updateBulkFile(lbf.fileId, lbfInfo);
+        const merged = {
+            ...lbf,
+            count: lbf.count + vbf.count,
+            lastHash: vbf.lastHash,
+            lastChainWork: vbf.lastChainWork,
+            data: newData,
+            fileHash: (0, utilityHelpers_noBuffer_1.asString)((0, Hash_1.sha256)((0, utilityHelpers_noBuffer_1.asArray)(newData)), 'base64'),
+            mru: Date.now()
+        };
+        const lbfInfo = bfdToInfo(merged, true);
+        if (this.storage != null) {
+            if (!merged.fileId)
+                throw new sdk_1.WERR_INVALID_OPERATION('stored incremental bulk file is missing its fileId');
+            const affected = await this.storage.updateBulkFile(merged.fileId, lbfInfo);
+            if (affected !== 1)
+                throw new sdk_1.WERR_INVALID_OPERATION(`failed to update bulk file ${merged.fileId}`);
         }
+        this.replaceBfdAtIndex(this.bfds.length - 1, merged);
+        r.updated.push(lbfInfo);
     }
     toLogString(what) {
         let log = '';
@@ -233,8 +533,51 @@ class BulkFileDataManager {
         return log;
     }
     async mergeIncrementalBlockHeaders(newBulkHeaders, incrementalChainWork) {
+        if (!Array.isArray(newBulkHeaders) || newBulkHeaders.length > MAX_BULK_FILE_HEADERS) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('newBulkHeaders', `a dense array of no more than ${MAX_BULK_FILE_HEADERS} headers`);
+        }
+        const authenticatedHeaders = [];
+        for (let index = 0; index < newBulkHeaders.length; index++) {
+            if (!Object.hasOwn(newBulkHeaders, index)) {
+                throw new sdk_1.WERR_INVALID_PARAMETER('newBulkHeaders', 'a dense array');
+            }
+            const source = newBulkHeaders[index];
+            if (source == null || typeof source !== 'object' || Array.isArray(source)) {
+                throw new sdk_1.WERR_INVALID_PARAMETER('newBulkHeaders', 'plain header data objects');
+            }
+            const prototype = Object.getPrototypeOf(source);
+            const descriptors = Object.getOwnPropertyDescriptors(source);
+            if ((prototype !== Object.prototype && prototype !== null) ||
+                Object.values(descriptors).some(descriptor => descriptor.get != null || descriptor.set != null)) {
+                throw new sdk_1.WERR_INVALID_PARAMETER('newBulkHeaders', 'accessor-free plain header data objects');
+            }
+            const value = (name) => {
+                const descriptor = descriptors[name];
+                if (descriptor == null || !('value' in descriptor)) {
+                    throw new sdk_1.WERR_INVALID_PARAMETER('newBulkHeaders', `headers with an own ${name} data property`);
+                }
+                return descriptor.value;
+            };
+            const candidate = {
+                version: value('version'),
+                previousHash: value('previousHash'),
+                merkleRoot: value('merkleRoot'),
+                time: value('time'),
+                bits: value('bits'),
+                nonce: value('nonce'),
+                height: value('height'),
+                hash: value('hash')
+            };
+            (0, blockHeaderUtilities_1.validateHeaderFormat)(candidate);
+            (0, blockHeaderUtilities_1.validateHeaderProofOfWork)(candidate);
+            authenticatedHeaders.push(candidate);
+        }
+        newBulkHeaders = authenticatedHeaders;
         if (newBulkHeaders.length === 0)
             return;
+        if (incrementalChainWork !== undefined && !HEX_32_BYTES.test(incrementalChainWork)) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('incrementalChainWork', 'exactly 32 lowercase hexadecimal bytes');
+        }
         return await this.lock.withWriteLock(async () => {
             const lbf = this.getLastFileNoLock();
             const nextHeight = lbf != null ? lbf.firstHeight + lbf.count : 0;
@@ -253,7 +596,7 @@ class BulkFileDataManager {
                 ? (0, blockHeaderUtilities_1.addWork)(incrementalChainWork, prevChainWork)
                 : computeChainWorkFromHeaders(newBulkHeaders, lbf);
             const data = (0, blockHeaderUtilities_1.serializeBaseBlockHeaders)(newBulkHeaders);
-            const fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(data)), 'base64');
+            const fileHash = (0, utilityHelpers_noBuffer_1.asString)((0, Hash_1.sha256)((0, utilityHelpers_noBuffer_1.asArray)(data)), 'base64');
             const bf = {
                 fileId: undefined,
                 chain: this.chain,
@@ -286,50 +629,78 @@ class BulkFileDataManager {
         });
     }
     async getDataFromFile(file, offset, length) {
-        const bfd = this.getBfdForHeight(file.firstHeight);
-        if (bfd == null || bfd.count < file.count) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('file', `a match for ${file.firstHeight}, ${file.count} in the BulkFileDataManager.`);
+        for (const [name, value] of [
+            ['offset', offset],
+            ['length', length]
+        ]) {
+            if (value !== undefined && (!Number.isSafeInteger(value) || value < 0 || value > MAX_BULK_FILE_HEADERS * 80)) {
+                throw new sdk_1.WERR_INVALID_PARAMETER(name, `an integer from 0 through ${MAX_BULK_FILE_HEADERS * 80}`);
+            }
+        }
+        if (offset !== undefined && length !== undefined && !Number.isSafeInteger(offset + length)) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('offset and length', 'values with a safe sum');
+        }
+        const resolved = await this.lock.withReadLock(async () => {
+            const resolved = this.getBfdForHeight(file.firstHeight);
+            if (resolved == null || resolved.count < file.count) {
+                throw new sdk_1.WERR_INVALID_PARAMETER('file', `a match for ${file.firstHeight}, ${file.count} in the BulkFileDataManager.`);
+            }
+            return { current: resolved, snapshot: snapshotBfd(resolved) };
+        });
+        // The immutable descriptor is snapshotted under the reader lock, but disk,
+        // network, and validation work happens after releasing it. A concurrent
+        // writer can publish a newer descriptor without invalidating this read.
+        return await this.getDataFromSnapshot(resolved.current, resolved.snapshot, offset, length);
+    }
+    async getDataFromSnapshot(original, snapshot, offset, length) {
+        const data = await this.getDataFromFileNoLock(snapshot, offset, length);
+        if (snapshot.data != null) {
+            await this.lock.withWriteLock(async () => {
+                if (this.bfds.includes(original) &&
+                    original.fileHash === snapshot.fileHash &&
+                    original.firstHeight === snapshot.firstHeight &&
+                    original.count === snapshot.count) {
+                    original.data = snapshot.data;
+                    original.validated = true;
+                    original.lastHash = snapshot.lastHash;
+                    original.lastChainWork = snapshot.lastChainWork;
+                    original.mru = Date.now();
+                    this.ensureMaxRetained();
+                }
+            });
         }
-        return await this.lock.withReadLock(async () => await this.getDataFromFileNoLock(bfd, offset, length));
+        return data;
     }
     async getDataFromFileNoLock(bfd, offset, length) {
         const fileLength = bfd.count * 80;
-        offset = offset || 0;
+        offset ??= 0;
         if (offset > fileLength - 1)
             return undefined;
-        length = length || bfd.count * 80 - offset;
+        length ??= bfd.count * 80 - offset;
         length = Math.min(length, fileLength - offset);
-        let data;
-        if (bfd.data != null) {
-            data = bfd.data.slice(offset, offset + length);
-        }
-        else if (bfd.fileId && this.storage != null) {
-            data = await this.storage.getBulkFileData(bfd.fileId, offset, length);
-        }
-        if (data == null) {
-            await this.ensureData(bfd);
-            if (bfd.data != null)
-                data = bfd.data.slice(offset, offset + length);
-        }
-        if (data == null)
-            return undefined;
-        return data;
+        if (length === 0)
+            return new Uint8Array();
+        // Never serve a partial storage read before validating the complete
+        // immutable object. Metadata and advisory validation state may have crossed
+        // a storage boundary, and neither can establish the digest, linkage, or
+        // proof-of-work of an isolated 80-byte slice.
+        const data = await this.ensureData(bfd);
+        return data.slice(offset, offset + length);
     }
     async findHeaderForHeightOrUndefined(height) {
-        return await this.lock.withReadLock(async () => {
-            if (!Number.isInteger(height) || height < 0) {
-                throw new sdk_1.WERR_INVALID_PARAMETER('height', `a non-negative integer (${height}).`);
+        const resolved = await this.lock.withReadLock(async () => {
+            if (!Number.isSafeInteger(height) || height < 0 || height > 0x7fffffff) {
+                throw new sdk_1.WERR_INVALID_PARAMETER('height', `an integer from 0 through 2147483647 (${height}).`);
             }
             const file = this.bfds.find(f => f.firstHeight <= height && f.firstHeight + f.count > height);
             if (file == null)
                 return undefined;
-            const offset = (height - file.firstHeight) * 80;
-            const data = await this.getDataFromFileNoLock(file, offset, 80);
-            if (data == null)
-                return undefined;
-            const header = (0, blockHeaderUtilities_1.deserializeBlockHeader)(data, height, 0);
-            return header;
+            return { current: file, snapshot: snapshotBfd(file), offset: (height - file.firstHeight) * 80 };
         });
+        if (resolved == null)
+            return undefined;
+        const data = await this.getDataFromSnapshot(resolved.current, resolved.snapshot, resolved.offset, 80);
+        return data == null ? undefined : (0, blockHeaderUtilities_1.deserializeBlockHeader)(data, height, 0);
     }
     async getFileForHeight(height) {
         return await this.lock.withReadLock(async () => {
@@ -340,13 +711,16 @@ class BulkFileDataManager {
         });
     }
     getBfdForHeight(height) {
-        if (!Number.isInteger(height) || height < 0) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('height', `a non-negative integer (${height}).`);
+        if (!Number.isSafeInteger(height) || height < 0 || height > 0x7fffffff) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('height', `an integer from 0 through 2147483647 (${height}).`);
         }
         const file = this.bfds.find(f => f.firstHeight <= height && f.firstHeight + f.count > height);
         return file;
     }
     getLastBfd(fromEnd = 1) {
+        if (!Number.isSafeInteger(fromEnd) || fromEnd < 1 || fromEnd > MAX_BULK_MANIFEST_FILES) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('fromEnd', `an integer from 1 through ${MAX_BULK_MANIFEST_FILES}`);
+        }
         if (this.bfds.length < fromEnd)
             return undefined;
         const bfd = this.bfds[this.bfds.length - fromEnd];
@@ -379,11 +753,9 @@ class BulkFileDataManager {
         return data;
     }
     async validateFileInfo(file) {
+        file = normalizeBulkHeaderFileInfo(file, true);
         if (file.chain !== this.chain)
             throw new sdk_1.WERR_INVALID_PARAMETER('chain', `${this.chain}`);
-        if (file.count <= 0) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('bf.count', `expected count to be greater than 0, but got ${file.count}`);
-        }
         if (file.count > this.maxPerFile && file.fileName !== 'incremental') {
             throw new sdk_1.WERR_INVALID_PARAMETER('count', `less than or equal to maxPerFile ${this.maxPerFile}`);
         }
@@ -392,44 +764,51 @@ class BulkFileDataManager {
         if (!file.sourceUrl && !file.fileId && file.data == null) {
             throw new sdk_1.WERR_INVALID_PARAMETER('data', 'defined when sourceUrl and fileId are undefined');
         }
-        const bfd = { ...file, fileHash: file.fileHash, mru: Date.now() };
-        if (!bfd.validated) {
+        // `validated` may have crossed a storage or network boundary. Never trust
+        // that flag. Data supplied with the metadata is verified immediately;
+        // remotely or persistently backed data remains lazy and is verified by
+        // `loadAndValidateData` before its first use.
+        const bfd = {
+            ...file,
+            fileHash: file.fileHash,
+            validated: false,
+            mru: Date.now()
+        };
+        if (bfd.data != null) {
             await this.validateBfdData(bfd, file.fileHash);
             bfd.validated = true;
         }
         return bfd;
     }
     async validateBfdData(bfd, expectedFileHash) {
-        await this.ensureData(bfd);
-        if (bfd.data?.length !== bfd.count * 80) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('file.data', `bulk file ${bfd.fileName} data length ${bfd.data?.length} does not match expected count ${bfd.count}`);
-        }
-        bfd.fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(bfd.data)), 'base64');
-        if (expectedFileHash && expectedFileHash !== bfd.fileHash) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('file.fileHash', `expected ${expectedFileHash} but got ${bfd.fileHash}`);
-        }
-        if (!(0, validBulkHeaderFilesByFileHash_1.isKnownValidBulkHeaderFile)(bfd)) {
-            this.validateBfdHeaders(bfd);
-        }
+        const data = await this.ensureData(bfd);
+        const validated = await this.validateRetrievedData(bfd, data, expectedFileHash);
+        bfd.data = validated;
     }
-    validateBfdHeaders(bfd) {
+    async validateBfdHeaders(bfd, expectedFileHash = bfd.fileHash) {
         const pbf = bfd.firstHeight > 0 ? this.getBfdForHeight(bfd.firstHeight - 1) : undefined;
         const prevHash = pbf?.lastHash ?? '00'.repeat(32);
         const prevChainWork = pbf?.lastChainWork ?? '00'.repeat(32);
-        const { lastHeaderHash, lastChainWork } = (0, blockHeaderUtilities_1.validateBufferOfHeaders)(bfd.data, prevHash, 0, undefined, prevChainWork);
-        if (bfd.lastHash && bfd.lastHash !== lastHeaderHash) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('file.lastHash', `expected ${bfd.lastHash} but got ${lastHeaderHash}`);
-        }
-        if (bfd.lastChainWork && bfd.lastChainWork !== lastChainWork) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('file.lastChainWork', `expected ${bfd.lastChainWork} but got ${lastChainWork}`);
-        }
-        bfd.lastHash = lastHeaderHash;
-        bfd.lastChainWork = lastChainWork;
-        if (bfd.firstHeight === 0)
-            (0, blockHeaderUtilities_1.validateGenesisHeader)(bfd.data, bfd.chain);
+        const result = await this.validator.validate({
+            fileName: bfd.fileName,
+            data: bfd.data,
+            count: bfd.count,
+            fileHash: expectedFileHash,
+            firstHeight: bfd.firstHeight,
+            prevHash,
+            prevChainWork,
+            lastHash: bfd.lastHash,
+            lastChainWork: bfd.lastChainWork,
+            chain: bfd.chain
+        });
+        bfd.data = result.data;
+        bfd.fileHash = result.fileHash;
+        bfd.lastHash = result.lastHeaderHash;
+        bfd.lastChainWork = result.lastChainWork;
+        return result.data;
     }
     async ReValidate() {
-        return await this.lock.withReadLock(async () => await this.ReValidateNoLock());
+        return await this.lock.withWriteLock(async () => await this.ReValidateNoLock());
     }
     async ReValidateNoLock() {
         for (const file of this.bfds) {
@@ -442,6 +821,9 @@ class BulkFileDataManager {
         }
     }
     validateBfdForAdd(bfd) {
+        if (this.fileHashToIndex[bfd.fileHash] !== undefined) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('fileHash', 'unique within bulk storage');
+        }
         if (this.bfds.length === 0 && bfd.firstHeight !== 0) {
             throw new sdk_1.WERR_INVALID_PARAMETER('firstHeight', '0 for the first file');
         }
@@ -457,14 +839,14 @@ class BulkFileDataManager {
     }
     async add(bfd) {
         this.validateBfdForAdd(bfd);
-        const index = this.bfds.length;
-        this.bfds.push(bfd);
-        this.fileHashToIndex[bfd.fileHash] = index;
-        this.ensureMaxRetained();
         const info = bfdToInfo(bfd, true);
         if (this.storage != null) {
             info.fileId = bfd.fileId = await this.storage.insertBulkFile(info);
         }
+        const index = this.bfds.length;
+        this.bfds.push(bfd);
+        this.fileHashToIndex[bfd.fileHash] = index;
+        this.ensureMaxRetained();
         return info;
     }
     replaceBfdAtIndex(index, update) {
@@ -473,6 +855,40 @@ class BulkFileDataManager {
         this.bfds[index] = update;
         this.fileHashToIndex[update.fileHash] = index;
     }
+    setBulkFilesState(files) {
+        const index = Object.create(null);
+        for (let i = 0; i < files.length; i++) {
+            if (index[files[i].fileHash] !== undefined) {
+                throw new sdk_1.WERR_INVALID_OPERATION('bulk-file state contains a duplicate file hash');
+            }
+            index[files[i].fileHash] = i;
+        }
+        this.bfds = files;
+        this.fileHashToIndex = index;
+    }
+    applyCommittedBulkFiles(files) {
+        if (!Array.isArray(files) || files.length !== this.bfds.length) {
+            throw new sdk_1.WERR_INVALID_OPERATION('storage returned an incomplete atomic bulk-file replacement');
+        }
+        const ids = new Set();
+        for (let index = 0; index < files.length; index++) {
+            if (!Object.hasOwn(files, index)) {
+                throw new sdk_1.WERR_INVALID_OPERATION('storage returned a sparse atomic bulk-file replacement');
+            }
+            const committed = normalizeBulkHeaderFileInfo(files[index], true);
+            const expected = this.bfds[index];
+            if (committed.fileId === undefined ||
+                ids.has(committed.fileId) ||
+                committed.chain !== expected.chain ||
+                committed.firstHeight !== expected.firstHeight ||
+                committed.count !== expected.count ||
+                committed.fileHash !== expected.fileHash) {
+                throw new sdk_1.WERR_INVALID_OPERATION('storage returned a mismatched atomic bulk-file replacement');
+            }
+            ids.add(committed.fileId);
+            expected.fileId = committed.fileId;
+        }
+    }
     /**
      * Updating an existing file occurs in two specific contexts:
      *
@@ -508,15 +924,31 @@ class BulkFileDataManager {
         if (isBdfCdn(update) === isBdfCdn(hbf) && update.count <= hbf.count) {
             throw new sdk_1.WERR_INVALID_PARAMETER('file.count', `greater than the current count ${hbf.count}`);
         }
+        if (update.fileId !== undefined && hbf.fileId !== undefined && update.fileId !== hbf.fileId) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('file.fileId', 'undefined or equal to the existing bulk file id');
+        }
+        if (update.fileId === undefined)
+            update.fileId = hbf.fileId;
+        const stateBefore = this.bfds.map(snapshotBfd);
         const { index, truncate, replaced, drop } = await this.resolveUpdatePlan(update, hbf);
-        this.replaceBfdAtIndex(index, update);
-        if (truncate != null)
-            await this.shiftWork(update, truncate, replaced);
-        if (drop != null)
-            this.dropLastBulkFile(drop);
-        await this.persistUpdate(update, truncate, replaced, drop);
-        this.recordUpdateResults(r, update, truncate, replaced, drop);
-        this.ensureMaxRetained();
+        const requiresAtomicReplacement = truncate != null || drop != null;
+        if (this.storage != null && requiresAtomicReplacement && typeof this.storage.replaceBulkFiles !== 'function') {
+            throw new sdk_1.WERR_INVALID_OPERATION('storage must implement atomic replaceBulkFiles for a multi-file bulk update');
+        }
+        try {
+            this.replaceBfdAtIndex(index, update);
+            if (truncate != null)
+                await this.shiftWork(update, truncate, replaced);
+            if (drop != null)
+                this.dropLastBulkFile(drop);
+            await this.persistUpdate(update, truncate, drop);
+            this.recordUpdateResults(r, update, truncate, replaced, drop);
+            this.ensureMaxRetained();
+        }
+        catch (error) {
+            this.setBulkFilesState(stateBefore);
+            throw error;
+        }
     }
     async resolveUpdatePlan(update, hbf) {
         const lbf = this.getLastBfd();
@@ -568,23 +1000,19 @@ class BulkFileDataManager {
             replaced: penultimateFile
         };
     }
-    async persistUpdate(update, truncate, replaced, drop) {
+    async persistUpdate(update, truncate, drop) {
         if (this.storage == null)
             return;
-        if (update.fileId)
-            await this.storage.updateBulkFile(update.fileId, bfdToInfo(update, true));
-        if (truncate != null) {
-            const truncateInfo = bfdToInfo(truncate, true);
-            if (replaced != null) {
-                await this.storage.updateBulkFile(truncate.fileId, truncateInfo);
-            }
-            else {
-                truncateInfo.fileId = undefined;
-                truncate.fileId = await this.storage.insertBulkFile(truncateInfo);
-            }
+        if (truncate != null || drop != null) {
+            const committed = await this.storage.replaceBulkFiles(this.bfds.map(bfd => bfdToInfo(bfd, true)));
+            this.applyCommittedBulkFiles(committed);
+            return;
         }
-        if (drop?.fileId)
-            await this.storage.deleteBulkFile(drop.fileId);
+        if (!update.fileId)
+            throw new sdk_1.WERR_INVALID_OPERATION('stored bulk-file update is missing its fileId');
+        const affected = await this.storage.updateBulkFile(update.fileId, bfdToInfo(update, true));
+        if (affected !== 1)
+            throw new sdk_1.WERR_INVALID_OPERATION(`failed to update bulk file ${update.fileId}`);
     }
     recordUpdateResults(r, update, truncate, replaced, drop) {
         if (!r)
@@ -644,7 +1072,7 @@ class BulkFileDataManager {
         truncate.firstHeight += count;
         truncate.data = truncate.data?.slice(count * 80);
         delete this.fileHashToIndex[truncate.fileHash];
-        truncate.fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(truncate.data)), 'base64');
+        truncate.fileHash = (0, utilityHelpers_noBuffer_1.asString)((0, Hash_1.sha256)((0, utilityHelpers_noBuffer_1.asArray)(truncate.data)), 'base64');
         this.fileHashToIndex[truncate.fileHash] = updateIndex + 1;
     }
     /**
@@ -653,36 +1081,126 @@ class BulkFileDataManager {
      * @returns
      */
     async ensureData(bfd) {
-        if (bfd.data != null)
+        if (bfd.data != null) {
+            this.stats.memoryHits++;
+            bfd.mru = Date.now();
             return bfd.data;
-        if (this.storage != null && bfd.fileId) {
-            bfd.data = await this.storage.getBulkFileData(bfd.fileId);
-            if (bfd.data == null)
-                throw new sdk_1.WERR_INVALID_PARAMETER('fileId', `valid, data not found for fileId ${bfd.fileId}`);
         }
-        if (bfd.data == null && this.fetch != null && bfd.sourceUrl) {
-            const url = this.fetch.pathJoin(bfd.sourceUrl, bfd.fileName);
-            try {
-                bfd.data = await this.fetch.download(url);
-            }
-            catch (firstAttemptErr) {
-                // First download attempt failed (e.g. transient network error); retry once.
-                console.debug(`BulkFileDataManager: first download attempt failed for ${url}, retrying`, firstAttemptErr);
-                bfd.data = await this.fetch.download(url);
+        const key = `${bfd.fileHash}:${bfd.fileName}`;
+        const existing = this.inFlightLoads.get(key);
+        if (existing != null) {
+            this.stats.coalescedLoads++;
+            const data = await existing;
+            bfd.data = data;
+            bfd.mru = Date.now();
+            this.ensureMaxRetained();
+            return data;
+        }
+        const failed = this.failedLoads.get(key);
+        if (failed != null) {
+            if (Date.now() < failed.retryAt) {
+                this.stats.loadBackoffs++;
+                throw failed.error;
             }
-            if (!bfd.data)
-                throw new sdk_1.WERR_INVALID_PARAMETER('sourceUrl', `data not found for sourceUrl ${url}`);
+            this.failedLoads.delete(key);
         }
-        if (bfd.data == null)
-            throw new sdk_1.WERR_INVALID_PARAMETER('data', `defined. Unable to retrieve data for ${bfd.fileName}`);
-        bfd.mru = Date.now();
-        // Validate retrieved data.
-        const fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(bfd.data)), 'base64');
-        if (fileHash !== bfd.fileHash) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('fileHash', `a match for retrieved data for ${bfd.fileName}`);
+        const load = this.loadAndValidateData(bfd);
+        this.inFlightLoads.set(key, load);
+        try {
+            const data = await load;
+            bfd.data = data;
+            bfd.validated = true;
+            this.failedLoads.delete(key);
+            bfd.mru = Date.now();
+            this.ensureMaxRetained();
+            return data;
         }
-        this.ensureMaxRetained();
-        return bfd.data;
+        catch (error) {
+            const resolved = error instanceof Error ? error : new Error(String(error));
+            this.failedLoads.set(key, { retryAt: Date.now() + this.failedLoadRetryMsecs, error: resolved });
+            throw resolved;
+        }
+        finally {
+            if (this.inFlightLoads.get(key) === load)
+                this.inFlightLoads.delete(key);
+        }
+    }
+    async loadAndValidateData(bfd) {
+        const stored = await this.loadFromStorage(bfd);
+        if (stored != null)
+            return stored;
+        const cached = await this.loadFromCache(bfd);
+        if (cached != null)
+            return cached;
+        const downloaded = await this.loadFromRemote(bfd);
+        if (downloaded != null)
+            return downloaded;
+        throw new sdk_1.WERR_INVALID_PARAMETER('data', `defined. Unable to retrieve data for ${bfd.fileName}`);
+    }
+    async loadFromStorage(bfd) {
+        if (this.storage == null || !bfd.fileId)
+            return undefined;
+        const stored = await this.storage.getBulkFileData(bfd.fileId);
+        if (stored == null) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('fileId', `valid, data not found for fileId ${bfd.fileId}`);
+        }
+        const validated = await this.validateRetrievedData(bfd, stored);
+        this.stats.storageHits++;
+        return validated;
+    }
+    async loadFromCache(bfd) {
+        if (this.cache == null)
+            return undefined;
+        const cached = await this.cache.get(bfd);
+        if (cached == null) {
+            this.stats.persistentCacheMisses++;
+            return undefined;
+        }
+        try {
+            const validated = await this.validateRetrievedData(bfd, cached);
+            this.stats.persistentCacheHits++;
+            await this.cache.promoteValidated?.(bfd, validated);
+            return validated;
+        }
+        catch (error) {
+            // Only deterministic rejection of these exact bytes may quarantine an
+            // immutable cache object. Operational validation failures preserve it.
+            if (!(error instanceof BulkFileDataValidatorApi_1.BulkFileDataValidationError))
+                throw error;
+            this.stats.persistentCacheRejects++;
+            let rejectedData = error.data;
+            if (!(rejectedData instanceof Uint8Array) && cached.byteLength > 0)
+                rejectedData = cached;
+            await this.cache.quarantine?.(bfd, String(error), rejectedData);
+            this.log(`Rejected corrupt bulk-header cache entry ${bfd.fileName}: ${String(error)}`);
+            return undefined;
+        }
+    }
+    async loadFromRemote(bfd) {
+        if (this.fetch == null || !bfd.sourceUrl)
+            return undefined;
+        const expectedBytes = bfd.count * 80;
+        await this.downloadBudget?.consume(expectedBytes);
+        const url = this.fetch.pathJoin(bfd.sourceUrl, bfd.fileName);
+        const downloaded = await this.fetch.download(url, expectedBytes, {
+            publicNetworkOnly: true,
+            beforeRetry: async () => await this.downloadBudget?.consume(expectedBytes)
+        });
+        if (downloaded == null) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('sourceUrl', `data not found for sourceUrl ${url}`);
+        }
+        const validated = await this.validateRetrievedData(bfd, downloaded);
+        this.stats.downloads++;
+        this.stats.downloadedBytes += validated.length;
+        await this.cache?.set(bfd, validated);
+        return validated;
+    }
+    async validateRetrievedData(bfd, data, expectedFileHash = bfd.fileHash) {
+        const candidate = { ...bfd, data };
+        const validated = await this.validateBfdHeaders(candidate, expectedFileHash);
+        bfd.lastHash = candidate.lastHash;
+        bfd.lastChainWork = candidate.lastChainWork;
+        return validated;
     }
     ensureMaxRetained() {
         if (this.maxRetained === undefined)
@@ -700,6 +1218,19 @@ class BulkFileDataManager {
         }
     }
     async exportHeadersToFs(toFs, toHeadersPerFile, toFolder, sourceUrl, maxHeight) {
+        if (!Number.isSafeInteger(toHeadersPerFile) || toHeadersPerFile < 1 || toHeadersPerFile > MAX_BULK_FILE_HEADERS) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('toHeadersPerFile', `an integer from 1 through ${MAX_BULK_FILE_HEADERS}`);
+        }
+        if (maxHeight !== undefined && (!Number.isSafeInteger(maxHeight) || maxHeight < 0 || maxHeight > 0x7fffffff)) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('maxHeight', 'an integer from 0 through 2147483647 when defined');
+        }
+        if (typeof toFolder !== 'string' || toFolder.trim() === '' || /[\r\n]/.test(toFolder)) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('toFolder', 'a non-empty path without control lines');
+        }
+        if (sourceUrl !== undefined &&
+            (typeof sourceUrl !== 'string' || sourceUrl.length > 2048 || /[\r\n]/.test(sourceUrl))) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('sourceUrl', 'a bounded string without control lines when defined');
+        }
         const chain = this.chain;
         const toFileName = (i) => `${chain}Net_${i}.headers`;
         const toPath = (i) => toFs.pathJoin(toFolder, toFileName(i));
@@ -711,7 +1242,7 @@ class BulkFileDataManager {
             files: []
         };
         let range = await this.getHeightRange();
-        if (maxHeight)
+        if (maxHeight !== undefined)
             range = range.intersect(new HeightRange_1.HeightRange(0, maxHeight));
         const reader = await this.createReader(range, toHeadersPerFile * 80);
         let firstHeight = 0;
@@ -724,17 +1255,24 @@ class BulkFileDataManager {
             if (data == null || data.length === 0) {
                 break;
             }
-            const last = (0, blockHeaderUtilities_1.validateBufferOfHeaders)(data, lastHeaderHash, 0, undefined, lastChainWork);
-            await toFs.writeFile(toPath(i), data);
-            const fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(data)), 'base64');
+            const validated = await this.validator.validate({
+                fileName: toFileName(i),
+                data,
+                count: data.length / 80,
+                firstHeight,
+                prevHash: lastHeaderHash,
+                prevChainWork: lastChainWork,
+                chain
+            });
+            await toFs.writeFile(toPath(i), validated.data);
             const file = {
                 chain,
-                count: data.length / 80,
-                fileHash,
+                count: validated.data.length / 80,
+                fileHash: validated.fileHash,
                 fileName: toFileName(i),
                 firstHeight,
-                lastChainWork: last.lastChainWork,
-                lastHash: last.lastHeaderHash,
+                lastChainWork: validated.lastChainWork,
+                lastHash: validated.lastHeaderHash,
                 prevChainWork: lastChainWork,
                 prevHash: lastHeaderHash,
                 sourceUrl
@@ -746,15 +1284,44 @@ class BulkFileDataManager {
         }
         await toFs.writeFile(toJsonPath(), (0, utilityHelpers_noBuffer_1.asUint8Array)(JSON.stringify(toBulkFiles), 'utf8'));
     }
+    async destroy() {
+        await this.validator.destroy?.();
+    }
 }
 exports.BulkFileDataManager = BulkFileDataManager;
+function snapshotBfd(file) {
+    return { ...file, data: file.data };
+}
 function selectBulkHeaderFiles(files, chain, maxPerFile) {
+    if (!SUPPORTED_CHAINS.has(chain))
+        throw new sdk_1.WERR_INVALID_PARAMETER('chain', 'a supported Chaintracks network');
+    if (!Number.isSafeInteger(maxPerFile) || maxPerFile < 1 || maxPerFile > MAX_BULK_FILE_HEADERS) {
+        throw new sdk_1.WERR_INVALID_PARAMETER('maxPerFile', `an integer from 1 through ${MAX_BULK_FILE_HEADERS}`);
+    }
+    if (!Array.isArray(files) || files.length > MAX_BULK_MANIFEST_FILES) {
+        throw new sdk_1.WERR_INVALID_PARAMETER('files', `a dense array of no more than ${MAX_BULK_MANIFEST_FILES} entries`);
+    }
+    const bestByHeight = new Map();
+    for (let index = 0; index < files.length; index++) {
+        const descriptor = Object.getOwnPropertyDescriptor(files, String(index));
+        if (descriptor == null || !('value' in descriptor)) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('files', 'a dense array of plain data entries');
+        }
+        const file = normalizeBulkHeaderFileInfo(descriptor.value);
+        if (file.chain !== chain || file.count > maxPerFile)
+            continue;
+        const current = bestByHeight.get(file.firstHeight);
+        if (current == null || current.count < file.count) {
+            bestByHeight.set(file.firstHeight, file);
+        }
+        else if (current.count === file.count && current.fileHash !== file.fileHash) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('files', 'unambiguous for each chain, first height, and count');
+        }
+    }
     const r = [];
     let height = 0;
-    for (;;) {
-        const choices = files.filter(f => f.firstHeight === height && f.count <= maxPerFile && f.chain === chain);
-        // Pick the file with the maximum count
-        const choice = choices.reduce((a, b) => (a.count > b.count ? a : b), choices[0]);
+    while (r.length <= bestByHeight.size) {
+        const choice = bestByHeight.get(height);
         if (!choice)
             break; // no more files to select
         r.push(choice);
@@ -782,7 +1349,7 @@ function bfdToInfo(bfd, keepData) {
         prevHash: bfd.prevHash,
         lastHash: bfd.lastHash,
         validated: bfd.validated || false,
-        data: keepData ? bfd.data : undefined
+        data: keepData && bfd.data != null ? Uint8Array.from(bfd.data) : undefined
     };
 }
 /**