@bsv/wallet-toolbox 2.8.0 → 2.10.0

package/out/src/services/chaintracker/chaintracks/util/BulkFileDataCacheFs.js

@@ -48,29 +48,70 @@ const path = __importStar(require("node:path"));
  */
 class BulkFileDataCacheFs {
     rootFolder;
-    constructor(rootFolder) {
-        this.rootFolder = rootFolder;
+    legacyRoots;
+    constructor(rootFolderOrOptions) {
+        const options = typeof rootFolderOrOptions === 'string' ? { rootFolder: rootFolderOrOptions } : rootFolderOrOptions;
+        this.rootFolder = options.rootFolder;
+        this.legacyRoots = options.legacyRoots ?? [];
     }
-    filePath(file) {
+    legacyFilePath(root, file) {
+        this.validateFileName(file);
+        return path.join(root, file.fileName);
+    }
+    validateFileName(file) {
         if (path.basename(file.fileName) !== file.fileName || file.fileName === '.' || file.fileName === '..') {
             throw new Error(`Invalid bulk-header cache file name: ${file.fileName}`);
         }
-        return path.join(this.rootFolder, file.fileName);
+    }
+    digestHex(file) {
+        if (file.fileHash == null)
+            throw new Error(`Missing bulk-header digest for ${file.fileName}`);
+        const digest = Buffer.from(file.fileHash, 'base64');
+        if (digest.length !== 32 || digest.toString('base64') !== file.fileHash) {
+            throw new Error(`Invalid bulk-header digest for ${file.fileName}`);
+        }
+        return digest.toString('hex');
+    }
+    objectPath(file) {
+        this.validateFileName(file);
+        const digest = this.digestHex(file);
+        return path.join(this.rootFolder, 'objects', digest.slice(0, 2), `${digest}.headers`);
+    }
+    legacyRejectionMarker(file) {
+        return path.join(this.rootFolder, 'quarantine', `${this.digestHex(file)}.legacy-rejected.json`);
     }
     async get(file) {
         try {
-            return new Uint8Array(await node_fs_1.promises.readFile(this.filePath(file)));
+            return new Uint8Array(await node_fs_1.promises.readFile(this.objectPath(file)));
         }
         catch (error) {
-            if (error.code === 'ENOENT')
-                return undefined;
-            throw error;
+            if (error.code !== 'ENOENT')
+                throw error;
+        }
+        try {
+            await node_fs_1.promises.access(this.legacyRejectionMarker(file));
+            return undefined;
         }
+        catch (error) {
+            if (error.code !== 'ENOENT')
+                throw error;
+        }
+        for (const root of this.legacyRoots) {
+            try {
+                return new Uint8Array(await node_fs_1.promises.readFile(this.legacyFilePath(root, file)));
+            }
+            catch (error) {
+                if (error.code !== 'ENOENT')
+                    throw error;
+            }
+        }
+        return undefined;
     }
     async set(file, data) {
-        const destination = this.filePath(file);
-        await node_fs_1.promises.mkdir(this.rootFolder, { recursive: true });
-        const temporary = path.join(this.rootFolder, `.${file.fileName}.${process.pid}.${(0, node_crypto_1.randomUUID)()}.tmp`);
+        const destination = this.objectPath(file);
+        const folder = path.dirname(destination);
+        await node_fs_1.promises.mkdir(folder, { recursive: true });
+        const temporary = path.join(folder, `.${path.basename(destination)}.${process.pid}.${(0, node_crypto_1.randomUUID)()}.tmp`);
         let handle;
         try {
             handle = await node_fs_1.promises.open(temporary, 'wx', 0o600);
@@ -79,6 +120,7 @@ class BulkFileDataCacheFs {
             await handle.close();
             handle = undefined;
             await node_fs_1.promises.rename(temporary, destination);
+            await syncDirectory(folder);
         }
         finally {
             await handle?.close().catch(() => undefined);
@@ -88,15 +130,75 @@ class BulkFileDataCacheFs {
             });
         }
     }
-    async delete(file) {
+    async promoteValidated(file, data) {
         try {
-            await node_fs_1.promises.unlink(this.filePath(file));
+            await node_fs_1.promises.access(this.objectPath(file));
         }
         catch (error) {
             if (error.code !== 'ENOENT')
                 throw error;
+            await this.set(file, data);
         }
     }
+    async quarantine(file, reason, rejectedData) {
+        const source = this.objectPath(file);
+        const quarantineFolder = path.join(this.rootFolder, 'quarantine');
+        const destination = path.join(quarantineFolder, `${this.digestHex(file)}.${Date.now()}.${(0, node_crypto_1.randomUUID)()}.invalid`);
+        await node_fs_1.promises.mkdir(quarantineFolder, { recursive: true });
+        try {
+            const candidate = await node_fs_1.promises.readFile(source);
+            if (rejectedData != null && !candidate.equals(Buffer.from(rejectedData)))
+                return;
+            await node_fs_1.promises.rename(source, destination);
+            await syncDirectory(path.dirname(source));
+            await syncDirectory(quarantineFolder);
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT')
+                throw error;
+            if (this.legacyRoots.length === 0)
+                return;
+            // Legacy entries are intentionally read-only. Once replacement succeeds,
+            // the content-addressed object takes precedence without destroying the
+            // old deployment's only retained bytes.
+            await writeAtomicFile(this.legacyRejectionMarker(file), Buffer.from(JSON.stringify({ fileName: file.fileName, fileHash: file.fileHash, reason })));
+        }
+    }
+    async delete(file) {
+        await this.quarantine(file, 'delete requested through legacy cache contract');
+    }
 }
 exports.BulkFileDataCacheFs = BulkFileDataCacheFs;
+async function writeAtomicFile(destination, data) {
+    const folder = path.dirname(destination);
+    await node_fs_1.promises.mkdir(folder, { recursive: true });
+    const temporary = path.join(folder, `.${path.basename(destination)}.${process.pid}.${(0, node_crypto_1.randomUUID)()}.tmp`);
+    let handle;
+    try {
+        handle = await node_fs_1.promises.open(temporary, 'wx', 0o600);
+        await handle.writeFile(data);
+        await handle.sync();
+        await handle.close();
+        handle = undefined;
+        await node_fs_1.promises.rename(temporary, destination);
+        await syncDirectory(folder);
+    }
+    finally {
+        await handle?.close().catch(() => undefined);
+        await node_fs_1.promises.unlink(temporary).catch(error => {
+            if (error.code !== 'ENOENT')
+                throw error;
+        });
+    }
+}
+async function syncDirectory(folder) {
+    let handle;
+    try {
+        handle = await node_fs_1.promises.open(folder, 'r');
+        await handle.sync();
+    }
+    finally {
+        await handle?.close();
+    }
+}
 //# sourceMappingURL=BulkFileDataCacheFs.js.map

package/out/src/services/chaintracker/chaintracks/util/BulkFileDataCacheFs.js.map

@@ -1 +1 @@
-{"version":3,"file":"BulkFileDataCacheFs.js","sourceRoot":"","sources":["../../../../../../src/services/chaintracker/chaintracks/util/BulkFileDataCacheFs.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAwC;AACxC,6CAAwC;AACxC,MAAY,IAAI,sCAAiB;AAIjC;;;;;;;;GAQG;AACH;IAC+B,UAAU;IAAvC,YAA6B,UAAkB;0BAAlB,UAAU;IAAW,CAAC;IAE3C,QAAQ,CAAC,IAAkC;QACjD,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;YACtG,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAA;QAC1E,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IAClD,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAkC;QAC1C,IAAI,CAAC;YACH,OAAO,IAAI,UAAU,CAAC,MAAM,kBAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC/D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,OAAO,SAAS,CAAA;YACxE,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAkC,EAAE,IAAgB;QAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACvC,MAAM,kBAAE,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACpD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,IAAI,IAAA,wBAAU,GAAE,MAAM,CAAC,CAAA;QACpG,IAAI,MAAiC,CAAA;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,CAAA;YAC9C,MAAM,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YAC5B,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YACpB,MAAM,GAAG,SAAS,CAAA;YAClB,MAAM,kBAAE,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;QACzC,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;YAC5C,MAAM,kBAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACvC,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;oBAAE,MAAM,KAAK,CAAA;YACrE,CAAC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAkC;QAC7C,IAAI,CAAC;YACH,MAAM,kBAAE,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAA;QACrE,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"BulkFileDataCacheFs.js","sourceRoot":"","sources":["../../../../../../src/services/chaintracker/chaintracks/util/BulkFileDataCacheFs.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAwC;AACxC,6CAAwC;AACxC,MAAY,IAAI,sCAAiB;AAWjC;;;;;;;;GAQG;AACH;IACmB,UAAU,CAAQ;IAClB,WAAW,CAAU;IAEtC,YAAY,mBAAwD;QAClE,MAAM,OAAO,GAAG,OAAO,mBAAmB,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,mBAAmB,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAA;QACnH,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAA;QACpC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,EAAE,CAAA;IAC9C,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,IAAkC;QACrE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAA;QAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IACvC,CAAC;IAEO,gBAAgB,CAAC,IAAkC;QACzD,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;YACtG,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAA;QAC1E,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,IAAkC;QAClD,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAA;QAC7F,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;QACnD,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;YACxE,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAA;QACpE,CAAC;QACD,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IAC/B,CAAC;IAEO,UAAU,CAAC,IAAkC;QACnD,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAA;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QACnC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,MAAM,UAAU,CAAC,CAAA;IACvF,CAAC;IAEO,qBAAqB,CAAC,IAAkC;QAC9D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAA;IACjG,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAkC;QAC1C,IAAI,CAAC;YACH,OAAO,IAAI,UAAU,CAAC,MAAM,kBAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAA;QACrE,CAAC;QACD,IAAI,CAAC;YACH,MAAM,kBAAE,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAA;YACjD,OAAO,SAAS,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAA;QACrE,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,OAAO,IAAI,UAAU,CAAC,MAAM,kBAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAC3E,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;oBAAE,MAAM,KAAK,CAAA;YACrE,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAkC,EAAE,IAAgB;QAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;QACxC,MAAM,kBAAE,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,GAAG,IAAI,IAAA,wBAAU,GAAE,MAAM,CAAC,CAAA;QACxG,IAAI,MAAiC,CAAA;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,CAAA;YAC9C,MAAM,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YAC5B,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YACpB,MAAM,GAAG,SAAS,CAAA;YAClB,MAAM,kBAAE,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;YACvC,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;QAC7B,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;YAC5C,MAAM,kBAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACvC,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;oBAAE,MAAM,KAAK,CAAA;YACrE,CAAC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAkC,EAAE,IAAgB;QACzE,IAAI,CAAC;YACH,MAAM,kBAAE,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAA;QACxC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAA;YACnE,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QAC5B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAkC,EAAE,MAAc,EAAE,YAAyB;QAC5F,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QACpC,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;QACjE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,IAAA,wBAAU,GAAE,UAAU,CAAC,CAAA;QAChH,MAAM,kBAAE,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACrD,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,kBAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;YAC3C,IAAI,YAAY,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAAE,OAAM;YAChF,MAAM,kBAAE,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;YACpC,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAA;YACzC,MAAM,aAAa,CAAC,gBAAgB,CAAC,CAAA;QACvC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAA;YACnE,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAM;YACzC,yEAAyE;YACzE,uEAAuE;YACvE,wCAAwC;YACxC,MAAM,eAAe,CACnB,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAChC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAC1F,CAAA;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAkC;QAC7C,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,gDAAgD,CAAC,CAAA;IAC/E,CAAC;CACF;;AAED,KAAK,UAAU,eAAe,CAAC,WAAmB,EAAE,IAAgB;IAClE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IACxC,MAAM,kBAAE,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,GAAG,IAAI,IAAA,wBAAU,GAAE,MAAM,CAAC,CAAA;IACxG,IAAI,MAAiC,CAAA;IACrC,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,CAAA;QAC9C,MAAM,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QAC5B,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;QACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;QACpB,MAAM,GAAG,SAAS,CAAA;QAClB,MAAM,kBAAE,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;QACvC,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IAC7B,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;QAC5C,MAAM,kBAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YACvC,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAA;QACrE,CAAC,CAAC,CAAA;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,MAAc;IACzC,IAAI,MAAiC,CAAA;IACrC,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QACnC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;IACrB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,EAAE,KAAK,EAAE,CAAA;IACvB,CAAC;AACH,CAAC"}

package/out/src/services/chaintracker/chaintracks/util/BulkFileDataManager.d.ts

@@ -5,7 +5,8 @@ import { BulkHeaderFileInfo } from './BulkHeaderFile';
 import { HeightRange } from './HeightRange';
 import { ChaintracksStorageBulkFileApi } from '../Api/ChaintracksStorageApi';
 import { ChaintracksFsApi } from '../Api/ChaintracksFsApi';
-import type { BulkFileDataCacheApi, BulkFileDownloadBudgetApi } from '../Api/BulkFileDataCacheApi';
+import type { BulkFileDataCacheApi, BulkFileDownloadBudgetApi, BulkFileDownloadBudgetSnapshot } from '../Api/BulkFileDataCacheApi';
+import { type BulkFileDataValidatorApi, type BulkFileDataValidatorStats } from '../Api/BulkFileDataValidatorApi';
 export interface BulkFileDataManagerOptions {
     chain: Chain;
     maxPerFile: number;
@@ -16,6 +17,10 @@ export interface BulkFileDataManagerOptions {
     cache?: BulkFileDataCacheApi;
     /** Optional process-local or shared reservation budget for remote bytes. */
     downloadBudget?: BulkFileDownloadBudgetApi;
+    /** Complete-object validator. Node services should inject a worker-backed implementation. */
+    validator?: BulkFileDataValidatorApi;
+    /** Cooldown after a failed load before the same immutable object can retry. */
+    failedLoadRetryMsecs?: number;
 }
 /** @public */
 export interface BulkFileDataManagerStats {
@@ -27,6 +32,9 @@ export interface BulkFileDataManagerStats {
     coalescedLoads: number;
     downloads: number;
     downloadedBytes: number;
+    loadBackoffs: number;
+    validation?: BulkFileDataValidatorStats;
+    downloadBudget?: BulkFileDownloadBudgetSnapshot;
 }
 /**
  * Manages bulk file data (typically 8MB chunks of 100,000 headers each).
@@ -44,6 +52,7 @@ export declare class BulkFileDataManager {
     private fileHashToIndex;
     private readonly lock;
     private readonly inFlightLoads;
+    private readonly failedLoads;
     private storage?;
     private readonly stats;
     readonly chain: Chain;
@@ -53,6 +62,8 @@ export declare class BulkFileDataManager {
     readonly fromKnownSourceUrl?: string;
     readonly cache?: BulkFileDataCacheApi;
     readonly downloadBudget?: BulkFileDownloadBudgetApi;
+    readonly validator: BulkFileDataValidatorApi;
+    readonly failedLoadRetryMsecs: number;
     constructor(options: BulkFileDataManagerOptions | Chain);
     getStats(): BulkFileDataManagerStats;
     deleteBulkFiles(): Promise<void>;
@@ -81,6 +92,7 @@ export declare class BulkFileDataManager {
     getBulkFiles(keepData?: boolean): Promise<BulkHeaderFileInfo[]>;
     getHeightRange(): Promise<HeightRange>;
     getDataFromFile(file: BulkHeaderFileInfo, offset?: number, length?: number): Promise<Uint8Array | undefined>;
+    private getDataFromSnapshot;
     private getDataFromFileNoLock;
     findHeaderForHeightOrUndefined(height: number): Promise<BlockHeader | undefined>;
     getFileForHeight(height: number): Promise<BulkHeaderFileInfo | undefined>;
@@ -148,9 +160,13 @@ export declare class BulkFileDataManager {
      */
     private ensureData;
     private loadAndValidateData;
+    private loadFromStorage;
+    private loadFromCache;
+    private loadFromRemote;
     private validateRetrievedData;
     private ensureMaxRetained;
     exportHeadersToFs(toFs: ChaintracksFsApi, toHeadersPerFile: number, toFolder: string, sourceUrl?: string, maxHeight?: number): Promise<void>;
+    destroy(): Promise<void>;
 }
 interface BulkFileData extends BulkHeaderFileInfo {
     mru: number;

package/out/src/services/chaintracker/chaintracks/util/BulkFileDataManager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"BulkFileDataManager.d.ts","sourceRoot":"","sources":["../../../../../../src/services/chaintracker/chaintracks/util/BulkFileDataManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAiE,MAAM,iBAAiB,CAAA;AAGnH,OAAO,EAAE,kBAAkB,EAAuB,MAAM,kBAAkB,CAAA;AAE1E,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAU3C,OAAO,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAA;AAE5E,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAE1D,OAAO,KAAK,EAAE,oBAAoB,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAA;AAElG,MAAM,WAAW,0BAA0B;IACzC,KAAK,EAAE,KAAK,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,mBAAmB,CAAA;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,uEAAuE;IACvE,KAAK,CAAC,EAAE,oBAAoB,CAAA;IAC5B,4EAA4E;IAC5E,cAAc,CAAC,EAAE,yBAAyB,CAAA;CAC3C;AAED,cAAc;AACd,MAAM,WAAW,wBAAwB;IACvC,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,qBAAqB,EAAE,MAAM,CAAA;IAC7B,sBAAsB,EAAE,MAAM,CAAA;IAC9B,cAAc,EAAE,MAAM,CAAA;IACtB,SAAS,EAAE,MAAM,CAAA;IACjB,eAAe,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;;GAQG;AACH,qBAAa,mBAAmB;IAC9B,MAAM,CAAC,oBAAoB,CAAC,KAAK,EAAE,KAAK,GAAG,0BAA0B,CAQpE;IAED,OAAO,CAAC,GAAG,CAAqC;IAEhD,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,eAAe,CAA6B;IACpD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiE;IACtF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyC;IACvE,OAAO,CAAC,OAAO,CAAC,CAA+B;IAC/C,OAAO,CAAC,QAAQ,CAAC,KAAK,CASrB;IAED,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAA;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,mBAAmB,CAAA;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;IACpC,QAAQ,CAAC,KAAK,CAAC,EAAE,oBAAoB,CAAA;IACrC,QAAQ,CAAC,cAAc,CAAC,EAAE,yBAAyB,CAAA;IAEnD,YAAY,OAAO,EAAE,0BAA0B,GAAG,KAAK,EAWtD;IAED,QAAQ,IAAI,wBAAwB,CAEnC;IAEK,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAErC;IAED,OAAO,CAAC,qBAAqB;IAc7B;;;;;;OAMG;IACG,UAAU,CAAC,OAAO,EAAE,6BAA6B,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAErG;YAEa,gBAAgB;IAiC9B,yBAAyB,CAAC,KAAK,EAAE,kBAAkB,EAAE,GAAG;QACtD,GAAG,EAAE,WAAW,CAAA;QAChB,GAAG,EAAE,WAAW,CAAA;QAChB,WAAW,EAAE,WAAW,CAAA;KACzB,CASA;IAEK,YAAY,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAI3F;IAEK,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCjD;IAEK,KAAK,CAAC,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,8BAA8B,CAAC,CAEhF;YAEa,WAAW;YAoCX,gBAAgB;IAoB9B,WAAW,CAAC,IAAI,CAAC,EAAE,8BAA8B,GAAG,YAAY,EAAE,GAAG,kBAAkB,EAAE,GAAG,MAAM,CA2BjG;IAEK,4BAA4B,CAAC,cAAc,EAAE,WAAW,EAAE,EAAE,oBAAoB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA8C9G;IAEK,YAAY,CAAC,QAAQ,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAIpE;IAEK,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,CAO3C;IAEK,eAAe,CAAC,IAAI,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC,CASjH;YAEa,qBAAqB;IAkB7B,8BAA8B,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAarF;IAEK,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAM9E;IAED,OAAO,CAAC,eAAe;IAQvB,OAAO,CAAC,UAAU;IAMZ,WAAW,CAAC,OAAO,SAAI,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAEtE;IAED,OAAO,CAAC,iBAAiB;YAMX,iBAAiB;YAUjB,eAAe;YASf,gBAAgB;YA+BhB,eAAe;IAkB7B,OAAO,CAAC,kBAAkB;IAoBpB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAEhC;YAEa,gBAAgB;IAU9B,OAAO,CAAC,iBAAiB;YAeX,GAAG;IAajB,OAAO,CAAC,iBAAiB;IAOzB;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;YACW,MAAM;YAuBN,iBAAiB;YAcjB,qBAAqB;YA4BrB,4BAA4B;YA6B5B,aAAa;IAoB3B,OAAO,CAAC,mBAAmB;IAiB3B,OAAO,CAAC,gBAAgB;IASxB;;;;;;;;OAQG;YACW,SAAS;IAgCvB;;;;OAIG;YACW,UAAU;YAgCV,mBAAmB;IA8CjC,OAAO,CAAC,qBAAqB;IAe7B,OAAO,CAAC,iBAAiB;IAcnB,iBAAiB,CACrB,IAAI,EAAE,gBAAgB,EACtB,gBAAgB,EAAE,MAAM,EACxB,QAAQ,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC,CAqDf;CACF;AAED,UAAU,YAAa,SAAQ,kBAAkB;IAC/C,GAAG,EAAE,MAAM,CAAA;IACX,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,kBAAkB,EAAE,EAC3B,KAAK,EAAE,KAAK,EACZ,UAAU,EAAE,MAAM,GACjB,kBAAkB,EAAE,CAYtB;AA4BD,MAAM,WAAW,8BAA8B;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAA;IAC/B,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,OAAO,EAAE,kBAAkB,EAAE,CAAA;IAC7B,OAAO,EAAE,kBAAkB,EAAE,CAAA;CAC9B;AAiDD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA"}
+{"version":3,"file":"BulkFileDataManager.d.ts","sourceRoot":"","sources":["../../../../../../src/services/chaintracker/chaintracks/util/BulkFileDataManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAiE,MAAM,iBAAiB,CAAA;AAGnH,OAAO,EAAE,kBAAkB,EAAuB,MAAM,kBAAkB,CAAA;AAE1E,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAQ3C,OAAO,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAA;AAE5E,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAE1D,OAAO,KAAK,EACV,oBAAoB,EACpB,yBAAyB,EACzB,8BAA8B,EAC/B,MAAM,6BAA6B,CAAA;AACpC,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,0BAA0B,EAChC,MAAM,iCAAiC,CAAA;AAGxC,MAAM,WAAW,0BAA0B;IACzC,KAAK,EAAE,KAAK,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,mBAAmB,CAAA;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,uEAAuE;IACvE,KAAK,CAAC,EAAE,oBAAoB,CAAA;IAC5B,4EAA4E;IAC5E,cAAc,CAAC,EAAE,yBAAyB,CAAA;IAC1C,6FAA6F;IAC7F,SAAS,CAAC,EAAE,wBAAwB,CAAA;IACpC,+EAA+E;IAC/E,oBAAoB,CAAC,EAAE,MAAM,CAAA;CAC9B;AAED,cAAc;AACd,MAAM,WAAW,wBAAwB;IACvC,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,qBAAqB,EAAE,MAAM,CAAA;IAC7B,sBAAsB,EAAE,MAAM,CAAA;IAC9B,cAAc,EAAE,MAAM,CAAA;IACtB,SAAS,EAAE,MAAM,CAAA;IACjB,eAAe,EAAE,MAAM,CAAA;IACvB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,0BAA0B,CAAA;IACvC,cAAc,CAAC,EAAE,8BAA8B,CAAA;CAChD;AAED;;;;;;;;GAQG;AACH,qBAAa,mBAAmB;IAC9B,MAAM,CAAC,oBAAoB,CAAC,KAAK,EAAE,KAAK,GAAG,0BAA0B,CAQpE;IAED,OAAO,CAAC,GAAG,CAAqC;IAEhD,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,eAAe,CAA6B;IACpD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiE;IACtF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyC;IACvE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuD;IACnF,OAAO,CAAC,OAAO,CAAC,CAA+B;IAC/C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAUrB;IAED,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAA;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,mBAAmB,CAAA;IACpC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAA;IACpC,QAAQ,CAAC,KAAK,CAAC,EAAE,oBAAoB,CAAA;IACrC,QAAQ,CAAC,cAAc,CAAC,EAAE,yBAAyB,CAAA;IACnD,QAAQ,CAAC,SAAS,EAAE,wBAAwB,CAAA;IAC5C,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAA;IAErC,YAAY,OAAO,EAAE,0BAA0B,GAAG,KAAK,EAgBtD;IAED,QAAQ,IAAI,wBAAwB,CAMnC;IAEK,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAErC;IAED,OAAO,CAAC,qBAAqB;IAc7B;;;;;;OAMG;IACG,UAAU,CAAC,OAAO,EAAE,6BAA6B,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAErG;YAEa,gBAAgB;IAiC9B,yBAAyB,CAAC,KAAK,EAAE,kBAAkB,EAAE,GAAG;QACtD,GAAG,EAAE,WAAW,CAAA;QAChB,GAAG,EAAE,WAAW,CAAA;QAChB,WAAW,EAAE,WAAW,CAAA;KACzB,CASA;IAEK,YAAY,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAI3F;IAEK,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCjD;IAEK,KAAK,CAAC,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,8BAA8B,CAAC,CAEhF;YAEa,WAAW;YAoCX,gBAAgB;IAoB9B,WAAW,CAAC,IAAI,CAAC,EAAE,8BAA8B,GAAG,YAAY,EAAE,GAAG,kBAAkB,EAAE,GAAG,MAAM,CA2BjG;IAEK,4BAA4B,CAAC,cAAc,EAAE,WAAW,EAAE,EAAE,oBAAoB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA8C9G;IAEK,YAAY,CAAC,QAAQ,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAIpE;IAEK,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,CAO3C;IAEK,eAAe,CAAC,IAAI,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC,CAejH;YAEa,mBAAmB;YA2BnB,qBAAqB;IAkB7B,8BAA8B,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAYrF;IAEK,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAM9E;IAED,OAAO,CAAC,eAAe;IAQvB,OAAO,CAAC,UAAU;IAMZ,WAAW,CAAC,OAAO,SAAI,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAEtE;IAED,OAAO,CAAC,iBAAiB;YAMX,iBAAiB;YAUjB,eAAe;YASf,gBAAgB;YA+BhB,eAAe;YAMf,kBAAkB;IAuB1B,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAEhC;YAEa,gBAAgB;IAU9B,OAAO,CAAC,iBAAiB;YAeX,GAAG;IAajB,OAAO,CAAC,iBAAiB;IAOzB;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;YACW,MAAM;YAuBN,iBAAiB;YAcjB,qBAAqB;YA4BrB,4BAA4B;YA6B5B,aAAa;IAoB3B,OAAO,CAAC,mBAAmB;IAiB3B,OAAO,CAAC,gBAAgB;IASxB;;;;;;;;OAQG;YACW,SAAS;IAgCvB;;;;OAIG;YACW,UAAU;YA8CV,mBAAmB;YAanB,eAAe;YAWf,aAAa;YAyBb,cAAc;YAkBd,qBAAqB;IAYnC,OAAO,CAAC,iBAAiB;IAcnB,iBAAiB,CACrB,IAAI,EAAE,gBAAgB,EACtB,gBAAgB,EAAE,MAAM,EACxB,QAAQ,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,EAClB,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC,CA4Df;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAE7B;CACF;AAED,UAAU,YAAa,SAAQ,kBAAkB;IAC/C,GAAG,EAAE,MAAM,CAAA;IACX,QAAQ,EAAE,MAAM,CAAA;CACjB;AAMD,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,kBAAkB,EAAE,EAC3B,KAAK,EAAE,KAAK,EACZ,UAAU,EAAE,MAAM,GACjB,kBAAkB,EAAE,CAYtB;AA4BD,MAAM,WAAW,8BAA8B;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAA;IAC/B,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,OAAO,EAAE,kBAAkB,EAAE,CAAA;IAC7B,OAAO,EAAE,kBAAkB,EAAE,CAAA;CAC9B;AAiDD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA"}

package/out/src/services/chaintracker/chaintracks/util/BulkFileDataManager.js

@@ -11,6 +11,8 @@ const HeightRange_1 = require("./HeightRange");
 const blockHeaderUtilities_1 = require("./blockHeaderUtilities");
 const ChaintracksFetch_1 = require("./ChaintracksFetch");
 const SingleWriterMultiReaderLock_1 = require("./SingleWriterMultiReaderLock");
+const BulkFileDataValidatorApi_1 = require("../Api/BulkFileDataValidatorApi");
+const InlineBulkFileDataValidator_1 = require("./InlineBulkFileDataValidator");
 /**
  * Manages bulk file data (typically 8MB chunks of 100,000 headers each).
  *
@@ -35,6 +37,7 @@ class BulkFileDataManager {
     fileHashToIndex = {};
     lock = new SingleWriterMultiReaderLock_1.SingleWriterMultiReaderLock();
     inFlightLoads = new Map();
+    failedLoads = new Map();
     storage;
     stats = {
         memoryHits: 0,
@@ -44,7 +47,8 @@ class BulkFileDataManager {
         persistentCacheRejects: 0,
         coalescedLoads: 0,
         downloads: 0,
-        downloadedBytes: 0
+        downloadedBytes: 0,
+        loadBackoffs: 0
     };
     chain;
     maxPerFile;
@@ -53,6 +57,8 @@ class BulkFileDataManager {
     fromKnownSourceUrl;
     cache;
     downloadBudget;
+    validator;
+    failedLoadRetryMsecs;
     constructor(options) {
         const resolvedOptions = typeof options === 'object' ? options : BulkFileDataManager.createDefaultOptions(options);
         this.chain = resolvedOptions.chain;
@@ -62,10 +68,19 @@ class BulkFileDataManager {
         this.fetch = resolvedOptions.fetch;
         this.cache = resolvedOptions.cache;
         this.downloadBudget = resolvedOptions.downloadBudget;
+        this.validator = resolvedOptions.validator ?? new InlineBulkFileDataValidator_1.InlineBulkFileDataValidator();
+        this.failedLoadRetryMsecs = resolvedOptions.failedLoadRetryMsecs ?? 30 * 1000;
+        if (!Number.isSafeInteger(this.failedLoadRetryMsecs) || this.failedLoadRetryMsecs < 0) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('failedLoadRetryMsecs', 'a non-negative safe integer');
+        }
         this.deleteBulkFilesNoLock();
     }
     getStats() {
-        return { ...this.stats };
+        return {
+            ...this.stats,
+            validation: this.validator.getStats?.(),
+            downloadBudget: this.downloadBudget?.snapshot?.()
+        };
     }
     async deleteBulkFiles() {
         return await this.lock.withWriteLock(async () => this.deleteBulkFilesNoLock());
@@ -310,11 +325,36 @@ class BulkFileDataManager {
         });
     }
     async getDataFromFile(file, offset, length) {
-        const bfd = this.getBfdForHeight(file.firstHeight);
-        if (bfd == null || bfd.count < file.count) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('file', `a match for ${file.firstHeight}, ${file.count} in the BulkFileDataManager.`);
+        const resolved = await this.lock.withReadLock(async () => {
+            const resolved = this.getBfdForHeight(file.firstHeight);
+            if (resolved == null || resolved.count < file.count) {
+                throw new sdk_1.WERR_INVALID_PARAMETER('file', `a match for ${file.firstHeight}, ${file.count} in the BulkFileDataManager.`);
+            }
+            return { current: resolved, snapshot: snapshotBfd(resolved) };
+        });
+        // The immutable descriptor is snapshotted under the reader lock, but disk,
+        // network, and validation work happens after releasing it. A concurrent
+        // writer can publish a newer descriptor without invalidating this read.
+        return await this.getDataFromSnapshot(resolved.current, resolved.snapshot, offset, length);
+    }
+    async getDataFromSnapshot(original, snapshot, offset, length) {
+        const data = await this.getDataFromFileNoLock(snapshot, offset, length);
+        if (snapshot.data != null) {
+            await this.lock.withWriteLock(async () => {
+                if (this.bfds.includes(original) &&
+                    original.fileHash === snapshot.fileHash &&
+                    original.firstHeight === snapshot.firstHeight &&
+                    original.count === snapshot.count) {
+                    original.data = snapshot.data;
+                    original.validated = true;
+                    original.lastHash = snapshot.lastHash;
+                    original.lastChainWork = snapshot.lastChainWork;
+                    original.mru = Date.now();
+                    this.ensureMaxRetained();
+                }
+            });
         }
-        return await this.lock.withReadLock(async () => await this.getDataFromFileNoLock(bfd, offset, length));
+        return data;
     }
     async getDataFromFileNoLock(bfd, offset, length) {
         const fileLength = bfd.count * 80;
@@ -331,20 +371,19 @@ class BulkFileDataManager {
         return data.slice(offset, offset + length);
     }
     async findHeaderForHeightOrUndefined(height) {
-        return await this.lock.withReadLock(async () => {
+        const resolved = await this.lock.withReadLock(async () => {
             if (!Number.isInteger(height) || height < 0) {
                 throw new sdk_1.WERR_INVALID_PARAMETER('height', `a non-negative integer (${height}).`);
             }
             const file = this.bfds.find(f => f.firstHeight <= height && f.firstHeight + f.count > height);
             if (file == null)
                 return undefined;
-            const offset = (height - file.firstHeight) * 80;
-            const data = await this.getDataFromFileNoLock(file, offset, 80);
-            if (data == null)
-                return undefined;
-            const header = (0, blockHeaderUtilities_1.deserializeBlockHeader)(data, height, 0);
-            return header;
+            return { current: file, snapshot: snapshotBfd(file), offset: (height - file.firstHeight) * 80 };
         });
+        if (resolved == null)
+            return undefined;
+        const data = await this.getDataFromSnapshot(resolved.current, resolved.snapshot, resolved.offset, 80);
+        return data == null ? undefined : (0, blockHeaderUtilities_1.deserializeBlockHeader)(data, height, 0);
     }
     async getFileForHeight(height) {
         return await this.lock.withReadLock(async () => {
@@ -424,31 +463,31 @@ class BulkFileDataManager {
         return bfd;
     }
     async validateBfdData(bfd, expectedFileHash) {
-        await this.ensureData(bfd);
-        if (bfd.data?.length !== bfd.count * 80) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('file.data', `bulk file ${bfd.fileName} data length ${bfd.data?.length} does not match expected count ${bfd.count}`);
-        }
-        bfd.fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(bfd.data)), 'base64');
-        if (expectedFileHash && expectedFileHash !== bfd.fileHash) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('file.fileHash', `expected ${expectedFileHash} but got ${bfd.fileHash}`);
-        }
-        this.validateBfdHeaders(bfd);
+        const data = await this.ensureData(bfd);
+        const validated = await this.validateRetrievedData(bfd, data, expectedFileHash);
+        bfd.data = validated;
     }
-    validateBfdHeaders(bfd) {
+    async validateBfdHeaders(bfd, expectedFileHash = bfd.fileHash) {
         const pbf = bfd.firstHeight > 0 ? this.getBfdForHeight(bfd.firstHeight - 1) : undefined;
         const prevHash = pbf?.lastHash ?? '00'.repeat(32);
         const prevChainWork = pbf?.lastChainWork ?? '00'.repeat(32);
-        const { lastHeaderHash, lastChainWork } = (0, blockHeaderUtilities_1.validateBufferOfHeaders)(bfd.data, prevHash, 0, undefined, prevChainWork);
-        if (bfd.lastHash && bfd.lastHash !== lastHeaderHash) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('file.lastHash', `expected ${bfd.lastHash} but got ${lastHeaderHash}`);
-        }
-        if (bfd.lastChainWork && bfd.lastChainWork !== lastChainWork) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('file.lastChainWork', `expected ${bfd.lastChainWork} but got ${lastChainWork}`);
-        }
-        bfd.lastHash = lastHeaderHash;
-        bfd.lastChainWork = lastChainWork;
-        if (bfd.firstHeight === 0)
-            (0, blockHeaderUtilities_1.validateGenesisHeader)(bfd.data, bfd.chain);
+        const result = await this.validator.validate({
+            fileName: bfd.fileName,
+            data: bfd.data,
+            count: bfd.count,
+            fileHash: expectedFileHash,
+            firstHeight: bfd.firstHeight,
+            prevHash,
+            prevChainWork,
+            lastHash: bfd.lastHash,
+            lastChainWork: bfd.lastChainWork,
+            chain: bfd.chain
+        });
+        bfd.data = result.data;
+        bfd.fileHash = result.fileHash;
+        bfd.lastHash = result.lastHeaderHash;
+        bfd.lastChainWork = result.lastChainWork;
+        return result.data;
     }
     async ReValidate() {
         return await this.lock.withReadLock(async () => await this.ReValidateNoLock());
@@ -690,75 +729,110 @@ class BulkFileDataManager {
             this.ensureMaxRetained();
             return data;
         }
+        const failed = this.failedLoads.get(key);
+        if (failed != null) {
+            if (Date.now() < failed.retryAt) {
+                this.stats.loadBackoffs++;
+                throw failed.error;
+            }
+            this.failedLoads.delete(key);
+        }
         const load = this.loadAndValidateData(bfd);
         this.inFlightLoads.set(key, load);
         try {
             const data = await load;
             bfd.data = data;
             bfd.validated = true;
+            this.failedLoads.delete(key);
             bfd.mru = Date.now();
             this.ensureMaxRetained();
             return data;
         }
+        catch (error) {
+            const resolved = error instanceof Error ? error : new Error(String(error));
+            this.failedLoads.set(key, { retryAt: Date.now() + this.failedLoadRetryMsecs, error: resolved });
+            throw resolved;
+        }
         finally {
             if (this.inFlightLoads.get(key) === load)
                 this.inFlightLoads.delete(key);
         }
     }
     async loadAndValidateData(bfd) {
-        if (this.storage != null && bfd.fileId) {
-            const stored = await this.storage.getBulkFileData(bfd.fileId);
-            if (stored == null) {
-                throw new sdk_1.WERR_INVALID_PARAMETER('fileId', `valid, data not found for fileId ${bfd.fileId}`);
-            }
-            this.validateRetrievedData(bfd, stored);
-            this.stats.storageHits++;
+        const stored = await this.loadFromStorage(bfd);
+        if (stored != null)
             return stored;
-        }
-        if (this.cache != null) {
-            const cached = await this.cache.get(bfd);
-            if (cached != null) {
-                try {
-                    this.validateRetrievedData(bfd, cached);
-                    this.stats.persistentCacheHits++;
-                    return cached;
-                }
-                catch (error) {
-                    this.stats.persistentCacheRejects++;
-                    await this.cache.delete?.(bfd);
-                    this.log(`Rejected corrupt bulk-header cache entry ${bfd.fileName}: ${String(error)}`);
-                }
-            }
-            else {
-                this.stats.persistentCacheMisses++;
-            }
-        }
-        if (this.fetch != null && bfd.sourceUrl) {
-            const expectedBytes = bfd.count * 80;
-            await this.downloadBudget?.consume(expectedBytes);
-            const url = this.fetch.pathJoin(bfd.sourceUrl, bfd.fileName);
-            const downloaded = await this.fetch.download(url, expectedBytes);
-            if (downloaded == null) {
-                throw new sdk_1.WERR_INVALID_PARAMETER('sourceUrl', `data not found for sourceUrl ${url}`);
-            }
-            this.validateRetrievedData(bfd, downloaded);
-            this.stats.downloads++;
-            this.stats.downloadedBytes += downloaded.length;
-            await this.cache?.set(bfd, downloaded);
+        const cached = await this.loadFromCache(bfd);
+        if (cached != null)
+            return cached;
+        const downloaded = await this.loadFromRemote(bfd);
+        if (downloaded != null)
             return downloaded;
-        }
         throw new sdk_1.WERR_INVALID_PARAMETER('data', `defined. Unable to retrieve data for ${bfd.fileName}`);
     }
-    validateRetrievedData(bfd, data) {
-        if (data.length !== bfd.count * 80) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('file.data', `bulk file ${bfd.fileName} data length ${data.length} does not match expected count ${bfd.count}`);
+    async loadFromStorage(bfd) {
+        if (this.storage == null || !bfd.fileId)
+            return undefined;
+        const stored = await this.storage.getBulkFileData(bfd.fileId);
+        if (stored == null) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('fileId', `valid, data not found for fileId ${bfd.fileId}`);
         }
-        const fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(data)), 'base64');
-        if (fileHash !== bfd.fileHash) {
-            throw new sdk_1.WERR_INVALID_PARAMETER('fileHash', `a match for retrieved data for ${bfd.fileName}`);
+        const validated = await this.validateRetrievedData(bfd, stored);
+        this.stats.storageHits++;
+        return validated;
+    }
+    async loadFromCache(bfd) {
+        if (this.cache == null)
+            return undefined;
+        const cached = await this.cache.get(bfd);
+        if (cached == null) {
+            this.stats.persistentCacheMisses++;
+            return undefined;
         }
+        try {
+            const validated = await this.validateRetrievedData(bfd, cached);
+            this.stats.persistentCacheHits++;
+            await this.cache.promoteValidated?.(bfd, validated);
+            return validated;
+        }
+        catch (error) {
+            // Only deterministic rejection of these exact bytes may quarantine an
+            // immutable cache object. Operational validation failures preserve it.
+            if (!(error instanceof BulkFileDataValidatorApi_1.BulkFileDataValidationError))
+                throw error;
+            this.stats.persistentCacheRejects++;
+            let rejectedData = error.data;
+            if (!(rejectedData instanceof Uint8Array) && cached.byteLength > 0)
+                rejectedData = cached;
+            await this.cache.quarantine?.(bfd, String(error), rejectedData);
+            this.log(`Rejected corrupt bulk-header cache entry ${bfd.fileName}: ${String(error)}`);
+            return undefined;
+        }
+    }
+    async loadFromRemote(bfd) {
+        if (this.fetch == null || !bfd.sourceUrl)
+            return undefined;
+        const expectedBytes = bfd.count * 80;
+        await this.downloadBudget?.consume(expectedBytes);
+        const url = this.fetch.pathJoin(bfd.sourceUrl, bfd.fileName);
+        const downloaded = await this.fetch.download(url, expectedBytes, {
+            beforeRetry: async () => await this.downloadBudget?.consume(expectedBytes)
+        });
+        if (downloaded == null) {
+            throw new sdk_1.WERR_INVALID_PARAMETER('sourceUrl', `data not found for sourceUrl ${url}`);
+        }
+        const validated = await this.validateRetrievedData(bfd, downloaded);
+        this.stats.downloads++;
+        this.stats.downloadedBytes += validated.length;
+        await this.cache?.set(bfd, validated);
+        return validated;
+    }
+    async validateRetrievedData(bfd, data, expectedFileHash = bfd.fileHash) {
         const candidate = { ...bfd, data };
-        this.validateBfdHeaders(candidate);
+        const validated = await this.validateBfdHeaders(candidate, expectedFileHash);
+        bfd.lastHash = candidate.lastHash;
+        bfd.lastChainWork = candidate.lastChainWork;
+        return validated;
     }
     ensureMaxRetained() {
         if (this.maxRetained === undefined)
@@ -800,17 +874,24 @@ class BulkFileDataManager {
             if (data == null || data.length === 0) {
                 break;
             }
-            const last = (0, blockHeaderUtilities_1.validateBufferOfHeaders)(data, lastHeaderHash, 0, undefined, lastChainWork);
-            await toFs.writeFile(toPath(i), data);
-            const fileHash = (0, utilityHelpers_noBuffer_1.asString)(sdk_2.Hash.sha256((0, utilityHelpers_noBuffer_1.asArray)(data)), 'base64');
+            const validated = await this.validator.validate({
+                fileName: toFileName(i),
+                data,
+                count: data.length / 80,
+                firstHeight,
+                prevHash: lastHeaderHash,
+                prevChainWork: lastChainWork,
+                chain
+            });
+            await toFs.writeFile(toPath(i), validated.data);
             const file = {
                 chain,
-                count: data.length / 80,
-                fileHash,
+                count: validated.data.length / 80,
+                fileHash: validated.fileHash,
                 fileName: toFileName(i),
                 firstHeight,
-                lastChainWork: last.lastChainWork,
-                lastHash: last.lastHeaderHash,
+                lastChainWork: validated.lastChainWork,
+                lastHash: validated.lastHeaderHash,
                 prevChainWork: lastChainWork,
                 prevHash: lastHeaderHash,
                 sourceUrl
@@ -822,8 +903,14 @@ class BulkFileDataManager {
         }
         await toFs.writeFile(toJsonPath(), (0, utilityHelpers_noBuffer_1.asUint8Array)(JSON.stringify(toBulkFiles), 'utf8'));
     }
+    async destroy() {
+        await this.validator.destroy?.();
+    }
 }
 exports.BulkFileDataManager = BulkFileDataManager;
+function snapshotBfd(file) {
+    return { ...file, data: file.data };
+}
 function selectBulkHeaderFiles(files, chain, maxPerFile) {
     const r = [];
     let height = 0;