@bopen-io/wallet-toolbox 2.6.2-brc153.5 → 2.14.5

package/out/src/services/chaintracker/chaintracks/Chaintracks.js

@@ -1,15 +1,27 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Chaintracks = void 0;
+exports.Chaintracks = exports.ChaintracksQueueCapacityError = void 0;
 const dirtyHashes_1 = require("./util/dirtyHashes");
 const blockHeaderUtilities_1 = require("./util/blockHeaderUtilities");
 const utilityHelpers_noBuffer_1 = require("../../../utility/utilityHelpers.noBuffer");
 const HeightRange_1 = require("./util/HeightRange");
 const SingleWriterMultiReaderLock_1 = require("./util/SingleWriterMultiReaderLock");
+class ChaintracksQueueCapacityError extends Error {
+    code = 'ERR_CHAINTRACKS_QUEUE_CAPACITY';
+    constructor() {
+        super('The submitted block-header queue is at capacity.');
+        this.name = 'ChaintracksQueueCapacityError';
+    }
+}
+exports.ChaintracksQueueCapacityError = ChaintracksQueueCapacityError;
 const utilityHelpers_1 = require("../../../utility/utilityHelpers");
 const WalletError_1 = require("../../../sdk/WalletError");
+const WERR_errors_1 = require("../../../sdk/WERR_errors");
+const safeDiagnostic_1 = require("./util/safeDiagnostic");
 class Chaintracks {
     options;
+    static maxSubscribersPerKind = 100_000;
+    static maxBulkLiveHeaders = 100_000;
     static createOptions(chain) {
         return {
             chain,
@@ -17,7 +29,7 @@ class Chaintracks {
             bulkIngestors: [],
             liveIngestors: [],
             addLiveRecursionLimit: 36,
-            logging: (...args) => console.log(new Date().toISOString(), ...args),
+            maxQueuedBaseHeaders: 4096,
             readonly: false
         };
     }
@@ -32,27 +44,47 @@ class Chaintracks {
     bulkIngestors;
     liveIngestors;
     baseHeaders = [];
+    queuedBaseHeaderHashes = new Set();
     liveHeaders = [];
     addLiveRecursionLimit = 11;
+    maxQueuedBaseHeaders;
     available = false;
+    cleanupComplete = false;
     startupError = null;
     subscriberCallbacksEnabled = false;
     stopMainThread = true;
     lastPresentHeight = -1;
     lastPresentHeightMsecs = 0;
     lastPresentHeightMaxAge = 60 * 1000; // 1 minute, in milliseconds
+    presentHeightRefresh;
+    mainLoopHeartbeatMsecs = 0;
     lock = new SingleWriterMultiReaderLock_1.SingleWriterMultiReaderLock();
     sourceStatus = new Map();
     constructor(options) {
         this.options = options;
+        if (!['main', 'test', 'stn', 'ttn', 'tstn', 'mock'].includes(options.chain)) {
+            throw new Error('chain must be a supported Chain value.');
+        }
         if (options.storage == null)
             throw new Error('storage is required.');
-        if (!options.bulkIngestors || options.bulkIngestors.length < 1) {
+        if (!Array.isArray(options.bulkIngestors) || options.bulkIngestors.length < 1) {
             throw new Error('At least one bulk ingestor is required.');
         }
-        if (!options.liveIngestors || options.liveIngestors.length < 1) {
+        if (!Array.isArray(options.liveIngestors) || options.liveIngestors.length < 1) {
             throw new Error('At least one live ingestor is required.');
         }
+        this.validateIngestors('bulkIngestors', options.bulkIngestors);
+        this.validateIngestors('liveIngestors', options.liveIngestors);
+        if (!Number.isSafeInteger(options.addLiveRecursionLimit) ||
+            options.addLiveRecursionLimit < 0 ||
+            options.addLiveRecursionLimit > 10_000) {
+            throw new Error('addLiveRecursionLimit must be an integer between 0 and 10000.');
+        }
+        if (typeof options.readonly !== 'boolean')
+            throw new Error('readonly must be a boolean.');
+        if (options.logging != null && typeof options.logging !== 'function') {
+            throw new Error('logging must be a function when supplied.');
+        }
         this.chain = options.chain;
         this.readonly = options.readonly;
         this.storage = options.storage;
@@ -67,38 +99,81 @@ class Chaintracks {
             this.sourceStatus.set(name, { name, role: 'live', state: 'unknown' });
         }
         this.addLiveRecursionLimit = options.addLiveRecursionLimit;
+        this.maxQueuedBaseHeaders = options.maxQueuedBaseHeaders ?? 4096;
+        if (!Number.isSafeInteger(this.maxQueuedBaseHeaders) ||
+            this.maxQueuedBaseHeaders < 1 ||
+            this.maxQueuedBaseHeaders > 1_000_000) {
+            throw new Error('maxQueuedBaseHeaders must be an integer between 1 and 1000000.');
+        }
         if (options.logging != null)
             this.log = options.logging;
         this.storage.log = this.log;
         this.log(`New ChaintracksBase Instance Constructed ${options.chain}Net`);
     }
+    validateIngestors(name, ingestors) {
+        if (ingestors.length > 1_000)
+            throw new Error(`${name} cannot contain more than 1000 entries.`);
+        for (let i = 0; i < ingestors.length; i++) {
+            if (!Object.prototype.hasOwnProperty.call(ingestors, i) || ingestors[i] == null) {
+                throw new Error(`${name} must be a dense array of ingestors.`);
+            }
+        }
+    }
     async getChain() {
         return this.chain;
     }
     /**
-     * Caches and returns most recently sourced value if less than one minute old.
-     * @returns the current externally available chain height (via bulk ingestors).
+     * Returns the last known valid height immediately and refreshes stale state
+     * once in the background. Cold start waits for the single shared refresh.
      */
     async getPresentHeight() {
         const now = Date.now();
         if (this.lastPresentHeight >= 0 && now - this.lastPresentHeightMsecs < this.lastPresentHeightMaxAge) {
             return this.lastPresentHeight;
         }
+        if (this.lastPresentHeight >= 0) {
+            void this.refreshPresentHeight().catch(error => {
+                this.log(`Background present-height refresh failed: ${this.safeDiagnostic(error)}`);
+            });
+            return this.lastPresentHeight;
+        }
+        return await this.refreshPresentHeight();
+    }
+    async refreshPresentHeight() {
+        if (this.presentHeightRefresh != null)
+            return await this.presentHeightRefresh;
+        const refresh = this.loadPresentHeight();
+        this.presentHeightRefresh = refresh;
+        try {
+            return await refresh;
+        }
+        finally {
+            if (this.presentHeightRefresh === refresh)
+                this.presentHeightRefresh = undefined;
+        }
+    }
+    async loadPresentHeight() {
         for (const [index, bulk] of this.bulkIngestors.entries()) {
             const source = this.sourceName('bulk', index, bulk);
             try {
                 const presentHeight = await bulk.getPresentHeight();
-                if (presentHeight != null && Number.isInteger(presentHeight) && presentHeight >= 0) {
+                if (presentHeight != null &&
+                    Number.isSafeInteger(presentHeight) &&
+                    presentHeight >= 0 &&
+                    presentHeight <= 0x7fffffff) {
                     this.markSourceSuccess(source, 'bulk');
                     this.lastPresentHeight = presentHeight;
-                    this.lastPresentHeightMsecs = now;
+                    this.lastPresentHeightMsecs = Date.now();
                     return presentHeight;
                 }
+                if (presentHeight != null) {
+                    this.markSourceFailure(source, 'bulk', new Error('source returned an invalid present height'));
+                }
             }
             catch (uerr) {
                 const error = WalletError_1.WalletError.fromUnknown(uerr);
                 this.markSourceFailure(source, 'bulk', error);
-                this.log(`Present-height source ${source} failed: ${error.message}`);
+                this.log(`Present-height source ${source} failed: ${this.safeDiagnostic(error)}`);
             }
         }
         // A provider outage must not make an already synchronized tracker unusable.
@@ -109,37 +184,66 @@ class Chaintracks {
             const localHeight = Math.max(ranges.bulk.maxHeight, ranges.live.maxHeight);
             if (localHeight >= 0) {
                 this.lastPresentHeight = localHeight;
-                this.lastPresentHeightMsecs = now;
+                this.lastPresentHeightMsecs = Date.now();
                 return localHeight;
             }
         }
         catch (error) {
-            this.log(`Unable to read the locally validated ChainTracks height: ${WalletError_1.WalletError.fromUnknown(error).message}`);
+            this.log(`Unable to read the locally validated ChainTracks height: ${this.safeDiagnostic(error)}`);
         }
         throw new Error('No present-height source or locally validated headers are available.');
     }
     async currentHeight() {
         return await this.getPresentHeight();
     }
+    /** Returns local process state without locks, storage reads, or network I/O. */
+    getAvailabilitySnapshot() {
+        return {
+            available: this.available,
+            startupError: this.startupError == null ? undefined : this.safeDiagnostic(this.startupError),
+            presentHeight: this.lastPresentHeight >= 0 ? this.lastPresentHeight : undefined,
+            presentHeightUpdatedAt: this.lastPresentHeightMsecs > 0 ? new Date(this.lastPresentHeightMsecs).toISOString() : undefined,
+            presentHeightRefreshInFlight: this.presentHeightRefresh != null,
+            mainLoopHeartbeatAt: this.mainLoopHeartbeatMsecs > 0 ? new Date(this.mainLoopHeartbeatMsecs).toISOString() : undefined,
+            sources: Array.from(this.sourceStatus.values()).map(status => ({ ...status })),
+            bulkData: this.storage.bulkManager.getStats()
+        };
+    }
     async subscribeHeaders(listener) {
+        this.validateListener(listener, 'header listener');
+        this.requireSubscriberCapacity(this.callbacks.header, 'header');
         const ID = (0, utilityHelpers_1.randomBytesBase64)(8);
         this.callbacks.header[ID] = listener;
         return ID;
     }
     async subscribeReorgs(listener) {
+        this.validateListener(listener, 'reorg listener');
+        this.requireSubscriberCapacity(this.callbacks.reorg, 'reorg');
         const ID = (0, utilityHelpers_1.randomBytesBase64)(8);
         this.callbacks.reorg[ID] = listener;
         return ID;
     }
     async unsubscribe(subscriptionId) {
-        let success = true;
-        if (this.callbacks.header[subscriptionId])
-            this.callbacks.header[subscriptionId] = null;
-        else if (this.callbacks.reorg[subscriptionId])
-            this.callbacks.reorg[subscriptionId] = null;
-        else
-            success = false;
-        return success;
+        if (typeof subscriptionId !== 'string' || subscriptionId.length === 0 || subscriptionId.length > 128)
+            return false;
+        if (Object.prototype.hasOwnProperty.call(this.callbacks.header, subscriptionId)) {
+            delete this.callbacks.header[subscriptionId];
+            return true;
+        }
+        if (Object.prototype.hasOwnProperty.call(this.callbacks.reorg, subscriptionId)) {
+            delete this.callbacks.reorg[subscriptionId];
+            return true;
+        }
+        return false;
+    }
+    validateListener(listener, name) {
+        if (typeof listener !== 'function')
+            throw new WERR_errors_1.WERR_INVALID_PARAMETER(name, 'a function');
+    }
+    requireSubscriberCapacity(listeners, kind) {
+        if (Object.keys(listeners).length >= Chaintracks.maxSubscribersPerKind) {
+            throw new WERR_errors_1.WERR_INVALID_PARAMETER(`${kind} subscriptions`, `fewer than ${Chaintracks.maxSubscribersPerKind}`);
+        }
     }
     /**
      * Queues a potentially new, unknown header for consideration as an addition to the chain.
@@ -151,7 +255,23 @@ class Chaintracks {
      * @param header
      */
     async addHeader(header) {
-        this.baseHeaders.push(header);
+        (0, blockHeaderUtilities_1.validateBaseBlockHeaderFormat)(header);
+        const pendingHeader = {
+            version: header.version,
+            previousHash: header.previousHash,
+            merkleRoot: header.merkleRoot,
+            time: header.time,
+            bits: header.bits,
+            nonce: header.nonce
+        };
+        const hash = (0, blockHeaderUtilities_1.blockHash)(pendingHeader);
+        if (this.queuedBaseHeaderHashes.has(hash))
+            return;
+        if (this.baseHeaders.length >= this.maxQueuedBaseHeaders) {
+            throw new ChaintracksQueueCapacityError();
+        }
+        this.queuedBaseHeaderHashes.add(hash);
+        this.baseHeaders.push(pendingHeader);
     }
     /**
      * If not already available, takes a writer lock to queue calls until available.
@@ -169,50 +289,84 @@ class Chaintracks {
             // Only the first call proceeds to initialize...
             if (this.available)
                 return;
-            // Make sure database schema exists and is updated...
-            await this.storage.migrateLatest();
-            for (const bulkIn of this.bulkIngestors)
-                await bulkIn.setStorage(this.storage, this.log);
-            for (const liveIn of this.liveIngestors)
-                await liveIn.setStorage(this.storage, this.log);
-            // Start all live ingestors to push new headers onto liveHeaders... each long running.
-            this.stopMainThread = false;
-            for (const [index, liveIngestor] of this.liveIngestors.entries()) {
-                this.promises.push(this.runLiveIngestor(liveIngestor, index));
+            this.cleanupComplete = false;
+            this.startupError = null;
+            try {
+                // Make sure database schema exists and is updated...
+                await this.storage.migrateLatest();
+                for (const bulkIn of this.bulkIngestors)
+                    await bulkIn.setStorage(this.storage, this.log);
+                for (const liveIn of this.liveIngestors)
+                    await liveIn.setStorage(this.storage, this.log);
+                // Start all live ingestors to push new headers onto liveHeaders... each long running.
+                this.stopMainThread = false;
+                for (const [index, liveIngestor] of this.liveIngestors.entries()) {
+                    this.promises.push(this.runLiveIngestor(liveIngestor, index));
+                }
+                // Start main loop to shift out liveHeaders...once sync'd, will set `available` true.
+                this.promises.push(this.mainThreadShiftLiveHeaders());
+                // Wait for the main thread to finish initial sync.
+                while (!this.available && this.startupError == null) {
+                    await (0, utilityHelpers_1.wait)(100);
+                }
+                if (this.startupError != null)
+                    throw this.startupError;
             }
-            // Start mai loop to shift out liveHeaders...once sync'd, will set `available` true.
-            this.promises.push(this.mainThreadShiftLiveHeaders());
-            // Wait for the main thread to finish initial sync.
-            while (!this.available && this.startupError == null) {
-                await (0, utilityHelpers_1.wait)(100);
+            catch (error) {
+                this.startupError = WalletError_1.WalletError.fromUnknown(error);
+                await this.stopRuntimeForRetry();
+                throw error;
             }
-            if (this.startupError != null)
-                throw this.startupError;
         });
     }
     async startPromises() {
-        if (this.promises.length > 0 || !this.stopMainThread)
-            return;
+        await this.makeAvailable();
     }
     async destroy() {
-        if (!this.available)
-            return;
         await this.lock.withWriteLock(async () => {
-            if (!this.available || this.stopMainThread)
+            if (this.cleanupComplete)
+                return;
+            if (!this.available && this.promises.length === 0 && this.startupError == null)
                 return;
             this.log('Shutting Down');
             this.stopMainThread = true;
             for (const liveIn of this.liveIngestors)
-                await liveIn.shutdown();
-            for (const bulkIn of this.bulkIngestors)
-                await bulkIn.shutdown();
-            await Promise.all(this.promises);
-            await this.storage.destroy();
+                this.stopLiveIngestor(liveIn);
+            const cleanup = [
+                ...this.promises,
+                ...this.liveIngestors.map(async (liveIn) => await liveIn.shutdown()),
+                ...this.bulkIngestors.map(async (bulkIn) => await bulkIn.shutdown())
+            ];
+            const results = await Promise.allSettled(cleanup);
+            results.push(...(await Promise.allSettled([this.storage.bulkManager.destroy()])));
+            results.push(...(await Promise.allSettled([this.storage.destroy()])));
+            this.promises.length = 0;
             this.available = false;
-            this.stopMainThread = false;
+            this.subscriberCallbacksEnabled = false;
+            this.cleanupComplete = true;
+            const failure = results.find((result) => result.status === 'rejected');
+            if (failure != null)
+                throw WalletError_1.WalletError.fromUnknown(failure.reason);
             this.log('Shutdown');
         });
     }
+    async stopRuntimeForRetry() {
+        this.stopMainThread = true;
+        for (const liveIngestor of this.liveIngestors)
+            this.stopLiveIngestor(liveIngestor);
+        await Promise.allSettled(this.promises);
+        this.promises.length = 0;
+        this.available = false;
+        this.subscriberCallbacksEnabled = false;
+    }
+    stopLiveIngestor(liveIngestor) {
+        try {
+            liveIngestor.stopListening();
+        }
+        catch {
+            // Cleanup continues so one faulty source cannot prevent the remaining resources from stopping.
+        }
+    }
     async listening() {
         return await this.makeAvailable();
     }
@@ -229,7 +383,7 @@ class Chaintracks {
                 restartCount++;
                 const waitMsecs = this.liveIngestorRestartWaitMsecs(restartCount);
                 this.log(`Live ingestor ${name} stopped unexpectedly restart=${restartCount} retryMsecs=${waitMsecs}`);
-                await (0, utilityHelpers_1.wait)(waitMsecs);
+                await this.waitUnlessStopping(waitMsecs);
             }
             catch (error_) {
                 if (this.stopMainThread)
@@ -238,11 +392,19 @@ class Chaintracks {
                 const e = WalletError_1.WalletError.fromUnknown(error_);
                 this.markSourceFailure(source, 'live', e);
                 const waitMsecs = this.liveIngestorRestartWaitMsecs(restartCount);
-                this.log(`Live ingestor ${name} failed restart=${restartCount} retryMsecs=${waitMsecs}: ${e.stack ?? e.message}`);
-                await (0, utilityHelpers_1.wait)(waitMsecs);
+                this.log(`Live ingestor ${name} failed restart=${restartCount} retryMsecs=${waitMsecs}: ${this.safeDiagnostic(e)}`);
+                await this.waitUnlessStopping(waitMsecs);
             }
         }
     }
+    async waitUnlessStopping(msecs) {
+        let remaining = msecs;
+        while (remaining > 0 && !this.stopMainThread) {
+            const chunk = Math.min(250, remaining);
+            await (0, utilityHelpers_1.wait)(chunk);
+            remaining -= chunk;
+        }
+    }
     liveIngestorRestartWaitMsecs(restartCount) {
         return Math.min(1000 * Math.min(2 ** Math.max(restartCount - 1, 0), 60), 60000);
     }
@@ -254,6 +416,7 @@ class Chaintracks {
         return true;
     }
     async findHeaderForHeight(height) {
+        this.validatePublicHeight(height);
         await this.makeAvailable();
         return await this.lock.withReadLock(async () => await this.findHeaderForHeightNoLock(height));
     }
@@ -261,6 +424,7 @@ class Chaintracks {
         return await this.storage.findHeaderForHeightOrUndefined(height);
     }
     async findHeaderForBlockHash(hash) {
+        this.validatePublicHash(hash);
         await this.makeAvailable();
         return await this.lock.withReadLock(async () => await this.findHeaderForBlockHashNoLock(hash));
     }
@@ -268,6 +432,7 @@ class Chaintracks {
         return (await this.storage.findLiveHeaderForBlockHash(hash)) || undefined;
     }
     async isValidRootForHeight(root, height) {
+        this.validatePublicHash(root);
         const r = await this.findHeaderForHeight(height);
         if (r == null)
             return false;
@@ -288,11 +453,16 @@ class Chaintracks {
             bulkIngestors: this.bulkIngestors.map(bulkIngestor => bulkIngestor.constructor.name),
             liveIngestors: this.liveIngestors.map(liveIngestor => liveIngestor.constructor.name),
             packages: [],
-            sources: Array.from(this.sourceStatus.values()).map(status => ({ ...status }))
+            sources: Array.from(this.sourceStatus.values()).map(status => ({ ...status })),
+            bulkData: this.storage.bulkManager.getStats()
         };
         return info;
     }
     async getHeaders(height, count) {
+        this.validatePublicHeight(height);
+        if (!Number.isSafeInteger(count) || count < 1 || count > 100000 || height + count - 1 > 0x7fffffff) {
+            throw new WERR_errors_1.WERR_INVALID_PARAMETER('count', 'an integer from 1 through 100000 within the supported height range');
+        }
         await this.makeAvailable();
         return await this.lock.withReadLock(async () => (0, utilityHelpers_noBuffer_1.asString)(await this.storage.getHeadersUint8Array(height, count)));
     }
@@ -305,6 +475,7 @@ class Chaintracks {
         return await this.lock.withReadLock(async () => await this.storage.findChainTipHash());
     }
     async findLiveHeaderForBlockHash(hash) {
+        this.validatePublicHash(hash);
         await this.makeAvailable();
         const header = await this.lock.withReadLock(async () => await this.storage.findLiveHeaderForBlockHash(hash));
         return header || undefined;
@@ -313,6 +484,16 @@ class Chaintracks {
         const header = await this.findLiveHeaderForBlockHash(hash);
         return header?.chainWork;
     }
+    validatePublicHeight(height) {
+        if (!Number.isSafeInteger(height) || height < 0 || height > 0x7fffffff) {
+            throw new WERR_errors_1.WERR_INVALID_PARAMETER('height', 'an integer from 0 through 2147483647');
+        }
+    }
+    validatePublicHash(hash) {
+        if (typeof hash !== 'string' || !/^[0-9a-fA-F]{64}$/.test(hash)) {
+            throw new WERR_errors_1.WERR_INVALID_PARAMETER('hash', 'exactly 32 hexadecimal bytes');
+        }
+    }
     /**
      * @returns true iff all headers from height zero through current chainTipHeader height can be retreived and form a valid chain.
      */
@@ -335,7 +516,7 @@ class Chaintracks {
         await bulk.exportHeadersToFs(toFs, toHeadersPerFile, toFolder, sourceUrl, maxHeight);
     }
     async startListening() {
-        this.makeAvailable();
+        await this.makeAvailable();
     }
     async syncBulkStorage(presentHeight, initialRanges) {
         await this.lock.withWriteLock(async () => await this.syncBulkStorageNoLock(presentHeight, initialRanges));
@@ -365,7 +546,13 @@ class Chaintracks {
             }
         }
         if (this.startupError == null) {
-            this.liveHeaders.unshift(...newLiveHeaders);
+            if (this.liveHeaders.length + newLiveHeaders.length > Chaintracks.maxBulkLiveHeaders) {
+                throw new Error(`Bulk sync cannot queue more than ${Chaintracks.maxBulkLiveHeaders} live headers.`);
+            }
+            for (let end = newLiveHeaders.length; end > 0; end -= 10_000) {
+                const start = Math.max(0, end - 10_000);
+                this.liveHeaders.unshift(...newLiveHeaders.slice(start, end));
+            }
             added = after.bulk.above(initialRanges.bulk);
             this.log(`syncBulkStorage done
   Before sync: bulk ${initialRanges.bulk}, live ${initialRanges.live}
@@ -389,7 +576,9 @@ class Chaintracks {
                 const r = await bulk.synchronize(presentHeight, before, newLiveHeaders);
                 hadSuccess = true;
                 this.markSourceSuccess(source, 'bulk');
-                newLiveHeaders = r.liveHeaders;
+                newLiveHeaders = this.normalizeBulkLiveHeaders(r.liveHeaders);
+                if (typeof r.done !== 'boolean')
+                    throw new Error('Bulk ingestor result done must be a boolean.');
                 after = await this.storage.getAvailableHeightRanges();
                 const added = after.bulk.above(before.bulk);
                 const afterLiveRange = HeightRange_1.HeightRange.from(newLiveHeaders);
@@ -405,7 +594,7 @@ class Chaintracks {
             catch (error_) {
                 const e = (bulkSyncError = WalletError_1.WalletError.fromUnknown(error_));
                 this.markSourceFailure(source, 'bulk', e);
-                this.log(`bulk sync error: ${e.message}`);
+                this.log(`bulk sync error: ${this.safeDiagnostic(e)}`);
             }
         }
         if (!this.available && bulkSyncError != null && !hadSuccess)
@@ -415,6 +604,30 @@ class Chaintracks {
     sourceName(role, index, source) {
         return `${role}[${index}]:${source.constructor.name}`;
     }
+    normalizeBulkLiveHeaders(value) {
+        if (!Array.isArray(value) || value.length > Chaintracks.maxBulkLiveHeaders) {
+            throw new Error(`Bulk ingestor liveHeaders must be an array of at most ${Chaintracks.maxBulkLiveHeaders} entries.`);
+        }
+        const headers = [];
+        for (let index = 0; index < value.length; index++) {
+            if (!Object.prototype.hasOwnProperty.call(value, index)) {
+                throw new Error('Bulk ingestor liveHeaders must be a dense array.');
+            }
+            const header = value[index];
+            (0, blockHeaderUtilities_1.validateHeaderFormat)(header);
+            headers.push({
+                version: header.version,
+                previousHash: header.previousHash,
+                merkleRoot: header.merkleRoot,
+                time: header.time,
+                bits: header.bits,
+                nonce: header.nonce,
+                height: header.height,
+                hash: header.hash
+            });
+        }
+        return headers;
+    }
     markSourceSuccess(name, role) {
         this.sourceStatus.set(name, {
             ...this.sourceStatus.get(name),
@@ -432,7 +645,7 @@ class Chaintracks {
             role,
             state: 'degraded',
             lastFailure: new Date().toISOString(),
-            error: error.message
+            error: this.safeDiagnostic(error)
         });
     }
     async getMissingBlockHeader(hash) {
@@ -447,7 +660,7 @@ class Chaintracks {
             catch (error) {
                 const resolved = WalletError_1.WalletError.fromUnknown(error);
                 this.markSourceFailure(source, 'live', resolved);
-                this.log(`Header lookup source ${source} failed: ${resolved.message}`);
+                this.log(`Header lookup source ${source} failed: ${this.safeDiagnostic(resolved)}`);
             }
         }
         return undefined;
@@ -457,6 +670,7 @@ class Chaintracks {
     }
     async addLiveHeader(header) {
         (0, blockHeaderUtilities_1.validateHeaderFormat)(header);
+        (0, blockHeaderUtilities_1.validateHeaderProofOfWork)(header);
         (0, dirtyHashes_1.validateAgainstDirtyHashes)(header.hash);
         const ihr = this.available
             ? await this.lock.withWriteLock(async () => await this.storage.insertHeader(header))
@@ -476,7 +690,7 @@ class Chaintracks {
             const listener = this.callbacks.header[id];
             if (listener != null) {
                 try {
-                    listener(header);
+                    listener({ ...header });
                 }
                 catch {
                     /* ignore all errors thrown */
@@ -485,13 +699,11 @@ class Chaintracks {
         }
     }
     notifyReorgListeners(ihr, header) {
-        const priorTip = { ...ihr.priorTip };
-        const deactivated = ihr.deactivatedHeaders.map(lbh => ({ ...lbh }));
         for (const id in this.callbacks.reorg) {
             const listener = this.callbacks.reorg[id];
             if (listener != null) {
                 try {
-                    listener(ihr.reorgDepth, priorTip, header, deactivated);
+                    listener(ihr.reorgDepth, { ...ihr.priorTip }, { ...header }, ihr.deactivatedHeaders.map(lbh => ({ ...lbh })));
                 }
                 catch {
                     /* ignore all errors thrown */
@@ -520,14 +732,16 @@ class Chaintracks {
         while (!this.stopMainThread) {
             try {
                 const now = Date.now();
+                this.mainLoopHeartbeatMsecs = now;
                 lastSyncCheck = now;
                 lastBulkSync = await this.runBulkSyncIfNeeded(now, lastBulkSync, cdnSyncRepeatMsecs);
                 await this.processLiveHeaderQueue(lastSyncCheck, syncCheckRepeatMsecs);
+                this.mainLoopHeartbeatMsecs = Date.now();
             }
             catch (error_) {
                 const e = WalletError_1.WalletError.fromUnknown(error_);
                 if (this.available) {
-                    this.log(`Error occurred during chaintracks main thread processing: ${e.stack || e.message}`);
+                    this.log(`Error occurred during chaintracks main thread processing: ${this.safeDiagnostic(e)}`);
                 }
                 else {
                     this.startupError = e;
@@ -538,7 +752,7 @@ class Chaintracks {
     }
     /** Returns (potentially updated) lastBulkSync timestamp. */
     async runBulkSyncIfNeeded(now, lastBulkSync, cdnSyncRepeatMsecs) {
-        const presentHeight = await this.getPresentHeight();
+        const presentHeight = await this.refreshPresentHeight();
         const before = await this.storage.getAvailableHeightRanges();
         let skipBulkSync = !before.live.isEmpty && before.live.maxHeight >= presentHeight - this.addLiveRecursionLimit / 2;
         // If we haven't re-synced in a long time, do it just to check for a CDN update.
@@ -577,6 +791,7 @@ class Chaintracks {
         const baseHeader = this.baseHeaders.shift();
         if (baseHeader == null)
             return null;
+        this.queuedBaseHeaderHashes.delete((0, blockHeaderUtilities_1.blockHash)(baseHeader));
         if (await this.processOneBaseHeader(baseHeader))
             stats.count++;
         return false;
@@ -671,6 +886,9 @@ class Chaintracks {
             this.log(`listening at height of ${live.maxHeight}`);
         }
     }
+    safeDiagnostic(error) {
+        return (0, safeDiagnostic_1.safeDiagnostic)(error, 1024);
+    }
 }
 exports.Chaintracks = Chaintracks;
 //# sourceMappingURL=Chaintracks.js.map