@bopen-io/wallet-toolbox 2.1.21-parity-fix.2 → 2.6.2-brc153.2

package/out/src/services/chaintracker/chaintracks/Chaintracks.js

@@ -9,6 +9,7 @@ const SingleWriterMultiReaderLock_1 = require("./util/SingleWriterMultiReaderLoc
 const utilityHelpers_1 = require("../../../utility/utilityHelpers");
 const WalletError_1 = require("../../../sdk/WalletError");
 class Chaintracks {
+    options;
     static createOptions(chain) {
         return {
             chain,
@@ -20,37 +21,53 @@ class Chaintracks {
             readonly: false
         };
     }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    log = () => { };
+    chain;
+    readonly;
+    // Collection of all long running "threads": main thread (liveHeaders consumer / monitor) and each live header ingestor.
+    promises = [];
+    callbacks = { header: {}, reorg: {} };
+    storage;
+    bulkIngestors;
+    liveIngestors;
+    baseHeaders = [];
+    liveHeaders = [];
+    addLiveRecursionLimit = 11;
+    available = false;
+    startupError = null;
+    subscriberCallbacksEnabled = false;
+    stopMainThread = true;
+    lastPresentHeight = -1;
+    lastPresentHeightMsecs = 0;
+    lastPresentHeightMaxAge = 60 * 1000; // 1 minute, in milliseconds
+    lock = new SingleWriterMultiReaderLock_1.SingleWriterMultiReaderLock();
+    sourceStatus = new Map();
     constructor(options) {
         this.options = options;
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        this.log = () => { };
-        // Collection of all long running "threads": main thread (liveHeaders consumer / monitor) and each live header ingestor.
-        this.promises = [];
-        this.callbacks = { header: [], reorg: [] };
-        this.baseHeaders = [];
-        this.liveHeaders = [];
-        this.addLiveRecursionLimit = 11;
-        this.available = false;
-        this.startupError = null;
-        this.subscriberCallbacksEnabled = false;
-        this.stopMainThread = true;
-        this.lastPresentHeight = 0;
-        this.lastPresentHeightMsecs = 0;
-        this.lastPresentHeightMaxAge = 60 * 1000; // 1 minute, in milliseconds
-        this.lock = new SingleWriterMultiReaderLock_1.SingleWriterMultiReaderLock();
-        if (!options.storage)
+        if (options.storage == null)
             throw new Error('storage is required.');
-        if (!options.bulkIngestors || options.bulkIngestors.length < 1)
+        if (!options.bulkIngestors || options.bulkIngestors.length < 1) {
             throw new Error('At least one bulk ingestor is required.');
-        if (!options.liveIngestors || options.liveIngestors.length < 1)
+        }
+        if (!options.liveIngestors || options.liveIngestors.length < 1) {
             throw new Error('At least one live ingestor is required.');
+        }
         this.chain = options.chain;
         this.readonly = options.readonly;
         this.storage = options.storage;
         this.bulkIngestors = options.bulkIngestors;
         this.liveIngestors = options.liveIngestors;
+        for (const [index, source] of this.bulkIngestors.entries()) {
+            const name = this.sourceName('bulk', index, source);
+            this.sourceStatus.set(name, { name, role: 'bulk', state: 'unknown' });
+        }
+        for (const [index, source] of this.liveIngestors.entries()) {
+            const name = this.sourceName('live', index, source);
+            this.sourceStatus.set(name, { name, role: 'live', state: 'unknown' });
+        }
         this.addLiveRecursionLimit = options.addLiveRecursionLimit;
-        if (options.logging)
+        if (options.logging != null)
             this.log = options.logging;
         this.storage.log = this.log;
         this.log(`New ChaintracksBase Instance Constructed ${options.chain}Net`);
@@ -64,26 +81,42 @@ class Chaintracks {
      */
     async getPresentHeight() {
         const now = Date.now();
-        if (this.lastPresentHeight && now - this.lastPresentHeightMsecs < this.lastPresentHeightMaxAge) {
+        if (this.lastPresentHeight >= 0 && now - this.lastPresentHeightMsecs < this.lastPresentHeightMaxAge) {
             return this.lastPresentHeight;
         }
-        const presentHeights = [];
-        for (const bulk of this.bulkIngestors) {
+        for (const [index, bulk] of this.bulkIngestors.entries()) {
+            const source = this.sourceName('bulk', index, bulk);
             try {
                 const presentHeight = await bulk.getPresentHeight();
-                if (presentHeight)
-                    presentHeights.push(presentHeight);
+                if (presentHeight != null && Number.isInteger(presentHeight) && presentHeight >= 0) {
+                    this.markSourceSuccess(source, 'bulk');
+                    this.lastPresentHeight = presentHeight;
+                    this.lastPresentHeightMsecs = now;
+                    return presentHeight;
+                }
             }
             catch (uerr) {
-                console.error(uerr);
+                const error = WalletError_1.WalletError.fromUnknown(uerr);
+                this.markSourceFailure(source, 'bulk', error);
+                this.log(`Present-height source ${source} failed: ${error.message}`);
             }
         }
-        const presentHeight = presentHeights.length ? Math.max(...presentHeights) : undefined;
-        if (!presentHeight)
-            throw new Error('At least one bulk ingestor must implement getPresentHeight.');
-        this.lastPresentHeight = presentHeight;
-        this.lastPresentHeightMsecs = now;
-        return presentHeight;
+        // A provider outage must not make an already synchronized tracker unusable.
+        if (this.lastPresentHeight >= 0)
+            return this.lastPresentHeight;
+        try {
+            const ranges = await this.storage.getAvailableHeightRanges();
+            const localHeight = Math.max(ranges.bulk.maxHeight, ranges.live.maxHeight);
+            if (localHeight >= 0) {
+                this.lastPresentHeight = localHeight;
+                this.lastPresentHeightMsecs = now;
+                return localHeight;
+            }
+        }
+        catch (error) {
+            this.log(`Unable to read the locally validated ChainTracks height: ${WalletError_1.WalletError.fromUnknown(error).message}`);
+        }
+        throw new Error('No present-height source or locally validated headers are available.');
     }
     async currentHeight() {
         return await this.getPresentHeight();
@@ -101,9 +134,9 @@ class Chaintracks {
     async unsubscribe(subscriptionId) {
         let success = true;
         if (this.callbacks.header[subscriptionId])
-            delete this.callbacks.header[subscriptionId];
+            this.callbacks.header[subscriptionId] = null;
         else if (this.callbacks.reorg[subscriptionId])
-            delete this.callbacks.reorg[subscriptionId];
+            this.callbacks.reorg[subscriptionId] = null;
         else
             success = false;
         return success;
@@ -143,20 +176,22 @@ class Chaintracks {
             for (const liveIn of this.liveIngestors)
                 await liveIn.setStorage(this.storage, this.log);
             // Start all live ingestors to push new headers onto liveHeaders... each long running.
-            for (const liveIngestor of this.liveIngestors)
-                this.promises.push(liveIngestor.startListening(this.liveHeaders));
+            this.stopMainThread = false;
+            for (const [index, liveIngestor] of this.liveIngestors.entries()) {
+                this.promises.push(this.runLiveIngestor(liveIngestor, index));
+            }
             // Start mai loop to shift out liveHeaders...once sync'd, will set `available` true.
             this.promises.push(this.mainThreadShiftLiveHeaders());
             // Wait for the main thread to finish initial sync.
-            while (!this.available && !this.startupError) {
+            while (!this.available && this.startupError == null) {
                 await (0, utilityHelpers_1.wait)(100);
             }
-            if (this.startupError)
+            if (this.startupError != null)
                 throw this.startupError;
         });
     }
     async startPromises() {
-        if (this.promises.length > 0 || this.stopMainThread !== true)
+        if (this.promises.length > 0 || !this.stopMainThread)
             return;
     }
     async destroy() {
@@ -179,40 +214,69 @@ class Chaintracks {
         });
     }
     async listening() {
-        return this.makeAvailable();
+        return await this.makeAvailable();
+    }
+    async runLiveIngestor(liveIngestor, index) {
+        let restartCount = 0;
+        const name = liveIngestor.constructor.name;
+        const source = this.sourceName('live', index, liveIngestor);
+        while (!this.stopMainThread) {
+            try {
+                this.markSourceSuccess(source, 'live');
+                await liveIngestor.startListening(this.liveHeaders);
+                if (this.stopMainThread)
+                    return;
+                restartCount++;
+                const waitMsecs = this.liveIngestorRestartWaitMsecs(restartCount);
+                this.log(`Live ingestor ${name} stopped unexpectedly restart=${restartCount} retryMsecs=${waitMsecs}`);
+                await (0, utilityHelpers_1.wait)(waitMsecs);
+            }
+            catch (error_) {
+                if (this.stopMainThread)
+                    return;
+                restartCount++;
+                const e = WalletError_1.WalletError.fromUnknown(error_);
+                this.markSourceFailure(source, 'live', e);
+                const waitMsecs = this.liveIngestorRestartWaitMsecs(restartCount);
+                this.log(`Live ingestor ${name} failed restart=${restartCount} retryMsecs=${waitMsecs}: ${e.stack ?? e.message}`);
+                await (0, utilityHelpers_1.wait)(waitMsecs);
+            }
+        }
+    }
+    liveIngestorRestartWaitMsecs(restartCount) {
+        return Math.min(1000 * Math.min(2 ** Math.max(restartCount - 1, 0), 60), 60000);
     }
     async isListening() {
         return this.available;
     }
     async isSynchronized() {
         await this.makeAvailable();
-        // TODO add synchronized flag... false while bulksyncing...
         return true;
     }
     async findHeaderForHeight(height) {
         await this.makeAvailable();
-        return this.lock.withReadLock(async () => this.findHeaderForHeightNoLock(height));
+        return await this.lock.withReadLock(async () => await this.findHeaderForHeightNoLock(height));
     }
     async findHeaderForHeightNoLock(height) {
         return await this.storage.findHeaderForHeightOrUndefined(height);
     }
     async findHeaderForBlockHash(hash) {
         await this.makeAvailable();
-        return this.lock.withReadLock(async () => this.findHeaderForBlockHashNoLock(hash));
+        return await this.lock.withReadLock(async () => await this.findHeaderForBlockHashNoLock(hash));
     }
     async findHeaderForBlockHashNoLock(hash) {
         return (await this.storage.findLiveHeaderForBlockHash(hash)) || undefined;
     }
     async isValidRootForHeight(root, height) {
         const r = await this.findHeaderForHeight(height);
-        if (!r)
+        if (r == null)
             return false;
         const isValid = root === r.merkleRoot;
         return isValid;
     }
     async getInfo() {
         await this.makeAvailable();
-        return this.lock.withReadLock(async () => this.getInfoNoLock());
+        return await this.lock.withReadLock(async () => await this.getInfoNoLock());
     }
     async getInfoNoLock() {
         const liveRange = await this.storage.findLiveHeightRange();
@@ -223,21 +287,22 @@ class Chaintracks {
             storage: this.storage.constructor.name,
             bulkIngestors: this.bulkIngestors.map(bulkIngestor => bulkIngestor.constructor.name),
             liveIngestors: this.liveIngestors.map(liveIngestor => liveIngestor.constructor.name),
-            packages: []
+            packages: [],
+            sources: Array.from(this.sourceStatus.values()).map(status => ({ ...status }))
         };
         return info;
     }
     async getHeaders(height, count) {
         await this.makeAvailable();
-        return this.lock.withReadLock(async () => (0, utilityHelpers_noBuffer_1.asString)(await this.storage.getHeadersUint8Array(height, count)));
+        return await this.lock.withReadLock(async () => (0, utilityHelpers_noBuffer_1.asString)(await this.storage.getHeadersUint8Array(height, count)));
     }
     async findChainTipHeader() {
         await this.makeAvailable();
-        return this.lock.withReadLock(async () => await this.storage.findChainTipHeader());
+        return await this.lock.withReadLock(async () => await this.storage.findChainTipHeader());
     }
     async findChainTipHash() {
         await this.makeAvailable();
-        return this.lock.withReadLock(async () => await this.storage.findChainTipHash());
+        return await this.lock.withReadLock(async () => await this.storage.findChainTipHash());
     }
     async findLiveHeaderForBlockHash(hash) {
         await this.makeAvailable();
@@ -246,7 +311,7 @@ class Chaintracks {
     }
     async findChainWorkForBlockHash(hash) {
         const header = await this.findLiveHeaderForBlockHash(hash);
-        return header === null || header === void 0 ? void 0 : header.chainWork;
+        return header?.chainWork;
     }
     /**
      * @returns true iff all headers from height zero through current chainTipHeader height can be retreived and form a valid chain.
@@ -255,7 +320,7 @@ class Chaintracks {
         let h = await this.findChainTipHeader();
         while (h.height > 0) {
             const hp = await this.findHeaderForHeight(h.height - 1);
-            if (!hp || hp.hash !== h.previousHash)
+            if (hp?.hash !== h.previousHash)
                 throw new Error(`validation fails at height ${h.height}`);
             h = hp;
             if (10000 * Math.floor(h.height / 10000) === h.height)
@@ -265,7 +330,7 @@ class Chaintracks {
         return true;
     }
     async exportBulkHeaders(toFolder, toFs, sourceUrl, toHeadersPerFile, maxHeight) {
-        toHeadersPerFile || (toHeadersPerFile = 100000);
+        toHeadersPerFile ||= 100000;
         const bulk = this.storage.bulkManager;
         await bulk.exportHeadersToFs(toFs, toHeadersPerFile, toFolder, sourceUrl, maxHeight);
     }
@@ -281,46 +346,17 @@ class Chaintracks {
         let after = before;
         let added = HeightRange_1.HeightRange.empty;
         const maxSyncRounds = Math.max(1, this.bulkIngestors.length * 2);
-        let done = false;
-        for (let round = 1; !done && round <= maxSyncRounds; round++) {
-            let bulkSyncError;
-            let roundMadeProgress = false;
-            let roundHadSuccess = false;
-            for (const bulk of this.bulkIngestors) {
-                try {
-                    const beforeBulkMax = before.bulk.maxHeight;
-                    const beforeLiveRange = HeightRange_1.HeightRange.from(newLiveHeaders);
-                    const r = await bulk.synchronize(presentHeight, before, newLiveHeaders);
-                    roundHadSuccess = true;
-                    newLiveHeaders = r.liveHeaders;
-                    after = await this.storage.getAvailableHeightRanges();
-                    added = after.bulk.above(before.bulk);
-                    const afterLiveRange = HeightRange_1.HeightRange.from(newLiveHeaders);
-                    if (after.bulk.maxHeight > beforeBulkMax || afterLiveRange.maxHeight > beforeLiveRange.maxHeight) {
-                        roundMadeProgress = true;
-                    }
-                    before = after;
-                    this.log(`Bulk Ingestor: ${added.length} added with ${newLiveHeaders.length} live headers from ${bulk.constructor.name}`);
-                    if (r.done) {
-                        done = true;
-                        break;
-                    }
-                }
-                catch (eu) {
-                    const e = (bulkSyncError = WalletError_1.WalletError.fromUnknown(eu));
-                    this.log(`bulk sync error: ${e.message}`);
-                    if (!this.available)
-                        // During initial startup, bulk ingestors must be available.
-                        break;
-                }
-            }
-            if (!this.available && bulkSyncError && !roundHadSuccess) {
-                this.startupError = bulkSyncError;
+        for (let round = 1; round <= maxSyncRounds; round++) {
+            const result = await this.runBulkSyncRound(before, presentHeight, newLiveHeaders);
+            after = result.after;
+            newLiveHeaders = result.newLiveHeaders;
+            added = after.bulk.above(before.bulk);
+            before = after;
+            if (this.startupError != null)
                 break;
-            }
-            if (done)
+            if (result.done)
                 break;
-            if (!roundMadeProgress) {
+            if (!result.madeProgress) {
                 this.log(`Bulk sync stalled after round ${round}. Deferring further bulk sync attempts to continue live header processing.`);
                 break;
             }
@@ -328,7 +364,7 @@ class Chaintracks {
                 this.log(`Bulk sync paused after ${maxSyncRounds} rounds to avoid runaway retries. Will retry in a later sync cycle.`);
             }
         }
-        if (!this.startupError) {
+        if (this.startupError == null) {
             this.liveHeaders.unshift(...newLiveHeaders);
             added = after.bulk.above(initialRanges.bulk);
             this.log(`syncBulkStorage done
@@ -339,11 +375,80 @@ class Chaintracks {
 `);
         }
     }
+    async runBulkSyncRound(before, presentHeight, newLiveHeaders) {
+        let after = before;
+        let bulkSyncError;
+        let madeProgress = false;
+        let hadSuccess = false;
+        let done = false;
+        for (const [index, bulk] of this.bulkIngestors.entries()) {
+            const source = this.sourceName('bulk', index, bulk);
+            try {
+                const beforeBulkMax = before.bulk.maxHeight;
+                const beforeLiveRange = HeightRange_1.HeightRange.from(newLiveHeaders);
+                const r = await bulk.synchronize(presentHeight, before, newLiveHeaders);
+                hadSuccess = true;
+                this.markSourceSuccess(source, 'bulk');
+                newLiveHeaders = r.liveHeaders;
+                after = await this.storage.getAvailableHeightRanges();
+                const added = after.bulk.above(before.bulk);
+                const afterLiveRange = HeightRange_1.HeightRange.from(newLiveHeaders);
+                if (after.bulk.maxHeight > beforeBulkMax || afterLiveRange.maxHeight > beforeLiveRange.maxHeight)
+                    madeProgress = true;
+                before = after;
+                this.log(`Bulk Ingestor: ${added.length} added with ${newLiveHeaders.length} live headers from ${bulk.constructor.name}`);
+                if (r.done) {
+                    done = true;
+                    break;
+                }
+            }
+            catch (error_) {
+                const e = (bulkSyncError = WalletError_1.WalletError.fromUnknown(error_));
+                this.markSourceFailure(source, 'bulk', e);
+                this.log(`bulk sync error: ${e.message}`);
+            }
+        }
+        if (!this.available && bulkSyncError != null && !hadSuccess)
+            this.startupError = bulkSyncError;
+        return { after, newLiveHeaders, done, madeProgress };
+    }
+    sourceName(role, index, source) {
+        return `${role}[${index}]:${source.constructor.name}`;
+    }
+    markSourceSuccess(name, role) {
+        this.sourceStatus.set(name, {
+            ...this.sourceStatus.get(name),
+            name,
+            role,
+            state: 'healthy',
+            lastSuccess: new Date().toISOString(),
+            error: undefined
+        });
+    }
+    markSourceFailure(name, role, error) {
+        this.sourceStatus.set(name, {
+            ...this.sourceStatus.get(name),
+            name,
+            role,
+            state: 'degraded',
+            lastFailure: new Date().toISOString(),
+            error: error.message
+        });
+    }
     async getMissingBlockHeader(hash) {
-        for (const live of this.liveIngestors) {
-            const header = await live.getHeaderByHash(hash);
-            if (header)
-                return header;
+        for (const [index, live] of this.liveIngestors.entries()) {
+            const source = this.sourceName('live', index, live);
+            try {
+                const header = await live.getHeaderByHash(hash);
+                this.markSourceSuccess(source, 'live');
+                if (header != null)
+                    return header;
+            }
+            catch (error) {
+                const resolved = WalletError_1.WalletError.fromUnknown(error);
+                this.markSourceFailure(source, 'live', resolved);
+                this.log(`Header lookup source ${source} failed: ${resolved.message}`);
+            }
         }
         return undefined;
     }
@@ -359,36 +464,40 @@ class Chaintracks {
         if (this.invalidInsertHeaderResult(ihr))
             return ihr;
         if (this.subscriberCallbacksEnabled && ihr.added && ihr.isActiveTip) {
-            // If a new active chaintip has been added, notify subscribed event listeners...
-            for (const id in this.callbacks.header) {
-                const addListener = this.callbacks.header[id];
-                if (addListener) {
-                    try {
-                        addListener(header);
-                    }
-                    catch (_a) {
-                        /* ignore all errors thrown */
-                    }
+            this.notifyHeaderListeners(header);
+            if (ihr.reorgDepth > 0 && ihr.priorTip != null) {
+                this.notifyReorgListeners(ihr, header);
+            }
+        }
+        return ihr;
+    }
+    notifyHeaderListeners(header) {
+        for (const id in this.callbacks.header) {
+            const listener = this.callbacks.header[id];
+            if (listener != null) {
+                try {
+                    listener(header);
+                }
+                catch {
+                    /* ignore all errors thrown */
                 }
             }
-            if (ihr.reorgDepth > 0 && ihr.priorTip) {
-                // If the new header was also a reorg, notify subscribed event listeners...
-                for (const id in this.callbacks.reorg) {
-                    const reorgListener = this.callbacks.reorg[id];
-                    if (reorgListener) {
-                        try {
-                            const priorTip = { ...ihr.priorTip };
-                            const deactivated = ihr.deactivatedHeaders.map(lbh => ({ ...lbh }));
-                            reorgListener(ihr.reorgDepth, priorTip, header, deactivated);
-                        }
-                        catch (_b) {
-                            /* ignore all errors thrown */
-                        }
-                    }
+        }
+    }
+    notifyReorgListeners(ihr, header) {
+        const priorTip = { ...ihr.priorTip };
+        const deactivated = ihr.deactivatedHeaders.map(lbh => ({ ...lbh }));
+        for (const id in this.callbacks.reorg) {
+            const listener = this.callbacks.reorg[id];
+            if (listener != null) {
+                try {
+                    listener(ihr.reorgDepth, priorTip, header, deactivated);
+                }
+                catch {
+                    /* ignore all errors thrown */
                 }
             }
         }
-        return ihr;
     }
     /**
      * Long running method terminated by setting `stopMainThread` false.
@@ -404,162 +513,163 @@ class Chaintracks {
      */
     async mainThreadShiftLiveHeaders() {
         this.stopMainThread = false;
-        let lastSyncCheck = Date.now();
+        let lastSyncCheck = 0;
         let lastBulkSync = Date.now();
         const cdnSyncRepeatMsecs = 24 * 60 * 60 * 1000; // 24 hours
         const syncCheckRepeatMsecs = 30 * 60 * 1000; // 30 minutes
         while (!this.stopMainThread) {
             try {
-                // Review the need for bulk sync...
                 const now = Date.now();
                 lastSyncCheck = now;
-                const presentHeight = await this.getPresentHeight();
-                const before = await this.storage.getAvailableHeightRanges();
-                // Skip bulk sync if within less than half the recursion limit of present height
-                let skipBulkSync = !before.live.isEmpty && before.live.maxHeight >= presentHeight - this.addLiveRecursionLimit / 2;
-                if (skipBulkSync && now - lastBulkSync > cdnSyncRepeatMsecs) {
-                    // If we haven't re-synced in a long time, do it just to check for a CDN update.
-                    skipBulkSync = false;
+                lastBulkSync = await this.runBulkSyncIfNeeded(now, lastBulkSync, cdnSyncRepeatMsecs);
+                await this.processLiveHeaderQueue(lastSyncCheck, syncCheckRepeatMsecs);
+            }
+            catch (error_) {
+                const e = WalletError_1.WalletError.fromUnknown(error_);
+                if (this.available) {
+                    this.log(`Error occurred during chaintracks main thread processing: ${e.stack || e.message}`);
+                }
+                else {
+                    this.startupError = e;
+                    this.stopMainThread = true;
                 }
-                this.log(`Chaintracks Update Services: Bulk Header Sync Review
+            }
+        }
+    }
+    /** Returns (potentially updated) lastBulkSync timestamp. */
+    async runBulkSyncIfNeeded(now, lastBulkSync, cdnSyncRepeatMsecs) {
+        const presentHeight = await this.getPresentHeight();
+        const before = await this.storage.getAvailableHeightRanges();
+        let skipBulkSync = !before.live.isEmpty && before.live.maxHeight >= presentHeight - this.addLiveRecursionLimit / 2;
+        // If we haven't re-synced in a long time, do it just to check for a CDN update.
+        if (skipBulkSync && now - lastBulkSync > cdnSyncRepeatMsecs)
+            skipBulkSync = false;
+        this.log(`Chaintracks Update Services: Bulk Header Sync Review
   presentHeight=${presentHeight}   addLiveRecursionLimit=${this.addLiveRecursionLimit}
   Before synchronize: bulk ${before.bulk}, live ${before.live}
   ${skipBulkSync ? 'Skipping' : 'Starting'} syncBulkStorage.
 `);
-                if (!skipBulkSync) {
-                    // Bring bulk storage up-to-date and (re-)initialize liveHeaders
-                    lastBulkSync = now;
-                    if (this.available)
-                        // Once available, initial write lock is released, take a new one to update bulk storage.
-                        await this.syncBulkStorage(presentHeight, before);
-                    else
-                        // While still not available, the makeAvailable write lock is held.
-                        await this.syncBulkStorageNoLock(presentHeight, before);
-                    if (this.startupError)
-                        throw this.startupError;
-                }
-                let count = 0;
-                let liveHeaderDupes = 0;
-                let needSyncCheck = false;
-                for (; !needSyncCheck && !this.stopMainThread;) {
-                    let header = this.liveHeaders.shift();
-                    if (header) {
-                        // Process a "live" block header...
-                        let recursions = this.addLiveRecursionLimit;
-                        for (; !needSyncCheck && !this.stopMainThread;) {
-                            //console.log(`Processing liveHeader: height: ${header.height} hash: ${header.hash} ${new Date().toISOString()}`)
-                            const ihr = await this.addLiveHeader(header);
-                            if (this.invalidInsertHeaderResult(ihr)) {
-                                this.log(`Ignoring liveHeader ${header.height} ${header.hash} due to invalid insert result.`);
-                                needSyncCheck = true;
-                            }
-                            else if (ihr.noPrev) {
-                                // Previous header is unknown, request it by hash from the network and try adding it first...
-                                if (recursions-- <= 0) {
-                                    // Ignore this header...
-                                    this.log(`Ignoring liveHeader ${header.height} ${header.hash} addLiveRecursionLimit=${this.addLiveRecursionLimit} exceeded.`);
-                                    needSyncCheck = true;
-                                }
-                                else {
-                                    const hash = header.previousHash;
-                                    const prevHeader = await this.getMissingBlockHeader(hash);
-                                    if (!prevHeader) {
-                                        this.log(`Ignoring liveHeader ${header.height} ${header.hash} failed to find previous header by hash ${(0, utilityHelpers_noBuffer_1.asString)(hash)}`);
-                                        needSyncCheck = true;
-                                    }
-                                    else {
-                                        // Switch to trying to add prevHeader, unshifting current header to try it again after prevHeader exists.
-                                        this.liveHeaders.unshift(header);
-                                        header = prevHeader;
-                                    }
-                                }
-                            }
-                            else {
-                                if (this.subscriberCallbacksEnabled)
-                                    this.log(`addLiveHeader ${header.height}${ihr.added ? ' added' : ''}${ihr.dupe ? ' dupe' : ''}${ihr.isActiveTip ? ' isActiveTip' : ''}${ihr.reorgDepth ? ' reorg depth ' + ihr.reorgDepth : ''}${ihr.noPrev ? ' noPrev' : ''}${ihr.noActiveAncestor || ihr.noTip || ihr.badPrev ? ' error' : ''}`);
-                                if (ihr.dupe) {
-                                    liveHeaderDupes++;
-                                }
-                                // Header wasn't invalid and previous header is known. If it was successfully added, count it as a win.
-                                if (ihr.added) {
-                                    count++;
-                                }
-                                break;
-                            }
-                        }
-                    }
-                    else {
-                        // There are no liveHeaders currently to process, check the out-of-band baseHeaders channel (`addHeader` method called by a client).
-                        const bheader = this.baseHeaders.shift();
-                        if (bheader) {
-                            const prev = await this.storage.findLiveHeaderForBlockHash(bheader.previousHash);
-                            if (!prev) {
-                                // Ignoring attempt to add a baseHeader with unknown previous hash, no attempt made to find previous header(s).
-                                this.log(`Ignoring header with unknown previousHash ${bheader.previousHash} in live storage.`);
-                                // Does not trigger a re-sync.
-                            }
-                            else {
-                                const header = {
-                                    ...bheader,
-                                    height: prev.height + 1,
-                                    hash: (0, blockHeaderUtilities_1.blockHash)(bheader)
-                                };
-                                const ihr = await this.addLiveHeader(header);
-                                if (this.invalidInsertHeaderResult(ihr)) {
-                                    this.log(`Ignoring invalid baseHeader ${header.height} ${header.hash}.`);
-                                }
-                                else {
-                                    if (this.subscriberCallbacksEnabled)
-                                        this.log(`addBaseHeader ${header.height}${ihr.added ? ' added' : ''}${ihr.dupe ? ' dupe' : ''}${ihr.isActiveTip ? ' isActiveTip' : ''}${ihr.reorgDepth ? ' reorg depth ' + ihr.reorgDepth : ''}${ihr.noPrev ? ' noPrev' : ''}${ihr.noActiveAncestor || ihr.noTip || ihr.badPrev ? ' error' : ''}`);
-                                    // baseHeader was successfully added.
-                                    if (ihr.added) {
-                                        count++;
-                                    }
-                                }
-                            }
-                        }
-                        else {
-                            // There are no liveHeaders and no baseHeaders to add,
-                            if (count > 0) {
-                                if (liveHeaderDupes > 0) {
-                                    this.log(`${liveHeaderDupes} duplicate headers ignored.`);
-                                    liveHeaderDupes = 0;
-                                }
-                                const updated = await this.storage.getAvailableHeightRanges();
-                                this.log(`After adding ${count} live headers
-   After live: bulk ${updated.bulk}, live ${updated.live}
-`);
-                                count = 0;
-                            }
-                            if (!this.subscriberCallbacksEnabled) {
-                                const live = await this.storage.findLiveHeightRange();
-                                if (!live.isEmpty) {
-                                    this.subscriberCallbacksEnabled = true;
-                                    this.log(`listening at height of ${live.maxHeight}`);
-                                }
-                            }
-                            if (!this.available) {
-                                this.available = true;
-                            }
-                            needSyncCheck = Date.now() - lastSyncCheck > syncCheckRepeatMsecs;
-                            // If we aren't going to review sync, wait before checking input queues again
-                            if (!needSyncCheck)
-                                await (0, utilityHelpers_1.wait)(1000);
-                        }
-                    }
-                }
+        if (!skipBulkSync) {
+            // Once available, the initial write lock is released; take a new one to update bulk storage.
+            // While not yet available, the makeAvailable write lock is still held.
+            if (this.available)
+                await this.syncBulkStorage(presentHeight, before);
+            else
+                await this.syncBulkStorageNoLock(presentHeight, before);
+            if (this.startupError != null)
+                throw this.startupError;
+            return now;
+        }
+        return lastBulkSync;
+    }
+    async processNextQueuedHeader(stats) {
+        const liveHeader = this.liveHeaders.shift();
+        if (liveHeader != null) {
+            const result = await this.processOneLiveHeader(liveHeader);
+            if (result.needSyncCheck)
+                return true;
+            if (result.dupe)
+                stats.liveHeaderDupes++;
+            if (result.added)
+                stats.count++;
+            return false;
+        }
+        const baseHeader = this.baseHeaders.shift();
+        if (baseHeader == null)
+            return null;
+        if (await this.processOneBaseHeader(baseHeader))
+            stats.count++;
+        return false;
+    }
+    async flushLiveHeaderProgress(stats) {
+        if (stats.count === 0)
+            return;
+        if (stats.liveHeaderDupes > 0) {
+            this.log(`${stats.liveHeaderDupes} duplicate headers ignored.`);
+            stats.liveHeaderDupes = 0;
+        }
+        const updated = await this.storage.getAvailableHeightRanges();
+        this.log(`After adding ${stats.count} live headers\n   After live: bulk ${updated.bulk}, live ${updated.live}\n`);
+        stats.count = 0;
+    }
+    async waitForQueuedHeaders(stats, lastSyncCheck, syncCheckRepeatMsecs) {
+        await this.flushLiveHeaderProgress(stats);
+        await this.checkAndEnableSubscribers();
+        if (!this.available)
+            this.available = true;
+        const needSyncCheck = Date.now() - lastSyncCheck > syncCheckRepeatMsecs;
+        if (!needSyncCheck)
+            await (0, utilityHelpers_1.wait)(1000);
+        return needSyncCheck;
+    }
+    async processLiveHeaderQueue(lastSyncCheck, syncCheckRepeatMsecs) {
+        const stats = { count: 0, liveHeaderDupes: 0 };
+        let needSyncCheck = false;
+        while (!needSyncCheck && !this.stopMainThread) {
+            const queuedResult = await this.processNextQueuedHeader(stats);
+            needSyncCheck = queuedResult ?? (await this.waitForQueuedHeaders(stats, lastSyncCheck, syncCheckRepeatMsecs));
+        }
+    }
+    formatIhrLog(prefix, header, ihr) {
+        return `${prefix} ${header.height}${ihr.added ? ' added' : ''}${ihr.dupe ? ' dupe' : ''}${ihr.isActiveTip ? ' isActiveTip' : ''}${ihr.reorgDepth ? ' reorg depth ' + ihr.reorgDepth : ''}${ihr.noPrev ? ' noPrev' : ''}${ihr.noActiveAncestor || ihr.noTip || ihr.badPrev ? ' error' : ''}`;
+    }
+    async processOneLiveHeader(startHeader) {
+        let header = startHeader;
+        let recursions = this.addLiveRecursionLimit;
+        while (!this.stopMainThread) {
+            const ihr = await this.addLiveHeader(header);
+            if (this.invalidInsertHeaderResult(ihr)) {
+                this.log(`Ignoring liveHeader ${header.height} ${header.hash} due to invalid insert result.`);
+                return { needSyncCheck: true, dupe: false, added: false };
             }
-            catch (eu) {
-                const e = WalletError_1.WalletError.fromUnknown(eu);
-                if (!this.available) {
-                    this.startupError = e;
-                    this.stopMainThread = true;
+            if (ihr.noPrev) {
+                // Previous header is unknown; request it by hash from the network and try adding it first.
+                if (recursions-- <= 0) {
+                    this.log(`Ignoring liveHeader ${header.height} ${header.hash} addLiveRecursionLimit=${this.addLiveRecursionLimit} exceeded.`);
+                    return { needSyncCheck: true, dupe: false, added: false };
                 }
-                else {
-                    this.log(`Error occurred during chaintracks main thread processing: ${e.stack || e.message}`);
+                const prevHeader = await this.getMissingBlockHeader(header.previousHash);
+                if (prevHeader == null) {
+                    this.log(`Ignoring liveHeader ${header.height} ${header.hash} failed to find previous header by hash ${(0, utilityHelpers_noBuffer_1.asString)(header.previousHash)}`);
+                    return { needSyncCheck: true, dupe: false, added: false };
                 }
+                // Retry adding prevHeader first; then re-queue current header.
+                this.liveHeaders.unshift(header);
+                header = prevHeader;
+            }
+            else {
+                if (this.subscriberCallbacksEnabled)
+                    this.log(this.formatIhrLog('addLiveHeader', header, ihr));
+                return { needSyncCheck: false, dupe: ihr.dupe, added: ihr.added };
             }
         }
+        return { needSyncCheck: false, dupe: false, added: false };
+    }
+    async processOneBaseHeader(bheader) {
+        const prev = await this.storage.findLiveHeaderForBlockHash(bheader.previousHash);
+        if (prev == null) {
+            // Unknown previous hash — ignore without triggering a re-sync.
+            this.log(`Ignoring header with unknown previousHash ${bheader.previousHash} in live storage.`);
+            return false;
+        }
+        const header = { ...bheader, height: prev.height + 1, hash: (0, blockHeaderUtilities_1.blockHash)(bheader) };
+        const ihr = await this.addLiveHeader(header);
+        if (this.invalidInsertHeaderResult(ihr)) {
+            this.log(`Ignoring invalid baseHeader ${header.height} ${header.hash}.`);
+            return false;
+        }
+        if (this.subscriberCallbacksEnabled)
+            this.log(this.formatIhrLog('addBaseHeader', header, ihr));
+        return ihr.added;
+    }
+    async checkAndEnableSubscribers() {
+        if (this.subscriberCallbacksEnabled)
+            return;
+        const live = await this.storage.findLiveHeightRange();
+        if (!live.isEmpty) {
+            this.subscriberCallbacksEnabled = true;
+            this.log(`listening at height of ${live.maxHeight}`);
+        }
     }
 }
 exports.Chaintracks = Chaintracks;