@cogcoin/client 0.5.5 → 0.5.7

package/dist/bitcoind/bootstrap/headers.js

@@ -1,5 +1,9 @@
+import { open } from "node:fs/promises";
 import { formatManagedSyncErrorMessage } from "../errors.js";
+import { MANAGED_RPC_RETRY_MESSAGE, consumeManagedRpcRetryDelayMs, createManagedRpcRetryState, describeManagedRpcRetryError, resetManagedRpcRetryState, isRetryableManagedRpcError, } from "../retryable-rpc.js";
 import { DEFAULT_SNAPSHOT_METADATA, HEADER_NO_PEER_TIMEOUT_MS, HEADER_POLL_MS, } from "./constants.js";
+const DEBUG_LOG_TAIL_BYTES = 64 * 1024;
+const HEADER_SYNC_DEBUG_LINE_PATTERN = /Pre-synchronizing blockheaders,\s*height:\s*([\d,]+)\s*\(~(\d+(?:\.\d+)?)%\)/u;
 function createAbortError(signal) {
     const reason = signal?.reason;
     if (reason instanceof Error) {
@@ -33,7 +37,52 @@ function resolvePeerCount(networkInfo) {
         ? networkInfo.connections
         : (networkInfo.connections_in ?? 0) + (networkInfo.connections_out ?? 0);
 }
-function resolveHeaderWaitMessage(headers, peerCount, networkActive) {
+async function readDebugLogTail(filePath, maxBytes = DEBUG_LOG_TAIL_BYTES) {
+    let handle = null;
+    try {
+        handle = await open(filePath, "r");
+        const stats = await handle.stat();
+        const bytesToRead = Math.min(maxBytes, Math.max(0, stats.size));
+        if (bytesToRead === 0) {
+            return null;
+        }
+        const buffer = Buffer.alloc(bytesToRead);
+        await handle.read(buffer, 0, bytesToRead, stats.size - bytesToRead);
+        return buffer.toString("utf8");
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT") {
+            return null;
+        }
+        return null;
+    }
+    finally {
+        await handle?.close().catch(() => { });
+    }
+}
+async function readHeaderSyncProgressFromDebugLog(debugLogPath) {
+    const tail = await readDebugLogTail(debugLogPath);
+    if (tail === null) {
+        return null;
+    }
+    const lines = tail.split(/\r?\n/u).reverse();
+    for (const line of lines) {
+        const match = HEADER_SYNC_DEBUG_LINE_PATTERN.exec(line);
+        if (match === null) {
+            continue;
+        }
+        const height = Number(match[1].replaceAll(",", ""));
+        if (!Number.isFinite(height) || height < 0) {
+            return null;
+        }
+        return {
+            height,
+            message: `Pre-synchronizing blockheaders, height: ${height.toLocaleString()} (~${match[2]}%)`,
+        };
+    }
+    return null;
+}
+function resolveHeaderWaitMessage(headers, peerCount, networkActive, rpcHeaders, headerSyncMessage) {
     if (!networkActive) {
         return "Bitcoin networking is inactive for the managed node.";
     }
@@ -43,33 +92,69 @@ function resolveHeaderWaitMessage(headers, peerCount, networkActive) {
     if (peerCount === 0) {
         return `Waiting for peers to continue header sync (${headers.toLocaleString()} headers, 0 peers).`;
     }
-    return "Waiting for Bitcoin headers to reach the snapshot height.";
+    if (rpcHeaders > 0) {
+        return "Waiting for Bitcoin headers to reach the snapshot height.";
+    }
+    return headerSyncMessage ?? "Pre-synchronizing blockheaders.";
 }
 export async function waitForHeaders(rpc, snapshot, progress, options = {}) {
     const now = options.now ?? Date.now;
     const sleepImpl = options.sleep ?? sleep;
     const noPeerTimeoutMs = options.noPeerTimeoutMs ?? HEADER_NO_PEER_TIMEOUT_MS;
     const { signal } = options;
+    const retryState = options.retryState ?? createManagedRpcRetryState();
+    const readDebugLogProgress = options.readDebugLogProgress ?? readHeaderSyncProgressFromDebugLog;
     let noPeerSince = null;
+    let lastBlocks = 0;
+    let lastHeaders = 0;
     while (true) {
         throwIfAborted(signal);
-        const [info, networkInfo] = await Promise.all([
-            rpc.getBlockchainInfo(),
-            rpc.getNetworkInfo(),
-        ]);
+        let info;
+        let networkInfo;
+        try {
+            [info, networkInfo] = await Promise.all([
+                rpc.getBlockchainInfo(),
+                rpc.getNetworkInfo(),
+            ]);
+            resetManagedRpcRetryState(retryState);
+        }
+        catch (error) {
+            if (!isRetryableManagedRpcError(error)) {
+                throw error;
+            }
+            await progress.setPhase("wait_headers_for_snapshot", {
+                headers: lastHeaders,
+                targetHeight: snapshot.height,
+                blocks: lastBlocks,
+                percent: (Math.min(lastHeaders, snapshot.height) / snapshot.height) * 100,
+                lastError: describeManagedRpcRetryError(error),
+                message: MANAGED_RPC_RETRY_MESSAGE,
+            });
+            await sleepImpl(consumeManagedRpcRetryDelayMs(retryState), signal);
+            continue;
+        }
+        lastBlocks = info.blocks;
+        const debugLogProgress = info.headers === 0 && options.debugLogPath !== undefined
+            ? await readDebugLogProgress(options.debugLogPath)
+            : null;
+        const observedHeaders = info.headers > 0
+            ? info.headers
+            : Math.max(info.headers, debugLogProgress?.height ?? 0);
+        lastHeaders = observedHeaders;
         const peerCount = resolvePeerCount(networkInfo);
-        const message = resolveHeaderWaitMessage(info.headers, peerCount, networkInfo.networkactive);
+        const message = resolveHeaderWaitMessage(observedHeaders, peerCount, networkInfo.networkactive, info.headers, debugLogProgress?.message ?? null);
         await progress.setPhase("wait_headers_for_snapshot", {
-            headers: info.headers,
+            headers: observedHeaders,
             targetHeight: snapshot.height,
             blocks: info.blocks,
-            percent: (Math.min(info.headers, snapshot.height) / snapshot.height) * 100,
+            percent: (Math.min(observedHeaders, snapshot.height) / snapshot.height) * 100,
+            lastError: null,
             message,
         });
         if (info.headers >= snapshot.height) {
             return;
         }
-        if (info.headers === 0 && peerCount === 0) {
+        if (observedHeaders === 0 && peerCount === 0) {
             noPeerSince ??= now();
             if (now() - noPeerSince >= noPeerTimeoutMs) {
                 throw new Error(formatManagedSyncErrorMessage("bitcoind_no_peers_for_header_sync_check_internet_or_firewall"));

package/dist/bitcoind/bootstrap.d.ts

@@ -1,6 +1,7 @@
 export { DEFAULT_SNAPSHOT_METADATA } from "./bootstrap/constants.js";
 export { AssumeUtxoBootstrapController } from "./bootstrap/controller.js";
 export { downloadSnapshotFileForTesting } from "./bootstrap/download.js";
+export { prepareLatestGetblockArchive, prepareLatestGetblockArchiveForTesting, resolveGetblockArchivePathsForTesting, resolveReadyGetblockArchiveForTesting, waitForGetblockArchiveImport, waitForGetblockArchiveImportForTesting, } from "./bootstrap/getblock-archive.js";
 export { waitForHeadersForTesting } from "./bootstrap/headers.js";
 export { resolveBootstrapPathsForTesting } from "./bootstrap/paths.js";
 export { loadBootstrapStateForTesting, saveBootstrapStateForTesting, createBootstrapStateForTesting, } from "./bootstrap/state.js";

package/dist/bitcoind/bootstrap.js

@@ -1,6 +1,7 @@
 export { DEFAULT_SNAPSHOT_METADATA } from "./bootstrap/constants.js";
 export { AssumeUtxoBootstrapController } from "./bootstrap/controller.js";
 export { downloadSnapshotFileForTesting } from "./bootstrap/download.js";
+export { prepareLatestGetblockArchive, prepareLatestGetblockArchiveForTesting, resolveGetblockArchivePathsForTesting, resolveReadyGetblockArchiveForTesting, waitForGetblockArchiveImport, waitForGetblockArchiveImportForTesting, } from "./bootstrap/getblock-archive.js";
 export { waitForHeadersForTesting } from "./bootstrap/headers.js";
 export { resolveBootstrapPathsForTesting } from "./bootstrap/paths.js";
 export { loadBootstrapStateForTesting, saveBootstrapStateForTesting, createBootstrapStateForTesting, } from "./bootstrap/state.js";

package/dist/bitcoind/client/factory.js

@@ -1,47 +1,110 @@
 import { loadBundledGenesisParameters } from "@cogcoin/indexer";
 import { resolveDefaultBitcoindDataDirForTesting } from "../../app-paths.js";
 import { openClient } from "../../client.js";
-import { AssumeUtxoBootstrapController, DEFAULT_SNAPSHOT_METADATA, resolveBootstrapPathsForTesting } from "../bootstrap.js";
+import { AssumeUtxoBootstrapController, DEFAULT_SNAPSHOT_METADATA, prepareLatestGetblockArchiveForTesting, resolveBootstrapPathsForTesting, } from "../bootstrap.js";
 import { attachOrStartIndexerDaemon } from "../indexer-daemon.js";
 import { createRpcClient } from "../node.js";
+import { assertCogcoinProcessingStartHeight, resolveCogcoinProcessingStartHeight, } from "../processing-start-height.js";
 import { ManagedProgressController } from "../progress.js";
-import { attachOrStartManagedBitcoindService } from "../service.js";
+import { attachOrStartManagedBitcoindService, probeManagedBitcoindService, stopManagedBitcoindService, } from "../service.js";
 import { DefaultManagedBitcoindClient } from "./managed-client.js";
 const DEFAULT_SYNC_DEBOUNCE_MS = 250;
 async function createManagedBitcoindClient(options) {
     const genesisParameters = options.genesisParameters ?? await loadBundledGenesisParameters();
-    const dataDir = options.dataDir ?? resolveDefaultBitcoindDataDirForTesting();
-    const node = await attachOrStartManagedBitcoindService({
-        ...options,
-        dataDir,
+    assertCogcoinProcessingStartHeight({
+        chain: options.chain,
+        startHeight: options.startHeight,
+        genesisParameters,
     });
-    const rpc = createRpcClient(node.rpc);
+    const dataDir = options.dataDir ?? resolveDefaultBitcoindDataDirForTesting();
     const progress = new ManagedProgressController({
         onProgress: options.onProgress,
         progressOutput: options.progressOutput,
-        quoteStatePath: resolveBootstrapPathsForTesting(node.dataDir, DEFAULT_SNAPSHOT_METADATA).quoteStatePath,
+        quoteStatePath: resolveBootstrapPathsForTesting(dataDir, DEFAULT_SNAPSHOT_METADATA).quoteStatePath,
         snapshot: DEFAULT_SNAPSHOT_METADATA,
     });
-    const bootstrap = new AssumeUtxoBootstrapController({
-        rpc,
-        dataDir: node.dataDir,
-        progress,
-        snapshot: DEFAULT_SNAPSHOT_METADATA,
-    });
-    const client = await openClient({
-        store: options.store,
-        genesisParameters,
-        snapshotInterval: options.snapshotInterval,
-    });
-    const indexerDaemon = options.databasePath
-        ? await attachOrStartIndexerDaemon({
+    let progressStarted = false;
+    try {
+        await progress.start();
+        progressStarted = true;
+        let getblockArchive = options.chain === "main"
+            ? await prepareLatestGetblockArchiveForTesting({
+                dataDir,
+                progress,
+                fetchImpl: options.fetchImpl,
+            })
+            : null;
+        if (options.chain === "main" && getblockArchive !== null) {
+            const existingProbe = await probeManagedBitcoindService({
+                ...options,
+                dataDir,
+            });
+            if (existingProbe.compatibility === "compatible" && existingProbe.status !== null) {
+                const currentArchiveEndHeight = existingProbe.status.getblockArchiveEndHeight ?? null;
+                const currentArchiveSha256 = existingProbe.status.getblockArchiveSha256 ?? null;
+                const nextArchiveEndHeight = getblockArchive.manifest.endHeight;
+                const nextArchiveSha256 = getblockArchive.manifest.artifactSha256;
+                const needsRestart = currentArchiveEndHeight !== nextArchiveEndHeight
+                    || currentArchiveSha256 !== nextArchiveSha256;
+                if (needsRestart) {
+                    const restartApproved = options.confirmGetblockArchiveRestart === undefined
+                        ? false
+                        : await options.confirmGetblockArchiveRestart({
+                            currentArchiveEndHeight,
+                            nextArchiveEndHeight,
+                        });
+                    if (restartApproved) {
+                        await stopManagedBitcoindService({
+                            dataDir,
+                            walletRootId: options.walletRootId,
+                            shutdownTimeoutMs: options.shutdownTimeoutMs,
+                        });
+                    }
+                    else {
+                        getblockArchive = null;
+                    }
+                }
+            }
+        }
+        const node = await attachOrStartManagedBitcoindService({
+            ...options,
             dataDir,
-            databasePath: options.databasePath,
-            walletRootId: options.walletRootId,
-            startupTimeoutMs: options.startupTimeoutMs,
-        })
-        : null;
-    return new DefaultManagedBitcoindClient(client, options.store, node, rpc, progress, bootstrap, indexerDaemon, options.syncDebounceMs ?? DEFAULT_SYNC_DEBOUNCE_MS);
+            getblockArchivePath: getblockArchive?.artifactPath ?? null,
+            getblockArchiveEndHeight: getblockArchive?.manifest.endHeight ?? null,
+            getblockArchiveSha256: getblockArchive?.manifest.artifactSha256 ?? null,
+        });
+        const rpc = createRpcClient(node.rpc);
+        const bootstrap = new AssumeUtxoBootstrapController({
+            rpc,
+            dataDir: node.dataDir,
+            progress,
+            snapshot: DEFAULT_SNAPSHOT_METADATA,
+        });
+        const client = await openClient({
+            store: options.store,
+            genesisParameters,
+            snapshotInterval: options.snapshotInterval,
+        });
+        const indexerDaemon = options.databasePath
+            ? await attachOrStartIndexerDaemon({
+                dataDir,
+                databasePath: options.databasePath,
+                walletRootId: options.walletRootId,
+                startupTimeoutMs: options.startupTimeoutMs,
+            })
+            : null;
+        await indexerDaemon?.pauseBackgroundFollow();
+        // The persistent service may already exist from a non-processing attach path
+        // that used startHeight 0. Cogcoin replay still begins at the requested
+        // processing boundary for this managed client.
+        return new DefaultManagedBitcoindClient(client, options.store, node, rpc, progress, bootstrap, indexerDaemon, options.startHeight, options.syncDebounceMs ?? DEFAULT_SYNC_DEBOUNCE_MS);
+    }
+    catch (error) {
+        if (progressStarted) {
+            await progress.close().catch(() => undefined);
+        }
+        throw error;
+    }
 }
 export async function openManagedBitcoindClient(options) {
     const genesisParameters = options.genesisParameters ?? await loadBundledGenesisParameters();
@@ -49,7 +112,7 @@ export async function openManagedBitcoindClient(options) {
         ...options,
         genesisParameters,
         chain: "main",
-        startHeight: genesisParameters.genesisBlock,
+        startHeight: resolveCogcoinProcessingStartHeight(genesisParameters),
     });
 }
 export async function openManagedBitcoindClientInternal(options) {

package/dist/bitcoind/client/managed-client.d.ts

@@ -7,7 +7,7 @@ import type { BitcoinRpcClient } from "../rpc.js";
 import type { ManagedBitcoindClient, ManagedBitcoindNodeHandle, ManagedBitcoindStatus, SyncResult } from "../types.js";
 export declare class DefaultManagedBitcoindClient implements ManagedBitcoindClient {
     #private;
-    constructor(client: Client, store: ClientStoreAdapter, node: ManagedBitcoindNodeHandle, rpc: BitcoinRpcClient, progress: ManagedProgressController, bootstrap: AssumeUtxoBootstrapController, indexerDaemon: IndexerDaemonClient | null, syncDebounceMs: number);
+    constructor(client: Client, store: ClientStoreAdapter, node: ManagedBitcoindNodeHandle, rpc: BitcoinRpcClient, progress: ManagedProgressController, bootstrap: AssumeUtxoBootstrapController, indexerDaemon: IndexerDaemonClient | null, startHeight: number, syncDebounceMs: number);
     getTip(): Promise<import("../../types.js").ClientTip | null>;
     getState(): Promise<import("@cogcoin/indexer/types").IndexerState>;
     applyBlock(block: BitcoinBlock): Promise<import("../../types.js").ApplyBlockResult>;

package/dist/bitcoind/client/managed-client.js

@@ -22,7 +22,7 @@ export class DefaultManagedBitcoindClient {
     #syncPromise = Promise.resolve(createInitialSyncResult());
     #debounceTimer = null;
     #syncAbortControllers = new Set();
-    constructor(client, store, node, rpc, progress, bootstrap, indexerDaemon, syncDebounceMs) {
+    constructor(client, store, node, rpc, progress, bootstrap, indexerDaemon, startHeight, syncDebounceMs) {
         this.#client = client;
         this.#store = store;
         this.#node = node;
@@ -30,7 +30,7 @@ export class DefaultManagedBitcoindClient {
         this.#progress = progress;
         this.#bootstrap = bootstrap;
         this.#indexerDaemon = indexerDaemon;
-        this.#startHeight = node.startHeight;
+        this.#startHeight = startHeight;
         this.#syncDebounceMs = syncDebounceMs;
     }
     async getTip() {
@@ -176,8 +176,9 @@ export class DefaultManagedBitcoindClient {
         await this.#syncPromise.catch(() => undefined);
         await this.#progress.close();
         await this.#node.stop();
-        await this.#indexerDaemon?.close();
         await this.#client.close();
+        await this.#indexerDaemon?.resumeBackgroundFollow().catch(() => undefined);
+        await this.#indexerDaemon?.close();
     }
     async playSyncCompletionScene() {
         this.#assertOpen();

package/dist/bitcoind/client/sync-engine.js

@@ -1,5 +1,7 @@
+import { waitForGetblockArchiveImport } from "../bootstrap.js";
 import { formatManagedSyncErrorMessage } from "../errors.js";
 import { normalizeRpcBlock } from "../normalize.js";
+import { MANAGED_RPC_RETRY_MESSAGE, consumeManagedRpcRetryDelayMs, createManagedRpcRetryState, describeManagedRpcRetryError, isRetryableManagedRpcError, resetManagedRpcRetryState, } from "../retryable-rpc.js";
 import { estimateEtaSeconds } from "./rate-tracker.js";
 const DEFAULT_SYNC_CATCH_UP_POLL_MS = 2_000;
 function createAbortError(signal) {
@@ -44,12 +46,42 @@ async function setBitcoinSyncProgress(dependencies, info) {
         headers: info.headers,
         targetHeight: info.headers,
         etaSeconds,
+        lastError: null,
         message: dependencies.node.expectedChain === "main"
             ? "Bitcoin Core is syncing blocks after assumeutxo bootstrap."
             : "Reading blocks from the managed Bitcoin node.",
     });
 }
-async function findCommonAncestor(dependencies, tip, bestHeight) {
+async function setRetryingProgress(dependencies, error) {
+    const status = dependencies.progress.getStatusSnapshot();
+    const { phase: _phase, updatedAt: _updatedAt, ...progress } = status.bootstrapProgress;
+    await dependencies.progress.setPhase(status.bootstrapPhase, {
+        ...progress,
+        lastError: describeManagedRpcRetryError(error),
+        message: MANAGED_RPC_RETRY_MESSAGE,
+    });
+}
+async function runWithManagedRpcRetry(dependencies, retryState, operation) {
+    while (true) {
+        throwIfAborted(dependencies.abortSignal);
+        try {
+            const result = await operation();
+            resetManagedRpcRetryState(retryState);
+            return result;
+        }
+        catch (error) {
+            if (isAbortError(error, dependencies.abortSignal)) {
+                throw createAbortError(dependencies.abortSignal);
+            }
+            if (!isRetryableManagedRpcError(error)) {
+                throw error;
+            }
+            await setRetryingProgress(dependencies, error);
+            await sleep(consumeManagedRpcRetryDelayMs(retryState), dependencies.abortSignal);
+        }
+    }
+}
+async function findCommonAncestor(dependencies, tip, bestHeight, runRpc) {
     const startHeight = Math.min(tip.height, bestHeight);
     for (let height = startHeight; height >= dependencies.startHeight; height -= 1) {
         const localHashHex = height === tip.height
@@ -58,14 +90,14 @@ async function findCommonAncestor(dependencies, tip, bestHeight) {
         if (localHashHex === null) {
             continue;
         }
-        const chainHashHex = await dependencies.rpc.getBlockHash(height);
+        const chainHashHex = await runRpc(() => dependencies.rpc.getBlockHash(height));
         if (chainHashHex === localHashHex) {
             return height;
         }
     }
     return dependencies.startHeight - 1;
 }
-async function syncAgainstBestHeight(dependencies, bestHeight) {
+async function syncAgainstBestHeight(dependencies, bestHeight, runRpc) {
     if (bestHeight < dependencies.startHeight) {
         return {
             appliedBlocks: 0,
@@ -77,7 +109,7 @@ async function syncAgainstBestHeight(dependencies, bestHeight) {
     let rewoundBlocks = 0;
     let commonAncestorHeight = null;
     if (startTip !== null) {
-        const rewindTarget = await findCommonAncestor(dependencies, startTip, bestHeight);
+        const rewindTarget = await findCommonAncestor(dependencies, startTip, bestHeight, runRpc);
         if (rewindTarget < startTip.height) {
             commonAncestorHeight = rewindTarget < dependencies.startHeight ? null : rewindTarget;
             await dependencies.client.rewindToHeight(rewindTarget);
@@ -93,8 +125,8 @@ async function syncAgainstBestHeight(dependencies, bestHeight) {
         await dependencies.progress.setCogcoinSync(nextHeight - 1, bestHeight, estimateEtaSeconds(dependencies.cogcoinRateTracker, nextHeight - 1, bestHeight));
     }
     for (let height = nextHeight; height <= bestHeight; height += 1) {
-        const blockHashHex = await dependencies.rpc.getBlockHash(height);
-        const rpcBlock = await dependencies.rpc.getBlock(blockHashHex);
+        const blockHashHex = await runRpc(() => dependencies.rpc.getBlockHash(height));
+        const rpcBlock = await runRpc(() => dependencies.rpc.getBlock(blockHashHex));
         const normalizedBlock = normalizeRpcBlock(rpcBlock);
         await dependencies.client.applyBlock(normalizedBlock);
         if (typeof rpcBlock.time === "number") {
@@ -111,12 +143,21 @@ async function syncAgainstBestHeight(dependencies, bestHeight) {
 }
 export async function syncToTip(dependencies) {
     try {
+        const retryState = createManagedRpcRetryState();
+        const runRpc = (operation) => runWithManagedRpcRetry(dependencies, retryState, operation);
         throwIfAborted(dependencies.abortSignal);
-        await dependencies.node.validate();
+        await runRpc(() => dependencies.node.validate());
         const indexedTipBeforeBootstrap = await dependencies.client.getTip();
-        await dependencies.bootstrap.ensureReady(indexedTipBeforeBootstrap, dependencies.node.expectedChain, {
+        await runRpc(() => dependencies.bootstrap.ensureReady(indexedTipBeforeBootstrap, dependencies.node.expectedChain, {
             signal: dependencies.abortSignal,
-        });
+            retryState,
+        }));
+        if (dependencies.node.expectedChain === "main"
+            && dependencies.node.getblockArchiveEndHeight !== null) {
+            await waitForGetblockArchiveImport({
+                getBlockchainInfo: () => runRpc(() => dependencies.rpc.getBlockchainInfo()),
+            }, dependencies.progress, dependencies.node.getblockArchiveEndHeight, dependencies.abortSignal);
+        }
         const startTip = await dependencies.client.getTip();
         const aggregate = {
             appliedBlocks: 0,
@@ -129,9 +170,9 @@ export async function syncToTip(dependencies) {
         };
         while (true) {
             throwIfAborted(dependencies.abortSignal);
-            const startInfo = await dependencies.rpc.getBlockchainInfo();
+            const startInfo = await runRpc(() => dependencies.rpc.getBlockchainInfo());
             await setBitcoinSyncProgress(dependencies, startInfo);
-            const pass = await syncAgainstBestHeight(dependencies, startInfo.blocks);
+            const pass = await syncAgainstBestHeight(dependencies, startInfo.blocks, runRpc);
             aggregate.appliedBlocks += pass.appliedBlocks;
             aggregate.rewoundBlocks += pass.rewoundBlocks;
             if (pass.commonAncestorHeight !== null) {
@@ -140,19 +181,20 @@ export async function syncToTip(dependencies) {
                     : Math.min(aggregate.commonAncestorHeight, pass.commonAncestorHeight);
             }
             const finalTip = await dependencies.client.getTip();
-            const endInfo = await dependencies.rpc.getBlockchainInfo();
+            const endInfo = await runRpc(() => dependencies.rpc.getBlockchainInfo());
             const caughtUpCogcoin = endInfo.blocks < dependencies.startHeight || finalTip?.height === endInfo.blocks;
             aggregate.endingHeight = finalTip?.height ?? null;
             aggregate.bestHeight = endInfo.blocks;
             aggregate.bestHashHex = endInfo.bestblockhash;
             if (endInfo.blocks === endInfo.headers && caughtUpCogcoin) {
                 if (dependencies.isFollowing()) {
-                    dependencies.progress.replaceFollowBlockTimes(await dependencies.loadVisibleFollowBlockTimes(finalTip));
+                    dependencies.progress.replaceFollowBlockTimes(await runRpc(() => dependencies.loadVisibleFollowBlockTimes(finalTip)));
                 }
                 await dependencies.progress.setPhase(dependencies.isFollowing() ? "follow_tip" : "complete", {
                     blocks: endInfo.blocks,
                     headers: endInfo.headers,
                     targetHeight: endInfo.headers,
+                    lastError: null,
                     message: dependencies.isFollowing()
                         ? "Following the live Bitcoin tip."
                         : "Managed sync fully caught up to the live tip.",

package/dist/bitcoind/errors.js

@@ -5,6 +5,24 @@ function appendNextStep(message, nextStep) {
     return `${message} Next: ${nextStep}`;
 }
 export function formatManagedSyncErrorMessage(message) {
+    if (message.startsWith("managed_getblock_archive_manifest_http_")) {
+        return appendNextStep(`Getblock archive manifest request failed (${message.replace("managed_getblock_archive_manifest_http_", "HTTP ")}).`, "Wait a moment, confirm the snapshot host is reachable, then rerun sync.");
+    }
+    if (message.startsWith("managed_getblock_archive_http_")) {
+        return appendNextStep(`Getblock archive request failed (${message.replace("managed_getblock_archive_http_", "HTTP ")}).`, "Wait a moment, confirm the snapshot host is reachable, then rerun sync.");
+    }
+    if (message === "managed_getblock_archive_response_body_missing") {
+        return appendNextStep("Getblock archive server returned an empty response body.", "Wait a moment, confirm the snapshot host is reachable, then rerun sync.");
+    }
+    if (message === "managed_getblock_archive_resume_requires_partial_content") {
+        return appendNextStep("Getblock archive server ignored the resume request for a partial download.", "Wait a moment and rerun sync. If this keeps happening, confirm the snapshot host supports HTTP range requests.");
+    }
+    if (message.startsWith("managed_getblock_archive_chunk_sha256_mismatch_")) {
+        return appendNextStep("A downloaded getblock archive chunk was corrupted and was rolled back to the last verified checkpoint.", "Wait a moment and rerun sync. If this keeps happening, check local disk health and the stability of the archive download.");
+    }
+    if (message === "managed_getblock_archive_sha256_mismatch" || message === "managed_getblock_archive_truncated") {
+        return appendNextStep("The downloaded getblock archive did not match the published manifest.", "Rerun sync so the archive can be downloaded again. If this keeps happening, check local disk health and the snapshot host.");
+    }
     if (message === "bitcoind_no_peers_for_header_sync_check_internet_or_firewall") {
         return appendNextStep("No Bitcoin peers were available for header sync.", "Check your internet access and firewall rules for outbound Bitcoin connections, then rerun sync.");
     }

package/dist/bitcoind/indexer-daemon-main.js

@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
 import net from "node:net";
 import { access, constants, mkdir, readFile, rm } from "node:fs/promises";
 import { serializeIndexerState } from "@cogcoin/indexer";
+import { openManagedBitcoindClientInternal } from "./client.js";
 import { openClient } from "../client.js";
 import { openSqliteStore } from "../sqlite/index.js";
 import { writeRuntimeStatusFile } from "../wallet/fs/status-file.js";
@@ -29,6 +30,9 @@ async function readJsonFile(filePath) {
         throw error;
     }
 }
+async function readManagedBitcoindStatus(paths) {
+    return readJsonFile(paths.bitcoindStatusPath);
+}
 async function readPackageVersionFromDisk() {
     try {
         const raw = await readFile(new URL("../../package.json", import.meta.url), "utf8");
@@ -168,6 +172,9 @@ async function main() {
     let lastAppliedAtUnixMs = null;
     let lastError = null;
     let hasSuccessfulCoreTipRefresh = false;
+    let backgroundStore = null;
+    let backgroundClient = null;
+    let backgroundResumePromise = null;
     await mkdir(paths.indexerServiceRoot, { recursive: true });
     await rm(paths.indexerDaemonSocketPath, { force: true }).catch(() => undefined);
     const observeAppliedTip = (appliedTip, now) => {
@@ -264,6 +271,58 @@ async function main() {
         lastError = leaseState.lastError;
         return writeStatus();
     };
+    const pauseBackgroundFollow = async () => {
+        const pendingResume = backgroundResumePromise;
+        backgroundResumePromise = null;
+        await pendingResume?.catch(() => undefined);
+        const client = backgroundClient;
+        const store = backgroundStore;
+        backgroundClient = null;
+        backgroundStore = null;
+        await client?.close().catch(() => undefined);
+        await store?.close().catch(() => undefined);
+    };
+    const resumeBackgroundFollow = async () => {
+        if (backgroundClient !== null) {
+            return;
+        }
+        if (backgroundResumePromise !== null) {
+            return backgroundResumePromise;
+        }
+        backgroundResumePromise = (async () => {
+            const bitcoindStatus = await readManagedBitcoindStatus(paths);
+            const store = await openSqliteStore({ filename: databasePath });
+            try {
+                const client = await openManagedBitcoindClientInternal({
+                    store,
+                    dataDir,
+                    chain: bitcoindStatus?.chain ?? "main",
+                    startHeight: bitcoindStatus?.startHeight ?? 0,
+                    walletRootId,
+                    progressOutput: "none",
+                });
+                try {
+                    await client.startFollowingTip();
+                    backgroundStore = store;
+                    backgroundClient = client;
+                }
+                catch (error) {
+                    await client.close().catch(() => undefined);
+                    throw error;
+                }
+            }
+            catch (error) {
+                await store.close().catch(() => undefined);
+                throw error;
+            }
+        })();
+        try {
+            await backgroundResumePromise;
+        }
+        finally {
+            backgroundResumePromise = null;
+        }
+    };
     const heartbeat = setInterval(() => {
         void refreshStatus().catch(() => undefined);
         const now = Date.now();
@@ -427,6 +486,24 @@ async function main() {
                             });
                             return;
                         }
+                        if (request.method === "PauseBackgroundFollow") {
+                            await pauseBackgroundFollow();
+                            writeResponse({
+                                id: request.id,
+                                ok: true,
+                                result: null,
+                            });
+                            return;
+                        }
+                        if (request.method === "ResumeBackgroundFollow") {
+                            await resumeBackgroundFollow();
+                            writeResponse({
+                                id: request.id,
+                                ok: true,
+                                result: null,
+                            });
+                            return;
+                        }
                         throw new Error(`indexer_daemon_unknown_method_${request.method}`);
                     }
                     catch (error) {
@@ -443,6 +520,7 @@ async function main() {
     });
     const shutdown = async () => {
         clearInterval(heartbeat);
+        await pauseBackgroundFollow().catch(() => undefined);
         state = "stopping";
         heartbeatAtUnixMs = Date.now();
         updatedAtUnixMs = heartbeatAtUnixMs;