@cogcoin/client 1.1.1 → 1.1.3

package/dist/wallet/mining/runner.js

@@ -32,6 +32,8 @@ const BEST_BLOCK_POLL_INTERVAL_MS = 500;
 const BACKGROUND_START_TIMEOUT_MS = 15_000;
 const MINING_BITCOIN_RECOVERY_GRACE_MS = 15_000;
 const MINING_BITCOIN_RECOVERY_RESTART_COOLDOWN_MS = 60_000;
+const MINING_SUSPEND_HEARTBEAT_INTERVAL_MS = 1_000;
+const MINING_MEMPOOL_COOPERATIVE_YIELD_EVERY = 25;
 const MINING_BITCOIN_RECOVERY_NOTE = "Mining lost contact with the local Bitcoin RPC service and is waiting for it to recover.";
 function resolveBip39WordsFromIndices(indices) {
     if (indices === null || indices === undefined) {
@@ -62,6 +64,38 @@ function resolveSettledWinnerRequiredWords(options) {
 function resolveSnapshotOverride(override, fallback) {
     return override === undefined ? fallback : override;
 }
+function resolveMiningProviderBackoffDelayMs(consecutiveFailureCount) {
+    const exponent = Math.max(consecutiveFailureCount - 1, 0);
+    return Math.min(MINING_PROVIDER_BACKOFF_BASE_MS * (2 ** exponent), MINING_PROVIDER_BACKOFF_MAX_MS);
+}
+function clearMiningProviderWait(loopState, resetTransientFailureCount = true) {
+    loopState.providerWaitState = null;
+    loopState.providerWaitLastError = null;
+    loopState.providerWaitNextRetryAtUnixMs = null;
+    if (resetTransientFailureCount) {
+        loopState.providerTransientFailureCount = 0;
+    }
+}
+function recordTransientMiningProviderWait(options) {
+    options.loopState.providerTransientFailureCount += 1;
+    options.loopState.providerWaitState = options.error.providerState === "rate-limited"
+        ? "rate-limited"
+        : "backoff";
+    options.loopState.providerWaitLastError = options.error.message;
+    options.loopState.providerWaitNextRetryAtUnixMs = options.nowUnixMs
+        + resolveMiningProviderBackoffDelayMs(options.loopState.providerTransientFailureCount);
+}
+function recordTerminalMiningProviderWait(options) {
+    clearMiningProviderWait(options.loopState);
+    if (options.error.providerState !== "auth-error" && options.error.providerState !== "not-found") {
+        throw new Error("mining_provider_wait_state_invalid");
+    }
+    options.loopState.providerWaitState = options.error.providerState;
+    options.loopState.providerWaitLastError = options.error.message;
+}
+function isTransientMiningProviderError(error) {
+    return error.providerState === "unavailable" || error.providerState === "rate-limited";
+}
 class MiningSuspendDetectedError extends Error {
     detectedAtUnixMs;
     constructor(detectedAtUnixMs) {
@@ -78,20 +112,75 @@ class MiningPublishRejectedError extends Error {
     }
 }
 const miningGateCache = new Map();
-function createMiningSuspendDetector(monotonicNow = performance.now()) {
-    return {
-        lastMonotonicMs: monotonicNow,
+const defaultMiningSuspendScheduler = {
+    every(intervalMs, callback) {
+        const timer = setInterval(callback, intervalMs);
+        timer.unref?.();
+        return {
+            clear() {
+                clearInterval(timer);
+            },
+        };
+    },
+};
+function refreshMiningSuspendDetector(detector) {
+    if (detector === undefined) {
+        return;
+    }
+    const monotonicNow = detector.monotonicNow();
+    const gapMs = monotonicNow - detector.lastHeartbeatMonotonicMs;
+    detector.lastHeartbeatMonotonicMs = monotonicNow;
+    if (gapMs > MINING_SUSPEND_GAP_THRESHOLD_MS
+        && detector.detectedAtUnixMs === null) {
+        detector.detectedAtUnixMs = detector.nowUnixMs();
+    }
+}
+function createMiningSuspendDetector(options = {}) {
+    const monotonicNow = options.monotonicNow ?? (() => performance.now());
+    const nowUnixMs = options.nowUnixMs ?? Date.now;
+    const scheduler = options.scheduler ?? defaultMiningSuspendScheduler;
+    let heartbeat = null;
+    const detector = {
+        lastHeartbeatMonotonicMs: monotonicNow(),
+        detectedAtUnixMs: null,
+        monotonicNow,
+        nowUnixMs,
+        stop() {
+            heartbeat?.clear();
+            heartbeat = null;
+        },
     };
+    heartbeat = scheduler.every(MINING_SUSPEND_HEARTBEAT_INTERVAL_MS, () => {
+        refreshMiningSuspendDetector(detector);
+    });
+    return detector;
 }
-function checkpointMiningSuspendDetector(detector, monotonicNow = performance.now()) {
+function throwIfMiningSuspendDetected(detector) {
     if (detector === undefined) {
         return;
     }
-    const gapMs = monotonicNow - detector.lastMonotonicMs;
-    detector.lastMonotonicMs = monotonicNow;
-    if (gapMs > MINING_SUSPEND_GAP_THRESHOLD_MS) {
-        throw new MiningSuspendDetectedError(Date.now());
+    refreshMiningSuspendDetector(detector);
+    if (detector.detectedAtUnixMs === null) {
+        return;
     }
+    const detectedAtUnixMs = detector.detectedAtUnixMs;
+    detector.detectedAtUnixMs = null;
+    throw new MiningSuspendDetectedError(detectedAtUnixMs);
+}
+function stopMiningSuspendDetector(detector) {
+    detector?.stop();
+}
+function defaultMiningCooperativeYield() {
+    return new Promise((resolve) => {
+        setImmediate(resolve);
+    });
+}
+async function maybeYieldDuringMempoolScan(options) {
+    const yieldEvery = options.cooperativeYieldEvery ?? MINING_MEMPOOL_COOPERATIVE_YIELD_EVERY;
+    if (yieldEvery <= 0 || options.iteration === 0 || (options.iteration % yieldEvery) !== 0) {
+        return;
+    }
+    await (options.cooperativeYield ?? defaultMiningCooperativeYield)();
 }
 function clearMiningGateCache(walletRootId) {
     if (walletRootId === null || walletRootId === undefined) {
@@ -473,6 +562,10 @@ function createMiningLoopState() {
         selectedCandidate: null,
         ui: createEmptyMiningFollowVisualizerState(),
         waitingNote: null,
+        providerWaitState: null,
+        providerWaitLastError: null,
+        providerWaitNextRetryAtUnixMs: null,
+        providerTransientFailureCount: 0,
         bitcoinRecoveryFirstFailureAtUnixMs: null,
         bitcoinRecoveryFirstUnreachableAtUnixMs: null,
         bitcoinRecoveryLastRestartAttemptAtUnixMs: null,
@@ -527,6 +620,25 @@ function resetMiningUiForTip(loopState, targetBlockHeight) {
 export function resetMiningUiForTipForTesting(loopState, targetBlockHeight) {
     resetMiningUiForTip(loopState, targetBlockHeight);
 }
+function resolveProvisionalBroadcastTxidForCandidate(options) {
+    if (options.liveState === null || options.liveState === undefined) {
+        return null;
+    }
+    const liveState = normalizeMiningStateRecord(options.liveState);
+    if (liveState.currentTxid === null
+        || liveState.currentPublishState !== "in-mempool"
+        || liveState.livePublishInMempool !== true) {
+        return null;
+    }
+    if (liveState.currentDomain !== options.candidate.domainName
+        || liveState.currentDomainId !== options.candidate.domainId
+        || liveState.currentSentence !== options.candidate.sentence
+        || liveState.currentBlockTargetHeight !== options.candidate.targetBlockHeight
+        || liveState.currentReferencedBlockHashDisplay !== options.candidate.referencedBlockHashDisplay) {
+        return null;
+    }
+    return liveState.currentTxid;
+}
 function fallbackSettledWinnerDomainName(domainId) {
     return `domain-${domainId}`;
 }
@@ -600,13 +712,17 @@ function syncMiningUiForCurrentTip(options) {
         tipChanged,
     };
 }
-function setMiningUiCandidate(loopState, candidate) {
+function setMiningUiCandidate(loopState, candidate, liveState) {
     loopState.ui.latestSentence = candidate.sentence;
     loopState.ui.provisionalRequiredWords = [...candidate.bip39Words];
     loopState.ui.provisionalEntry = {
         domainName: candidate.domainName,
         sentence: candidate.sentence,
     };
+    loopState.ui.provisionalBroadcastTxid = resolveProvisionalBroadcastTxidForCandidate({
+        candidate,
+        liveState,
+    });
 }
 function getSelectedCandidateForTip(loopState, tipKey) {
     if (tipKey === null || loopState.selectedCandidateTipKey !== tipKey) {
@@ -617,13 +733,13 @@ function getSelectedCandidateForTip(loopState, tipKey) {
 export function getSelectedCandidateForTipForTesting(loopState, tipKey) {
     return getSelectedCandidateForTip(loopState, tipKey);
 }
-function cacheSelectedCandidateForTip(loopState, tipKey, candidate) {
+function cacheSelectedCandidateForTip(loopState, tipKey, candidate, liveState) {
     loopState.selectedCandidateTipKey = tipKey;
     loopState.selectedCandidate = candidate;
-    setMiningUiCandidate(loopState, candidate);
+    setMiningUiCandidate(loopState, candidate, liveState);
 }
-export function cacheSelectedCandidateForTipForTesting(loopState, tipKey, candidate) {
-    cacheSelectedCandidateForTip(loopState, tipKey, candidate);
+export function cacheSelectedCandidateForTipForTesting(loopState, tipKey, candidate, liveState) {
+    cacheSelectedCandidateForTip(loopState, tipKey, candidate, liveState);
 }
 function clearSelectedCandidate(loopState) {
     loopState.selectedCandidateTipKey = null;
@@ -635,6 +751,7 @@ function clearMiningUiTransientCandidate(loopState) {
         domainName: null,
         sentence: null,
     };
+    loopState.ui.provisionalBroadcastTxid = null;
     loopState.ui.latestSentence = null;
 }
 function discardMiningLoopTransientWork(loopState, walletRootId) {
@@ -642,6 +759,7 @@ function discardMiningLoopTransientWork(loopState, walletRootId) {
     clearSelectedCandidate(loopState);
     clearMiningUiTransientCandidate(loopState);
     loopState.waitingNote = null;
+    clearMiningProviderWait(loopState);
 }
 function resolveMiningBitcoindRecoveryIdentity(value) {
     const raw = (value ?? {});
@@ -948,39 +1066,61 @@ function getAncestorTxids(context, txContexts) {
         .filter((txid) => txid !== null && txContexts.has(txid));
 }
 function topologicallyOrderAncestorContexts(options) {
-    const visited = new Set();
-    const visiting = new Set();
+    const visited = new Map();
     const ordered = [];
-    const visit = (txid) => {
-        if (visited.has(txid)) {
-            return true;
-        }
-        if (visiting.has(txid)) {
-            return false;
-        }
-        const context = options.txContexts.get(txid);
-        if (context === undefined) {
-            return true;
-        }
-        visiting.add(txid);
-        for (const parentTxid of getAncestorTxids(context, options.txContexts)) {
-            if (!visit(parentTxid)) {
-                return false;
-            }
-        }
-        visiting.delete(txid);
-        visited.add(txid);
-        ordered.push(context);
-        return true;
-    };
     const root = options.txContexts.get(options.txid);
     if (root === undefined) {
         return [];
     }
-    for (const parentTxid of getAncestorTxids(root, options.txContexts)) {
-        if (!visit(parentTxid)) {
+    const stack = getAncestorTxids(root, options.txContexts)
+        .reverse()
+        .map((txid) => ({
+        txid,
+        expanded: false,
+    }));
+    while (stack.length > 0) {
+        const frame = stack.pop();
+        const state = visited.get(frame.txid);
+        if (frame.expanded) {
+            if (state !== "visiting") {
+                continue;
+            }
+            visited.set(frame.txid, "visited");
+            const context = options.txContexts.get(frame.txid);
+            if (context !== undefined) {
+                ordered.push(context);
+            }
+            continue;
+        }
+        if (state === "visited") {
+            continue;
+        }
+        if (state === "visiting") {
             return null;
         }
+        const context = options.txContexts.get(frame.txid);
+        if (context === undefined) {
+            continue;
+        }
+        visited.set(frame.txid, "visiting");
+        stack.push({
+            txid: frame.txid,
+            expanded: true,
+        });
+        const parents = getAncestorTxids(context, options.txContexts);
+        for (let index = parents.length - 1; index >= 0; index -= 1) {
+            const parentTxid = parents[index];
+            const parentState = visited.get(parentTxid);
+            if (parentState === "visiting") {
+                return null;
+            }
+            if (parentState !== "visited") {
+                stack.push({
+                    txid: parentTxid,
+                    expanded: false,
+                });
+            }
+        }
     }
     return ordered;
 }
@@ -1096,13 +1236,17 @@ async function resolveOverlayAuthorizedMiningDomain(options) {
     return authorized ? domain : null;
 }
 function buildStatusSnapshot(view, overrides = {}) {
+    const resolvedCurrentPhase = resolveSnapshotOverride(overrides.currentPhase, view.runtime.currentPhase);
+    const clearProviderWaitCarryover = overrides.currentPhase !== undefined
+        && overrides.currentPhase !== "waiting-provider"
+        && view.runtime.currentPhase === "waiting-provider";
     return {
         ...view.runtime,
         runMode: resolveSnapshotOverride(overrides.runMode, view.runtime.runMode),
         backgroundWorkerPid: resolveSnapshotOverride(overrides.backgroundWorkerPid, view.runtime.backgroundWorkerPid),
         backgroundWorkerRunId: resolveSnapshotOverride(overrides.backgroundWorkerRunId, view.runtime.backgroundWorkerRunId),
         backgroundWorkerHeartbeatAtUnixMs: resolveSnapshotOverride(overrides.backgroundWorkerHeartbeatAtUnixMs, view.runtime.backgroundWorkerHeartbeatAtUnixMs),
-        currentPhase: resolveSnapshotOverride(overrides.currentPhase, view.runtime.currentPhase),
+        currentPhase: resolvedCurrentPhase,
         currentPublishState: resolveSnapshotOverride(overrides.currentPublishState, view.runtime.currentPublishState),
         targetBlockHeight: resolveSnapshotOverride(overrides.targetBlockHeight, view.runtime.targetBlockHeight),
         referencedBlockHashDisplay: resolveSnapshotOverride(overrides.referencedBlockHashDisplay, view.runtime.referencedBlockHashDisplay),
@@ -1118,7 +1262,9 @@ function buildStatusSnapshot(view, overrides = {}) {
         lastSuspendDetectedAtUnixMs: resolveSnapshotOverride(overrides.lastSuspendDetectedAtUnixMs, view.runtime.lastSuspendDetectedAtUnixMs),
         reconnectSettledUntilUnixMs: resolveSnapshotOverride(overrides.reconnectSettledUntilUnixMs, view.runtime.reconnectSettledUntilUnixMs),
         tipSettledUntilUnixMs: resolveSnapshotOverride(overrides.tipSettledUntilUnixMs, view.runtime.tipSettledUntilUnixMs),
-        providerState: resolveSnapshotOverride(overrides.providerState, view.runtime.providerState),
+        providerState: resolveSnapshotOverride(overrides.providerState, clearProviderWaitCarryover
+            ? (view.provider.status === "ready" ? "ready" : "unavailable")
+            : view.runtime.providerState),
         corePublishState: resolveSnapshotOverride(overrides.corePublishState, view.runtime.corePublishState),
         currentPublishDecision: resolveSnapshotOverride(overrides.currentPublishDecision, view.runtime.currentPublishDecision),
         sameDomainCompetitorSuppressed: resolveSnapshotOverride(overrides.sameDomainCompetitorSuppressed, view.runtime.sameDomainCompetitorSuppressed),
@@ -1128,8 +1274,8 @@ function buildStatusSnapshot(view, overrides = {}) {
         mempoolSequenceCacheStatus: resolveSnapshotOverride(overrides.mempoolSequenceCacheStatus, view.runtime.mempoolSequenceCacheStatus),
         lastMempoolSequence: resolveSnapshotOverride(overrides.lastMempoolSequence, view.runtime.lastMempoolSequence),
         lastCompetitivenessGateAtUnixMs: resolveSnapshotOverride(overrides.lastCompetitivenessGateAtUnixMs, view.runtime.lastCompetitivenessGateAtUnixMs),
-        lastError: resolveSnapshotOverride(overrides.lastError, view.runtime.lastError),
-        note: resolveSnapshotOverride(overrides.note, view.runtime.note),
+        lastError: resolveSnapshotOverride(overrides.lastError, clearProviderWaitCarryover ? null : view.runtime.lastError),
+        note: resolveSnapshotOverride(overrides.note, clearProviderWaitCarryover ? null : view.runtime.note),
         livePublishInMempool: resolveSnapshotOverride(overrides.livePublishInMempool, view.runtime.livePublishInMempool),
         updatedAtUnixMs: Date.now(),
     };
@@ -1740,6 +1886,7 @@ async function runCompetitivenessGate(options) {
         candidateRank: overrides.candidateRank ?? null,
     });
     const walletRootId = options.readContext.localState.walletRootId ?? "uninitialized-wallet-root";
+    const assaySentencesImpl = options.assaySentencesImpl ?? assaySentences;
     const indexerTruthKey = getIndexerTruthKey(options.readContext);
     const excludedTxids = [options.currentTxid].filter((value) => value !== null).sort();
     const localAssayTupleKey = [
@@ -1789,7 +1936,13 @@ async function runCompetitivenessGate(options) {
             txContexts.delete(txid);
         }
     }
-    for (const txid of visibleTxids) {
+    for (let index = 0; index < visibleTxids.length; index += 1) {
+        await maybeYieldDuringMempoolScan({
+            iteration: index,
+            cooperativeYield: options.cooperativeYield,
+            cooperativeYieldEvery: options.cooperativeYieldEvery,
+        });
+        const txid = visibleTxids[index];
         if (txContexts.has(txid)) {
             continue;
         }
@@ -1815,7 +1968,13 @@ async function runCompetitivenessGate(options) {
         });
     }
     const entries = new Map();
-    for (const txid of visibleTxids) {
+    for (let index = 0; index < visibleTxids.length; index += 1) {
+        await maybeYieldDuringMempoolScan({
+            iteration: index,
+            cooperativeYield: options.cooperativeYield,
+            cooperativeYieldEvery: options.cooperativeYieldEvery,
+        });
+        const txid = visibleTxids[index];
         const context = txContexts.get(txid);
         if (context === undefined || context.payload === null || context.senderScriptHex === null) {
             continue;
@@ -1854,7 +2013,7 @@ async function runCompetitivenessGate(options) {
         if (overlayDomain === null || overlayDomain.name === null || !rootDomain(overlayDomain.name)) {
             continue;
         }
-        const assayed = await assaySentences(decoded.domainId, options.candidate.referencedBlockHashInternal, [Buffer.from(decoded.sentenceBytes).toString("utf8")]).catch(() => []);
+        const assayed = await assaySentencesImpl(decoded.domainId, options.candidate.referencedBlockHashInternal, [Buffer.from(decoded.sentenceBytes).toString("utf8")]).catch(() => []);
         const scored = assayed[0];
         if (scored === undefined || !scored.gatesPass || scored.encodedSentenceBytes === null || scored.canonicalBlend === null) {
             continue;
@@ -2475,6 +2634,8 @@ export async function ensureBuiltInMiningSetupIfNeeded(options) {
 }
 async function performMiningCycle(options) {
     const now = options.nowImpl ?? Date.now;
+    const generateCandidatesForDomainsImpl = options.generateCandidatesForDomainsImpl ?? generateCandidatesForDomains;
+    const runCompetitivenessGateImpl = options.runCompetitivenessGateImpl ?? runCompetitivenessGate;
     let readContext = await options.openReadContext({
         dataDir: options.dataDir,
         databasePath: options.databasePath,
@@ -2483,6 +2644,7 @@ async function performMiningCycle(options) {
     });
     let readContextClosed = false;
     try {
+        throwIfMiningSuspendDetected(options.suspendDetector);
         let clearRecoveredBitcoindError = false;
         const saveCycleStatus = async (readContext, overrides, includeVisualizer = true) => {
             const statusNowUnixMs = now();
@@ -2504,7 +2666,6 @@ async function performMiningCycle(options) {
                 visualizerState: includeVisualizer ? options.loopState.ui : undefined,
             });
         };
-        checkpointMiningSuspendDetector(options.suspendDetector);
         await saveCycleStatus(readContext, {
             runMode: options.runMode,
             backgroundWorkerPid: options.backgroundWorkerPid,
@@ -2512,9 +2673,11 @@ async function performMiningCycle(options) {
             backgroundWorkerHeartbeatAtUnixMs: options.runMode === "background" ? now() : null,
         }, false);
         if (readContext.localState.availability !== "ready" || readContext.localState.state === null) {
+            clearMiningProviderWait(options.loopState);
             await saveCycleStatus(readContext, {
                 runMode: options.runMode,
                 currentPhase: "waiting",
+                lastError: null,
                 note: "Wallet state must be locally available for mining to continue.",
             });
             return;
@@ -2525,7 +2688,7 @@ async function performMiningCycle(options) {
             startHeight: 0,
             walletRootId: readContext.localState.state.walletRootId,
         });
-        checkpointMiningSuspendDetector(options.suspendDetector);
+        throwIfMiningSuspendDetected(options.suspendDetector);
         const rpc = options.rpcFactory(service.rpc);
         const reconciliation = await reconcileLiveMiningState({
             state: readContext.localState.state,
@@ -2535,7 +2698,7 @@ async function performMiningCycle(options) {
             snapshotState: readContext.snapshot?.state ?? null,
         });
         const reconciledState = reconciliation.state;
-        checkpointMiningSuspendDetector(options.suspendDetector);
+        throwIfMiningSuspendDetected(options.suspendDetector);
         let effectiveReadContext = readContext;
         if (JSON.stringify(reconciledState.miningState) !== JSON.stringify(readContext.localState.state.miningState)) {
             await saveWalletStatePreservingUnlock({
@@ -2576,18 +2739,24 @@ async function performMiningCycle(options) {
         });
         if (tipChanged) {
             setMiningTipSettleWindow(options.loopState, now());
+            if (options.loopState.providerWaitNextRetryAtUnixMs === null) {
+                clearMiningProviderWait(options.loopState);
+            }
         }
         const displaySats = await resolveFundingDisplaySats(effectiveReadContext.localState.state, rpc).catch(() => null);
         syncMiningVisualizerBalances(options.loopState, effectiveReadContext, displaySats);
         if (effectiveReadContext.localState.state.miningState.state === "repair-required") {
+            clearMiningProviderWait(options.loopState);
             await saveCycleStatus(effectiveReadContext, {
                 runMode: options.runMode,
                 currentPhase: "waiting",
+                lastError: null,
                 note: "Mining is blocked until the current mining publish is repaired or reconciled.",
             });
             return;
         }
         if (hasBlockingMutation(effectiveReadContext.localState.state)) {
+            clearMiningProviderWait(options.loopState);
             const nextState = defaultMiningStatePatch(effectiveReadContext.localState.state, {
                 state: "paused",
                 pauseReason: "wallet-busy",
@@ -2608,12 +2777,14 @@ async function performMiningCycle(options) {
             await saveCycleStatus(effectiveReadContext, {
                 runMode: options.runMode,
                 currentPhase: "waiting",
+                lastError: null,
                 note: "Mining is paused while another wallet mutation is active.",
             });
             return;
         }
         const preemptionRequest = await readMiningPreemptionRequest(options.paths);
         if (preemptionRequest !== null) {
+            clearMiningProviderWait(options.loopState);
             const nextState = defaultMiningStatePatch(effectiveReadContext.localState.state, {
                 state: effectiveReadContext.localState.state.miningState.livePublishInMempool
                     && effectiveReadContext.localState.state.miningState.state === "paused-stale"
@@ -2635,6 +2806,7 @@ async function performMiningCycle(options) {
             }, {
                 runMode: options.runMode,
                 currentPhase: "waiting",
+                lastError: null,
                 note: "Mining is paused while another wallet command is preempting sentence generation.",
             });
             return;
@@ -2644,7 +2816,7 @@ async function performMiningCycle(options) {
             rpc.getNetworkInfo(),
             rpc.getMempoolInfo(),
         ]);
-        checkpointMiningSuspendDetector(options.suspendDetector);
+        throwIfMiningSuspendDetected(options.suspendDetector);
         const corePublishState = determineCorePublishState({
             blockchain: blockchainInfo,
             network: networkInfo,
@@ -2652,6 +2824,7 @@ async function performMiningCycle(options) {
         });
         clearRecoveredBitcoindError = resetMiningBitcoindRecoveryState(options.loopState, effectiveReadContext.nodeStatus?.serviceStatus ?? { pid: service.pid });
         if (corePublishState !== "healthy") {
+            clearMiningProviderWait(options.loopState);
             await saveCycleStatus(effectiveReadContext, {
                 runMode: options.runMode,
                 currentPhase: "waiting-bitcoin-network",
@@ -2661,6 +2834,7 @@ async function performMiningCycle(options) {
             return;
         }
         if (effectiveReadContext.indexer.health !== "synced" || effectiveReadContext.nodeHealth !== "synced") {
+            clearMiningProviderWait(options.loopState);
             await saveCycleStatus(effectiveReadContext, {
                 runMode: options.runMode,
                 currentPhase: effectiveReadContext.indexer.health !== "synced"
@@ -2673,6 +2847,7 @@ async function performMiningCycle(options) {
             return;
         }
         if (targetBlockHeight === null) {
+            clearMiningProviderWait(options.loopState);
             await saveCycleStatus(effectiveReadContext, {
                 runMode: options.runMode,
                 currentPhase: "waiting-bitcoin-network",
@@ -2681,6 +2856,7 @@ async function performMiningCycle(options) {
             return;
         }
         if (getBlockRewardCogtoshi(targetBlockHeight) === 0n) {
+            clearMiningProviderWait(options.loopState);
             const nextState = defaultMiningStatePatch(effectiveReadContext.localState.state, {
                 state: "paused",
                 pauseReason: "zero-reward",
@@ -2700,6 +2876,7 @@ async function performMiningCycle(options) {
                 runMode: options.runMode,
                 currentPhase: "idle",
                 currentPublishDecision: "publish-skipped-zero-reward",
+                lastError: null,
                 note: "Mining is disabled because the target block reward is zero.",
             });
             await appendEvent(options.paths, createEvent("publish-skipped-zero-reward", "Skipped mining because the target block reward is zero.", {
@@ -2709,10 +2886,38 @@ async function performMiningCycle(options) {
             }));
             return;
         }
+        if (options.loopState.providerWaitState !== null
+            && options.loopState.providerWaitLastError !== null) {
+            if (options.loopState.providerWaitNextRetryAtUnixMs !== null
+                && now() < options.loopState.providerWaitNextRetryAtUnixMs) {
+                await saveCycleStatus(effectiveReadContext, {
+                    runMode: options.runMode,
+                    currentPhase: "waiting-provider",
+                    providerState: options.loopState.providerWaitState,
+                    lastError: options.loopState.providerWaitLastError,
+                    note: "Mining is waiting for the sentence provider to recover.",
+                });
+                return;
+            }
+            if (options.loopState.providerWaitNextRetryAtUnixMs === null
+                && tipKey !== null
+                && options.loopState.attemptedTipKey === tipKey) {
+                await saveCycleStatus(effectiveReadContext, {
+                    runMode: options.runMode,
+                    currentPhase: "waiting-provider",
+                    providerState: options.loopState.providerWaitState,
+                    lastError: options.loopState.providerWaitLastError,
+                    note: "Mining is waiting for the sentence provider to recover.",
+                });
+                return;
+            }
+            clearMiningProviderWait(options.loopState, options.loopState.providerWaitNextRetryAtUnixMs === null);
+        }
         if (tipKey !== null && options.loopState.attemptedTipKey === tipKey) {
             await saveCycleStatus(effectiveReadContext, {
                 runMode: options.runMode,
                 currentPhase: "waiting",
+                lastError: null,
                 note: options.loopState.waitingNote ?? "Waiting for the next block after the last mining attempt on this tip.",
             });
             return;
@@ -2751,9 +2956,11 @@ async function performMiningCycle(options) {
         if (selectedCandidate === null) {
             const domains = resolveEligibleAnchoredRoots(effectiveReadContext);
             if (domains.length === 0) {
+                clearMiningProviderWait(options.loopState);
                 await saveCycleStatus(effectiveReadContext, {
                     runMode: options.runMode,
                     currentPhase: "idle",
+                    lastError: null,
                     note: "No locally controlled anchored root domains are currently eligible to mine.",
                 });
                 return;
@@ -2761,6 +2968,7 @@ async function performMiningCycle(options) {
             await saveCycleStatus(effectiveReadContext, {
                 runMode: options.runMode,
                 currentPhase: "generating",
+                lastError: null,
                 note: "Generating mining sentences for eligible root domains.",
             });
             await appendEvent(options.paths, createEvent("sentence-generation-start", "Started mining sentence generation.", {
@@ -2770,7 +2978,7 @@ async function performMiningCycle(options) {
             }));
             let candidates;
             try {
-                candidates = await generateCandidatesForDomains({
+                candidates = await generateCandidatesForDomainsImpl({
                     rpc,
                     readContext: effectiveReadContext,
                     domains,
@@ -2780,18 +2988,30 @@ async function performMiningCycle(options) {
                     runId: options.backgroundWorkerRunId,
                     fetchImpl: options.fetchImpl,
                 });
-                checkpointMiningSuspendDetector(options.suspendDetector);
+                throwIfMiningSuspendDetected(options.suspendDetector);
             }
             catch (error) {
                 if (error instanceof MiningProviderRequestError) {
-                    if (tipKey !== null) {
+                    if (isTransientMiningProviderError(error)) {
+                        recordTransientMiningProviderWait({
+                            loopState: options.loopState,
+                            error,
+                            nowUnixMs: now(),
+                        });
+                    }
+                    else {
+                        recordTerminalMiningProviderWait({
+                            loopState: options.loopState,
+                            error,
+                        });
+                    }
+                    if (!isTransientMiningProviderError(error) && tipKey !== null) {
                         options.loopState.attemptedTipKey = tipKey;
-                        options.loopState.waitingNote = "Mining is waiting for the sentence provider to recover.";
                     }
                     await saveCycleStatus(effectiveReadContext, {
                         runMode: options.runMode,
                         currentPhase: "waiting-provider",
-                        providerState: error.providerState,
+                        providerState: options.loopState.providerWaitState ?? error.providerState,
                         lastError: error.message,
                         note: "Mining is waiting for the sentence provider to recover.",
                     });
@@ -2813,6 +3033,7 @@ async function performMiningCycle(options) {
                     return;
                 }
                 if (error instanceof Error && error.message === "mining_generation_stale_indexer_truth") {
+                    clearMiningProviderWait(options.loopState);
                     clearMiningGateCache(walletRootId);
                     await appendEvent(options.paths, createEvent("generation-restarted-indexer-truth", "Detected updated coherent indexer truth during mining; restarting on the next tick.", {
                         level: "warn",
@@ -2823,6 +3044,7 @@ async function performMiningCycle(options) {
                     return;
                 }
                 if (error instanceof Error && error.message === "mining_generation_preempted") {
+                    clearMiningProviderWait(options.loopState);
                     await appendEvent(options.paths, createEvent("generation-paused-preempted", "Stopped sentence generation because another wallet command requested mining preemption.", {
                         level: "warn",
                         targetBlockHeight,
@@ -2831,6 +3053,7 @@ async function performMiningCycle(options) {
                     }));
                     return;
                 }
+                clearMiningProviderWait(options.loopState);
                 const failureMessage = error instanceof Error ? error.message : String(error);
                 if (tipKey !== null) {
                     options.loopState.attemptedTipKey = tipKey;
@@ -2851,9 +3074,11 @@ async function performMiningCycle(options) {
                 }));
                 return;
             }
+            clearMiningProviderWait(options.loopState);
             await saveCycleStatus(effectiveReadContext, {
                 runMode: options.runMode,
                 currentPhase: "scoring",
+                lastError: null,
                 note: "Scoring mining candidates for the current tip.",
             });
             const best = await chooseBestLocalCandidate(candidates);
@@ -2880,7 +3105,7 @@ async function performMiningCycle(options) {
                 return;
             }
             options.loopState.ui.recentWin = null;
-            cacheSelectedCandidateForTip(options.loopState, tipKey, best);
+            cacheSelectedCandidateForTip(options.loopState, tipKey, best, effectiveReadContext.localState.state.miningState);
             selectedCandidate = best;
             await appendEvent(options.paths, createEvent("candidate-selected", `Selected ${best.domainName} with score ${best.canonicalBlend.toString()}.`, {
                 targetBlockHeight: best.targetBlockHeight,
@@ -2890,13 +3115,16 @@ async function performMiningCycle(options) {
                 score: best.canonicalBlend.toString(),
                 runId: options.backgroundWorkerRunId,
             }));
-            const gate = await runCompetitivenessGate({
+            const gate = await runCompetitivenessGateImpl({
                 rpc,
                 readContext: effectiveReadContext,
                 candidate: best,
                 currentTxid: effectiveReadContext.localState.state.miningState.currentTxid,
+                assaySentencesImpl: options.assaySentencesImpl,
+                cooperativeYield: options.cooperativeYieldImpl,
+                cooperativeYieldEvery: options.cooperativeYieldEvery,
             });
-            checkpointMiningSuspendDetector(options.suspendDetector);
+            throwIfMiningSuspendDetected(options.suspendDetector);
             gateSnapshot = {
                 higherRankedCompetitorDomainCount: gate.higherRankedCompetitorDomainCount,
                 dedupedCompetitorDomainCount: gate.dedupedCompetitorDomainCount,
@@ -2908,7 +3136,7 @@ async function performMiningCycle(options) {
                     options.loopState.attemptedTipKey = tipKey;
                 }
                 clearSelectedCandidate(options.loopState);
-                setMiningUiCandidate(options.loopState, best);
+                setMiningUiCandidate(options.loopState, best, effectiveReadContext.localState.state.miningState);
                 options.loopState.waitingNote = gate.decision === "suppressed-same-domain-mempool"
                     ? "Best local sentence found, but a same-domain mempool competitor already matches or beats it."
                     : gate.decision === "suppressed-top5-mempool"
@@ -2949,7 +3177,7 @@ async function performMiningCycle(options) {
         }
         else {
             options.loopState.ui.recentWin = null;
-            setMiningUiCandidate(options.loopState, selectedCandidate);
+            setMiningUiCandidate(options.loopState, selectedCandidate, effectiveReadContext.localState.state.miningState);
         }
         if (!await ensureCurrentIndexerTruthOrRestart()) {
             return;
@@ -2965,12 +3193,11 @@ async function performMiningCycle(options) {
             purpose: "wallet-mine",
             walletRootId: effectiveReadContext.localState.state.walletRootId,
         });
-        checkpointMiningSuspendDetector(options.suspendDetector);
         try {
             if (!await ensureCurrentIndexerTruthOrRestart()) {
                 return;
             }
-            checkpointMiningSuspendDetector(options.suspendDetector);
+            throwIfMiningSuspendDetected(options.suspendDetector);
             const published = await publishCandidate({
                 dataDir: options.dataDir,
                 databasePath: options.databasePath,
@@ -2983,12 +3210,11 @@ async function performMiningCycle(options) {
                 candidate: selectedCandidate,
                 runId: options.backgroundWorkerRunId,
             });
-            checkpointMiningSuspendDetector(options.suspendDetector);
             if (tipKey !== null && published.retryable !== true) {
                 options.loopState.attemptedTipKey = tipKey;
             }
             if (published.retryable === true) {
-                cacheSelectedCandidateForTip(options.loopState, tipKey, published.candidate);
+                cacheSelectedCandidateForTip(options.loopState, tipKey, published.candidate, published.state.miningState);
                 options.loopState.waitingNote = published.note;
                 await saveCycleStatus({
                     ...effectiveReadContext,
@@ -3014,7 +3240,7 @@ async function performMiningCycle(options) {
             }
             if (published.skipped === true) {
                 clearSelectedCandidate(options.loopState);
-                setMiningUiCandidate(options.loopState, selectedCandidate);
+                setMiningUiCandidate(options.loopState, selectedCandidate, published.state.miningState);
                 options.loopState.waitingNote = published.note;
                 const lastError = published.decision === "publish-paused-insufficient-funds"
                     ? published.lastError ?? createInsufficientFundsMiningPublishErrorMessage()
@@ -3046,7 +3272,7 @@ async function performMiningCycle(options) {
             if (published.txid !== null) {
                 options.loopState.ui.latestTxid = published.txid;
             }
-            setMiningUiCandidate(options.loopState, published.candidate);
+            setMiningUiCandidate(options.loopState, published.candidate, published.state.miningState);
             options.loopState.waitingNote = published.decision === "kept-live-publish"
                 ? "Existing live mining publish already covers this block attempt. Waiting for the next block."
                 : published.txid === null
@@ -3214,59 +3440,71 @@ async function attemptSaveMempool(rpc, paths, runId) {
     }
 }
 async function runMiningLoop(options) {
-    const suspendDetector = createMiningSuspendDetector();
-    const loopState = createMiningLoopState();
+    const suspendDetector = createMiningSuspendDetector({
+        monotonicNow: options.suspendMonotonicNowImpl,
+        nowUnixMs: options.nowImpl ?? Date.now,
+        scheduler: options.suspendScheduler,
+    });
+    const loopState = options.loopState ?? createMiningLoopState();
     const probeService = options.probeService ?? probeManagedBitcoindService;
     const stopService = options.stopService ?? stopManagedBitcoindService;
     const sleepImpl = options.sleepImpl ?? sleep;
-    await appendEvent(options.paths, createEvent("runtime-start", `Started ${options.runMode} mining runtime.`, {
-        runId: options.backgroundWorkerRunId,
-    }));
-    while (!options.signal?.aborted) {
-        try {
-            checkpointMiningSuspendDetector(suspendDetector);
-        }
-        catch (error) {
-            if (!(error instanceof MiningSuspendDetectedError)) {
-                throw error;
+    try {
+        await appendEvent(options.paths, createEvent("runtime-start", `Started ${options.runMode} mining runtime.`, {
+            runId: options.backgroundWorkerRunId,
+        }));
+        while (!options.signal?.aborted) {
+            try {
+                throwIfMiningSuspendDetected(suspendDetector);
             }
-            discardMiningLoopTransientWork(loopState, null);
-            await handleDetectedMiningRuntimeResume({
-                dataDir: options.dataDir,
-                databasePath: options.databasePath,
-                provider: options.provider,
-                paths: options.paths,
-                runMode: options.runMode,
-                backgroundWorkerPid: options.backgroundWorkerPid,
-                backgroundWorkerRunId: options.backgroundWorkerRunId,
-                detectedAtUnixMs: error.detectedAtUnixMs,
-                openReadContext: options.openReadContext,
-                visualizer: options.visualizer,
+            catch (error) {
+                if (!(error instanceof MiningSuspendDetectedError)) {
+                    throw error;
+                }
+                discardMiningLoopTransientWork(loopState, null);
+                await handleDetectedMiningRuntimeResume({
+                    dataDir: options.dataDir,
+                    databasePath: options.databasePath,
+                    provider: options.provider,
+                    paths: options.paths,
+                    runMode: options.runMode,
+                    backgroundWorkerPid: options.backgroundWorkerPid,
+                    backgroundWorkerRunId: options.backgroundWorkerRunId,
+                    detectedAtUnixMs: error.detectedAtUnixMs,
+                    openReadContext: options.openReadContext,
+                    visualizer: options.visualizer,
+                    loopState,
+                });
+                continue;
+            }
+            await performMiningCycle({
+                ...options,
+                suspendDetector,
+                assaySentencesImpl: options.assaySentencesImpl,
+                cooperativeYieldImpl: options.cooperativeYieldImpl,
+                cooperativeYieldEvery: options.cooperativeYieldEvery,
                 loopState,
+                probeService,
+                stopService,
             });
-            continue;
+            await sleepImpl(Math.min(MINING_LOOP_INTERVAL_MS, MINING_STATUS_HEARTBEAT_INTERVAL_MS), options.signal);
         }
-        await performMiningCycle({
-            ...options,
-            suspendDetector,
-            loopState,
-            probeService,
-            stopService,
-        });
-        await sleepImpl(Math.min(MINING_LOOP_INTERVAL_MS, MINING_STATUS_HEARTBEAT_INTERVAL_MS), options.signal);
+        const service = await options.attachService({
+            dataDir: options.dataDir,
+            chain: "main",
+            startHeight: 0,
+            walletRootId: undefined,
+        }).catch(() => null);
+        if (service !== null) {
+            await attemptSaveMempool(options.rpcFactory(service.rpc), options.paths, options.backgroundWorkerRunId);
+        }
+        await appendEvent(options.paths, createEvent("runtime-stop", `Stopped ${options.runMode} mining runtime.`, {
+            runId: options.backgroundWorkerRunId,
+        }));
     }
-    const service = await options.attachService({
-        dataDir: options.dataDir,
-        chain: "main",
-        startHeight: 0,
-        walletRootId: undefined,
-    }).catch(() => null);
-    if (service !== null) {
-        await attemptSaveMempool(options.rpcFactory(service.rpc), options.paths, options.backgroundWorkerRunId);
+    finally {
+        stopMiningSuspendDetector(suspendDetector);
     }
-    await appendEvent(options.paths, createEvent("runtime-stop", `Stopped ${options.runMode} mining runtime.`, {
-        runId: options.backgroundWorkerRunId,
-    }));
 }
 async function waitForBackgroundHealthy(paths) {
     const deadline = Date.now() + BACKGROUND_START_TIMEOUT_MS;
@@ -3579,6 +3817,30 @@ export async function runMiningLoopForTesting(options) {
         ...options,
     });
 }
+export async function runCompetitivenessGateForTesting(options) {
+    return await runCompetitivenessGate({
+        rpc: options.rpc,
+        readContext: options.readContext,
+        candidate: options.candidate,
+        currentTxid: options.currentTxid,
+        assaySentencesImpl: options.assaySentencesImpl,
+        cooperativeYield: options.cooperativeYieldImpl,
+        cooperativeYieldEvery: options.cooperativeYieldEvery,
+    });
+}
+export function createMiningSuspendDetectorForTesting(options = {}) {
+    return createMiningSuspendDetector(options);
+}
+export function throwIfMiningSuspendDetectedForTesting(detector) {
+    throwIfMiningSuspendDetected(detector);
+}
+export function topologicallyOrderAncestorTxidsForTesting(options) {
+    const ordered = topologicallyOrderAncestorContexts({
+        txid: options.txid,
+        txContexts: options.txContexts,
+    });
+    return ordered?.map((context) => context.txid) ?? null;
+}
 export function buildPrePublishStatusOverridesForTesting(options) {
     return buildPrePublishStatusOverrides(options);
 }