@cogcoin/client 1.0.0 → 1.0.2

package/dist/wallet/mining/runner.js

@@ -1,5 +1,7 @@
 import { createHash, randomBytes } from "node:crypto";
 import { spawn } from "node:child_process";
+import { rm } from "node:fs/promises";
+import { join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { getBalance, getBlockWinners, lookupDomain, lookupDomainById, } from "@cogcoin/indexer/queries";
 import { assaySentences, deriveBlendSeed, displayToInternalBlockhash, getWords, settleBlock, } from "@cogcoin/scoring";
@@ -9,8 +11,8 @@ import { attachOrStartManagedBitcoindService } from "../../bitcoind/service.js";
 import { createRpcClient } from "../../bitcoind/node.js";
 import { COG_OPCODES, COG_PREFIX } from "../cogop/constants.js";
 import { extractOpReturnPayloadFromScriptHex } from "../tx/register.js";
-import { assertFixedInputPrefixMatches, buildWalletMutationTransaction, outpointKey as walletMutationOutpointKey, isAlreadyAcceptedError, isBroadcastUnknownError, reconcilePersistentPolicyLocks, resolveWalletMutationFeeSelection, saveWalletStatePreservingUnlock, } from "../tx/common.js";
-import { acquireFileLock } from "../fs/lock.js";
+import { assertFixedInputPrefixMatches, buildWalletMutationTransaction, isInsufficientFundsError, outpointKey as walletMutationOutpointKey, isAlreadyAcceptedError, isBroadcastUnknownError, reconcilePersistentPolicyLocks, resolveWalletMutationFeeSelection, saveWalletStatePreservingUnlock, } from "../tx/common.js";
+import { FileLockBusyError, acquireFileLock, clearOrphanedFileLock, readLockMetadata, } from "../fs/lock.js";
 import { isMineableWalletDomain, openWalletReadContext, } from "../read/index.js";
 import { resolveWalletRuntimePathsForTesting } from "../runtime.js";
 import { createDefaultWalletSecretProvider, } from "../state/provider.js";
@@ -19,13 +21,16 @@ import { appendMiningEvent, loadMiningRuntimeStatus, saveMiningRuntimeStatus, }
 import { loadClientConfig } from "./config.js";
 import { MINING_LOOP_INTERVAL_MS, MINING_NETWORK_SETTLE_WINDOW_MS, MINING_PROVIDER_BACKOFF_BASE_MS, MINING_PROVIDER_BACKOFF_MAX_MS, MINING_SHUTDOWN_GRACE_MS, MINING_STATUS_HEARTBEAT_INTERVAL_MS, MINING_SUSPEND_GAP_THRESHOLD_MS, MINING_TIP_SETTLE_WINDOW_MS, MINING_WORKER_API_VERSION, } from "./constants.js";
 import { inspectMiningControlPlane, setupBuiltInMining } from "./control.js";
-import { isMiningGenerationAbortRequested, markMiningGenerationActive, markMiningGenerationInactive, readMiningPreemptionRequest, requestMiningGenerationPreemption, } from "./coordination.js";
+import { isMiningGenerationAbortRequested, markMiningGenerationActive, markMiningGenerationInactive, readMiningGenerationActivity, readMiningPreemptionRequest, requestMiningGenerationPreemption, } from "./coordination.js";
 import { clearMiningPublishState, miningPublishIsInMempool, miningPublishMayStillExist, normalizeMiningPublishState, normalizeMiningStateRecord, } from "./state.js";
 import { createMiningSentenceRequestLimits } from "./sentence-protocol.js";
 import { generateMiningSentences, MiningProviderRequestError } from "./sentences.js";
 import { createEmptyMiningFollowVisualizerState, MiningFollowVisualizer, } from "./visualizer.js";
 const BEST_BLOCK_POLL_INTERVAL_MS = 500;
 const BACKGROUND_START_TIMEOUT_MS = 15_000;
+function resolveSnapshotOverride(override, fallback) {
+    return override === undefined ? fallback : override;
+}
 class MiningSuspendDetectedError extends Error {
     detectedAtUnixMs;
     constructor(detectedAtUnixMs) {
@@ -88,6 +93,239 @@ async function isProcessAlive(pid) {
         return true;
     }
 }
+function normalizeMiningPid(value) {
+    return typeof value === "number" && Number.isInteger(value) && value > 0
+        ? value
+        : null;
+}
+function resolveMiningGenerationRequestPath(paths) {
+    return join(paths.miningRoot, "generation-request.json");
+}
+function resolveMiningGenerationActivityPath(paths) {
+    return join(paths.miningRoot, "generation-activity.json");
+}
+function createTakeoverStoppedMiningNote(livePublishInMempool) {
+    return livePublishInMempool
+        ? "Mining runtime replaced. The last mining transaction may still confirm from mempool."
+        : "Mining runtime replaced.";
+}
+function createStoppedMiningRuntimeSnapshotForTakeover(options) {
+    const note = createTakeoverStoppedMiningNote(options.snapshot?.livePublishInMempool);
+    if (options.snapshot !== null) {
+        return {
+            ...options.snapshot,
+            updatedAtUnixMs: options.nowUnixMs,
+            runMode: "stopped",
+            backgroundWorkerPid: null,
+            backgroundWorkerRunId: null,
+            backgroundWorkerHeartbeatAtUnixMs: null,
+            backgroundWorkerHealth: null,
+            currentPhase: "idle",
+            note,
+        };
+    }
+    return {
+        schemaVersion: 1,
+        walletRootId: options.walletRootId,
+        workerApiVersion: null,
+        workerBinaryVersion: null,
+        workerBuildId: null,
+        updatedAtUnixMs: options.nowUnixMs,
+        runMode: "stopped",
+        backgroundWorkerPid: null,
+        backgroundWorkerRunId: null,
+        backgroundWorkerHeartbeatAtUnixMs: null,
+        backgroundWorkerHealth: null,
+        indexerDaemonState: null,
+        indexerDaemonInstanceId: null,
+        indexerSnapshotSeq: null,
+        indexerSnapshotOpenedAtUnixMs: null,
+        indexerTruthSource: undefined,
+        indexerHeartbeatAtUnixMs: null,
+        coreBestHeight: null,
+        coreBestHash: null,
+        indexerTipHeight: null,
+        indexerTipHash: null,
+        indexerReorgDepth: null,
+        indexerTipAligned: null,
+        corePublishState: null,
+        providerState: null,
+        lastSuspendDetectedAtUnixMs: null,
+        reconnectSettledUntilUnixMs: null,
+        tipSettledUntilUnixMs: null,
+        miningState: "idle",
+        currentPhase: "idle",
+        currentPublishState: "none",
+        targetBlockHeight: null,
+        referencedBlockHashDisplay: null,
+        currentDomainId: null,
+        currentDomainName: null,
+        currentSentenceDisplay: null,
+        currentCanonicalBlend: null,
+        currentTxid: null,
+        currentWtxid: null,
+        livePublishInMempool: null,
+        currentFeeRateSatVb: null,
+        currentAbsoluteFeeSats: null,
+        currentBlockFeeSpentSats: "0",
+        sessionFeeSpentSats: "0",
+        lifetimeFeeSpentSats: "0",
+        sameDomainCompetitorSuppressed: null,
+        higherRankedCompetitorDomainCount: null,
+        dedupedCompetitorDomainCount: null,
+        competitivenessGateIndeterminate: null,
+        mempoolSequenceCacheStatus: null,
+        currentPublishDecision: null,
+        lastMempoolSequence: null,
+        lastCompetitivenessGateAtUnixMs: null,
+        pauseReason: null,
+        providerConfigured: false,
+        providerKind: null,
+        bitcoindHealth: "unavailable",
+        bitcoindServiceState: null,
+        bitcoindReplicaStatus: null,
+        nodeHealth: "unavailable",
+        indexerHealth: "unavailable",
+        tipsAligned: null,
+        lastEventAtUnixMs: null,
+        lastError: null,
+        note,
+    };
+}
+async function waitForMiningProcessExit(pid, timeoutMs, sleepImpl = sleep) {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        if (!await isProcessAlive(pid)) {
+            return true;
+        }
+        await sleepImpl(Math.min(250, Math.max(timeoutMs, 1)));
+    }
+    return !await isProcessAlive(pid);
+}
+async function terminateMiningRuntimePid(options) {
+    if (!await isProcessAlive(options.pid)) {
+        return false;
+    }
+    try {
+        process.kill(options.pid, "SIGTERM");
+    }
+    catch (error) {
+        if (!(error instanceof Error && "code" in error && error.code === "ESRCH")) {
+            throw error;
+        }
+    }
+    if (await waitForMiningProcessExit(options.pid, options.shutdownGraceMs, options.sleepImpl)) {
+        return true;
+    }
+    try {
+        process.kill(options.pid, "SIGKILL");
+    }
+    catch (error) {
+        if (!(error instanceof Error && "code" in error && error.code === "ESRCH")) {
+            throw error;
+        }
+    }
+    if (await waitForMiningProcessExit(options.pid, options.shutdownGraceMs, options.sleepImpl)) {
+        return true;
+    }
+    throw new Error("mining_process_stop_timeout");
+}
+async function takeOverMiningRuntime(options) {
+    const snapshot = await loadMiningRuntimeStatus(options.paths.miningStatusPath).catch(() => null);
+    const controlLockMetadata = options.controlLockMetadata ?? (options.clearControlLockFile === true
+        ? await readLockMetadata(options.paths.miningControlLockPath).catch(() => null)
+        : null);
+    const generationActivity = await readMiningGenerationActivity(options.paths).catch(() => null);
+    const shutdownGraceMs = options.shutdownGraceMs ?? MINING_SHUTDOWN_GRACE_MS;
+    const requestPreemption = options.requestMiningPreemption ?? requestMiningGenerationPreemption;
+    const controlLockPid = normalizeMiningPid(controlLockMetadata?.processId);
+    const backgroundWorkerPid = normalizeMiningPid(snapshot?.backgroundWorkerPid);
+    const generationOwnerPid = normalizeMiningPid(generationActivity?.generationOwnerPid);
+    const terminatedPids = [];
+    const discoveredPids = new Set();
+    for (const pid of [controlLockPid, backgroundWorkerPid, generationOwnerPid]) {
+        if (pid === null
+            || pid === process.pid
+            || discoveredPids.has(pid)
+            || !await isProcessAlive(pid)) {
+            continue;
+        }
+        discoveredPids.add(pid);
+    }
+    const shouldPreemptGeneration = discoveredPids.size > 0 && (generationActivity?.generationActive === true
+        || snapshot?.currentPhase === "generating"
+        || snapshot?.currentPhase === "scoring");
+    const preemption = shouldPreemptGeneration
+        ? await requestPreemption({
+            paths: options.paths,
+            reason: options.reason,
+            timeoutMs: Math.min(shutdownGraceMs, 15_000),
+        }).catch(() => null)
+        : null;
+    try {
+        for (const pid of discoveredPids) {
+            if (await terminateMiningRuntimePid({
+                pid,
+                shutdownGraceMs,
+                sleepImpl: options.sleepImpl,
+            })) {
+                terminatedPids.push(pid);
+            }
+        }
+    }
+    finally {
+        await preemption?.release().catch(() => undefined);
+    }
+    const controlLockCleared = options.clearControlLockFile === true
+        ? await clearOrphanedFileLock(options.paths.miningControlLockPath, isProcessAlive).catch(() => false)
+        : false;
+    await rm(resolveMiningGenerationRequestPath(options.paths), { force: true }).catch(() => undefined);
+    await rm(resolveMiningGenerationActivityPath(options.paths), { force: true }).catch(() => undefined);
+    const walletRootId = snapshot?.walletRootId
+        ?? (typeof controlLockMetadata?.walletRootId === "string" ? controlLockMetadata.walletRootId : null);
+    if (snapshot !== null || walletRootId !== null || terminatedPids.length > 0 || controlLockCleared) {
+        await saveMiningRuntimeStatus(options.paths.miningStatusPath, createStoppedMiningRuntimeSnapshotForTakeover({
+            snapshot,
+            walletRootId,
+            nowUnixMs: Date.now(),
+        }));
+    }
+    return {
+        controlLockCleared,
+        replaced: terminatedPids.length > 0,
+        snapshot,
+        terminatedPids,
+    };
+}
+async function acquireMiningStartControlLock(options) {
+    while (true) {
+        try {
+            return await acquireFileLock(options.paths.miningControlLockPath, {
+                purpose: options.purpose,
+            });
+        }
+        catch (error) {
+            if (!(error instanceof FileLockBusyError)) {
+                throw error;
+            }
+            if (error.existingMetadata?.processId === process.pid) {
+                throw error;
+            }
+            const takeover = await takeOverMiningRuntime({
+                paths: options.paths,
+                reason: options.takeoverReason,
+                clearControlLockFile: true,
+                controlLockMetadata: error.existingMetadata,
+                requestMiningPreemption: options.requestMiningPreemption,
+                shutdownGraceMs: options.shutdownGraceMs,
+                sleepImpl: options.sleepImpl,
+            });
+            if (!takeover.replaced && !takeover.controlLockCleared) {
+                throw error;
+            }
+        }
+    }
+}
 function writeStdout(stream, line) {
     if (stream === undefined) {
         return;
@@ -232,7 +470,7 @@ function fallbackSettledWinnerDomainName(domainId) {
     return `domain-${domainId}`;
 }
 function resolveCurrentMinedBlockBoard(options) {
-    const settledBlockHeight = options.nodeBestHeight ?? options.snapshotTipHeight ?? null;
+    const settledBlockHeight = options.snapshotTipHeight ?? null;
     if (settledBlockHeight === null) {
         return {
             settledBlockHeight,
@@ -245,12 +483,6 @@ function resolveCurrentMinedBlockBoard(options) {
             settledBoardEntries: [],
         };
     }
-    if (options.nodeBestHeight !== null && (options.snapshotTipHeight ?? -1) < options.nodeBestHeight) {
-        return {
-            settledBlockHeight,
-            settledBoardEntries: [],
-        };
-    }
     const settledBoardEntries = (getBlockWinners(options.snapshotState, settledBlockHeight) ?? [])
         .slice()
         .sort((left, right) => left.rank - right.rank || left.txIndex - right.txIndex)
@@ -268,15 +500,33 @@ function resolveCurrentMinedBlockBoard(options) {
 export function resolveSettledBoardForTesting(options) {
     return resolveCurrentMinedBlockBoard(options);
 }
-function syncMiningUiSettledBoard(loopState, snapshotState, snapshotTipHeight, nodeBestHeight) {
+function syncMiningUiSettledBoard(loopState, snapshotState, snapshotTipHeight) {
     const settledBoard = resolveCurrentMinedBlockBoard({
         snapshotState,
         snapshotTipHeight,
-        nodeBestHeight,
+        nodeBestHeight: null,
     });
     loopState.ui.settledBlockHeight = settledBoard.settledBlockHeight;
     loopState.ui.settledBoardEntries = settledBoard.settledBoardEntries;
 }
+function syncMiningUiForCurrentTip(options) {
+    const targetBlockHeight = options.nodeBestHeight === null
+        ? null
+        : options.nodeBestHeight + 1;
+    const tipKey = buildMiningTipKey(options.nodeBestHash, targetBlockHeight);
+    if (tipKey !== options.loopState.currentTipKey) {
+        options.loopState.currentTipKey = tipKey;
+        resetMiningUiForTip(options.loopState, targetBlockHeight);
+        if (options.recentWin !== null) {
+            options.loopState.ui.recentWin = options.recentWin;
+        }
+    }
+    syncMiningUiSettledBoard(options.loopState, options.snapshotState, options.snapshotTipHeight);
+    return {
+        targetBlockHeight,
+        tipKey,
+    };
+}
 function setMiningUiCandidate(loopState, candidate) {
     loopState.ui.latestSentence = candidate.sentence;
     loopState.ui.provisionalRequiredWords = [...candidate.bip39Words];
@@ -348,6 +598,21 @@ function syncMiningVisualizerBalances(loopState, readContext, balanceSats) {
         : getBalance(readContext.snapshot.state, readContext.localState.state.funding.scriptPubKeyHex);
     loopState.ui.balanceSats = balanceSats;
 }
+function createIndexedMiningFollowVisualizerState(readContext) {
+    const uiState = createEmptyMiningFollowVisualizerState();
+    const localState = readContext.localState;
+    const settledBoard = resolveCurrentMinedBlockBoard({
+        snapshotState: readContext.snapshot?.state ?? null,
+        snapshotTipHeight: readContext.snapshot?.tip?.height ?? readContext.indexer.snapshotTip?.height ?? null,
+        nodeBestHeight: readContext.nodeStatus?.nodeBestHeight ?? null,
+    });
+    uiState.settledBlockHeight = settledBoard.settledBlockHeight;
+    uiState.settledBoardEntries = settledBoard.settledBoardEntries;
+    if (readContext.snapshot !== null && localState.availability === "ready" && localState.state !== null) {
+        uiState.balanceCogtoshi = getBalance(readContext.snapshot.state, localState.state.funding.scriptPubKeyHex);
+    }
+    return uiState;
+}
 function syncMiningVisualizerBlockTimes(loopState, blockTimesByHeight) {
     loopState.ui.visibleBlockTimesByHeight = { ...blockTimesByHeight };
 }
@@ -642,28 +907,71 @@ async function resolveOverlayAuthorizedMiningDomain(options) {
 function buildStatusSnapshot(view, overrides = {}) {
     return {
         ...view.runtime,
-        runMode: overrides.runMode ?? view.runtime.runMode,
-        backgroundWorkerPid: overrides.backgroundWorkerPid ?? view.runtime.backgroundWorkerPid,
-        backgroundWorkerRunId: overrides.backgroundWorkerRunId ?? view.runtime.backgroundWorkerRunId,
-        backgroundWorkerHeartbeatAtUnixMs: overrides.backgroundWorkerHeartbeatAtUnixMs ?? view.runtime.backgroundWorkerHeartbeatAtUnixMs,
-        currentPhase: overrides.currentPhase ?? view.runtime.currentPhase,
-        lastSuspendDetectedAtUnixMs: overrides.lastSuspendDetectedAtUnixMs ?? view.runtime.lastSuspendDetectedAtUnixMs,
-        providerState: overrides.providerState ?? view.runtime.providerState,
-        corePublishState: overrides.corePublishState ?? view.runtime.corePublishState,
-        currentPublishDecision: overrides.currentPublishDecision ?? view.runtime.currentPublishDecision,
-        sameDomainCompetitorSuppressed: overrides.sameDomainCompetitorSuppressed ?? view.runtime.sameDomainCompetitorSuppressed,
-        higherRankedCompetitorDomainCount: overrides.higherRankedCompetitorDomainCount ?? view.runtime.higherRankedCompetitorDomainCount,
-        dedupedCompetitorDomainCount: overrides.dedupedCompetitorDomainCount ?? view.runtime.dedupedCompetitorDomainCount,
-        competitivenessGateIndeterminate: overrides.competitivenessGateIndeterminate ?? view.runtime.competitivenessGateIndeterminate,
-        mempoolSequenceCacheStatus: overrides.mempoolSequenceCacheStatus ?? view.runtime.mempoolSequenceCacheStatus,
-        lastMempoolSequence: overrides.lastMempoolSequence ?? view.runtime.lastMempoolSequence,
-        lastCompetitivenessGateAtUnixMs: overrides.lastCompetitivenessGateAtUnixMs ?? view.runtime.lastCompetitivenessGateAtUnixMs,
-        lastError: overrides.lastError ?? view.runtime.lastError,
-        note: overrides.note ?? view.runtime.note,
-        livePublishInMempool: overrides.livePublishInMempool ?? view.runtime.livePublishInMempool,
+        runMode: resolveSnapshotOverride(overrides.runMode, view.runtime.runMode),
+        backgroundWorkerPid: resolveSnapshotOverride(overrides.backgroundWorkerPid, view.runtime.backgroundWorkerPid),
+        backgroundWorkerRunId: resolveSnapshotOverride(overrides.backgroundWorkerRunId, view.runtime.backgroundWorkerRunId),
+        backgroundWorkerHeartbeatAtUnixMs: resolveSnapshotOverride(overrides.backgroundWorkerHeartbeatAtUnixMs, view.runtime.backgroundWorkerHeartbeatAtUnixMs),
+        currentPhase: resolveSnapshotOverride(overrides.currentPhase, view.runtime.currentPhase),
+        currentPublishState: resolveSnapshotOverride(overrides.currentPublishState, view.runtime.currentPublishState),
+        targetBlockHeight: resolveSnapshotOverride(overrides.targetBlockHeight, view.runtime.targetBlockHeight),
+        referencedBlockHashDisplay: resolveSnapshotOverride(overrides.referencedBlockHashDisplay, view.runtime.referencedBlockHashDisplay),
+        currentDomainId: resolveSnapshotOverride(overrides.currentDomainId, view.runtime.currentDomainId),
+        currentDomainName: resolveSnapshotOverride(overrides.currentDomainName, view.runtime.currentDomainName),
+        currentSentenceDisplay: resolveSnapshotOverride(overrides.currentSentenceDisplay, view.runtime.currentSentenceDisplay),
+        currentCanonicalBlend: resolveSnapshotOverride(overrides.currentCanonicalBlend, view.runtime.currentCanonicalBlend),
+        currentTxid: resolveSnapshotOverride(overrides.currentTxid, view.runtime.currentTxid),
+        currentWtxid: resolveSnapshotOverride(overrides.currentWtxid, view.runtime.currentWtxid),
+        currentFeeRateSatVb: resolveSnapshotOverride(overrides.currentFeeRateSatVb, view.runtime.currentFeeRateSatVb),
+        currentAbsoluteFeeSats: resolveSnapshotOverride(overrides.currentAbsoluteFeeSats, view.runtime.currentAbsoluteFeeSats),
+        currentBlockFeeSpentSats: resolveSnapshotOverride(overrides.currentBlockFeeSpentSats, view.runtime.currentBlockFeeSpentSats),
+        lastSuspendDetectedAtUnixMs: resolveSnapshotOverride(overrides.lastSuspendDetectedAtUnixMs, view.runtime.lastSuspendDetectedAtUnixMs),
+        providerState: resolveSnapshotOverride(overrides.providerState, view.runtime.providerState),
+        corePublishState: resolveSnapshotOverride(overrides.corePublishState, view.runtime.corePublishState),
+        currentPublishDecision: resolveSnapshotOverride(overrides.currentPublishDecision, view.runtime.currentPublishDecision),
+        sameDomainCompetitorSuppressed: resolveSnapshotOverride(overrides.sameDomainCompetitorSuppressed, view.runtime.sameDomainCompetitorSuppressed),
+        higherRankedCompetitorDomainCount: resolveSnapshotOverride(overrides.higherRankedCompetitorDomainCount, view.runtime.higherRankedCompetitorDomainCount),
+        dedupedCompetitorDomainCount: resolveSnapshotOverride(overrides.dedupedCompetitorDomainCount, view.runtime.dedupedCompetitorDomainCount),
+        competitivenessGateIndeterminate: resolveSnapshotOverride(overrides.competitivenessGateIndeterminate, view.runtime.competitivenessGateIndeterminate),
+        mempoolSequenceCacheStatus: resolveSnapshotOverride(overrides.mempoolSequenceCacheStatus, view.runtime.mempoolSequenceCacheStatus),
+        lastMempoolSequence: resolveSnapshotOverride(overrides.lastMempoolSequence, view.runtime.lastMempoolSequence),
+        lastCompetitivenessGateAtUnixMs: resolveSnapshotOverride(overrides.lastCompetitivenessGateAtUnixMs, view.runtime.lastCompetitivenessGateAtUnixMs),
+        lastError: resolveSnapshotOverride(overrides.lastError, view.runtime.lastError),
+        note: resolveSnapshotOverride(overrides.note, view.runtime.note),
+        livePublishInMempool: resolveSnapshotOverride(overrides.livePublishInMempool, view.runtime.livePublishInMempool),
         updatedAtUnixMs: Date.now(),
     };
 }
+function buildPrePublishStatusOverrides(options) {
+    const replacing = options.state.miningState.currentTxid !== null;
+    const replacingAcrossTips = replacing && !livePublishTargetsCandidateTip({
+        liveState: options.state.miningState,
+        candidate: options.candidate,
+    });
+    return {
+        currentPhase: replacing ? "replacing" : "publishing",
+        currentPublishDecision: replacing ? "replacing" : "publishing",
+        targetBlockHeight: options.candidate.targetBlockHeight,
+        referencedBlockHashDisplay: options.candidate.referencedBlockHashDisplay,
+        currentDomainId: options.candidate.domainId,
+        currentDomainName: options.candidate.domainName,
+        currentSentenceDisplay: options.candidate.sentence,
+        currentCanonicalBlend: options.candidate.canonicalBlend.toString(),
+        note: replacing
+            ? "Replacing the live mining transaction for the current tip."
+            : "Broadcasting the best mining candidate for the current tip.",
+        ...(replacingAcrossTips
+            ? {
+                currentPublishState: "none",
+                currentTxid: null,
+                currentWtxid: null,
+                livePublishInMempool: false,
+                currentFeeRateSatVb: null,
+                currentAbsoluteFeeSats: null,
+                currentBlockFeeSpentSats: "0",
+            }
+            : {}),
+    };
+}
 async function refreshAndSaveStatus(options) {
     const view = await inspectMiningControlPlane({
         provider: options.provider,
@@ -705,6 +1013,7 @@ async function handleDetectedMiningRuntimeResume(options) {
                 note: "Mining discarded stale in-flight work after a large local runtime gap and is rechecking health.",
             },
             visualizer: options.visualizer,
+            visualizerState: createIndexedMiningFollowVisualizerState(readContext),
         });
     }
     finally {
@@ -897,6 +1206,12 @@ function createStaleMiningCandidateWaitingNote() {
 function createRetryableMiningPublishWaitingNote() {
     return "Selected mining candidate did not reach mempool and will be retried on the current tip with refreshed wallet state.";
 }
+function createInsufficientFundsMiningPublishWaitingNote() {
+    return "Mining is waiting for enough confirmed safe BTC funding that Bitcoin Core can use for the next publish.";
+}
+function createInsufficientFundsMiningPublishErrorMessage() {
+    return "Bitcoin Core could not fund the next mining publish with confirmed safe BTC.";
+}
 async function generateCandidatesForDomains(options) {
     const bestBlockHash = options.readContext.nodeStatus?.nodeBestHashHex;
     if (bestBlockHash === null || bestBlockHash === undefined) {
@@ -1524,6 +1839,7 @@ async function publishCandidateOnce(options) {
         dataDir: options.dataDir,
         chain: "main",
         startHeight: 0,
+        serviceLifetime: "ephemeral",
         walletRootId: options.readContext.localState.state.walletRootId,
     });
     const rpc = options.rpcFactory(service.rpc);
@@ -1801,6 +2117,29 @@ async function publishCandidate(options) {
                     candidate: refreshedCandidate,
                 };
             }
+            if (isInsufficientFundsError(error)) {
+                const note = createInsufficientFundsMiningPublishWaitingNote();
+                const lastError = createInsufficientFundsMiningPublishErrorMessage();
+                await appendEventFn(options.paths, createEvent("publish-paused-insufficient-funds", "Paused mining publish because Bitcoin Core could not fund the next mining transaction with confirmed safe BTC.", {
+                    level: "warn",
+                    runId: options.runId,
+                    targetBlockHeight: refreshedCandidate.targetBlockHeight,
+                    referencedBlockHashDisplay: refreshedCandidate.referencedBlockHashDisplay,
+                    domainId: refreshedCandidate.domainId,
+                    domainName: refreshedCandidate.domainName,
+                    score: refreshedCandidate.canonicalBlend.toString(),
+                    reason: "insufficient-funds",
+                }));
+                return {
+                    state: readyReadContext.localState.state,
+                    txid: null,
+                    decision: "publish-paused-insufficient-funds",
+                    note,
+                    lastError,
+                    skipped: true,
+                    candidate: null,
+                };
+            }
             throw error;
         }
     }
@@ -1811,7 +2150,7 @@ async function publishCandidate(options) {
 export async function publishCandidateForTesting(options) {
     return await publishCandidate(options);
 }
-async function ensureBuiltInSetupIfNeeded(options) {
+export async function ensureBuiltInMiningSetupIfNeeded(options) {
     const config = await loadClientConfig({
         path: options.paths.clientConfigPath,
         provider: options.provider,
@@ -1869,6 +2208,7 @@ async function performMiningCycle(options) {
             dataDir: options.dataDir,
             chain: "main",
             startHeight: 0,
+            serviceLifetime: "ephemeral",
             walletRootId: readContext.localState.state.walletRootId,
         });
         checkpointMiningSuspendDetector(options.suspendDetector);
@@ -1911,6 +2251,16 @@ async function performMiningCycle(options) {
             indexedTipHashHex: indexedTip?.blockHashHex ?? null,
         }).catch(() => ({}));
         syncMiningVisualizerBlockTimes(options.loopState, visibleBlockTimes);
+        const { targetBlockHeight, tipKey } = syncMiningUiForCurrentTip({
+            loopState: options.loopState,
+            snapshotState: effectiveReadContext.snapshot?.state ?? null,
+            snapshotTipHeight: effectiveReadContext.snapshot?.tip?.height ?? effectiveReadContext.indexer.snapshotTip?.height ?? null,
+            nodeBestHeight: effectiveReadContext.nodeStatus?.nodeBestHeight ?? null,
+            nodeBestHash: effectiveReadContext.nodeStatus?.nodeBestHashHex ?? null,
+            recentWin: reconciliation.recentWin,
+        });
+        const displaySats = await resolveFundingDisplaySats(effectiveReadContext.localState.state, rpc).catch(() => null);
+        syncMiningVisualizerBalances(options.loopState, effectiveReadContext, displaySats);
         if (effectiveReadContext.localState.state.miningState.state === "repair-required") {
             await refreshAndSaveStatus({
                 paths: options.paths,
@@ -2038,18 +2388,21 @@ async function performMiningCycle(options) {
             });
             return;
         }
-        const targetBlockHeight = (effectiveReadContext.nodeStatus?.nodeBestHeight ?? 0) + 1;
-        const tipKey = buildMiningTipKey(effectiveReadContext.nodeStatus?.nodeBestHashHex ?? null, targetBlockHeight);
-        if (tipKey !== options.loopState.currentTipKey) {
-            options.loopState.currentTipKey = tipKey;
-            resetMiningUiForTip(options.loopState, targetBlockHeight);
-            if (reconciliation.recentWin !== null) {
-                options.loopState.ui.recentWin = reconciliation.recentWin;
-            }
+        if (targetBlockHeight === null) {
+            await refreshAndSaveStatus({
+                paths: options.paths,
+                provider: options.provider,
+                readContext: effectiveReadContext,
+                overrides: {
+                    runMode: options.runMode,
+                    currentPhase: "waiting-bitcoin-network",
+                    note: "Mining is waiting for the local Bitcoin node to become publishable.",
+                },
+                visualizer: options.visualizer,
+                visualizerState: options.loopState.ui,
+            });
+            return;
         }
-        syncMiningUiSettledBoard(options.loopState, effectiveReadContext.snapshot?.state ?? null, effectiveReadContext.snapshot?.tip?.height ?? effectiveReadContext.indexer.snapshotTip?.height ?? null, effectiveReadContext.nodeStatus?.nodeBestHeight ?? null);
-        const displaySats = await resolveFundingDisplaySats(effectiveReadContext.localState.state, rpc).catch(() => null);
-        syncMiningVisualizerBalances(options.loopState, effectiveReadContext, displaySats);
         if (getBlockRewardCogtoshi(targetBlockHeight) === 0n) {
             const nextState = defaultMiningStatePatch(effectiveReadContext.localState.state, {
                 state: "paused",
@@ -2393,12 +2746,10 @@ async function performMiningCycle(options) {
             readContext: effectiveReadContext,
             overrides: {
                 runMode: options.runMode,
-                currentPhase: effectiveReadContext.localState.state.miningState.currentTxid === null
-                    ? "publishing"
-                    : "replacing",
-                note: effectiveReadContext.localState.state.miningState.currentTxid === null
-                    ? "Broadcasting the best mining candidate for the current tip."
-                    : "Replacing the live mining transaction for the current tip.",
+                ...buildPrePublishStatusOverrides({
+                    state: effectiveReadContext.localState.state,
+                    candidate: selectedCandidate,
+                }),
             },
             visualizer: options.visualizer,
             visualizerState: options.loopState.ui,
@@ -2465,6 +2816,9 @@ async function performMiningCycle(options) {
                 clearSelectedCandidate(options.loopState);
                 setMiningUiCandidate(options.loopState, selectedCandidate);
                 options.loopState.waitingNote = published.note;
+                const lastError = published.decision === "publish-paused-insufficient-funds"
+                    ? published.lastError ?? createInsufficientFundsMiningPublishErrorMessage()
+                    : undefined;
                 await refreshAndSaveStatus({
                     paths: options.paths,
                     provider: options.provider,
@@ -2486,6 +2840,7 @@ async function performMiningCycle(options) {
                         mempoolSequenceCacheStatus: gateSnapshot.mempoolSequenceCacheStatus,
                         lastMempoolSequence: gateSnapshot.lastMempoolSequence,
                         lastCompetitivenessGateAtUnixMs: Date.now(),
+                        lastError,
                         note: published.note,
                         livePublishInMempool: published.state.miningState.livePublishInMempool,
                     },
@@ -2580,6 +2935,7 @@ async function saveStopSnapshot(options) {
                 dataDir: options.dataDir,
                 chain: "main",
                 startHeight: 0,
+                serviceLifetime: "ephemeral",
                 walletRootId: localState.state.walletRootId,
             }).catch(() => null);
             if (service !== null) {
@@ -2692,6 +3048,7 @@ async function runMiningLoop(options) {
         dataDir: options.dataDir,
         chain: "main",
         startHeight: 0,
+        serviceLifetime: "ephemeral",
         walletRootId: undefined,
     }).catch(() => null);
     if (service !== null) {
@@ -2723,34 +3080,50 @@ export async function runForegroundMining(options) {
     const openReadContext = options.openReadContext ?? openWalletReadContext;
     const attachService = options.attachService ?? attachOrStartManagedBitcoindService;
     const rpcFactory = options.rpcFactory ?? createRpcClient;
-    const controlLock = await acquireFileLock(paths.miningControlLockPath, {
-        purpose: "mine-foreground",
-    });
+    const requestMiningPreemption = options.requestMiningPreemption ?? requestMiningGenerationPreemption;
+    const runMiningLoopImpl = options.runMiningLoopImpl ?? runMiningLoop;
+    const saveStopSnapshotImpl = options.saveStopSnapshotImpl ?? saveStopSnapshot;
     let visualizer = null;
-    try {
-        const existing = await loadMiningRuntimeStatus(paths.miningStatusPath).catch(() => null);
-        if (existing?.runMode === "background") {
-            throw new Error("Background mining is already active. Run `cogcoin mine stop` first.");
-        }
-        const setupReady = await ensureBuiltInSetupIfNeeded({
+    const setupReady = options.builtInSetupEnsured === true
+        ? true
+        : await ensureBuiltInMiningSetupIfNeeded({
             provider,
             prompter: options.prompter,
             paths,
         });
-        if (!setupReady) {
-            throw new Error("Built-in mining provider is not configured. Run `cogcoin mine setup`.");
-        }
+    if (!setupReady) {
+        throw new Error("Built-in mining provider is not configured. Run `cogcoin mine setup`.");
+    }
+    const controlLock = await acquireMiningStartControlLock({
+        paths,
+        purpose: "mine-foreground",
+        takeoverReason: "mine-foreground-replace",
+        requestMiningPreemption,
+        shutdownGraceMs: options.shutdownGraceMs,
+        sleepImpl: options.sleepImpl,
+    });
+    const abortController = new AbortController();
+    const abortListener = () => {
+        abortController.abort();
+    };
+    const handleSigint = () => abortController.abort();
+    const handleSigterm = () => abortController.abort();
+    try {
+        await takeOverMiningRuntime({
+            paths,
+            reason: "mine-foreground-replace",
+            requestMiningPreemption,
+            shutdownGraceMs: options.shutdownGraceMs,
+            sleepImpl: options.sleepImpl,
+        });
         visualizer = new MiningFollowVisualizer({
             progressOutput: options.progressOutput ?? "auto",
             stream: options.stderr,
         });
-        const abortController = new AbortController();
-        options.signal?.addEventListener("abort", () => {
-            abortController.abort();
-        }, { once: true });
-        process.on("SIGINT", () => abortController.abort());
-        process.on("SIGTERM", () => abortController.abort());
-        await runMiningLoop({
+        options.signal?.addEventListener("abort", abortListener, { once: true });
+        process.on("SIGINT", handleSigint);
+        process.on("SIGTERM", handleSigterm);
+        await runMiningLoopImpl({
             dataDir: options.dataDir,
             databasePath: options.databasePath,
             provider,
@@ -2766,7 +3139,7 @@ export async function runForegroundMining(options) {
             stdout: options.stdout,
             visualizer,
         });
-        await saveStopSnapshot({
+        await saveStopSnapshotImpl({
             dataDir: options.dataDir,
             databasePath: options.databasePath,
             provider,
@@ -2778,6 +3151,9 @@ export async function runForegroundMining(options) {
         });
     }
     finally {
+        options.signal?.removeEventListener("abort", abortListener);
+        process.off("SIGINT", handleSigint);
+        process.off("SIGTERM", handleSigterm);
         visualizer?.close();
         await controlLock.release();
     }
@@ -2785,33 +3161,50 @@ export async function runForegroundMining(options) {
 export async function startBackgroundMining(options) {
     const provider = options.provider ?? createDefaultWalletSecretProvider();
     const paths = options.paths ?? resolveWalletRuntimePathsForTesting();
-    const controlLock = await acquireFileLock(paths.miningControlLockPath, {
-        purpose: "mine-start",
-    });
+    const requestMiningPreemption = options.requestMiningPreemption ?? requestMiningGenerationPreemption;
+    const spawnWorkerProcess = options.spawnWorkerProcess ?? spawn;
+    const waitForBackgroundHealthyImpl = options.waitForBackgroundHealthyImpl ?? waitForBackgroundHealthy;
+    const setupReady = options.builtInSetupEnsured === true
+        ? true
+        : await ensureBuiltInMiningSetupIfNeeded({
+            provider,
+            prompter: options.prompter,
+            paths,
+        });
+    if (!setupReady) {
+        throw new Error("Built-in mining provider is not configured. Run `cogcoin mine setup`.");
+    }
+    let controlLock;
     try {
-        const existing = await loadMiningRuntimeStatus(paths.miningStatusPath).catch(() => null);
-        if (existing?.runMode === "background"
-            && existing.backgroundWorkerPid !== null
-            && await isProcessAlive(existing.backgroundWorkerPid)) {
+        controlLock = await acquireMiningStartControlLock({
+            paths,
+            purpose: "mine-start",
+            takeoverReason: "mine-start-replace",
+            requestMiningPreemption,
+            shutdownGraceMs: options.shutdownGraceMs,
+            sleepImpl: options.sleepImpl,
+        });
+    }
+    catch (error) {
+        if (error instanceof FileLockBusyError && error.existingMetadata?.processId === process.pid) {
             return {
                 started: false,
-                snapshot: existing,
+                snapshot: await loadMiningRuntimeStatus(paths.miningStatusPath).catch(() => null),
             };
         }
-        if (existing?.runMode === "foreground") {
-            throw new Error("Foreground mining is already active. Interrupt that process directly.");
-        }
-        const setupReady = await ensureBuiltInSetupIfNeeded({
-            provider,
-            prompter: options.prompter,
+        throw error;
+    }
+    try {
+        await takeOverMiningRuntime({
             paths,
+            reason: "mine-start-replace",
+            requestMiningPreemption,
+            shutdownGraceMs: options.shutdownGraceMs,
+            sleepImpl: options.sleepImpl,
         });
-        if (!setupReady) {
-            throw new Error("Built-in mining provider is not configured. Run `cogcoin mine setup`.");
-        }
         const runId = randomBytes(16).toString("hex");
         const workerMainPath = fileURLToPath(new URL("./worker-main.js", import.meta.url));
-        const child = spawn(process.execPath, [
+        const child = spawnWorkerProcess(process.execPath, [
             workerMainPath,
             `--data-dir=${options.dataDir}`,
             `--database-path=${options.databasePath}`,
@@ -2821,7 +3214,7 @@ export async function startBackgroundMining(options) {
             stdio: "ignore",
         });
         child.unref();
-        const snapshot = await waitForBackgroundHealthy(paths);
+        const snapshot = await waitForBackgroundHealthyImpl(paths);
         return {
             started: true,
             snapshot,
@@ -2955,12 +3348,21 @@ export async function runBackgroundMiningWorker(options) {
 export async function handleDetectedMiningRuntimeResumeForTesting(options) {
     await handleDetectedMiningRuntimeResume(options);
 }
+export async function takeOverMiningRuntimeForTesting(options) {
+    return await takeOverMiningRuntime(options);
+}
 export async function performMiningCycleForTesting(options) {
     await performMiningCycle({
         ...options,
         loopState: options.loopState ?? createMiningLoopState(),
     });
 }
+export function buildPrePublishStatusOverridesForTesting(options) {
+    return buildPrePublishStatusOverrides(options);
+}
+export function buildStatusSnapshotForTesting(view, overrides = {}) {
+    return buildStatusSnapshot(view, overrides);
+}
 export function shouldKeepCurrentTipLivePublishForTesting(options) {
     return livePublishTargetsCandidateTip(options);
 }