@lodestar/beacon-node 1.49.0-dev.4d35c8b82e → 1.49.0-dev.535f5dd45f

package/lib/sync/unknownBlock.js

@@ -2,7 +2,7 @@ import { routes } from "@lodestar/api";
 import { ForkSeq, SLOTS_PER_EPOCH, isForkPostGloas } from "@lodestar/params";
 import { RequestError, RequestErrorCode } from "@lodestar/reqresp";
 import { computeTimeAtSlot } from "@lodestar/state-transition";
-import { fromHex, prettyPrintIndices, pruneSetToMax, sleep, toRootHex } from "@lodestar/utils";
+import { fromHex, isErrorAborted, prettyPrintIndices, pruneSetToMax, sleep, toRootHex } from "@lodestar/utils";
 import { isBlockInputBlobs, isBlockInputColumns } from "../chain/blocks/blockInput/blockInput.js";
 import { BlockInputSource } from "../chain/blocks/blockInput/types.js";
 import { PayloadError, PayloadErrorCode } from "../chain/blocks/importExecutionPayload.js";
@@ -13,11 +13,12 @@ import { validateGloasBlockDataColumnSidecars } from "../chain/validation/dataCo
 import { validateGossipExecutionPayloadEnvelope } from "../chain/validation/executionPayloadEnvelope.js";
 import { NetworkEvent, prettyPrintPeerIdStr } from "../network/index.js";
 import { MAX_PEERS_PER_ROOT } from "../network/processor/constants.js";
+import { ClockEvent } from "../util/clock.js";
 import { shuffle } from "../util/shuffle.js";
 import { sortBy } from "../util/sortBy.js";
 import { wrapError } from "../util/wrapError.js";
 import { MAX_CONCURRENT_REQUESTS } from "./constants.js";
-import { DroppedItemReason, PendingBlockInputStatus, PendingBlockType, PendingPayloadInputStatus, getBlockInputSyncCacheItemRootHex, getBlockInputSyncCacheItemSlot, getPayloadSyncCacheItemRootHex, getPayloadSyncCacheItemSlot, isPendingBlockInput, isPendingPayloadEnvelope, isPendingPayloadInput, } from "./types.js";
+import { DeferredPayloadResult, DownloadResult, DroppedItemReason, FetchResult, PendingBlockInputStatus, PendingBlockType, PendingPayloadInputStatus, getBlockInputSyncCacheItemRootHex, getBlockInputSyncCacheItemSlot, getPayloadSyncCacheItemRootHex, getPayloadSyncCacheItemSlot, isPendingBlockInput, isPendingPayloadEnvelope, isPendingPayloadInput, } from "./types.js";
 import { DownloadByRootError, downloadByRoot } from "./utils/downloadByRoot.js";
 import { getAllDescendantBlocks, getUnknownAndAncestorBlocks } from "./utils/pendingBlocksTree.js";
 import { getRateLimitedUntilMs } from "./utils/rateLimit.js";
@@ -28,14 +29,10 @@ const MAX_PENDING_BLOCKS = 100;
  * A pending payload without slot can only stay at most 2 epochs.
  */
 const PRUNE_UNRESOLVED_SLOT_EPOCHS = 2;
-var FetchResult;
-(function (FetchResult) {
-    FetchResult["SuccessResolved"] = "success_resolved";
-    FetchResult["SuccessMissingParent"] = "success_missing_parent";
-    FetchResult["SuccessLate"] = "success_late";
-    FetchResult["FailureTriedAllPeers"] = "failure_tried_all_peers";
-    FetchResult["FailureMaxAttempts"] = "failure_max_attempts";
-})(FetchResult || (FetchResult = {}));
+/**
+ * Poll pending payloads of the current slot this often, between PAYLOAD_DUE and the end of the slot.
+ */
+export const PAYLOAD_POLL_INTERVAL_MS = 1_000;
 class UnknownBlockRateLimitedError extends Error {
     constructor(message) {
         super(message);
@@ -92,6 +89,11 @@ export class BlockInputSync {
     subscribedToNetworkEvents = false;
     peerBalancer;
     rateLimitBackoffTimeout;
+    /**
+     * Search unknown payload starting at PAYLOAD_DUE_BPS.
+     */
+    deferredPayloadRoots = new Map();
+    payloadPollAbortController;
     constructor(config, network, chain, logger, metrics, opts) {
         this.config = config;
         this.network = network;
@@ -117,6 +119,7 @@ export class BlockInputSync {
             this.logger.verbose("BlockInputSync enabled.");
             this.chain.emitter.on(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
             this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
+            this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRootSlot, this.onUnknownEnvelopeBlockRootSlot);
             this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
             this.chain.emitter.on(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
             this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
@@ -125,6 +128,7 @@ export class BlockInputSync {
             this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadImported);
             this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
             this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
+            this.chain.clock.on(ClockEvent.slot, this.startPayloadPollAtSlot);
             // Seed the balancer with peers that connected before we subscribed
             for (const peerId of this.network.getConnectedPeers()) {
                 try {
@@ -135,13 +139,21 @@ export class BlockInputSync {
                 }
             }
             this.subscribedToNetworkEvents = true;
+            // we subscribe mid-slot, and a root can be deferred at any point in the slot, so start this
+            // slot's loop now instead of waiting for the next ClockEvent.slot
+            this.startPayloadPollAtSlot(this.chain.clock.currentSlot);
         }
     }
     unsubscribeFromNetwork() {
         this.logger.verbose("BlockInputSync disabled.");
         this.clearRateLimitBackoffTimer();
+        this.chain.clock.off(ClockEvent.slot, this.startPayloadPollAtSlot);
+        this.payloadPollAbortController?.abort();
+        this.payloadPollAbortController = undefined;
+        this.deferredPayloadRoots.clear();
         this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
         this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
+        this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRootSlot, this.onUnknownEnvelopeBlockRootSlot);
         this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
         this.chain.emitter.off(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
         this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
@@ -165,8 +177,11 @@ export class BlockInputSync {
     onUnknownBlockRoot = (data) => {
         try {
             const isNewRoot = this.addByRootHex(data.rootHex, data.peer);
-            if (isNewRoot) {
+            // A retained entry (an earlier download failed, #10018) is not new but still needs a retry pass.
+            if (this.pendingBlocks.get(data.rootHex)?.status === PendingBlockInputStatus.pending) {
                 this.triggerUnknownBlockSearch();
+            }
+            if (isNewRoot) {
                 this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_BLOCK_ROOT });
                 this.metrics?.blockInputSync.source.inc({ source: data.source });
             }
@@ -202,6 +217,33 @@ export class BlockInputSync {
             this.logger.debug("Error handling unknownEnvelopeBlockRoot event", {}, e);
         }
     };
+    onUnknownEnvelopeBlockRootSlot = (data) => {
+        try {
+            if (data.slot === this.chain.clock.currentSlot) {
+                // we receive the optimistic search, the poll loop searches it at PAYLOAD_DUE or on its next
+                // tick. If that loop already returned - every root it watched was imported - this root waits
+                // for the next slot, where startPayloadPollAtSlot moves it to pendingPayloads. That is not the normal
+                // scenario, and in a forking condition it only delays the search of the extra root
+                const deferred = this.deferredPayloadRoots.get(data.rootHex);
+                if (deferred === undefined) {
+                    this.deferredPayloadRoots.set(data.rootHex, { slot: data.slot, source: data.source, peer: data.peer });
+                }
+                else if (deferred.peer === undefined) {
+                    deferred.peer = data.peer;
+                }
+                return;
+            }
+            const isNewRoot = this.addByPayloadRootHex(data.rootHex, data.peer, data.slot);
+            if (isNewRoot) {
+                this.triggerUnknownBlockSearch();
+                this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_PAYLOAD_BLOCK_ROOT_SLOT });
+                this.metrics?.blockInputSync.payloadSource.inc({ source: data.source });
+            }
+        }
+        catch (e) {
+            this.logger.debug("Error handling unknownEnvelopeBlockRootSlot event", {}, e);
+        }
+    };
     onIncompletePayloadEnvelope = (data) => {
         try {
             this.addByPayloadInput(data.payloadInput, data.peer);
@@ -296,12 +338,21 @@ export class BlockInputSync {
                 belowFinalizedPayloads++;
             }
         }
-        let deletedBlocks = 0;
+        let belowFinalizedBlocks = 0;
+        let agedOutBlocks = 0;
         for (const [rootHex, block] of this.pendingBlocks) {
             const slot = getBlockInputSyncCacheItemSlot(block);
-            if (typeof slot === "number" && slot < finalizedSlot) {
+            if (typeof slot !== "number") {
+                // we may not be able to download a root, hence there is no slot for it
+                if (nowSec - block.timeAddedSec > maxSecInCache) {
+                    this.pendingBlocks.delete(rootHex);
+                    agedOutBlocks++;
+                }
+                continue;
+            }
+            if (slot < finalizedSlot) {
                 this.pendingBlocks.delete(rootHex);
-                deletedBlocks++;
+                belowFinalizedBlocks++;
             }
         }
         if (belowFinalizedPayloads > 0) {
@@ -310,15 +361,19 @@ export class BlockInputSync {
         if (agedOutPayloads > 0) {
             this.metrics?.blockInputSync.removedPayloads.inc({ reason: DroppedItemReason.agedOut }, agedOutPayloads);
         }
-        if (deletedBlocks > 0) {
-            this.metrics?.blockInputSync.removedBlocks.inc({ reason: DroppedItemReason.belowFinalized }, deletedBlocks);
+        if (belowFinalizedBlocks > 0) {
+            this.metrics?.blockInputSync.removedBlocks.inc({ reason: DroppedItemReason.belowFinalized }, belowFinalizedBlocks);
         }
-        this.logger.debug("BlockInputSync.pruneFinalized dropped pre-finalized pending items", {
+        if (agedOutBlocks > 0) {
+            this.metrics?.blockInputSync.removedBlocks.inc({ reason: DroppedItemReason.agedOut }, agedOutBlocks);
+        }
+        this.logger.debug("BlockInputSync.pruneFinalized dropped stale pending items", {
             finalizedSlot,
             finalizedRoot: checkpoint.rootHex,
             belowFinalizedPayloads,
             agedOutPayloads,
-            deletedBlocks,
+            belowFinalizedBlocks,
+            agedOutBlocks,
             unresolvedSlotPayloads,
         });
     };
@@ -660,6 +715,115 @@ export class BlockInputSync {
             this.rateLimitBackoffTimeout = undefined;
         }
     }
+    /**
+     * Start this slot's payload poll: end the previous slot's loop, move roots it never got to
+     * into pendingPayloads, then poll from PAYLOAD_DUE to the end of the slot.
+     */
+    startPayloadPollAtSlot = (slot) => {
+        if (!isForkPostGloas(this.config.getForkName(slot))) {
+            return;
+        }
+        // end the previous slot's loop, wherever it is: its sleep is bound to this same controller
+        this.payloadPollAbortController?.abort();
+        this.payloadPollAbortController = new AbortController();
+        // entries from an earlier slot is not polled
+        for (const [rootHex, { slot: deferredSlot, source, peer }] of this.deferredPayloadRoots) {
+            if (deferredSlot < slot) {
+                if (!this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+                    this.addDeferredPayloadRoot(rootHex, deferredSlot, source, peer);
+                }
+                this.deferredPayloadRoots.delete(rootHex);
+            }
+        }
+        this.pollPayloadsAtSlot(slot, this.payloadPollAbortController.signal).catch((e) => {
+            if (!isErrorAborted(e)) {
+                this.logger.debug("Error polling payloads", { slot }, e);
+            }
+        });
+    };
+    /**
+     * Poll unknown payloads of the current slot.
+     */
+    async pollPayloadsAtSlot(slot, signal) {
+        // fixed tick count so the loop cannot outlive its slot: for mainnet, it's 6 ticks
+        const tickCount = Math.floor((this.config.SLOT_DURATION_MS - this.config.getPayloadDueMs()) / PAYLOAD_POLL_INTERVAL_MS);
+        // a root still missing on several ticks is searched on each of them, recorded once
+        const searchedRoots = new Set();
+        let polls = 0;
+        try {
+            await sleep(Math.max(0, this.config.getPayloadDueMs() - this.chain.clock.msFromSlot(slot)), signal);
+            for (polls = 0; polls < tickCount; polls++) {
+                let importedCount = 0;
+                for (const [rootHex, { slot: deferredSlot, source, peer }] of this.deferredPayloadRoots) {
+                    if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+                        importedCount++;
+                        continue;
+                    }
+                    // copy to the regular pendingPayloads so that we can search
+                    this.addDeferredPayloadRoot(rootHex, deferredSlot, source, peer);
+                    searchedRoots.add(rootHex);
+                }
+                // an empty deferredPayloadRoots means there is no optimistic search comes before PAYLOAD_DUE
+                // we have to poll until the payload of current slot is imported or slot end
+                if (this.deferredPayloadRoots.size > 0 && importedCount === this.deferredPayloadRoots.size) {
+                    if (searchedRoots.size > 0) {
+                        // the backup search was needed
+                        this.metrics?.blockInputSync.deferredPayloadResult.inc({
+                            result: DeferredPayloadResult.ImportedAfterSearch,
+                        });
+                        this.metrics?.blockInputSync.deferredPayloadPolls.observe(polls);
+                        this.logger.verbose("Payload imported after unknown sync search", {
+                            slot,
+                            roots: Array.from(searchedRoots).join(","),
+                            polls,
+                        });
+                    }
+                    else {
+                        // this happens most of the time on devnet
+                        // gossip delivered everything before PAYLOAD_DUE
+                        this.metrics?.blockInputSync.deferredPayloadResult.inc({
+                            result: DeferredPayloadResult.ImportedBeforeSearch,
+                        });
+                    }
+                    return;
+                }
+                this.triggerUnknownBlockSearch();
+                if (polls < tickCount - 1) {
+                    // no sleep after the last tick: the loop would otherwise end right at the slot boundary,
+                    // where startPayloadPollAtSlot aborts it
+                    await sleep(PAYLOAD_POLL_INTERVAL_MS, signal);
+                }
+            }
+            // reaching here means the early return never fired
+            if (this.deferredPayloadRoots.size > 0) {
+                this.metrics?.blockInputSync.deferredPayloadResult.inc({ result: DeferredPayloadResult.Unresolved });
+                this.logger.debug("Deferred payloads still missing at end of slot", {
+                    slot,
+                    roots: Array.from(this.deferredPayloadRoots.keys()).join(","),
+                });
+            }
+            else {
+                // no optimistic search and no payload import ended the loop, eg a skipped slot
+                this.metrics?.blockInputSync.deferredPayloadResult.inc({ result: DeferredPayloadResult.ResolvedNoRoot });
+                this.logger.debug("No deferred payloads at slot", {
+                    slot,
+                });
+            }
+        }
+        catch (e) {
+            if (!isErrorAborted(e)) {
+                throw e;
+            }
+            this.logger.verbose("Payload poll aborted", { slot, polls, msIntoSlot: this.chain.clock.msFromSlot(slot) });
+        }
+    }
+    addDeferredPayloadRoot(rootHex, slot, source, peer) {
+        const isNewRoot = this.addByPayloadRootHex(rootHex, peer, slot);
+        if (isNewRoot) {
+            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.DEFERRED_PAYLOAD_BLOCK_ROOT });
+            this.metrics?.blockInputSync.payloadSource.inc({ source });
+        }
+    }
     async downloadBlock(block) {
         if (block.status !== PendingBlockInputStatus.pending) {
             return;
@@ -678,7 +842,12 @@ export class BlockInputSync {
         if (!res.err) {
             this.metrics?.blockInputSync.downloadedBlocksSuccess.inc();
             const pending = res.result;
-            this.pendingBlocks.set(pending.blockInput.blockRootHex, pending);
+            // pruning may have deleted this entry while we were awaiting the network, do not resurrect it via the set below
+            if (!this.pendingBlocks.has(rootHex)) {
+                this.logger.verbose("Dropping downloaded block, entry pruned during fetch", logCtx);
+                return;
+            }
+            this.pendingBlocks.set(rootHex, pending);
             const blockSlot = pending.blockInput.slot;
             const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
             const delaySec = Date.now() / 1000 - computeTimeAtSlot(this.config, blockSlot, this.chain.genesisTime);
@@ -738,8 +907,11 @@ export class BlockInputSync {
                 return;
             }
             this.metrics?.blockInputSync.downloadedBlocksError.inc();
-            this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
-            this.removeAndDownScoreAllDescendants(block, DroppedItemReason.unavailable);
+            this.logger.debug("Failed to download unknown block root, retained for retry", logCtx, res.err);
+            const pendingBlock = this.pendingBlocks.get(rootHex);
+            if (pendingBlock && pendingBlock.status === PendingBlockInputStatus.fetching) {
+                pendingBlock.status = PendingBlockInputStatus.pending;
+            }
         }
     }
     /**
@@ -946,15 +1118,23 @@ export class BlockInputSync {
                 return;
             }
             this.pendingPayloads.set(resultRootHex, pendingPayload);
+            const result = isPendingPayloadEnvelope(pendingPayload)
+                ? DownloadResult.WaitingForBlock
+                : this.chain.forkChoice.hasPayloadHexUnsafe(resultRootHex)
+                    ? DownloadResult.Late
+                    : DownloadResult.Resolved;
+            this.metrics?.blockInputSync.downloadedPayloadsResult.inc({ result });
             if (isPendingPayloadEnvelope(pendingPayload)) {
                 await this.reconcilePayloadEnvelope(pendingPayload);
             }
             else if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
+                this.metrics?.blockInputSync.elapsedTimeTillPayloadReceived.observe(Date.now() / 1000 - computeTimeAtSlot(this.config, pendingPayload.payloadInput.slot, this.chain.genesisTime));
                 await this.processPayload(pendingPayload);
             }
             return;
         }
-        this.logger.debug("Ignoring unknown payload root after failed download", logCtx, res.err);
+        this.metrics?.blockInputSync.downloadedPayloadsResult.inc({ result: DownloadResult.Failed });
+        this.logger.debug("Failed to download unknown payload root, retained for retry", logCtx, res.err);
         if (!isPendingPayloadEnvelope(payload)) {
             payload.status = PendingPayloadInputStatus.pending;
         }
@@ -1050,6 +1230,10 @@ export class BlockInputSync {
             ? cacheItem.payloadInput
             : await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
         let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
+        const fetchStartSec = Date.now() / 1000;
+        if (typeof slot === "number") {
+            this.metrics?.blockInputSync.payloadFetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
+        }
         let i = 0;
         let deferredByRateLimit = false;
         while (i++ < this.getMaxDownloadAttempts()) {
@@ -1065,6 +1249,8 @@ export class BlockInputSync {
                 const reason = pendingColumns.size > 0
                     ? `cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
                     : "no peer available to download pending payload";
+                this.metrics?.blockInputSync.payloadFetchTime.observe({ result: FetchResult.FailureTriedAllPeers }, Date.now() / 1000 - fetchStartSec);
+                this.metrics?.blockInputSync.payloadFetchPeers.set({ result: FetchResult.FailureTriedAllPeers }, i);
                 throw Error(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: ${reason}`);
             }
             const { peerId, client: peerClient } = peerMeta;
@@ -1072,7 +1258,12 @@ export class BlockInputSync {
             try {
                 if (!envelope) {
                     envelope = await this.fetchExecutionPayloadEnvelope(peerId, blockRoot, rootHex);
+                    const slotWasUnknown = typeof slot !== "number";
                     slot = envelope.message.payload.slotNumber;
+                    if (slotWasUnknown) {
+                        // we were not able to observe the time into slot when starting the fetch, do it now
+                        this.metrics?.blockInputSync.payloadFetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
+                    }
                 }
                 payloadInput ??= await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
                 if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
@@ -1080,6 +1271,8 @@ export class BlockInputSync {
                     // envelope and wait for the block body; reconcilePayloadEnvelope validates once the block lands.
                     // payloadInput may be seeded from the block body during download, so a non-null payloadInput does not
                     // imply the block is imported.
+                    this.metrics?.blockInputSync.payloadFetchTime.observe({ result: FetchResult.SuccessWaitingForBlock }, Date.now() / 1000 - fetchStartSec);
+                    this.metrics?.blockInputSync.payloadFetchPeers.set({ result: FetchResult.SuccessWaitingForBlock }, i);
                     return {
                         status: PendingPayloadInputStatus.waitingForBlock,
                         envelope,
@@ -1124,6 +1317,12 @@ export class BlockInputSync {
                     hasAllData: pendingPayload.payloadInput.hasAllData(),
                 });
                 if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
+                    // late = the payload got imported (via gossip) while we were downloading, same as the block path
+                    const result = this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)
+                        ? FetchResult.SuccessLate
+                        : FetchResult.SuccessResolved;
+                    this.metrics?.blockInputSync.payloadFetchTime.observe({ result }, Date.now() / 1000 - fetchStartSec);
+                    this.metrics?.blockInputSync.payloadFetchPeers.set({ result }, i);
                     return pendingPayload;
                 }
                 cacheItem = pendingPayload;
@@ -1155,6 +1354,8 @@ export class BlockInputSync {
                 this.peerBalancer.onRequestCompleted(peerId);
             }
         }
+        this.metrics?.blockInputSync.payloadFetchTime.observe({ result: FetchResult.FailureMaxAttempts }, Date.now() / 1000 - fetchStartSec);
+        this.metrics?.blockInputSync.payloadFetchPeers.set({ result: FetchResult.FailureMaxAttempts }, i - 1);
         if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
             throw new UnknownBlockRateLimitedError(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`);
         }
@@ -1227,7 +1428,7 @@ export class BlockInputSync {
                 }
                 // no more peer with needed columns to try, throw error
                 const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
-                this.metrics?.blockInputSync.fetchTimeSec.observe({ result: FetchResult.FailureTriedAllPeers }, Date.now() / 1000 - fetchStartSec);
+                this.metrics?.blockInputSync.fetchTime.observe({ result: FetchResult.FailureTriedAllPeers }, Date.now() / 1000 - fetchStartSec);
                 this.metrics?.blockInputSync.fetchPeers.set({ result: FetchResult.FailureTriedAllPeers }, i);
                 throw Error(message);
             }
@@ -1301,7 +1502,11 @@ export class BlockInputSync {
             finally {
                 this.peerBalancer.onRequestCompleted(peerId);
             }
-            this.pendingBlocks.set(getBlockInputSyncCacheItemRootHex(cacheItem), cacheItem);
+            const cacheItemRootHex = getBlockInputSyncCacheItemRootHex(cacheItem);
+            // pruning may have dropped it mid-fetch
+            if (this.pendingBlocks.has(cacheItemRootHex)) {
+                this.pendingBlocks.set(cacheItemRootHex, cacheItem);
+            }
             if (cacheItem.status === PendingBlockInputStatus.downloaded) {
                 // download was successful, no need to go with another peer, return
                 const result = this.chain.forkChoice.hasBlockHex(cacheItem.blockInput.blockRootHex)
@@ -1309,7 +1514,7 @@ export class BlockInputSync {
                     : this.chain.forkChoice.hasBlockHex(cacheItem.blockInput.parentRootHex)
                         ? FetchResult.SuccessResolved
                         : FetchResult.SuccessMissingParent;
-                this.metrics?.blockInputSync.fetchTimeSec.observe({ result }, Date.now() / 1000 - fetchStartSec);
+                this.metrics?.blockInputSync.fetchTime.observe({ result }, Date.now() / 1000 - fetchStartSec);
                 this.metrics?.blockInputSync.fetchPeers.set({ result }, i);
                 return cacheItem;
             }
@@ -1336,7 +1541,7 @@ export class BlockInputSync {
                 throw new Error(`${message} Missing column indices=${prettyPrintIndices(missing)}.`);
             }
         }
-        this.metrics?.blockInputSync.fetchTimeSec.observe({ result: FetchResult.FailureMaxAttempts }, Date.now() / 1000 - fetchStartSec);
+        this.metrics?.blockInputSync.fetchTime.observe({ result: FetchResult.FailureMaxAttempts }, Date.now() / 1000 - fetchStartSec);
         this.metrics?.blockInputSync.fetchPeers.set({ result: FetchResult.FailureMaxAttempts }, i - 1);
         throw Error(message);
     }