@lodestar/beacon-node 1.47.0-dev.ee197a8d7f → 1.47.0-dev.f0f26cd2b6

package/lib/sync/unknownBlock.js

@@ -1,5 +1,5 @@
 import { routes } from "@lodestar/api";
-import { ForkSeq, isForkPostGloas } from "@lodestar/params";
+import { ForkSeq, SLOTS_PER_EPOCH, isForkPostGloas } from "@lodestar/params";
 import { RequestError, RequestErrorCode } from "@lodestar/reqresp";
 import { computeTimeAtSlot } from "@lodestar/state-transition";
 import { fromHex, prettyPrintIndices, pruneSetToMax, sleep, toRootHex } from "@lodestar/utils";
@@ -12,17 +12,22 @@ import { ChainEvent } from "../chain/index.js";
 import { validateGloasBlockDataColumnSidecars } from "../chain/validation/dataColumnSidecar.js";
 import { validateGossipExecutionPayloadEnvelope } from "../chain/validation/executionPayloadEnvelope.js";
 import { NetworkEvent, prettyPrintPeerIdStr } from "../network/index.js";
+import { MAX_PEERS_PER_ROOT } from "../network/processor/constants.js";
 import { shuffle } from "../util/shuffle.js";
 import { sortBy } from "../util/sortBy.js";
 import { wrapError } from "../util/wrapError.js";
 import { MAX_CONCURRENT_REQUESTS } from "./constants.js";
-import { PendingBlockInputStatus, PendingBlockType, PendingPayloadInputStatus, getBlockInputSyncCacheItemRootHex, getBlockInputSyncCacheItemSlot, getPayloadSyncCacheItemRootHex, getPayloadSyncCacheItemSlot, isPendingBlockInput, isPendingPayloadEnvelope, isPendingPayloadInput, } from "./types.js";
+import { DroppedItemReason, PendingBlockInputStatus, PendingBlockType, PendingPayloadInputStatus, getBlockInputSyncCacheItemRootHex, getBlockInputSyncCacheItemSlot, getPayloadSyncCacheItemRootHex, getPayloadSyncCacheItemSlot, isPendingBlockInput, isPendingPayloadEnvelope, isPendingPayloadInput, } from "./types.js";
 import { DownloadByRootError, downloadByRoot } from "./utils/downloadByRoot.js";
 import { getAllDescendantBlocks, getUnknownAndAncestorBlocks } from "./utils/pendingBlocksTree.js";
 import { getRateLimitedUntilMs } from "./utils/rateLimit.js";
 const MAX_ATTEMPTS_PER_BLOCK = 5;
 const MAX_KNOWN_BAD_BLOCKS = 500;
 const MAX_PENDING_BLOCKS = 100;
+/**
+ * A pending payload without slot can only stay at most 2 epochs.
+ */
+const PRUNE_UNRESOLVED_SLOT_EPOCHS = 2;
 var FetchResult;
 (function (FetchResult) {
     FetchResult["SuccessResolved"] = "success_resolved";
@@ -115,10 +120,20 @@ export class BlockInputSync {
             this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
             this.chain.emitter.on(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
             this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
+            this.chain.emitter.on(ChainEvent.forkChoiceFinalized, this.pruneFinalized);
             this.chain.emitter.on(routes.events.EventType.block, this.onBlockImported);
             this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadImported);
             this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
             this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
+            // Seed the balancer with peers that connected before we subscribed
+            for (const peerId of this.network.getConnectedPeers()) {
+                try {
+                    this.peerBalancer.onPeerConnected(peerId, this.network.getConnectedPeerSyncMeta(peerId));
+                }
+                catch (e) {
+                    this.logger.debug("Failed to seed connected peer into BlockInputSync balancer", { peerId }, e);
+                }
+            }
             this.subscribedToNetworkEvents = true;
         }
     }
@@ -130,10 +145,12 @@ export class BlockInputSync {
         this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
         this.chain.emitter.off(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
         this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
+        this.chain.emitter.off(ChainEvent.forkChoiceFinalized, this.pruneFinalized);
         this.chain.emitter.off(routes.events.EventType.block, this.onBlockImported);
         this.chain.emitter.off(routes.events.EventType.executionPayload, this.onPayloadImported);
         this.network.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
         this.network.events.off(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
+        this.peerBalancer.clear();
         this.subscribedToNetworkEvents = false;
     }
     close() {
@@ -147,10 +164,12 @@ export class BlockInputSync {
      */
     onUnknownBlockRoot = (data) => {
         try {
-            this.addByRootHex(data.rootHex, data.peer);
-            this.triggerUnknownBlockSearch();
-            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_BLOCK_ROOT });
-            this.metrics?.blockInputSync.source.inc({ source: data.source });
+            const isNewRoot = this.addByRootHex(data.rootHex, data.peer);
+            if (isNewRoot) {
+                this.triggerUnknownBlockSearch();
+                this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_BLOCK_ROOT });
+                this.metrics?.blockInputSync.source.inc({ source: data.source });
+            }
         }
         catch (e) {
             this.logger.debug("Error handling unknownBlockRoot event", {}, e);
@@ -172,10 +191,12 @@ export class BlockInputSync {
     };
     onUnknownEnvelopeBlockRoot = (data) => {
         try {
-            this.addByPayloadRootHex(data.rootHex, data.peer, data.slot);
-            this.triggerUnknownBlockSearch();
-            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_PAYLOAD_BLOCK_ROOT });
-            this.metrics?.blockInputSync.payloadSource.inc({ source: data.source });
+            const isNewRoot = this.addByPayloadRootHex(data.rootHex, data.peer, this.resolvePayloadSlot(data.rootHex));
+            if (isNewRoot) {
+                this.triggerUnknownBlockSearch();
+                this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_PAYLOAD_BLOCK_ROOT });
+                this.metrics?.blockInputSync.payloadSource.inc({ source: data.source });
+            }
         }
         catch (e) {
             this.logger.debug("Error handling unknownEnvelopeBlockRoot event", {}, e);
@@ -208,12 +229,12 @@ export class BlockInputSync {
                         parentRoot: missingDependency.parentRootHex,
                         parentBlockHash: missingDependency.parentBlockHashHex,
                     });
-                    this.removeAndDownScoreAllDescendants(pendingBlock);
+                    this.removeAndDownScoreAllDescendants(pendingBlock, DroppedItemReason.invalidParentPayload);
                 }
                 return;
             }
             if (missingDependency.kind === "parentPayload") {
-                this.addByPayloadRootHex(missingDependency.rootHex, data.peer);
+                this.addByPayloadRootHex(missingDependency.rootHex, data.peer, missingDependency.slot);
             }
             else if (missingDependency.kind === "parentBlock") {
                 this.addByRootHex(missingDependency.rootHex, data.peer);
@@ -236,6 +257,71 @@ export class BlockInputSync {
         this.pendingPayloads.delete(blockRoot);
         this.triggerUnknownBlockSearch();
     };
+    /**
+     * Drop pending payloads and blocks that fell below the finalized slot.
+     * See https://github.com/ChainSafe/lodestar/issues/9800
+     */
+    pruneFinalized = (checkpoint) => {
+        // Use the actual finalized block slot, not computeStartSlotAtEpoch(checkpoint.epoch)
+        // because the payload at the finalized block is not finalized
+        const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
+        const nowSec = Date.now() / 1000;
+        const maxSecInCache = (PRUNE_UNRESOLVED_SLOT_EPOCHS * SLOTS_PER_EPOCH * this.config.SLOT_DURATION_MS) / 1000;
+        let belowFinalizedPayloads = 0;
+        let agedOutPayloads = 0;
+        let unresolvedSlotPayloads = 0;
+        for (const [rootHex, payload] of this.pendingPayloads) {
+            // Last-effort resolve for a slot-less PendingPayloadRootHex, and store it back for later passes.
+            if (!isPendingPayloadInput(payload) && !isPendingPayloadEnvelope(payload) && payload.slot === undefined) {
+                payload.slot = this.resolvePayloadSlot(rootHex);
+                if (payload.slot === undefined) {
+                    // Still unresolvable: age it out so the MAX_PENDING_BLOCKS cap cannot stay saturated.
+                    if (nowSec - payload.timeAddedSec > maxSecInCache) {
+                        this.pendingPayloads.delete(rootHex);
+                        agedOutPayloads++;
+                    }
+                    else {
+                        unresolvedSlotPayloads++;
+                        this.logger.debug("BlockInputSync.pruneFinalized: pending payload with unresolved slot", {
+                            root: rootHex,
+                            finalizedSlot,
+                        });
+                    }
+                    continue;
+                }
+            }
+            const slot = getPayloadSyncCacheItemSlot(payload);
+            if (typeof slot === "number" && slot < finalizedSlot) {
+                this.pendingPayloads.delete(rootHex);
+                belowFinalizedPayloads++;
+            }
+        }
+        let deletedBlocks = 0;
+        for (const [rootHex, block] of this.pendingBlocks) {
+            const slot = getBlockInputSyncCacheItemSlot(block);
+            if (typeof slot === "number" && slot < finalizedSlot) {
+                this.pendingBlocks.delete(rootHex);
+                deletedBlocks++;
+            }
+        }
+        if (belowFinalizedPayloads > 0) {
+            this.metrics?.blockInputSync.removedPayloads.inc({ reason: DroppedItemReason.belowFinalized }, belowFinalizedPayloads);
+        }
+        if (agedOutPayloads > 0) {
+            this.metrics?.blockInputSync.removedPayloads.inc({ reason: DroppedItemReason.agedOut }, agedOutPayloads);
+        }
+        if (deletedBlocks > 0) {
+            this.metrics?.blockInputSync.removedBlocks.inc({ reason: DroppedItemReason.belowFinalized }, deletedBlocks);
+        }
+        this.logger.debug("BlockInputSync.pruneFinalized dropped pre-finalized pending items", {
+            finalizedSlot,
+            finalizedRoot: checkpoint.rootHex,
+            belowFinalizedPayloads,
+            agedOutPayloads,
+            deletedBlocks,
+            unresolvedSlotPayloads,
+        });
+    };
     addByRootHex = (rootHex, peerIdStr) => {
         let pendingBlock = this.pendingBlocks.get(rootHex);
         let added = false;
@@ -253,13 +339,16 @@ export class BlockInputSync {
                 peerIdStr: peerIdStr ?? "unknown peer",
             });
         }
-        if (peerIdStr) {
+        // bound peers tracked per root: propagationSource is recorded pre-validation, so cap the set (#9923).
+        // Once at the cap, ignore further peers rather than evicting already-tracked ones.
+        if (peerIdStr && pendingBlock.peerIdStrings.size < MAX_PEERS_PER_ROOT) {
             pendingBlock.peerIdStrings.add(peerIdStr);
         }
         // TODO: check this prune methodology
         // Limit pending blocks to prevent DOS attacks that cause OOM
         const prunedItemCount = pruneSetToMax(this.pendingBlocks, this.maxPendingBlocks);
         if (prunedItemCount > 0) {
+            this.metrics?.blockInputSync.removedBlocks.inc({ reason: DroppedItemReason.capacity }, prunedItemCount);
             this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingBlocks`);
         }
         return added;
@@ -283,13 +372,16 @@ export class BlockInputSync {
                 delaySec: this.chain.clock.secFromSlot(blockInput.slot),
             });
         }
-        if (peerIdStr) {
+        // bound peers tracked per root: propagationSource is recorded pre-validation, so cap the set (#9923).
+        // Once at the cap, ignore further peers rather than evicting already-tracked ones.
+        if (peerIdStr && pendingBlock.peerIdStrings.size < MAX_PEERS_PER_ROOT) {
             pendingBlock.peerIdStrings.add(peerIdStr);
         }
         // TODO: check this prune methodology
         // Limit pending blocks to prevent DOS attacks that cause OOM
         const prunedItemCount = pruneSetToMax(this.pendingBlocks, this.maxPendingBlocks);
         if (prunedItemCount > 0) {
+            this.metrics?.blockInputSync.removedBlocks.inc({ reason: DroppedItemReason.capacity }, prunedItemCount);
             this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingBlocks`);
         }
     };
@@ -313,18 +405,47 @@ export class BlockInputSync {
                 peerIdStr: peerIdStr ?? "unknown peer",
             });
         }
-        if (peerIdStr) {
+        else if (
+        // update a previously slot-less entry once a slot becomes available
+        slot !== undefined &&
+            !isPendingPayloadInput(pendingPayload) &&
+            !isPendingPayloadEnvelope(pendingPayload) &&
+            pendingPayload.slot === undefined) {
+            pendingPayload.slot = slot;
+        }
+        if (peerIdStr && pendingPayload.peerIdStrings.size < MAX_PEERS_PER_ROOT) {
             pendingPayload.peerIdStrings.add(peerIdStr);
         }
         const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
         if (prunedItemCount > 0) {
+            this.metrics?.blockInputSync.removedPayloads.inc({ reason: DroppedItemReason.capacity }, prunedItemCount);
             this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
         }
         return added;
     };
+    /**
+     * Resolve the block slot for a payload root.
+     */
+    resolvePayloadSlot(rootHex) {
+        // a pending block of the same root
+        const pendingBlock = this.pendingBlocks.get(rootHex);
+        if (pendingBlock) {
+            const pendingBlockSlot = getBlockInputSyncCacheItemSlot(pendingBlock);
+            if (typeof pendingBlockSlot === "number") {
+                return pendingBlockSlot;
+            }
+        }
+        // the block is in fork choice
+        const forkChoiceSlot = this.chain.forkChoice.getBlockHexDefaultStatus(rootHex)?.slot;
+        if (forkChoiceSlot !== undefined) {
+            return forkChoiceSlot;
+        }
+        // a seen payload envelope input
+        return this.chain.seenPayloadEnvelopeInputCache.get(rootHex)?.slot;
+    }
     addByPayloadInput = (payloadInput, peerIdStr, envelope) => {
         const pendingPayload = this.toPendingPayloadInput(payloadInput, this.pendingPayloads.get(payloadInput.blockRootHex), envelope);
-        if (peerIdStr) {
+        if (peerIdStr && pendingPayload.peerIdStrings.size < MAX_PEERS_PER_ROOT) {
             pendingPayload.peerIdStrings.add(peerIdStr);
         }
         this.pendingPayloads.set(payloadInput.blockRootHex, pendingPayload);
@@ -335,6 +456,7 @@ export class BlockInputSync {
         });
         const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
         if (prunedItemCount > 0) {
+            this.metrics?.blockInputSync.removedPayloads.inc({ reason: DroppedItemReason.capacity }, prunedItemCount);
             this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
         }
     };
@@ -360,7 +482,8 @@ export class BlockInputSync {
      */
     getMissingBlockDependency(blockInput) {
         const parentRootHex = blockInput.parentRootHex;
-        if (!this.chain.forkChoice.hasBlockHex(parentRootHex)) {
+        const parentBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentRootHex);
+        if (!parentBlock) {
             return { kind: "parentBlock", rootHex: parentRootHex };
         }
         if (!blockInput.hasBlock()) {
@@ -377,14 +500,8 @@ export class BlockInputSync {
         if (this.chain.forkChoice.hasPayloadHexUnsafe(parentRootHex)) {
             return { kind: "invalidParentPayload", parentRootHex, parentBlockHashHex };
         }
-        const parentPayloadInput = this.chain.seenPayloadEnvelopeInputCache.get(parentRootHex);
-        if (parentPayloadInput) {
-            if (parentPayloadInput.getBlockHashHex() === parentBlockHashHex) {
-                return { kind: "parentPayload", rootHex: parentRootHex };
-            }
-            return { kind: "invalidParentPayload", parentRootHex, parentBlockHashHex };
-        }
-        return { kind: "parentPayload", rootHex: parentRootHex };
+        // Parent is in fork choice but its payload `parentBlockHashHex` isn't revealed yet
+        return { kind: "parentPayload", rootHex: parentRootHex, slot: parentBlock.slot };
     }
     advancePendingBlock(pendingBlock) {
         const missingDependency = this.getMissingBlockDependency(pendingBlock.blockInput);
@@ -402,9 +519,9 @@ export class BlockInputSync {
                 return added ? "queued_parent_block" : "blocked";
             }
             case "parentPayload": {
-                let added = this.addByPayloadRootHex(missingDependency.rootHex);
+                let added = this.addByPayloadRootHex(missingDependency.rootHex, undefined, missingDependency.slot);
                 for (const peerIdStr of pendingBlock.peerIdStrings) {
-                    added = this.addByPayloadRootHex(missingDependency.rootHex, peerIdStr) || added;
+                    added = this.addByPayloadRootHex(missingDependency.rootHex, peerIdStr, missingDependency.slot) || added;
                 }
                 return added ? "queued_parent_payload" : "blocked";
             }
@@ -415,7 +532,7 @@ export class BlockInputSync {
                     parentRoot: missingDependency.parentRootHex,
                     parentBlockHash: missingDependency.parentBlockHashHex,
                 });
-                this.removeAndDownScoreAllDescendants(pendingBlock);
+                this.removeAndDownScoreAllDescendants(pendingBlock, DroppedItemReason.invalidParentPayload);
                 return "removed";
         }
     }
@@ -604,7 +721,7 @@ export class BlockInputSync {
                     ...logCtx2,
                     finalizedSlot,
                 });
-                this.removeAndDownScoreAllDescendants(block);
+                this.removeAndDownScoreAllDescendants(block, DroppedItemReason.belowFinalized);
             }
             else {
                 this.onUnknownBlockRoot({ rootHex: pending.blockInput.parentRootHex, source: BlockInputSource.byRoot });
@@ -622,7 +739,7 @@ export class BlockInputSync {
             }
             this.metrics?.blockInputSync.downloadedBlocksError.inc();
             this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
-            this.removeAndDownScoreAllDescendants(block);
+            this.removeAndDownScoreAllDescendants(block, DroppedItemReason.unavailable);
         }
     }
     /**
@@ -679,9 +796,9 @@ export class BlockInputSync {
             // no need to update status to "processed", delete anyway
             this.pendingBlocks.delete(blockRootHex);
             if (isForkPostGloas(fork)) {
-                const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
+                const payloadInput = await this.chain.seenPayloadEnvelopeInputCache.getOrReload(blockRootHex);
                 if (!payloadInput) {
-                    this.logger.warn("PayloadEnvelopeInput not seeded during unknown sync processReadyBlock()", logCtx);
+                    this.logger.debug("PayloadEnvelopeInput unavailable during unknown sync processReadyBlock()", logCtx);
                 }
                 else {
                     // Similar to the gossip path: immediately attempt fetch of data columns from the execution engine.
@@ -717,17 +834,17 @@ export class BlockInputSync {
                     case BlockErrorCode.EXECUTION_ENGINE_ERROR:
                         // Removing the block(s) without penalizing the peers, hoping for EL to
                         // recover on a latter download + verify attempt.
-                        this.removeAllDescendants(pendingBlock);
+                        this.removeAllDescendants(pendingBlock, DroppedItemReason.elError, DroppedItemReason.elError);
                         break;
                     case BlockErrorCode.EXECUTION_ENGINE_INVALID:
                         // the peer served a bad block
                         this.logger.debug("Execution engine rejected block from unknown parent sync", errorData, res.err);
-                        this.removeAndDownScoreAllDescendants(pendingBlock);
+                        this.removeAndDownScoreAllDescendants(pendingBlock, DroppedItemReason.elInvalid);
                         break;
                     default:
                         // Block is not correct with respect to our chain. Log error loudly
                         this.logger.debug("Error processing block from unknown parent sync", errorData, res.err);
-                        this.removeAndDownScoreAllDescendants(pendingBlock);
+                        this.removeAndDownScoreAllDescendants(pendingBlock, DroppedItemReason.invalidBlock);
                 }
             }
             // Probably a queue error or something unwanted happened, mark as pending to try again latter
@@ -748,7 +865,6 @@ export class BlockInputSync {
             this.pendingPayloads.delete(rootHex);
             return;
         }
-        const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
         if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
             // Block not in fork choice yet. payloadInput may be seeded from the block body during download, so a
             // non-null payloadInput does not imply the block is imported; defer regardless and pull the block first.
@@ -762,8 +878,9 @@ export class BlockInputSync {
             }
             return;
         }
+        const payloadInput = await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
         if (!payloadInput) {
-            this.logger.debug("Missing PayloadEnvelopeInput for known block while reconciling payload envelope", {
+            this.logger.debug("PayloadEnvelopeInput not yet reloadable for imported block, will retry", {
                 root: rootHex,
             });
             return;
@@ -775,6 +892,7 @@ export class BlockInputSync {
                 const pendingPayloadByRoot = {
                     status: PendingPayloadInputStatus.pending,
                     rootHex,
+                    slot: pendingPayload.envelope.message.payload.slotNumber,
                     timeAddedSec: pendingPayload.timeAddedSec,
                     peerIdStrings: new Set(pendingPayload.peerIdStrings),
                 };
@@ -806,6 +924,9 @@ export class BlockInputSync {
         if (payload.status !== PendingPayloadInputStatus.pending) {
             return;
         }
+        if (!isPendingPayloadInput(payload) && payload.slot === undefined) {
+            payload.slot = this.resolvePayloadSlot(rootHex);
+        }
         const payloadSlot = getPayloadSyncCacheItemSlot(payload);
         const logCtx = {
             slot: payloadSlot,
@@ -818,7 +939,13 @@ export class BlockInputSync {
         const res = await wrapError(this.fetchPayloadInput(payload));
         if (!res.err) {
             const pendingPayload = res.result;
-            this.pendingPayloads.set(getPayloadSyncCacheItemRootHex(pendingPayload), pendingPayload);
+            const resultRootHex = getPayloadSyncCacheItemRootHex(pendingPayload);
+            // finalization may have deleted this entry while we were awaiting the network, do not resurrect it via the set below
+            if (!this.pendingPayloads.has(resultRootHex)) {
+                this.logger.verbose("Dropping downloaded payload, entry pruned during fetch", logCtx);
+                return;
+            }
+            this.pendingPayloads.set(resultRootHex, pendingPayload);
             if (isPendingPayloadEnvelope(pendingPayload)) {
                 await this.reconcilePayloadEnvelope(pendingPayload);
             }
@@ -885,16 +1012,24 @@ export class BlockInputSync {
                     pendingPayload.status = PendingPayloadInputStatus.downloaded;
                     break;
                 case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
+                    this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+                    this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.elInvalid);
+                    break;
                 case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
+                    this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+                    this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.invalidEnvelope);
+                    break;
                 case PayloadErrorCode.INVALID_SIGNATURE:
                     // TODO GLOAS: Decide how invalid payload inputs should eventually leave memory without
                     // reintroducing envelope replacement / recreation flows.
                     this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
-                    this.removePendingPayloadAndDescendants(rootHex);
+                    this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.invalidSignature);
                     break;
                 default:
                     this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
-                    this.pendingPayloads.delete(rootHex);
+                    if (this.pendingPayloads.delete(rootHex)) {
+                        this.metrics?.blockInputSync.removedPayloads.inc({ reason: DroppedItemReason.unknown }, 1);
+                    }
             }
             return;
         }
@@ -913,7 +1048,7 @@ export class BlockInputSync {
         let slot = getPayloadSyncCacheItemSlot(cacheItem);
         let payloadInput = isPendingPayloadInput(cacheItem)
             ? cacheItem.payloadInput
-            : this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
+            : await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
         let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
         let i = 0;
         let deferredByRateLimit = false;
@@ -921,12 +1056,16 @@ export class BlockInputSync {
             const pendingColumns = payloadInput?.hasAllData()
                 ? new Set()
                 : new Set(payloadInput?.getMissingSampledColumnMeta().missing ?? []);
-            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
+            // prefer peers that gossiped this payload root to us (#9923)
+            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers, cacheItem.peerIdStrings);
             if (peerMeta === null) {
                 if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
                     throw new UnknownBlockRateLimitedError(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`);
                 }
-                throw Error(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`);
+                const reason = pendingColumns.size > 0
+                    ? `cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
+                    : "no peer available to download pending payload";
+                throw Error(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: ${reason}`);
             }
             const { peerId, client: peerClient } = peerMeta;
             cacheItem.peerIdStrings.add(peerId);
@@ -935,7 +1074,7 @@ export class BlockInputSync {
                     envelope = await this.fetchExecutionPayloadEnvelope(peerId, blockRoot, rootHex);
                     slot = envelope.message.payload.slotNumber;
                 }
-                payloadInput ??= this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
+                payloadInput ??= await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
                 if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
                     // Block not in fork choice yet. Validating now would throw BLOCK_ROOT_UNKNOWN, so keep the downloaded
                     // envelope and wait for the block body; reconcilePayloadEnvelope validates once the block lands.
@@ -1080,7 +1219,8 @@ export class BlockInputSync {
             const pendingColumns = isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput)
                 ? new Set(cacheItem.blockInput.getMissingSampledColumnMeta().missing)
                 : defaultPendingColumns;
-            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
+            // prefer peers that gossiped this block root to us (#9923)
+            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers, cacheItem.peerIdStrings);
             if (peerMeta === null) {
                 if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
                     throw new UnknownBlockRateLimitedError(`Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`);
@@ -1206,9 +1346,9 @@ export class BlockInputSync {
      * Downscore all peers that have referenced any of this bad blocks. May report peers multiple times if they have
      * referenced more than one bad block.
      */
-    removeAndDownScoreAllDescendants(block) {
+    removeAndDownScoreAllDescendants(block, headReason) {
         // Get all blocks that are a descendant of this one
-        const badPendingBlocks = this.removeAllDescendants(block);
+        const badPendingBlocks = this.removeAllDescendants(block, headReason, DroppedItemReason.invalidParent);
         // just console log and do not penalize on pending/bad blocks for debugging
         // console.log("removeAndDownscoreAllDescendants", {block});
         for (const block of badPendingBlocks) {
@@ -1232,16 +1372,21 @@ export class BlockInputSync {
         pruneSetToMax(this.knownBadBlocks, MAX_KNOWN_BAD_BLOCKS);
     }
     // Once a parent payload is invalid, every descendant waiting on that payload lineage becomes unrecoverable too.
-    removePendingPayloadAndDescendants(rootHex) {
+    removePendingPayloadAndDescendants(rootHex, headReason) {
         // Keep PayloadEnvelopeInput resident in the seen cache. importBlock() owns that object and
         // later validation/finalization logic decides when it can leave memory.
-        this.pendingPayloads.delete(rootHex);
+        if (this.pendingPayloads.delete(rootHex)) {
+            this.metrics?.blockInputSync.removedPayloads.inc({ reason: headReason }, 1);
+        }
         const badPendingBlocks = getAllDescendantBlocks(rootHex, this.pendingBlocks);
-        this.metrics?.blockInputSync.removedBlocks.inc(badPendingBlocks.length);
         for (const block of badPendingBlocks) {
             const descendantRootHex = getBlockInputSyncCacheItemRootHex(block);
-            this.pendingBlocks.delete(descendantRootHex);
-            this.pendingPayloads.delete(descendantRootHex);
+            if (this.pendingBlocks.delete(descendantRootHex)) {
+                this.metrics?.blockInputSync.removedBlocks.inc({ reason: DroppedItemReason.invalidParent }, 1);
+            }
+            if (this.pendingPayloads.delete(descendantRootHex)) {
+                this.metrics?.blockInputSync.removedPayloads.inc({ reason: DroppedItemReason.invalidParent }, 1);
+            }
             this.chain.seenBlockInputCache.prune(descendantRootHex);
             this.logger.debug("Removing pending descendant after invalid parent payload", {
                 slot: getBlockInputSyncCacheItemSlot(block),
@@ -1250,16 +1395,20 @@ export class BlockInputSync {
             });
         }
     }
-    removeAllDescendants(block) {
+    removeAllDescendants(block, headReason, descendantReason = DroppedItemReason.invalidParent) {
         const rootHex = getBlockInputSyncCacheItemRootHex(block);
         const slot = getBlockInputSyncCacheItemSlot(block);
-        // Get all blocks that are a descendant of this one
+        // Get all blocks that are a descendant of this one (index 0 is the head item itself)
         const badPendingBlocks = [block, ...getAllDescendantBlocks(rootHex, this.pendingBlocks)];
-        this.metrics?.blockInputSync.removedBlocks.inc(badPendingBlocks.length);
-        for (const block of badPendingBlocks) {
+        for (const [i, block] of badPendingBlocks.entries()) {
             const rootHex = getBlockInputSyncCacheItemRootHex(block);
-            this.pendingBlocks.delete(rootHex);
-            this.pendingPayloads.delete(rootHex);
+            const reason = i === 0 ? headReason : descendantReason;
+            if (this.pendingBlocks.delete(rootHex)) {
+                this.metrics?.blockInputSync.removedBlocks.inc({ reason }, 1);
+            }
+            if (this.pendingPayloads.delete(rootHex)) {
+                this.metrics?.blockInputSync.removedPayloads.inc({ reason }, 1);
+            }
             this.chain.seenBlockInputCache.prune(rootHex);
             // Keep PayloadEnvelopeInput resident in the seen cache for consistency with the
             // importBlock()-owned lifecycle.
@@ -1303,6 +1452,12 @@ export class UnknownBlockPeerBalancer {
         this.activeRequests.delete(peerId);
         this.rateLimitedUntilByPeer.delete(peerId);
     }
+    /** Reset all tracked peer state, e.g. when BlockInputSync unsubscribes from the network. */
+    clear() {
+        this.peersMeta.clear();
+        this.activeRequests.clear();
+        this.rateLimitedUntilByPeer.clear();
+    }
     onRateLimited(peerId, rateLimitedUntilMs) {
         this.rateLimitedUntilByPeer.set(peerId, rateLimitedUntilMs);
     }
@@ -1330,17 +1485,21 @@ export class UnknownBlockPeerBalancer {
         return retryAt;
     }
     /**
-     * called from fetchBlockInput() where we only have block root and nothing else
+     * called from fetchBlockInput()/fetchPayloadInput() with a block/payload root.
      * excludedPeers are the peers that we requested already so we don't want to try again
      * pendingColumns is empty for prefulu, or the 1st time we we download a block by root
+     * preferredPeers are the peers NetworkProcessor told us gossiped this root (from
+     *   pendingBlock/cacheItem.peerIdStrings). They are preferred but not required.
      */
-    bestPeerForPendingColumns(pendingColumns, excludedPeers) {
+    bestPeerForPendingColumns(pendingColumns, excludedPeers, preferredPeers) {
         const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers);
         if (eligiblePeers.length === 0) {
             return null;
         }
         const sortedEligiblePeers = sortBy(shuffle(eligiblePeers), 
-        // prefer peers with least active req
+        // 1st key: prefer peers we heard about this root from
+        (peerId) => (preferredPeers?.has(peerId) ? 0 : 1), 
+        // 2nd key (tiebreak): prefer peers with least active req
         (peerId) => this.activeRequests.get(peerId) ?? 0);
         const bestPeerId = sortedEligiblePeers[0];
         this.onRequest(bestPeerId);