@lodestar/beacon-node 1.50.0-dev.ebecebb9ba → 1.50.0-dev.f0a5c36612

package/lib/sync/unknownBlock.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"unknownBlock.d.ts","sourceRoot":"","sources":["../../src/sync/unknownBlock.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,MAAM,EAA+E,MAAM,iBAAiB,CAAC;AAMrH,OAAO,EAA6B,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAG3E,OAAO,EAAC,OAAO,EAAC,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAC,QAAQ,EAAuD,MAAM,qBAAqB,CAAC;AACnG,OAAO,EAAC,YAAY,EAAC,MAAM,+BAA+B,CAAC;AAG3D,OAAO,EAAC,SAAS,EAAC,MAAM,mBAAmB,CAAC;AAK5C,OAAO,EAAC,WAAW,EAAC,MAAM,cAAc,CAAC;AAmCzC;;GAEG;AACH,eAAO,MAAM,wBAAwB,OAAQ,CAAC;AAiB9C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,cAAc;IAmBvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;IAvBxB;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+C;IAE7E,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA4C;IAC5E,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAClC,OAAO,CAAC,yBAAyB,CAAS;IAC1C,OAAO,CAAC,YAAY,CAA2B;IAC/C,OAAO,CAAC,uBAAuB,CAA6B;IAC5D;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAgF;IACrH,OAAO,CAAC,0BAA0B,CAA8B;IAEhE,YACmB,MAAM,EAAE,eAAe,EACvB,OAAO,EAAE,QAAQ,EACjB,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,IAAI,CAAC,EAAE,WAAW,YAAA,EAgBpC;IAED,kBAAkB,IAAI,IAAI,CAoCzB;IAED,sBAAsB,IAAI,IAAI,CAoB7B;IAED,KAAK,IAAI,IAAI,CAEZ;IAED,qBAAqB,IAAI,OAAO,CAE/B;IAED;;OAEG;IACH,OAAO,CAAC,kBAAkB,CAcxB;IAEF;;OAEG;IACH,OAAO,CAAC,sBAAsB,CAS5B;IAEF,OAAO,CAAC,0BAA0B,CAWhC;IAEF,OAAO,CAAC,8BAA8B,CAyBpC;IAEF,OAAO,CAAC,2BAA2B,CASjC;IAEF;;OAEG;IACH,OAAO,CAAC,eAAe,CA+BrB;IAEF,OAAO,CAAC,eAAe,CAIrB;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF;;;OAGG;IACH,OAAO,CAAC,cAAc,CAiFpB;IAEF,OAAO,CAAC,YAAY,CAiClB;IAEF,OAAO,CAAC,eAAe,CAkCrB;IAEF,OAAO,CAAC,mBAAmB,CAuCzB;IAEF;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAoB1B,OAAO,CAAC,iBAAiB,CA4BvB;IAEF,OAAO,CAAC,eAAe,CASrB;IAEF,OAAO,CAAC,kBAAkB,CAIxB;IAEF;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAmCjC,OAAO,CAAC,mBAAmB;IAuC3B,OAAO,CAAC,qBAAqB;IA0B7B;;OAEG;IACH,OAAO,CAAC,yBAAyB,CAwF/B;IAEF,OAAO,CAAC,6BAA6B;IAuBrC,OAAO,CAAC,0BAA0B;IAOlC;;;OAGG;IACH,OAAO,CAAC,sBAAsB,CAqB5B;IAEF;;OAEG;YACW,kBAAkB;IA8EhC,OAAO,CAAC,sBAAsB;YAQhB,aAAa;IAkG3B;;;OAGG;YACW,iBAAiB;IA4H/B;;;;OAIG;YACW,wBAAwB;YAqExB,eAAe;YAmEf,cAAc;IA0F5B;;;;OAIG;YACW,iBAAiB;YAkLjB,6BAA6B;YAmB7B,mBAAmB;IAwCjC;;;;;;;;OAQG;YACW,eAAe;IAyK7B;;;;;OAKG;IACH,OAAO,CAAC,gCAAgC;IA6BxC,OAAO,CAAC,kCAAkC;IAyB1C,OAAO,CAAC,oBAAoB;IA+B5B,OAAO,CAAC,sBAAsB;CAY/B;AAED;;GAEG;AACH,qBAAa,wBAAwB;IACnC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IACjD,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAChD,QAAQ,CAAC,sBAAsB,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAExD,cAIC;IAED,qCAAqC;IACrC,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,GAAG,IAAI,CAM/D;IAED,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAI1C;IAED,4FAA4F;IAC5F,KAAK,IAAI,IAAI,CAIZ;IAED,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAEjE;IAED,uBAAuB,CAAC,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,GAAG,MAAM,GAAG,IAAI,CA4BnG;IAED;;;;;;OAMG;IACH,yBAAyB,CACvB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,aAAa,EAAE,GAAG,CAAC,SAAS,CAAC,EAC7B,cAAc,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,GAC9B,YAAY,GAAG,IAAI,CAiBrB;IAED;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAEjC;IAED;;OAEG;IACH,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAE1C;IAED,sBAAsB,IAAI,MAAM,CAM/B;IAED,OAAO,CAAC,WAAW;IA8DnB,OAAO,CAAC,qBAAqB;CAO9B"}
+{"version":3,"file":"unknownBlock.d.ts","sourceRoot":"","sources":["../../src/sync/unknownBlock.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,MAAM,EAA+E,MAAM,iBAAiB,CAAC;AAMrH,OAAO,EAA6B,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAG3E,OAAO,EAAC,OAAO,EAAC,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAC,QAAQ,EAAmE,MAAM,qBAAqB,CAAC;AAC/G,OAAO,EAAC,YAAY,EAAC,MAAM,+BAA+B,CAAC;AAG3D,OAAO,EAAC,SAAS,EAAC,MAAM,mBAAmB,CAAC;AAK5C,OAAO,EAAC,WAAW,EAAC,MAAM,cAAc,CAAC;AAmCzC;;GAEG;AACH,eAAO,MAAM,wBAAwB,OAAQ,CAAC;AAqB9C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,cAAc;IAmBvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;IAvBxB;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+C;IAE7E,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA4C;IAC5E,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAClC,OAAO,CAAC,yBAAyB,CAAS;IAC1C,OAAO,CAAC,YAAY,CAA2B;IAC/C,OAAO,CAAC,uBAAuB,CAA6B;IAC5D;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAgF;IACrH,OAAO,CAAC,0BAA0B,CAA8B;IAEhE,YACmB,MAAM,EAAE,eAAe,EACvB,OAAO,EAAE,QAAQ,EACjB,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,IAAI,CAAC,EAAE,WAAW,YAAA,EAgBpC;IAED,kBAAkB,IAAI,IAAI,CAoCzB;IAED,sBAAsB,IAAI,IAAI,CAoB7B;IAED,KAAK,IAAI,IAAI,CAEZ;IAED,qBAAqB,IAAI,OAAO,CAE/B;IAED;;OAEG;IACH,OAAO,CAAC,kBAAkB,CAcxB;IAEF;;OAEG;IACH,OAAO,CAAC,sBAAsB,CAS5B;IAEF,OAAO,CAAC,0BAA0B,CAWhC;IAEF,OAAO,CAAC,8BAA8B,CAyBpC;IAEF,OAAO,CAAC,2BAA2B,CASjC;IAEF;;OAEG;IACH,OAAO,CAAC,eAAe,CA+BrB;IAEF,OAAO,CAAC,eAAe,CAIrB;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF;;;OAGG;IACH,OAAO,CAAC,cAAc,CAiFpB;IAEF,OAAO,CAAC,YAAY,CAsClB;IAEF,OAAO,CAAC,eAAe,CAuCrB;IAEF,OAAO,CAAC,mBAAmB,CAgDzB;IAEF;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAoB1B,OAAO,CAAC,iBAAiB,CAqCvB;IAEF,OAAO,CAAC,eAAe,CASrB;IAEF,OAAO,CAAC,kBAAkB,CAIxB;IAEF;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAoCjC,OAAO,CAAC,mBAAmB;IAuC3B,OAAO,CAAC,qBAAqB;IA0B7B;;OAEG;IACH,OAAO,CAAC,yBAAyB,CAwF/B;IAEF;;OAEG;IACH,OAAO,CAAC,6BAA6B;IAwBrC,OAAO,CAAC,0BAA0B;IAOlC;;;OAGG;IACH,OAAO,CAAC,sBAAsB,CAqB5B;IAEF;;OAEG;YACW,kBAAkB;IA8EhC,OAAO,CAAC,sBAAsB;YAQhB,aAAa;IAuG3B;;;OAGG;YACW,iBAAiB;IA+H/B;;;;OAIG;YACW,wBAAwB;YAqExB,eAAe;YA2Ef,cAAc;IA6F5B;;;;OAIG;YACW,iBAAiB;YA2LjB,6BAA6B;YAmB7B,mBAAmB;IAwCjC;;;;;;;;OAQG;YACW,eAAe;IA+K7B,OAAO,CAAC,kCAAkC;IAuB1C;;;;OAIG;IACH,OAAO,CAAC,gCAAgC;IAaxC;;;OAGG;IACH,OAAO,CAAC,0BAA0B;IA2BlC,OAAO,CAAC,oBAAoB;IA8B5B,OAAO,CAAC,sBAAsB;CAY/B;AAED;;GAEG;AACH,qBAAa,wBAAwB;IACnC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IACjD,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAChD,QAAQ,CAAC,sBAAsB,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAExD,cAIC;IAED,qCAAqC;IACrC,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,GAAG,IAAI,CAM/D;IAED,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAI1C;IAED,4FAA4F;IAC5F,KAAK,IAAI,IAAI,CAIZ;IAED,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAEjE;IAED,uBAAuB,CACrB,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAC5B,aAAa,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,EAC9B,GAAG,SAAa,GACf,MAAM,GAAG,IAAI,CA2Bf;IAED;;;;;;OAMG;IACH,yBAAyB,CACvB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,aAAa,EAAE,GAAG,CAAC,SAAS,CAAC,EAC7B,cAAc,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,EAC/B,GAAG,SAAa,GACf,YAAY,GAAG,IAAI,CAiBrB;IAED;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAEjC;IAED;;OAEG;IACH,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAE1C;IAED,sBAAsB,IAAI,MAAM,CAM/B;IAED,OAAO,CAAC,WAAW;IA6DnB,OAAO,CAAC,qBAAqB;CAO9B"}

package/lib/sync/unknownBlock.js

@@ -11,7 +11,7 @@ import { BlockError, BlockErrorCode } from "../chain/errors/index.js";
 import { ChainEvent } from "../chain/index.js";
 import { validateGloasBlockDataColumnSidecars } from "../chain/validation/dataColumnSidecar.js";
 import { validateGossipExecutionPayloadEnvelope } from "../chain/validation/executionPayloadEnvelope.js";
-import { NetworkEvent, prettyPrintPeerIdStr } from "../network/index.js";
+import { NetworkEvent, PeerAction, prettyPrintPeerIdStr } from "../network/index.js";
 import { MAX_PEERS_PER_ROOT } from "../network/processor/constants.js";
 import { ClockEvent } from "../util/clock.js";
 import { shuffle } from "../util/shuffle.js";
@@ -34,8 +34,12 @@ const PRUNE_UNRESOLVED_SLOT_EPOCHS = 2;
  */
 export const PAYLOAD_POLL_INTERVAL_MS = 1_000;
 class UnknownBlockRateLimitedError extends Error {
-    constructor(message) {
+    retryAt;
+    constructor(message, 
+    /** Earliest backoff expiry seen when the download was deferred, it is retried once that has passed */
+    retryAt) {
         super(message);
+        this.retryAt = retryAt;
         this.name = "UnknownBlockRateLimitedError";
     }
 }
@@ -378,6 +382,10 @@ export class BlockInputSync {
         });
     };
     addByRootHex = (rootHex, peerIdStr) => {
+        if (this.knownBadBlocks.has(rootHex)) {
+            this.logger.debug("Ignoring known bad block root", { root: rootHex, peerIdStr: peerIdStr ?? "unknown peer" });
+            return false;
+        }
         let pendingBlock = this.pendingBlocks.get(rootHex);
         let added = false;
         if (!pendingBlock) {
@@ -409,6 +417,10 @@ export class BlockInputSync {
         return added;
     };
     addByBlockInput = (blockInput, peerIdStr) => {
+        if (this.knownBadBlocks.has(blockInput.blockRootHex)) {
+            this.logger.debug("Ignoring known bad block", { slot: blockInput.slot, root: blockInput.blockRootHex, peerIdStr });
+            return;
+        }
         let pendingBlock = this.pendingBlocks.get(blockInput.blockRootHex);
         // if entry is missing or was added via rootHex and now we have more complete information overwrite
         // the existing information with the more complete cache entry
@@ -441,6 +453,14 @@ export class BlockInputSync {
         }
     };
     addByPayloadRootHex = (rootHex, peerIdStr, slot) => {
+        if (this.knownBadBlocks.has(rootHex)) {
+            this.logger.debug("Ignoring payload for known bad block root", {
+                slot: slot ?? "unknown",
+                root: rootHex,
+                peerIdStr,
+            });
+            return false;
+        }
         let pendingPayload = this.pendingPayloads.get(rootHex);
         let added = false;
         if (!pendingPayload) {
@@ -499,6 +519,14 @@ export class BlockInputSync {
         return this.chain.seenPayloadEnvelopeInputCache.get(rootHex)?.slot;
     }
     addByPayloadInput = (payloadInput, peerIdStr, envelope) => {
+        if (this.knownBadBlocks.has(payloadInput.blockRootHex)) {
+            this.logger.debug("Ignoring payload input for known bad block root", {
+                slot: payloadInput.slot,
+                root: payloadInput.blockRootHex,
+                peerIdStr,
+            });
+            return;
+        }
         const pendingPayload = this.toPendingPayloadInput(payloadInput, this.pendingPayloads.get(payloadInput.blockRootHex), envelope);
         if (peerIdStr && pendingPayload.peerIdStrings.size < MAX_PEERS_PER_ROOT) {
             pendingPayload.peerIdStrings.add(peerIdStr);
@@ -552,6 +580,7 @@ export class BlockInputSync {
         if (this.chain.forkChoice.getBlockHexAndBlockHash(parentRootHex, parentBlockHashHex) !== null) {
             return { kind: "ready" };
         }
+        // The parent's payload is known but does not match `parentBlockHashHex`
         if (this.chain.forkChoice.hasPayloadHexUnsafe(parentRootHex)) {
             return { kind: "invalidParentPayload", parentRootHex, parentBlockHashHex };
         }
@@ -693,21 +722,25 @@ export class BlockInputSync {
             this.triggerUnknownBlockSearch();
         }
     };
+    /**
+     * Returns false if there is no backoff left to wait for, in which case no retry is scheduled.
+     */
     scheduleRateLimitBackoffRetry() {
         this.clearRateLimitBackoffTimer();
         if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
-            return;
+            return false;
         }
         const now = Date.now();
         const retryAt = this.peerBalancer.getNextRateLimitRetryAt();
         if (retryAt === null) {
-            return;
+            return false;
         }
         this.rateLimitBackoffTimeout = setTimeout(() => {
             this.rateLimitBackoffTimeout = undefined;
             this.triggerUnknownBlockSearch();
             this.scheduleRateLimitBackoffRetry();
         }, Math.max(0, retryAt - now));
+        return true;
     }
     clearRateLimitBackoffTimer() {
         if (this.rateLimitBackoffTimeout !== undefined) {
@@ -882,7 +915,8 @@ export class BlockInputSync {
             }
             else if (blockSlot <= finalizedSlot) {
                 // the common ancestor of the downloading chain and canonical chain should be at least the finalized slot and
-                // we should found it through forkchoice. If not, we should penalize all peers sending us this block chain
+                // we should found it through forkchoice. If not, drop the chain without penalizing peers because pending entries are
+                // block retained on failed download, so finalization can pass the block while it is still pending
                 // 0 - 1 - ... - n - finalizedSlot
                 //                \
                 //                parent 1 - parent 2 - ... - unknownParent block
@@ -890,7 +924,7 @@ export class BlockInputSync {
                     ...logCtx2,
                     finalizedSlot,
                 });
-                this.removeAndDownScoreAllDescendants(block, DroppedItemReason.belowFinalized);
+                this.removeAllDescendants(block, DroppedItemReason.belowFinalized);
             }
             else {
                 this.onUnknownBlockRoot({ rootHex: pending.blockInput.parentRootHex, source: BlockInputSource.byRoot });
@@ -903,7 +937,11 @@ export class BlockInputSync {
                     pendingBlock.status = PendingBlockInputStatus.pending;
                 }
                 this.logger.debug("Deferring unknown block download due to peer rate limit", logCtx, res.err);
-                this.scheduleRateLimitBackoffRetry();
+                if (!this.scheduleRateLimitBackoffRetry() || res.err.retryAt <= Date.now()) {
+                    // The backoff this download waited for expired in the meantime, a timer is only armed if another
+                    // peer is still in backoff and would retry it later than needed
+                    this.triggerUnknownBlockSearch();
+                }
                 return;
             }
             this.metrics?.blockInputSync.downloadedBlocksError.inc();
@@ -1009,14 +1047,17 @@ export class BlockInputSync {
                         this.removeAllDescendants(pendingBlock, DroppedItemReason.elError, DroppedItemReason.elError);
                         break;
                     case BlockErrorCode.EXECUTION_ENGINE_INVALID:
-                        // the peer served a bad block
+                        // Ban but do not downscore: optimistic-sync peers may forward and serve blocks whose payload is later
+                        // rejected by the execution engine. Per the spec, an INVALID execution payload SHOULD NOT cause a peer
+                        // to be down-scored or disconnected, only consensus-layer invalidity MAY.
+                        // https://github.com/ethereum/consensus-specs/blob/master/specs/bellatrix/p2p-interface.md#the-reqresp-domain
                         this.logger.debug("Execution engine rejected block from unknown parent sync", errorData, res.err);
-                        this.removeAndDownScoreAllDescendants(pendingBlock, DroppedItemReason.elInvalid);
+                        this.removeAndBanAllDescendants(pendingBlock, DroppedItemReason.elInvalid);
                         break;
                     default:
                         // Block is not correct with respect to our chain. Log error loudly
                         this.logger.debug("Error processing block from unknown parent sync", errorData, res.err);
-                        this.removeAndDownScoreAllDescendants(pendingBlock, DroppedItemReason.invalidBlock);
+                        this.removeAllDescendants(pendingBlock, DroppedItemReason.invalidBlock);
                 }
             }
             // Probably a queue error or something unwanted happened, mark as pending to try again latter
@@ -1138,6 +1179,12 @@ export class BlockInputSync {
         if (!isPendingPayloadEnvelope(payload)) {
             payload.status = PendingPayloadInputStatus.pending;
         }
+        if (res.err instanceof UnknownBlockRateLimitedError &&
+            (!this.scheduleRateLimitBackoffRetry() || res.err.retryAt <= Date.now())) {
+            // The backoff this download waited for expired in the meantime, a timer is only armed if another
+            // peer is still in backoff and would retry it later than needed
+            this.triggerUnknownBlockSearch();
+        }
     }
     async processPayload(pendingPayload) {
         const rootHex = pendingPayload.payloadInput.blockRootHex;
@@ -1191,18 +1238,21 @@ export class BlockInputSync {
                     this.logger.debug("Execution engine error while processing payload from unknown sync", logCtx, res.err);
                     pendingPayload.status = PendingPayloadInputStatus.downloaded;
                     break;
+                // The invalid envelope is evicted from the seen cache in the cases below, otherwise the next
+                // attempt would reuse it instead of fetching another envelope from peers
                 case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
                     this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+                    this.chain.seenPayloadEnvelopeInputCache.removeInvalid(pendingPayload.payloadInput);
                     this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.elInvalid);
                     break;
                 case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
                     this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+                    this.chain.seenPayloadEnvelopeInputCache.removeInvalid(pendingPayload.payloadInput);
                     this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.invalidEnvelope);
                     break;
                 case PayloadErrorCode.INVALID_SIGNATURE:
-                    // TODO GLOAS: Decide how invalid payload inputs should eventually leave memory without
-                    // reintroducing envelope replacement / recreation flows.
                     this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+                    this.chain.seenPayloadEnvelopeInputCache.removeInvalid(pendingPayload.payloadInput);
                     this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.invalidSignature);
                     break;
                 default:
@@ -1240,11 +1290,14 @@ export class BlockInputSync {
             const pendingColumns = payloadInput?.hasAllData()
                 ? new Set()
                 : new Set(payloadInput?.getMissingSampledColumnMeta().missing ?? []);
+            // Same `now` for both calls, a backoff expiring in between would be reported as no peer available
+            const now = Date.now();
             // prefer peers that gossiped this payload root to us (#9923)
-            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers, cacheItem.peerIdStrings);
+            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers, cacheItem.peerIdStrings, now);
             if (peerMeta === null) {
-                if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
-                    throw new UnknownBlockRateLimitedError(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`);
+                const retryAt = this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers, now);
+                if (retryAt !== null) {
+                    throw new UnknownBlockRateLimitedError(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`, retryAt);
                 }
                 const reason = pendingColumns.size > 0
                     ? `cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
@@ -1254,10 +1307,10 @@ export class BlockInputSync {
                 throw Error(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: ${reason}`);
             }
             const { peerId, client: peerClient } = peerMeta;
-            cacheItem.peerIdStrings.add(peerId);
             try {
                 if (!envelope) {
                     envelope = await this.fetchExecutionPayloadEnvelope(peerId, blockRoot, rootHex);
+                    cacheItem.peerIdStrings.add(peerId);
                     const slotWasUnknown = typeof slot !== "number";
                     slot = envelope.message.payload.slotNumber;
                     if (slotWasUnknown) {
@@ -1292,6 +1345,8 @@ export class BlockInputSync {
                     const missing = pendingPayload.payloadInput.getMissingSampledColumnMeta().missing;
                     if (missing.length > 0) {
                         const columnSidecars = await this.fetchPayloadColumns(peerMeta, pendingPayload.payloadInput, missing);
+                        // toPendingPayloadInput copied cacheItem's set, so add to the object that is carried forward
+                        pendingPayload.peerIdStrings.add(peerId);
                         const seenTimestampSec = Date.now() / 1000;
                         for (const columnSidecar of columnSidecars) {
                             if (pendingPayload.payloadInput.hasColumn(columnSidecar.index)) {
@@ -1356,8 +1411,9 @@ export class BlockInputSync {
         }
         this.metrics?.blockInputSync.payloadFetchTime.observe({ result: FetchResult.FailureMaxAttempts }, Date.now() / 1000 - fetchStartSec);
         this.metrics?.blockInputSync.payloadFetchPeers.set({ result: FetchResult.FailureMaxAttempts }, i - 1);
-        if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
-            throw new UnknownBlockRateLimitedError(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`);
+        const retryAt = deferredByRateLimit ? this.peerBalancer.getNextRateLimitRetryAt() : null;
+        if (retryAt !== null) {
+            throw new UnknownBlockRateLimitedError(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`, retryAt);
         }
         throw Error(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts.`);
     }
@@ -1420,11 +1476,14 @@ export class BlockInputSync {
             const pendingColumns = isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput)
                 ? new Set(cacheItem.blockInput.getMissingSampledColumnMeta().missing)
                 : defaultPendingColumns;
+            // Same `now` for both calls, a backoff expiring in between would be reported as no peer available
+            const now = Date.now();
             // prefer peers that gossiped this block root to us (#9923)
-            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers, cacheItem.peerIdStrings);
+            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers, cacheItem.peerIdStrings, now);
             if (peerMeta === null) {
-                if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
-                    throw new UnknownBlockRateLimitedError(`Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`);
+                const retryAt = this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers, now);
+                if (retryAt !== null) {
+                    throw new UnknownBlockRateLimitedError(`Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`, retryAt);
                 }
                 // no more peer with needed columns to try, throw error
                 const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
@@ -1433,7 +1492,6 @@ export class BlockInputSync {
                 throw Error(message);
             }
             const { peerId, client: peerClient } = peerMeta;
-            cacheItem.peerIdStrings.add(peerId);
             try {
                 const downloadResult = await downloadByRoot({
                     config: this.config,
@@ -1444,6 +1502,7 @@ export class BlockInputSync {
                     cacheItem,
                 });
                 cacheItem = downloadResult.result;
+                cacheItem.peerIdStrings.add(peerId);
                 if (slot === undefined) {
                     slot = cacheItem.blockInput.slot;
                     // we were not able to observe the time into slot when starting the fetch, do it now
@@ -1520,8 +1579,9 @@ export class BlockInputSync {
             }
         } // end while loop over peers
         const message = `Error fetching BlockInput with slot=${slot} root=${rootHex} after ${i - 1} attempts.`;
-        if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
-            throw new UnknownBlockRateLimitedError(`${message} Peers are rate-limited.`);
+        const retryAt = deferredByRateLimit ? this.peerBalancer.getNextRateLimitRetryAt() : null;
+        if (retryAt !== null) {
+            throw new UnknownBlockRateLimitedError(`${message} Peers are rate-limited.`, retryAt);
         }
         if (!isPendingBlockInput(cacheItem)) {
             throw Error(`${message} No block and no data was found.`);
@@ -1545,41 +1605,8 @@ export class BlockInputSync {
         this.metrics?.blockInputSync.fetchPeers.set({ result: FetchResult.FailureMaxAttempts }, i - 1);
         throw Error(message);
     }
-    /**
-     * Gets all descendant blocks of `block` recursively from `pendingBlocks`.
-     * Assumes that if a parent block does not exist or is not processable, all descendant blocks are bad too.
-     * Downscore all peers that have referenced any of this bad blocks. May report peers multiple times if they have
-     * referenced more than one bad block.
-     */
-    removeAndDownScoreAllDescendants(block, headReason) {
-        // Get all blocks that are a descendant of this one
-        const badPendingBlocks = this.removeAllDescendants(block, headReason, DroppedItemReason.invalidParent);
-        // just console log and do not penalize on pending/bad blocks for debugging
-        // console.log("removeAndDownscoreAllDescendants", {block});
-        for (const block of badPendingBlocks) {
-            //
-            // TODO(fulu): why is this commented out here?
-            //
-            //   this.knownBadBlocks.add(block.blockRootHex);
-            //   for (const peerIdStr of block.peerIdStrings) {
-            //     // TODO: Refactor peerRpcScores to work with peerIdStr only
-            //     this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadBlockByRoot");
-            //   }
-            this.logger.debug("ignored Banning unknown block", {
-                slot: getBlockInputSyncCacheItemSlot(block),
-                root: getBlockInputSyncCacheItemRootHex(block),
-                peerIdStrings: Array.from(block.peerIdStrings)
-                    .map((id) => prettyPrintPeerIdStr(id))
-                    .join(","),
-            });
-        }
-        // Prune knownBadBlocks
-        pruneSetToMax(this.knownBadBlocks, MAX_KNOWN_BAD_BLOCKS);
-    }
     // Once a parent payload is invalid, every descendant waiting on that payload lineage becomes unrecoverable too.
     removePendingPayloadAndDescendants(rootHex, headReason) {
-        // Keep PayloadEnvelopeInput resident in the seen cache. importBlock() owns that object and
-        // later validation/finalization logic decides when it can leave memory.
         if (this.pendingPayloads.delete(rootHex)) {
             this.metrics?.blockInputSync.removedPayloads.inc({ reason: headReason }, 1);
         }
@@ -1600,9 +1627,50 @@ export class BlockInputSync {
             });
         }
     }
+    /**
+     * Same as `removeAndBanAllDescendants`, and downscores every peer that referenced or served a banned item, once per
+     * peer. Only call for consensus-layer invalidity, never for an INVALID execution payload:
+     * https://github.com/ethereum/consensus-specs/blob/master/specs/bellatrix/p2p-interface.md#the-reqresp-domain
+     */
+    removeAndDownScoreAllDescendants(block, headReason) {
+        const peersToReport = new Set();
+        for (const bannedBlock of this.removeAndBanAllDescendants(block, headReason)) {
+            for (const peerIdStr of bannedBlock.peerIdStrings) {
+                peersToReport.add(peerIdStr);
+            }
+        }
+        for (const peerIdStr of peersToReport) {
+            this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, headReason);
+        }
+    }
+    /**
+     * Removes `block` and all its pending descendants and bans their roots in `knownBadBlocks`. Only call when the chain
+     * is provably bad: a banned root is never searched for again. Returns the banned items.
+     */
+    removeAndBanAllDescendants(block, headReason) {
+        const bannedBlocks = [];
+        for (const badBlock of this.removeAllDescendants(block, headReason, DroppedItemReason.invalidParent)) {
+            const rootHex = getBlockInputSyncCacheItemRootHex(badBlock);
+            // a gossip import may race a failing sync path, never ban a root fork choice has imported
+            if (this.chain.forkChoice.hasBlockHex(rootHex)) {
+                continue;
+            }
+            this.knownBadBlocks.add(rootHex);
+            bannedBlocks.push(badBlock);
+            this.logger.debug("Banning unknown block", {
+                slot: getBlockInputSyncCacheItemSlot(badBlock),
+                root: rootHex,
+                reason: headReason,
+                peerIdStrings: Array.from(badBlock.peerIdStrings)
+                    .map((id) => prettyPrintPeerIdStr(id))
+                    .join(","),
+            });
+        }
+        pruneSetToMax(this.knownBadBlocks, MAX_KNOWN_BAD_BLOCKS);
+        return bannedBlocks;
+    }
     removeAllDescendants(block, headReason, descendantReason = DroppedItemReason.invalidParent) {
         const rootHex = getBlockInputSyncCacheItemRootHex(block);
-        const slot = getBlockInputSyncCacheItemSlot(block);
         // Get all blocks that are a descendant of this one (index 0 is the head item itself)
         const badPendingBlocks = [block, ...getAllDescendantBlocks(rootHex, this.pendingBlocks)];
         for (const [i, block] of badPendingBlocks.entries()) {
@@ -1618,8 +1686,8 @@ export class BlockInputSync {
             // Keep PayloadEnvelopeInput resident in the seen cache for consistency with the
             // importBlock()-owned lifecycle.
             this.logger.debug("Removing bad/unknown/incomplete BlockInputSyncCacheItem", {
-                slot,
-                blockRoot: rootHex,
+                slot: getBlockInputSyncCacheItemSlot(block),
+                root: rootHex,
             });
         }
         return badPendingBlocks;
@@ -1666,8 +1734,7 @@ export class UnknownBlockPeerBalancer {
     onRateLimited(peerId, rateLimitedUntilMs) {
         this.rateLimitedUntilByPeer.set(peerId, rateLimitedUntilMs);
     }
-    getNextRateLimitRetryAt(pendingColumns, excludedPeers) {
-        const now = Date.now();
+    getNextRateLimitRetryAt(pendingColumns, excludedPeers, now = Date.now()) {
         let retryAt = null;
         for (const [peerId, rateLimitedUntil] of this.rateLimitedUntilByPeer.entries()) {
             if (rateLimitedUntil <= now) {
@@ -1696,8 +1763,8 @@ export class UnknownBlockPeerBalancer {
      * preferredPeers are the peers NetworkProcessor told us gossiped this root (from
      *   pendingBlock/cacheItem.peerIdStrings). They are preferred but not required.
      */
-    bestPeerForPendingColumns(pendingColumns, excludedPeers, preferredPeers) {
-        const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers);
+    bestPeerForPendingColumns(pendingColumns, excludedPeers, preferredPeers, now = Date.now()) {
+        const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers, now);
         if (eligiblePeers.length === 0) {
             return null;
         }
@@ -1730,8 +1797,7 @@ export class UnknownBlockPeerBalancer {
         }
         return totalActiveRequests;
     }
-    filterPeers(pendingDataColumns, excludedPeers) {
-        const now = Date.now();
+    filterPeers(pendingDataColumns, excludedPeers, now) {
         let maxColumnCount = 0;
         const considerPeers = [];
         for (const [peerId, syncMeta] of this.peersMeta.entries()) {