@lodestar/beacon-node 1.42.0-rc.0 → 1.43.0-dev.05a33e512f

package/src/sync/unknownBlock.ts

@@ -1,13 +1,18 @@
+import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkSeq} from "@lodestar/params";
 import {RequestError, RequestErrorCode} from "@lodestar/reqresp";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
-import {RootHex} from "@lodestar/types";
-import {Logger, prettyPrintIndices, pruneSetToMax, sleep} from "@lodestar/utils";
+import {RootHex, gloas} from "@lodestar/types";
+import {Logger, fromHex, prettyPrintIndices, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../chain/blocks/blockInput/types.js";
+import {PayloadError, PayloadErrorCode} from "../chain/blocks/importExecutionPayload.js";
+import {PayloadEnvelopeInput, PayloadEnvelopeInputSource} from "../chain/blocks/payloadEnvelopeInput/index.js";
 import {BlockError, BlockErrorCode} from "../chain/errors/index.js";
 import {ChainEvent, ChainEventData, IBeaconChain} from "../chain/index.js";
+import {validateGloasBlockDataColumnSidecars} from "../chain/validation/dataColumnSidecar.js";
+import {validateGossipExecutionPayloadEnvelope} from "../chain/validation/executionPayloadEnvelope.js";
 import {Metrics} from "../metrics/index.js";
 import {INetwork, NetworkEvent, NetworkEventData, prettyPrintPeerIdStr} from "../network/index.js";
 import {PeerSyncMeta} from "../network/peers/peersData.js";
@@ -19,20 +24,37 @@ import {MAX_CONCURRENT_REQUESTS} from "./constants.js";
 import {SyncOptions} from "./options.js";
 import {
   BlockInputSyncCacheItem,
+  PayloadSyncCacheItem,
   PendingBlockInput,
   PendingBlockInputStatus,
   PendingBlockType,
+  PendingPayloadEnvelope,
+  PendingPayloadInput,
+  PendingPayloadInputStatus,
+  PendingPayloadRootHex,
   getBlockInputSyncCacheItemRootHex,
   getBlockInputSyncCacheItemSlot,
+  getPayloadSyncCacheItemRootHex,
+  getPayloadSyncCacheItemSlot,
   isPendingBlockInput,
+  isPendingPayloadEnvelope,
+  isPendingPayloadInput,
 } from "./types.js";
 import {DownloadByRootError, downloadByRoot} from "./utils/downloadByRoot.js";
-import {getAllDescendantBlocks, getDescendantBlocks, getUnknownAndAncestorBlocks} from "./utils/pendingBlocksTree.js";
+import {getAllDescendantBlocks, getUnknownAndAncestorBlocks} from "./utils/pendingBlocksTree.js";
 
 const MAX_ATTEMPTS_PER_BLOCK = 5;
 const MAX_KNOWN_BAD_BLOCKS = 500;
 const MAX_PENDING_BLOCKS = 100;
 
+type AdvancePendingBlockResult =
+  | "ready"
+  | "queued_block"
+  | "queued_parent_block"
+  | "queued_parent_payload"
+  | "blocked"
+  | "removed";
+
 enum FetchResult {
   SuccessResolved = "success_resolved",
   SuccessMissingParent = "success_missing_parent",
@@ -41,6 +63,27 @@ enum FetchResult {
   FailureMaxAttempts = "failure_max_attempts",
 }
 
+class UnknownBlockRateLimitedError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "UnknownBlockRateLimitedError";
+  }
+}
+
+function getRateLimitedUntilMs(e: unknown): number | null {
+  if (!(e instanceof RequestError)) {
+    return null;
+  }
+
+  switch (e.type.code) {
+    case RequestErrorCode.RESP_RATE_LIMITED:
+    case RequestErrorCode.REQUEST_SELF_RATE_LIMITED:
+      return e.type.rateLimitedUntilMs ?? null;
+    default:
+      return null;
+  }
+}
+
 /**
  * BlockInputSync is a class that handles ReqResp to find blocks and data related to a specific blockRoot.  The
  * blockRoot may have been found via object gossip, or the API.  Gossip objects that can trigger a search are block,
@@ -78,10 +121,13 @@ export class BlockInputSync {
    * block RootHex -> PendingBlock. To avoid finding same root at the same time
    */
   private readonly pendingBlocks = new Map<RootHex, BlockInputSyncCacheItem>();
+  // Payload sync is keyed by beacon block root as well, so block and payload queues can unblock each other.
+  private readonly pendingPayloads = new Map<RootHex, PayloadSyncCacheItem>();
   private readonly knownBadBlocks = new Set<RootHex>();
   private readonly maxPendingBlocks;
   private subscribedToNetworkEvents = false;
   private peerBalancer: UnknownBlockPeerBalancer;
+  private rateLimitBackoffTimeout: NodeJS.Timeout | undefined;
 
   constructor(
     private readonly config: ChainForkConfig,
@@ -98,6 +144,9 @@ export class BlockInputSync {
       metrics.blockInputSync.pendingBlocks.addCollect(() =>
         metrics.blockInputSync.pendingBlocks.set(this.pendingBlocks.size)
       );
+      metrics.blockInputSync.pendingPayloads.addCollect(() =>
+        metrics.blockInputSync.pendingPayloads.set(this.pendingPayloads.size)
+      );
       metrics.blockInputSync.knownBadBlocks.addCollect(() =>
         metrics.blockInputSync.knownBadBlocks.set(this.knownBadBlocks.size)
       );
@@ -114,8 +163,12 @@ export class BlockInputSync {
     if (!this.subscribedToNetworkEvents) {
       this.logger.verbose("BlockInputSync enabled.");
       this.chain.emitter.on(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
+      this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
       this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
+      this.chain.emitter.on(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
       this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
+      this.chain.emitter.on(routes.events.EventType.block, this.onBlockImported);
+      this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadImported);
       this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
       this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
       this.subscribedToNetworkEvents = true;
@@ -124,9 +177,14 @@ export class BlockInputSync {
 
   unsubscribeFromNetwork(): void {
     this.logger.verbose("BlockInputSync disabled.");
+    this.clearRateLimitBackoffTimer();
     this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
+    this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
     this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
+    this.chain.emitter.off(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
     this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
+    this.chain.emitter.off(routes.events.EventType.block, this.onBlockImported);
+    this.chain.emitter.off(routes.events.EventType.executionPayload, this.onPayloadImported);
     this.network.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
     this.network.events.off(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
     this.subscribedToNetworkEvents = false;
@@ -168,12 +226,55 @@ export class BlockInputSync {
     }
   };
 
+  private onUnknownEnvelopeBlockRoot = (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRoot]): void => {
+    try {
+      this.addByPayloadRootHex(data.rootHex, data.peer);
+      this.triggerUnknownBlockSearch();
+      this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_DATA});
+      this.metrics?.blockInputSync.source.inc({source: data.source});
+    } catch (e) {
+      this.logger.debug("Error handling unknownEnvelopeBlockRoot event", {}, e as Error);
+    }
+  };
+
+  private onIncompletePayloadEnvelope = (data: ChainEventData[ChainEvent.incompletePayloadEnvelope]): void => {
+    try {
+      this.addByPayloadInput(data.payloadInput, data.peer);
+      this.triggerUnknownBlockSearch();
+      this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_DATA});
+      this.metrics?.blockInputSync.source.inc({source: data.source});
+    } catch (e) {
+      this.logger.debug("Error handling incompletePayloadEnvelope event", {}, e as Error);
+    }
+  };
+
   /**
    * Process an unknownBlockParent event and register the block in `pendingBlocks` Map.
    */
   private onUnknownParent = (data: ChainEventData[ChainEvent.blockUnknownParent]): void => {
     try {
-      this.addByRootHex(data.blockInput.parentRootHex, data.peer);
+      const missingDependency = this.getMissingBlockDependency(data.blockInput);
+      if (missingDependency.kind === "invalidParentPayload") {
+        this.addByBlockInput(data.blockInput, data.peer);
+
+        const pendingBlock = this.pendingBlocks.get(data.blockInput.blockRootHex);
+        if (pendingBlock && isPendingBlockInput(pendingBlock)) {
+          this.logger.debug("Ignoring block with conflicting parent payload hash", {
+            slot: pendingBlock.blockInput.slot,
+            root: pendingBlock.blockInput.blockRootHex,
+            parentRoot: missingDependency.parentRootHex,
+            parentBlockHash: missingDependency.parentBlockHashHex,
+          });
+          this.removeAndDownScoreAllDescendants(pendingBlock);
+        }
+        return;
+      }
+
+      if (missingDependency.kind === "parentPayload") {
+        this.addByPayloadRootHex(missingDependency.rootHex, data.peer);
+      } else if (missingDependency.kind === "parentBlock") {
+        this.addByRootHex(missingDependency.rootHex, data.peer);
+      }
       this.addByBlockInput(data.blockInput, data.peer);
       this.triggerUnknownBlockSearch();
       this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_PARENT});
@@ -183,8 +284,22 @@ export class BlockInputSync {
     }
   };
 
-  private addByRootHex = (rootHex: RootHex, peerIdStr?: PeerIdStr): void => {
+  private onBlockImported = (): void => {
+    if (this.pendingPayloads.size > 0) {
+      this.triggerUnknownBlockSearch();
+    }
+  };
+
+  private onPayloadImported = ({
+    blockRoot,
+  }: routes.events.EventData[routes.events.EventType.executionPayload]): void => {
+    this.pendingPayloads.delete(blockRoot);
+    this.triggerUnknownBlockSearch();
+  };
+
+  private addByRootHex = (rootHex: RootHex, peerIdStr?: PeerIdStr): boolean => {
     let pendingBlock = this.pendingBlocks.get(rootHex);
+    let added = false;
     if (!pendingBlock) {
       pendingBlock = {
         status: PendingBlockInputStatus.pending,
@@ -193,6 +308,7 @@ export class BlockInputSync {
         timeAddedSec: Date.now() / 1000,
       };
       this.pendingBlocks.set(rootHex, pendingBlock);
+      added = true;
 
       this.logger.verbose("Added new rootHex to BlockInputSync.pendingBlocks", {
         root: pendingBlock.rootHex,
@@ -210,6 +326,7 @@ export class BlockInputSync {
     if (prunedItemCount > 0) {
       this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingBlocks`);
     }
+    return added;
   };
 
   private addByBlockInput = (blockInput: IBlockInput, peerIdStr?: string): void => {
@@ -242,6 +359,58 @@ export class BlockInputSync {
     }
   };
 
+  private addByPayloadRootHex = (rootHex: RootHex, peerIdStr?: PeerIdStr): boolean => {
+    let pendingPayload = this.pendingPayloads.get(rootHex);
+    let added = false;
+    if (!pendingPayload) {
+      pendingPayload = {
+        status: PendingPayloadInputStatus.pending,
+        rootHex,
+        peerIdStrings: new Set(),
+        timeAddedSec: Date.now() / 1000,
+      };
+      this.pendingPayloads.set(rootHex, pendingPayload);
+      added = true;
+
+      this.logger.verbose("Added new payload rootHex to BlockInputSync.pendingPayloads", {
+        root: rootHex,
+        peerIdStr: peerIdStr ?? "unknown peer",
+      });
+    }
+
+    if (peerIdStr) {
+      pendingPayload.peerIdStrings.add(peerIdStr);
+    }
+
+    const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
+    if (prunedItemCount > 0) {
+      this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
+    }
+    return added;
+  };
+
+  private addByPayloadInput = (
+    payloadInput: PayloadEnvelopeInput,
+    peerIdStr?: PeerIdStr,
+    envelope?: gloas.SignedExecutionPayloadEnvelope
+  ): void => {
+    const pendingPayload = this.toPendingPayloadInput(
+      payloadInput,
+      this.pendingPayloads.get(payloadInput.blockRootHex),
+      envelope
+    );
+
+    if (peerIdStr) {
+      pendingPayload.peerIdStrings.add(peerIdStr);
+    }
+
+    this.pendingPayloads.set(payloadInput.blockRootHex, pendingPayload);
+    const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
+    if (prunedItemCount > 0) {
+      this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
+    }
+  };
+
   private onPeerConnected = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
     try {
       const peerId = data.peer;
@@ -256,56 +425,242 @@ export class BlockInputSync {
   private onPeerDisconnected = (data: NetworkEventData[NetworkEvent.peerDisconnected]): void => {
     const peerId = data.peer;
     this.peerBalancer.onPeerDisconnected(peerId);
+    this.scheduleRateLimitBackoffRetry();
   };
 
+  /**
+   * Post-gloas, a locally complete block can still be blocked on its parent's execution payload lineage.
+   * Distinguish which dependency is missing so the scheduler can enqueue the right follow-up work.
+   */
+  private getMissingBlockDependency(
+    blockInput: IBlockInput
+  ):
+    | {kind: "ready"}
+    | {kind: "block" | "parentBlock" | "parentPayload"; rootHex: RootHex}
+    | {kind: "invalidParentPayload"; parentRootHex: RootHex; parentBlockHashHex: RootHex} {
+    const parentRootHex = blockInput.parentRootHex;
+    if (!this.chain.forkChoice.hasBlockHex(parentRootHex)) {
+      return {kind: "parentBlock", rootHex: parentRootHex};
+    }
+
+    if (!blockInput.hasBlock()) {
+      return {kind: "block", rootHex: blockInput.blockRootHex};
+    }
+
+    if (this.config.getForkSeq(blockInput.slot) < ForkSeq.gloas) {
+      return {kind: "ready"};
+    }
+
+    const block = blockInput.getBlock() as gloas.SignedBeaconBlock;
+    const parentBlockHashHex = toRootHex(block.message.body.signedExecutionPayloadBid.message.parentBlockHash);
+    if (this.chain.forkChoice.getBlockHexAndBlockHash(parentRootHex, parentBlockHashHex) !== null) {
+      return {kind: "ready"};
+    }
+
+    if (this.chain.forkChoice.hasPayloadHexUnsafe(parentRootHex)) {
+      return {kind: "invalidParentPayload", parentRootHex, parentBlockHashHex};
+    }
+
+    const parentPayloadInput = this.chain.seenPayloadEnvelopeInputCache.get(parentRootHex);
+    if (parentPayloadInput) {
+      if (parentPayloadInput.getBlockHashHex() === parentBlockHashHex) {
+        return {kind: "parentPayload", rootHex: parentRootHex};
+      }
+
+      return {kind: "invalidParentPayload", parentRootHex, parentBlockHashHex};
+    }
+
+    return {kind: "parentPayload", rootHex: parentRootHex};
+  }
+
+  private advancePendingBlock(pendingBlock: PendingBlockInput): AdvancePendingBlockResult {
+    const missingDependency = this.getMissingBlockDependency(pendingBlock.blockInput);
+
+    switch (missingDependency.kind) {
+      case "ready":
+        return "ready";
+
+      case "block":
+        pendingBlock.status = PendingBlockInputStatus.pending;
+        return "queued_block";
+
+      case "parentBlock": {
+        let added = this.addByRootHex(missingDependency.rootHex);
+        for (const peerIdStr of pendingBlock.peerIdStrings) {
+          added = this.addByRootHex(missingDependency.rootHex, peerIdStr) || added;
+        }
+        return added ? "queued_parent_block" : "blocked";
+      }
+
+      case "parentPayload": {
+        let added = this.addByPayloadRootHex(missingDependency.rootHex);
+        for (const peerIdStr of pendingBlock.peerIdStrings) {
+          added = this.addByPayloadRootHex(missingDependency.rootHex, peerIdStr) || added;
+        }
+        return added ? "queued_parent_payload" : "blocked";
+      }
+
+      case "invalidParentPayload":
+        this.logger.debug("Removing block with conflicting parent payload hash", {
+          slot: pendingBlock.blockInput.slot,
+          root: pendingBlock.blockInput.blockRootHex,
+          parentRoot: missingDependency.parentRootHex,
+          parentBlockHash: missingDependency.parentBlockHashHex,
+        });
+        this.removeAndDownScoreAllDescendants(pendingBlock);
+        return "removed";
+    }
+  }
+
+  private toPendingPayloadInput(
+    payloadInput: PayloadEnvelopeInput,
+    previous?: PayloadSyncCacheItem,
+    envelope?: gloas.SignedExecutionPayloadEnvelope
+  ): PendingPayloadInput {
+    // Normalize every payload queueing path into the same cache shape while preserving first-seen
+    // timing and peer provenance from any earlier by-root or envelope-only entry.
+    const queuedEnvelope = envelope ?? (previous && isPendingPayloadEnvelope(previous) ? previous.envelope : undefined);
+
+    if (queuedEnvelope && !payloadInput.hasPayloadEnvelope()) {
+      payloadInput.addPayloadEnvelope({
+        envelope: queuedEnvelope,
+        source: PayloadEnvelopeInputSource.byRoot,
+        seenTimestampSec: Date.now() / 1000,
+      });
+    }
+
+    return {
+      status: payloadInput.isComplete() ? PendingPayloadInputStatus.downloaded : PendingPayloadInputStatus.pending,
+      payloadInput,
+      timeAddedSec: previous?.timeAddedSec ?? Date.now() / 1000,
+      timeSyncedSec: payloadInput.isComplete() ? Date.now() / 1000 : undefined,
+      peerIdStrings: new Set(previous?.peerIdStrings ?? []),
+    };
+  }
+
   /**
    * Gather tip parent blocks with unknown parent and do a search for all of them
    */
   private triggerUnknownBlockSearch = (): void => {
     // Cheap early stop to prevent calling the network.getConnectedPeers()
-    if (this.pendingBlocks.size === 0) {
+    if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
       return;
     }
 
-    // If the node loses all peers with pending unknown blocks, the sync will stall
+    // If the node loses all peers with pending unknown blocks or payloads, the sync will stall
     const connectedPeers = this.network.getConnectedPeers();
-    if (connectedPeers.length === 0) {
-      this.logger.debug("No connected peers, skipping unknown block search.");
-      return;
-    }
+    const hasConnectedPeers = connectedPeers.length > 0;
 
     const {unknowns, ancestors} = getUnknownAndAncestorBlocks(this.pendingBlocks);
-    // it's rare when there is no unknown block
-    // see https://github.com/ChainSafe/lodestar/issues/5649#issuecomment-1594213550
-    if (unknowns.length === 0) {
-      let processedBlocks = 0;
-
-      for (const block of ancestors) {
-        // when this happens, it's likely the block and parent block are processed by head sync
-        if (this.chain.forkChoice.hasBlockHex(block.blockInput.parentRootHex)) {
+    let processedBlocks = 0;
+    let shouldRerunBlockSearch = false;
+
+    for (const block of ancestors) {
+      const advanceResult = this.advancePendingBlock(block);
+      switch (advanceResult) {
+        case "ready":
           processedBlocks++;
-          this.processBlock(block).catch((e) => {
+          this.processReadyBlock(block).catch((e) => {
             this.logger.debug("Unexpected error - process old downloaded block", {}, e);
           });
-        }
+          break;
+
+        case "queued_block":
+        case "queued_parent_block":
+          shouldRerunBlockSearch = true;
+          break;
+
+        case "queued_parent_payload":
+        case "blocked":
+        case "removed":
+          break;
       }
+    }
 
+    if (unknowns.length > 0) {
+      if (!hasConnectedPeers) {
+        this.logger.debug("No connected peers, skipping unknown block download.");
+      } else {
+        // Most of the time there is exactly 1 unknown block
+        for (const block of unknowns) {
+          this.downloadBlock(block).catch((e) => {
+            this.logger.debug("Unexpected error - downloadBlock", {root: getBlockInputSyncCacheItemRootHex(block)}, e);
+          });
+        }
+      }
+    } else if (ancestors.length > 0) {
+      // It's rare when there is no unknown block
+      // see https://github.com/ChainSafe/lodestar/issues/5649#issuecomment-1594213550
       this.logger.verbose("No unknown block, process ancestor downloaded blocks", {
         pendingBlocks: this.pendingBlocks.size,
         ancestorBlocks: ancestors.length,
         processedBlocks,
       });
-      return;
     }
 
-    // most of the time there is exactly 1 unknown block
-    for (const block of unknowns) {
-      this.downloadBlock(block).catch((e) => {
-        this.logger.debug("Unexpected error - downloadBlock", {root: getBlockInputSyncCacheItemRootHex(block)}, e);
+    // Blocks can unblock payloads and payloads can unblock blocks, so every scheduler pass services both queues.
+    for (const payload of Array.from(this.pendingPayloads.values())) {
+      if (isPendingPayloadInput(payload) && payload.status === PendingPayloadInputStatus.downloaded) {
+        this.processPayload(payload).catch((e) => {
+          this.logger.debug("Unexpected error - process downloaded payload", {}, e);
+        });
+        continue;
+      }
+
+      if (isPendingPayloadEnvelope(payload)) {
+        this.reconcilePayloadEnvelope(payload).catch((e) => {
+          this.logger.debug("Unexpected error - reconcile pending payload envelope", {}, e);
+        });
+        continue;
+      }
+
+      if (!hasConnectedPeers) {
+        this.logger.debug("No connected peers, skipping unknown payload download.", {
+          root: getPayloadSyncCacheItemRootHex(payload),
+        });
+        continue;
+      }
+
+      this.downloadPayload(payload).catch((e) => {
+        this.logger.debug("Unexpected error - downloadPayload", {root: getPayloadSyncCacheItemRootHex(payload)}, e);
       });
     }
+
+    if (shouldRerunBlockSearch) {
+      this.triggerUnknownBlockSearch();
+    }
   };
 
+  private scheduleRateLimitBackoffRetry(): void {
+    this.clearRateLimitBackoffTimer();
+
+    if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
+      return;
+    }
+
+    const now = Date.now();
+    const retryAt = this.peerBalancer.getNextRateLimitRetryAt();
+    if (retryAt === null) {
+      return;
+    }
+
+    this.rateLimitBackoffTimeout = setTimeout(
+      () => {
+        this.rateLimitBackoffTimeout = undefined;
+        this.triggerUnknownBlockSearch();
+        this.scheduleRateLimitBackoffRetry();
+      },
+      Math.max(0, retryAt - now)
+    );
+  }
+
+  private clearRateLimitBackoffTimer(): void {
+    if (this.rateLimitBackoffTimeout !== undefined) {
+      clearTimeout(this.rateLimitBackoffTimeout);
+      this.rateLimitBackoffTimeout = undefined;
+    }
+  }
+
   private async downloadBlock(block: BlockInputSyncCacheItem): Promise<void> {
     if (block.status !== PendingBlockInputStatus.pending) {
       return;
@@ -342,10 +697,26 @@ export class BlockInputSync {
       this.logger.verbose("Downloaded unknown block", logCtx2);
 
       if (parentInForkChoice) {
-        // Bingo! Process block. Add to pending blocks anyway for recycle the cache that prevents duplicate processing
-        this.processBlock(pending).catch((e) => {
-          this.logger.debug("Unexpected error - process newly downloaded block", logCtx2, e);
-        });
+        // If the direct parent is already in fork choice, let the block state machine decide if
+        // the next step is block import, parent payload download, or branch removal.
+        const advanceResult = this.advancePendingBlock(pending);
+        switch (advanceResult) {
+          case "ready":
+            this.processReadyBlock(pending).catch((e) => {
+              this.logger.debug("Unexpected error - process newly downloaded block", logCtx2, e);
+            });
+            break;
+
+          case "queued_block":
+          case "queued_parent_block":
+          case "queued_parent_payload":
+            this.triggerUnknownBlockSearch();
+            break;
+
+          case "blocked":
+          case "removed":
+            break;
+        }
       } else if (blockSlot <= finalizedSlot) {
         // the common ancestor of the downloading chain and canonical chain should be at least the finalized slot and
         // we should found it through forkchoice. If not, we should penalize all peers sending us this block chain
@@ -361,6 +732,16 @@ export class BlockInputSync {
         this.onUnknownBlockRoot({rootHex: pending.blockInput.parentRootHex, source: BlockInputSource.byRoot});
       }
     } else {
+      if (res.err instanceof UnknownBlockRateLimitedError) {
+        const pendingBlock = this.pendingBlocks.get(rootHex);
+        if (pendingBlock) {
+          pendingBlock.status = PendingBlockInputStatus.pending;
+        }
+        this.logger.debug("Deferring unknown block download due to peer rate limit", logCtx, res.err);
+        this.scheduleRateLimitBackoffRetry();
+        return;
+      }
+
       this.metrics?.blockInputSync.downloadedBlocksError.inc();
       this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
       this.removeAndDownScoreAllDescendants(block);
@@ -368,26 +749,11 @@ export class BlockInputSync {
   }
 
   /**
-   * Send block to the processor awaiting completition. If processed successfully, send all children to the processor.
-   * On error, remove and downscore all descendants.
-   * This function could run recursively for all descendant blocks
+   * Import a block that has already passed the local dependency checks in BlockInputSync.
+   * On error, remove and downscore descendants as appropriate for the failure type.
    */
-  private async processBlock(pendingBlock: PendingBlockInput): Promise<void> {
-    // pending block status is `downloaded` right after `downloadBlock`
-    // but could be `pending` if added by `onUnknownBlockParent` event and this function is called recursively
+  private async processReadyBlock(pendingBlock: PendingBlockInput): Promise<void> {
     if (pendingBlock.status !== PendingBlockInputStatus.downloaded) {
-      if (pendingBlock.status === PendingBlockInputStatus.pending) {
-        const connectedPeers = this.network.getConnectedPeers();
-        if (connectedPeers.length === 0) {
-          this.logger.debug("No connected peers, skipping download block", {
-            slot: pendingBlock.blockInput.slot,
-            blockRoot: pendingBlock.blockInput.blockRootHex,
-          });
-          return;
-        }
-        // if the download is a success we'll call `processBlock()` for this block
-        await this.downloadBlock(pendingBlock);
-      }
       return;
     }
 
@@ -432,15 +798,9 @@ export class BlockInputSync {
     if (!res.err) {
       // no need to update status to "processed", delete anyway
       this.pendingBlocks.delete(pendingBlock.blockInput.blockRootHex);
-
-      // Send child blocks to the processor
-      for (const descendantBlock of getDescendantBlocks(pendingBlock.blockInput.blockRootHex, this.pendingBlocks)) {
-        if (isPendingBlockInput(descendantBlock)) {
-          this.processBlock(descendantBlock).catch((e) => {
-            this.logger.debug("Unexpected error - process descendant block", {}, e);
-          });
-        }
-      }
+      // Re-enter the scheduler so descendants blocked on either parent blocks or parent payloads
+      // are advanced through the same dependency checks as every other pending item.
+      this.triggerUnknownBlockSearch();
     } else {
       const errorData = {slot: pendingBlock.blockInput.slot, root: pendingBlock.blockInput.blockRootHex};
       if (res.err instanceof BlockError) {
@@ -456,6 +816,19 @@ export class BlockInputSync {
             pendingBlock.status = PendingBlockInputStatus.downloaded;
             break;
 
+          case BlockErrorCode.PARENT_PAYLOAD_UNKNOWN:
+            this.logger.error(
+              "processReadyBlock() hit unexpected parent payload dependency after readiness checks",
+              {
+                ...errorData,
+                parentRoot: pendingBlock.blockInput.parentRootHex,
+                parentBlockHash: res.err.type.parentBlockHash,
+              },
+              res.err
+            );
+            pendingBlock.status = PendingBlockInputStatus.downloaded;
+            break;
+
           case BlockErrorCode.EXECUTION_ENGINE_ERROR:
             // Removing the block(s) without penalizing the peers, hoping for EL to
             // recover on a latter download + verify attempt
@@ -477,6 +850,393 @@ export class BlockInputSync {
     }
   }
 
+  /**
+   * Reconcile an envelope-first payload entry once the block import path has seeded its
+   * PayloadEnvelopeInput. This may queue block download, validate the speculative envelope, or
+   * downgrade back to by-root fetching when the cached envelope does not match the imported block.
+   */
+  private async reconcilePayloadEnvelope(pendingPayload: PendingPayloadEnvelope): Promise<void> {
+    const rootHex = getPayloadSyncCacheItemRootHex(pendingPayload);
+    if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+      this.pendingPayloads.delete(rootHex);
+      return;
+    }
+
+    const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
+    if (!payloadInput) {
+      if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
+        // Column commitments live on the block body, so an envelope-only entry has to pull the block first.
+        if (!this.pendingBlocks.has(rootHex)) {
+          this.addByRootHex(rootHex);
+        }
+
+        const pendingBlock = this.pendingBlocks.get(rootHex);
+        if (pendingBlock && this.network.getConnectedPeers().length > 0) {
+          await this.downloadBlock(pendingBlock);
+        }
+      } else {
+        this.logger.debug("Missing PayloadEnvelopeInput for known block while reconciling payload envelope", {
+          root: rootHex,
+        });
+      }
+      return;
+    }
+
+    if (!payloadInput.hasPayloadEnvelope()) {
+      const validationResult = await wrapError(
+        validateGossipExecutionPayloadEnvelope(this.chain, pendingPayload.envelope)
+      );
+      if (validationResult.err) {
+        this.logger.debug(
+          "Pending payload envelope failed validation after block import, refetching by root",
+          {slot: pendingPayload.envelope.message.payload.slotNumber, root: rootHex},
+          validationResult.err
+        );
+
+        const pendingPayloadByRoot: PendingPayloadRootHex = {
+          status: PendingPayloadInputStatus.pending,
+          rootHex,
+          timeAddedSec: pendingPayload.timeAddedSec,
+          peerIdStrings: new Set(pendingPayload.peerIdStrings),
+        };
+        this.pendingPayloads.set(rootHex, pendingPayloadByRoot);
+
+        if (this.network.getConnectedPeers().length > 0) {
+          await this.downloadPayload(pendingPayloadByRoot);
+        }
+        return;
+      }
+    }
+
+    const upgradedPayload = this.toPendingPayloadInput(payloadInput, pendingPayload, pendingPayload.envelope);
+    this.pendingPayloads.set(rootHex, upgradedPayload);
+
+    if (upgradedPayload.status === PendingPayloadInputStatus.downloaded) {
+      await this.processPayload(upgradedPayload);
+      return;
+    }
+
+    await this.downloadPayload(upgradedPayload);
+  }
+
+  private async downloadPayload(payload: PayloadSyncCacheItem): Promise<void> {
+    if (isPendingPayloadEnvelope(payload)) {
+      await this.reconcilePayloadEnvelope(payload);
+      return;
+    }
+
+    const rootHex = getPayloadSyncCacheItemRootHex(payload);
+    if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+      this.pendingPayloads.delete(rootHex);
+      return;
+    }
+
+    if (payload.status !== PendingPayloadInputStatus.pending) {
+      return;
+    }
+
+    const logCtx = {
+      slot: getPayloadSyncCacheItemSlot(payload),
+      root: rootHex,
+      pendingPayloads: this.pendingPayloads.size,
+    };
+
+    this.logger.verbose("BlockInputSync.downloadPayload()", logCtx);
+
+    payload.status = PendingPayloadInputStatus.fetching;
+
+    const res = await wrapError(this.fetchPayloadInput(payload));
+    if (!res.err) {
+      const pendingPayload = res.result;
+      this.pendingPayloads.set(getPayloadSyncCacheItemRootHex(pendingPayload), pendingPayload);
+
+      if (isPendingPayloadEnvelope(pendingPayload)) {
+        await this.reconcilePayloadEnvelope(pendingPayload);
+      } else if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
+        await this.processPayload(pendingPayload);
+      }
+      return;
+    }
+
+    this.logger.debug("Ignoring unknown payload root after failed download", logCtx, res.err);
+    if (!isPendingPayloadEnvelope(payload)) {
+      payload.status = PendingPayloadInputStatus.pending;
+    }
+  }
+
+  private async processPayload(pendingPayload: PendingPayloadInput): Promise<void> {
+    const rootHex = pendingPayload.payloadInput.blockRootHex;
+    const logCtx = {slot: pendingPayload.payloadInput.slot, root: rootHex};
+
+    if (pendingPayload.status !== PendingPayloadInputStatus.downloaded) {
+      this.logger.debug("Skipping payload processing before payload input is downloaded", {
+        ...logCtx,
+        status: pendingPayload.status,
+      });
+      return;
+    }
+
+    if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+      this.logger.debug("Payload already imported while processing unknown payload", logCtx);
+      this.pendingPayloads.delete(rootHex);
+      return;
+    }
+
+    if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
+      this.logger.debug("Payload input is ready before its block is in fork choice", logCtx);
+      const added = this.addByRootHex(rootHex);
+      pendingPayload.status = PendingPayloadInputStatus.downloaded;
+      if (added) {
+        this.triggerUnknownBlockSearch();
+      }
+      return;
+    }
+
+    pendingPayload.status = PendingPayloadInputStatus.processing;
+
+    const res = await wrapError(this.chain.processExecutionPayload(pendingPayload.payloadInput));
+    if (!res.err) {
+      this.logger.debug("Processed payload from unknown sync", logCtx);
+      this.pendingPayloads.delete(rootHex);
+      this.triggerUnknownBlockSearch();
+      return;
+    }
+
+    if (res.err instanceof PayloadError) {
+      switch (res.err.type.code) {
+        case PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE:
+          // Payload sync discovered the block dependency before the block queue did. Re-enqueue the
+          // block and keep the payload ready so the scheduler can retry once the block reaches fork choice.
+          if (this.addByRootHex(rootHex)) {
+            this.triggerUnknownBlockSearch();
+          }
+          // Keep the payload out of any synchronous requeue pass; a later scheduler pass will retry it.
+          pendingPayload.status = PendingPayloadInputStatus.downloaded;
+          break;
+
+        case PayloadErrorCode.EXECUTION_ENGINE_ERROR:
+          this.logger.debug("Execution engine error while processing payload from unknown sync", logCtx, res.err);
+          pendingPayload.status = PendingPayloadInputStatus.downloaded;
+          break;
+
+        case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
+        case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
+        case PayloadErrorCode.INVALID_SIGNATURE:
+          // TODO GLOAS: Decide how invalid payload inputs should eventually leave memory without
+          // reintroducing envelope replacement / recreation flows.
+          this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+          this.removePendingPayloadAndDescendants(rootHex);
+          break;
+
+        default:
+          this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+          this.pendingPayloads.delete(rootHex);
+      }
+      return;
+    }
+
+    this.logger.debug("Unknown error processing payload from unknown sync", logCtx, res.err);
+    pendingPayload.status = PendingPayloadInputStatus.downloaded;
+  }
+
+  /**
+   * Download payload material keyed by beacon block root. Unlike block download, payload sync may
+   * already have a locally cached envelope or partial columns, so each attempt starts from local state
+   * and only asks peers for the remaining pieces.
+   */
+  private async fetchPayloadInput(
+    cacheItem: PendingPayloadInput | PendingPayloadRootHex
+  ): Promise<PendingPayloadInput | PendingPayloadEnvelope> {
+    const rootHex = getPayloadSyncCacheItemRootHex(cacheItem);
+    const blockRoot = fromHex(rootHex);
+    const excludedPeers = new Set<PeerIdStr>();
+
+    let slot = getPayloadSyncCacheItemSlot(cacheItem);
+    let payloadInput = isPendingPayloadInput(cacheItem)
+      ? cacheItem.payloadInput
+      : this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
+    let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
+
+    let i = 0;
+    let deferredByRateLimit = false;
+    while (i++ < this.getMaxDownloadAttempts()) {
+      const pendingColumns = payloadInput?.hasAllData()
+        ? new Set<number>()
+        : new Set(payloadInput?.getMissingSampledColumnMeta().missing ?? []);
+      const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
+      if (peerMeta === null) {
+        if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
+          throw new UnknownBlockRateLimitedError(
+            `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`
+          );
+        }
+
+        throw Error(
+          `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
+        );
+      }
+
+      const {peerId, client: peerClient} = peerMeta;
+      cacheItem.peerIdStrings.add(peerId);
+
+      try {
+        if (!envelope) {
+          envelope = await this.fetchExecutionPayloadEnvelope(peerId, blockRoot, rootHex);
+          slot = envelope.message.payload.slotNumber;
+        }
+
+        payloadInput ??= this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
+        if (!payloadInput) {
+          if (this.chain.forkChoice.hasBlockHex(rootHex)) {
+            throw new Error(`Missing PayloadEnvelopeInput for known block ${rootHex}`);
+          }
+          // Keep the validated envelope around, but wait for the block body before turning it into a full payload input.
+          return {
+            status: PendingPayloadInputStatus.waitingForBlock,
+            envelope,
+            timeAddedSec: cacheItem.timeAddedSec,
+            peerIdStrings: cacheItem.peerIdStrings,
+          };
+        }
+
+        if (!payloadInput.hasPayloadEnvelope()) {
+          await validateGossipExecutionPayloadEnvelope(this.chain, envelope);
+        }
+
+        let pendingPayload = this.toPendingPayloadInput(payloadInput, cacheItem, envelope);
+        if (!pendingPayload.payloadInput.hasAllData()) {
+          const missing = pendingPayload.payloadInput.getMissingSampledColumnMeta().missing;
+          if (missing.length > 0) {
+            const columnSidecars = await this.fetchPayloadColumns(peerMeta, pendingPayload.payloadInput, missing);
+            const seenTimestampSec = Date.now() / 1000;
+            for (const columnSidecar of columnSidecars) {
+              if (pendingPayload.payloadInput.hasColumn(columnSidecar.index)) {
+                continue;
+              }
+
+              pendingPayload.payloadInput.addColumn({
+                columnSidecar,
+                source: PayloadEnvelopeInputSource.byRoot,
+                seenTimestampSec,
+                peerIdStr: peerId,
+              });
+            }
+            pendingPayload = this.toPendingPayloadInput(pendingPayload.payloadInput, pendingPayload);
+          }
+        }
+
+        this.logger.verbose("BlockInputSync.fetchPayloadInput: successful download", {
+          slot,
+          rootHex,
+          peerId,
+          peerClient,
+          hasPayload: pendingPayload.payloadInput.hasPayloadEnvelope(),
+          hasAllData: pendingPayload.payloadInput.hasAllData(),
+        });
+
+        if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
+          return pendingPayload;
+        }
+
+        cacheItem = pendingPayload;
+        payloadInput = pendingPayload.payloadInput;
+      } catch (e) {
+        this.logger.debug(
+          "Error downloading payload in BlockInputSync.fetchPayloadInput",
+          {slot, rootHex, attempt: i, peer: peerId, peerClient},
+          e as Error
+        );
+
+        const rateLimitedUntilMs = getRateLimitedUntilMs(e);
+        if (rateLimitedUntilMs !== null) {
+          deferredByRateLimit = true;
+          this.peerBalancer.onRateLimited(peerId, rateLimitedUntilMs);
+          this.scheduleRateLimitBackoffRetry();
+        } else if (e instanceof RequestError) {
+          switch (e.type.code) {
+            case RequestErrorCode.REQUEST_RATE_LIMITED:
+            case RequestErrorCode.REQUEST_TIMEOUT:
+              break;
+            default:
+              excludedPeers.add(peerId);
+              break;
+          }
+        } else {
+          excludedPeers.add(peerId);
+        }
+      } finally {
+        this.peerBalancer.onRequestCompleted(peerId);
+      }
+    }
+
+    if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
+      throw new UnknownBlockRateLimitedError(
+        `Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`
+      );
+    }
+
+    throw Error(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts.`);
+  }
+
+  private async fetchExecutionPayloadEnvelope(
+    peerIdStr: PeerIdStr,
+    blockRoot: Uint8Array,
+    rootHex: RootHex
+  ): Promise<gloas.SignedExecutionPayloadEnvelope> {
+    const response = await this.network.sendExecutionPayloadEnvelopesByRoot(peerIdStr, [blockRoot]);
+    const envelope = response.at(0);
+    if (!envelope) {
+      throw new Error(`Missing execution payload envelope for root=${rootHex}`);
+    }
+
+    const receivedRootHex = toRootHex(envelope.message.beaconBlockRoot);
+    if (receivedRootHex !== rootHex) {
+      throw new Error(`Execution payload envelope root mismatch requested=${rootHex} received=${receivedRootHex}`);
+    }
+
+    return envelope;
+  }
+
+  private async fetchPayloadColumns(
+    peerMeta: PeerSyncMeta,
+    payloadInput: PayloadEnvelopeInput,
+    missing: number[]
+  ): Promise<gloas.DataColumnSidecar[]> {
+    const {peerId: peerIdStr} = peerMeta;
+    const peerColumns = new Set(peerMeta.custodyColumns ?? []);
+    const requestedColumns = missing.filter((columnIndex) => peerColumns.has(columnIndex));
+    if (requestedColumns.length === 0) {
+      return [];
+    }
+
+    const columnSidecars = (await this.network.sendDataColumnSidecarsByRoot(peerIdStr, [
+      {blockRoot: fromHex(payloadInput.blockRootHex), columns: requestedColumns},
+    ])) as gloas.DataColumnSidecar[];
+
+    if (columnSidecars.length === 0) {
+      throw new Error(`No data column sidecars returned for payload root=${payloadInput.blockRootHex}`);
+    }
+
+    const requestedColumnsSet = new Set(requestedColumns);
+    const extraColumns = columnSidecars.filter((columnSidecar) => !requestedColumnsSet.has(columnSidecar.index));
+    if (extraColumns.length > 0) {
+      throw new Error(
+        `Received unexpected payload data columns indices=${prettyPrintIndices(extraColumns.map((column) => column.index))}`
+      );
+    }
+
+    // PayloadEnvelopeInput already carries the block slot, root, and commitments, so reuse the
+    // block-based Gloas validator rather than maintaining a second payload-specific variant.
+    await validateGloasBlockDataColumnSidecars(
+      payloadInput.slot,
+      fromHex(payloadInput.blockRootHex),
+      payloadInput.getBlobKzgCommitments(),
+      columnSidecars,
+      this.chain.metrics?.peerDas
+    );
+    return columnSidecars;
+  }
+
   /**
    * From a set of shuffled peers:
    *   - fetch the block
@@ -498,6 +1258,7 @@ export class BlockInputSync {
     }
 
     let i = 0;
+    let deferredByRateLimit = false;
     while (i++ < this.getMaxDownloadAttempts()) {
       const pendingColumns =
         isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput)
@@ -505,6 +1266,12 @@ export class BlockInputSync {
           : defaultPendingColumns;
       const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
       if (peerMeta === null) {
+        if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
+          throw new UnknownBlockRateLimitedError(
+            `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`
+          );
+        }
+
         // no more peer with needed columns to try, throw error
         const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
         this.metrics?.blockInputSync.fetchTimeSec.observe(
@@ -560,14 +1327,23 @@ export class BlockInputSync {
         } else if (e instanceof RequestError) {
           // should look into req_resp metrics in this case
           downloadByRootMetrics?.error.inc({code: "req_resp", client: peerClient});
-          switch (e.type.code) {
-            case RequestErrorCode.REQUEST_RATE_LIMITED:
-            case RequestErrorCode.REQUEST_TIMEOUT:
-              // do not exclude peer for these errors
-              break;
-            default:
-              excludedPeers.add(peerId);
-              break;
+          const rateLimitedUntilMs = getRateLimitedUntilMs(e);
+          if (rateLimitedUntilMs !== null) {
+            deferredByRateLimit = true;
+            this.peerBalancer.onRateLimited(peerId, rateLimitedUntilMs);
+            this.scheduleRateLimitBackoffRetry();
+          } else {
+            switch (e.type.code) {
+              case RequestErrorCode.REQUEST_RATE_LIMITED:
+              case RequestErrorCode.RESP_RATE_LIMITED:
+              case RequestErrorCode.REQUEST_SELF_RATE_LIMITED:
+              case RequestErrorCode.REQUEST_TIMEOUT:
+                // do not exclude peer for these errors
+                break;
+              default:
+                excludedPeers.add(peerId);
+                break;
+            }
           }
         } else {
           // investigate if this happens
@@ -595,6 +1371,10 @@ export class BlockInputSync {
 
     const message = `Error fetching BlockInput with slot=${slot} root=${rootHex} after ${i - 1} attempts.`;
 
+    if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
+      throw new UnknownBlockRateLimitedError(`${message} Peers are rate-limited.`);
+    }
+
     if (!isPendingBlockInput(cacheItem)) {
       throw Error(`${message} No block and no data was found.`);
     }
@@ -660,6 +1440,28 @@ export class BlockInputSync {
     pruneSetToMax(this.knownBadBlocks, MAX_KNOWN_BAD_BLOCKS);
   }
 
+  // Once a parent payload is invalid, every descendant waiting on that payload lineage becomes unrecoverable too.
+  private removePendingPayloadAndDescendants(rootHex: RootHex): void {
+    // Keep PayloadEnvelopeInput resident in the seen cache. importBlock() owns that object and
+    // later validation/finalization logic decides when it can leave memory.
+    this.pendingPayloads.delete(rootHex);
+
+    const badPendingBlocks = getAllDescendantBlocks(rootHex, this.pendingBlocks);
+    this.metrics?.blockInputSync.removedBlocks.inc(badPendingBlocks.length);
+
+    for (const block of badPendingBlocks) {
+      const descendantRootHex = getBlockInputSyncCacheItemRootHex(block);
+      this.pendingBlocks.delete(descendantRootHex);
+      this.pendingPayloads.delete(descendantRootHex);
+      this.chain.seenBlockInputCache.prune(descendantRootHex);
+      this.logger.debug("Removing pending descendant after invalid parent payload", {
+        slot: getBlockInputSyncCacheItemSlot(block),
+        blockRoot: descendantRootHex,
+        parentPayloadRoot: rootHex,
+      });
+    }
+  }
+
   private removeAllDescendants(block: BlockInputSyncCacheItem): BlockInputSyncCacheItem[] {
     const rootHex = getBlockInputSyncCacheItemRootHex(block);
     const slot = getBlockInputSyncCacheItemSlot(block);
@@ -671,7 +1473,10 @@ export class BlockInputSync {
     for (const block of badPendingBlocks) {
       const rootHex = getBlockInputSyncCacheItemRootHex(block);
       this.pendingBlocks.delete(rootHex);
+      this.pendingPayloads.delete(rootHex);
       this.chain.seenBlockInputCache.prune(rootHex);
+      // Keep PayloadEnvelopeInput resident in the seen cache for consistency with the
+      // importBlock()-owned lifecycle.
       this.logger.debug("Removing bad/unknown/incomplete BlockInputSyncCacheItem", {
         slot,
         blockRoot: rootHex,
@@ -701,10 +1506,12 @@ export class BlockInputSync {
 export class UnknownBlockPeerBalancer {
   readonly peersMeta: Map<PeerIdStr, PeerSyncMeta>;
   readonly activeRequests: Map<PeerIdStr, number>;
+  readonly rateLimitedUntilByPeer: Map<PeerIdStr, number>;
 
   constructor() {
     this.peersMeta = new Map();
     this.activeRequests = new Map();
+    this.rateLimitedUntilByPeer = new Map();
   }
 
   /** Trigger on each peer re-status */
@@ -719,6 +1526,41 @@ export class UnknownBlockPeerBalancer {
   onPeerDisconnected(peerId: PeerIdStr): void {
     this.peersMeta.delete(peerId);
     this.activeRequests.delete(peerId);
+    this.rateLimitedUntilByPeer.delete(peerId);
+  }
+
+  onRateLimited(peerId: PeerIdStr, rateLimitedUntilMs: number): void {
+    this.rateLimitedUntilByPeer.set(peerId, rateLimitedUntilMs);
+  }
+
+  getNextRateLimitRetryAt(pendingColumns?: Set<number>, excludedPeers?: Set<PeerIdStr>): number | null {
+    const now = Date.now();
+    let retryAt: number | null = null;
+
+    for (const [peerId, rateLimitedUntil] of this.rateLimitedUntilByPeer.entries()) {
+      if (rateLimitedUntil <= now) {
+        this.rateLimitedUntilByPeer.delete(peerId);
+        continue;
+      }
+
+      if (excludedPeers?.has(peerId)) {
+        continue;
+      }
+
+      const syncMeta = this.peersMeta.get(peerId);
+      if (syncMeta === undefined) {
+        this.rateLimitedUntilByPeer.delete(peerId);
+        continue;
+      }
+
+      if (pendingColumns !== undefined && !this.peerHasPendingColumns(syncMeta, pendingColumns)) {
+        continue;
+      }
+
+      retryAt = Math.min(retryAt ?? rateLimitedUntil, rateLimitedUntil);
+    }
+
+    return retryAt;
   }
 
   /**
@@ -767,6 +1609,7 @@ export class UnknownBlockPeerBalancer {
   }
 
   private filterPeers(pendingDataColumns: Set<number>, excludedPeers: Set<PeerIdStr>): PeerIdStr[] {
+    const now = Date.now();
     let maxColumnCount = 0;
     const considerPeers: {peerId: PeerIdStr; columnCount: number}[] = [];
     for (const [peerId, syncMeta] of this.peersMeta.entries()) {
@@ -775,12 +1618,24 @@ export class UnknownBlockPeerBalancer {
         continue;
       }
 
+      const rateLimitedUntil = this.rateLimitedUntilByPeer.get(peerId);
+      if (rateLimitedUntil !== undefined) {
+        if (now < rateLimitedUntil) {
+          continue;
+        }
+        this.rateLimitedUntilByPeer.delete(peerId);
+      }
+
       const activeRequests = this.activeRequests.get(peerId) ?? 0;
       if (activeRequests >= MAX_CONCURRENT_REQUESTS) {
         // should return peer with no more than MAX_CONCURRENT_REQUESTS active requests
         continue;
       }
 
+      if (!this.peerHasPendingColumns(syncMeta, pendingDataColumns)) {
+        continue;
+      }
+
       if (pendingDataColumns.size === 0) {
         considerPeers.push({peerId, columnCount: 0});
         continue;
@@ -814,4 +1669,12 @@ export class UnknownBlockPeerBalancer {
 
     return eligiblePeers;
   }
+
+  private peerHasPendingColumns(syncMeta: PeerSyncMeta, pendingDataColumns: Set<number>): boolean {
+    if (pendingDataColumns.size === 0) {
+      return true;
+    }
+
+    return syncMeta.custodyColumns.some((column) => pendingDataColumns.has(column));
+  }
 }