@lodestar/beacon-node 1.43.0-rc.1 → 1.43.0-rc.5

package/src/metrics/metrics/lodestar.ts

@@ -3,6 +3,7 @@ import {NotReorgedReason} from "@lodestar/fork-choice";
 import {ArchiveStoreTask} from "../../chain/archiveStore/archiveStore.js";
 import {FrequencyStateArchiveStep} from "../../chain/archiveStore/strategies/frequencyStateArchiveStrategy.js";
 import {BlockInputSource} from "../../chain/blocks/blockInput/index.js";
+import {PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
 import {PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js";
 import {JobQueueItemType} from "../../chain/bls/index.js";
 import {AttestationErrorCode, BlockErrorCode} from "../../chain/errors/index.js";
@@ -890,6 +891,11 @@ export function createLodestarMetrics(
         labelNames: ["source"],
         buckets: [0.5, 1, 2, 4, 6, 12],
       }),
+      processPayloadErrors: register.gauge<{error: PayloadErrorCode | "NOT_PAYLOAD_ERROR"}>({
+        name: "lodestar_gossip_execution_payload_envelope_process_payload_errors",
+        help: "Count of errors, by error type, while processing execution payload envelopes",
+        labelNames: ["error"],
+      }),
     },
     // recovery in the case of specific blob rows required
     recoverBlobSidecars: {

package/src/network/interface.ts

@@ -78,6 +78,7 @@ export interface INetwork extends INetworkCorePublic {
   // ReqResp
   sendBeaconBlocksByRange(peerId: PeerIdStr, request: phase0.BeaconBlocksByRangeRequest): Promise<SignedBeaconBlock[]>;
   sendBeaconBlocksByRoot(peerId: PeerIdStr, request: BeaconBlocksByRootRequest): Promise<SignedBeaconBlock[]>;
+  sendBeaconBlocksByHead(peerId: PeerIdStr, request: fulu.BeaconBlocksByHeadRequest): Promise<SignedBeaconBlock[]>;
   sendBlobSidecarsByRange(peerId: PeerIdStr, request: deneb.BlobSidecarsByRangeRequest): Promise<deneb.BlobSidecar[]>;
   sendBlobSidecarsByRoot(peerId: PeerIdStr, request: BlobSidecarsByRootRequest): Promise<deneb.BlobSidecar[]>;
   sendDataColumnSidecarsByRange(

package/src/network/network.ts

@@ -578,6 +578,18 @@ export class Network implements INetwork {
     );
   }
 
+  async sendBeaconBlocksByHead(
+    peerId: PeerIdStr,
+    request: fulu.BeaconBlocksByHeadRequest
+  ): Promise<SignedBeaconBlock[]> {
+    return collectMaxResponseTypedWithBytes(
+      this.sendReqRespRequest(peerId, ReqRespMethod.BeaconBlocksByHead, [Version.V1], request),
+      Math.min(request.count, this.config.MAX_REQUEST_BLOCKS_DENEB),
+      responseSszTypeByMethod[ReqRespMethod.BeaconBlocksByHead],
+      this.chain.serializedCache
+    );
+  }
+
   async sendLightClientBootstrap(peerId: PeerIdStr, request: Root): Promise<LightClientBootstrap> {
     return collectExactOneTyped(
       this.sendReqRespRequest(peerId, ReqRespMethod.LightClientBootstrap, [Version.V1], request),

package/src/network/processor/gossipHandlers.ts

@@ -35,6 +35,7 @@ import {
   IBlockInput,
   isBlockInputColumns,
 } from "../../chain/blocks/blockInput/index.js";
+import {PayloadError, PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
 import {PayloadEnvelopeInput, PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js";
 import {BlobSidecarValidation} from "../../chain/blocks/types.js";
 import {ChainEvent} from "../../chain/emitter.js";
@@ -1148,7 +1149,40 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       });
 
       chain.processExecutionPayload(payloadInput, {validSignature: true}).catch((e) => {
-        chain.logger.debug("Error processing execution payload from gossip", {slot, root: blockRootHex}, e as Error);
+        // Adjust verbosity based on error type
+        let logLevel: LogLevel;
+
+        if (e instanceof PayloadError) {
+          switch (e.type.code) {
+            // BLOCK_NOT_IN_FORK_CHOICE should not happen, validateGossipExecutionPayloadEnvelope above
+            // already verified the block is in fork choice
+            case PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE:
+            case PayloadErrorCode.MISS_BLOCK_STATE:
+            case PayloadErrorCode.EXECUTION_ENGINE_ERROR:
+              // Errors might indicate an issue with our node or the connected EL client
+              logLevel = LogLevel.error;
+              break;
+            // INVALID_SIGNATURE should not happen, signature is verified during gossip validation
+            case PayloadErrorCode.INVALID_SIGNATURE:
+            case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
+            case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
+              core.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadGossipPayload");
+              // Misbehaving peer, but could highlight an issue in another client
+              logLevel = LogLevel.warn;
+              break;
+          }
+        } else {
+          // Any unexpected error
+          logLevel = LogLevel.error;
+        }
+        metrics?.gossipExecutionPayloadEnvelope.processPayloadErrors.inc({
+          error: e instanceof PayloadError ? e.type.code : "NOT_PAYLOAD_ERROR",
+        });
+        chain.logger[logLevel](
+          "Error processing execution payload from gossip",
+          {slot, peer: peerIdStr, root: blockRootHex},
+          e as Error
+        );
       });
     },
     [GossipType.payload_attestation_message]: async ({

package/src/network/reqresp/ReqRespBeaconNode.ts

@@ -286,6 +286,7 @@ export class ReqRespBeaconNode extends ReqResp {
         // instead of protocol version. This is not easily fixable with our current architecture.
         // See https://github.com/ChainSafe/lodestar/pull/8168 for more details.
         [protocols.StatusV2(fork, this.config), this.onStatus.bind(this)],
+        [protocols.BeaconBlocksByHead(fork, this.config), this.getHandler(ReqRespMethod.BeaconBlocksByHead)],
         [
           protocols.DataColumnSidecarsByRoot(fork, this.config),
           this.getHandler(ReqRespMethod.DataColumnSidecarsByRoot),

package/src/network/reqresp/handlers/beaconBlocksByHead.ts

@@ -0,0 +1,91 @@
+import {PeerId} from "@libp2p/interface";
+import {BeaconConfig} from "@lodestar/config";
+import {ForkName, GENESIS_EPOCH, GENESIS_SLOT, isForkPostDeneb} from "@lodestar/params";
+import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
+import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {fulu} from "@lodestar/types";
+import {toRootHex} from "@lodestar/utils";
+import {IBeaconChain} from "../../../chain/index.js";
+import {getParentRootFromSignedBeaconBlockSerialized} from "../../../util/sszBytes.js";
+import {prettyPrintPeerId} from "../../util.js";
+
+// See https://github.com/ethereum/consensus-specs/pull/5181
+export async function* onBeaconBlocksByHead(
+  request: fulu.BeaconBlocksByHeadRequest,
+  chain: IBeaconChain,
+  peerId: PeerId,
+  peerClient: string
+): AsyncIterable<ResponseOutgoing> {
+  const currentFork = chain.config.getForkName(chain.clock.currentSlot);
+  const {beaconRoot, count} = validateBeaconBlocksByHeadRequest(currentFork, chain.config, request);
+
+  const requestedRootHex = toRootHex(beaconRoot);
+  let blockRootHex = requestedRootHex;
+  const minimumRequestEpoch = Math.max(
+    GENESIS_EPOCH,
+    chain.clock.currentEpoch - chain.config.MIN_EPOCHS_FOR_BLOCK_REQUESTS
+  );
+  const minimumRequestSlot = computeStartSlotAtEpoch(minimumRequestEpoch);
+
+  for (let blocksSent = 0; blocksSent < count; blocksSent++) {
+    const blockBytes = await chain.getSerializedBlockByRoot(blockRootHex);
+    if (!blockBytes) {
+      if (blocksSent === 0) {
+        throw new ResponseError(RespStatus.RESOURCE_UNAVAILABLE, `Unknown block root ${requestedRootHex}`);
+      }
+      return;
+    }
+
+    if (blockBytes.slot < minimumRequestSlot) {
+      if (blocksSent === 0) {
+        chain.logger.verbose("Peer requested unavailable block for BeaconBlocksByHead", {
+          peer: prettyPrintPeerId(peerId),
+          client: peerClient,
+          requestedRoot: requestedRootHex,
+          slot: blockBytes.slot,
+          minimumRequestSlot,
+        });
+      }
+      return;
+    }
+
+    yield {
+      data: blockBytes.block,
+      boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(blockBytes.slot)),
+    };
+
+    if (blockBytes.slot === GENESIS_SLOT) {
+      return;
+    }
+
+    const parentRootHex = getParentRootFromSignedBeaconBlockSerialized(blockBytes.block);
+    if (parentRootHex === null) {
+      throw new ResponseError(
+        RespStatus.SERVER_ERROR,
+        `Invalid block bytes for root ${blockRootHex} slot ${blockBytes.slot}`
+      );
+    }
+    blockRootHex = parentRootHex;
+  }
+}
+
+export function validateBeaconBlocksByHeadRequest(
+  fork: ForkName,
+  config: BeaconConfig,
+  request: fulu.BeaconBlocksByHeadRequest
+): fulu.BeaconBlocksByHeadRequest {
+  const {beaconRoot} = request;
+  let {count} = request;
+
+  if (count < 1) {
+    throw new ResponseError(RespStatus.INVALID_REQUEST, "count < 1");
+  }
+
+  const maxRequestBlocks = isForkPostDeneb(fork) ? config.MAX_REQUEST_BLOCKS_DENEB : config.MAX_REQUEST_BLOCKS;
+
+  if (count > maxRequestBlocks) {
+    count = maxRequestBlocks;
+  }
+
+  return {beaconRoot, count};
+}

package/src/network/reqresp/handlers/index.ts

@@ -9,6 +9,7 @@ import {
   ExecutionPayloadEnvelopesByRootRequestType,
 } from "../../../util/types.js";
 import {GetReqRespHandlerFn, ReqRespMethod} from "../types.js";
+import {onBeaconBlocksByHead} from "./beaconBlocksByHead.js";
 import {onBeaconBlocksByRange} from "./beaconBlocksByRange.js";
 import {onBeaconBlocksByRoot} from "./beaconBlocksByRoot.js";
 import {onBlobSidecarsByRange} from "./blobSidecarsByRange.js";
@@ -47,6 +48,10 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
       const body = BeaconBlocksByRootRequestType(fork, chain.config).deserialize(req.data);
       return onBeaconBlocksByRoot(body, chain);
     },
+    [ReqRespMethod.BeaconBlocksByHead]: (req, peerId, peerClient) => {
+      const body = ssz.fulu.BeaconBlocksByHeadRequest.deserialize(req.data);
+      return onBeaconBlocksByHead(body, chain, peerId, peerClient);
+    },
     [ReqRespMethod.BlobSidecarsByRoot]: (req) => {
       const fork = chain.config.getForkName(chain.clock.currentSlot);
       const body = BlobSidecarsByRootRequestType(fork, chain.config).deserialize(req.data);

package/src/network/reqresp/interface.ts

@@ -33,7 +33,7 @@ export enum RespStatus {
    */
   SERVER_ERROR = 2,
   /**
-   * The responder does not have requested resource.  The response payload adheres to the ErrorMessage schema (described below). Note: This response code is only valid as a response to BlocksByRange
+   * The responder does not have requested resource. The response payload adheres to the ErrorMessage schema.
    */
   RESOURCE_UNAVAILABLE = 3,
   /**

package/src/network/reqresp/protocols.ts

@@ -70,6 +70,12 @@ export const BeaconBlocksByRootV2 = toProtocol({
   contextBytesType: ContextBytesType.ForkDigest,
 });
 
+export const BeaconBlocksByHead = toProtocol({
+  method: ReqRespMethod.BeaconBlocksByHead,
+  version: Version.V1,
+  contextBytesType: ContextBytesType.ForkDigest,
+});
+
 export const BlobSidecarsByRange = toProtocol({
   method: ReqRespMethod.BlobSidecarsByRange,
   version: Version.V1,

package/src/network/reqresp/rateLimit.ts

@@ -40,6 +40,10 @@ export const rateLimitQuotas: (fork: ForkName, config: BeaconConfig) => Record<R
     },
     getRequestCount: getRequestCountFn(fork, config, ReqRespMethod.BeaconBlocksByRoot, (req) => req.length),
   },
+  [ReqRespMethod.BeaconBlocksByHead]: {
+    byPeer: {quota: config.MAX_REQUEST_BLOCKS_DENEB, quotaTimeMs: 10_000},
+    getRequestCount: getRequestCountFn(fork, config, ReqRespMethod.BeaconBlocksByHead, (req) => req.count),
+  },
   [ReqRespMethod.BlobSidecarsByRange]: {
     // Rationale: MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK
     byPeer: {

package/src/network/reqresp/score.ts

@@ -46,6 +46,7 @@ export function onOutgoingReqRespError(e: RequestError, method: ReqRespMethod):
           return PeerAction.LowToleranceError;
         case ReqRespMethod.BeaconBlocksByRange:
         case ReqRespMethod.BeaconBlocksByRoot:
+        case ReqRespMethod.BeaconBlocksByHead:
         case ReqRespMethod.ExecutionPayloadEnvelopesByRoot:
         case ReqRespMethod.ExecutionPayloadEnvelopesByRange:
           return PeerAction.MidToleranceError;

package/src/network/reqresp/types.ts

@@ -42,6 +42,7 @@ export enum ReqRespMethod {
   Metadata = "metadata",
   BeaconBlocksByRange = "beacon_blocks_by_range",
   BeaconBlocksByRoot = "beacon_blocks_by_root",
+  BeaconBlocksByHead = "beacon_blocks_by_head",
   BlobSidecarsByRange = "blob_sidecars_by_range",
   BlobSidecarsByRoot = "blob_sidecars_by_root",
   DataColumnSidecarsByRange = "data_column_sidecars_by_range",
@@ -62,6 +63,7 @@ export type RequestBodyByMethod = {
   [ReqRespMethod.Metadata]: null;
   [ReqRespMethod.BeaconBlocksByRange]: phase0.BeaconBlocksByRangeRequest;
   [ReqRespMethod.BeaconBlocksByRoot]: BeaconBlocksByRootRequest;
+  [ReqRespMethod.BeaconBlocksByHead]: fulu.BeaconBlocksByHeadRequest;
   [ReqRespMethod.BlobSidecarsByRange]: deneb.BlobSidecarsByRangeRequest;
   [ReqRespMethod.BlobSidecarsByRoot]: BlobSidecarsByRootRequest;
   [ReqRespMethod.DataColumnSidecarsByRange]: fulu.DataColumnSidecarsByRangeRequest;
@@ -82,6 +84,7 @@ type ResponseBodyByMethod = {
   // Do not matter
   [ReqRespMethod.BeaconBlocksByRange]: SignedBeaconBlock;
   [ReqRespMethod.BeaconBlocksByRoot]: SignedBeaconBlock;
+  [ReqRespMethod.BeaconBlocksByHead]: SignedBeaconBlock;
   [ReqRespMethod.BlobSidecarsByRange]: deneb.BlobSidecar;
   [ReqRespMethod.BlobSidecarsByRoot]: deneb.BlobSidecar;
   [ReqRespMethod.DataColumnSidecarsByRange]: DataColumnSidecar;
@@ -111,6 +114,7 @@ export const requestSszTypeByMethod: (
 
   [ReqRespMethod.BeaconBlocksByRange]: ssz.phase0.BeaconBlocksByRangeRequest,
   [ReqRespMethod.BeaconBlocksByRoot]: BeaconBlocksByRootRequestType(fork, config),
+  [ReqRespMethod.BeaconBlocksByHead]: ssz.fulu.BeaconBlocksByHeadRequest,
   [ReqRespMethod.BlobSidecarsByRange]: ssz.deneb.BlobSidecarsByRangeRequest,
   [ReqRespMethod.BlobSidecarsByRoot]: BlobSidecarsByRootRequestType(fork, config),
   [ReqRespMethod.DataColumnSidecarsByRange]: ssz.fulu.DataColumnSidecarsByRangeRequest,
@@ -142,6 +146,7 @@ export const responseSszTypeByMethod: {[K in ReqRespMethod]: ResponseTypeGetter<
     version === Version.V1 ? ssz.phase0.Metadata : version === Version.V2 ? ssz.altair.Metadata : ssz.fulu.Metadata,
   [ReqRespMethod.BeaconBlocksByRange]: blocksResponseType,
   [ReqRespMethod.BeaconBlocksByRoot]: blocksResponseType,
+  [ReqRespMethod.BeaconBlocksByHead]: (fork) => ssz[fork].SignedBeaconBlock,
   [ReqRespMethod.BlobSidecarsByRange]: () => ssz.deneb.BlobSidecar,
   [ReqRespMethod.BlobSidecarsByRoot]: () => ssz.deneb.BlobSidecar,
   [ReqRespMethod.LightClientBootstrap]: (fork) => sszTypesFor(onlyPostAltairFork(fork)).LightClientBootstrap,

package/src/sync/constants.ts

@@ -7,7 +7,11 @@ export const MIN_FINALIZED_CHAIN_VALIDATED_EPOCHS = 10;
 /** The number of times to retry a batch before it is considered failed. */
 export const MAX_BATCH_DOWNLOAD_ATTEMPTS = 5;
 
-/** Backoff before assigning more range-sync batches to a peer that rate-limited us. */
+/**
+ * Backoff before assigning more range-sync batches to a peer that rate-limited us.
+ *
+ * Note: this is used when rate limited due to MAX_CONCURRENT_REQUESTS
+ */
 export const RATE_LIMITED_PEER_BACKOFF_MS = 5_000;
 
 /**

package/src/sync/range/batch.ts

@@ -45,6 +45,15 @@ export type Attempt = {
   hash: RootHex;
 };
 
+type TrackedRequest = {
+  /** only happen for the 1st batch in checkpoint sync */
+  parentPayload: boolean;
+  /**
+   * we always issue by_range before parent_payload, so we don't model this as null
+   */
+  byRangeColumns: Set<number>;
+};
+
 export type AwaitingDownloadState = {
   status: BatchStatus.AwaitingDownload;
   blocks: IBlockInput[];
@@ -62,6 +71,7 @@ export type BatchState =
   | {
       status: BatchStatus.Downloading;
       peer: PeerIdStr;
+      request: TrackedRequest;
       blocks: IBlockInput[];
       payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null;
     }
@@ -110,6 +120,13 @@ function formatColumnsReq(req: {startSlot: Slot; count: number; columns: number[
   return `startSlot=${req.startSlot},count=${req.count},cols=${prettyPrintIndices(req.columns)}`;
 }
 
+function getTrackedRequest({parentPayloadRequest, columnsRequest}: DownloadByRangeRequests): TrackedRequest {
+  return {
+    parentPayload: parentPayloadRequest != null,
+    byRangeColumns: new Set(parentPayloadRequest == null ? (columnsRequest?.columns ?? []) : []),
+  };
+}
+
 /**
  * Batches are downloaded at the first block of the epoch.
  *
@@ -131,8 +148,8 @@ export class Batch {
   requests: DownloadByRangeRequests;
   /** State of the batch. */
   state: BatchState = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
-  /** Peers that provided good data */
-  goodPeers: PeerIdStr[] = [];
+  /** Peers that provided good data, with column coverage for by_range requests */
+  private readonly successfulDownloads = new Map<PeerIdStr, TrackedRequest>();
   /** The `Attempts` that have been made and failed to send us this batch. */
   readonly failedProcessingAttempts: Attempt[] = [];
   /** The `Attempts` that have been made and failed because of execution malfunction. */
@@ -406,6 +423,35 @@ export class Batch {
     return [...this.failedDownloadAttempts, ...this.failedProcessingAttempts.flatMap((a) => a.peers)];
   }
 
+  /**
+   * True only if the peer has already returned a successful response for the current request.
+   * A by_range success may update `this.requests` to parent_payload, and the same peer is then
+   * still eligible for the newly discovered parent payload data.
+   * For by_range, a peer that previously succeeded with a superset of requested columns is skipped.
+   */
+  hasPeerSucceededCurrentRequest(peer: PeerSyncMeta): boolean {
+    const successfulDownload = this.successfulDownloads.get(peer.peerId);
+    if (successfulDownload == null) return false;
+
+    const request = getTrackedRequest(this.getRequestsForPeer(peer));
+    if (request.parentPayload) return successfulDownload.parentPayload;
+
+    const requestByRangeColumns = request.byRangeColumns;
+
+    if (requestByRangeColumns.size === 0) {
+      // this means a download blocks/envelops by_range only
+      // don't do that again if we already did it
+      // see https://github.com/ChainSafe/lodestar/issues/9357
+      return true;
+    }
+
+    return [...requestByRangeColumns].every((column) => successfulDownload.byRangeColumns.has(column));
+  }
+
+  private getSuccessfulPeers(): PeerIdStr[] {
+    return Array.from(this.successfulDownloads.keys());
+  }
+
   getMetadata(): BatchMetadata {
     const {blocksRequest, blobsRequest, columnsRequest, envelopesRequest} = this.requests;
     const failedProcessingPeerList = this.failedProcessingAttempts.flatMap((a) => a.peers);
@@ -440,14 +486,17 @@ export class Batch {
   /**
    * AwaitingDownload -> Downloading
    */
-  startDownloading(peer: PeerIdStr): void {
+  startDownloading(peer: PeerSyncMeta): void {
     if (this.state.status !== BatchStatus.AwaitingDownload) {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.AwaitingDownload));
     }
 
+    const request = getTrackedRequest(this.getRequestsForPeer(peer));
+
     this.state = {
       status: BatchStatus.Downloading,
-      peer,
+      peer: peer.peerId,
+      request,
       blocks: this.state.blocks,
       payloadEnvelopes: this.state.payloadEnvelopes,
     };
@@ -468,7 +517,17 @@ export class Batch {
     // ensure that blocks are always sorted before getting stored on the batch.state or being used to getRequests
     blocks.sort((a, b) => a.slot - b.slot);
 
-    this.goodPeers.push(peer);
+    const successfulDownload = this.successfulDownloads.get(peer) ?? {
+      parentPayload: false,
+      byRangeColumns: new Set<number>(),
+    };
+    successfulDownload.parentPayload ||= this.state.request.parentPayload;
+    if (!this.state.request.parentPayload) {
+      for (const column of this.state.request.byRangeColumns) {
+        successfulDownload.byRangeColumns.add(column);
+      }
+    }
+    this.successfulDownloads.set(peer, successfulDownload);
 
     let allComplete = true;
     const slots = new Set<number>();
@@ -497,8 +556,9 @@ export class Batch {
     if (allComplete && isForkPostGloas(this.forkName)) {
       for (const block of blocks) {
         const payloadInput = newPayloadEnvelopes?.get(block.slot);
-        // by_range needs every block's envelope and all sampled columns.
-        if (!payloadInput?.hasPayloadEnvelope() || !payloadInput.hasComputedAllData()) {
+        // only need to make sure envelope has all columns, not all blocks have payload
+        // assertLinearChainSegment() was called before reaching this
+        if (payloadInput?.hasPayloadEnvelope() && !payloadInput.hasComputedAllData()) {
           allComplete = false;
           break;
         }
@@ -589,10 +649,10 @@ export class Batch {
     const blocks = this.state.blocks;
     const payloadEnvelopes = this.state.payloadEnvelopes;
     const hash = hashBlocks(blocks, this.config); // tracks blocks to report peer on processing error
-    // Reset goodPeers in case another download attempt needs to be made.  When Attempt is successful or not the peers
-    // that the data came from will be handled by the Attempt that goes for processing
-    const peers = this.goodPeers;
-    this.goodPeers = [];
+    // Reset successfulDownloads in case another download attempt needs to be made. When Attempt is successful or not
+    // the peers that the data came from will be handled by the Attempt that goes for processing.
+    const peers = this.getSuccessfulPeers();
+    this.successfulDownloads.clear();
     this.state = {status: BatchStatus.Processing, blocks, payloadEnvelopes, attempt: {peers, hash}};
     return {blocks, payloadEnvelopes, peers};
   }

package/src/sync/range/chain.ts

@@ -1,5 +1,4 @@
 import {ChainForkConfig} from "@lodestar/config";
-import {RequestErrorCode} from "@lodestar/reqresp";
 import {Epoch, Root, Slot, gloas} from "@lodestar/types";
 import {ErrorAborted, LodestarError, Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js";
@@ -16,13 +15,9 @@ import {CustodyConfig} from "../../util/dataColumns.js";
 import {ItTrigger} from "../../util/itTrigger.js";
 import {PeerIdStr} from "../../util/peerId.js";
 import {WarnResult, wrapError} from "../../util/wrapError.js";
-import {
-  BATCH_BUFFER_SIZE,
-  EPOCHS_PER_BATCH,
-  MAX_LOOK_AHEAD_EPOCHS,
-  RATE_LIMITED_PEER_BACKOFF_MS,
-} from "../constants.js";
+import {BATCH_BUFFER_SIZE, EPOCHS_PER_BATCH, MAX_LOOK_AHEAD_EPOCHS} from "../constants.js";
 import {DownloadByRangeError, DownloadByRangeErrorCode} from "../utils/downloadByRange.js";
+import {getRateLimitedUntilMs} from "../utils/rateLimit.js";
 import {RangeSyncType} from "../utils/remoteSyncType.js";
 import {Batch, BatchError, BatchErrorCode, BatchMetadata, BatchStatus} from "./batch.js";
 import {
@@ -156,6 +151,7 @@ export class SyncChain {
    * The reqresp SelfRateLimiter independently enforces backoff at the protocol level as a safety net.
    */
   private readonly rateLimitedPeers = new Map<PeerIdStr, number>();
+  private rateLimitBackoffTimeout: NodeJS.Timeout | undefined;
 
   private readonly logger: Logger;
   private readonly config: ChainForkConfig;
@@ -241,6 +237,7 @@ export class SyncChain {
    */
   stopSyncing(): void {
     this.status = SyncChainStatus.Stopped;
+    this.clearRateLimitBackoffTimer();
     this.logger.debug("SyncChain stopSyncing", {id: this.logId});
   }
 
@@ -249,6 +246,7 @@ export class SyncChain {
    */
   remove(): void {
     this.logger.debug("SyncChain remove", {id: this.logId});
+    this.clearRateLimitBackoffTimer();
     this.batchProcessor.end(new ErrorAborted("SyncChain"));
   }
 
@@ -371,6 +369,8 @@ export class SyncChain {
       }
 
       throw e;
+    } finally {
+      this.clearRateLimitBackoffTimer();
     }
   }
 
@@ -394,6 +394,44 @@ export class SyncChain {
     }
   }
 
+  private scheduleRateLimitBackoffRetry(): void {
+    this.clearRateLimitBackoffTimer();
+
+    if (this.status !== SyncChainStatus.Syncing || this.rateLimitedPeers.size === 0) {
+      return;
+    }
+
+    const now = Date.now();
+    let retryAt: number | null = null;
+    for (const [peerId, rateLimitedUntil] of this.rateLimitedPeers.entries()) {
+      if (rateLimitedUntil <= now) {
+        this.rateLimitedPeers.delete(peerId);
+        continue;
+      }
+      retryAt = Math.min(retryAt ?? rateLimitedUntil, rateLimitedUntil);
+    }
+
+    if (retryAt === null) {
+      return;
+    }
+
+    this.rateLimitBackoffTimeout = setTimeout(
+      () => {
+        this.rateLimitBackoffTimeout = undefined;
+        this.triggerBatchDownloader();
+        this.scheduleRateLimitBackoffRetry();
+      },
+      Math.max(0, retryAt - now)
+    );
+  }
+
+  private clearRateLimitBackoffTimer(): void {
+    if (this.rateLimitBackoffTimeout !== undefined) {
+      clearTimeout(this.rateLimitBackoffTimeout);
+      this.rateLimitBackoffTimeout = undefined;
+    }
+  }
+
   /**
    * Attempts to request the next required batches from the peer pool if the chain is syncing.
    * It will exhaust the peer pool and left over batches until the batch buffer is reached.
@@ -513,7 +551,7 @@ export class SyncChain {
       peer: prettyPrintPeerIdStr(peer.peerId),
     });
     try {
-      batch.startDownloading(peer.peerId);
+      batch.startDownloading(peer);
 
       // wrapError ensures to never call both batch success() and batch error()
       const res = await wrapError(this.downloadByRange(peer, batch, this.syncType));
@@ -543,6 +581,8 @@ export class SyncChain {
           case DownloadByRangeErrorCode.OUT_OF_ORDER_BLOCKS:
           case DownloadByRangeErrorCode.OUT_OF_RANGE_BLOCKS:
           case DownloadByRangeErrorCode.PARENT_ROOT_MISMATCH:
+          case DownloadByRangeErrorCode.INVALID_ENVELOPE_BEACON_BLOCK_ROOT:
+          case DownloadByRangeErrorCode.INVALID_CHAIN_SEGMENT:
           case BlobSidecarErrorCode.INCLUSION_PROOF_INVALID:
           case BlobSidecarErrorCode.INVALID_KZG_PROOF_BATCH:
           case DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT:
@@ -556,9 +596,11 @@ export class SyncChain {
           {id: this.logId, ...batch.getMetadata(), peer: prettyPrintPeerIdStr(peer.peerId)},
           res.err
         );
-        if (errCode === RequestErrorCode.RESP_RATE_LIMITED || errCode === RequestErrorCode.REQUEST_SELF_RATE_LIMITED) {
+        const rateLimitedUntilMs = getRateLimitedUntilMs(res.err);
+        if (rateLimitedUntilMs !== null) {
           // Peer rate-limited us — don't count as a failed download attempt and mark peer for backoff
-          this.rateLimitedPeers.set(peer.peerId, Date.now() + RATE_LIMITED_PEER_BACKOFF_MS);
+          this.rateLimitedPeers.set(peer.peerId, rateLimitedUntilMs);
+          this.scheduleRateLimitBackoffRetry();
           batch.downloadingRateLimited();
           this.triggerBatchDownloader();
         } else {