@lodestar/beacon-node 1.43.0-dev.b05ea98d04 → 1.43.0-dev.b0e1143d06

package/src/sync/range/batch.ts

@@ -1,18 +1,19 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkName, isForkPostDeneb, isForkPostFulu, isForkPostGloas} from "@lodestar/params";
-import {Epoch, RootHex, Slot, phase0} from "@lodestar/types";
-import {LodestarError, prettyPrintIndices} from "@lodestar/utils";
+import {Epoch, RootHex, SignedBeaconBlock, Slot, gloas, phase0} from "@lodestar/types";
+import {LodestarError, byteArrayEquals, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
 import {isDaOutOfRange} from "../../chain/blocks/blockInput/utils.js";
 import {PayloadEnvelopeInput} from "../../chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js";
 import {BlockError, BlockErrorCode} from "../../chain/errors/index.js";
+import {ZERO_HASH} from "../../constants/constants.js";
 import {PeerSyncMeta} from "../../network/peers/peersData.js";
 import {IClock} from "../../util/clock.js";
 import {CustodyConfig} from "../../util/dataColumns.js";
 import {PeerIdStr} from "../../util/peerId.js";
 import {MAX_BATCH_DOWNLOAD_ATTEMPTS, MAX_BATCH_PROCESSING_ATTEMPTS} from "../constants.js";
-import {DownloadByRangeRequests} from "../utils/downloadByRange.js";
+import {DownloadByRangeRequests, ParentPayloadCommitments} from "../utils/downloadByRange.js";
 import {getBatchSlotRange, hashBlocks} from "./utils/index.js";
 
 /**
@@ -44,6 +45,15 @@ export type Attempt = {
   hash: RootHex;
 };
 
+type TrackedRequest = {
+  /** only happen for the 1st batch in checkpoint sync */
+  parentPayload: boolean;
+  /**
+   * we always issue by_range before parent_payload, so we don't model this as null
+   */
+  byRangeColumns: Set<number>;
+};
+
 export type AwaitingDownloadState = {
   status: BatchStatus.AwaitingDownload;
   blocks: IBlockInput[];
@@ -61,6 +71,7 @@ export type BatchState =
   | {
       status: BatchStatus.Downloading;
       peer: PeerIdStr;
+      request: TrackedRequest;
       blocks: IBlockInput[];
       payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null;
     }
@@ -109,6 +120,13 @@ function formatColumnsReq(req: {startSlot: Slot; count: number; columns: number[
   return `startSlot=${req.startSlot},count=${req.count},cols=${prettyPrintIndices(req.columns)}`;
 }
 
+function getTrackedRequest({parentPayloadRequest, columnsRequest}: DownloadByRangeRequests): TrackedRequest {
+  return {
+    parentPayload: parentPayloadRequest != null,
+    byRangeColumns: new Set(parentPayloadRequest == null ? (columnsRequest?.columns ?? []) : []),
+  };
+}
+
 /**
  * Batches are downloaded at the first block of the epoch.
  *
@@ -130,8 +148,8 @@ export class Batch {
   requests: DownloadByRangeRequests;
   /** State of the batch. */
   state: BatchState = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
-  /** Peers that provided good data */
-  goodPeers: PeerIdStr[] = [];
+  /** Peers that provided good data, with column coverage for by_range requests */
+  private readonly successfulDownloads = new Map<PeerIdStr, TrackedRequest>();
   /** The `Attempts` that have been made and failed to send us this batch. */
   readonly failedProcessingAttempts: Attempt[] = [];
   /** The `Attempts` that have been made and failed because of execution malfunction. */
@@ -141,8 +159,18 @@ export class Batch {
   private readonly config: ChainForkConfig;
   private readonly clock: IClock;
   private readonly custodyConfig: CustodyConfig;
-
-  constructor(startEpoch: Epoch, config: ChainForkConfig, clock: IClock, custodyConfig: CustodyConfig) {
+  private readonly isFirstBatchInChain: boolean;
+  private readonly latestBid: gloas.ExecutionPayloadBid | undefined;
+
+  constructor(
+    startEpoch: Epoch,
+    config: ChainForkConfig,
+    clock: IClock,
+    custodyConfig: CustodyConfig,
+    isFirstBatchInChain: boolean,
+    latestBid: gloas.ExecutionPayloadBid | undefined,
+    targetSlot: Slot
+  ) {
     this.config = config;
     this.clock = clock;
     this.custodyConfig = custodyConfig;
@@ -151,10 +179,40 @@ export class Batch {
     this.forkName = this.config.getForkName(startSlot);
     this.startEpoch = startEpoch;
     this.startSlot = startSlot;
-    this.count = count;
+    this.count = Math.min(count, targetSlot - startSlot + 1);
+    this.isFirstBatchInChain = isFirstBatchInChain;
+    this.latestBid = latestBid;
     this.requests = this.getRequests([]);
   }
 
+  private shouldDownloadParentEnvelope(firstBlock?: SignedBeaconBlock): boolean {
+    if (!this.isFirstBatchInChain) return false;
+
+    if (this.startSlot === 0 || !isForkPostGloas(this.config.getForkName(this.startSlot - 1))) {
+      return false;
+    }
+
+    // we only know if we should download parent envelope if firstBlock is downloaded
+    if (firstBlock === undefined) return false;
+    if (this.latestBid === undefined) return false;
+    const firstBlockBidParentHash = (firstBlock.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message
+      .parentBlockHash;
+    return byteArrayEquals(firstBlockBidParentHash, this.latestBid.blockHash);
+  }
+
+  getParentPayloadCommitments(parentBlockRoot: Uint8Array): ParentPayloadCommitments {
+    if (this.latestBid === undefined) {
+      throw new Error(
+        `Coding error: getParentPayloadCommitments called without latestBid for parentBlockRoot=${toRootHex(parentBlockRoot)}`
+      );
+    }
+    return {
+      blockRoot: parentBlockRoot,
+      blockRootHex: toRootHex(parentBlockRoot),
+      kzgCommitments: this.latestBid.blobKzgCommitments,
+    };
+  }
+
   /**
    * Builds ByRange requests for block, blobs and columns
    */
@@ -292,6 +350,36 @@ export class Batch {
       };
     }
 
+    // Only the first batch of a SyncChain may need the dangling-parent payload by-root.
+    if (blocks.length > 0 && this.shouldDownloadParentEnvelope(blocks[0].getBlock())) {
+      // shouldDownloadParentEnvelope() = true means there are at least 1 block
+      const parentRoot = blocks[0].getBlock().message.parentRoot;
+      if (!byteArrayEquals(parentRoot, ZERO_HASH)) {
+        const parentRootHex = toRootHex(parentRoot);
+        let parentPayloadInput: PayloadEnvelopeInput | undefined;
+        if (this.state.payloadEnvelopes) {
+          for (const pi of this.state.payloadEnvelopes.values()) {
+            if (pi.blockRootHex === parentRootHex) {
+              parentPayloadInput = pi;
+              break;
+            }
+          }
+        }
+
+        const needsEnvelope = !parentPayloadInput?.hasPayloadEnvelope();
+        const missingColumns = parentPayloadInput
+          ? parentPayloadInput.getMissingSampledColumnMeta().missing
+          : this.custodyConfig.sampledColumns;
+
+        if (needsEnvelope || missingColumns.length > 0) {
+          requests.parentPayloadRequest = {
+            ...(needsEnvelope ? {envelopeBlockRoot: parentRoot} : {}),
+            ...(missingColumns.length > 0 ? {blockRoot: parentRoot, columns: missingColumns} : {}),
+          };
+        }
+      }
+    }
+
     return requests;
   }
 
@@ -303,24 +391,28 @@ export class Batch {
       return this.requests;
     }
 
-    // post-fulu we need to ensure that we only request columns that the peer has advertised
-    const {columnsRequest} = this.requests;
-    if (columnsRequest == null) {
-      return this.requests;
-    }
+    // post-fulu we need to ensure that we only request columns that the peer has advertised.
+    const {columnsRequest, parentPayloadRequest} = this.requests;
 
     const peerColumns = new Set(peer.custodyColumns ?? []);
-    const requestedColumns = columnsRequest.columns.filter((c) => peerColumns.has(c));
-    if (requestedColumns.length === columnsRequest.columns.length) {
-      return this.requests;
-    }
+    const filteredColumnsRequest =
+      columnsRequest != null ? columnsRequest.columns.filter((c) => peerColumns.has(c)) : null;
+    const parentColumns = parentPayloadRequest?.columns;
+    const filteredParentColumns = parentColumns != null ? parentColumns.filter((c) => peerColumns.has(c)) : null;
+
+    const updatedColumnRequest =
+      columnsRequest != null && filteredColumnsRequest != null
+        ? {columnsRequest: {...columnsRequest, columns: filteredColumnsRequest}}
+        : {};
+    const updatedParentPayloadRequest =
+      parentPayloadRequest != null && filteredParentColumns != null
+        ? {parentPayloadRequest: {...parentPayloadRequest, columns: filteredParentColumns}}
+        : {};
 
     return {
       ...this.requests,
-      columnsRequest: {
-        ...columnsRequest,
-        columns: requestedColumns,
-      },
+      ...updatedColumnRequest,
+      ...updatedParentPayloadRequest,
     };
   }
 
@@ -331,6 +423,35 @@ export class Batch {
     return [...this.failedDownloadAttempts, ...this.failedProcessingAttempts.flatMap((a) => a.peers)];
   }
 
+  /**
+   * True only if the peer has already returned a successful response for the current request.
+   * A by_range success may update `this.requests` to parent_payload, and the same peer is then
+   * still eligible for the newly discovered parent payload data.
+   * For by_range, a peer that previously succeeded with a superset of requested columns is skipped.
+   */
+  hasPeerSucceededCurrentRequest(peer: PeerSyncMeta): boolean {
+    const successfulDownload = this.successfulDownloads.get(peer.peerId);
+    if (successfulDownload == null) return false;
+
+    const request = getTrackedRequest(this.getRequestsForPeer(peer));
+    if (request.parentPayload) return successfulDownload.parentPayload;
+
+    const requestByRangeColumns = request.byRangeColumns;
+
+    if (requestByRangeColumns.size === 0) {
+      // this means a download blocks/envelops by_range only
+      // don't do that again if we already did it
+      // see https://github.com/ChainSafe/lodestar/issues/9357
+      return true;
+    }
+
+    return [...requestByRangeColumns].every((column) => successfulDownload.byRangeColumns.has(column));
+  }
+
+  private getSuccessfulPeers(): PeerIdStr[] {
+    return Array.from(this.successfulDownloads.keys());
+  }
+
   getMetadata(): BatchMetadata {
     const {blocksRequest, blobsRequest, columnsRequest, envelopesRequest} = this.requests;
     const failedProcessingPeerList = this.failedProcessingAttempts.flatMap((a) => a.peers);
@@ -365,14 +486,17 @@ export class Batch {
   /**
    * AwaitingDownload -> Downloading
    */
-  startDownloading(peer: PeerIdStr): void {
+  startDownloading(peer: PeerSyncMeta): void {
     if (this.state.status !== BatchStatus.AwaitingDownload) {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.AwaitingDownload));
     }
 
+    const request = getTrackedRequest(this.getRequestsForPeer(peer));
+
     this.state = {
       status: BatchStatus.Downloading,
-      peer,
+      peer: peer.peerId,
+      request,
       blocks: this.state.blocks,
       payloadEnvelopes: this.state.payloadEnvelopes,
     };
@@ -393,7 +517,17 @@ export class Batch {
     // ensure that blocks are always sorted before getting stored on the batch.state or being used to getRequests
     blocks.sort((a, b) => a.slot - b.slot);
 
-    this.goodPeers.push(peer);
+    const successfulDownload = this.successfulDownloads.get(peer) ?? {
+      parentPayload: false,
+      byRangeColumns: new Set<number>(),
+    };
+    successfulDownload.parentPayload ||= this.state.request.parentPayload;
+    if (!this.state.request.parentPayload) {
+      for (const column of this.state.request.byRangeColumns) {
+        successfulDownload.byRangeColumns.add(column);
+      }
+    }
+    this.successfulDownloads.set(peer, successfulDownload);
 
     let allComplete = true;
     const slots = new Set<number>();
@@ -422,14 +556,38 @@ export class Batch {
     if (allComplete && isForkPostGloas(this.forkName)) {
       for (const block of blocks) {
         const payloadInput = newPayloadEnvelopes?.get(block.slot);
-        // by_range needs to download all columns
-        if (!payloadInput?.hasPayloadEnvelope() || !payloadInput.hasComputedAllData()) {
+        // only need to make sure envelope has all columns, not all blocks have payload
+        // assertLinearChainSegment() was called before reaching this
+        if (payloadInput?.hasPayloadEnvelope() && !payloadInput.hasComputedAllData()) {
           allComplete = false;
           break;
         }
       }
     }
 
+    // First batch of a sync chain must additionally have the dangling-parent payload fully
+    // present, otherwise `processBlocks` will throw PARENT_PAYLOAD_UNKNOWN. The parent's
+    // `PayloadEnvelopeInput` is identified by `blockRootHex` matching `blocks[0].parentRoot`.
+    if (allComplete && blocks.length > 0 && this.shouldDownloadParentEnvelope(blocks[0].getBlock())) {
+      const parentRoot = blocks[0].getBlock().message.parentRoot;
+      // Genesis has no parent payload — nothing to wait for.
+      if (!byteArrayEquals(parentRoot, ZERO_HASH)) {
+        const parentRootHex = toRootHex(parentRoot);
+        let parentPayloadComplete = false;
+        if (newPayloadEnvelopes) {
+          for (const payloadInput of newPayloadEnvelopes.values()) {
+            if (payloadInput.blockRootHex === parentRootHex) {
+              parentPayloadComplete = payloadInput.hasPayloadEnvelope() && payloadInput.hasComputedAllData();
+              break;
+            }
+          }
+        }
+        if (!parentPayloadComplete) {
+          allComplete = false;
+        }
+      }
+    }
+
     if (allComplete) {
       this.state = {status: BatchStatus.AwaitingProcessing, blocks, payloadEnvelopes: newPayloadEnvelopes};
     } else {
@@ -491,10 +649,10 @@ export class Batch {
     const blocks = this.state.blocks;
     const payloadEnvelopes = this.state.payloadEnvelopes;
     const hash = hashBlocks(blocks, this.config); // tracks blocks to report peer on processing error
-    // Reset goodPeers in case another download attempt needs to be made.  When Attempt is successful or not the peers
-    // that the data came from will be handled by the Attempt that goes for processing
-    const peers = this.goodPeers;
-    this.goodPeers = [];
+    // Reset successfulDownloads in case another download attempt needs to be made. When Attempt is successful or not
+    // the peers that the data came from will be handled by the Attempt that goes for processing.
+    const peers = this.getSuccessfulPeers();
+    this.successfulDownloads.clear();
     this.state = {status: BatchStatus.Processing, blocks, payloadEnvelopes, attempt: {peers, hash}};
     return {blocks, payloadEnvelopes, peers};
   }

package/src/sync/range/chain.ts

@@ -1,6 +1,5 @@
 import {ChainForkConfig} from "@lodestar/config";
-import {RequestErrorCode} from "@lodestar/reqresp";
-import {Epoch, Root, Slot} from "@lodestar/types";
+import {Epoch, Root, Slot, gloas} from "@lodestar/types";
 import {ErrorAborted, LodestarError, Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js";
 import {BlockInputErrorCode} from "../../chain/blocks/blockInput/errors.js";
@@ -16,13 +15,9 @@ import {CustodyConfig} from "../../util/dataColumns.js";
 import {ItTrigger} from "../../util/itTrigger.js";
 import {PeerIdStr} from "../../util/peerId.js";
 import {WarnResult, wrapError} from "../../util/wrapError.js";
-import {
-  BATCH_BUFFER_SIZE,
-  EPOCHS_PER_BATCH,
-  MAX_LOOK_AHEAD_EPOCHS,
-  RATE_LIMITED_PEER_BACKOFF_MS,
-} from "../constants.js";
+import {BATCH_BUFFER_SIZE, EPOCHS_PER_BATCH, MAX_LOOK_AHEAD_EPOCHS} from "../constants.js";
 import {DownloadByRangeError, DownloadByRangeErrorCode} from "../utils/downloadByRange.js";
+import {getRateLimitedUntilMs} from "../utils/rateLimit.js";
 import {RangeSyncType} from "../utils/remoteSyncType.js";
 import {Batch, BatchError, BatchErrorCode, BatchMetadata, BatchStatus} from "./batch.js";
 import {
@@ -145,6 +140,10 @@ export class SyncChain {
   private readonly batchProcessor = new ItTrigger();
   /** Sorted map of batches undergoing some kind of processing. */
   private readonly batches = new Map<Epoch, Batch>();
+  /**
+   * `true` until the first `Batch` is constructed via `includeNextBatch`
+   */
+  private isFirstBatch = true;
   private readonly peerset = new Map<PeerIdStr, ChainTarget>();
   /**
    * Tracks peers that have rate-limited us, mapped to the timestamp (ms) until which we should avoid them.
@@ -152,19 +151,22 @@ export class SyncChain {
    * The reqresp SelfRateLimiter independently enforces backoff at the protocol level as a safety net.
    */
   private readonly rateLimitedPeers = new Map<PeerIdStr, number>();
+  private rateLimitBackoffTimeout: NodeJS.Timeout | undefined;
 
   private readonly logger: Logger;
   private readonly config: ChainForkConfig;
   private readonly clock: IClock;
   private readonly metrics: Metrics | null;
   private readonly custodyConfig: CustodyConfig;
+  private readonly latestBid: gloas.ExecutionPayloadBid | undefined;
 
   constructor(
     initialBatchEpoch: Epoch,
     initialTarget: ChainTarget,
     syncType: RangeSyncType,
     fns: SyncChainFns,
-    modules: SyncChainModules
+    modules: SyncChainModules,
+    latestBid: gloas.ExecutionPayloadBid | undefined
   ) {
     const {config, clock, custodyConfig, logger, metrics} = modules;
     this.firstBatchEpoch = initialBatchEpoch;
@@ -180,6 +182,7 @@ export class SyncChain {
     this.clock = clock;
     this.metrics = metrics;
     this.custodyConfig = custodyConfig;
+    this.latestBid = latestBid;
     this.logger = logger;
     this.logId = `${syncType}-${nextChainId++}`;
 
@@ -234,6 +237,7 @@ export class SyncChain {
    */
   stopSyncing(): void {
     this.status = SyncChainStatus.Stopped;
+    this.clearRateLimitBackoffTimer();
     this.logger.debug("SyncChain stopSyncing", {id: this.logId});
   }
 
@@ -242,6 +246,7 @@ export class SyncChain {
    */
   remove(): void {
     this.logger.debug("SyncChain remove", {id: this.logId});
+    this.clearRateLimitBackoffTimer();
     this.batchProcessor.end(new ErrorAborted("SyncChain"));
   }
 
@@ -364,6 +369,8 @@ export class SyncChain {
       }
 
       throw e;
+    } finally {
+      this.clearRateLimitBackoffTimer();
     }
   }
 
@@ -387,6 +394,44 @@ export class SyncChain {
     }
   }
 
+  private scheduleRateLimitBackoffRetry(): void {
+    this.clearRateLimitBackoffTimer();
+
+    if (this.status !== SyncChainStatus.Syncing || this.rateLimitedPeers.size === 0) {
+      return;
+    }
+
+    const now = Date.now();
+    let retryAt: number | null = null;
+    for (const [peerId, rateLimitedUntil] of this.rateLimitedPeers.entries()) {
+      if (rateLimitedUntil <= now) {
+        this.rateLimitedPeers.delete(peerId);
+        continue;
+      }
+      retryAt = Math.min(retryAt ?? rateLimitedUntil, rateLimitedUntil);
+    }
+
+    if (retryAt === null) {
+      return;
+    }
+
+    this.rateLimitBackoffTimeout = setTimeout(
+      () => {
+        this.rateLimitBackoffTimeout = undefined;
+        this.triggerBatchDownloader();
+        this.scheduleRateLimitBackoffRetry();
+      },
+      Math.max(0, retryAt - now)
+    );
+  }
+
+  private clearRateLimitBackoffTimer(): void {
+    if (this.rateLimitBackoffTimeout !== undefined) {
+      clearTimeout(this.rateLimitBackoffTimeout);
+      this.rateLimitBackoffTimeout = undefined;
+    }
+  }
+
   /**
    * Attempts to request the next required batches from the peer pool if the chain is syncing.
    * It will exhaust the peer pool and left over batches until the batch buffer is reached.
@@ -481,7 +526,17 @@ export class SyncChain {
       return null;
     }
 
-    const batch = new Batch(startEpoch, this.config, this.clock, this.custodyConfig);
+    const batch = new Batch(
+      startEpoch,
+      this.config,
+      this.clock,
+      this.custodyConfig,
+      this.isFirstBatch,
+      // `latestBid` is only meaningful for the first batch's parent-payload check
+      this.isFirstBatch ? this.latestBid : undefined,
+      this.target.slot
+    );
+    this.isFirstBatch = false;
     this.batches.set(startEpoch, batch);
     return batch;
   }
@@ -496,7 +551,7 @@ export class SyncChain {
       peer: prettyPrintPeerIdStr(peer.peerId),
     });
     try {
-      batch.startDownloading(peer.peerId);
+      batch.startDownloading(peer);
 
       // wrapError ensures to never call both batch success() and batch error()
       const res = await wrapError(this.downloadByRange(peer, batch, this.syncType));
@@ -526,6 +581,8 @@ export class SyncChain {
           case DownloadByRangeErrorCode.OUT_OF_ORDER_BLOCKS:
           case DownloadByRangeErrorCode.OUT_OF_RANGE_BLOCKS:
           case DownloadByRangeErrorCode.PARENT_ROOT_MISMATCH:
+          case DownloadByRangeErrorCode.INVALID_ENVELOPE_BEACON_BLOCK_ROOT:
+          case DownloadByRangeErrorCode.INVALID_CHAIN_SEGMENT:
           case BlobSidecarErrorCode.INCLUSION_PROOF_INVALID:
           case BlobSidecarErrorCode.INVALID_KZG_PROOF_BATCH:
           case DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT:
@@ -539,9 +596,11 @@ export class SyncChain {
           {id: this.logId, ...batch.getMetadata(), peer: prettyPrintPeerIdStr(peer.peerId)},
           res.err
         );
-        if (errCode === RequestErrorCode.RESP_RATE_LIMITED || errCode === RequestErrorCode.REQUEST_SELF_RATE_LIMITED) {
+        const rateLimitedUntilMs = getRateLimitedUntilMs(res.err);
+        if (rateLimitedUntilMs !== null) {
           // Peer rate-limited us — don't count as a failed download attempt and mark peer for backoff
-          this.rateLimitedPeers.set(peer.peerId, Date.now() + RATE_LIMITED_PEER_BACKOFF_MS);
+          this.rateLimitedPeers.set(peer.peerId, rateLimitedUntilMs);
+          this.scheduleRateLimitBackoffRetry();
           batch.downloadingRateLimited();
           this.triggerBatchDownloader();
         } else {

package/src/sync/range/range.ts

@@ -1,16 +1,23 @@
 import {EventEmitter} from "node:events";
 import {StrictEventEmitter} from "strict-event-emitter-types";
 import {BeaconConfig} from "@lodestar/config";
-import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {IBeaconStateViewGloas, computeStartSlotAtEpoch, isStatePostGloas} from "@lodestar/state-transition";
 import {Epoch, Status, fulu} from "@lodestar/types";
-import {Logger, toRootHex} from "@lodestar/utils";
+import {Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
 import {AttestationImportOpt, ImportBlockOpts} from "../../chain/blocks/index.js";
+import {assertLinearChainSegment} from "../../chain/blocks/utils/chainSegment.js";
+import {BlockError} from "../../chain/errors/index.js";
 import {IBeaconChain} from "../../chain/index.js";
 import {Metrics} from "../../metrics/index.js";
 import {INetwork} from "../../network/index.js";
 import {PeerIdStr} from "../../util/peerId.js";
-import {cacheByRangeResponses, downloadByRange} from "../utils/downloadByRange.js";
+import {
+  DownloadByRangeError,
+  DownloadByRangeErrorCode,
+  cacheByRangeResponses,
+  downloadByRange,
+} from "../utils/downloadByRange.js";
 import {RangeSyncType, getRangeSyncTarget, rangeSyncTypes} from "../utils/remoteSyncType.js";
 import {ChainTarget, SyncChain, SyncChainDebugState, SyncChainFns} from "./chain.js";
 import {updateChains} from "./utils/index.js";
@@ -206,14 +213,18 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
 
   private downloadByRange: SyncChainFns["downloadByRange"] = async (peer, batch) => {
     const batchBlocks = batch.getBlocks();
+    const requests = batch.getRequestsForPeer(peer);
+    const parentRoot = requests.parentPayloadRequest?.envelopeBlockRoot ?? requests.parentPayloadRequest?.blockRoot;
+    const parentPayloadCommitments = parentRoot ? batch.getParentPayloadCommitments(parentRoot) : undefined;
     const {result, warnings} = await downloadByRange({
       config: this.config,
       network: this.network,
       logger: this.logger,
       peerIdStr: peer.peerId,
       batchBlocks,
+      parentPayloadCommitments,
       peerDasMetrics: this.chain.metrics?.peerDas,
-      ...batch.getRequestsForPeer(peer),
+      ...requests,
     });
     const {responses, payloadEnvelopes: downloadedPayloadEnvelopes} = result;
     const {blocks, payloadEnvelopes} = cacheByRangeResponses({
@@ -227,6 +238,49 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
       custodyConfig: this.chain.custodyConfig,
       seenTimestampSec: Date.now() / 1000,
     });
+
+    const segmentBlocks = blocks.filter((b) => b.hasBlock()).sort((a, b) => a.slot - b.slot);
+    const envelopeSlots = payloadEnvelopes
+      ? Array.from(payloadEnvelopes.entries())
+          .filter(([, pi]) => pi.hasPayloadEnvelope())
+          .map(([slot]) => slot)
+          .sort((a, b) => a - b)
+      : [];
+    this.logger.verbose("downloadByRange batch ready", {
+      peer: peer.peerId,
+      blockSlots: prettyPrintIndices(segmentBlocks.map((b) => b.slot)),
+      envelopeSlots: prettyPrintIndices(envelopeSlots),
+      ...batch.getMetadata(),
+    });
+
+    if (segmentBlocks.length > 1) {
+      try {
+        assertLinearChainSegment(this.config, segmentBlocks, payloadEnvelopes, null);
+      } catch (err) {
+        if (err instanceof BlockError) {
+          this.logger.debug(
+            "downloadByRange segment validation failed",
+            {
+              peer: peer.peerId,
+              reason: err.type.code,
+              slot: err.signedBlock.message.slot,
+              detail: JSON.stringify(err.type),
+              ...batch.getMetadata(),
+            },
+            err
+          );
+          // with this error, the peer will be penalized inside SyncChain
+          throw new DownloadByRangeError({
+            code: DownloadByRangeErrorCode.INVALID_CHAIN_SEGMENT,
+            slot: err.signedBlock.message.slot,
+            reason: err.type.code,
+          });
+        }
+
+        throw err;
+      }
+    }
+
     return {result: {blocks, payloadEnvelopes}, warnings};
   };
 
@@ -258,6 +312,14 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
   private addPeerOrCreateChain(startEpoch: Epoch, target: ChainTarget, peer: PeerIdStr, syncType: RangeSyncType): void {
     let syncChain = this.chains.get(syncType);
     if (!syncChain) {
+      // The first batch of a new sync chain may need to detect whether the parent block was an
+      // gloas "empty" block (no envelope produced). It does so by comparing the first
+      // downloaded block's `bid.parentBlockHash` against the head state's `latestExecutionPayloadBid.blockHash`.
+      const headState = this.chain.getHeadState();
+      const latestBid = isStatePostGloas(headState)
+        ? (headState as IBeaconStateViewGloas).latestExecutionPayloadBid
+        : undefined;
+
       syncChain = new SyncChain(
         startEpoch,
         target,
@@ -276,7 +338,8 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
           logger: this.logger,
           custodyConfig: this.chain.custodyConfig,
           metrics: this.metrics,
-        }
+        },
+        latestBid
       );
       this.chains.set(syncType, syncChain);
 

package/src/sync/range/utils/peerBalancer.ts

@@ -52,7 +52,8 @@ export class ChainPeersBalancer {
 
   /**
    * Return the most suitable peer to retry
-   * Sort peers by (1) no failed request (2) less active requests, then pick first
+   * Sort peers by (1) less active requests (2) most columns we need.
+   * Peers that failed this batch or already succeeded for the same request are excluded inside `filterPeers`.
    */
   bestPeerToRetryBatch(batch: Batch): PeerSyncMeta | undefined {
     if (batch.state.status !== BatchStatus.AwaitingDownload) {
@@ -63,10 +64,8 @@ export class ChainPeersBalancer {
     const pendingDataColumns = columnsRequest?.columns ?? this.custodyConfig.sampledColumns;
     const eligiblePeers = this.filterPeers(batch, pendingDataColumns, false);
 
-    const failedPeers = new Set(batch.getFailedPeers());
     const sortedBestPeers = sortBy(
       eligiblePeers,
-      ({syncInfo}) => (failedPeers.has(syncInfo.peerId) ? 1 : 0), // prefer peers without failed requests
       ({syncInfo}) => this.activeRequestsByPeer.get(syncInfo.peerId) ?? 0, // prefer peers with least active req
       ({columns}) => -1 * columns // prefer peers with the most columns
     );
@@ -117,9 +116,16 @@ export class ChainPeersBalancer {
       return eligiblePeers;
     }
 
+    // Skip peers that failed this batch, or that already returned the exact current request shape.
+    const failedPeers = new Set<PeerIdStr>(batch.getFailedPeers());
+
     for (const peer of this.peers) {
       const {earliestAvailableSlot, target, peerId} = peer;
 
+      if (failedPeers.has(peerId) || batch.hasPeerSucceededCurrentRequest(peer)) {
+        continue;
+      }
+
       const activeRequest = this.activeRequestsByPeer.get(peerId) ?? 0;
       if (noActiveRequest && activeRequest > 0) {
         // consumer wants to find peer with no active request, but this peer has active request