@lodestar/beacon-node 1.43.0-dev.be5f70c0c8 → 1.43.0-dev.c9dc2b2332

package/src/sync/range/batch.ts

@@ -45,6 +45,15 @@ export type Attempt = {
   hash: RootHex;
 };
 
+type TrackedRequest = {
+  /** only happen for the 1st batch in checkpoint sync */
+  parentPayload: boolean;
+  /**
+   * we always issue by_range before parent_payload, so we don't model this as null
+   */
+  byRangeColumns: Set<number>;
+};
+
 export type AwaitingDownloadState = {
   status: BatchStatus.AwaitingDownload;
   blocks: IBlockInput[];
@@ -62,6 +71,7 @@ export type BatchState =
   | {
       status: BatchStatus.Downloading;
       peer: PeerIdStr;
+      request: TrackedRequest;
       blocks: IBlockInput[];
       payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null;
     }
@@ -110,6 +120,13 @@ function formatColumnsReq(req: {startSlot: Slot; count: number; columns: number[
   return `startSlot=${req.startSlot},count=${req.count},cols=${prettyPrintIndices(req.columns)}`;
 }
 
+function getTrackedRequest({parentPayloadRequest, columnsRequest}: DownloadByRangeRequests): TrackedRequest {
+  return {
+    parentPayload: parentPayloadRequest != null,
+    byRangeColumns: new Set(parentPayloadRequest == null ? (columnsRequest?.columns ?? []) : []),
+  };
+}
+
 /**
  * Batches are downloaded at the first block of the epoch.
  *
@@ -131,8 +148,8 @@ export class Batch {
   requests: DownloadByRangeRequests;
   /** State of the batch. */
   state: BatchState = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
-  /** Peers that provided good data */
-  goodPeers: PeerIdStr[] = [];
+  /** Peers that provided good data, with column coverage for by_range requests */
+  private readonly successfulDownloads = new Map<PeerIdStr, TrackedRequest>();
   /** The `Attempts` that have been made and failed to send us this batch. */
   readonly failedProcessingAttempts: Attempt[] = [];
   /** The `Attempts` that have been made and failed because of execution malfunction. */
@@ -406,6 +423,35 @@ export class Batch {
     return [...this.failedDownloadAttempts, ...this.failedProcessingAttempts.flatMap((a) => a.peers)];
   }
 
+  /**
+   * True only if the peer has already returned a successful response for the current request.
+   * A by_range success may update `this.requests` to parent_payload, and the same peer is then
+   * still eligible for the newly discovered parent payload data.
+   * For by_range, a peer that previously succeeded with a superset of requested columns is skipped.
+   */
+  hasPeerSucceededCurrentRequest(peer: PeerSyncMeta): boolean {
+    const successfulDownload = this.successfulDownloads.get(peer.peerId);
+    if (successfulDownload == null) return false;
+
+    const request = getTrackedRequest(this.getRequestsForPeer(peer));
+    if (request.parentPayload) return successfulDownload.parentPayload;
+
+    const requestByRangeColumns = request.byRangeColumns;
+
+    if (requestByRangeColumns.size === 0) {
+      // this means a download blocks/envelops by_range only
+      // don't do that again if we already did it
+      // see https://github.com/ChainSafe/lodestar/issues/9357
+      return true;
+    }
+
+    return [...requestByRangeColumns].every((column) => successfulDownload.byRangeColumns.has(column));
+  }
+
+  private getSuccessfulPeers(): PeerIdStr[] {
+    return Array.from(this.successfulDownloads.keys());
+  }
+
   getMetadata(): BatchMetadata {
     const {blocksRequest, blobsRequest, columnsRequest, envelopesRequest} = this.requests;
     const failedProcessingPeerList = this.failedProcessingAttempts.flatMap((a) => a.peers);
@@ -440,14 +486,17 @@ export class Batch {
   /**
    * AwaitingDownload -> Downloading
    */
-  startDownloading(peer: PeerIdStr): void {
+  startDownloading(peer: PeerSyncMeta): void {
     if (this.state.status !== BatchStatus.AwaitingDownload) {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.AwaitingDownload));
     }
 
+    const request = getTrackedRequest(this.getRequestsForPeer(peer));
+
     this.state = {
       status: BatchStatus.Downloading,
-      peer,
+      peer: peer.peerId,
+      request,
       blocks: this.state.blocks,
       payloadEnvelopes: this.state.payloadEnvelopes,
     };
@@ -468,7 +517,17 @@ export class Batch {
     // ensure that blocks are always sorted before getting stored on the batch.state or being used to getRequests
     blocks.sort((a, b) => a.slot - b.slot);
 
-    this.goodPeers.push(peer);
+    const successfulDownload = this.successfulDownloads.get(peer) ?? {
+      parentPayload: false,
+      byRangeColumns: new Set<number>(),
+    };
+    successfulDownload.parentPayload ||= this.state.request.parentPayload;
+    if (!this.state.request.parentPayload) {
+      for (const column of this.state.request.byRangeColumns) {
+        successfulDownload.byRangeColumns.add(column);
+      }
+    }
+    this.successfulDownloads.set(peer, successfulDownload);
 
     let allComplete = true;
     const slots = new Set<number>();
@@ -497,8 +556,9 @@ export class Batch {
     if (allComplete && isForkPostGloas(this.forkName)) {
       for (const block of blocks) {
         const payloadInput = newPayloadEnvelopes?.get(block.slot);
-        // by_range needs every block's envelope and all sampled columns.
-        if (!payloadInput?.hasPayloadEnvelope() || !payloadInput.hasComputedAllData()) {
+        // only need to make sure envelope has all columns, not all blocks have payload
+        // assertLinearChainSegment() was called before reaching this
+        if (payloadInput?.hasPayloadEnvelope() && !payloadInput.hasComputedAllData()) {
           allComplete = false;
           break;
         }
@@ -589,10 +649,10 @@ export class Batch {
     const blocks = this.state.blocks;
     const payloadEnvelopes = this.state.payloadEnvelopes;
     const hash = hashBlocks(blocks, this.config); // tracks blocks to report peer on processing error
-    // Reset goodPeers in case another download attempt needs to be made.  When Attempt is successful or not the peers
-    // that the data came from will be handled by the Attempt that goes for processing
-    const peers = this.goodPeers;
-    this.goodPeers = [];
+    // Reset successfulDownloads in case another download attempt needs to be made. When Attempt is successful or not
+    // the peers that the data came from will be handled by the Attempt that goes for processing.
+    const peers = this.getSuccessfulPeers();
+    this.successfulDownloads.clear();
     this.state = {status: BatchStatus.Processing, blocks, payloadEnvelopes, attempt: {peers, hash}};
     return {blocks, payloadEnvelopes, peers};
   }

package/src/sync/range/chain.ts

@@ -551,7 +551,7 @@ export class SyncChain {
       peer: prettyPrintPeerIdStr(peer.peerId),
     });
     try {
-      batch.startDownloading(peer.peerId);
+      batch.startDownloading(peer);
 
       // wrapError ensures to never call both batch success() and batch error()
       const res = await wrapError(this.downloadByRange(peer, batch, this.syncType));
@@ -581,6 +581,8 @@ export class SyncChain {
           case DownloadByRangeErrorCode.OUT_OF_ORDER_BLOCKS:
           case DownloadByRangeErrorCode.OUT_OF_RANGE_BLOCKS:
           case DownloadByRangeErrorCode.PARENT_ROOT_MISMATCH:
+          case DownloadByRangeErrorCode.INVALID_ENVELOPE_BEACON_BLOCK_ROOT:
+          case DownloadByRangeErrorCode.INVALID_CHAIN_SEGMENT:
           case BlobSidecarErrorCode.INCLUSION_PROOF_INVALID:
           case BlobSidecarErrorCode.INVALID_KZG_PROOF_BATCH:
           case DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT:

package/src/sync/range/range.ts

@@ -3,14 +3,21 @@ import {StrictEventEmitter} from "strict-event-emitter-types";
 import {BeaconConfig} from "@lodestar/config";
 import {IBeaconStateViewGloas, computeStartSlotAtEpoch, isStatePostGloas} from "@lodestar/state-transition";
 import {Epoch, Status, fulu} from "@lodestar/types";
-import {Logger, toRootHex} from "@lodestar/utils";
+import {Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
 import {AttestationImportOpt, ImportBlockOpts} from "../../chain/blocks/index.js";
+import {assertLinearChainSegment} from "../../chain/blocks/utils/chainSegment.js";
+import {BlockError} from "../../chain/errors/index.js";
 import {IBeaconChain} from "../../chain/index.js";
 import {Metrics} from "../../metrics/index.js";
 import {INetwork} from "../../network/index.js";
 import {PeerIdStr} from "../../util/peerId.js";
-import {cacheByRangeResponses, downloadByRange} from "../utils/downloadByRange.js";
+import {
+  DownloadByRangeError,
+  DownloadByRangeErrorCode,
+  cacheByRangeResponses,
+  downloadByRange,
+} from "../utils/downloadByRange.js";
 import {RangeSyncType, getRangeSyncTarget, rangeSyncTypes} from "../utils/remoteSyncType.js";
 import {ChainTarget, SyncChain, SyncChainDebugState, SyncChainFns} from "./chain.js";
 import {updateChains} from "./utils/index.js";
@@ -231,6 +238,49 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
       custodyConfig: this.chain.custodyConfig,
       seenTimestampSec: Date.now() / 1000,
     });
+
+    const segmentBlocks = blocks.filter((b) => b.hasBlock()).sort((a, b) => a.slot - b.slot);
+    const envelopeSlots = payloadEnvelopes
+      ? Array.from(payloadEnvelopes.entries())
+          .filter(([, pi]) => pi.hasPayloadEnvelope())
+          .map(([slot]) => slot)
+          .sort((a, b) => a - b)
+      : [];
+    this.logger.verbose("downloadByRange batch ready", {
+      peer: peer.peerId,
+      blockSlots: prettyPrintIndices(segmentBlocks.map((b) => b.slot)),
+      envelopeSlots: prettyPrintIndices(envelopeSlots),
+      ...batch.getMetadata(),
+    });
+
+    if (segmentBlocks.length > 1) {
+      try {
+        assertLinearChainSegment(this.config, segmentBlocks, payloadEnvelopes, null);
+      } catch (err) {
+        if (err instanceof BlockError) {
+          this.logger.debug(
+            "downloadByRange segment validation failed",
+            {
+              peer: peer.peerId,
+              reason: err.type.code,
+              slot: err.signedBlock.message.slot,
+              detail: JSON.stringify(err.type),
+              ...batch.getMetadata(),
+            },
+            err
+          );
+          // with this error, the peer will be penalized inside SyncChain
+          throw new DownloadByRangeError({
+            code: DownloadByRangeErrorCode.INVALID_CHAIN_SEGMENT,
+            slot: err.signedBlock.message.slot,
+            reason: err.type.code,
+          });
+        }
+
+        throw err;
+      }
+    }
+
     return {result: {blocks, payloadEnvelopes}, warnings};
   };
 

package/src/sync/range/utils/peerBalancer.ts

@@ -52,7 +52,8 @@ export class ChainPeersBalancer {
 
   /**
    * Return the most suitable peer to retry
-   * Sort peers by (1) no failed request (2) less active requests, then pick first
+   * Sort peers by (1) less active requests (2) most columns we need.
+   * Peers that failed this batch or already succeeded for the same request are excluded inside `filterPeers`.
    */
   bestPeerToRetryBatch(batch: Batch): PeerSyncMeta | undefined {
     if (batch.state.status !== BatchStatus.AwaitingDownload) {
@@ -63,10 +64,8 @@ export class ChainPeersBalancer {
     const pendingDataColumns = columnsRequest?.columns ?? this.custodyConfig.sampledColumns;
     const eligiblePeers = this.filterPeers(batch, pendingDataColumns, false);
 
-    const failedPeers = new Set(batch.getFailedPeers());
     const sortedBestPeers = sortBy(
       eligiblePeers,
-      ({syncInfo}) => (failedPeers.has(syncInfo.peerId) ? 1 : 0), // prefer peers without failed requests
       ({syncInfo}) => this.activeRequestsByPeer.get(syncInfo.peerId) ?? 0, // prefer peers with least active req
       ({columns}) => -1 * columns // prefer peers with the most columns
     );
@@ -117,9 +116,16 @@ export class ChainPeersBalancer {
       return eligiblePeers;
     }
 
+    // Skip peers that failed this batch, or that already returned the exact current request shape.
+    const failedPeers = new Set<PeerIdStr>(batch.getFailedPeers());
+
     for (const peer of this.peers) {
       const {earliestAvailableSlot, target, peerId} = peer;
 
+      if (failedPeers.has(peerId) || batch.hasPeerSucceededCurrentRequest(peer)) {
+        continue;
+      }
+
       const activeRequest = this.activeRequestsByPeer.get(peerId) ?? 0;
       if (noActiveRequest && activeRequest > 0) {
         // consumer wants to find peer with no active request, but this peer has active request

package/src/sync/utils/downloadByRange.ts

@@ -1155,6 +1155,9 @@ export enum DownloadByRangeErrorCode {
 
   /** Envelope beaconBlockRoot does not match the block's root */
   INVALID_ENVELOPE_BEACON_BLOCK_ROOT = "DOWNLOAD_BY_RANGE_ERROR_INVALID_ENVELOPE_BEACON_BLOCK_ROOT",
+
+  /** Block segment + envelopes failed chain-segment linearity / FULL-chain checks */
+  INVALID_CHAIN_SEGMENT = "DOWNLOAD_BY_RANGE_ERROR_INVALID_CHAIN_SEGMENT",
 }
 
 export type DownloadByRangeErrorType =
@@ -1252,6 +1255,11 @@ export type DownloadByRangeErrorType =
       slot: Slot;
       expected: string;
       actual: string;
+    }
+  | {
+      code: DownloadByRangeErrorCode.INVALID_CHAIN_SEGMENT;
+      slot: Slot;
+      reason: string;
     };
 
 export class DownloadByRangeError extends LodestarError<DownloadByRangeErrorType> {}