@lodestar/beacon-node 1.43.0-dev.87cbe69c66 → 1.43.0-dev.8ad5d8be4c

package/src/sync/range/range.ts

@@ -1,16 +1,23 @@
 import {EventEmitter} from "node:events";
 import {StrictEventEmitter} from "strict-event-emitter-types";
 import {BeaconConfig} from "@lodestar/config";
-import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {IBeaconStateViewGloas, computeStartSlotAtEpoch, isStatePostGloas} from "@lodestar/state-transition";
 import {Epoch, Status, fulu} from "@lodestar/types";
-import {Logger, toRootHex} from "@lodestar/utils";
+import {Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
 import {AttestationImportOpt, ImportBlockOpts} from "../../chain/blocks/index.js";
+import {assertLinearChainSegment} from "../../chain/blocks/utils/chainSegment.js";
+import {BlockError} from "../../chain/errors/index.js";
 import {IBeaconChain} from "../../chain/index.js";
 import {Metrics} from "../../metrics/index.js";
 import {INetwork} from "../../network/index.js";
 import {PeerIdStr} from "../../util/peerId.js";
-import {cacheByRangeResponses, downloadByRange} from "../utils/downloadByRange.js";
+import {
+  DownloadByRangeError,
+  DownloadByRangeErrorCode,
+  cacheByRangeResponses,
+  downloadByRange,
+} from "../utils/downloadByRange.js";
 import {RangeSyncType, getRangeSyncTarget, rangeSyncTypes} from "../utils/remoteSyncType.js";
 import {ChainTarget, SyncChain, SyncChainDebugState, SyncChainFns} from "./chain.js";
 import {updateChains} from "./utils/index.js";
@@ -206,14 +213,18 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
 
   private downloadByRange: SyncChainFns["downloadByRange"] = async (peer, batch) => {
     const batchBlocks = batch.getBlocks();
+    const requests = batch.getRequestsForPeer(peer);
+    const parentRoot = requests.parentPayloadRequest?.envelopeBlockRoot ?? requests.parentPayloadRequest?.blockRoot;
+    const parentPayloadCommitments = parentRoot ? batch.getParentPayloadCommitments(parentRoot) : undefined;
     const {result, warnings} = await downloadByRange({
       config: this.config,
       network: this.network,
       logger: this.logger,
       peerIdStr: peer.peerId,
       batchBlocks,
+      parentPayloadCommitments,
       peerDasMetrics: this.chain.metrics?.peerDas,
-      ...batch.getRequestsForPeer(peer),
+      ...requests,
     });
     const {responses, payloadEnvelopes: downloadedPayloadEnvelopes} = result;
     const {blocks, payloadEnvelopes} = cacheByRangeResponses({
@@ -227,6 +238,49 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
       custodyConfig: this.chain.custodyConfig,
       seenTimestampSec: Date.now() / 1000,
     });
+
+    const segmentBlocks = blocks.filter((b) => b.hasBlock()).sort((a, b) => a.slot - b.slot);
+    const envelopeSlots = payloadEnvelopes
+      ? Array.from(payloadEnvelopes.entries())
+          .filter(([, pi]) => pi.hasPayloadEnvelope())
+          .map(([slot]) => slot)
+          .sort((a, b) => a - b)
+      : [];
+    this.logger.verbose("downloadByRange batch ready", {
+      peer: peer.peerId,
+      blockSlots: prettyPrintIndices(segmentBlocks.map((b) => b.slot)),
+      envelopeSlots: prettyPrintIndices(envelopeSlots),
+      ...batch.getMetadata(),
+    });
+
+    if (segmentBlocks.length > 1) {
+      try {
+        assertLinearChainSegment(this.config, segmentBlocks, payloadEnvelopes, null);
+      } catch (err) {
+        if (err instanceof BlockError) {
+          this.logger.debug(
+            "downloadByRange segment validation failed",
+            {
+              peer: peer.peerId,
+              reason: err.type.code,
+              slot: err.signedBlock.message.slot,
+              detail: JSON.stringify(err.type),
+              ...batch.getMetadata(),
+            },
+            err
+          );
+          // with this error, the peer will be penalized inside SyncChain
+          throw new DownloadByRangeError({
+            code: DownloadByRangeErrorCode.INVALID_CHAIN_SEGMENT,
+            slot: err.signedBlock.message.slot,
+            reason: err.type.code,
+          });
+        }
+
+        throw err;
+      }
+    }
+
     return {result: {blocks, payloadEnvelopes}, warnings};
   };
 
@@ -258,6 +312,14 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
   private addPeerOrCreateChain(startEpoch: Epoch, target: ChainTarget, peer: PeerIdStr, syncType: RangeSyncType): void {
     let syncChain = this.chains.get(syncType);
     if (!syncChain) {
+      // The first batch of a new sync chain may need to detect whether the parent block was an
+      // gloas "empty" block (no envelope produced). It does so by comparing the first
+      // downloaded block's `bid.parentBlockHash` against the head state's `latestExecutionPayloadBid.blockHash`.
+      const headState = this.chain.getHeadState();
+      const latestBid = isStatePostGloas(headState)
+        ? (headState as IBeaconStateViewGloas).latestExecutionPayloadBid
+        : undefined;
+
       syncChain = new SyncChain(
         startEpoch,
         target,
@@ -276,7 +338,8 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
           logger: this.logger,
           custodyConfig: this.chain.custodyConfig,
           metrics: this.metrics,
-        }
+        },
+        latestBid
       );
       this.chains.set(syncType, syncChain);
 

package/src/sync/range/utils/peerBalancer.ts

@@ -52,7 +52,8 @@ export class ChainPeersBalancer {
 
   /**
    * Return the most suitable peer to retry
-   * Sort peers by (1) no failed request (2) less active requests, then pick first
+   * Sort peers by (1) less active requests (2) most columns we need.
+   * Peers that failed this batch or already succeeded for the same request are excluded inside `filterPeers`.
    */
   bestPeerToRetryBatch(batch: Batch): PeerSyncMeta | undefined {
     if (batch.state.status !== BatchStatus.AwaitingDownload) {
@@ -63,10 +64,8 @@ export class ChainPeersBalancer {
     const pendingDataColumns = columnsRequest?.columns ?? this.custodyConfig.sampledColumns;
     const eligiblePeers = this.filterPeers(batch, pendingDataColumns, false);
 
-    const failedPeers = new Set(batch.getFailedPeers());
     const sortedBestPeers = sortBy(
       eligiblePeers,
-      ({syncInfo}) => (failedPeers.has(syncInfo.peerId) ? 1 : 0), // prefer peers without failed requests
       ({syncInfo}) => this.activeRequestsByPeer.get(syncInfo.peerId) ?? 0, // prefer peers with least active req
       ({columns}) => -1 * columns // prefer peers with the most columns
     );
@@ -117,9 +116,16 @@ export class ChainPeersBalancer {
       return eligiblePeers;
     }
 
+    // Skip peers that failed this batch, or that already returned the exact current request shape.
+    const failedPeers = new Set<PeerIdStr>(batch.getFailedPeers());
+
     for (const peer of this.peers) {
       const {earliestAvailableSlot, target, peerId} = peer;
 
+      if (failedPeers.has(peerId) || batch.hasPeerSucceededCurrentRequest(peer)) {
+        continue;
+      }
+
       const activeRequest = this.activeRequestsByPeer.get(peerId) ?? 0;
       if (noActiveRequest && activeRequest > 0) {
         // consumer wants to find peer with no active request, but this peer has active request

package/src/sync/sync.ts

@@ -1,6 +1,6 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {Slot} from "@lodestar/types";
-import {Logger} from "@lodestar/utils";
+import {Logger, toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../chain/index.js";
 import {GENESIS_SLOT} from "../constants/constants.js";
 import {ExecutionEngineState} from "../execution/index.js";
@@ -188,6 +188,18 @@ export class BeaconSync implements IBeaconSync {
   private addPeer = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
     const localStatus = this.chain.getStatus();
     const syncType = getPeerSyncType(localStatus, data.status, this.chain.forkChoice, this.slotImportTolerance);
+    this.logger.verbose("Peer sync type classified", {
+      peer: data.peer,
+      syncType,
+      localFinalizedEpoch: localStatus.finalizedEpoch,
+      localFinalizedRoot: toRootHex(localStatus.finalizedRoot),
+      localHeadSlot: localStatus.headSlot,
+      localHeadRoot: toRootHex(localStatus.headRoot),
+      remoteFinalizedEpoch: data.status.finalizedEpoch,
+      remoteFinalizedRoot: toRootHex(data.status.finalizedRoot),
+      remoteHeadSlot: data.status.headSlot,
+      remoteHeadRoot: toRootHex(data.status.headRoot),
+    });
 
     // For metrics only
     this.peerSyncType.set(data.peer, syncType);

package/src/sync/unknownBlock.ts

@@ -42,6 +42,7 @@ import {
 } from "./types.js";
 import {DownloadByRootError, downloadByRoot} from "./utils/downloadByRoot.js";
 import {getAllDescendantBlocks, getUnknownAndAncestorBlocks} from "./utils/pendingBlocksTree.js";
+import {getRateLimitedUntilMs} from "./utils/rateLimit.js";
 
 const MAX_ATTEMPTS_PER_BLOCK = 5;
 const MAX_KNOWN_BAD_BLOCKS = 500;
@@ -63,6 +64,13 @@ enum FetchResult {
   FailureMaxAttempts = "failure_max_attempts",
 }
 
+class UnknownBlockRateLimitedError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "UnknownBlockRateLimitedError";
+  }
+}
+
 /**
  * BlockInputSync is a class that handles ReqResp to find blocks and data related to a specific blockRoot.  The
  * blockRoot may have been found via object gossip, or the API.  Gossip objects that can trigger a search are block,
@@ -106,6 +114,7 @@ export class BlockInputSync {
   private readonly maxPendingBlocks;
   private subscribedToNetworkEvents = false;
   private peerBalancer: UnknownBlockPeerBalancer;
+  private rateLimitBackoffTimeout: NodeJS.Timeout | undefined;
 
   constructor(
     private readonly config: ChainForkConfig,
@@ -155,6 +164,7 @@ export class BlockInputSync {
 
   unsubscribeFromNetwork(): void {
     this.logger.verbose("BlockInputSync disabled.");
+    this.clearRateLimitBackoffTimer();
     this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
     this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
     this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
@@ -402,6 +412,7 @@ export class BlockInputSync {
   private onPeerDisconnected = (data: NetworkEventData[NetworkEvent.peerDisconnected]): void => {
     const peerId = data.peer;
     this.peerBalancer.onPeerDisconnected(peerId);
+    this.scheduleRateLimitBackoffRetry();
   };
 
   /**
@@ -519,7 +530,7 @@ export class BlockInputSync {
    */
   private triggerUnknownBlockSearch = (): void => {
     // Cheap early stop to prevent calling the network.getConnectedPeers()
-    if (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0) {
+    if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
       return;
     }
 
@@ -607,6 +618,36 @@ export class BlockInputSync {
     }
   };
 
+  private scheduleRateLimitBackoffRetry(): void {
+    this.clearRateLimitBackoffTimer();
+
+    if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
+      return;
+    }
+
+    const now = Date.now();
+    const retryAt = this.peerBalancer.getNextRateLimitRetryAt();
+    if (retryAt === null) {
+      return;
+    }
+
+    this.rateLimitBackoffTimeout = setTimeout(
+      () => {
+        this.rateLimitBackoffTimeout = undefined;
+        this.triggerUnknownBlockSearch();
+        this.scheduleRateLimitBackoffRetry();
+      },
+      Math.max(0, retryAt - now)
+    );
+  }
+
+  private clearRateLimitBackoffTimer(): void {
+    if (this.rateLimitBackoffTimeout !== undefined) {
+      clearTimeout(this.rateLimitBackoffTimeout);
+      this.rateLimitBackoffTimeout = undefined;
+    }
+  }
+
   private async downloadBlock(block: BlockInputSyncCacheItem): Promise<void> {
     if (block.status !== PendingBlockInputStatus.pending) {
       return;
@@ -678,6 +719,16 @@ export class BlockInputSync {
         this.onUnknownBlockRoot({rootHex: pending.blockInput.parentRootHex, source: BlockInputSource.byRoot});
       }
     } else {
+      if (res.err instanceof UnknownBlockRateLimitedError) {
+        const pendingBlock = this.pendingBlocks.get(rootHex);
+        if (pendingBlock) {
+          pendingBlock.status = PendingBlockInputStatus.pending;
+        }
+        this.logger.debug("Deferring unknown block download due to peer rate limit", logCtx, res.err);
+        this.scheduleRateLimitBackoffRetry();
+        return;
+      }
+
       this.metrics?.blockInputSync.downloadedBlocksError.inc();
       this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
       this.removeAndDownScoreAllDescendants(block);
@@ -994,12 +1045,19 @@ export class BlockInputSync {
     let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
 
     let i = 0;
+    let deferredByRateLimit = false;
     while (i++ < this.getMaxDownloadAttempts()) {
       const pendingColumns = payloadInput?.hasAllData()
         ? new Set<number>()
         : new Set(payloadInput?.getMissingSampledColumnMeta().missing ?? []);
       const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
       if (peerMeta === null) {
+        if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
+          throw new UnknownBlockRateLimitedError(
+            `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`
+          );
+        }
+
         throw Error(
           `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
         );
@@ -1076,7 +1134,12 @@ export class BlockInputSync {
           e as Error
         );
 
-        if (e instanceof RequestError) {
+        const rateLimitedUntilMs = getRateLimitedUntilMs(e);
+        if (rateLimitedUntilMs !== null) {
+          deferredByRateLimit = true;
+          this.peerBalancer.onRateLimited(peerId, rateLimitedUntilMs);
+          this.scheduleRateLimitBackoffRetry();
+        } else if (e instanceof RequestError) {
           switch (e.type.code) {
             case RequestErrorCode.REQUEST_RATE_LIMITED:
             case RequestErrorCode.REQUEST_TIMEOUT:
@@ -1093,6 +1156,12 @@ export class BlockInputSync {
       }
     }
 
+    if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
+      throw new UnknownBlockRateLimitedError(
+        `Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`
+      );
+    }
+
     throw Error(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts.`);
   }
 
@@ -1176,6 +1245,7 @@ export class BlockInputSync {
     }
 
     let i = 0;
+    let deferredByRateLimit = false;
     while (i++ < this.getMaxDownloadAttempts()) {
       const pendingColumns =
         isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput)
@@ -1183,6 +1253,12 @@ export class BlockInputSync {
           : defaultPendingColumns;
       const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
       if (peerMeta === null) {
+        if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
+          throw new UnknownBlockRateLimitedError(
+            `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`
+          );
+        }
+
         // no more peer with needed columns to try, throw error
         const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
         this.metrics?.blockInputSync.fetchTimeSec.observe(
@@ -1238,14 +1314,23 @@ export class BlockInputSync {
         } else if (e instanceof RequestError) {
           // should look into req_resp metrics in this case
           downloadByRootMetrics?.error.inc({code: "req_resp", client: peerClient});
-          switch (e.type.code) {
-            case RequestErrorCode.REQUEST_RATE_LIMITED:
-            case RequestErrorCode.REQUEST_TIMEOUT:
-              // do not exclude peer for these errors
-              break;
-            default:
-              excludedPeers.add(peerId);
-              break;
+          const rateLimitedUntilMs = getRateLimitedUntilMs(e);
+          if (rateLimitedUntilMs !== null) {
+            deferredByRateLimit = true;
+            this.peerBalancer.onRateLimited(peerId, rateLimitedUntilMs);
+            this.scheduleRateLimitBackoffRetry();
+          } else {
+            switch (e.type.code) {
+              case RequestErrorCode.REQUEST_RATE_LIMITED:
+              case RequestErrorCode.RESP_RATE_LIMITED:
+              case RequestErrorCode.REQUEST_SELF_RATE_LIMITED:
+              case RequestErrorCode.REQUEST_TIMEOUT:
+                // do not exclude peer for these errors
+                break;
+              default:
+                excludedPeers.add(peerId);
+                break;
+            }
           }
         } else {
           // investigate if this happens
@@ -1273,6 +1358,10 @@ export class BlockInputSync {
 
     const message = `Error fetching BlockInput with slot=${slot} root=${rootHex} after ${i - 1} attempts.`;
 
+    if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
+      throw new UnknownBlockRateLimitedError(`${message} Peers are rate-limited.`);
+    }
+
     if (!isPendingBlockInput(cacheItem)) {
       throw Error(`${message} No block and no data was found.`);
     }
@@ -1404,10 +1493,12 @@ export class BlockInputSync {
 export class UnknownBlockPeerBalancer {
   readonly peersMeta: Map<PeerIdStr, PeerSyncMeta>;
   readonly activeRequests: Map<PeerIdStr, number>;
+  readonly rateLimitedUntilByPeer: Map<PeerIdStr, number>;
 
   constructor() {
     this.peersMeta = new Map();
     this.activeRequests = new Map();
+    this.rateLimitedUntilByPeer = new Map();
   }
 
   /** Trigger on each peer re-status */
@@ -1422,6 +1513,41 @@ export class UnknownBlockPeerBalancer {
   onPeerDisconnected(peerId: PeerIdStr): void {
     this.peersMeta.delete(peerId);
     this.activeRequests.delete(peerId);
+    this.rateLimitedUntilByPeer.delete(peerId);
+  }
+
+  onRateLimited(peerId: PeerIdStr, rateLimitedUntilMs: number): void {
+    this.rateLimitedUntilByPeer.set(peerId, rateLimitedUntilMs);
+  }
+
+  getNextRateLimitRetryAt(pendingColumns?: Set<number>, excludedPeers?: Set<PeerIdStr>): number | null {
+    const now = Date.now();
+    let retryAt: number | null = null;
+
+    for (const [peerId, rateLimitedUntil] of this.rateLimitedUntilByPeer.entries()) {
+      if (rateLimitedUntil <= now) {
+        this.rateLimitedUntilByPeer.delete(peerId);
+        continue;
+      }
+
+      if (excludedPeers?.has(peerId)) {
+        continue;
+      }
+
+      const syncMeta = this.peersMeta.get(peerId);
+      if (syncMeta === undefined) {
+        this.rateLimitedUntilByPeer.delete(peerId);
+        continue;
+      }
+
+      if (pendingColumns !== undefined && !this.peerHasPendingColumns(syncMeta, pendingColumns)) {
+        continue;
+      }
+
+      retryAt = Math.min(retryAt ?? rateLimitedUntil, rateLimitedUntil);
+    }
+
+    return retryAt;
   }
 
   /**
@@ -1470,6 +1596,7 @@ export class UnknownBlockPeerBalancer {
   }
 
   private filterPeers(pendingDataColumns: Set<number>, excludedPeers: Set<PeerIdStr>): PeerIdStr[] {
+    const now = Date.now();
     let maxColumnCount = 0;
     const considerPeers: {peerId: PeerIdStr; columnCount: number}[] = [];
     for (const [peerId, syncMeta] of this.peersMeta.entries()) {
@@ -1478,12 +1605,24 @@ export class UnknownBlockPeerBalancer {
         continue;
       }
 
+      const rateLimitedUntil = this.rateLimitedUntilByPeer.get(peerId);
+      if (rateLimitedUntil !== undefined) {
+        if (now < rateLimitedUntil) {
+          continue;
+        }
+        this.rateLimitedUntilByPeer.delete(peerId);
+      }
+
       const activeRequests = this.activeRequests.get(peerId) ?? 0;
       if (activeRequests >= MAX_CONCURRENT_REQUESTS) {
         // should return peer with no more than MAX_CONCURRENT_REQUESTS active requests
         continue;
       }
 
+      if (!this.peerHasPendingColumns(syncMeta, pendingDataColumns)) {
+        continue;
+      }
+
       if (pendingDataColumns.size === 0) {
         considerPeers.push({peerId, columnCount: 0});
         continue;
@@ -1517,4 +1656,12 @@ export class UnknownBlockPeerBalancer {
 
     return eligiblePeers;
   }
+
+  private peerHasPendingColumns(syncMeta: PeerSyncMeta, pendingDataColumns: Set<number>): boolean {
+    if (pendingDataColumns.size === 0) {
+      return true;
+    }
+
+    return syncMeta.custodyColumns.some((column) => pendingDataColumns.has(column));
+  }
 }