@lodestar/beacon-node 1.43.0-dev.a45ba75824 → 1.43.0-dev.a58ad80359

package/src/sync/range/range.ts

@@ -1,7 +1,7 @@
 import {EventEmitter} from "node:events";
 import {StrictEventEmitter} from "strict-event-emitter-types";
 import {BeaconConfig} from "@lodestar/config";
-import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {IBeaconStateViewGloas, computeStartSlotAtEpoch, isStatePostGloas} from "@lodestar/state-transition";
 import {Epoch, Status, fulu} from "@lodestar/types";
 import {Logger, toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
@@ -206,14 +206,18 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
 
   private downloadByRange: SyncChainFns["downloadByRange"] = async (peer, batch) => {
     const batchBlocks = batch.getBlocks();
+    const requests = batch.getRequestsForPeer(peer);
+    const parentRoot = requests.parentPayloadRequest?.envelopeBlockRoot ?? requests.parentPayloadRequest?.blockRoot;
+    const parentPayloadCommitments = parentRoot ? batch.getParentPayloadCommitments(parentRoot) : undefined;
     const {result, warnings} = await downloadByRange({
       config: this.config,
       network: this.network,
       logger: this.logger,
       peerIdStr: peer.peerId,
       batchBlocks,
+      parentPayloadCommitments,
       peerDasMetrics: this.chain.metrics?.peerDas,
-      ...batch.getRequestsForPeer(peer),
+      ...requests,
     });
     const {responses, payloadEnvelopes: downloadedPayloadEnvelopes} = result;
     const {blocks, payloadEnvelopes} = cacheByRangeResponses({
@@ -258,6 +262,14 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
   private addPeerOrCreateChain(startEpoch: Epoch, target: ChainTarget, peer: PeerIdStr, syncType: RangeSyncType): void {
     let syncChain = this.chains.get(syncType);
     if (!syncChain) {
+      // The first batch of a new sync chain may need to detect whether the parent block was an
+      // gloas "empty" block (no envelope produced). It does so by comparing the first
+      // downloaded block's `bid.parentBlockHash` against the head state's `latestExecutionPayloadBid.blockHash`.
+      const headState = this.chain.getHeadState();
+      const latestBid = isStatePostGloas(headState)
+        ? (headState as IBeaconStateViewGloas).latestExecutionPayloadBid
+        : undefined;
+
       syncChain = new SyncChain(
         startEpoch,
         target,
@@ -276,7 +288,8 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
           logger: this.logger,
           custodyConfig: this.chain.custodyConfig,
           metrics: this.metrics,
-        }
+        },
+        latestBid
       );
       this.chains.set(syncType, syncChain);
 

package/src/sync/sync.ts

@@ -1,6 +1,6 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {Slot} from "@lodestar/types";
-import {Logger} from "@lodestar/utils";
+import {Logger, toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../chain/index.js";
 import {GENESIS_SLOT} from "../constants/constants.js";
 import {ExecutionEngineState} from "../execution/index.js";
@@ -188,6 +188,18 @@ export class BeaconSync implements IBeaconSync {
   private addPeer = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
     const localStatus = this.chain.getStatus();
     const syncType = getPeerSyncType(localStatus, data.status, this.chain.forkChoice, this.slotImportTolerance);
+    this.logger.verbose("Peer sync type classified", {
+      peer: data.peer,
+      syncType,
+      localFinalizedEpoch: localStatus.finalizedEpoch,
+      localFinalizedRoot: toRootHex(localStatus.finalizedRoot),
+      localHeadSlot: localStatus.headSlot,
+      localHeadRoot: toRootHex(localStatus.headRoot),
+      remoteFinalizedEpoch: data.status.finalizedEpoch,
+      remoteFinalizedRoot: toRootHex(data.status.finalizedRoot),
+      remoteHeadSlot: data.status.headSlot,
+      remoteHeadRoot: toRootHex(data.status.headRoot),
+    });
 
     // For metrics only
     this.peerSyncType.set(data.peer, syncType);

package/src/sync/unknownBlock.ts

@@ -63,6 +63,27 @@ enum FetchResult {
   FailureMaxAttempts = "failure_max_attempts",
 }
 
+class UnknownBlockRateLimitedError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "UnknownBlockRateLimitedError";
+  }
+}
+
+function getRateLimitedUntilMs(e: unknown): number | null {
+  if (!(e instanceof RequestError)) {
+    return null;
+  }
+
+  switch (e.type.code) {
+    case RequestErrorCode.RESP_RATE_LIMITED:
+    case RequestErrorCode.REQUEST_SELF_RATE_LIMITED:
+      return e.type.rateLimitedUntilMs ?? null;
+    default:
+      return null;
+  }
+}
+
 /**
  * BlockInputSync is a class that handles ReqResp to find blocks and data related to a specific blockRoot.  The
  * blockRoot may have been found via object gossip, or the API.  Gossip objects that can trigger a search are block,
@@ -106,6 +127,7 @@ export class BlockInputSync {
   private readonly maxPendingBlocks;
   private subscribedToNetworkEvents = false;
   private peerBalancer: UnknownBlockPeerBalancer;
+  private rateLimitBackoffTimeout: NodeJS.Timeout | undefined;
 
   constructor(
     private readonly config: ChainForkConfig,
@@ -155,6 +177,7 @@ export class BlockInputSync {
 
   unsubscribeFromNetwork(): void {
     this.logger.verbose("BlockInputSync disabled.");
+    this.clearRateLimitBackoffTimer();
     this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
     this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
     this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
@@ -402,6 +425,7 @@ export class BlockInputSync {
   private onPeerDisconnected = (data: NetworkEventData[NetworkEvent.peerDisconnected]): void => {
     const peerId = data.peer;
     this.peerBalancer.onPeerDisconnected(peerId);
+    this.scheduleRateLimitBackoffRetry();
   };
 
   /**
@@ -519,7 +543,7 @@ export class BlockInputSync {
    */
   private triggerUnknownBlockSearch = (): void => {
     // Cheap early stop to prevent calling the network.getConnectedPeers()
-    if (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0) {
+    if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
       return;
     }
 
@@ -607,6 +631,36 @@ export class BlockInputSync {
     }
   };
 
+  private scheduleRateLimitBackoffRetry(): void {
+    this.clearRateLimitBackoffTimer();
+
+    if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
+      return;
+    }
+
+    const now = Date.now();
+    const retryAt = this.peerBalancer.getNextRateLimitRetryAt();
+    if (retryAt === null) {
+      return;
+    }
+
+    this.rateLimitBackoffTimeout = setTimeout(
+      () => {
+        this.rateLimitBackoffTimeout = undefined;
+        this.triggerUnknownBlockSearch();
+        this.scheduleRateLimitBackoffRetry();
+      },
+      Math.max(0, retryAt - now)
+    );
+  }
+
+  private clearRateLimitBackoffTimer(): void {
+    if (this.rateLimitBackoffTimeout !== undefined) {
+      clearTimeout(this.rateLimitBackoffTimeout);
+      this.rateLimitBackoffTimeout = undefined;
+    }
+  }
+
   private async downloadBlock(block: BlockInputSyncCacheItem): Promise<void> {
     if (block.status !== PendingBlockInputStatus.pending) {
       return;
@@ -678,6 +732,16 @@ export class BlockInputSync {
         this.onUnknownBlockRoot({rootHex: pending.blockInput.parentRootHex, source: BlockInputSource.byRoot});
       }
     } else {
+      if (res.err instanceof UnknownBlockRateLimitedError) {
+        const pendingBlock = this.pendingBlocks.get(rootHex);
+        if (pendingBlock) {
+          pendingBlock.status = PendingBlockInputStatus.pending;
+        }
+        this.logger.debug("Deferring unknown block download due to peer rate limit", logCtx, res.err);
+        this.scheduleRateLimitBackoffRetry();
+        return;
+      }
+
       this.metrics?.blockInputSync.downloadedBlocksError.inc();
       this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
       this.removeAndDownScoreAllDescendants(block);
@@ -994,12 +1058,19 @@ export class BlockInputSync {
     let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
 
     let i = 0;
+    let deferredByRateLimit = false;
     while (i++ < this.getMaxDownloadAttempts()) {
       const pendingColumns = payloadInput?.hasAllData()
         ? new Set<number>()
         : new Set(payloadInput?.getMissingSampledColumnMeta().missing ?? []);
       const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
       if (peerMeta === null) {
+        if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
+          throw new UnknownBlockRateLimitedError(
+            `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`
+          );
+        }
+
         throw Error(
           `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
         );
@@ -1076,7 +1147,12 @@ export class BlockInputSync {
           e as Error
         );
 
-        if (e instanceof RequestError) {
+        const rateLimitedUntilMs = getRateLimitedUntilMs(e);
+        if (rateLimitedUntilMs !== null) {
+          deferredByRateLimit = true;
+          this.peerBalancer.onRateLimited(peerId, rateLimitedUntilMs);
+          this.scheduleRateLimitBackoffRetry();
+        } else if (e instanceof RequestError) {
           switch (e.type.code) {
             case RequestErrorCode.REQUEST_RATE_LIMITED:
             case RequestErrorCode.REQUEST_TIMEOUT:
@@ -1093,6 +1169,12 @@ export class BlockInputSync {
       }
     }
 
+    if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
+      throw new UnknownBlockRateLimitedError(
+        `Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`
+      );
+    }
+
     throw Error(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts.`);
   }
 
@@ -1176,6 +1258,7 @@ export class BlockInputSync {
     }
 
     let i = 0;
+    let deferredByRateLimit = false;
     while (i++ < this.getMaxDownloadAttempts()) {
       const pendingColumns =
         isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput)
@@ -1183,6 +1266,12 @@ export class BlockInputSync {
           : defaultPendingColumns;
       const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
       if (peerMeta === null) {
+        if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
+          throw new UnknownBlockRateLimitedError(
+            `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`
+          );
+        }
+
         // no more peer with needed columns to try, throw error
         const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
         this.metrics?.blockInputSync.fetchTimeSec.observe(
@@ -1238,14 +1327,23 @@ export class BlockInputSync {
         } else if (e instanceof RequestError) {
           // should look into req_resp metrics in this case
           downloadByRootMetrics?.error.inc({code: "req_resp", client: peerClient});
-          switch (e.type.code) {
-            case RequestErrorCode.REQUEST_RATE_LIMITED:
-            case RequestErrorCode.REQUEST_TIMEOUT:
-              // do not exclude peer for these errors
-              break;
-            default:
-              excludedPeers.add(peerId);
-              break;
+          const rateLimitedUntilMs = getRateLimitedUntilMs(e);
+          if (rateLimitedUntilMs !== null) {
+            deferredByRateLimit = true;
+            this.peerBalancer.onRateLimited(peerId, rateLimitedUntilMs);
+            this.scheduleRateLimitBackoffRetry();
+          } else {
+            switch (e.type.code) {
+              case RequestErrorCode.REQUEST_RATE_LIMITED:
+              case RequestErrorCode.RESP_RATE_LIMITED:
+              case RequestErrorCode.REQUEST_SELF_RATE_LIMITED:
+              case RequestErrorCode.REQUEST_TIMEOUT:
+                // do not exclude peer for these errors
+                break;
+              default:
+                excludedPeers.add(peerId);
+                break;
+            }
           }
         } else {
           // investigate if this happens
@@ -1273,6 +1371,10 @@ export class BlockInputSync {
 
     const message = `Error fetching BlockInput with slot=${slot} root=${rootHex} after ${i - 1} attempts.`;
 
+    if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
+      throw new UnknownBlockRateLimitedError(`${message} Peers are rate-limited.`);
+    }
+
     if (!isPendingBlockInput(cacheItem)) {
       throw Error(`${message} No block and no data was found.`);
     }
@@ -1404,10 +1506,12 @@ export class BlockInputSync {
 export class UnknownBlockPeerBalancer {
   readonly peersMeta: Map<PeerIdStr, PeerSyncMeta>;
   readonly activeRequests: Map<PeerIdStr, number>;
+  readonly rateLimitedUntilByPeer: Map<PeerIdStr, number>;
 
   constructor() {
     this.peersMeta = new Map();
     this.activeRequests = new Map();
+    this.rateLimitedUntilByPeer = new Map();
   }
 
   /** Trigger on each peer re-status */
@@ -1422,6 +1526,41 @@ export class UnknownBlockPeerBalancer {
   onPeerDisconnected(peerId: PeerIdStr): void {
     this.peersMeta.delete(peerId);
     this.activeRequests.delete(peerId);
+    this.rateLimitedUntilByPeer.delete(peerId);
+  }
+
+  onRateLimited(peerId: PeerIdStr, rateLimitedUntilMs: number): void {
+    this.rateLimitedUntilByPeer.set(peerId, rateLimitedUntilMs);
+  }
+
+  getNextRateLimitRetryAt(pendingColumns?: Set<number>, excludedPeers?: Set<PeerIdStr>): number | null {
+    const now = Date.now();
+    let retryAt: number | null = null;
+
+    for (const [peerId, rateLimitedUntil] of this.rateLimitedUntilByPeer.entries()) {
+      if (rateLimitedUntil <= now) {
+        this.rateLimitedUntilByPeer.delete(peerId);
+        continue;
+      }
+
+      if (excludedPeers?.has(peerId)) {
+        continue;
+      }
+
+      const syncMeta = this.peersMeta.get(peerId);
+      if (syncMeta === undefined) {
+        this.rateLimitedUntilByPeer.delete(peerId);
+        continue;
+      }
+
+      if (pendingColumns !== undefined && !this.peerHasPendingColumns(syncMeta, pendingColumns)) {
+        continue;
+      }
+
+      retryAt = Math.min(retryAt ?? rateLimitedUntil, rateLimitedUntil);
+    }
+
+    return retryAt;
   }
 
   /**
@@ -1470,6 +1609,7 @@ export class UnknownBlockPeerBalancer {
   }
 
   private filterPeers(pendingDataColumns: Set<number>, excludedPeers: Set<PeerIdStr>): PeerIdStr[] {
+    const now = Date.now();
     let maxColumnCount = 0;
     const considerPeers: {peerId: PeerIdStr; columnCount: number}[] = [];
     for (const [peerId, syncMeta] of this.peersMeta.entries()) {
@@ -1478,12 +1618,24 @@ export class UnknownBlockPeerBalancer {
         continue;
       }
 
+      const rateLimitedUntil = this.rateLimitedUntilByPeer.get(peerId);
+      if (rateLimitedUntil !== undefined) {
+        if (now < rateLimitedUntil) {
+          continue;
+        }
+        this.rateLimitedUntilByPeer.delete(peerId);
+      }
+
       const activeRequests = this.activeRequests.get(peerId) ?? 0;
       if (activeRequests >= MAX_CONCURRENT_REQUESTS) {
         // should return peer with no more than MAX_CONCURRENT_REQUESTS active requests
         continue;
       }
 
+      if (!this.peerHasPendingColumns(syncMeta, pendingDataColumns)) {
+        continue;
+      }
+
       if (pendingDataColumns.size === 0) {
         considerPeers.push({peerId, columnCount: 0});
         continue;
@@ -1517,4 +1669,12 @@ export class UnknownBlockPeerBalancer {
 
     return eligiblePeers;
   }
+
+  private peerHasPendingColumns(syncMeta: PeerSyncMeta, pendingDataColumns: Set<number>): boolean {
+    if (pendingDataColumns.size === 0) {
+      return true;
+    }
+
+    return syncMeta.custodyColumns.some((column) => pendingDataColumns.has(column));
+  }
 }