@lodestar/beacon-node 1.43.0-dev.b05ea98d04 → 1.43.0-dev.b0e1143d06

package/src/sync/sync.ts

@@ -1,6 +1,6 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {Slot} from "@lodestar/types";
-import {Logger} from "@lodestar/utils";
+import {Logger, toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../chain/index.js";
 import {GENESIS_SLOT} from "../constants/constants.js";
 import {ExecutionEngineState} from "../execution/index.js";
@@ -188,6 +188,18 @@ export class BeaconSync implements IBeaconSync {
   private addPeer = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
     const localStatus = this.chain.getStatus();
     const syncType = getPeerSyncType(localStatus, data.status, this.chain.forkChoice, this.slotImportTolerance);
+    this.logger.verbose("Peer sync type classified", {
+      peer: data.peer,
+      syncType,
+      localFinalizedEpoch: localStatus.finalizedEpoch,
+      localFinalizedRoot: toRootHex(localStatus.finalizedRoot),
+      localHeadSlot: localStatus.headSlot,
+      localHeadRoot: toRootHex(localStatus.headRoot),
+      remoteFinalizedEpoch: data.status.finalizedEpoch,
+      remoteFinalizedRoot: toRootHex(data.status.finalizedRoot),
+      remoteHeadSlot: data.status.headSlot,
+      remoteHeadRoot: toRootHex(data.status.headRoot),
+    });
 
     // For metrics only
     this.peerSyncType.set(data.peer, syncType);

package/src/sync/unknownBlock.ts

@@ -42,6 +42,7 @@ import {
 } from "./types.js";
 import {DownloadByRootError, downloadByRoot} from "./utils/downloadByRoot.js";
 import {getAllDescendantBlocks, getUnknownAndAncestorBlocks} from "./utils/pendingBlocksTree.js";
+import {getRateLimitedUntilMs} from "./utils/rateLimit.js";
 
 const MAX_ATTEMPTS_PER_BLOCK = 5;
 const MAX_KNOWN_BAD_BLOCKS = 500;
@@ -63,6 +64,13 @@ enum FetchResult {
   FailureMaxAttempts = "failure_max_attempts",
 }
 
+class UnknownBlockRateLimitedError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "UnknownBlockRateLimitedError";
+  }
+}
+
 /**
  * BlockInputSync is a class that handles ReqResp to find blocks and data related to a specific blockRoot.  The
  * blockRoot may have been found via object gossip, or the API.  Gossip objects that can trigger a search are block,
@@ -106,6 +114,7 @@ export class BlockInputSync {
   private readonly maxPendingBlocks;
   private subscribedToNetworkEvents = false;
   private peerBalancer: UnknownBlockPeerBalancer;
+  private rateLimitBackoffTimeout: NodeJS.Timeout | undefined;
 
   constructor(
     private readonly config: ChainForkConfig,
@@ -155,6 +164,7 @@ export class BlockInputSync {
 
   unsubscribeFromNetwork(): void {
     this.logger.verbose("BlockInputSync disabled.");
+    this.clearRateLimitBackoffTimer();
     this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
     this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
     this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
@@ -402,6 +412,7 @@ export class BlockInputSync {
   private onPeerDisconnected = (data: NetworkEventData[NetworkEvent.peerDisconnected]): void => {
     const peerId = data.peer;
     this.peerBalancer.onPeerDisconnected(peerId);
+    this.scheduleRateLimitBackoffRetry();
   };
 
   /**
@@ -519,7 +530,7 @@ export class BlockInputSync {
    */
   private triggerUnknownBlockSearch = (): void => {
     // Cheap early stop to prevent calling the network.getConnectedPeers()
-    if (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0) {
+    if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
       return;
     }
 
@@ -607,6 +618,36 @@ export class BlockInputSync {
     }
   };
 
+  private scheduleRateLimitBackoffRetry(): void {
+    this.clearRateLimitBackoffTimer();
+
+    if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
+      return;
+    }
+
+    const now = Date.now();
+    const retryAt = this.peerBalancer.getNextRateLimitRetryAt();
+    if (retryAt === null) {
+      return;
+    }
+
+    this.rateLimitBackoffTimeout = setTimeout(
+      () => {
+        this.rateLimitBackoffTimeout = undefined;
+        this.triggerUnknownBlockSearch();
+        this.scheduleRateLimitBackoffRetry();
+      },
+      Math.max(0, retryAt - now)
+    );
+  }
+
+  private clearRateLimitBackoffTimer(): void {
+    if (this.rateLimitBackoffTimeout !== undefined) {
+      clearTimeout(this.rateLimitBackoffTimeout);
+      this.rateLimitBackoffTimeout = undefined;
+    }
+  }
+
   private async downloadBlock(block: BlockInputSyncCacheItem): Promise<void> {
     if (block.status !== PendingBlockInputStatus.pending) {
       return;
@@ -678,6 +719,16 @@ export class BlockInputSync {
         this.onUnknownBlockRoot({rootHex: pending.blockInput.parentRootHex, source: BlockInputSource.byRoot});
       }
     } else {
+      if (res.err instanceof UnknownBlockRateLimitedError) {
+        const pendingBlock = this.pendingBlocks.get(rootHex);
+        if (pendingBlock) {
+          pendingBlock.status = PendingBlockInputStatus.pending;
+        }
+        this.logger.debug("Deferring unknown block download due to peer rate limit", logCtx, res.err);
+        this.scheduleRateLimitBackoffRetry();
+        return;
+      }
+
       this.metrics?.blockInputSync.downloadedBlocksError.inc();
       this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
       this.removeAndDownScoreAllDescendants(block);
@@ -994,12 +1045,19 @@ export class BlockInputSync {
     let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
 
     let i = 0;
+    let deferredByRateLimit = false;
     while (i++ < this.getMaxDownloadAttempts()) {
       const pendingColumns = payloadInput?.hasAllData()
         ? new Set<number>()
         : new Set(payloadInput?.getMissingSampledColumnMeta().missing ?? []);
       const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
       if (peerMeta === null) {
+        if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
+          throw new UnknownBlockRateLimitedError(
+            `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`
+          );
+        }
+
         throw Error(
           `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
         );
@@ -1076,7 +1134,12 @@ export class BlockInputSync {
           e as Error
         );
 
-        if (e instanceof RequestError) {
+        const rateLimitedUntilMs = getRateLimitedUntilMs(e);
+        if (rateLimitedUntilMs !== null) {
+          deferredByRateLimit = true;
+          this.peerBalancer.onRateLimited(peerId, rateLimitedUntilMs);
+          this.scheduleRateLimitBackoffRetry();
+        } else if (e instanceof RequestError) {
           switch (e.type.code) {
             case RequestErrorCode.REQUEST_RATE_LIMITED:
             case RequestErrorCode.REQUEST_TIMEOUT:
@@ -1093,6 +1156,12 @@ export class BlockInputSync {
       }
     }
 
+    if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
+      throw new UnknownBlockRateLimitedError(
+        `Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`
+      );
+    }
+
     throw Error(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts.`);
   }
 
@@ -1176,6 +1245,7 @@ export class BlockInputSync {
     }
 
     let i = 0;
+    let deferredByRateLimit = false;
     while (i++ < this.getMaxDownloadAttempts()) {
       const pendingColumns =
         isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput)
@@ -1183,6 +1253,12 @@ export class BlockInputSync {
           : defaultPendingColumns;
       const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
       if (peerMeta === null) {
+        if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
+          throw new UnknownBlockRateLimitedError(
+            `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`
+          );
+        }
+
         // no more peer with needed columns to try, throw error
         const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
         this.metrics?.blockInputSync.fetchTimeSec.observe(
@@ -1238,16 +1314,23 @@ export class BlockInputSync {
         } else if (e instanceof RequestError) {
           // should look into req_resp metrics in this case
           downloadByRootMetrics?.error.inc({code: "req_resp", client: peerClient});
-          switch (e.type.code) {
-            case RequestErrorCode.REQUEST_RATE_LIMITED:
-            case RequestErrorCode.RESP_RATE_LIMITED:
-            case RequestErrorCode.REQUEST_SELF_RATE_LIMITED:
-            case RequestErrorCode.REQUEST_TIMEOUT:
-              // do not exclude peer for these errors
-              break;
-            default:
-              excludedPeers.add(peerId);
-              break;
+          const rateLimitedUntilMs = getRateLimitedUntilMs(e);
+          if (rateLimitedUntilMs !== null) {
+            deferredByRateLimit = true;
+            this.peerBalancer.onRateLimited(peerId, rateLimitedUntilMs);
+            this.scheduleRateLimitBackoffRetry();
+          } else {
+            switch (e.type.code) {
+              case RequestErrorCode.REQUEST_RATE_LIMITED:
+              case RequestErrorCode.RESP_RATE_LIMITED:
+              case RequestErrorCode.REQUEST_SELF_RATE_LIMITED:
+              case RequestErrorCode.REQUEST_TIMEOUT:
+                // do not exclude peer for these errors
+                break;
+              default:
+                excludedPeers.add(peerId);
+                break;
+            }
           }
         } else {
           // investigate if this happens
@@ -1275,6 +1358,10 @@ export class BlockInputSync {
 
     const message = `Error fetching BlockInput with slot=${slot} root=${rootHex} after ${i - 1} attempts.`;
 
+    if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
+      throw new UnknownBlockRateLimitedError(`${message} Peers are rate-limited.`);
+    }
+
     if (!isPendingBlockInput(cacheItem)) {
       throw Error(`${message} No block and no data was found.`);
     }
@@ -1406,10 +1493,12 @@ export class BlockInputSync {
 export class UnknownBlockPeerBalancer {
   readonly peersMeta: Map<PeerIdStr, PeerSyncMeta>;
   readonly activeRequests: Map<PeerIdStr, number>;
+  readonly rateLimitedUntilByPeer: Map<PeerIdStr, number>;
 
   constructor() {
     this.peersMeta = new Map();
     this.activeRequests = new Map();
+    this.rateLimitedUntilByPeer = new Map();
   }
 
   /** Trigger on each peer re-status */
@@ -1424,6 +1513,41 @@ export class UnknownBlockPeerBalancer {
   onPeerDisconnected(peerId: PeerIdStr): void {
     this.peersMeta.delete(peerId);
     this.activeRequests.delete(peerId);
+    this.rateLimitedUntilByPeer.delete(peerId);
+  }
+
+  onRateLimited(peerId: PeerIdStr, rateLimitedUntilMs: number): void {
+    this.rateLimitedUntilByPeer.set(peerId, rateLimitedUntilMs);
+  }
+
+  getNextRateLimitRetryAt(pendingColumns?: Set<number>, excludedPeers?: Set<PeerIdStr>): number | null {
+    const now = Date.now();
+    let retryAt: number | null = null;
+
+    for (const [peerId, rateLimitedUntil] of this.rateLimitedUntilByPeer.entries()) {
+      if (rateLimitedUntil <= now) {
+        this.rateLimitedUntilByPeer.delete(peerId);
+        continue;
+      }
+
+      if (excludedPeers?.has(peerId)) {
+        continue;
+      }
+
+      const syncMeta = this.peersMeta.get(peerId);
+      if (syncMeta === undefined) {
+        this.rateLimitedUntilByPeer.delete(peerId);
+        continue;
+      }
+
+      if (pendingColumns !== undefined && !this.peerHasPendingColumns(syncMeta, pendingColumns)) {
+        continue;
+      }
+
+      retryAt = Math.min(retryAt ?? rateLimitedUntil, rateLimitedUntil);
+    }
+
+    return retryAt;
   }
 
   /**
@@ -1472,6 +1596,7 @@ export class UnknownBlockPeerBalancer {
   }
 
   private filterPeers(pendingDataColumns: Set<number>, excludedPeers: Set<PeerIdStr>): PeerIdStr[] {
+    const now = Date.now();
     let maxColumnCount = 0;
     const considerPeers: {peerId: PeerIdStr; columnCount: number}[] = [];
     for (const [peerId, syncMeta] of this.peersMeta.entries()) {
@@ -1480,12 +1605,24 @@ export class UnknownBlockPeerBalancer {
         continue;
       }
 
+      const rateLimitedUntil = this.rateLimitedUntilByPeer.get(peerId);
+      if (rateLimitedUntil !== undefined) {
+        if (now < rateLimitedUntil) {
+          continue;
+        }
+        this.rateLimitedUntilByPeer.delete(peerId);
+      }
+
       const activeRequests = this.activeRequests.get(peerId) ?? 0;
       if (activeRequests >= MAX_CONCURRENT_REQUESTS) {
         // should return peer with no more than MAX_CONCURRENT_REQUESTS active requests
         continue;
       }
 
+      if (!this.peerHasPendingColumns(syncMeta, pendingDataColumns)) {
+        continue;
+      }
+
       if (pendingDataColumns.size === 0) {
         considerPeers.push({peerId, columnCount: 0});
         continue;
@@ -1519,4 +1656,12 @@ export class UnknownBlockPeerBalancer {
 
     return eligiblePeers;
   }
+
+  private peerHasPendingColumns(syncMeta: PeerSyncMeta, pendingDataColumns: Set<number>): boolean {
+    if (pendingDataColumns.size === 0) {
+      return true;
+    }
+
+    return syncMeta.custodyColumns.some((column) => pendingDataColumns.has(column));
+  }
 }