@lodestar/beacon-node 1.43.0-dev.bc569affb9 → 1.43.0-dev.be5f70c0c8

package/src/sync/range/chain.ts

@@ -1,9 +1,10 @@
 import {ChainForkConfig} from "@lodestar/config";
-import {Epoch, Root, Slot} from "@lodestar/types";
-import {ErrorAborted, LodestarError, Logger, toRootHex} from "@lodestar/utils";
+import {Epoch, Root, Slot, gloas} from "@lodestar/types";
+import {ErrorAborted, LodestarError, Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js";
 import {BlockInputErrorCode} from "../../chain/blocks/blockInput/errors.js";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
+import {PayloadEnvelopeInput} from "../../chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js";
 import {BlobSidecarErrorCode} from "../../chain/errors/blobSidecarError.js";
 import {DataColumnSidecarErrorCode} from "../../chain/errors/dataColumnSidecarError.js";
 import {Metrics} from "../../metrics/metrics.js";
@@ -16,6 +17,7 @@ import {PeerIdStr} from "../../util/peerId.js";
 import {WarnResult, wrapError} from "../../util/wrapError.js";
 import {BATCH_BUFFER_SIZE, EPOCHS_PER_BATCH, MAX_LOOK_AHEAD_EPOCHS} from "../constants.js";
 import {DownloadByRangeError, DownloadByRangeErrorCode} from "../utils/downloadByRange.js";
+import {getRateLimitedUntilMs} from "../utils/rateLimit.js";
 import {RangeSyncType} from "../utils/remoteSyncType.js";
 import {Batch, BatchError, BatchErrorCode, BatchMetadata, BatchStatus} from "./batch.js";
 import {
@@ -44,13 +46,19 @@ export type SyncChainFns = {
    * Must return if ALL blocks are processed successfully
    * If SOME blocks are processed must throw BlockProcessorError()
    */
-  processChainSegment: (blocks: IBlockInput[], syncType: RangeSyncType) => Promise<void>;
+  processChainSegment: (
+    blocks: IBlockInput[],
+    payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null,
+    syncType: RangeSyncType
+  ) => Promise<void>;
   /** Must download blocks, and validate their range */
   downloadByRange: (
     peer: PeerSyncMeta,
     batch: Batch,
     syncType: RangeSyncType
-  ) => Promise<WarnResult<IBlockInput[], DownloadByRangeError>>;
+  ) => Promise<
+    WarnResult<{blocks: IBlockInput[]; payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null}, DownloadByRangeError>
+  >;
   /** Report peer for negative actions. Decouples from the full network instance */
   reportPeer: (peer: PeerIdStr, action: PeerAction, actionName: string) => void;
   /** Gets current peer custodyColumns and earliestAvailableSlot */
@@ -132,20 +140,33 @@ export class SyncChain {
   private readonly batchProcessor = new ItTrigger();
   /** Sorted map of batches undergoing some kind of processing. */
   private readonly batches = new Map<Epoch, Batch>();
+  /**
+   * `true` until the first `Batch` is constructed via `includeNextBatch`
+   */
+  private isFirstBatch = true;
   private readonly peerset = new Map<PeerIdStr, ChainTarget>();
+  /**
+   * Tracks peers that have rate-limited us, mapped to the timestamp (ms) until which we should avoid them.
+   * This is a sync-layer optimization to avoid assigning batches to backed-off peers.
+   * The reqresp SelfRateLimiter independently enforces backoff at the protocol level as a safety net.
+   */
+  private readonly rateLimitedPeers = new Map<PeerIdStr, number>();
+  private rateLimitBackoffTimeout: NodeJS.Timeout | undefined;
 
   private readonly logger: Logger;
   private readonly config: ChainForkConfig;
   private readonly clock: IClock;
   private readonly metrics: Metrics | null;
   private readonly custodyConfig: CustodyConfig;
+  private readonly latestBid: gloas.ExecutionPayloadBid | undefined;
 
   constructor(
     initialBatchEpoch: Epoch,
     initialTarget: ChainTarget,
     syncType: RangeSyncType,
     fns: SyncChainFns,
-    modules: SyncChainModules
+    modules: SyncChainModules,
+    latestBid: gloas.ExecutionPayloadBid | undefined
   ) {
     const {config, clock, custodyConfig, logger, metrics} = modules;
     this.firstBatchEpoch = initialBatchEpoch;
@@ -161,6 +182,7 @@ export class SyncChain {
     this.clock = clock;
     this.metrics = metrics;
     this.custodyConfig = custodyConfig;
+    this.latestBid = latestBid;
     this.logger = logger;
     this.logId = `${syncType}-${nextChainId++}`;
 
@@ -215,12 +237,16 @@ export class SyncChain {
    */
   stopSyncing(): void {
     this.status = SyncChainStatus.Stopped;
+    this.clearRateLimitBackoffTimer();
+    this.logger.debug("SyncChain stopSyncing", {id: this.logId});
   }
 
   /**
    * Permanently remove this chain. Throws the main AsyncIterable
    */
   remove(): void {
+    this.logger.debug("SyncChain remove", {id: this.logId});
+    this.clearRateLimitBackoffTimer();
     this.batchProcessor.end(new ErrorAborted("SyncChain"));
   }
 
@@ -239,6 +265,7 @@ export class SyncChain {
    */
   removePeer(peerId: PeerIdStr): boolean {
     const deleted = this.peerset.delete(peerId);
+    this.rateLimitedPeers.delete(peerId);
     this.computeTarget();
     return deleted;
   }
@@ -342,6 +369,8 @@ export class SyncChain {
       }
 
       throw e;
+    } finally {
+      this.clearRateLimitBackoffTimer();
     }
   }
 
@@ -365,6 +394,44 @@ export class SyncChain {
     }
   }
 
+  private scheduleRateLimitBackoffRetry(): void {
+    this.clearRateLimitBackoffTimer();
+
+    if (this.status !== SyncChainStatus.Syncing || this.rateLimitedPeers.size === 0) {
+      return;
+    }
+
+    const now = Date.now();
+    let retryAt: number | null = null;
+    for (const [peerId, rateLimitedUntil] of this.rateLimitedPeers.entries()) {
+      if (rateLimitedUntil <= now) {
+        this.rateLimitedPeers.delete(peerId);
+        continue;
+      }
+      retryAt = Math.min(retryAt ?? rateLimitedUntil, rateLimitedUntil);
+    }
+
+    if (retryAt === null) {
+      return;
+    }
+
+    this.rateLimitBackoffTimeout = setTimeout(
+      () => {
+        this.rateLimitBackoffTimeout = undefined;
+        this.triggerBatchDownloader();
+        this.scheduleRateLimitBackoffRetry();
+      },
+      Math.max(0, retryAt - now)
+    );
+  }
+
+  private clearRateLimitBackoffTimer(): void {
+    if (this.rateLimitBackoffTimeout !== undefined) {
+      clearTimeout(this.rateLimitBackoffTimeout);
+      this.rateLimitBackoffTimeout = undefined;
+    }
+  }
+
   /**
    * Attempts to request the next required batches from the peer pool if the chain is syncing.
    * It will exhaust the peer pool and left over batches until the batch buffer is reached.
@@ -374,8 +441,18 @@ export class SyncChain {
       return;
     }
 
+    const now = Date.now();
     const peersSyncInfo: PeerSyncInfo[] = [];
     for (const [peerId, target] of this.peerset.entries()) {
+      // Skip peers that are currently in rate-limit backoff
+      const rateLimitedUntil = this.rateLimitedPeers.get(peerId);
+      if (rateLimitedUntil !== undefined) {
+        if (now < rateLimitedUntil) {
+          continue;
+        }
+        this.rateLimitedPeers.delete(peerId);
+      }
+
       try {
         peersSyncInfo.push({...this.getConnectedPeerSyncMeta(peerId), target});
       } catch (e) {
@@ -449,7 +526,17 @@ export class SyncChain {
       return null;
     }
 
-    const batch = new Batch(startEpoch, this.config, this.clock, this.custodyConfig);
+    const batch = new Batch(
+      startEpoch,
+      this.config,
+      this.clock,
+      this.custodyConfig,
+      this.isFirstBatch,
+      // `latestBid` is only meaningful for the first batch's parent-payload check
+      this.isFirstBatch ? this.latestBid : undefined,
+      this.target.slot
+    );
+    this.isFirstBatch = false;
     this.batches.set(startEpoch, batch);
     return batch;
   }
@@ -507,7 +594,16 @@ export class SyncChain {
           {id: this.logId, ...batch.getMetadata(), peer: prettyPrintPeerIdStr(peer.peerId)},
           res.err
         );
-        batch.downloadingError(peer.peerId); // Throws after MAX_DOWNLOAD_ATTEMPTS
+        const rateLimitedUntilMs = getRateLimitedUntilMs(res.err);
+        if (rateLimitedUntilMs !== null) {
+          // Peer rate-limited us — don't count as a failed download attempt and mark peer for backoff
+          this.rateLimitedPeers.set(peer.peerId, rateLimitedUntilMs);
+          this.scheduleRateLimitBackoffRetry();
+          batch.downloadingRateLimited();
+          this.triggerBatchDownloader();
+        } else {
+          batch.downloadingError(peer.peerId); // Throws after MAX_DOWNLOAD_ATTEMPTS
+        }
       } else {
         this.logger.verbose("Batch download success", {
           id: this.logId,
@@ -516,7 +612,8 @@ export class SyncChain {
         });
         this.metrics?.syncRange.downloadByRange.success.inc();
         const {warnings, result} = res.result;
-        const downloadSuccessOutput = batch.downloadingSuccess(peer.peerId, result);
+        const {blocks: downloadedBlocks, payloadEnvelopes} = result;
+        const downloadSuccessOutput = batch.downloadingSuccess(peer.peerId, downloadedBlocks, payloadEnvelopes);
         const logMeta: Record<string, number> = {
           blockCount: downloadSuccessOutput.blocks.length,
         };
@@ -526,7 +623,7 @@ export class SyncChain {
             this.metrics?.syncRange.downloadByRange.warn.inc({client: peer.client, code: warning.type.code});
             this.logger.debug(
               "Batch downloaded with warning",
-              {id: this.logId, epoch: batch.startEpoch, ...logMeta, peer: prettyPrintPeerIdStr(peer.peerId)},
+              {id: this.logId, ...batch.getMetadata(), ...logMeta, peer: prettyPrintPeerIdStr(peer.peerId)},
               warning
             );
           }
@@ -552,10 +649,17 @@ export class SyncChain {
           // the flow will continue to call triggerBatchDownloader() below
         }
 
+        const blockSlots = downloadSuccessOutput.blocks.map((b) => b.slot);
+        const envelopeSlots = downloadSuccessOutput.payloadEnvelopes
+          ? Array.from(downloadSuccessOutput.payloadEnvelopes.keys())
+          : null;
+
         this.logger.debug(logMessage, {
           id: this.logId,
-          epoch: batch.startEpoch,
+          ...batch.getMetadata(),
           ...logMeta,
+          blockSlots: prettyPrintIndices(blockSlots),
+          ...(envelopeSlots ? {envelopeSlots: prettyPrintIndices(envelopeSlots)} : {}),
           peer: prettyPrintPeerIdStr(peer.peerId),
         });
       }
@@ -578,13 +682,24 @@ export class SyncChain {
    * Sends `batch` to the processor. Note: batch may be empty
    */
   private async processBatch(batch: Batch): Promise<void> {
-    const blocks = batch.startProcessing();
+    const {blocks, payloadEnvelopes, peers} = batch.startProcessing();
+
+    const logCtx = {
+      id: this.logId,
+      ...batch.getMetadata(),
+      blockCount: blocks.length,
+      blockSlots: prettyPrintIndices(blocks.map((b) => b.slot)),
+      ...(payloadEnvelopes ? {envelopeSlots: prettyPrintIndices(Array.from(payloadEnvelopes.keys()))} : {}),
+      peers: peers.map(prettyPrintPeerIdStr).join(","),
+    };
+    this.logger.verbose("Processing batch", logCtx);
 
     // wrapError ensures to never call both batch success() and batch error()
-    const res = await wrapError(this.processChainSegment(blocks, this.syncType));
+    const res = await wrapError(this.processChainSegment(blocks, payloadEnvelopes, this.syncType));
 
     if (!res.err) {
       batch.processingSuccess();
+      this.logger.verbose("Processed batch", {...logCtx, ...batch.getMetadata()});
 
       // If the processed batch is not empty, validate previous AwaitingValidation blocks.
       if (blocks.length > 0) {
@@ -594,7 +709,7 @@ export class SyncChain {
       // Potentially process next AwaitingProcessing batch
       this.triggerBatchProcessor();
     } else {
-      this.logger.verbose("Batch process error", {id: this.logId, ...batch.getMetadata()}, res.err);
+      this.logger.verbose("Batch process error", logCtx, res.err);
       batch.processingError(res.err); // Throws after MAX_BATCH_PROCESSING_ATTEMPTS
 
       // At least one block was successfully verified and imported, so we can be sure all

package/src/sync/range/range.ts

@@ -1,7 +1,7 @@
 import {EventEmitter} from "node:events";
 import {StrictEventEmitter} from "strict-event-emitter-types";
 import {BeaconConfig} from "@lodestar/config";
-import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {IBeaconStateViewGloas, computeStartSlotAtEpoch, isStatePostGloas} from "@lodestar/state-transition";
 import {Epoch, Status, fulu} from "@lodestar/types";
 import {Logger, toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
@@ -172,7 +172,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
   }
 
   /** Convenience method for `SyncChain` */
-  private processChainSegment: SyncChainFns["processChainSegment"] = async (blocks, syncType) => {
+  private processChainSegment: SyncChainFns["processChainSegment"] = async (blocks, payloadEnvelopes, syncType) => {
     // Not trusted, verify signatures
     const flags: ImportBlockOpts = {
       // Only skip importing attestations for finalized sync. For head sync attestation are valuable.
@@ -192,30 +192,46 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
 
     if (this.opts?.disableProcessAsChainSegment) {
       // Should only be used for debugging or testing
-      for (const block of blocks) await this.chain.processBlock(block, flags);
+      for (const block of blocks) {
+        await this.chain.processBlock(block, flags);
+        const payloadEnvelope = payloadEnvelopes?.get(block.slot);
+        if (payloadEnvelope) {
+          await this.chain.processExecutionPayload(payloadEnvelope);
+        }
+      }
     } else {
-      await this.chain.processChainSegment(blocks, flags);
+      await this.chain.processChainSegment(blocks, payloadEnvelopes, flags);
     }
   };
 
   private downloadByRange: SyncChainFns["downloadByRange"] = async (peer, batch) => {
     const batchBlocks = batch.getBlocks();
+    const requests = batch.getRequestsForPeer(peer);
+    const parentRoot = requests.parentPayloadRequest?.envelopeBlockRoot ?? requests.parentPayloadRequest?.blockRoot;
+    const parentPayloadCommitments = parentRoot ? batch.getParentPayloadCommitments(parentRoot) : undefined;
     const {result, warnings} = await downloadByRange({
       config: this.config,
       network: this.network,
       logger: this.logger,
       peerIdStr: peer.peerId,
       batchBlocks,
+      parentPayloadCommitments,
       peerDasMetrics: this.chain.metrics?.peerDas,
-      ...batch.getRequestsForPeer(peer),
+      ...requests,
     });
-    const cached = cacheByRangeResponses({
+    const {responses, payloadEnvelopes: downloadedPayloadEnvelopes} = result;
+    const {blocks, payloadEnvelopes} = cacheByRangeResponses({
       cache: this.chain.seenBlockInputCache,
+      seenPayloadEnvelopeInputCache: this.chain.seenPayloadEnvelopeInputCache,
       peerIdStr: peer.peerId,
-      responses: result,
+      responses,
       batchBlocks,
+      downloadedPayloadEnvelopes,
+      existingPayloadEnvelopes: batch.getPayloadEnvelopes(),
+      custodyConfig: this.chain.custodyConfig,
+      seenTimestampSec: Date.now() / 1000,
     });
-    return {result: cached, warnings};
+    return {result: {blocks, payloadEnvelopes}, warnings};
   };
 
   private pruneBlockInputs: SyncChainFns["pruneBlockInputs"] = (blocks: IBlockInput[]) => {
@@ -246,6 +262,14 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
   private addPeerOrCreateChain(startEpoch: Epoch, target: ChainTarget, peer: PeerIdStr, syncType: RangeSyncType): void {
     let syncChain = this.chains.get(syncType);
     if (!syncChain) {
+      // The first batch of a new sync chain may need to detect whether the parent block was an
+      // gloas "empty" block (no envelope produced). It does so by comparing the first
+      // downloaded block's `bid.parentBlockHash` against the head state's `latestExecutionPayloadBid.blockHash`.
+      const headState = this.chain.getHeadState();
+      const latestBid = isStatePostGloas(headState)
+        ? (headState as IBeaconStateViewGloas).latestExecutionPayloadBid
+        : undefined;
+
       syncChain = new SyncChain(
         startEpoch,
         target,
@@ -264,7 +288,8 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
           logger: this.logger,
           custodyConfig: this.chain.custodyConfig,
           metrics: this.metrics,
-        }
+        },
+        latestBid
       );
       this.chains.set(syncType, syncChain);
 

package/src/sync/sync.ts

@@ -1,6 +1,6 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {Slot} from "@lodestar/types";
-import {Logger} from "@lodestar/utils";
+import {Logger, toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../chain/index.js";
 import {GENESIS_SLOT} from "../constants/constants.js";
 import {ExecutionEngineState} from "../execution/index.js";
@@ -188,6 +188,18 @@ export class BeaconSync implements IBeaconSync {
   private addPeer = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
     const localStatus = this.chain.getStatus();
     const syncType = getPeerSyncType(localStatus, data.status, this.chain.forkChoice, this.slotImportTolerance);
+    this.logger.verbose("Peer sync type classified", {
+      peer: data.peer,
+      syncType,
+      localFinalizedEpoch: localStatus.finalizedEpoch,
+      localFinalizedRoot: toRootHex(localStatus.finalizedRoot),
+      localHeadSlot: localStatus.headSlot,
+      localHeadRoot: toRootHex(localStatus.headRoot),
+      remoteFinalizedEpoch: data.status.finalizedEpoch,
+      remoteFinalizedRoot: toRootHex(data.status.finalizedRoot),
+      remoteHeadSlot: data.status.headSlot,
+      remoteHeadRoot: toRootHex(data.status.headRoot),
+    });
 
     // For metrics only
     this.peerSyncType.set(data.peer, syncType);

package/src/sync/types.ts

@@ -1,5 +1,8 @@
 import {RootHex, Slot} from "@lodestar/types";
+import {SignedExecutionPayloadEnvelope} from "@lodestar/types/gloas";
+import {toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../chain/blocks/blockInput/index.js";
+import {PayloadEnvelopeInput} from "../chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js";
 
 export enum PendingBlockType {
   /**
@@ -26,6 +29,14 @@ export enum PendingBlockInputStatus {
   processing = "processing",
 }
 
+export enum PendingPayloadInputStatus {
+  pending = "pending",
+  fetching = "fetching",
+  waitingForBlock = "waiting_for_block",
+  downloaded = "downloaded",
+  processing = "processing",
+}
+
 export type PendingBlockInput = {
   status: PendingBlockInputStatus;
   blockInput: IBlockInput;
@@ -44,10 +55,47 @@ export type PendingRootHex = {
 
 export type BlockInputSyncCacheItem = PendingBlockInput | PendingRootHex;
 
+export type PendingPayloadInput = {
+  status:
+    | PendingPayloadInputStatus.pending
+    | PendingPayloadInputStatus.fetching
+    | PendingPayloadInputStatus.downloaded
+    | PendingPayloadInputStatus.processing;
+  payloadInput: PayloadEnvelopeInput;
+  timeAddedSec: number;
+  timeSyncedSec?: number;
+  peerIdStrings: Set<string>;
+};
+
+export type PendingPayloadRootHex = {
+  status: PendingPayloadInputStatus.pending | PendingPayloadInputStatus.fetching;
+  rootHex: RootHex;
+  timeAddedSec: number;
+  timeSyncedSec?: number;
+  peerIdStrings: Set<string>;
+};
+
+export type PendingPayloadEnvelope = {
+  status: PendingPayloadInputStatus.waitingForBlock;
+  envelope: SignedExecutionPayloadEnvelope;
+  timeAddedSec: number;
+  peerIdStrings: Set<string>;
+};
+
+export type PayloadSyncCacheItem = PendingPayloadInput | PendingPayloadRootHex | PendingPayloadEnvelope;
+
 export function isPendingBlockInput(pending: BlockInputSyncCacheItem): pending is PendingBlockInput {
   return "blockInput" in pending;
 }
 
+export function isPendingPayloadInput(pending: PayloadSyncCacheItem): pending is PendingPayloadInput {
+  return "payloadInput" in pending;
+}
+
+export function isPendingPayloadEnvelope(pending: PayloadSyncCacheItem): pending is PendingPayloadEnvelope {
+  return "envelope" in pending;
+}
+
 export function getBlockInputSyncCacheItemRootHex(block: BlockInputSyncCacheItem): RootHex {
   return isPendingBlockInput(block) ? block.blockInput.blockRootHex : block.rootHex;
 }
@@ -55,3 +103,27 @@ export function getBlockInputSyncCacheItemRootHex(block: BlockInputSyncCacheItem
 export function getBlockInputSyncCacheItemSlot(block: BlockInputSyncCacheItem): Slot | string {
   return isPendingBlockInput(block) ? block.blockInput.slot : "unknown";
 }
+
+export function getPayloadSyncCacheItemRootHex(payload: PayloadSyncCacheItem): RootHex {
+  if (isPendingPayloadInput(payload)) {
+    return payload.payloadInput.blockRootHex;
+  }
+
+  if (isPendingPayloadEnvelope(payload)) {
+    return toRootHex(payload.envelope.message.beaconBlockRoot);
+  }
+
+  return payload.rootHex;
+}
+
+export function getPayloadSyncCacheItemSlot(payload: PayloadSyncCacheItem): Slot | string {
+  if (isPendingPayloadInput(payload)) {
+    return payload.payloadInput.slot;
+  }
+
+  if (isPendingPayloadEnvelope(payload)) {
+    return payload.envelope.message.payload.slotNumber;
+  }
+
+  return "unknown";
+}