@lodestar/beacon-node 1.43.0-dev.aef3645690 → 1.43.0-dev.b0e1143d06

package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts

@@ -1,6 +1,6 @@
 import {PeerId} from "@libp2p/interface";
 import {ChainConfig} from "@lodestar/config";
-import {GENESIS_SLOT} from "@lodestar/params";
+import {ForkSeq, GENESIS_SLOT} from "@lodestar/params";
 import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {ColumnIndex, Epoch, fulu} from "@lodestar/types";
@@ -43,10 +43,19 @@ export async function* onDataColumnSidecarsByRange(
 
   const finalized = db.dataColumnSidecarArchive;
   const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
+  // Columns of the last finalized block live in different DBs depending on fork:
+  // - Pre-gloas (fulu): migrated to dataColumnSidecarArchive in the same finalization run.
+  // - Post-gloas: stay in the hot db (db.dataColumnSidecar) until the next finalization run,
+  //   because the migration filter requires payloadStatus === FULL for gloas blocks.
+  // archiveMaxSlot is the last slot whose columns are served by the archive loop below;
+  // anything above it is served by the headChain loop.
+  const isPostGloasFinalized = chain.config.getForkSeq(finalizedSlot) >= ForkSeq.gloas;
+  const archiveMaxSlot = isPostGloasFinalized ? finalizedSlot - 1 : finalizedSlot;
 
   // Finalized range of columns
-  if (startSlot <= finalizedSlot) {
-    for (let slot = startSlot; slot < endSlot; slot++) {
+  if (startSlot <= archiveMaxSlot) {
+    const archiveEnd = Math.min(endSlot, archiveMaxSlot + 1);
+    for (let slot = startSlot; slot < archiveEnd; slot++) {
       const dataColumnSidecars = await finalized.getManyBinary(slot, availableColumns);
 
       const unavailableColumnIndices: ColumnIndex[] = [];
@@ -81,9 +90,12 @@ export async function* onDataColumnSidecarsByRange(
   }
 
   // Non-finalized range of columns
-  if (endSlot > finalizedSlot) {
+  if (endSlot > archiveMaxSlot) {
     const headBlock = chain.forkChoice.getHead();
     const headRoot = headBlock.blockRoot;
+    // getAllAncestorBlocks includes the last finalized block as its final element.
+    // Skip anything the archive loop above already served via the block.slot > archiveMaxSlot
+    // filter below (pre-gloas this skips finalizedSlot, post-gloas it keeps it).
     const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
 
     // Iterate head chain with ascending block numbers
@@ -91,7 +103,7 @@ export async function* onDataColumnSidecarsByRange(
       const block = headChain[i];
 
       // Must include only columns in the range requested
-      if (block.slot >= startSlot && block.slot < endSlot) {
+      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
         // Note: Here the forkChoice head may change due to a re-org, so the headChain reflects the canonical chain
         // at the time of the start of the request. Spec is clear the chain of columns must be consistent, but on
         // re-org there's no need to abort the request

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRange.ts

@@ -21,12 +21,15 @@ export async function* onExecutionPayloadEnvelopesByRange(
 
   const finalized = db.executionPayloadEnvelopeArchive;
   const finalizedSlot = chain.forkChoice.getFinalizedCheckpointSlot();
+  // The current finalized block's envelope is still in the hot db; archive migration happens
+  // in the next finalization run (see migrateExecutionPayloadEnvelopesFromHotToColdDb).
+  const archiveMaxSlot = finalizedSlot - 1;
 
   // Finalized range of envelopes
-  if (startSlot <= finalizedSlot) {
+  if (startSlot <= archiveMaxSlot) {
     for await (const {key, value: envelopeBytes} of finalized.binaryEntriesStream({
       gte: startSlot,
-      lt: endSlot,
+      lt: Math.min(endSlot, archiveMaxSlot + 1),
     })) {
       const slot = finalized.decodeKey(key);
       yield {
@@ -37,7 +40,7 @@ export async function* onExecutionPayloadEnvelopesByRange(
   }
 
   // Non-finalized range of envelopes
-  if (endSlot > finalizedSlot) {
+  if (endSlot > archiveMaxSlot) {
     const headBlock = chain.forkChoice.getHead();
     const headRoot = headBlock.blockRoot;
     const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
@@ -46,7 +49,7 @@ export async function* onExecutionPayloadEnvelopesByRange(
     for (let i = headChain.length - 1; i >= 0; i--) {
       const block = headChain[i];
 
-      if (block.slot >= startSlot && block.slot < endSlot) {
+      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
         // Skip EMPTY blocks
         if (block.payloadStatus !== PayloadStatus.FULL) {
           continue;

package/src/network/reqresp/handlers/index.ts

@@ -9,6 +9,7 @@ import {
   ExecutionPayloadEnvelopesByRootRequestType,
 } from "../../../util/types.js";
 import {GetReqRespHandlerFn, ReqRespMethod} from "../types.js";
+import {onBeaconBlocksByHead} from "./beaconBlocksByHead.js";
 import {onBeaconBlocksByRange} from "./beaconBlocksByRange.js";
 import {onBeaconBlocksByRoot} from "./beaconBlocksByRoot.js";
 import {onBlobSidecarsByRange} from "./blobSidecarsByRange.js";
@@ -47,6 +48,10 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
       const body = BeaconBlocksByRootRequestType(fork, chain.config).deserialize(req.data);
       return onBeaconBlocksByRoot(body, chain);
     },
+    [ReqRespMethod.BeaconBlocksByHead]: (req, peerId, peerClient) => {
+      const body = ssz.fulu.BeaconBlocksByHeadRequest.deserialize(req.data);
+      return onBeaconBlocksByHead(body, chain, peerId, peerClient);
+    },
     [ReqRespMethod.BlobSidecarsByRoot]: (req) => {
       const fork = chain.config.getForkName(chain.clock.currentSlot);
       const body = BlobSidecarsByRootRequestType(fork, chain.config).deserialize(req.data);

package/src/network/reqresp/interface.ts

@@ -33,7 +33,7 @@ export enum RespStatus {
    */
   SERVER_ERROR = 2,
   /**
-   * The responder does not have requested resource.  The response payload adheres to the ErrorMessage schema (described below). Note: This response code is only valid as a response to BlocksByRange
+   * The responder does not have requested resource. The response payload adheres to the ErrorMessage schema.
    */
   RESOURCE_UNAVAILABLE = 3,
   /**

package/src/network/reqresp/protocols.ts

@@ -70,6 +70,12 @@ export const BeaconBlocksByRootV2 = toProtocol({
   contextBytesType: ContextBytesType.ForkDigest,
 });
 
+export const BeaconBlocksByHead = toProtocol({
+  method: ReqRespMethod.BeaconBlocksByHead,
+  version: Version.V1,
+  contextBytesType: ContextBytesType.ForkDigest,
+});
+
 export const BlobSidecarsByRange = toProtocol({
   method: ReqRespMethod.BlobSidecarsByRange,
   version: Version.V1,

package/src/network/reqresp/rateLimit.ts

@@ -40,6 +40,10 @@ export const rateLimitQuotas: (fork: ForkName, config: BeaconConfig) => Record<R
     },
     getRequestCount: getRequestCountFn(fork, config, ReqRespMethod.BeaconBlocksByRoot, (req) => req.length),
   },
+  [ReqRespMethod.BeaconBlocksByHead]: {
+    byPeer: {quota: config.MAX_REQUEST_BLOCKS_DENEB, quotaTimeMs: 10_000},
+    getRequestCount: getRequestCountFn(fork, config, ReqRespMethod.BeaconBlocksByHead, (req) => req.count),
+  },
   [ReqRespMethod.BlobSidecarsByRange]: {
     // Rationale: MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK
     byPeer: {

package/src/network/reqresp/score.ts

@@ -46,6 +46,7 @@ export function onOutgoingReqRespError(e: RequestError, method: ReqRespMethod):
           return PeerAction.LowToleranceError;
         case ReqRespMethod.BeaconBlocksByRange:
         case ReqRespMethod.BeaconBlocksByRoot:
+        case ReqRespMethod.BeaconBlocksByHead:
         case ReqRespMethod.ExecutionPayloadEnvelopesByRoot:
         case ReqRespMethod.ExecutionPayloadEnvelopesByRange:
           return PeerAction.MidToleranceError;

package/src/network/reqresp/types.ts

@@ -42,6 +42,7 @@ export enum ReqRespMethod {
   Metadata = "metadata",
   BeaconBlocksByRange = "beacon_blocks_by_range",
   BeaconBlocksByRoot = "beacon_blocks_by_root",
+  BeaconBlocksByHead = "beacon_blocks_by_head",
   BlobSidecarsByRange = "blob_sidecars_by_range",
   BlobSidecarsByRoot = "blob_sidecars_by_root",
   DataColumnSidecarsByRange = "data_column_sidecars_by_range",
@@ -62,6 +63,7 @@ export type RequestBodyByMethod = {
   [ReqRespMethod.Metadata]: null;
   [ReqRespMethod.BeaconBlocksByRange]: phase0.BeaconBlocksByRangeRequest;
   [ReqRespMethod.BeaconBlocksByRoot]: BeaconBlocksByRootRequest;
+  [ReqRespMethod.BeaconBlocksByHead]: fulu.BeaconBlocksByHeadRequest;
   [ReqRespMethod.BlobSidecarsByRange]: deneb.BlobSidecarsByRangeRequest;
   [ReqRespMethod.BlobSidecarsByRoot]: BlobSidecarsByRootRequest;
   [ReqRespMethod.DataColumnSidecarsByRange]: fulu.DataColumnSidecarsByRangeRequest;
@@ -82,6 +84,7 @@ type ResponseBodyByMethod = {
   // Do not matter
   [ReqRespMethod.BeaconBlocksByRange]: SignedBeaconBlock;
   [ReqRespMethod.BeaconBlocksByRoot]: SignedBeaconBlock;
+  [ReqRespMethod.BeaconBlocksByHead]: SignedBeaconBlock;
   [ReqRespMethod.BlobSidecarsByRange]: deneb.BlobSidecar;
   [ReqRespMethod.BlobSidecarsByRoot]: deneb.BlobSidecar;
   [ReqRespMethod.DataColumnSidecarsByRange]: DataColumnSidecar;
@@ -111,6 +114,7 @@ export const requestSszTypeByMethod: (
 
   [ReqRespMethod.BeaconBlocksByRange]: ssz.phase0.BeaconBlocksByRangeRequest,
   [ReqRespMethod.BeaconBlocksByRoot]: BeaconBlocksByRootRequestType(fork, config),
+  [ReqRespMethod.BeaconBlocksByHead]: ssz.fulu.BeaconBlocksByHeadRequest,
   [ReqRespMethod.BlobSidecarsByRange]: ssz.deneb.BlobSidecarsByRangeRequest,
   [ReqRespMethod.BlobSidecarsByRoot]: BlobSidecarsByRootRequestType(fork, config),
   [ReqRespMethod.DataColumnSidecarsByRange]: ssz.fulu.DataColumnSidecarsByRangeRequest,
@@ -142,6 +146,7 @@ export const responseSszTypeByMethod: {[K in ReqRespMethod]: ResponseTypeGetter<
     version === Version.V1 ? ssz.phase0.Metadata : version === Version.V2 ? ssz.altair.Metadata : ssz.fulu.Metadata,
   [ReqRespMethod.BeaconBlocksByRange]: blocksResponseType,
   [ReqRespMethod.BeaconBlocksByRoot]: blocksResponseType,
+  [ReqRespMethod.BeaconBlocksByHead]: (fork) => ssz[fork].SignedBeaconBlock,
   [ReqRespMethod.BlobSidecarsByRange]: () => ssz.deneb.BlobSidecar,
   [ReqRespMethod.BlobSidecarsByRoot]: () => ssz.deneb.BlobSidecar,
   [ReqRespMethod.LightClientBootstrap]: (fork) => sszTypesFor(onlyPostAltairFork(fork)).LightClientBootstrap,

package/src/node/nodejs.ts

@@ -6,7 +6,7 @@ import {BeaconApiMethods} from "@lodestar/api/beacon/server";
 import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ZERO_HASH_HEX} from "@lodestar/params";
-import {IBeaconStateView, PubkeyCache, isStatePostBellatrix} from "@lodestar/state-transition";
+import {IBeaconStateView, PubkeyCache, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
 import {phase0} from "@lodestar/types";
 import {sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
@@ -221,14 +221,16 @@ export class BeaconNode {
 
     let executionEngineOpts = opts.executionEngine;
     if (opts.executionEngine.mode === "mock") {
-      const eth1BlockHash =
+      const latestEth1BlockHash =
         isStatePostBellatrix(anchorState) && anchorState.isExecutionStateType
-          ? toRootHex(anchorState.latestBlockHash)
+          ? isStatePostGloas(anchorState)
+            ? toRootHex(anchorState.latestBlockHash)
+            : toRootHex(anchorState.latestExecutionPayloadHeader.blockHash)
           : undefined;
       executionEngineOpts = {
         ...opts.executionEngine,
         genesisBlockHash: ZERO_HASH_HEX,
-        eth1BlockHash,
+        eth1BlockHash: opts.executionEngine.eth1BlockHash ?? latestEth1BlockHash,
         genesisTime: anchorState.genesisTime,
         config,
       };

package/src/sync/constants.ts

@@ -5,10 +5,14 @@ export const PARALLEL_HEAD_CHAINS = 2;
 export const MIN_FINALIZED_CHAIN_VALIDATED_EPOCHS = 10;
 
 /** The number of times to retry a batch before it is considered failed. */
-// export const MAX_BATCH_DOWNLOAD_ATTEMPTS = 5;
-// this constant is increased a lot for peerDAS because we may have many failed download due to rate limit not implemented yet
-// TODO: change it back to 5 when this issue is implemented https://github.com/ChainSafe/lodestar/issues/8033
-export const MAX_BATCH_DOWNLOAD_ATTEMPTS = 20;
+export const MAX_BATCH_DOWNLOAD_ATTEMPTS = 5;
+
+/**
+ * Backoff before assigning more range-sync batches to a peer that rate-limited us.
+ *
+ * Note: this is used when rate limited due to MAX_CONCURRENT_REQUESTS
+ */
+export const RATE_LIMITED_PEER_BACKOFF_MS = 5_000;
 
 /**
  * Consider batch faulty after downloading and processing this number of times