@lodestar/beacon-node 1.42.0 → 1.43.0-dev.05a33e512f

package/src/network/reqresp/handlers/blobSidecarsByRange.ts

@@ -2,7 +2,7 @@ import {ChainConfig} from "@lodestar/config";
 import {BLOB_SIDECAR_FIXED_SIZE, GENESIS_SLOT} from "@lodestar/params";
 import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
-import {Slot, deneb} from "@lodestar/types";
+import {Epoch, Slot, deneb} from "@lodestar/types";
 import {fromHex} from "@lodestar/utils";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
@@ -14,37 +14,43 @@ export async function* onBlobSidecarsByRange(
   db: IBeaconDb
 ): AsyncIterable<ResponseOutgoing> {
   // Non-finalized range of blobs
-  const {startSlot, count} = validateBlobSidecarsByRangeRequest(chain.config, request);
+  const {startSlot, count} = validateBlobSidecarsByRangeRequest(chain.config, chain.clock.currentEpoch, request);
   const endSlot = startSlot + count;
 
   const finalized = db.blobSidecarsArchive;
   const unfinalized = db.blobSidecars;
   const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
+  // Blobs are migrated to blobSidecarsArchive at finalization (including the finalized block
+  // itself), so the archive loop serves up to AND INCLUDING finalizedSlot and the headChain
+  // loop starts above it to avoid duplicate yields. See archiveBlocks.ts for the migration logic.
+  const archiveMaxSlot = finalizedSlot;
 
   // Finalized range of blobs
-  if (startSlot <= finalizedSlot) {
+  if (startSlot <= archiveMaxSlot) {
     // Chain of blobs won't change
     for await (const {key, value: blobSideCarsBytesWrapped} of finalized.binaryEntriesStream({
       gte: startSlot,
-      lt: endSlot,
+      lt: Math.min(endSlot, archiveMaxSlot + 1),
     })) {
       yield* iterateBlobBytesFromWrapper(chain, blobSideCarsBytesWrapped, finalized.decodeKey(key));
     }
   }
 
   // Non-finalized range of blobs
-  if (endSlot > finalizedSlot) {
+  if (endSlot > archiveMaxSlot) {
     const headBlock = chain.forkChoice.getHead();
     const headRoot = headBlock.blockRoot;
     // TODO DENEB: forkChoice should mantain an array of canonical blocks, and change only on reorg
     const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
+    // `getAllAncestorBlocks` includes both the head and the previous-finalized boundary.
 
     // Iterate head chain with ascending block numbers
     for (let i = headChain.length - 1; i >= 0; i--) {
       const block = headChain[i];
 
-      // Must include only blobs in the range requested
-      if (block.slot >= startSlot && block.slot < endSlot) {
+      // Must include only blobs in the range requested, and skip anything the archive loop
+      // above already served via the block.slot > archiveMaxSlot filter.
+      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
         // Note: Here the forkChoice head may change due to a re-org, so the headChain reflects the canonical chain
         // at the time of the start of the request. Spec is clear the chain of blobs must be consistent, but on
         // re-org there's no need to abort the request
@@ -94,6 +100,7 @@ export function* iterateBlobBytesFromWrapper(
 
 export function validateBlobSidecarsByRangeRequest(
   config: ChainConfig,
+  currentEpoch: Epoch,
   request: deneb.BlobSidecarsByRangeRequest
 ): deneb.BlobSidecarsByRangeRequest {
   const {startSlot} = request;
@@ -102,11 +109,22 @@ export function validateBlobSidecarsByRangeRequest(
   if (count < 1) {
     throw new ResponseError(RespStatus.INVALID_REQUEST, "count < 1");
   }
-  // TODO: validate against MIN_EPOCHS_FOR_BLOCK_REQUESTS
   if (startSlot < GENESIS_SLOT) {
     throw new ResponseError(RespStatus.INVALID_REQUEST, "startSlot < genesis");
   }
 
+  // Spec: [max(current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, DENEB_FORK_EPOCH), current_epoch]
+  const minimumRequestEpoch = Math.max(
+    currentEpoch - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS,
+    config.DENEB_FORK_EPOCH
+  );
+  if (computeEpochAtSlot(startSlot) < minimumRequestEpoch) {
+    throw new ResponseError(
+      RespStatus.RESOURCE_UNAVAILABLE,
+      "startSlot is before MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS"
+    );
+  }
+
   if (count > config.MAX_REQUEST_BLOCKS_DENEB) {
     count = config.MAX_REQUEST_BLOCKS_DENEB;
   }

package/src/network/reqresp/handlers/blobSidecarsByRoot.ts

@@ -12,6 +12,13 @@ export async function* onBlobSidecarsByRoot(
 ): AsyncIterable<ResponseOutgoing> {
   const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
 
+  // Spec: [max(current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, DENEB_FORK_EPOCH), current_epoch]
+  const currentEpoch = chain.clock.currentEpoch;
+  const minimumRequestEpoch = Math.max(
+    currentEpoch - chain.config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS,
+    chain.config.DENEB_FORK_EPOCH
+  );
+
   // In sidecars by root request, it can be expected that sidecar requests will be come
   // clustured by blockroots, and this helps us save db lookups once we load sidecars
   // for a root
@@ -29,6 +36,10 @@ export async function* onBlobSidecarsByRoot(
       continue;
     }
 
+    if (computeEpochAtSlot(block.slot) < minimumRequestEpoch) {
+      continue;
+    }
+
     // Check if we need to load sidecars for a new block root
     if (lastFetchedSideCars === null || lastFetchedSideCars.blockRoot !== blockRootHex) {
       const blobSidecarsBytes = await chain.getSerializedBlobSidecars(block.slot, blockRootHex);

package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts

@@ -1,9 +1,9 @@
 import {PeerId} from "@libp2p/interface";
 import {ChainConfig} from "@lodestar/config";
-import {GENESIS_SLOT} from "@lodestar/params";
+import {ForkSeq, GENESIS_SLOT} from "@lodestar/params";
 import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
-import {ColumnIndex, fulu} from "@lodestar/types";
+import {ColumnIndex, Epoch, fulu} from "@lodestar/types";
 import {fromHex} from "@lodestar/utils";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
@@ -21,7 +21,11 @@ export async function* onDataColumnSidecarsByRange(
   peerClient: string
 ): AsyncIterable<ResponseOutgoing> {
   // Non-finalized range of columns
-  const {startSlot, count, columns: requestedColumns} = validateDataColumnSidecarsByRangeRequest(chain.config, request);
+  const {
+    startSlot,
+    count,
+    columns: requestedColumns,
+  } = validateDataColumnSidecarsByRangeRequest(chain.config, chain.clock.currentEpoch, request);
   const availableColumns = validateRequestedDataColumns(chain, requestedColumns);
   const endSlot = startSlot + count;
 
@@ -39,10 +43,19 @@ export async function* onDataColumnSidecarsByRange(
 
   const finalized = db.dataColumnSidecarArchive;
   const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
+  // Columns of the last finalized block live in different DBs depending on fork:
+  // - Pre-gloas (fulu): migrated to dataColumnSidecarArchive in the same finalization run.
+  // - Post-gloas: stay in the hot db (db.dataColumnSidecar) until the next finalization run,
+  //   because the migration filter requires payloadStatus === FULL for gloas blocks.
+  // archiveMaxSlot is the last slot whose columns are served by the archive loop below;
+  // anything above it is served by the headChain loop.
+  const isPostGloasFinalized = chain.config.getForkSeq(finalizedSlot) >= ForkSeq.gloas;
+  const archiveMaxSlot = isPostGloasFinalized ? finalizedSlot - 1 : finalizedSlot;
 
   // Finalized range of columns
-  if (startSlot <= finalizedSlot) {
-    for (let slot = startSlot; slot < endSlot; slot++) {
+  if (startSlot <= archiveMaxSlot) {
+    const archiveEnd = Math.min(endSlot, archiveMaxSlot + 1);
+    for (let slot = startSlot; slot < archiveEnd; slot++) {
       const dataColumnSidecars = await finalized.getManyBinary(slot, availableColumns);
 
       const unavailableColumnIndices: ColumnIndex[] = [];
@@ -77,9 +90,12 @@ export async function* onDataColumnSidecarsByRange(
   }
 
   // Non-finalized range of columns
-  if (endSlot > finalizedSlot) {
+  if (endSlot > archiveMaxSlot) {
     const headBlock = chain.forkChoice.getHead();
     const headRoot = headBlock.blockRoot;
+    // getAllAncestorBlocks includes the last finalized block as its final element.
+    // Skip anything the archive loop above already served via the block.slot > archiveMaxSlot
+    // filter below (pre-gloas this skips finalizedSlot, post-gloas it keeps it).
     const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
 
     // Iterate head chain with ascending block numbers
@@ -87,7 +103,7 @@ export async function* onDataColumnSidecarsByRange(
       const block = headChain[i];
 
       // Must include only columns in the range requested
-      if (block.slot >= startSlot && block.slot < endSlot) {
+      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
         // Note: Here the forkChoice head may change due to a re-org, so the headChain reflects the canonical chain
         // at the time of the start of the request. Spec is clear the chain of columns must be consistent, but on
         // re-org there's no need to abort the request
@@ -139,6 +155,7 @@ export async function* onDataColumnSidecarsByRange(
 
 export function validateDataColumnSidecarsByRangeRequest(
   config: ChainConfig,
+  currentEpoch: Epoch,
   request: fulu.DataColumnSidecarsByRangeRequest
 ): fulu.DataColumnSidecarsByRangeRequest {
   const {startSlot, columns} = request;
@@ -147,11 +164,22 @@ export function validateDataColumnSidecarsByRangeRequest(
   if (count < 1) {
     throw new ResponseError(RespStatus.INVALID_REQUEST, "count < 1");
   }
-  // TODO: validate against MIN_EPOCHS_FOR_BLOCK_REQUESTS
   if (startSlot < GENESIS_SLOT) {
     throw new ResponseError(RespStatus.INVALID_REQUEST, "startSlot < genesis");
   }
 
+  // Spec: [max(current_epoch - MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS, FULU_FORK_EPOCH), current_epoch]
+  const minimumRequestEpoch = Math.max(
+    currentEpoch - config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS,
+    config.FULU_FORK_EPOCH
+  );
+  if (computeEpochAtSlot(startSlot) < minimumRequestEpoch) {
+    throw new ResponseError(
+      RespStatus.RESOURCE_UNAVAILABLE,
+      "startSlot is before MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS"
+    );
+  }
+
   if (count > config.MAX_REQUEST_BLOCKS_DENEB) {
     count = config.MAX_REQUEST_BLOCKS_DENEB;
   }

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRange.ts

@@ -21,12 +21,15 @@ export async function* onExecutionPayloadEnvelopesByRange(
 
   const finalized = db.executionPayloadEnvelopeArchive;
   const finalizedSlot = chain.forkChoice.getFinalizedCheckpointSlot();
+  // The current finalized block's envelope is still in the hot db; archive migration happens
+  // in the next finalization run (see migrateExecutionPayloadEnvelopesFromHotToColdDb).
+  const archiveMaxSlot = finalizedSlot - 1;
 
   // Finalized range of envelopes
-  if (startSlot <= finalizedSlot) {
+  if (startSlot <= archiveMaxSlot) {
     for await (const {key, value: envelopeBytes} of finalized.binaryEntriesStream({
       gte: startSlot,
-      lt: endSlot,
+      lt: Math.min(endSlot, archiveMaxSlot + 1),
     })) {
       const slot = finalized.decodeKey(key);
       yield {
@@ -37,7 +40,7 @@ export async function* onExecutionPayloadEnvelopesByRange(
   }
 
   // Non-finalized range of envelopes
-  if (endSlot > finalizedSlot) {
+  if (endSlot > archiveMaxSlot) {
     const headBlock = chain.forkChoice.getHead();
     const headRoot = headBlock.blockRoot;
     const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
@@ -46,7 +49,7 @@ export async function* onExecutionPayloadEnvelopesByRange(
     for (let i = headChain.length - 1; i >= 0; i--) {
       const block = headChain[i];
 
-      if (block.slot >= startSlot && block.slot < endSlot) {
+      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
         // Skip EMPTY blocks
         if (block.payloadStatus !== PayloadStatus.FULL) {
           continue;
@@ -81,11 +84,13 @@ export function validateExecutionPayloadEnvelopesByRangeRequest(
   if (count < 1) {
     throw new ResponseError(RespStatus.INVALID_REQUEST, "count < 1");
   }
-  // TODO: validate against MIN_EPOCHS_FOR_BLOCK_REQUESTS
   if (startSlot < GENESIS_SLOT) {
     throw new ResponseError(RespStatus.INVALID_REQUEST, "startSlot < genesis");
   }
 
+  // The gloas req/resp spec uses MIN_EPOCHS_FOR_BLOCK_REQUESTS to define the minimum range peers MUST serve.
+  // Archival nodes may still serve older retained payloads to allow genesis sync.
+
   if (count > config.MAX_REQUEST_BLOCKS_DENEB) {
     count = config.MAX_REQUEST_BLOCKS_DENEB;
   }

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRoot.ts

@@ -10,12 +10,8 @@ export async function* onExecutionPayloadEnvelopesByRoot(
   chain: IBeaconChain,
   db: IBeaconDb
 ): AsyncIterable<ResponseOutgoing> {
-  // Spec: [max(GLOAS_FORK_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOCK_REQUESTS), current_epoch]
-  const currentEpoch = chain.clock.currentEpoch;
-  const minimumRequestEpoch = Math.max(
-    currentEpoch - chain.config.MIN_EPOCHS_FOR_BLOCK_REQUESTS,
-    chain.config.GLOAS_FORK_EPOCH
-  );
+  // The gloas req/resp spec uses MIN_EPOCHS_FOR_BLOCK_REQUESTS to define the minimum range peers MUST serve.
+  // Archival nodes may still serve older retained payloads to allow genesis sync.
 
   for (const root of requestBody) {
     const rootHex = toRootHex(root);
@@ -27,16 +23,11 @@ export async function* onExecutionPayloadEnvelopesByRoot(
       continue;
     }
 
-    const requestedEpoch = computeEpochAtSlot(slot);
-    if (requestedEpoch < minimumRequestEpoch) {
-      continue;
-    }
-
     const envelopeBytes = await chain.getSerializedExecutionPayloadEnvelope(slot, rootHex);
     if (envelopeBytes) {
       yield {
         data: envelopeBytes,
-        boundary: chain.config.getForkBoundaryAtEpoch(requestedEpoch),
+        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(slot)),
       };
     }
   }

package/src/node/nodejs.ts

@@ -6,7 +6,7 @@ import {BeaconApiMethods} from "@lodestar/api/beacon/server";
 import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ZERO_HASH_HEX} from "@lodestar/params";
-import {IBeaconStateView, PubkeyCache} from "@lodestar/state-transition";
+import {IBeaconStateView, PubkeyCache, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
 import {phase0} from "@lodestar/types";
 import {sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
@@ -221,11 +221,16 @@ export class BeaconNode {
 
     let executionEngineOpts = opts.executionEngine;
     if (opts.executionEngine.mode === "mock") {
-      const eth1BlockHash = anchorState.isExecutionStateType ? toRootHex(anchorState.latestBlockHash) : undefined;
+      const latestEth1BlockHash =
+        isStatePostBellatrix(anchorState) && anchorState.isExecutionStateType
+          ? isStatePostGloas(anchorState)
+            ? toRootHex(anchorState.latestBlockHash)
+            : toRootHex(anchorState.latestExecutionPayloadHeader.blockHash)
+          : undefined;
       executionEngineOpts = {
         ...opts.executionEngine,
         genesisBlockHash: ZERO_HASH_HEX,
-        eth1BlockHash,
+        eth1BlockHash: opts.executionEngine.eth1BlockHash ?? latestEth1BlockHash,
         genesisTime: anchorState.genesisTime,
         config,
       };

package/src/node/notifier.ts

@@ -1,7 +1,12 @@
 import {BeaconConfig} from "@lodestar/config";
 import {ExecutionStatus, ProtoBlock} from "@lodestar/fork-choice";
 import {EPOCHS_PER_SYNC_COMMITTEE_PERIOD, SLOTS_PER_EPOCH} from "@lodestar/params";
-import {IBeaconStateView, computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {
+  IBeaconStateView,
+  computeEpochAtSlot,
+  computeStartSlotAtEpoch,
+  isStatePostBellatrix,
+} from "@lodestar/state-transition";
 import {Epoch} from "@lodestar/types";
 import {ErrorAborted, Logger, prettyBytes, prettyBytesShort, sleep} from "@lodestar/utils";
 import {IBeaconChain} from "../chain/index.js";
@@ -165,7 +170,7 @@ function getHeadExecutionInfo(
   const executionStatusStr = headInfo.executionStatus.toLowerCase();
 
   // Add execution status to notifier only if head is on/post bellatrix
-  if (headState.isExecutionStateType) {
+  if (isStatePostBellatrix(headState) && headState.isExecutionStateType) {
     if (headState.isMergeTransitionComplete) {
       const executionPayloadHashInfo =
         headInfo.executionStatus !== ExecutionStatus.PreMerge ? headInfo.executionPayloadBlockHash : "empty";

package/src/sync/constants.ts

@@ -5,10 +5,10 @@ export const PARALLEL_HEAD_CHAINS = 2;
 export const MIN_FINALIZED_CHAIN_VALIDATED_EPOCHS = 10;
 
 /** The number of times to retry a batch before it is considered failed. */
-// export const MAX_BATCH_DOWNLOAD_ATTEMPTS = 5;
-// this constant is increased a lot for peerDAS because we may have many failed download due to rate limit not implemented yet
-// TODO: change it back to 5 when this issue is implemented https://github.com/ChainSafe/lodestar/issues/8033
-export const MAX_BATCH_DOWNLOAD_ATTEMPTS = 20;
+export const MAX_BATCH_DOWNLOAD_ATTEMPTS = 5;
+
+/** Backoff before assigning more range-sync batches to a peer that rate-limited us. */
+export const RATE_LIMITED_PEER_BACKOFF_MS = 5_000;
 
 /**
  * Consider batch faulty after downloading and processing this number of times