@lodestar/beacon-node 1.43.0-dev.a6d8600051 → 1.43.0-dev.ac258a8def

package/src/sync/utils/downloadByRange.ts

@@ -8,7 +8,9 @@ import {
   isForkPostGloas,
 } from "@lodestar/params";
 import {
+  ColumnIndex,
   DataColumnSidecar,
+  RootHex,
   SignedBeaconBlock,
   Slot,
   deneb,
@@ -45,6 +47,22 @@ export type DownloadByRangeRequests = {
   blobsRequest?: deneb.BlobSidecarsByRangeRequest;
   columnsRequest?: fulu.DataColumnSidecarsByRangeRequest;
   envelopesRequest?: gloas.ExecutionPayloadEnvelopesByRangeRequest;
+  /**
+   * Post-Gloas only. Fetches the dangling-parent's payload envelope and/or its missing sampled
+   * data columns by-root. Set by `Batch` for the first batch of a `SyncChain` after the first
+   * block's parentRoot is known.
+   */
+  parentPayloadRequest?: {
+    blockRoot?: Uint8Array;
+    columns?: ColumnIndex[];
+    envelopeBlockRoot?: Uint8Array;
+  };
+};
+
+export type ParentPayloadCommitments = {
+  blockRoot: Uint8Array;
+  blockRootHex: RootHex;
+  kzgCommitments: deneb.BlobKzgCommitments;
 };
 
 export type DownloadByRangeResponses = {
@@ -60,6 +78,8 @@ export type DownloadAndCacheByRangeProps = DownloadByRangeRequests & {
   logger: Logger;
   peerIdStr: string;
   batchBlocks?: IBlockInput[];
+  /** Required when `parentPayloadRequest` is set; supplies the data needed to validate the parent's columns. */
+  parentPayloadCommitments?: ParentPayloadCommitments;
   peerDasMetrics?: BeaconMetrics["peerDas"] | null;
 };
 
@@ -185,33 +205,33 @@ export function cacheByRangeResponses({
     }
   }
 
-  // Build payloadEnvelopes map for gloas: start from existing (partial download) state.
-  // The entries are wrappers around (block + envelope + sampled columns) and also seeded into
-  // seenPayloadEnvelopeInputCache so importBlock can find them without creating a duplicate.
-  let payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null = null;
-  if (downloadedPayloadEnvelopes !== null) {
-    payloadEnvelopes = new Map(existingPayloadEnvelopes ?? []);
+  // Seed seenPayloadEnvelopeInputCache for every gloas block in the batch, regardless of whether
+  // the peer returned its envelope. Without this, a block returned without its envelope would be
+  // imported with no cache entry, and later payload-by-root sync would throw
+  // "Missing PayloadEnvelopeInput for known block" (see issue #9306).
+  for (const blockInput of updatedBatchBlocks.values()) {
+    if (!blockInput.hasBlock() || !isForkPostGloas(blockInput.forkName)) continue;
+    seenPayloadEnvelopeInputCache.add({
+      blockRootHex: blockInput.blockRootHex,
+      block: blockInput.getBlock() as SignedBeaconBlock<ForkPostGloas>,
+      forkName: blockInput.forkName,
+      sampledColumns: custodyConfig.sampledColumns,
+      custodyColumns: custodyConfig.custodyColumns,
+      timeCreatedSec: seenTimestampSec,
+    });
+  }
 
+  let payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null =
+    existingPayloadEnvelopes !== null ? new Map(existingPayloadEnvelopes) : null;
+  if (downloadedPayloadEnvelopes !== null) {
+    payloadEnvelopes ??= new Map();
     for (const [slot, envelope] of downloadedPayloadEnvelopes) {
-      const blockInput = updatedBatchBlocks.get(slot);
-      if (!blockInput?.hasBlock() || !isForkPostGloas(blockInput.forkName)) {
-        // No block to pair this envelope with; drop silently
-        continue;
-      }
-      const {blockRootHex} = blockInput;
-
-      // Reuse any existing PayloadEnvelopeInput (e.g. gossip arrived first) to avoid
-      // duplicate cache entries. If missing, create a fresh one from the block's bid.
-      let payloadInput = seenPayloadEnvelopeInputCache.get(blockRootHex);
+      const envelopeBlockRootHex = toRootHex(envelope.message.beaconBlockRoot);
+      const payloadInput = seenPayloadEnvelopeInputCache.get(envelopeBlockRootHex);
       if (payloadInput === undefined) {
-        payloadInput = seenPayloadEnvelopeInputCache.add({
-          blockRootHex,
-          block: blockInput.getBlock() as SignedBeaconBlock<ForkPostGloas>,
-          forkName: blockInput.forkName,
-          sampledColumns: custodyConfig.sampledColumns,
-          custodyColumns: custodyConfig.custodyColumns,
-          timeCreatedSec: seenTimestampSec,
-        });
+        // Unreachable given the loop above seeded an entry for every gloas block in the batch.
+        // for the parent block, it's populated at BeaconChain init
+        throw new Error(`Missing PayloadEnvelopeInput for block ${envelopeBlockRootHex}`);
       }
 
       if (!payloadInput.hasPayloadEnvelope()) {
@@ -301,6 +321,8 @@ export async function downloadByRange({
   blobsRequest,
   columnsRequest,
   envelopesRequest,
+  parentPayloadRequest,
+  parentPayloadCommitments,
   peerDasMetrics,
 }: DownloadAndCacheByRangeProps): Promise<
   WarnResult<
@@ -317,6 +339,7 @@ export async function downloadByRange({
       blobsRequest,
       columnsRequest,
       envelopesRequest,
+      parentPayloadRequest,
     });
   } catch (err) {
     throw new DownloadByRangeError({
@@ -333,6 +356,8 @@ export async function downloadByRange({
     blobsRequest,
     columnsRequest,
     envelopesRequest,
+    parentPayloadRequest,
+    parentPayloadCommitments,
     peerDasMetrics,
     ...response,
   });
@@ -348,14 +373,15 @@ export async function requestByRange({
   blobsRequest,
   columnsRequest,
   envelopesRequest,
+  parentPayloadRequest,
 }: DownloadByRangeRequests & {
   network: INetwork;
   peerIdStr: PeerIdStr;
 }): Promise<DownloadByRangeResponses> {
   let blocks: undefined | SignedBeaconBlock[];
   let blobSidecars: undefined | deneb.BlobSidecars;
-  let columnSidecars: undefined | DataColumnSidecar[];
-  let payloadEnvelopes: undefined | gloas.SignedExecutionPayloadEnvelope[];
+  const columnSidecars: DataColumnSidecar[] = [];
+  const payloadEnvelopes: gloas.SignedExecutionPayloadEnvelope[] = [];
 
   const requests: Promise<unknown>[] = [];
 
@@ -378,7 +404,7 @@ export async function requestByRange({
   if (columnsRequest) {
     requests.push(
       network.sendDataColumnSidecarsByRange(peerIdStr, columnsRequest).then((columnResponse) => {
-        columnSidecars = columnResponse;
+        columnSidecars.push(...columnResponse);
       })
     );
   }
@@ -386,11 +412,34 @@ export async function requestByRange({
   if (envelopesRequest) {
     requests.push(
       network.sendExecutionPayloadEnvelopesByRange(peerIdStr, envelopesRequest).then((envelopeResponse) => {
-        payloadEnvelopes = envelopeResponse;
+        payloadEnvelopes.push(...envelopeResponse);
       })
     );
   }
 
+  // Only happens on the 1st batch of a SyncChain — fetches whichever pieces of the
+  // dangling-parent's payload are still missing from the seen-cache entry.
+  if (parentPayloadRequest?.envelopeBlockRoot) {
+    requests.push(
+      network
+        .sendExecutionPayloadEnvelopesByRoot(peerIdStr, [parentPayloadRequest.envelopeBlockRoot])
+        .then((envelopeResponse) => {
+          payloadEnvelopes.push(...envelopeResponse);
+        })
+    );
+  }
+  if (parentPayloadRequest?.blockRoot && parentPayloadRequest.columns && parentPayloadRequest.columns.length > 0) {
+    requests.push(
+      network
+        .sendDataColumnSidecarsByRoot(peerIdStr, [
+          {blockRoot: parentPayloadRequest.blockRoot, columns: parentPayloadRequest.columns},
+        ])
+        .then((columnResponse) => {
+          columnSidecars.push(...columnResponse);
+        })
+    );
+  }
+
   await Promise.all(requests);
 
   return {
@@ -411,6 +460,8 @@ export async function validateResponses({
   blobsRequest,
   columnsRequest,
   envelopesRequest,
+  parentPayloadRequest,
+  parentPayloadCommitments,
   blocks,
   blobSidecars,
   columnSidecars,
@@ -420,6 +471,7 @@ export async function validateResponses({
   DownloadByRangeResponses & {
     config: ChainForkConfig;
     batchBlocks?: IBlockInput[];
+    parentPayloadCommitments?: ParentPayloadCommitments;
     peerDasMetrics?: BeaconMetrics["peerDas"] | null;
   }): Promise<
   WarnResult<
@@ -440,6 +492,11 @@ export async function validateResponses({
     );
   }
 
+  // `parentPayloadRequest` and `parentPayloadCommitments` must be supplied together
+  if ((parentPayloadRequest === undefined) !== (parentPayloadCommitments === undefined)) {
+    throw new Error("Coding error: parentPayloadRequest and parentPayloadCommitments must be both set or both unset");
+  }
+
   const validatedResponses: ValidatedResponses = {};
   let warnings: DownloadByRangeError[] | null = null;
 
@@ -451,21 +508,30 @@ export async function validateResponses({
     validatedResponses.validatedBlocks = result.result;
   }
 
+  const needsEnvelopeValidation = !!envelopesRequest || parentPayloadCommitments !== undefined;
   const dataRequest = blobsRequest ?? columnsRequest;
-  if (!dataRequest && !envelopesRequest) {
+  if (!dataRequest && !needsEnvelopeValidation) {
     return {result: {responses: validatedResponses, payloadEnvelopes: null}, warnings};
   }
 
   if (!dataRequest) {
-    // Only envelope validation needed
-    let validatedPayloadEnvelopes: Map<Slot, gloas.SignedExecutionPayloadEnvelope> | null = null;
-    if (envelopesRequest) {
-      validatedPayloadEnvelopes = validateEnvelopesByRangeResponse(
-        validatedResponses.validatedBlocks ?? [],
-        batchBlocks,
-        payloadEnvelopes ?? []
+    // Only envelope and/or parent-by-root validation needed
+    if (parentPayloadCommitments !== undefined) {
+      const parentValidated = await validateParentPayloadColumns(
+        parentPayloadCommitments,
+        columnSidecars ?? [],
+        peerDasMetrics
       );
+      if (parentValidated) {
+        validatedResponses.validatedColumnSidecars = [parentValidated];
+      }
     }
+    const validatedPayloadEnvelopes = validateEnvelopesByRangeResponse(
+      validatedResponses.validatedBlocks ?? [],
+      batchBlocks,
+      payloadEnvelopes ?? [],
+      parentPayloadCommitments
+    );
     return {result: {responses: validatedResponses, payloadEnvelopes: validatedPayloadEnvelopes}, warnings};
   }
 
@@ -524,19 +590,64 @@ export async function validateResponses({
     warnings = validatedColumnSidecarsResult.warnings;
   }
 
-  // Validate envelopes if an envelopes request was made
+  // Parent columns (by-root): KZG-validate against parent's bid commitments and append.
+  if (parentPayloadCommitments !== undefined) {
+    const parentValidated = await validateParentPayloadColumns(
+      parentPayloadCommitments,
+      columnSidecars ?? [],
+      peerDasMetrics
+    );
+    if (parentValidated) {
+      validatedResponses.validatedColumnSidecars = [
+        ...(validatedResponses.validatedColumnSidecars ?? []),
+        parentValidated,
+      ];
+    }
+  }
+
   let validatedPayloadEnvelopes: Map<Slot, gloas.SignedExecutionPayloadEnvelope> | null = null;
-  if (envelopesRequest) {
+  if (needsEnvelopeValidation) {
     validatedPayloadEnvelopes = validateEnvelopesByRangeResponse(
       validatedResponses.validatedBlocks ?? [],
       batchBlocks,
-      payloadEnvelopes ?? []
+      payloadEnvelopes ?? [],
+      parentPayloadCommitments
     );
   }
 
   return {result: {responses: validatedResponses, payloadEnvelopes: validatedPayloadEnvelopes}, warnings};
 }
 
+async function validateParentPayloadColumns(
+  parentPayloadCommitments: ParentPayloadCommitments,
+  columnSidecars: DataColumnSidecar[],
+  peerDasMetrics?: BeaconMetrics["peerDas"] | null
+): Promise<ValidatedColumnSidecars | null> {
+  const parentColumns: gloas.DataColumnSidecar[] = [];
+  for (const cs of columnSidecars) {
+    if (isGloasDataColumnSidecar(cs) && byteArrayEquals(cs.beaconBlockRoot, parentPayloadCommitments.blockRoot)) {
+      parentColumns.push(cs);
+    }
+  }
+
+  if (parentColumns.length === 0) {
+    return null;
+  }
+
+  parentColumns.sort((a, b) => a.index - b.index);
+  const parentSlot = parentColumns[0].slot;
+
+  await validateGloasBlockDataColumnSidecars(
+    parentSlot,
+    parentPayloadCommitments.blockRoot,
+    parentPayloadCommitments.kzgCommitments,
+    parentColumns,
+    peerDasMetrics
+  );
+
+  return {blockRoot: parentPayloadCommitments.blockRoot, columnSidecars: parentColumns};
+}
+
 /**
  * Should not be called directly. Only exported for unit testing purposes
  *
@@ -1044,6 +1155,9 @@ export enum DownloadByRangeErrorCode {
 
   /** Envelope beaconBlockRoot does not match the block's root */
   INVALID_ENVELOPE_BEACON_BLOCK_ROOT = "DOWNLOAD_BY_RANGE_ERROR_INVALID_ENVELOPE_BEACON_BLOCK_ROOT",
+
+  /** Block segment + envelopes failed chain-segment linearity / FULL-chain checks */
+  INVALID_CHAIN_SEGMENT = "DOWNLOAD_BY_RANGE_ERROR_INVALID_CHAIN_SEGMENT",
 }
 
 export type DownloadByRangeErrorType =
@@ -1141,20 +1255,29 @@ export type DownloadByRangeErrorType =
       slot: Slot;
       expected: string;
       actual: string;
+    }
+  | {
+      code: DownloadByRangeErrorCode.INVALID_CHAIN_SEGMENT;
+      slot: Slot;
+      reason: string;
     };
 
 export class DownloadByRangeError extends LodestarError<DownloadByRangeErrorType> {}
 
 /**
  * Validates SignedExecutionPayloadEnvelopes received for a range request.
- * For each envelope whose slot appears in the downloaded blocks, verifies that
- * envelope.message.beaconBlockRoot matches the corresponding block's root.
- * Envelopes for slots not in the batch (orphaned payloads) are silently ignored.
+ *
+ * Three categories of envelope slots:
+ *   - In-batch slot whose block we have: verify envelope.beaconBlockRoot matches.
+ *   - Dangling-parent envelope (only when `parentPayloadCommitments` is set, i.e. the first
+ *     batch of a `SyncChain`): keep if `envelope.beaconBlockRoot === parentPayloadCommitments.blockRoot`.
+ *   - Other "orphan" envelopes (e.g. unrelated slots): ignored.
  */
 export function validateEnvelopesByRangeResponse(
   validatedBlocks: ValidatedBlock[],
   batchBlocks: IBlockInput[] | undefined,
-  payloadEnvelopes: gloas.SignedExecutionPayloadEnvelope[]
+  payloadEnvelopes: gloas.SignedExecutionPayloadEnvelope[],
+  parentPayloadCommitments?: ParentPayloadCommitments
 ): Map<Slot, gloas.SignedExecutionPayloadEnvelope> {
   // Build a map of slot -> blockRoot for all blocks in the batch
   const batchBlockRoots = new Map<Slot, Uint8Array>();
@@ -1173,8 +1296,15 @@ export function validateEnvelopesByRangeResponse(
     const slot = payloadEnvelope.message.payload.slotNumber;
     const batchBlockRoot = batchBlockRoots.get(slot);
 
-    // Envelopes for slots not in the batch are silently ignored (orphaned payloads)
     if (batchBlockRoot === undefined) {
+      // Keep the requested dangling-parent envelope only when its beaconBlockRoot matches
+      // exactly. All other unrelated envelopes are dropped.
+      if (
+        parentPayloadCommitments !== undefined &&
+        byteArrayEquals(payloadEnvelope.message.beaconBlockRoot, parentPayloadCommitments.blockRoot)
+      ) {
+        payloadEnvelopeMap.set(slot, payloadEnvelope);
+      }
       continue;
     }
 

package/src/sync/utils/downloadByRoot.ts

@@ -3,6 +3,7 @@ import {ChainForkConfig} from "@lodestar/config";
 import {
   ForkPostDeneb,
   ForkPostFulu,
+  ForkPostGloas,
   ForkPreFulu,
   isForkPostDeneb,
   isForkPostFulu,
@@ -114,6 +115,17 @@ export async function downloadByRoot({
     });
   }
 
+  if (isForkPostGloas(blockInput.forkName)) {
+    chain.seenPayloadEnvelopeInputCache.add({
+      blockRootHex: rootHex,
+      block: blockInput.getBlock() as SignedBeaconBlock<ForkPostGloas>,
+      forkName: blockInput.forkName,
+      sampledColumns: chain.custodyConfig.sampledColumns,
+      custodyColumns: chain.custodyConfig.custodyColumns,
+      timeCreatedSec: Date.now() / 1000,
+    });
+  }
+
   const hasAllDataPreDownload = blockInput.hasBlockAndAllData();
 
   if (isBlockInputBlobs(blockInput) && !hasAllDataPreDownload) {

package/src/sync/utils/rateLimit.ts

@@ -0,0 +1,16 @@
+import {RequestError, RequestErrorCode} from "@lodestar/reqresp";
+import {RATE_LIMITED_PEER_BACKOFF_MS} from "../constants.js";
+
+export function getRateLimitedUntilMs(e: unknown): number | null {
+  if (!(e instanceof RequestError)) {
+    return null;
+  }
+
+  switch (e.type.code) {
+    case RequestErrorCode.RESP_RATE_LIMITED:
+    case RequestErrorCode.REQUEST_SELF_RATE_LIMITED:
+      return e.type.rateLimitedUntilMs ?? Date.now() + RATE_LIMITED_PEER_BACKOFF_MS;
+    default:
+      return null;
+  }
+}

package/src/util/sszBytes.ts

@@ -558,9 +558,10 @@ export function getBeaconBlockRootFromDataColumnSidecarSerialized(data: Uint8Arr
  * └─ ExecutionPayloadEnvelope (starts at byte 100):
  *    ├─ 4 bytes: payload offset
  *    ├─ 4 bytes: executionRequests offset
- *    ├─ 8 bytes: builderIndex        (offset 108-115)
- *    ├─ 32 bytes: beaconBlockRoot    (offset 116-147)
- *    └─ variable: payload data (starts at envelope + 48)
+ *    ├─ 8 bytes: builderIndex          (offset 108-115)
+ *    ├─ 32 bytes: beaconBlockRoot      (offset 116-147)
+ *    ├─ 32 bytes: parentBeaconBlockRoot (offset 148-179) — new in Gloas alpha.6 (consensus-specs#5152)
+ *    └─ variable: payload data (starts at envelope + 80)
  *       └─ ExecutionPayload fixed portion includes slotNumber at offset 532
  */
 const SIGNED_EXECUTION_PAYLOAD_ENVELOPE_MESSAGE_OFFSET = 4;
@@ -576,12 +577,13 @@ const BEACON_BLOCK_ROOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE =
   EXECUTION_PAYLOAD_ENVELOPE_REQUESTS_OFFSET +
   EXECUTION_PAYLOAD_ENVELOPE_BUILDER_INDEX_SIZE; // 116
 
-// Envelope fixed portion (without slot): payload_offset(4) + requests_offset(4) + builderIndex(8) + beaconBlockRoot(32) = 48
+// Envelope fixed portion: payload_offset(4) + requests_offset(4) + builderIndex(8) + beaconBlockRoot(32) + parentBeaconBlockRoot(32) = 80
 const EXECUTION_PAYLOAD_ENVELOPE_FIXED_SIZE =
   EXECUTION_PAYLOAD_ENVELOPE_PAYLOAD_OFFSET +
   EXECUTION_PAYLOAD_ENVELOPE_REQUESTS_OFFSET +
   EXECUTION_PAYLOAD_ENVELOPE_BUILDER_INDEX_SIZE +
-  ROOT_SIZE; // 48
+  ROOT_SIZE +
+  ROOT_SIZE; // 80
 
 // slotNumber offset within ExecutionPayload fixed portion:
 // parentHash(32) + feeRecipient(20) + stateRoot(32) + receiptsRoot(32) + logsBloom(256) +
@@ -595,7 +597,7 @@ const ENVELOPE_START_IN_SIGNED =
   SIGNED_EXECUTION_PAYLOAD_ENVELOPE_MESSAGE_OFFSET + SIGNED_EXECUTION_PAYLOAD_ENVELOPE_SIGNATURE_SIZE; // 100
 
 const SLOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE =
-  ENVELOPE_START_IN_SIGNED + EXECUTION_PAYLOAD_ENVELOPE_FIXED_SIZE + SLOT_NUMBER_OFFSET_IN_EXECUTION_PAYLOAD; // 100 + 48 + 532 = 680
+  ENVELOPE_START_IN_SIGNED + EXECUTION_PAYLOAD_ENVELOPE_FIXED_SIZE + SLOT_NUMBER_OFFSET_IN_EXECUTION_PAYLOAD; // 100 + 80 + 532 = 712
 
 export function getSlotFromExecutionPayloadEnvelopeSerialized(data: Uint8Array): Slot | null {
   if (data.length < SLOT_OFFSET_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + SLOT_SIZE) {