@lodestar/beacon-node 1.48.0-dev.0dbea0de1b → 1.48.0-dev.3211c1bdd4

package/src/chain/blocks/importBlock.ts

@@ -25,7 +25,17 @@ import {
   isStatePostAltair,
   isStatePostBellatrix,
 } from "@lodestar/state-transition";
-import {Attestation, BeaconBlock, altair, capella, electra, isGloasBeaconBlock, phase0, ssz} from "@lodestar/types";
+import {
+  Attestation,
+  BeaconBlock,
+  Epoch,
+  altair,
+  capella,
+  electra,
+  isGloasBeaconBlock,
+  phase0,
+  ssz,
+} from "@lodestar/types";
 import {isErrorAborted, toRootHex} from "@lodestar/utils";
 import {GENESIS_SLOT, ZERO_HASH_HEX} from "../../constants/index.js";
 import {callInNextEventLoop} from "../../util/eventLoop.js";
@@ -44,6 +54,15 @@ import {getCheckpointFromState} from "./utils/checkpoint.js";
  * Fork-choice allows to import attestations from current (0) or past (1) epoch.
  */
 const FORK_CHOICE_ATT_EPOCH_LIMIT = 1;
+/**
+ * Whether the attestations of a block at `blockEpoch` are imported into fork choice
+ */
+export function importsBlockAttestations(opts: ImportBlockOpts, blockEpoch: Epoch, currentEpoch: Epoch): boolean {
+  return (
+    opts.importAttestations === AttestationImportOpt.Force ||
+    (opts.importAttestations !== AttestationImportOpt.Skip && blockEpoch >= currentEpoch - FORK_CHOICE_ATT_EPOCH_LIMIT)
+  );
+}
 /**
  * Emit eventstream events for block contents events only for blocks that are recent enough to clock
  */
@@ -85,7 +104,7 @@ export async function importBlock(
   const currentEpoch = computeEpochAtSlot(currentSlot);
   const blockEpoch = computeEpochAtSlot(blockSlot);
   const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
-  const blockDelaySec =
+  const receiveDelaySec =
     fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postState.genesisTime);
   const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
   const fork = this.config.getForkSeq(blockSlot);
@@ -122,10 +141,12 @@ export async function importBlock(
     executionStatus = parentBlock.executionStatus;
   }
 
+  const importDelaySec = this.clock.secFromSlot(blockSlot);
   const blockSummary = this.forkChoice.onBlock(
     block.message,
     postState,
-    blockDelaySec,
+    receiveDelaySec,
+    importDelaySec,
     currentSlot,
     executionStatus,
     dataAvailabilityStatus
@@ -147,10 +168,7 @@ export async function importBlock(
   // Only process attestations of blocks with relevant attestations for the fork-choice:
   // If current epoch is N, and block is epoch X, block may include attestations for epoch X or X - 1.
   // The latest block that is useful is at epoch N - 1 which may include attestations for epoch N - 1 or N - 2.
-  if (
-    opts.importAttestations === AttestationImportOpt.Force ||
-    (opts.importAttestations !== AttestationImportOpt.Skip && blockEpoch >= currentEpoch - FORK_CHOICE_ATT_EPOCH_LIMIT)
-  ) {
+  if (importsBlockAttestations(opts, blockEpoch, currentEpoch)) {
     const attestations = block.message.body.attestations;
     const rootCache = new RootCache(postState);
     const invalidAttestationErrorsByCode = new Map<string, {error: Error; count: number}>();

package/src/chain/blocks/index.ts

@@ -1,3 +1,5 @@
+import {PayloadStatus} from "@lodestar/fork-choice";
+import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {SignedBeaconBlock, Slot} from "@lodestar/types";
 import {isErrorAborted, toRootHex} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
@@ -5,15 +7,17 @@ import {nextEventLoop} from "../../util/eventLoop.js";
 import {JobItemQueue, isQueueErrorAborted} from "../../util/queue/index.js";
 import type {BeaconChain} from "../chain.js";
 import {BlockError, BlockErrorCode, isBlockErrorAborted} from "../errors/index.js";
+import {ForkchoiceCaller} from "../forkChoice/index.js";
 import {BlockProcessOpts} from "../options.js";
 import {IBlockInput} from "./blockInput/types.js";
-import {importBlock} from "./importBlock.js";
+import {importBlock, importsBlockAttestations} from "./importBlock.js";
 import {PayloadError, importExecutionPayload} from "./importExecutionPayload.js";
 import {PayloadEnvelopeInput} from "./payloadEnvelopeInput/payloadEnvelopeInput.js";
 import {FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
 import {assertLinearChainSegment} from "./utils/chainSegment.js";
 import {verifyBlocksInEpoch} from "./verifyBlock.js";
 import {verifyBlocksSanityChecks} from "./verifyBlocksSanityChecks.js";
+import {verifyPayloadsDataAvailability} from "./verifyPayloadsDataAvailability.js";
 
 export {AttestationImportOpt, type ImportBlockOpts} from "./types.js";
 
@@ -70,6 +74,7 @@ export async function processBlocks(
     // No relevant blocks, skip verifyBlocksInEpoch()
     if (relevantBlocks.length === 0 || parentBlock === null) {
       // parentBlock can only be null if relevantBlocks are empty
+      await importPayloadEnvelopesOfKnownBlocks.call(this, payloadEnvelopes, opts);
       return;
     }
 
@@ -79,9 +84,21 @@ export async function processBlocks(
       payloadEnvelopes,
       parentBlock
     );
-    if (orphanedPayloads != null) {
+    // Orphaned payloads of recent blocks are still imported, a reorg of the child could make the payload canonical
+    // again and later blocks build on it. Older ones are skipped: without the block attestations the orphaned FULL
+    // variant has no descendants and ties at zero weight with the EMPTY variant carrying the chain, the payload
+    // status tiebreaker then picks FULL and parks the head there.
+    const blockEpoch = computeEpochAtSlot(relevantBlocks[0].getBlock().message.slot);
+    const importsAttestations = importsBlockAttestations(opts, blockEpoch, this.clock.currentEpoch);
+    let payloadEnvelopesToImport = payloadEnvelopes;
+    if (orphanedPayloads != null && payloadEnvelopes !== null && !importsAttestations) {
+      payloadEnvelopesToImport = new Map(payloadEnvelopes);
       for (const orphaned of orphanedPayloads) {
-        this.logger.debug("Orphaned payload envelope in chain segment", {
+        payloadEnvelopesToImport.delete(orphaned.slot);
+        // never validated, drop it from the shared cache so it is not served to peers, by-root sync reloads the
+        // entry from db if the payload is ever needed
+        this.seenPayloadEnvelopeInputCache.prune(orphaned.payloadEnvelopeInput.blockRootHex);
+        this.logger.debug("Skipping orphaned payload envelope in chain segment", {
           slot: orphaned.slot,
           blockRoot: orphaned.payloadEnvelopeInput.blockRootHex,
         });
@@ -97,7 +114,7 @@ export async function processBlocks(
       proposerBalanceDeltas,
       segmentExecStatus,
       indexedAttestationsByBlock,
-    } = await verifyBlocksInEpoch.call(this, parentBlock, relevantBlocks, payloadEnvelopes, opts);
+    } = await verifyBlocksInEpoch.call(this, parentBlock, relevantBlocks, payloadEnvelopesToImport, opts);
 
     // If segmentExecStatus has lvhForkchoice then, the entire segment should be invalid
     // and we need to further propagate
@@ -132,8 +149,8 @@ export async function processBlocks(
     }
 
     const slotSet = new Set<Slot>(blocks.map((b) => b.getBlock().message.slot));
-    if (payloadEnvelopes) {
-      for (const slot of payloadEnvelopes.keys()) slotSet.add(slot);
+    if (payloadEnvelopesToImport) {
+      for (const slot of payloadEnvelopesToImport.keys()) slotSet.add(slot);
     }
     const slots = Array.from(slotSet).sort((a, b) => a - b);
     for (const slot of slots) {
@@ -146,7 +163,7 @@ export async function processBlocks(
       // PayloadEnvelopeInput is shared and may receive an envelope after the DA snapshot was taken.
       const payloadDA = payloadDAStatuses.get(slot);
       if (payloadDA !== undefined) {
-        const payloadInput = payloadEnvelopes?.get(slot);
+        const payloadInput = payloadEnvelopesToImport?.get(slot);
         if (payloadInput === undefined) {
           throw new Error(`Missing payload input for slot ${slot} after DA verification`);
         }
@@ -209,6 +226,69 @@ export async function processBlocks(
   }
 }
 
+/**
+ * A peer can serve a block without its envelope, the block is then imported with a PENDING payload. Later batches
+ * carry the envelope but all their blocks are known, without importing it here every child building on the FULL
+ * variant fails with PARENT_PAYLOAD_UNKNOWN and range sync never progresses. If our head already descends from the
+ * block's EMPTY variant the chain did not build on the payload, importing it would only add a dead FULL leaf. As for
+ * orphaned payloads in a segment this only applies while the block attestations are not imported, a recent payload
+ * is imported anyway so a reorg building on it does not have to fetch it through unknown block sync.
+ */
+async function importPayloadEnvelopesOfKnownBlocks(
+  this: BeaconChain,
+  payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null,
+  opts: ImportBlockOpts
+): Promise<void> {
+  if (payloadEnvelopes === null) {
+    return;
+  }
+
+  const head = this.forkChoice.getHead();
+  const currentEpoch = this.clock.currentEpoch;
+  const payloadInputs: PayloadEnvelopeInput[] = [];
+  const orphanedSlots: Slot[] = [];
+  for (const payloadInput of payloadEnvelopes.values()) {
+    const {blockRootHex, slot} = payloadInput;
+    if (
+      !payloadInput.hasPayloadEnvelope() ||
+      !this.forkChoice.hasBlockHex(blockRootHex) ||
+      this.forkChoice.getBlockHexAndBlockHash(blockRootHex, payloadInput.getBlockHashHex()) !== null
+    ) {
+      continue;
+    }
+    if (
+      head.blockRoot !== blockRootHex &&
+      !importsBlockAttestations(opts, computeEpochAtSlot(slot), currentEpoch) &&
+      this.forkChoice.isDescendant(blockRootHex, PayloadStatus.EMPTY, head.blockRoot, head.payloadStatus)
+    ) {
+      // never validated, see processBlocks()
+      this.seenPayloadEnvelopeInputCache.prune(blockRootHex);
+      orphanedSlots.push(slot);
+      continue;
+    }
+    payloadInputs.push(payloadInput);
+  }
+
+  if (orphanedSlots.length > 0) {
+    this.logger.debug("Skipping orphaned payload envelopes of known blocks", {slots: orphanedSlots.join(",")});
+  }
+
+  if (payloadInputs.length === 0) {
+    return;
+  }
+
+  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(payloadInputs, new AbortController().signal);
+  for (let i = 0; i < payloadInputs.length; i++) {
+    this.logger.debug("Importing payload envelope of known block", {
+      slot: payloadInputs[i].slot,
+      blockRoot: payloadInputs[i].blockRootHex,
+    });
+    await importExecutionPayload.call(this, payloadInputs[i], dataAvailabilityStatuses[i], {validSignature: false});
+  }
+  // the new FULL variants are not seen by the head cached from the last block import
+  this.recomputeForkChoiceHead(ForkchoiceCaller.importBlock);
+}
+
 function getBlockOrPayloadError(e: unknown, block: SignedBeaconBlock): BlockError | PayloadError {
   if (e instanceof BlockError) {
     return e;

package/src/chain/blocks/utils/chainSegment.ts

@@ -37,17 +37,6 @@ export function assertLinearChainSegment(
 
   const warnings: OrphanedPayloadEnvelope[] = [];
 
-  let parentExecHash: string | null = parentBlock?.executionPayloadBlockHash ?? null;
-  if (parentBlock !== null) {
-    const parentPayloadInput = payloadEnvelopes?.get(parentBlock.slot);
-    if (parentPayloadInput?.hasPayloadEnvelope()) {
-      // Checkpoint-sync first batch: the anchor is stored PENDING with the inherited (grandparent)
-      // hash, not its own payload. Its payload envelope is in this batch, so use that blockHash as
-      // the real parent EL head.
-      parentExecHash = parentPayloadInput.getBlockHashHex();
-    }
-  }
-
   // Track the expected execution payload block hash through the segment, seeded from the FIRST
   // block's own bid
   const firstBlockMessage = blocks[0].getBlock().message;
@@ -55,6 +44,22 @@ export function assertLinearChainSegment(
     ? toRootHex(firstBlockMessage.body.signedExecutionPayloadBid.message.parentBlockHash)
     : null;
 
+  let parentExecHash: string | null = parentBlock?.executionPayloadBlockHash ?? null;
+  if (parentBlock !== null) {
+    const parentPayloadInput = payloadEnvelopes?.get(parentBlock.slot);
+    if (parentPayloadInput?.hasPayloadEnvelope()) {
+      const parentPayloadBlockHash = parentPayloadInput.getBlockHashHex();
+      if (currentExecHash === parentPayloadBlockHash) {
+        // Checkpoint-sync first batch: the anchor is stored PENDING with the inherited (grandparent)
+        // hash, its own payload is in this batch and is the real parent EL head
+        parentExecHash = parentPayloadBlockHash;
+      } else {
+        // First block builds on the parent's EMPTY variant, the parent's payload in this batch is orphaned
+        warnings.push({slot: parentBlock.slot, payloadEnvelopeInput: parentPayloadInput});
+      }
+    }
+  }
+
   // First block must build on the parent's EL head. PARENT_PAYLOAD_UNKNOWN (vs the in-segment
   // NON_LINEAR_PAYLOAD_ROOTS below) tells range sync this is a parent-link problem, not an in-batch one.
   if (parentExecHash !== null && currentExecHash !== null && currentExecHash !== parentExecHash) {

package/src/chain/errors/executionPayloadBid.ts

@@ -5,6 +5,7 @@ export enum ExecutionPayloadBidErrorCode {
   BUILDER_NOT_ELIGIBLE = "EXECUTION_PAYLOAD_BID_ERROR_BUILDER_NOT_ELIGIBLE",
   INVALID_BUILDER_VERSION = "EXECUTION_PAYLOAD_BID_ERROR_INVALID_BUILDER_VERSION",
   NON_ZERO_EXECUTION_PAYMENT = "EXECUTION_PAYLOAD_BID_ERROR_NON_ZERO_EXECUTION_PAYMENT",
+  BLOCK_HASH_EQUALS_PARENT_BLOCK_HASH = "EXECUTION_PAYLOAD_BID_ERROR_BLOCK_HASH_EQUALS_PARENT_BLOCK_HASH",
   INCOMPATIBLE_WITH_HEAD = "EXECUTION_PAYLOAD_BID_ERROR_INCOMPATIBLE_WITH_HEAD",
   BID_ALREADY_KNOWN = "EXECUTION_PAYLOAD_BID_ERROR_BID_ALREADY_KNOWN",
   BID_TOO_LOW = "EXECUTION_PAYLOAD_BID_ERROR_BID_TOO_LOW",
@@ -63,6 +64,7 @@ export type ExecutionPayloadBidErrorType =
     }
   | {code: ExecutionPayloadBidErrorCode.UNKNOWN_BLOCK_ROOT; parentBlockRoot: RootHex}
   | {code: ExecutionPayloadBidErrorCode.UNKNOWN_PARENT_BLOCK_HASH; parentBlockHash: RootHex}
+  | {code: ExecutionPayloadBidErrorCode.BLOCK_HASH_EQUALS_PARENT_BLOCK_HASH; blockHash: RootHex}
   | {code: ExecutionPayloadBidErrorCode.INVALID_SLOT; builderIndex: BuilderIndex; slot: Slot}
   | {code: ExecutionPayloadBidErrorCode.NOT_LATER_THAN_PARENT; parentSlot: Slot; slot: Slot}
   | {

package/src/chain/forkChoice/index.ts

@@ -135,6 +135,7 @@ export function initializeForkChoiceFromFinalizedState(
         blockRoot: toRootHex(checkpoint.root),
         timeliness: true, // Optimistically assume is timely
         ptcTimeliness: true, // Spec: block_timeliness for anchor = [True, True]
+        importedTimely: true, // Optimistically assume is timely
         proposerIndex: blockHeader.proposerIndex,
 
         justifiedEpoch: justifiedCheckpoint.epoch,
@@ -233,6 +234,7 @@ export function initializeForkChoiceFromUnfinalizedState(
     targetRoot: headRoot,
     timeliness: true, // Optimistically assume is timely
     ptcTimeliness: true, // Spec: block_timeliness for anchor = [True, True]
+    importedTimely: true, // Optimistically assume is timely
     proposerIndex: blockHeader.proposerIndex,
 
     justifiedEpoch: justifiedCheckpoint.epoch,

package/src/chain/validation/executionPayloadBid.ts

@@ -113,6 +113,15 @@ export async function validateApiExecutionPayloadBid(
     });
   }
 
+  // [REJECT] The bid's block hash is not equal to its parent block hash -- i.e.
+  // `bid.block_hash != bid.parent_block_hash`.
+  if (byteArrayEquals(bid.blockHash, bid.parentBlockHash)) {
+    throw new ExecutionPayloadBidError(GossipAction.REJECT, {
+      code: ExecutionPayloadBidErrorCode.BLOCK_HASH_EQUALS_PARENT_BLOCK_HASH,
+      blockHash: toRootHex(bid.blockHash),
+    });
+  }
+
   // [REJECT] The length of KZG commitments is less than or equal to the limitation defined in the
   // consensus layer -- i.e. validate that
   // `len(bid.blob_kzg_commitments) <= get_blob_parameters(compute_epoch_at_slot(bid.slot)).max_blobs_per_block`.
@@ -266,6 +275,15 @@ async function validateExecutionPayloadBid(
     });
   }
 
+  // [REJECT] The bid's block hash is not equal to its parent block hash -- i.e.
+  // `bid.block_hash != bid.parent_block_hash`.
+  if (byteArrayEquals(bid.blockHash, bid.parentBlockHash)) {
+    throw new ExecutionPayloadBidError(GossipAction.REJECT, {
+      code: ExecutionPayloadBidErrorCode.BLOCK_HASH_EQUALS_PARENT_BLOCK_HASH,
+      blockHash: toRootHex(bid.blockHash),
+    });
+  }
+
   // [REJECT] The length of KZG commitments is less than or equal to the limitation defined in the
   // consensus layer -- i.e. validate that
   // `len(bid.blob_kzg_commitments) <= get_blob_parameters(compute_epoch_at_slot(bid.slot)).max_blobs_per_block`.

package/src/chain/validation/payloadAttestationMessage.ts

@@ -7,7 +7,6 @@ import {RootHex, gloas, ssz} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
 import {GossipAction, PayloadAttestationError, PayloadAttestationErrorCode} from "../errors/index.js";
 import {IBeaconChain} from "../index.js";
-import {RegenCaller} from "../regen/index.js";
 
 export type PayloadAttestationValidationResult = {
   attDataRootHex: RootHex;
@@ -82,15 +81,15 @@ async function validatePayloadAttestationMessage(
   // TODO GLOAS: implement this. Technically if we cannot get proto block from fork choice,
   // it is possible that the block didn't pass the validation
 
-  // Use the referenced block's branch state for the PTC committee check
-  const state = await chain.regen
-    .getBlockSlotState(block, data.slot, {dontTransferCache: true}, RegenCaller.validateGossipPayloadAttestationMessage)
-    .catch(() => {
-      throw new PayloadAttestationError(GossipAction.IGNORE, {
-        code: PayloadAttestationErrorCode.UNKNOWN_BLOCK_ROOT,
-        blockRoot: toRootHex(data.beaconBlockRoot),
-      });
+  // block.slot === data.slot is enforced above, so use the block's post-state directly to avoid
+  // getting through regen queue
+  const state = chain.regen.getStateSync(block.stateRoot);
+  if (state == null) {
+    throw new PayloadAttestationError(GossipAction.IGNORE, {
+      code: PayloadAttestationErrorCode.UNKNOWN_BLOCK_ROOT,
+      blockRoot: toRootHex(data.beaconBlockRoot),
     });
+  }
 
   if (!isStatePostGloas(state)) {
     throw new Error(`Expected gloas+ state for payload attestation validation, got fork=${state.forkName}`);

package/src/execution/builder/validateBid.ts

@@ -56,6 +56,10 @@ export async function validateBuilderApiExecutionPayloadBid(
     );
   }
 
+  if (byteArrayEquals(bid.blockHash, bid.parentBlockHash)) {
+    throw Error(`Bid block hash ${toRootHex(bid.blockHash)} must not equal its parent block hash`);
+  }
+
   const blobKzgCommitmentsLen = bid.blobKzgCommitments.length;
   const maxBlobsPerBlock = chain.config.getMaxBlobsPerBlock(computeEpochAtSlot(bid.slot));
   if (blobKzgCommitmentsLen > maxBlobsPerBlock) {

package/src/metrics/metrics/lodestar.ts

@@ -2,6 +2,7 @@
 import {NotReorgedReason} from "@lodestar/fork-choice";
 import {ArchiveStoreTask} from "../../chain/archiveStore/archiveStore.js";
 import {FrequencyStateArchiveStep} from "../../chain/archiveStore/strategies/frequencyStateArchiveStrategy.js";
+import type {LateCanonicalBlockReason} from "../../chain/archiveStore/utils/archiveBlocks.js";
 import {BlockInputSource} from "../../chain/blocks/blockInput/index.js";
 import {PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
 import {
@@ -1047,6 +1048,11 @@ export function createLodestarMetrics(
         name: "lodestar_import_block_set_head_after_cutoff_total",
         help: "Total times an imported block is set as head after ATTESTATION_DUE_BPS of the slot",
       }),
+      lateCanonicalBlock: register.counter<{reason: LateCanonicalBlockReason}>({
+        name: "lodestar_import_block_late_canonical_total",
+        help: "Total finalized-canonical blocks this node imported after the attestation cutoff; reason distinguishes this node's processing lag (slow_import) from late reception (late_receive)",
+        labelNames: ["reason"],
+      }),
       bySource: register.gauge<{source: BlockInputSource}>({
         name: "lodestar_import_block_by_source_total",
         help: "Total number of imported blocks by source",
@@ -1352,11 +1358,6 @@ export function createLodestarMetrics(
           help: "Total number of InsertOutcome as a result of adding an execution payload bid from gossip to the pool",
           labelNames: ["insertOutcome"],
         }),
-        apiInsertOutcome: register.counter<{insertOutcome: InsertOutcome}>({
-          name: "lodestar_oppool_execution_payload_bid_pool_api_insert_outcome_total",
-          help: "Total number of InsertOutcome as a result of adding an execution payload bid from api to the pool",
-          labelNames: ["insertOutcome"],
-        }),
         apiValidationTime: register.histogram({
           name: "lodestar_api_execution_payload_bid_validation_time_seconds",
           help: "Time elapsed for signed execution payload bid validation - api path",