@lodestar/beacon-node 1.48.0-dev.67df6eac67 → 1.48.0-dev.68ce7b06a0

package/src/chain/blocks/importBlock.ts

@@ -25,7 +25,17 @@ import {
   isStatePostAltair,
   isStatePostBellatrix,
 } from "@lodestar/state-transition";
-import {Attestation, BeaconBlock, altair, capella, electra, isGloasBeaconBlock, phase0, ssz} from "@lodestar/types";
+import {
+  Attestation,
+  BeaconBlock,
+  Epoch,
+  altair,
+  capella,
+  electra,
+  isGloasBeaconBlock,
+  phase0,
+  ssz,
+} from "@lodestar/types";
 import {isErrorAborted, toRootHex} from "@lodestar/utils";
 import {GENESIS_SLOT, ZERO_HASH_HEX} from "../../constants/index.js";
 import {callInNextEventLoop} from "../../util/eventLoop.js";
@@ -44,6 +54,15 @@ import {getCheckpointFromState} from "./utils/checkpoint.js";
  * Fork-choice allows to import attestations from current (0) or past (1) epoch.
  */
 const FORK_CHOICE_ATT_EPOCH_LIMIT = 1;
+/**
+ * Whether the attestations of a block at `blockEpoch` are imported into fork choice
+ */
+export function importsBlockAttestations(opts: ImportBlockOpts, blockEpoch: Epoch, currentEpoch: Epoch): boolean {
+  return (
+    opts.importAttestations === AttestationImportOpt.Force ||
+    (opts.importAttestations !== AttestationImportOpt.Skip && blockEpoch >= currentEpoch - FORK_CHOICE_ATT_EPOCH_LIMIT)
+  );
+}
 /**
  * Emit eventstream events for block contents events only for blocks that are recent enough to clock
  */
@@ -149,10 +168,7 @@ export async function importBlock(
   // Only process attestations of blocks with relevant attestations for the fork-choice:
   // If current epoch is N, and block is epoch X, block may include attestations for epoch X or X - 1.
   // The latest block that is useful is at epoch N - 1 which may include attestations for epoch N - 1 or N - 2.
-  if (
-    opts.importAttestations === AttestationImportOpt.Force ||
-    (opts.importAttestations !== AttestationImportOpt.Skip && blockEpoch >= currentEpoch - FORK_CHOICE_ATT_EPOCH_LIMIT)
-  ) {
+  if (importsBlockAttestations(opts, blockEpoch, currentEpoch)) {
     const attestations = block.message.body.attestations;
     const rootCache = new RootCache(postState);
     const invalidAttestationErrorsByCode = new Map<string, {error: Error; count: number}>();
@@ -508,19 +524,6 @@ export async function importBlock(
         this.metrics?.previousJustifiedEpoch.set(checkpointState.previousJustifiedCheckpoint.epoch);
         this.metrics?.currentJustifiedEpoch.set(justifiedCheckpoint.epoch);
       }
-      const finalizedCheckpoint = checkpointState.finalizedCheckpoint;
-      const finalizedEpoch = finalizedCheckpoint.epoch;
-      const preFinalizedEpoch = parentBlockSummary.finalizedEpoch;
-      if (finalizedEpoch > preFinalizedEpoch) {
-        this.emitter.emit(routes.events.EventType.finalizedCheckpoint, {
-          block: toRootHex(finalizedCheckpoint.root),
-          epoch: finalizedCheckpoint.epoch,
-          state: toRootHex(checkpointState.hashTreeRoot()),
-          executionOptimistic: false,
-        });
-        this.logger.verbose("Checkpoint finalized", toCheckpointHex(finalizedCheckpoint));
-        this.metrics?.finalizedEpoch.set(finalizedCheckpoint.epoch);
-      }
     }
   }
 

package/src/chain/blocks/index.ts

@@ -1,3 +1,5 @@
+import {PayloadStatus} from "@lodestar/fork-choice";
+import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {SignedBeaconBlock, Slot} from "@lodestar/types";
 import {isErrorAborted, toRootHex} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
@@ -5,15 +7,17 @@ import {nextEventLoop} from "../../util/eventLoop.js";
 import {JobItemQueue, isQueueErrorAborted} from "../../util/queue/index.js";
 import type {BeaconChain} from "../chain.js";
 import {BlockError, BlockErrorCode, isBlockErrorAborted} from "../errors/index.js";
+import {ForkchoiceCaller} from "../forkChoice/index.js";
 import {BlockProcessOpts} from "../options.js";
 import {IBlockInput} from "./blockInput/types.js";
-import {importBlock} from "./importBlock.js";
+import {importBlock, importsBlockAttestations} from "./importBlock.js";
 import {PayloadError, importExecutionPayload} from "./importExecutionPayload.js";
 import {PayloadEnvelopeInput} from "./payloadEnvelopeInput/payloadEnvelopeInput.js";
-import {FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
-import {assertLinearChainSegment} from "./utils/chainSegment.js";
+import {FullyVerifiedBlock, ImportBlockOpts, ProcessBlocksResult} from "./types.js";
+import {OrphanedPayloadEnvelope, assertLinearChainSegment} from "./utils/chainSegment.js";
 import {verifyBlocksInEpoch} from "./verifyBlock.js";
 import {verifyBlocksSanityChecks} from "./verifyBlocksSanityChecks.js";
+import {verifyPayloadsDataAvailability} from "./verifyPayloadsDataAvailability.js";
 
 export {AttestationImportOpt, type ImportBlockOpts} from "./types.js";
 
@@ -23,10 +27,16 @@ const QUEUE_MAX_LENGTH = 256;
  * BlockProcessor processes block jobs in a queued fashion, one after the other.
  */
 export class BlockProcessor {
-  readonly jobQueue: JobItemQueue<[IBlockInput[], Map<Slot, PayloadEnvelopeInput> | null, ImportBlockOpts], void>;
+  readonly jobQueue: JobItemQueue<
+    [IBlockInput[], Map<Slot, PayloadEnvelopeInput> | null, ImportBlockOpts],
+    ProcessBlocksResult
+  >;
 
   constructor(chain: BeaconChain, metrics: Metrics | null, opts: BlockProcessOpts, signal: AbortSignal) {
-    this.jobQueue = new JobItemQueue<[IBlockInput[], Map<Slot, PayloadEnvelopeInput> | null, ImportBlockOpts], void>(
+    this.jobQueue = new JobItemQueue<
+      [IBlockInput[], Map<Slot, PayloadEnvelopeInput> | null, ImportBlockOpts],
+      ProcessBlocksResult
+    >(
       (job, payloadEnvelopes, importOpts) => {
         return processBlocks.call(chain, job, payloadEnvelopes, {...opts, ...importOpts});
       },
@@ -39,8 +49,8 @@ export class BlockProcessor {
     job: IBlockInput[],
     payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null,
     opts: ImportBlockOpts = {}
-  ): Promise<void> {
-    await this.jobQueue.push(job, payloadEnvelopes, opts);
+  ): Promise<ProcessBlocksResult> {
+    return this.jobQueue.push(job, payloadEnvelopes, opts);
   }
 }
 
@@ -59,9 +69,10 @@ export async function processBlocks(
   blocks: IBlockInput[],
   payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null,
   opts: BlockProcessOpts & ImportBlockOpts
-): Promise<void> {
+): Promise<ProcessBlocksResult> {
+  const emptyResult: ProcessBlocksResult = {orphaned: [], skipped: false};
   if (blocks.length === 0) {
-    return; // TODO: or throw?
+    return emptyResult; // TODO: or throw?
   }
 
   try {
@@ -70,7 +81,7 @@ export async function processBlocks(
     // No relevant blocks, skip verifyBlocksInEpoch()
     if (relevantBlocks.length === 0 || parentBlock === null) {
       // parentBlock can only be null if relevantBlocks are empty
-      return;
+      return await importPayloadEnvelopesOfKnownBlocks.call(this, payloadEnvelopes, opts);
     }
 
     const {warnings: orphanedPayloads} = assertLinearChainSegment(
@@ -79,12 +90,21 @@ export async function processBlocks(
       payloadEnvelopes,
       parentBlock
     );
-    if (orphanedPayloads != null) {
-      for (const orphaned of orphanedPayloads) {
-        this.logger.debug("Orphaned payload envelope in chain segment", {
-          slot: orphaned.slot,
-          blockRoot: orphaned.payloadEnvelopeInput.blockRootHex,
-        });
+    // Orphaned payloads of recent blocks are still imported, a reorg of the child could make the payload canonical
+    // again and later blocks build on it. Older ones are skipped: without the block attestations the orphaned FULL
+    // variant has no descendants and ties at zero weight with the EMPTY variant carrying the chain, the payload
+    // status tiebreaker then picks FULL and parks the head there.
+    const blockEpoch = computeEpochAtSlot(relevantBlocks[0].getBlock().message.slot);
+    const importsAttestations = importsBlockAttestations(opts, blockEpoch, this.clock.currentEpoch);
+    const skipped = orphanedPayloads != null && payloadEnvelopes !== null && !importsAttestations;
+    let payloadEnvelopesToImport = payloadEnvelopes;
+    if (skipped) {
+      payloadEnvelopesToImport = new Map(payloadEnvelopes);
+      for (const orphaned of orphanedPayloads ?? []) {
+        payloadEnvelopesToImport.delete(orphaned.slot);
+        // never validated, drop it from the shared cache so it is not served to peers, by-root sync reloads the
+        // entry from db if the payload is ever needed
+        this.seenPayloadEnvelopeInputCache.prune(orphaned.payloadEnvelopeInput.blockRootHex);
       }
     }
 
@@ -97,7 +117,7 @@ export async function processBlocks(
       proposerBalanceDeltas,
       segmentExecStatus,
       indexedAttestationsByBlock,
-    } = await verifyBlocksInEpoch.call(this, parentBlock, relevantBlocks, payloadEnvelopes, opts);
+    } = await verifyBlocksInEpoch.call(this, parentBlock, relevantBlocks, payloadEnvelopesToImport, opts);
 
     // If segmentExecStatus has lvhForkchoice then, the entire segment should be invalid
     // and we need to further propagate
@@ -132,8 +152,8 @@ export async function processBlocks(
     }
 
     const slotSet = new Set<Slot>(blocks.map((b) => b.getBlock().message.slot));
-    if (payloadEnvelopes) {
-      for (const slot of payloadEnvelopes.keys()) slotSet.add(slot);
+    if (payloadEnvelopesToImport) {
+      for (const slot of payloadEnvelopesToImport.keys()) slotSet.add(slot);
     }
     const slots = Array.from(slotSet).sort((a, b) => a - b);
     for (const slot of slots) {
@@ -146,7 +166,7 @@ export async function processBlocks(
       // PayloadEnvelopeInput is shared and may receive an envelope after the DA snapshot was taken.
       const payloadDA = payloadDAStatuses.get(slot);
       if (payloadDA !== undefined) {
-        const payloadInput = payloadEnvelopes?.get(slot);
+        const payloadInput = payloadEnvelopesToImport?.get(slot);
         if (payloadInput === undefined) {
           throw new Error(`Missing payload input for slot ${slot} after DA verification`);
         }
@@ -159,9 +179,11 @@ export async function processBlocks(
 
       await nextEventLoop();
     }
+
+    return {orphaned: orphanedPayloads ?? [], skipped};
   } catch (e) {
     if (isErrorAborted(e) || isQueueErrorAborted(e) || isBlockErrorAborted(e)) {
-      return; // Ignore
+      return emptyResult; // Ignore
     }
 
     // above functions should only throw BlockError, or PayloadError from the gloas payload import
@@ -187,8 +209,19 @@ export async function processBlocks(
         const blockSlot = signedBlock.message.slot;
         const {state} = err.type;
         const forkTypes = this.config.getForkTypes(blockSlot);
-        this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `${blockSlot}_invalid_signature`);
-        this.persistInvalidSszBytes("BeaconState", state.serialize(), `${state.slot}_invalid_signature`);
+        const blockRootHex = toRootHex(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message));
+        this.persistInvalidSszValue(
+          forkTypes.SignedBeaconBlock,
+          signedBlock,
+          `${blockSlot}_invalid_signature`,
+          blockRootHex
+        );
+        this.persistInvalidSszBytes(
+          "BeaconState",
+          state.serialize(),
+          toRootHex(state.hashTreeRoot()),
+          `${state.slot}_invalid_signature`
+        );
       } else if (err.type.code === BlockErrorCode.INVALID_STATE_ROOT) {
         const {signedBlock} = err;
         const blockSlot = signedBlock.message.slot;
@@ -209,6 +242,70 @@ export async function processBlocks(
   }
 }
 
+/**
+ * A peer can serve a block without its envelope, the block is then imported with a PENDING payload. Later batches
+ * carry the envelope but all their blocks are known, without importing it here every child building on the FULL
+ * variant fails with PARENT_PAYLOAD_UNKNOWN and range sync never progresses. If our head already descends from the
+ * block's EMPTY variant the chain did not build on the payload, importing it would only add a dead FULL leaf. As for
+ * orphaned payloads in a segment this only applies while the block attestations are not imported, a recent payload
+ * is imported anyway so a reorg building on it does not have to fetch it through unknown block sync.
+ */
+async function importPayloadEnvelopesOfKnownBlocks(
+  this: BeaconChain,
+  payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null,
+  opts: ImportBlockOpts
+): Promise<ProcessBlocksResult> {
+  if (payloadEnvelopes === null) {
+    return {orphaned: [], skipped: false};
+  }
+
+  const head = this.forkChoice.getHead();
+  const currentEpoch = this.clock.currentEpoch;
+  const payloadInputs: PayloadEnvelopeInput[] = [];
+  const orphaned: OrphanedPayloadEnvelope[] = [];
+  for (const payloadInput of payloadEnvelopes.values()) {
+    const {blockRootHex, slot} = payloadInput;
+    if (
+      !payloadInput.hasPayloadEnvelope() ||
+      !this.forkChoice.hasBlockHex(blockRootHex) ||
+      this.forkChoice.getBlockHexAndBlockHash(blockRootHex, payloadInput.getBlockHashHex()) !== null
+    ) {
+      continue;
+    }
+    if (
+      head.blockRoot !== blockRootHex &&
+      !importsBlockAttestations(opts, computeEpochAtSlot(slot), currentEpoch) &&
+      this.forkChoice.isDescendant(blockRootHex, PayloadStatus.EMPTY, head.blockRoot, head.payloadStatus)
+    ) {
+      // never validated, see processBlocks()
+      this.seenPayloadEnvelopeInputCache.prune(blockRootHex);
+      orphaned.push({slot, payloadEnvelopeInput: payloadInput});
+      continue;
+    }
+    payloadInputs.push(payloadInput);
+  }
+
+  // orphaned envelopes on this path are always skipped, range sync logs them with peer/client context
+  const result: ProcessBlocksResult = {orphaned, skipped: orphaned.length > 0};
+
+  if (payloadInputs.length === 0) {
+    return result;
+  }
+
+  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(payloadInputs, new AbortController().signal);
+  for (let i = 0; i < payloadInputs.length; i++) {
+    this.logger.debug("Importing payload envelope of known block", {
+      slot: payloadInputs[i].slot,
+      blockRoot: payloadInputs[i].blockRootHex,
+    });
+    await importExecutionPayload.call(this, payloadInputs[i], dataAvailabilityStatuses[i], {validSignature: false});
+  }
+  // the new FULL variants are not seen by the head cached from the last block import
+  this.recomputeForkChoiceHead(ForkchoiceCaller.importBlock);
+
+  return result;
+}
+
 function getBlockOrPayloadError(e: unknown, block: SignedBeaconBlock): BlockError | PayloadError {
   if (e instanceof BlockError) {
     return e;

package/src/chain/blocks/types.ts

@@ -4,6 +4,7 @@ import {ForkSeq} from "@lodestar/params";
 import {DataAvailabilityStatus, IBeaconStateView, computeEpochAtSlot} from "@lodestar/state-transition";
 import type {IndexedAttestation, Slot, fulu} from "@lodestar/types";
 import {IBlockInput} from "./blockInput/types.js";
+import {OrphanedPayloadEnvelope} from "./utils/chainSegment.js";
 
 export enum GossipedInputType {
   block = "block",
@@ -110,3 +111,11 @@ export type FullyVerifiedBlock = {
   /** If the execution payload couldn't be verified because of EL syncing status, used in optimistic sync */
   executionStatus: BlockExecutionStatus | PayloadExecutionStatus;
 };
+
+/**
+ * Result of processing a chain segment.
+ */
+export type ProcessBlocksResult = {
+  orphaned: OrphanedPayloadEnvelope[];
+  skipped: boolean;
+};

package/src/chain/blocks/utils/chainSegment.ts

@@ -22,7 +22,7 @@ export type ChainSegmentResult = {warnings: OrphanedPayloadEnvelope[] | null};
  * - Verifies parent root + slot linearity
  * - For gloas: verifies bid.parentBlockHash matches the tracked execution hash; if not, the
  *   previous FULL envelope is treated as orphaned (segment continues as if previous slot was EMPTY)
- * - If an envelope exists for this slot: verifies it references this block's root
+ * - If an envelope exists for this slot (or the parent's slot): verifies it references this block's root
  * - Advances the tracked execution hash (FULL if envelope present, EMPTY if not)
  */
 export function assertLinearChainSegment(
@@ -48,6 +48,15 @@ export function assertLinearChainSegment(
   if (parentBlock !== null) {
     const parentPayloadInput = payloadEnvelopes?.get(parentBlock.slot);
     if (parentPayloadInput?.hasPayloadEnvelope()) {
+      // Envelopes are served by slot, the one at the parent's slot must reference the parent block. Without this
+      // check an envelope of a sibling block is taken as orphaned and only fails downstream in the payload import
+      if (parentPayloadInput.blockRootHex !== parentBlock.blockRoot) {
+        throw new BlockError(blocks[0].getBlock(), {
+          code: BlockErrorCode.ENVELOPE_BLOCK_ROOT_MISMATCH,
+          envelopeBlockRoot: parentPayloadInput.blockRootHex,
+          blockRoot: parentBlock.blockRoot,
+        });
+      }
       const parentPayloadBlockHash = parentPayloadInput.getBlockHashHex();
       if (currentExecHash === parentPayloadBlockHash) {
         // Checkpoint-sync first batch: the anchor is stored PENDING with the inherited (grandparent)

package/src/chain/chain.ts

@@ -82,7 +82,7 @@ import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blocks/bloc
 import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js";
 import {PayloadEnvelopeInputSource} from "./blocks/payloadEnvelopeInput/index.js";
 import {PayloadEnvelopeProcessor} from "./blocks/payloadEnvelopeProcessor.js";
-import {ImportPayloadOpts} from "./blocks/types.js";
+import {ImportPayloadOpts, ProcessBlocksResult} from "./blocks/types.js";
 import {persistBlockInput} from "./blocks/writeBlockInputToDb.js";
 import {persistPayloadEnvelopeInput} from "./blocks/writePayloadEnvelopeInputToDb.js";
 import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js";
@@ -1167,15 +1167,15 @@ export class BeaconChain implements IBeaconChain {
   }
 
   async processBlock(block: IBlockInput, opts?: ImportBlockOpts): Promise<void> {
-    return this.blockProcessor.processBlocksJob([block], null, opts);
+    await this.blockProcessor.processBlocksJob([block], null, opts);
   }
 
   async processChainSegment(
     blocks: IBlockInput[],
     payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null,
     opts?: ImportBlockOpts
-  ): Promise<void> {
-    await this.blockProcessor.processBlocksJob(blocks, payloadEnvelopes, opts);
+  ): Promise<ProcessBlocksResult> {
+    return this.blockProcessor.processBlocksJob(blocks, payloadEnvelopes, opts);
   }
 
   async processExecutionPayload(payloadInput: PayloadEnvelopeInput, opts?: ImportPayloadOpts): Promise<void> {
@@ -1271,59 +1271,65 @@ export class BeaconChain implements IBeaconChain {
     try {
       const prevHead = this.forkChoice.getHead();
       const head = this.forkChoice.updateAndGetHead({mode: UpdateHeadOpt.GetCanonicalHead}).head;
+      this.emitHeadEvents(prevHead, head);
+      return head;
+    } catch (e) {
+      this.metrics?.forkChoice.errors.inc({entrypoint: UpdateHeadOpt.GetCanonicalHead});
+      throw e;
+    } finally {
+      timer?.();
+    }
+  }
 
-      const headRootChanged = head.blockRoot !== prevHead.blockRoot;
-
-      if (!headRootChanged && prevHead.payloadStatus === head.payloadStatus) {
-        return head;
-      }
+  /**
+   * Emit `head` and `head_v2` when the fork choice head differs between two observations. Every caller that
+   * can move the head, `updateAndGetHead()` and `updateTime()`, must capture the head before and pass both.
+   */
+  private emitHeadEvents(prevHead: ProtoBlock, head: ProtoBlock): void {
+    const headRootChanged = head.blockRoot !== prevHead.blockRoot;
 
-      try {
-        const previousDutyDependentRoot = this.forkChoice.getDependentRoot(head, EpochDifference.previous);
-        const currentDutyDependentRoot = this.forkChoice.getDependentRoot(head, EpochDifference.current);
-        const epochTransition = computeStartSlotAtEpoch(computeEpochAtSlot(head.slot)) === head.slot;
-        const executionOptimistic = isOptimisticBlock(head);
-
-        if (headRootChanged) {
-          this.emitter.emit(routes.events.EventType.head, {
-            block: head.blockRoot,
-            epochTransition,
-            slot: head.slot,
-            state: head.stateRoot,
-            previousDutyDependentRoot,
-            currentDutyDependentRoot,
-            executionOptimistic,
-          });
-        }
+    if (!headRootChanged && prevHead.payloadStatus === head.payloadStatus) {
+      return;
+    }
 
-        this.emitter.emit(routes.events.EventType.headV2, {
-          version: this.config.getForkName(head.slot),
-          data: {
-            slot: head.slot,
-            block: head.blockRoot,
-            state: head.stateRoot,
-            payloadStatus: head.payloadStatus === PayloadStatus.FULL ? "full" : "empty",
-            epochTransition,
-            currentEpochDependentRoot: previousDutyDependentRoot,
-            nextEpochDependentRoot: currentDutyDependentRoot,
-            executionOptimistic,
-          },
+    try {
+      const previousDutyDependentRoot = this.forkChoice.getDependentRoot(head, EpochDifference.previous);
+      const currentDutyDependentRoot = this.forkChoice.getDependentRoot(head, EpochDifference.current);
+      const epochTransition = computeStartSlotAtEpoch(computeEpochAtSlot(head.slot)) === head.slot;
+      const executionOptimistic = isOptimisticBlock(head);
+
+      if (headRootChanged) {
+        this.emitter.emit(routes.events.EventType.head, {
+          block: head.blockRoot,
+          epochTransition,
+          slot: head.slot,
+          state: head.stateRoot,
+          previousDutyDependentRoot,
+          currentDutyDependentRoot,
+          executionOptimistic,
         });
-      } catch (e) {
-        // getDependentRoot() may fail with error: "No block for root" as we can see in holesky non-finality issue
-        this.logger.debug(
-          "Error emitting head/head_v2 event",
-          {slot: head.slot, root: head.blockRoot, headRootChanged},
-          e as Error
-        );
       }
 
-      return head;
+      this.emitter.emit(routes.events.EventType.headV2, {
+        version: this.config.getForkName(head.slot),
+        data: {
+          slot: head.slot,
+          block: head.blockRoot,
+          state: head.stateRoot,
+          payloadStatus: head.payloadStatus === PayloadStatus.FULL ? "full" : "empty",
+          epochTransition,
+          currentEpochDependentRoot: previousDutyDependentRoot,
+          nextEpochDependentRoot: currentDutyDependentRoot,
+          executionOptimistic,
+        },
+      });
     } catch (e) {
-      this.metrics?.forkChoice.errors.inc({entrypoint: UpdateHeadOpt.GetCanonicalHead});
-      throw e;
-    } finally {
-      timer?.();
+      // getDependentRoot() may fail with error: "No block for root" as we can see in holesky non-finality issue
+      this.logger.debug(
+        "Error emitting head/head_v2 event",
+        {slot: head.slot, root: head.blockRoot, headRootChanged},
+        e as Error
+      );
     }
   }
 
@@ -1381,10 +1387,15 @@ export class BeaconChain implements IBeaconChain {
     const slot = data.slot;
     if (isBlindedBeaconBlock(data)) {
       const sszType = this.config.getPostBellatrixForkTypes(slot).BlindedBeaconBlock;
-      void this.persistSszObject("BlindedBeaconBlock", sszType.serialize(data), sszType.hashTreeRoot(data), suffix);
+      void this.persistSszObject(
+        "BlindedBeaconBlock",
+        sszType.serialize(data),
+        toRootHex(sszType.hashTreeRoot(data)),
+        suffix
+      );
     } else {
       const sszType = this.config.getForkTypes(slot).BeaconBlock;
-      void this.persistSszObject("BeaconBlock", sszType.serialize(data), sszType.hashTreeRoot(data), suffix);
+      void this.persistSszObject("BeaconBlock", sszType.serialize(data), toRootHex(sszType.hashTreeRoot(data)), suffix);
     }
   }
 
@@ -1405,33 +1416,39 @@ export class BeaconChain implements IBeaconChain {
       this.persistSszObject(
         `SignedBeaconBlock_slot_${blockSlot}`,
         blockType.serialize(block),
-        blockType.hashTreeRoot(block),
+        toRootHex(this.config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message)),
         `${logStr}_block`
       ),
       this.persistSszObject(
         `preState_slot_${preState.slot}_BeaconState`,
         preState.serialize(),
-        preState.hashTreeRoot(),
+        toRootHex(preState.hashTreeRoot()),
         `${logStr}_pre_state`
       ),
       this.persistSszObject(
         `postState_slot_${postState.slot}_BeaconState`,
         postState.serialize(),
-        postState.hashTreeRoot(),
+        toRootHex(postStateRoot),
         `${logStr}_post_state`
       ),
     ]);
   }
 
-  persistInvalidSszValue<T>(type: Type<T>, sszObject: T, suffix?: string): void {
+  persistInvalidSszValue<T>(type: Type<T>, sszObject: T, suffix?: string, rootHex?: RootHex): void {
     if (this.opts.persistInvalidSszObjects) {
-      void this.persistSszObject(type.typeName, type.serialize(sszObject), type.hashTreeRoot(sszObject), suffix);
+      void this.persistSszObject(
+        type.typeName,
+        type.serialize(sszObject),
+        // in SignedBeaconBlock case, we want to use BeaconBlock root instead
+        rootHex ?? toRootHex(type.hashTreeRoot(sszObject)),
+        suffix
+      );
     }
   }
 
-  persistInvalidSszBytes(typeName: string, sszBytes: Uint8Array, suffix?: string): void {
+  persistInvalidSszBytes(typeName: string, sszBytes: Uint8Array, rootHex: RootHex, suffix?: string): void {
     if (this.opts.persistInvalidSszObjects) {
-      void this.persistSszObject(typeName, sszBytes, sszBytes, suffix);
+      void this.persistSszObject(typeName, sszBytes, rootHex, suffix);
     }
   }
 
@@ -1562,14 +1579,14 @@ export class BeaconChain implements IBeaconChain {
     return {state: blockState, stateId: "block_state_any_epoch", shouldWarn: true};
   }
 
-  private async persistSszObject(prefix: string, bytes: Uint8Array, root: Uint8Array, logStr?: string): Promise<void> {
+  private async persistSszObject(prefix: string, bytes: Uint8Array, rootHex: RootHex, logStr?: string): Promise<void> {
     const now = new Date();
     // yyyy-MM-dd
     const dateStr = now.toISOString().split("T")[0];
 
     // by default store to lodestar_archive of current dir
     const dirpath = path.join(this.opts.persistInvalidSszObjectsDir ?? "invalid_ssz_objects", dateStr);
-    const filepath = path.join(dirpath, `${prefix}_${toRootHex(root)}.ssz`);
+    const filepath = path.join(dirpath, `${prefix}_${rootHex}.ssz`);
 
     await ensureDir(dirpath);
 
@@ -1612,7 +1629,10 @@ export class BeaconChain implements IBeaconChain {
       this.processShutdownCallback(this.forkChoice.irrecoverableError);
     }
 
+    // updateTime() recomputes the head on epoch-boundary checkpoint pull-up and under fast confirmation
+    const prevHead = this.forkChoice.getHead();
     this.forkChoice.updateTime(slot);
+    this.emitHeadEvents(prevHead, this.forkChoice.getHead());
     this.metrics?.clockSlot.set(slot);
 
     this.attestationPool.prune(slot);
@@ -1670,6 +1690,25 @@ export class BeaconChain implements IBeaconChain {
 
   private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithHex): Promise<void> {
     this.logger.verbose("Fork choice finalized", {epoch: cp.epoch, root: cp.rootHex});
+    this.metrics?.finalizedEpoch.set(cp.epoch);
+    const finalizedBlock = this.forkChoice.getBlockHexDefaultStatus(cp.rootHex);
+    if (finalizedBlock) {
+      // The callback runs synchronously inside fork choice checkpoint updates, defer writing to subscribers
+      callInNextEventLoop(() => {
+        this.emitter.emit(routes.events.EventType.finalizedCheckpoint, {
+          block: cp.rootHex,
+          epoch: cp.epoch,
+          state: finalizedBlock.stateRoot,
+          executionOptimistic: isOptimisticBlock(finalizedBlock),
+        });
+      });
+    } else {
+      this.logger.debug("Finalized block not found in fork choice, skip finalized_checkpoint event", {
+        epoch: cp.epoch,
+        root: cp.rootHex,
+      });
+    }
+
     const finalizedSlot = computeStartSlotAtEpoch(cp.epoch);
     this.seenBlockProposers.prune(finalizedSlot);
 

package/src/chain/emitter.ts

@@ -33,7 +33,7 @@ export enum ChainEvent {
   /**
    * This event signals that the fork choice store has been updated.
    *
-   * This event is guaranteed to be triggered whenever the fork choice justified checkpoint is updated. This is in response to a newly processed block.
+   * This event is guaranteed to be triggered whenever the fork choice finalized checkpoint is updated. This is either in response to a newly processed block or a new clock tick.
    */
   forkChoiceFinalized = "forkChoice:finalized",
   /**

package/src/chain/errors/blockError.ts

@@ -78,8 +78,8 @@ export enum BlockErrorCode {
   TOO_MANY_KZG_COMMITMENTS = "BLOCK_ERROR_TOO_MANY_KZG_COMMITMENTS",
   /** Bid parent block root does not match block parent root */
   BID_PARENT_ROOT_MISMATCH = "BLOCK_ERROR_BID_PARENT_ROOT_MISMATCH",
-  /** A block body operation or parent execution request list exceeds its per-block limit */
-  TOO_MANY_BLOCK_OPERATIONS = "BLOCK_ERROR_TOO_MANY_BLOCK_OPERATIONS",
+  /** A block contains deposits where none are allowed */
+  NON_ZERO_DEPOSITS = "BLOCK_ERROR_NON_ZERO_DEPOSITS",
   /** The parent block's execution payload has been verified as invalid */
   PARENT_EXECUTION_INVALID = "BLOCK_ERROR_PARENT_EXECUTION_INVALID",
   /**
@@ -153,7 +153,7 @@ export type BlockErrorType =
   | {code: BlockErrorCode.DATA_UNAVAILABLE}
   | {code: BlockErrorCode.TOO_MANY_KZG_COMMITMENTS; blobKzgCommitmentsLen: number; commitmentLimit: number}
   | {code: BlockErrorCode.BID_PARENT_ROOT_MISMATCH; bidParentRoot: RootHex; blockParentRoot: RootHex}
-  | {code: BlockErrorCode.TOO_MANY_BLOCK_OPERATIONS; name: string; count: number; limit: number}
+  | {code: BlockErrorCode.NON_ZERO_DEPOSITS; count: number}
   | {code: BlockErrorCode.PARENT_EXECUTION_INVALID; parentRoot: RootHex}
   | {code: BlockErrorCode.PARENT_PAYLOAD_UNKNOWN; parentRoot: RootHex; parentBlockHash: RootHex}
   | {code: BlockErrorCode.NON_LINEAR_PAYLOAD_ROOTS; parentBlockHash: RootHex; expectedBlockHash: RootHex};

package/src/chain/errors/executionPayloadEnvelope.ts

@@ -12,8 +12,6 @@ export enum ExecutionPayloadEnvelopeErrorCode {
   BUILDER_INDEX_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BUILDER_INDEX_MISMATCH",
   BLOCK_HASH_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BLOCK_HASH_MISMATCH",
   EXECUTION_REQUESTS_ROOT_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_EXECUTION_REQUESTS_ROOT_MISMATCH",
-  EXECUTION_REQUESTS_COUNT_EXCEEDED = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_EXECUTION_REQUESTS_COUNT_EXCEEDED",
-  WITHDRAWALS_COUNT_EXCEEDED = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_WITHDRAWALS_COUNT_EXCEEDED",
   INVALID_SIGNATURE = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_INVALID_SIGNATURE",
   PAYLOAD_ENVELOPE_INPUT_MISSING = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_PAYLOAD_ENVELOPE_INPUT_MISSING",
 }
@@ -44,13 +42,6 @@ export type ExecutionPayloadEnvelopeErrorType =
       envelopeRequestsRoot: RootHex;
       bidRequestsRoot: RootHex;
     }
-  | {
-      code: ExecutionPayloadEnvelopeErrorCode.EXECUTION_REQUESTS_COUNT_EXCEEDED;
-      name: string;
-      count: number;
-      limit: number;
-    }
-  | {code: ExecutionPayloadEnvelopeErrorCode.WITHDRAWALS_COUNT_EXCEEDED; count: number; limit: number}
   | {code: ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE}
   | {code: ExecutionPayloadEnvelopeErrorCode.PAYLOAD_ENVELOPE_INPUT_MISSING; blockRoot: RootHex};