@lodestar/beacon-node 1.48.0-dev.b71ab45af3 → 1.48.0-dev.bb3ffba9e9

package/src/chain/archiveStore/historicalState/worker.ts

@@ -25,7 +25,10 @@ logger.info("Historical state worker started");
 
 const config = createBeaconConfig(chainConfigFromJson(workerData.chainConfigJson), workerData.genesisValidatorsRoot);
 
-const db = new BeaconDb(config, await LevelDbController.create({name: workerData.dbLocation}, {logger}));
+const db = new BeaconDb(config, await LevelDbController.create({name: workerData.dbLocation}, {logger}), {
+  dataColumnDir: workerData.dataColumnDir,
+  logger,
+});
 
 const abortController = new AbortController();
 

package/src/chain/archiveStore/utils/archiveBlocks.ts

@@ -134,6 +134,8 @@ export async function archiveBlocks(
       logger.verbose("Migrated blobSidecars from hot DB to cold DB", {...logCtx, migratedEntries});
     }
 
+    // Keep the normal LevelDB hot-to-cold flow for legacy columns. New columns are written
+    // directly to flat files, so this only moves fallback data left by older versions.
     if (finalizedPostFulu) {
       const migratedSlots = await migrateDataColumnSidecarsFromHotToColdDb(
         config,
@@ -142,7 +144,7 @@ export async function archiveBlocks(
         finalizedCanonicalBlocks,
         currentEpoch
       );
-      logger.verbose("Migrated dataColumnSidecars from hot DB to cold DB", {
+      logger.verbose("Migrated legacy dataColumnSidecars from hot DB to cold DB", {
         ...logCtx,
         migratedEntries: migratedSlots.length,
         slotRange: prettyPrintIndices(migratedSlots),
@@ -195,6 +197,23 @@ export async function archiveBlocks(
       slotRange: prettyPrintIndices(nonCanonicalSlots),
     };
 
+    const columnItems = finalizedNonCanonicalBlocks
+      // Gloas EMPTY and FULL variants share a block root. EMPTY has no sidecars, so deleting by its root could
+      // remove the canonical FULL variant's columns. Pre-Gloas blocks are always FULL.
+      .filter(
+        (summary) => config.getForkSeq(summary.slot) >= ForkSeq.fulu && summary.payloadStatus === PayloadStatus.FULL
+      )
+      .map((summary) => ({slot: summary.slot, blockRoot: summary.blockRoot}));
+    if (columnItems.length > 0) {
+      // Delete sidecars first so their block roots remain available to retry cleanup after a failure or crash.
+      await db.dataColumns.deleteMany(columnItems);
+      logger.verbose("Deleted non canonical data columns of blocks", {
+        ...logCtx,
+        blocks: columnItems.length,
+        slotRange: prettyPrintIndices(columnItems.map(({slot}) => slot).sort((a, b) => a - b)),
+      });
+    }
+
     await db.block.batchDelete(nonCanonicalBlockRoots);
     logger.verbose("Deleted non canonical blocks from hot DB", nonCanonicalLogCtx);
 
@@ -203,11 +222,6 @@ export async function archiveBlocks(
       logger.verbose("Deleted non canonical blobSidecars from hot DB", nonCanonicalLogCtx);
     }
 
-    if (finalizedPostFulu) {
-      await db.dataColumnSidecar.deleteMany(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical dataColumnSidecars from hot DB", nonCanonicalLogCtx);
-    }
-
     if (finalizedPostGloas) {
       await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots);
       logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", nonCanonicalLogCtx);
@@ -245,23 +259,16 @@ export async function archiveBlocks(
       );
       const dataColumnSidecarsMinEpoch = currentEpoch - dataColumnSidecarsArchiveWindow;
       if (dataColumnSidecarsMinEpoch >= config.FULU_FORK_EPOCH) {
-        const prefixedKeys = await db.dataColumnSidecarArchive.keys({
-          // The `id` value `0` refers to the column index. So we want to fetch all sidecars less than zero column of `dataColumnSidecarsMinEpoch`
-          lt: {prefix: computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch), id: 0},
-        });
-        // for each slot there could be multiple dataColumnSidecar, so we need to deduplicate it
-        const slotsToDelete = [...new Set(prefixedKeys.map(({prefix}) => prefix))].sort((a, b) => a - b);
-
-        if (slotsToDelete.length > 0) {
-          await db.dataColumnSidecarArchive.deleteMany(slotsToDelete);
+        const columnsPruneSlot = computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch);
+        const prunedColumnSlots = await db.dataColumns.pruneBefore(columnsPruneSlot);
+        if (prunedColumnSlots.length > 0) {
           logger.verbose("dataColumnSidecars prune", {
             ...logCtx,
-            slotRange: prettyPrintIndices(slotsToDelete),
-            numOfSlots: slotsToDelete.length,
-            totalNumOfSidecars: prefixedKeys.length,
+            slotRange: prettyPrintIndices(prunedColumnSlots),
+            numOfSlots: prunedColumnSlots.length,
           });
         } else {
-          logger.verbose(`dataColumnSidecars prune: no entries before epoch ${dataColumnSidecarsMinEpoch}`, logCtx);
+          logger.verbose("dataColumnSidecars prune: no entries before slot", {...logCtx, slot: columnsPruneSlot});
         }
       } else {
         logger.verbose(
@@ -389,7 +396,6 @@ async function migrateBlobSidecarsFromHotToColdDb(
       )
     ).filter((e): e is KeyValue<Slot, Uint8Array> => e !== null);
 
-    // put to blockArchive db and delete block db
     await Promise.all([
       db.blobSidecarsArchive.batchPutBinary(canonicalBlobSidecarsEntries),
       db.blobSidecars.batchDelete(canonicalBlocks.map((block) => block.root)),
@@ -402,11 +408,9 @@ async function migrateBlobSidecarsFromHotToColdDb(
 
 // TODO: This function can be simplified further by reducing layers of promises in a loop
 /**
- * Post-gloas the data columns of a Gloas block are tied to its execution payload envelope —
- * columns only exist once the FULL variant of the block is in the proto-array. Pre-Gloas (Fulu)
- * blocks only have a FULL variant, so the `payloadStatus === FULL` filter passes them all.
- * Blocks whose canonical variant is PENDING/EMPTY are skipped here — their columns will be picked
- * up on a later run once the FULL variant appears in the ancestor walk.
+ * Post-gloas the data columns of a Gloas block are tied to its execution payload envelope.
+ * Columns only exist once the FULL variant of the block is in the proto-array. Pre-Gloas
+ * blocks only have a FULL variant, so the payload-status filter passes them all.
  */
 async function migrateDataColumnSidecarsFromHotToColdDb(
   config: ChainForkConfig,
@@ -428,32 +432,24 @@ async function migrateDataColumnSidecarsFromHotToColdDb(
     if (batch.length === 0) break;
 
     const promises: Promise<void>[] = [];
-
-    // load Buffer instead of ssz deserialized to improve performance
     for (const block of batch) {
-      const blockSlot = block.slot;
-      const blockEpoch = computeEpochAtSlot(blockSlot);
-
+      const blockEpoch = computeEpochAtSlot(block.slot);
       if (
-        config.getForkSeq(blockSlot) < ForkSeq.fulu ||
-        // if block is out of ${config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS}, skip this step
+        config.getForkSeq(block.slot) < ForkSeq.fulu ||
         blockEpoch < currentEpoch - config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS
       ) {
         continue;
       }
 
-      // Here we assume the data column sidecars are already in the hot db
       const dataColumnSidecarBytes = await fromAsync(db.dataColumnSidecar.valuesStreamBinary(block.root));
       if (dataColumnSidecarBytes.length === 0) {
-        // Empty stream: either the block has no blobs, or columns were already archived on a
-        // previous run (boundary block). Nothing to migrate.
         logger.debug("DataColumnSidecars in forkchoice but missing in hot db, could be already archived", {
           slot: block.slot,
           root: toRootHex(block.root),
         });
         continue;
       }
-      logger.verbose("Migrated dataColumnSidecars for block", {
+      logger.verbose("Migrated legacy dataColumnSidecars for block", {
         currentEpoch,
         slot: block.slot,
         root: toRootHex(block.root),
@@ -462,15 +458,14 @@ async function migrateDataColumnSidecarsFromHotToColdDb(
       promises.push(
         db.dataColumnSidecarArchive.putManyBinary(
           block.slot,
-          dataColumnSidecarBytes.map((p) => ({key: p.id, value: p.value}))
+          dataColumnSidecarBytes.map((entry) => ({key: entry.id, value: entry.value}))
         )
       );
       migratedSlots.push(block.slot);
     }
 
-    promises.push(db.dataColumnSidecar.deleteMany(batch.map((block) => block.root)));
-
     await Promise.all(promises);
+    await db.dataColumnSidecar.deleteMany(batch.map((block) => block.root));
   }
 
   // Ancestor walk is newest to oldest; sort ascending so `prettyPrintIndices` renders cleanly.

package/src/chain/blocks/importBlock.ts

@@ -524,19 +524,6 @@ export async function importBlock(
         this.metrics?.previousJustifiedEpoch.set(checkpointState.previousJustifiedCheckpoint.epoch);
         this.metrics?.currentJustifiedEpoch.set(justifiedCheckpoint.epoch);
       }
-      const finalizedCheckpoint = checkpointState.finalizedCheckpoint;
-      const finalizedEpoch = finalizedCheckpoint.epoch;
-      const preFinalizedEpoch = parentBlockSummary.finalizedEpoch;
-      if (finalizedEpoch > preFinalizedEpoch) {
-        this.emitter.emit(routes.events.EventType.finalizedCheckpoint, {
-          block: toRootHex(finalizedCheckpoint.root),
-          epoch: finalizedCheckpoint.epoch,
-          state: toRootHex(checkpointState.hashTreeRoot()),
-          executionOptimistic: false,
-        });
-        this.logger.verbose("Checkpoint finalized", toCheckpointHex(finalizedCheckpoint));
-        this.metrics?.finalizedEpoch.set(finalizedCheckpoint.epoch);
-      }
     }
   }
 

package/src/chain/blocks/index.ts

@@ -209,8 +209,19 @@ export async function processBlocks(
         const blockSlot = signedBlock.message.slot;
         const {state} = err.type;
         const forkTypes = this.config.getForkTypes(blockSlot);
-        this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `${blockSlot}_invalid_signature`);
-        this.persistInvalidSszBytes("BeaconState", state.serialize(), `${state.slot}_invalid_signature`);
+        const blockRootHex = toRootHex(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message));
+        this.persistInvalidSszValue(
+          forkTypes.SignedBeaconBlock,
+          signedBlock,
+          `${blockSlot}_invalid_signature`,
+          blockRootHex
+        );
+        this.persistInvalidSszBytes(
+          "BeaconState",
+          state.serialize(),
+          toRootHex(state.hashTreeRoot()),
+          `${state.slot}_invalid_signature`
+        );
       } else if (err.type.code === BlockErrorCode.INVALID_STATE_ROOT) {
         const {signedBlock} = err;
         const blockSlot = signedBlock.message.slot;

package/src/chain/blocks/writeBlockInputToDb.ts

@@ -1,6 +1,6 @@
-import {ForkPostDeneb, isForkPostDeneb} from "@lodestar/params";
+import {ForkPostDeneb, ForkPostFulu, isForkPostDeneb} from "@lodestar/params";
 import {SignedBeaconBlock} from "@lodestar/types";
-import {fromHex, toRootHex} from "@lodestar/utils";
+import {toRootHex} from "@lodestar/utils";
 import {getBlobKzgCommitments} from "../../util/dataColumns.js";
 import {BeaconChain} from "../chain.js";
 import {IBlockInput, IDataColumnsInput, isBlockInputBlobs, isBlockInputColumns} from "./blockInput/index.js";
@@ -76,7 +76,6 @@ async function writeBlockAndBlobsToDb(this: BeaconChain, blockInput: IBlockInput
  */
 export async function writeDataColumnsToDb(this: BeaconChain, blockInput: IDataColumnsInput): Promise<void> {
   const {slot, blockRootHex} = blockInput;
-  const blockRoot = fromHex(blockRootHex);
 
   if (!blockInput.hasComputedAllData()) {
     // Supernodes may only have a subset of the data columns by the time the block begins to be imported
@@ -89,25 +88,19 @@ export async function writeDataColumnsToDb(this: BeaconChain, blockInput: IDataC
 
   const {custodyColumns} = this.custodyConfig;
   const dataColumnSidecars = blockInput.getCustodyColumns();
+  const dataColumnSidecarType = this.config.getForkTypes<ForkPostFulu>(slot).DataColumnSidecar;
 
-  const binaryPuts: {key: number; value: Uint8Array}[] = [];
-  const nonbinaryPuts = [];
+  const binaryColumns: {index: number; data: Uint8Array}[] = [];
   for (const dataColumnSidecar of dataColumnSidecars) {
-    // skip reserializing column if we already have it
     const serialized = this.serializedCache.get(dataColumnSidecar);
-    if (serialized) {
-      binaryPuts.push({key: dataColumnSidecar.index, value: serialized});
-    } else {
-      nonbinaryPuts.push(dataColumnSidecar);
-    }
+    binaryColumns.push({
+      index: dataColumnSidecar.index,
+      data: serialized ?? dataColumnSidecarType.serialize(dataColumnSidecar),
+    });
   }
+  await this.db.dataColumns.putManyBinary({slot, blockRoot: blockRootHex}, binaryColumns);
 
-  await Promise.all([
-    this.db.dataColumnSidecar.putManyBinary(blockRoot, binaryPuts),
-    this.db.dataColumnSidecar.putMany(blockRoot, nonbinaryPuts),
-  ]);
-
-  this.logger.debug("Persisted dataColumnSidecars to hot DB", {
+  this.logger.debug("Persisted dataColumnSidecars", {
     slot,
     root: blockRootHex,
     dataColumnSidecars: dataColumnSidecars.length,

package/src/chain/chain.ts

@@ -268,6 +268,7 @@ export class BeaconChain implements IBeaconChain {
       db,
       dbName,
       dataDir,
+      dataColumnDir,
       logger,
       processShutdownCallback,
       clock,
@@ -285,6 +286,7 @@ export class BeaconChain implements IBeaconChain {
       db: IBeaconDb;
       dbName: string;
       dataDir: string;
+      dataColumnDir?: string;
       logger: Logger;
       processShutdownCallback: ProcessShutdownCallback;
       /** Used for testing to supply fake clock */
@@ -505,7 +507,12 @@ export class BeaconChain implements IBeaconChain {
 
     this.archiveStore = new ArchiveStore(
       {db, chain: this, logger: logger as LoggerNode, metrics},
-      {...opts, dbName, anchorState: {finalizedCheckpoint: anchorState.finalizedCheckpoint}},
+      {
+        ...opts,
+        dbName,
+        dataColumnDir: dataColumnDir ?? path.join(dataDir, "data_columns"),
+        anchorState: {finalizedCheckpoint: anchorState.finalizedCheckpoint},
+      },
       signal
     );
 
@@ -969,12 +976,15 @@ export class BeaconChain implements IBeaconChain {
 
   async getDataColumnSidecars(blockSlot: Slot, blockRootHex: string): Promise<DataColumnSidecar[]> {
     const fork = this.config.getForkName(blockSlot);
+    const sidecarsByIndex = new Map<number, DataColumnSidecar>();
 
     if (isForkPostGloas(fork)) {
       // After gloas, columns are tracked in PayloadEnvelopeInput
       const payloadInput = this.seenPayloadEnvelopeInputCache.get(blockRootHex);
       if (payloadInput) {
-        return payloadInput.getAllColumns();
+        for (const sidecar of payloadInput.getAllColumns()) {
+          sidecarsByIndex.set(sidecar.index, sidecar);
+        }
       }
     } else {
       // Before gloas, columns are tracked in BlockInput
@@ -983,16 +993,17 @@ export class BeaconChain implements IBeaconChain {
         if (!isBlockInputColumns(blockInput)) {
           throw new Error(`Expected block input to have columns: slot=${blockSlot} root=${blockRootHex}`);
         }
-        return blockInput.getAllColumns();
+        for (const sidecar of blockInput.getAllColumns()) {
+          sidecarsByIndex.set(sidecar.index, sidecar);
+        }
       }
     }
 
-    const sidecarsUnfinalized = await this.db.dataColumnSidecar.values(fromHex(blockRootHex));
-    if (sidecarsUnfinalized.length > 0) {
-      return sidecarsUnfinalized;
+    for (const sidecar of await this.db.dataColumns.getAll({slot: blockSlot, blockRoot: blockRootHex})) {
+      if (!sidecarsByIndex.has(sidecar.index)) sidecarsByIndex.set(sidecar.index, sidecar);
     }
-    const sidecarsFinalized = await this.db.dataColumnSidecarArchive.values(blockSlot);
-    return sidecarsFinalized;
+
+    return [...sidecarsByIndex.values()].sort((a, b) => a.index - b.index);
   }
 
   async getSerializedDataColumnSidecars(
@@ -1001,22 +1012,25 @@ export class BeaconChain implements IBeaconChain {
     indices: number[]
   ): Promise<(Uint8Array | undefined)[]> {
     const fork = this.config.getForkName(blockSlot);
+    const dataColumnSidecars: (Uint8Array | undefined)[] = indices.map(() => undefined);
 
     if (isForkPostGloas(fork)) {
       // After gloas, columns are tracked in PayloadEnvelopeInput
       const payloadInput = this.seenPayloadEnvelopeInputCache.get(blockRootHex);
       if (payloadInput) {
-        return indices.map((index) => {
+        for (let i = 0; i < indices.length; i++) {
+          const index = indices[i];
           const sidecar = payloadInput.getColumn(index);
           if (!sidecar) {
-            return undefined;
+            continue;
           }
           const serialized = this.serializedCache.get(sidecar);
           if (serialized) {
-            return serialized;
+            dataColumnSidecars[i] = serialized;
+          } else {
+            dataColumnSidecars[i] = sszTypesFor(fork as ForkPostGloas).DataColumnSidecar.serialize(sidecar);
           }
-          return sszTypesFor(fork as ForkPostGloas).DataColumnSidecar.serialize(sidecar);
-        });
+        }
       }
     } else {
       // Before gloas, columns are tracked in BlockInput
@@ -1025,26 +1039,42 @@ export class BeaconChain implements IBeaconChain {
         if (!isBlockInputColumns(blockInput)) {
           throw new Error(`Expected block input to have columns: slot=${blockSlot} root=${blockRootHex}`);
         }
-        return indices.map((index) => {
+        for (let i = 0; i < indices.length; i++) {
+          const index = indices[i];
           const sidecar = blockInput.getColumn(index);
           if (!sidecar) {
-            return undefined;
+            continue;
           }
           const serialized = this.serializedCache.get(sidecar);
           if (serialized) {
-            return serialized;
+            dataColumnSidecars[i] = serialized;
+          } else {
+            dataColumnSidecars[i] = sszTypesFor(blockInput.forkName as ForkPostFulu).DataColumnSidecar.serialize(
+              sidecar
+            );
           }
-          return sszTypesFor(blockInput.forkName as ForkPostFulu).DataColumnSidecar.serialize(sidecar);
-        });
+        }
       }
     }
 
-    const sidecarsUnfinalized = await this.db.dataColumnSidecar.getManyBinary(fromHex(blockRootHex), indices);
-    if (sidecarsUnfinalized.some((sidecar) => sidecar != null)) {
-      return sidecarsUnfinalized;
+    const missingPositions = dataColumnSidecars
+      .map((sidecar, position) => (sidecar === undefined ? position : -1))
+      .filter((position) => position !== -1);
+
+    if (missingPositions.length > 0) {
+      const persistedSidecars = await this.db.dataColumns.getManyBinary(
+        {slot: blockSlot, blockRoot: blockRootHex},
+        missingPositions.map((position) => indices[position])
+      );
+      for (let i = 0; i < missingPositions.length; i++) {
+        const sidecar = persistedSidecars[i];
+        if (sidecar !== undefined) {
+          dataColumnSidecars[missingPositions[i]] = sidecar;
+        }
+      }
     }
-    const sidecarsFinalized = await this.db.dataColumnSidecarArchive.getManyBinary(blockSlot, indices);
-    return sidecarsFinalized;
+
+    return dataColumnSidecars;
   }
 
   async produceCommonBlockBody(blockAttributes: BlockAttributes): Promise<CommonBlockBody> {
@@ -1271,59 +1301,65 @@ export class BeaconChain implements IBeaconChain {
     try {
       const prevHead = this.forkChoice.getHead();
       const head = this.forkChoice.updateAndGetHead({mode: UpdateHeadOpt.GetCanonicalHead}).head;
+      this.emitHeadEvents(prevHead, head);
+      return head;
+    } catch (e) {
+      this.metrics?.forkChoice.errors.inc({entrypoint: UpdateHeadOpt.GetCanonicalHead});
+      throw e;
+    } finally {
+      timer?.();
+    }
+  }
 
-      const headRootChanged = head.blockRoot !== prevHead.blockRoot;
-
-      if (!headRootChanged && prevHead.payloadStatus === head.payloadStatus) {
-        return head;
-      }
+  /**
+   * Emit `head` and `head_v2` when the fork choice head differs between two observations. Every caller that
+   * can move the head, `updateAndGetHead()` and `updateTime()`, must capture the head before and pass both.
+   */
+  private emitHeadEvents(prevHead: ProtoBlock, head: ProtoBlock): void {
+    const headRootChanged = head.blockRoot !== prevHead.blockRoot;
 
-      try {
-        const previousDutyDependentRoot = this.forkChoice.getDependentRoot(head, EpochDifference.previous);
-        const currentDutyDependentRoot = this.forkChoice.getDependentRoot(head, EpochDifference.current);
-        const epochTransition = computeStartSlotAtEpoch(computeEpochAtSlot(head.slot)) === head.slot;
-        const executionOptimistic = isOptimisticBlock(head);
-
-        if (headRootChanged) {
-          this.emitter.emit(routes.events.EventType.head, {
-            block: head.blockRoot,
-            epochTransition,
-            slot: head.slot,
-            state: head.stateRoot,
-            previousDutyDependentRoot,
-            currentDutyDependentRoot,
-            executionOptimistic,
-          });
-        }
+    if (!headRootChanged && prevHead.payloadStatus === head.payloadStatus) {
+      return;
+    }
 
-        this.emitter.emit(routes.events.EventType.headV2, {
-          version: this.config.getForkName(head.slot),
-          data: {
-            slot: head.slot,
-            block: head.blockRoot,
-            state: head.stateRoot,
-            payloadStatus: head.payloadStatus === PayloadStatus.FULL ? "full" : "empty",
-            epochTransition,
-            currentEpochDependentRoot: previousDutyDependentRoot,
-            nextEpochDependentRoot: currentDutyDependentRoot,
-            executionOptimistic,
-          },
+    try {
+      const previousDutyDependentRoot = this.forkChoice.getDependentRoot(head, EpochDifference.previous);
+      const currentDutyDependentRoot = this.forkChoice.getDependentRoot(head, EpochDifference.current);
+      const epochTransition = computeStartSlotAtEpoch(computeEpochAtSlot(head.slot)) === head.slot;
+      const executionOptimistic = isOptimisticBlock(head);
+
+      if (headRootChanged) {
+        this.emitter.emit(routes.events.EventType.head, {
+          block: head.blockRoot,
+          epochTransition,
+          slot: head.slot,
+          state: head.stateRoot,
+          previousDutyDependentRoot,
+          currentDutyDependentRoot,
+          executionOptimistic,
         });
-      } catch (e) {
-        // getDependentRoot() may fail with error: "No block for root" as we can see in holesky non-finality issue
-        this.logger.debug(
-          "Error emitting head/head_v2 event",
-          {slot: head.slot, root: head.blockRoot, headRootChanged},
-          e as Error
-        );
       }
 
-      return head;
+      this.emitter.emit(routes.events.EventType.headV2, {
+        version: this.config.getForkName(head.slot),
+        data: {
+          slot: head.slot,
+          block: head.blockRoot,
+          state: head.stateRoot,
+          payloadStatus: head.payloadStatus === PayloadStatus.FULL ? "full" : "empty",
+          epochTransition,
+          currentEpochDependentRoot: previousDutyDependentRoot,
+          nextEpochDependentRoot: currentDutyDependentRoot,
+          executionOptimistic,
+        },
+      });
     } catch (e) {
-      this.metrics?.forkChoice.errors.inc({entrypoint: UpdateHeadOpt.GetCanonicalHead});
-      throw e;
-    } finally {
-      timer?.();
+      // getDependentRoot() may fail with error: "No block for root" as we can see in holesky non-finality issue
+      this.logger.debug(
+        "Error emitting head/head_v2 event",
+        {slot: head.slot, root: head.blockRoot, headRootChanged},
+        e as Error
+      );
     }
   }
 
@@ -1381,10 +1417,15 @@ export class BeaconChain implements IBeaconChain {
     const slot = data.slot;
     if (isBlindedBeaconBlock(data)) {
       const sszType = this.config.getPostBellatrixForkTypes(slot).BlindedBeaconBlock;
-      void this.persistSszObject("BlindedBeaconBlock", sszType.serialize(data), sszType.hashTreeRoot(data), suffix);
+      void this.persistSszObject(
+        "BlindedBeaconBlock",
+        sszType.serialize(data),
+        toRootHex(sszType.hashTreeRoot(data)),
+        suffix
+      );
     } else {
       const sszType = this.config.getForkTypes(slot).BeaconBlock;
-      void this.persistSszObject("BeaconBlock", sszType.serialize(data), sszType.hashTreeRoot(data), suffix);
+      void this.persistSszObject("BeaconBlock", sszType.serialize(data), toRootHex(sszType.hashTreeRoot(data)), suffix);
     }
   }
 
@@ -1405,33 +1446,39 @@ export class BeaconChain implements IBeaconChain {
       this.persistSszObject(
         `SignedBeaconBlock_slot_${blockSlot}`,
         blockType.serialize(block),
-        blockType.hashTreeRoot(block),
+        toRootHex(this.config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message)),
         `${logStr}_block`
       ),
       this.persistSszObject(
         `preState_slot_${preState.slot}_BeaconState`,
         preState.serialize(),
-        preState.hashTreeRoot(),
+        toRootHex(preState.hashTreeRoot()),
         `${logStr}_pre_state`
       ),
       this.persistSszObject(
         `postState_slot_${postState.slot}_BeaconState`,
         postState.serialize(),
-        postState.hashTreeRoot(),
+        toRootHex(postStateRoot),
         `${logStr}_post_state`
       ),
     ]);
   }
 
-  persistInvalidSszValue<T>(type: Type<T>, sszObject: T, suffix?: string): void {
+  persistInvalidSszValue<T>(type: Type<T>, sszObject: T, suffix?: string, rootHex?: RootHex): void {
     if (this.opts.persistInvalidSszObjects) {
-      void this.persistSszObject(type.typeName, type.serialize(sszObject), type.hashTreeRoot(sszObject), suffix);
+      void this.persistSszObject(
+        type.typeName,
+        type.serialize(sszObject),
+        // in SignedBeaconBlock case, we want to use BeaconBlock root instead
+        rootHex ?? toRootHex(type.hashTreeRoot(sszObject)),
+        suffix
+      );
     }
   }
 
-  persistInvalidSszBytes(typeName: string, sszBytes: Uint8Array, suffix?: string): void {
+  persistInvalidSszBytes(typeName: string, sszBytes: Uint8Array, rootHex: RootHex, suffix?: string): void {
     if (this.opts.persistInvalidSszObjects) {
-      void this.persistSszObject(typeName, sszBytes, sszBytes, suffix);
+      void this.persistSszObject(typeName, sszBytes, rootHex, suffix);
     }
   }
 
@@ -1562,14 +1609,14 @@ export class BeaconChain implements IBeaconChain {
     return {state: blockState, stateId: "block_state_any_epoch", shouldWarn: true};
   }
 
-  private async persistSszObject(prefix: string, bytes: Uint8Array, root: Uint8Array, logStr?: string): Promise<void> {
+  private async persistSszObject(prefix: string, bytes: Uint8Array, rootHex: RootHex, logStr?: string): Promise<void> {
     const now = new Date();
     // yyyy-MM-dd
     const dateStr = now.toISOString().split("T")[0];
 
     // by default store to lodestar_archive of current dir
     const dirpath = path.join(this.opts.persistInvalidSszObjectsDir ?? "invalid_ssz_objects", dateStr);
-    const filepath = path.join(dirpath, `${prefix}_${toRootHex(root)}.ssz`);
+    const filepath = path.join(dirpath, `${prefix}_${rootHex}.ssz`);
 
     await ensureDir(dirpath);
 
@@ -1612,7 +1659,10 @@ export class BeaconChain implements IBeaconChain {
       this.processShutdownCallback(this.forkChoice.irrecoverableError);
     }
 
+    // updateTime() recomputes the head on epoch-boundary checkpoint pull-up and under fast confirmation
+    const prevHead = this.forkChoice.getHead();
     this.forkChoice.updateTime(slot);
+    this.emitHeadEvents(prevHead, this.forkChoice.getHead());
     this.metrics?.clockSlot.set(slot);
 
     this.attestationPool.prune(slot);
@@ -1670,6 +1720,25 @@ export class BeaconChain implements IBeaconChain {
 
   private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithHex): Promise<void> {
     this.logger.verbose("Fork choice finalized", {epoch: cp.epoch, root: cp.rootHex});
+    this.metrics?.finalizedEpoch.set(cp.epoch);
+    const finalizedBlock = this.forkChoice.getBlockHexDefaultStatus(cp.rootHex);
+    if (finalizedBlock) {
+      // The callback runs synchronously inside fork choice checkpoint updates, defer writing to subscribers
+      callInNextEventLoop(() => {
+        this.emitter.emit(routes.events.EventType.finalizedCheckpoint, {
+          block: cp.rootHex,
+          epoch: cp.epoch,
+          state: finalizedBlock.stateRoot,
+          executionOptimistic: isOptimisticBlock(finalizedBlock),
+        });
+      });
+    } else {
+      this.logger.debug("Finalized block not found in fork choice, skip finalized_checkpoint event", {
+        epoch: cp.epoch,
+        root: cp.rootHex,
+      });
+    }
+
     const finalizedSlot = computeStartSlotAtEpoch(cp.epoch);
     this.seenBlockProposers.prune(finalizedSlot);
 

package/src/chain/emitter.ts

@@ -33,7 +33,7 @@ export enum ChainEvent {
   /**
    * This event signals that the fork choice store has been updated.
    *
-   * This event is guaranteed to be triggered whenever the fork choice justified checkpoint is updated. This is in response to a newly processed block.
+   * This event is guaranteed to be triggered whenever the fork choice finalized checkpoint is updated. This is either in response to a newly processed block or a new clock tick.
    */
   forkChoiceFinalized = "forkChoice:finalized",
   /**