@lodestar/beacon-node 1.36.0-dev.d42eb95bf0 → 1.36.0-dev.d8afb6dc39

package/src/chain/forkChoice/index.ts

@@ -5,17 +5,22 @@ import {
   ForkChoiceStore,
   JustifiedBalancesGetter,
   ProtoArray,
+  ProtoBlock,
   ForkChoiceOpts as RawForkChoiceOpts,
 } from "@lodestar/fork-choice";
+import {ZERO_HASH_HEX} from "@lodestar/params";
 import {
   CachedBeaconStateAllForks,
   DataAvailabilityStatus,
   computeAnchorCheckpoint,
+  computeEpochAtSlot,
+  computeStartSlotAtEpoch,
+  getBlockRootAtSlot,
   getEffectiveBalanceIncrementsZeroInactive,
   isExecutionStateType,
   isMergeTransitionComplete,
 } from "@lodestar/state-transition";
-import {Slot} from "@lodestar/types";
+import {Slot, ssz} from "@lodestar/types";
 import {Logger, toRootHex} from "@lodestar/utils";
 import {GENESIS_SLOT} from "../../constants/index.js";
 import {Metrics} from "../../metrics/index.js";
@@ -35,6 +40,43 @@ export enum ForkchoiceCaller {
  * Fork Choice extended with a ChainEventEmitter
  */
 export function initializeForkChoice(
+  config: ChainForkConfig,
+  emitter: ChainEventEmitter,
+  currentSlot: Slot,
+  state: CachedBeaconStateAllForks,
+  isFinalizedState: boolean,
+  opts: ForkChoiceOpts,
+  justifiedBalancesGetter: JustifiedBalancesGetter,
+  metrics: Metrics | null,
+  logger?: Logger
+): ForkChoice {
+  return isFinalizedState
+    ? initializeForkChoiceFromFinalizedState(
+        config,
+        emitter,
+        currentSlot,
+        state,
+        opts,
+        justifiedBalancesGetter,
+        metrics,
+        logger
+      )
+    : initializeForkChoiceFromUnfinalizedState(
+        config,
+        emitter,
+        currentSlot,
+        state,
+        opts,
+        justifiedBalancesGetter,
+        metrics,
+        logger
+      );
+}
+
+/**
+ * Initialize forkchoice from a finalized state.
+ */
+export function initializeForkChoiceFromFinalizedState(
   config: ChainForkConfig,
   emitter: ChainEventEmitter,
   currentSlot: Slot,
@@ -82,7 +124,7 @@ export function initializeForkChoice(
         parentRoot: toRootHex(blockHeader.parentRoot),
         stateRoot: toRootHex(blockHeader.stateRoot),
         blockRoot: toRootHex(checkpoint.root),
-        timeliness: true, // Optimisitcally assume is timely
+        timeliness: true, // Optimistically assume is timely
 
         justifiedEpoch: justifiedCheckpoint.epoch,
         justifiedRoot: toRootHex(justifiedCheckpoint.root),
@@ -105,6 +147,140 @@ export function initializeForkChoice(
       },
       currentSlot
     ),
+    state.validators.length,
+    metrics,
+    opts,
+    logger
+  );
+}
+
+/**
+ * Initialize forkchoice from an unfinalized state.
+ */
+export function initializeForkChoiceFromUnfinalizedState(
+  config: ChainForkConfig,
+  emitter: ChainEventEmitter,
+  currentSlot: Slot,
+  unfinalizedState: CachedBeaconStateAllForks,
+  opts: ForkChoiceOpts,
+  justifiedBalancesGetter: JustifiedBalancesGetter,
+  metrics: Metrics | null,
+  logger?: Logger
+): ForkChoice {
+  const {blockHeader} = computeAnchorCheckpoint(config, unfinalizedState);
+  const finalizedCheckpoint = unfinalizedState.finalizedCheckpoint.toValue();
+  const justifiedCheckpoint = unfinalizedState.currentJustifiedCheckpoint.toValue();
+  const headRoot = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader));
+
+  const logCtx = {
+    currentSlot: currentSlot,
+    stateSlot: unfinalizedState.slot,
+    headSlot: blockHeader.slot,
+    headRoot: headRoot,
+    finalizedEpoch: finalizedCheckpoint.epoch,
+    finalizedRoot: toRootHex(finalizedCheckpoint.root),
+    justifiedEpoch: justifiedCheckpoint.epoch,
+    justifiedRoot: toRootHex(justifiedCheckpoint.root),
+  };
+  logger?.warn("Initializing fork choice from unfinalized state", logCtx);
+
+  // this is not the justified state, but there is no other ways to get justified balances
+  const justifiedBalances = getEffectiveBalanceIncrementsZeroInactive(unfinalizedState);
+  const store = new ForkChoiceStore(
+    currentSlot,
+    justifiedCheckpoint,
+    finalizedCheckpoint,
+    justifiedBalances,
+    justifiedBalancesGetter,
+    {
+      onJustified: (cp) => emitter.emit(ChainEvent.forkChoiceJustified, cp),
+      onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp),
+    }
+  );
+
+  // this is the same to the finalized state
+  const headBlock: ProtoBlock = {
+    slot: blockHeader.slot,
+    parentRoot: toRootHex(blockHeader.parentRoot),
+    stateRoot: toRootHex(blockHeader.stateRoot),
+    blockRoot: headRoot,
+    targetRoot: headRoot,
+    timeliness: true, // Optimistically assume is timely
+
+    justifiedEpoch: justifiedCheckpoint.epoch,
+    justifiedRoot: toRootHex(justifiedCheckpoint.root),
+    finalizedEpoch: finalizedCheckpoint.epoch,
+    finalizedRoot: toRootHex(finalizedCheckpoint.root),
+    unrealizedJustifiedEpoch: justifiedCheckpoint.epoch,
+    unrealizedJustifiedRoot: toRootHex(justifiedCheckpoint.root),
+    unrealizedFinalizedEpoch: finalizedCheckpoint.epoch,
+    unrealizedFinalizedRoot: toRootHex(finalizedCheckpoint.root),
+
+    ...(isExecutionStateType(unfinalizedState) && isMergeTransitionComplete(unfinalizedState)
+      ? {
+          executionPayloadBlockHash: toRootHex(unfinalizedState.latestExecutionPayloadHeader.blockHash),
+          executionPayloadNumber: unfinalizedState.latestExecutionPayloadHeader.blockNumber,
+          executionStatus: blockHeader.slot === GENESIS_SLOT ? ExecutionStatus.Valid : ExecutionStatus.Syncing,
+        }
+      : {executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}),
+
+    dataAvailabilityStatus: DataAvailabilityStatus.PreData,
+  };
+
+  const parentSlot = blockHeader.slot - 1;
+  const parentEpoch = computeEpochAtSlot(parentSlot);
+  // parent of head block
+  const parentBlock: ProtoBlock = {
+    ...headBlock,
+    slot: parentSlot,
+    // link this to the dummy justified block
+    parentRoot: toRootHex(justifiedCheckpoint.root),
+    // dummy data, we're not able to regen state before headBlock
+    stateRoot: ZERO_HASH_HEX,
+    blockRoot: headBlock.parentRoot,
+    targetRoot: toRootHex(getBlockRootAtSlot(unfinalizedState, computeStartSlotAtEpoch(parentEpoch))),
+  };
+
+  const justifiedBlock: ProtoBlock = {
+    ...headBlock,
+    slot: computeStartSlotAtEpoch(justifiedCheckpoint.epoch),
+    // link this to the finalized root so that getAncestors can find the finalized block
+    parentRoot: toRootHex(finalizedCheckpoint.root),
+    // dummy data, we're not able to regen state before headBlock
+    stateRoot: ZERO_HASH_HEX,
+    blockRoot: toRootHex(justifiedCheckpoint.root),
+    // same to blockRoot
+    targetRoot: toRootHex(justifiedCheckpoint.root),
+  };
+
+  const finalizedBlock: ProtoBlock = {
+    ...headBlock,
+    slot: computeStartSlotAtEpoch(finalizedCheckpoint.epoch),
+    // we don't care parent of finalized block
+    parentRoot: ZERO_HASH_HEX,
+    // dummy data, we're not able to regen state before headBlock
+    stateRoot: ZERO_HASH_HEX,
+    blockRoot: toRootHex(finalizedCheckpoint.root),
+    // same to blockRoot
+    targetRoot: toRootHex(finalizedCheckpoint.root),
+  };
+
+  const protoArray = ProtoArray.initialize(finalizedBlock, currentSlot);
+  protoArray.onBlock(justifiedBlock, currentSlot);
+  protoArray.onBlock(parentBlock, currentSlot);
+  protoArray.onBlock(headBlock, currentSlot);
+
+  logger?.verbose("Initialized protoArray successfully", {...logCtx, length: protoArray.length()});
+
+  // forkchoiceConstructor is only used for some test cases
+  // production code use ForkChoice constructor directly
+  const forkchoiceConstructor = opts.forkchoiceConstructor ?? ForkChoice;
+
+  return new forkchoiceConstructor(
+    config,
+    store,
+    protoArray,
+    unfinalizedState.validators.length,
     metrics,
     opts,
     logger

package/src/chain/interface.ts

@@ -178,6 +178,8 @@ export interface IBeaconChain {
     stateRoot: RootHex,
     opts?: StateGetOpts
   ): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null>;
+  /** Return serialized bytes of a persisted checkpoint state */
+  getPersistedCheckpointState(checkpoint?: phase0.Checkpoint): Promise<Uint8Array | null>;
   /** Returns a cached state by checkpoint */
   getStateByCheckpoint(
     checkpoint: CheckpointWithHex

package/src/chain/options.ts

@@ -8,6 +8,7 @@ import {ShufflingCacheOpts} from "./shufflingCache.js";
 import {DEFAULT_MAX_BLOCK_STATES, FIFOBlockStateCacheOpts} from "./stateCache/fifoBlockStateCache.js";
 import {
   DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY,
+  DEFAULT_MAX_CP_STATE_ON_DISK,
   PersistentCheckpointStateCacheOpts,
 } from "./stateCache/persistentCheckpointsCache.js";
 import {ValidatorMonitorOpts} from "./validatorMonitor.js";
@@ -41,8 +42,6 @@ export type IChainOptions = BlockProcessOpts &
     maxCachedBlobSidecars?: number;
     /** Max number of produced block roots (blinded or full) cached for broadcast validations */
     maxCachedProducedRoots?: number;
-    /** Subscribe to and custody all data column sidecar subnets */
-    supernode?: boolean;
     initialCustodyGroupCount?: number;
     broadcastValidationStrictness?: string;
     minSameMessageSignatureSetsToBatch: number;
@@ -118,7 +117,6 @@ export const defaultChainOptions: IChainOptions = {
   archiveMode: DEFAULT_ARCHIVE_MODE,
   pruneHistory: false,
   emitPayloadAttributes: false,
-  supernode: false,
   // for gossip block validation, it's unlikely we see a reorg with 32 slots
   // for attestation validation, having this value ensures we don't have to regen states most of the time
   maxSkipSlots: 32,
@@ -135,4 +133,5 @@ export const defaultChainOptions: IChainOptions = {
   nHistoricalStatesFileDataStore: true,
   maxBlockStates: DEFAULT_MAX_BLOCK_STATES,
   maxCPStateEpochsInMemory: DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY,
+  maxCPStateEpochsOnDisk: DEFAULT_MAX_CP_STATE_ON_DISK,
 };

package/src/chain/stateCache/datastore/db.ts

@@ -1,5 +1,11 @@
-import {phase0, ssz} from "@lodestar/types";
+import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import {Epoch, phase0, ssz} from "@lodestar/types";
+import {MapDef} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/interface.js";
+import {
+  getLastProcessedSlotFromBeaconStateSerialized,
+  getSlotFromBeaconStateSerialized,
+} from "../../../util/sszBytes.js";
 import {CPStateDatastore, DatastoreKey} from "./types.js";
 
 /**
@@ -22,6 +28,13 @@ export class DbCPStateDatastore implements CPStateDatastore {
     return this.db.checkpointState.getBinary(serializedCheckpoint);
   }
 
+  async readLatestSafe(): Promise<Uint8Array | null> {
+    const allKeys = await this.readKeys();
+    if (allKeys.length === 0) return null;
+
+    return getLatestSafeDatastoreKey(allKeys, this.read.bind(this));
+  }
+
   async readKeys(): Promise<DatastoreKey[]> {
     return this.db.checkpointState.keys();
   }
@@ -34,3 +47,78 @@ export function datastoreKeyToCheckpoint(key: DatastoreKey): phase0.Checkpoint {
 export function checkpointToDatastoreKey(cp: phase0.Checkpoint): DatastoreKey {
   return ssz.phase0.Checkpoint.serialize(cp);
 }
+
+/**
+ * Get the latest safe checkpoint state the node can use to boot from
+ *   - it should be the checkpoint state that's unique in its epoch
+ *   - its last processed block slot should be at epoch boundary or last slot of previous epoch
+ *   - state slot should be at epoch boundary
+ *   - state slot should be equal to epoch * SLOTS_PER_EPOCH
+ *
+ * return the serialized data of Current Root Checkpoint State (CRCS) or Previous Root Checkpoint State (PRCS)
+ *
+ */
+export async function getLatestSafeDatastoreKey(
+  allKeys: DatastoreKey[],
+  readFn: (key: DatastoreKey) => Promise<Uint8Array | null>
+): Promise<Uint8Array | null> {
+  const checkpointsByEpoch = new MapDef<Epoch, DatastoreKey[]>(() => []);
+  for (const key of allKeys) {
+    const cp = datastoreKeyToCheckpoint(key);
+    checkpointsByEpoch.getOrDefault(cp.epoch).push(key);
+  }
+
+  const dataStoreKeyByEpoch: Map<Epoch, DatastoreKey> = new Map();
+  for (const [epoch, keys] of checkpointsByEpoch.entries()) {
+    // only consider epochs with a single checkpoint to avoid ambiguity from forks
+    if (keys.length === 1) {
+      dataStoreKeyByEpoch.set(epoch, keys[0]);
+    }
+  }
+
+  const epochsDesc = Array.from(dataStoreKeyByEpoch.keys()).sort((a, b) => b - a);
+  for (const epoch of epochsDesc) {
+    const datastoreKey = dataStoreKeyByEpoch.get(epoch);
+    if (datastoreKey == null) {
+      // should not happen
+      continue;
+    }
+
+    const stateBytes = await readFn(datastoreKey);
+    if (stateBytes == null) {
+      // should not happen
+      continue;
+    }
+
+    const lastProcessedSlot = getLastProcessedSlotFromBeaconStateSerialized(stateBytes);
+    if (lastProcessedSlot == null) {
+      // cannot extract last processed slot from serialized state, skip
+      continue;
+    }
+
+    const stateSlot = getSlotFromBeaconStateSerialized(stateBytes);
+    if (stateSlot == null) {
+      // cannot extract slot from serialized state, skip
+      continue;
+    }
+
+    if (lastProcessedSlot !== stateSlot && lastProcessedSlot !== stateSlot - 1) {
+      // not CRCS or PRCS, skip
+      continue;
+    }
+
+    if (stateSlot % SLOTS_PER_EPOCH !== 0) {
+      // not at epoch boundary, skip
+      continue;
+    }
+
+    if (stateSlot !== SLOTS_PER_EPOCH * epoch) {
+      // should not happen after above checks, but just to be safe
+      continue;
+    }
+
+    return stateBytes;
+  }
+
+  return null;
+}

package/src/chain/stateCache/datastore/file.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 import {phase0, ssz} from "@lodestar/types";
 import {fromHex, toHex} from "@lodestar/utils";
 import {ensureDir, readFile, readFileNames, removeFile, writeIfNotExist} from "../../../util/file.js";
+import {getLatestSafeDatastoreKey} from "./db.js";
 import {CPStateDatastore, DatastoreKey} from "./types.js";
 
 const CHECKPOINT_STATES_FOLDER = "checkpoint_states";
@@ -44,6 +45,13 @@ export class FileCPStateDatastore implements CPStateDatastore {
     return readFile(filePath);
   }
 
+  async readLatestSafe(): Promise<Uint8Array | null> {
+    const allKeys = await this.readKeys();
+    if (allKeys.length === 0) return null;
+
+    return getLatestSafeDatastoreKey(allKeys, this.read.bind(this));
+  }
+
   async readKeys(): Promise<DatastoreKey[]> {
     const fileNames = await readFileNames(this.folderPath);
     return fileNames

package/src/chain/stateCache/datastore/types.ts

@@ -8,6 +8,7 @@ export interface CPStateDatastore {
   write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array) => Promise<DatastoreKey>;
   remove: (key: DatastoreKey) => Promise<void>;
   read: (key: DatastoreKey) => Promise<Uint8Array | null>;
+  readLatestSafe: () => Promise<Uint8Array | null>;
   readKeys: () => Promise<DatastoreKey[]>;
   init?: () => Promise<void>;
 }

package/src/chain/stateCache/persistentCheckpointsCache.ts

@@ -17,8 +17,10 @@ import {MapTracker} from "./mapMetrics.js";
 import {BlockStateCache, CacheItemType, CheckpointHex, CheckpointStateCache} from "./types.js";
 
 export type PersistentCheckpointStateCacheOpts = {
-  /** Keep max n states in memory, persist the rest to disk */
+  /** Keep max n state epochs in memory, persist the rest to disk */
   maxCPStateEpochsInMemory?: number;
+  /** Keep max n state epochs on disk */
+  maxCPStateEpochsOnDisk?: number;
 };
 
 type PersistentCheckpointStateCacheModules = {
@@ -58,6 +60,14 @@ type LoadedStateBytesData = {persistedKey: DatastoreKey; stateBytes: Uint8Array}
  */
 export const DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY = 3;
 
+/**
+ * By default we don't prune any persistent checkpoint states as it's not safe to delete them during
+ * long non-finality as we don't know the state of the chain and there could be a deep (hundreds of epochs) reorg
+ * if there two competing chains with similar weight but we wouldn't have a close enough state to pivot to this chain
+ * and instead require a resync from last finalized checkpoint state which could be very far in the past.
+ */
+export const DEFAULT_MAX_CP_STATE_ON_DISK = Infinity;
+
 // TODO GLOAS: re-evaluate this timing
 const PROCESS_CHECKPOINT_STATES_BPS = 6667;
 
@@ -104,6 +114,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   private preComputedCheckpoint: string | null = null;
   private preComputedCheckpointHits: number | null = null;
   private readonly maxEpochsInMemory: number;
+  private readonly maxEpochsOnDisk: number;
   private readonly datastore: CPStateDatastore;
   private readonly blockStateCache: BlockStateCache;
   private readonly bufferPool?: BufferPool | null;
@@ -139,10 +150,16 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
     this.logger = logger;
     this.clock = clock;
     this.signal = signal;
+
     if (opts.maxCPStateEpochsInMemory !== undefined && opts.maxCPStateEpochsInMemory < 0) {
       throw new Error("maxEpochsInMemory must be >= 0");
     }
+    if (opts.maxCPStateEpochsOnDisk !== undefined && opts.maxCPStateEpochsOnDisk < 0) {
+      throw new Error("maxCPStateEpochsOnDisk must be >= 0");
+    }
+
     this.maxEpochsInMemory = opts.maxCPStateEpochsInMemory ?? DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY;
+    this.maxEpochsOnDisk = opts.maxCPStateEpochsOnDisk ?? DEFAULT_MAX_CP_STATE_ON_DISK;
     // Specify different datastore for testing
     this.datastore = datastore;
     this.blockStateCache = blockStateCache;
@@ -324,6 +341,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
       this.logger.verbose("Added checkpoint state to memory", {epoch: cp.epoch, rootHex: cpHex.rootHex});
     }
     this.epochIndex.getOrDefault(cp.epoch).add(cpHex.rootHex);
+    this.prunePersistedStates();
   }
 
   /**
@@ -766,13 +784,38 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
       this.cache.delete(key);
     }
     this.epochIndex.delete(epoch);
-    this.logger.verbose("Pruned finalized checkpoints states for epoch", {
+    this.logger.verbose("Pruned checkpoint states for epoch", {
       epoch,
       persistCount,
       rootHexes: Array.from(rootHexes).join(","),
     });
   }
 
+  /**
+   * Prune persisted checkpoint states from disk.
+   * Note that this should handle all possible errors and not throw.
+   */
+  private prunePersistedStates(): void {
+    //                epochsOnDisk                                   epochsInMemory
+    // |----------------------------------------------------------|----------------------|
+    const maxTrackedEpochs = this.maxEpochsOnDisk + this.maxEpochsInMemory;
+    if (this.epochIndex.size <= maxTrackedEpochs) {
+      return;
+    }
+
+    const sortedEpochs = Array.from(this.epochIndex.keys()).sort((a, b) => a - b);
+    const pruneEpochs = sortedEpochs.slice(0, sortedEpochs.length - maxTrackedEpochs);
+    for (const epoch of pruneEpochs) {
+      this.deleteAllEpochItems(epoch).catch((e) =>
+        this.logger.debug(
+          "Error delete all epoch items",
+          {epoch, maxEpochsOnDisk: this.maxEpochsOnDisk, maxEpochsInMemory: this.maxEpochsInMemory},
+          e as Error
+        )
+      );
+    }
+  }
+
   /**
    * Serialize validators to bytes leveraging the buffer pool to save memory allocation.
    *   - As monitored on holesky as of Jan 2024, it helps save ~500ms state reload time (4.3s vs 3.8s)

package/src/chain/validation/dataColumnSidecar.ts

@@ -39,7 +39,7 @@ export async function validateGossipDataColumnSidecar(
   if (computeSubnetForDataColumnSidecar(chain.config, dataColumnSidecar) !== gossipSubnet) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_SUBNET,
-      columnIdx: dataColumnSidecar.index,
+      columnIndex: dataColumnSidecar.index,
       gossipSubnet: gossipSubnet,
     });
   }
@@ -156,7 +156,7 @@ export async function validateGossipDataColumnSidecar(
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID,
       slot: dataColumnSidecar.signedBlockHeader.message.slot,
-      columnIdx: dataColumnSidecar.index,
+      columnIndex: dataColumnSidecar.index,
     });
   }
 
@@ -173,7 +173,7 @@ export async function validateGossipDataColumnSidecar(
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF,
       slot: blockHeader.slot,
-      columnIdx: dataColumnSidecar.index,
+      columnIndex: dataColumnSidecar.index,
     });
   } finally {
     kzgProofTimer?.();
@@ -193,7 +193,7 @@ function verifyDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar: ful
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_INDEX,
       slot: dataColumnSidecar.signedBlockHeader.message.slot,
-      columnIdx: dataColumnSidecar.index,
+      columnIndex: dataColumnSidecar.index,
     });
   }
 
@@ -201,7 +201,7 @@ function verifyDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar: ful
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.NO_COMMITMENTS,
       slot: dataColumnSidecar.signedBlockHeader.message.slot,
-      columnIdx: dataColumnSidecar.index,
+      columnIndex: dataColumnSidecar.index,
     });
   }
 
@@ -212,7 +212,7 @@ function verifyDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar: ful
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.TOO_MANY_KZG_COMMITMENTS,
       slot: dataColumnSidecar.signedBlockHeader.message.slot,
-      columnIdx: dataColumnSidecar.index,
+      columnIndex: dataColumnSidecar.index,
       count: dataColumnSidecar.kzgCommitments.length,
       limit: maxBlobsPerBlock,
     });
@@ -293,7 +293,6 @@ export async function validateBlockDataColumnSidecars(
       "Block has no blob commitments but data column sidecars were provided"
     );
   }
-
   // Hash the first sidecar block header and compare the rest via (cheaper) equality
   const firstSidecarBlockHeader = dataColumnSidecars[0].signedBlockHeader.message;
   const firstBlockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(firstSidecarBlockHeader);
@@ -302,7 +301,7 @@ export async function validateBlockDataColumnSidecars(
       {
         code: DataColumnSidecarErrorCode.INCORRECT_BLOCK,
         slot: blockSlot,
-        columnIdx: 0,
+        columnIndex: 0,
         expected: toRootHex(blockRoot),
         actual: toRootHex(firstBlockRoot),
       },
@@ -317,33 +316,52 @@ export async function validateBlockDataColumnSidecars(
   for (let i = 0; i < dataColumnSidecars.length; i++) {
     const columnSidecar = dataColumnSidecars[i];
 
+    if (!ssz.phase0.BeaconBlockHeader.equals(firstSidecarBlockHeader, columnSidecar.signedBlockHeader.message)) {
+      throw new DataColumnSidecarValidationError({
+        code: DataColumnSidecarErrorCode.INCORRECT_HEADER_ROOT,
+        slot: blockSlot,
+        expected: toRootHex(blockRoot),
+        actual: toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(columnSidecar.signedBlockHeader.message)),
+      });
+    }
+
     if (columnSidecar.index >= NUMBER_OF_COLUMNS) {
       throw new DataColumnSidecarValidationError(
         {
           code: DataColumnSidecarErrorCode.INVALID_INDEX,
           slot: blockSlot,
-          columnIdx: columnSidecar.index,
+          columnIndex: columnSidecar.index,
         },
         "DataColumnSidecar has invalid index"
       );
     }
 
-    if (columnSidecar.kzgCommitments.length !== blockBlobCount) {
+    if (columnSidecar.column.length !== blockBlobCount) {
       throw new DataColumnSidecarValidationError({
-        code: DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT,
+        code: DataColumnSidecarErrorCode.INCORRECT_CELL_COUNT,
         slot: blockSlot,
-        columnIdx: columnSidecar.index,
+        columnIndex: columnSidecar.index,
         expected: blockBlobCount,
+        actual: columnSidecar.column.length,
+      });
+    }
+
+    if (columnSidecar.column.length !== columnSidecar.kzgCommitments.length) {
+      throw new DataColumnSidecarValidationError({
+        code: DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT,
+        slot: blockSlot,
+        columnIndex: columnSidecar.index,
+        expected: columnSidecar.column.length,
         actual: columnSidecar.kzgCommitments.length,
       });
     }
 
-    if (columnSidecar.kzgProofs.length !== columnSidecar.kzgCommitments.length) {
+    if (columnSidecar.column.length !== columnSidecar.kzgProofs.length) {
       throw new DataColumnSidecarValidationError({
         code: DataColumnSidecarErrorCode.INCORRECT_KZG_PROOF_COUNT,
         slot: blockSlot,
-        columnIdx: columnSidecar.index,
-        expected: columnSidecar.kzgCommitments.length,
+        columnIndex: columnSidecar.index,
+        expected: columnSidecar.column.length,
         actual: columnSidecar.kzgProofs.length,
       });
     }
@@ -353,7 +371,7 @@ export async function validateBlockDataColumnSidecars(
         {
           code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID,
           slot: blockSlot,
-          columnIdx: columnSidecar.index,
+          columnIndex: columnSidecar.index,
         },
         "DataColumnSidecar has invalid inclusion proof"
       );

package/src/index.ts

@@ -3,6 +3,8 @@
 export type {RestApiServerMetrics, RestApiServerModules, RestApiServerOpts} from "./api/rest/base.js";
 export {RestApiServer} from "./api/rest/base.js";
 export {checkAndPersistAnchorState, initStateFromDb, initStateFromEth1} from "./chain/index.js";
+export {DbCPStateDatastore} from "./chain/stateCache/datastore/db.js";
+export {FileCPStateDatastore} from "./chain/stateCache/datastore/file.js";
 export {BeaconDb, type IBeaconDb} from "./db/index.js";
 export {Eth1Provider, type IEth1Provider} from "./eth1/index.js";
 // Export metrics utilities to de-duplicate validator metrics

package/src/metrics/metrics/lodestar.ts

@@ -1878,7 +1878,7 @@ export function createLodestarMetrics(
       fetchKeys: register.histogram({
         name: "lodestar_prune_history_fetch_keys_time_seconds",
         help: "Time to fetch keys in seconds",
-        buckets: [0.001, 0.01, 0.1, 1],
+        buckets: [0.001, 0.01, 0.1, 0.3, 0.5, 1],
       }),
 
       pruneKeys: register.histogram({

package/src/network/core/networkCore.ts

@@ -553,7 +553,11 @@ export class NetworkCore implements INetworkCore {
           // On fork boundary transition
           if (epoch === nextBoundaryEpoch) {
             // updateEth2Field() MUST be called with clock epoch, onEpoch event is emitted in response to clock events
-            this.metadata.updateEth2Field(epoch);
+            const {forkDigest} = this.metadata.updateEth2Field(epoch);
+            // Update local status to reflect the new fork digest, otherwise we will disconnect peers that re-status us
+            // right after the fork transition due to incompatible forks as our fork digest is stale since we only
+            // update it once we import a new head or when emitting update status event.
+            this.statusCache.update({...this.statusCache.get(), forkDigest});
             this.reqResp.registerProtocolsAtBoundary(nextBoundary);
           }