@lodestar/beacon-node 1.36.0-dev.43f2c7cbbd → 1.36.0-dev.52be14b24f

package/src/api/impl/lodestar/index.ts

@@ -1,5 +1,3 @@
-import fs from "node:fs";
-import path from "node:path";
 import {Tree} from "@chainsafe/persistent-merkle-tree";
 import {routes} from "@lodestar/api";
 import {ApplicationMethods} from "@lodestar/api/server";
@@ -8,13 +6,16 @@ import {Repository} from "@lodestar/db";
 import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {BeaconStateCapella, getLatestWeakSubjectivityCheckpointEpoch, loadState} from "@lodestar/state-transition";
 import {ssz} from "@lodestar/types";
-import {toHex, toRootHex} from "@lodestar/utils";
+import {Checkpoint} from "@lodestar/types/phase0";
+import {fromHex, toHex, toRootHex} from "@lodestar/utils";
 import {BeaconChain} from "../../../chain/index.js";
 import {QueuedStateRegenerator, RegenRequest} from "../../../chain/regen/index.js";
 import {IBeaconDb} from "../../../db/interface.js";
 import {GossipType} from "../../../network/index.js";
-import {profileNodeJS, writeHeapSnapshot} from "../../../util/profile.js";
+import {getStateSlotFromBytes} from "../../../util/multifork.js";
+import {ProfileThread, profileThread, writeHeapSnapshot} from "../../../util/profile.js";
 import {getStateResponseWithRegen} from "../beacon/state/utils.js";
+import {ApiError} from "../errors.js";
 import {ApiModules} from "../types.js";
 
 export function getLodestarApi({
@@ -26,7 +27,9 @@ export function getLodestarApi({
 }: Pick<ApiModules, "chain" | "config" | "db" | "network" | "sync">): ApplicationMethods<routes.lodestar.Endpoints> {
   let writingHeapdump = false;
   let writingProfile = false;
-  const defaultProfileMs = SLOTS_PER_EPOCH * config.SLOT_DURATION_MS;
+  // for NodeJS, profile the whole epoch
+  // for Bun, profile 1 slot. Otherwise it will either crash the app, and/or inspector cannot render the profile
+  const defaultProfileMs = globalThis.Bun ? config.SLOT_DURATION_MS : SLOTS_PER_EPOCH * config.SLOT_DURATION_MS;
 
   return {
     async writeHeapdump({thread = "main", dirpath = "."}) {
@@ -63,7 +66,6 @@ export function getLodestarApi({
 
       try {
         let filepath: string;
-        let profile: string;
         switch (thread) {
           case "network":
             filepath = await network.writeNetworkThreadProfile(duration, dirpath);
@@ -73,12 +75,10 @@ export function getLodestarApi({
             break;
           default:
             // main thread
-            profile = await profileNodeJS(duration);
-            filepath = path.join(dirpath, `main_thread_${new Date().toISOString()}.cpuprofile`);
-            fs.writeFileSync(filepath, profile);
+            filepath = await profileThread(ProfileThread.MAIN, duration, dirpath);
             break;
         }
-        return {data: {filepath}};
+        return {data: {result: filepath}};
       } finally {
         writingProfile = false;
       }
@@ -218,6 +218,26 @@ export function getLodestarApi({
         meta: {executionOptimistic, finalized, version: fork},
       };
     },
+
+    // the optional checkpoint is in root:epoch format
+    async getPersistedCheckpointState({checkpointId}) {
+      const checkpoint = checkpointId ? getCheckpointFromArg(checkpointId) : undefined;
+      const stateBytes = await chain.getPersistedCheckpointState(checkpoint);
+      if (stateBytes === null) {
+        throw new ApiError(
+          404,
+          checkpointId ? `Checkpoint state not found for id ${checkpointId}` : "Latest safe checkpoint state not found"
+        );
+      }
+
+      const slot = getStateSlotFromBytes(stateBytes);
+      return {
+        data: stateBytes,
+        meta: {
+          version: config.getForkName(slot),
+        },
+      };
+    },
   };
 }
 
@@ -247,6 +267,18 @@ function regenRequestToJson(config: ChainForkConfig, regenRequest: RegenRequest)
   }
 }
 
+const CHECKPOINT_REGEX = /^(?:0x)?([0-9a-f]{64}):([0-9]+)$/;
+/**
+ * Extract a checkpoint from a string in the format `rootHex:epoch`.
+ */
+export function getCheckpointFromArg(checkpointStr: string): Checkpoint {
+  const match = CHECKPOINT_REGEX.exec(checkpointStr.toLowerCase());
+  if (!match) {
+    throw new ApiError(400, `Could not parse checkpoint string: ${checkpointStr}`);
+  }
+  return {root: fromHex(match[1]), epoch: parseInt(match[2])};
+}
+
 function stringifyKeys(keys: (Uint8Array | number | string)[]): string[] {
   return keys.map((key) => {
     if (key instanceof Uint8Array) {

package/src/api/impl/node/utils.ts

@@ -10,7 +10,7 @@ export function formatNodePeer(peerIdStr: string, connections: Connection[]): ro
   return {
     peerId: conn ? conn.remotePeer.toString() : peerIdStr,
     // TODO: figure out how to get enr of peer
-    enr: "",
+    enr: null,
     lastSeenP2pAddress: conn ? conn.remoteAddr.toString() : "",
     direction: conn ? (conn.direction as routes.node.PeerDirection) : null,
     state: conn ? getPeerState(conn.status) : "disconnected",

package/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts

@@ -63,7 +63,9 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy {
     const lastStoredEpoch = computeEpochAtSlot(lastStoredSlot ?? 0);
     const {archiveStateEpochFrequency} = this.opts;
 
+    const logCtx = {finalizedEpoch: finalized.epoch, lastStoredEpoch, archiveStateEpochFrequency};
     if (finalized.epoch - lastStoredEpoch >= Math.min(PERSIST_TEMP_STATE_EVERY_EPOCHS, archiveStateEpochFrequency)) {
+      this.logger.verbose("Start archiving state", logCtx);
       await this.archiveState(finalized, metrics);
 
       // Only check the current and previous intervals
@@ -88,11 +90,13 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy {
 
       // More logs to investigate the rss spike issue https://github.com/ChainSafe/lodestar/issues/5591
       this.logger.verbose("Archived state completed", {
-        finalizedEpoch: finalized.epoch,
+        ...logCtx,
         minEpoch,
         storedStateSlots: storedStateSlots.join(","),
         statesSlotsToDelete: statesSlotsToDelete.join(","),
       });
+    } else {
+      this.logger.verbose("Skip archiving state", logCtx);
     }
   }
 

package/src/chain/chain.ts

@@ -37,6 +37,7 @@ import {
   ValidatorIndex,
   Wei,
   isBlindedBeaconBlock,
+  phase0,
 } from "@lodestar/types";
 import {Logger, fromHex, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
@@ -95,11 +96,13 @@ import {SeenBlockAttesters} from "./seenCache/seenBlockAttesters.js";
 import {SeenBlockInput} from "./seenCache/seenGossipBlockInput.js";
 import {ShufflingCache} from "./shufflingCache.js";
 import {BlockStateCacheImpl} from "./stateCache/blockStateCacheImpl.js";
-import {DbCPStateDatastore} from "./stateCache/datastore/db.js";
+import {DbCPStateDatastore, checkpointToDatastoreKey} from "./stateCache/datastore/db.js";
 import {FileCPStateDatastore} from "./stateCache/datastore/file.js";
+import {CPStateDatastore} from "./stateCache/datastore/types.js";
 import {FIFOBlockStateCache} from "./stateCache/fifoBlockStateCache.js";
 import {InMemoryCheckpointStateCache} from "./stateCache/inMemoryCheckpointsCache.js";
 import {PersistentCheckpointStateCache} from "./stateCache/persistentCheckpointsCache.js";
+import {CheckpointStateCache} from "./stateCache/types.js";
 import {ValidatorMonitor} from "./validatorMonitor.js";
 
 /**
@@ -181,6 +184,8 @@ export class BeaconChain implements IBeaconChain {
 
   protected readonly blockProcessor: BlockProcessor;
   protected readonly db: IBeaconDb;
+  // this is only available if nHistoricalStates is enabled
+  private readonly cpStateDatastore?: CPStateDatastore;
   private abortController = new AbortController();
   private processShutdownCallback: ProcessShutdownCallback;
   private _earliestAvailableSlot: Slot;
@@ -210,6 +215,7 @@ export class BeaconChain implements IBeaconChain {
       metrics,
       validatorMonitor,
       anchorState,
+      isAnchorStateFinalized,
       eth1,
       executionEngine,
       executionBuilder,
@@ -226,6 +232,7 @@ export class BeaconChain implements IBeaconChain {
       metrics: Metrics | null;
       validatorMonitor: ValidatorMonitor | null;
       anchorState: BeaconStateAllForks;
+      isAnchorStateFinalized: boolean;
       eth1: IEth1ForBlockProduction;
       executionEngine: IExecutionEngine;
       executionBuilder?: IExecutionBuilder;
@@ -326,23 +333,26 @@ export class BeaconChain implements IBeaconChain {
     this.bufferPool = this.opts.nHistoricalStates
       ? new BufferPool(anchorState.type.tree_serializedSize(anchorState.node), metrics)
       : null;
-    const checkpointStateCache = this.opts.nHistoricalStates
-      ? new PersistentCheckpointStateCache(
-          {
-            metrics,
-            logger,
-            clock,
-            blockStateCache,
-            bufferPool: this.bufferPool,
-            datastore: fileDataStore
-              ? // debug option if we want to investigate any issues with the DB
-                new FileCPStateDatastore(dataDir)
-              : // production option
-                new DbCPStateDatastore(this.db),
-          },
-          this.opts
-        )
-      : new InMemoryCheckpointStateCache({metrics});
+
+    let checkpointStateCache: CheckpointStateCache;
+    this.cpStateDatastore = undefined;
+    if (this.opts.nHistoricalStates) {
+      this.cpStateDatastore = fileDataStore ? new FileCPStateDatastore(dataDir) : new DbCPStateDatastore(this.db);
+      checkpointStateCache = new PersistentCheckpointStateCache(
+        {
+          metrics,
+          logger,
+          clock,
+          blockStateCache,
+          bufferPool: this.bufferPool,
+          datastore: this.cpStateDatastore,
+        },
+        this.opts
+      );
+    } else {
+      checkpointStateCache = new InMemoryCheckpointStateCache({metrics});
+    }
+
     const {checkpoint} = computeAnchorCheckpoint(config, anchorState);
     blockStateCache.add(cachedState);
     blockStateCache.setHeadState(cachedState);
@@ -353,6 +363,7 @@ export class BeaconChain implements IBeaconChain {
       emitter,
       clock.currentSlot,
       cachedState,
+      isAnchorStateFinalized,
       opts,
       this.justifiedBalancesGetter.bind(this),
       metrics,
@@ -604,6 +615,20 @@ export class BeaconChain implements IBeaconChain {
     return data && {state: data, executionOptimistic: false, finalized: true};
   }
 
+  async getPersistedCheckpointState(checkpoint?: phase0.Checkpoint): Promise<Uint8Array | null> {
+    if (!this.cpStateDatastore) {
+      throw new Error("n-historical-state flag is not enabled");
+    }
+
+    if (checkpoint == null) {
+      // return the last safe checkpoint state by default
+      return this.cpStateDatastore.readLatestSafe();
+    }
+
+    const persistedKey = checkpointToDatastoreKey(checkpoint);
+    return this.cpStateDatastore.read(persistedKey);
+  }
+
   getStateByCheckpoint(
     checkpoint: CheckpointWithHex
   ): {state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null {

package/src/chain/forkChoice/index.ts

@@ -5,17 +5,22 @@ import {
   ForkChoiceStore,
   JustifiedBalancesGetter,
   ProtoArray,
+  ProtoBlock,
   ForkChoiceOpts as RawForkChoiceOpts,
 } from "@lodestar/fork-choice";
+import {ZERO_HASH_HEX} from "@lodestar/params";
 import {
   CachedBeaconStateAllForks,
   DataAvailabilityStatus,
   computeAnchorCheckpoint,
+  computeEpochAtSlot,
+  computeStartSlotAtEpoch,
+  getBlockRootAtSlot,
   getEffectiveBalanceIncrementsZeroInactive,
   isExecutionStateType,
   isMergeTransitionComplete,
 } from "@lodestar/state-transition";
-import {Slot} from "@lodestar/types";
+import {Slot, ssz} from "@lodestar/types";
 import {Logger, toRootHex} from "@lodestar/utils";
 import {GENESIS_SLOT} from "../../constants/index.js";
 import {Metrics} from "../../metrics/index.js";
@@ -35,6 +40,43 @@ export enum ForkchoiceCaller {
  * Fork Choice extended with a ChainEventEmitter
  */
 export function initializeForkChoice(
+  config: ChainForkConfig,
+  emitter: ChainEventEmitter,
+  currentSlot: Slot,
+  state: CachedBeaconStateAllForks,
+  isFinalizedState: boolean,
+  opts: ForkChoiceOpts,
+  justifiedBalancesGetter: JustifiedBalancesGetter,
+  metrics: Metrics | null,
+  logger?: Logger
+): ForkChoice {
+  return isFinalizedState
+    ? initializeForkChoiceFromFinalizedState(
+        config,
+        emitter,
+        currentSlot,
+        state,
+        opts,
+        justifiedBalancesGetter,
+        metrics,
+        logger
+      )
+    : initializeForkChoiceFromUnfinalizedState(
+        config,
+        emitter,
+        currentSlot,
+        state,
+        opts,
+        justifiedBalancesGetter,
+        metrics,
+        logger
+      );
+}
+
+/**
+ * Initialize forkchoice from a finalized state.
+ */
+export function initializeForkChoiceFromFinalizedState(
   config: ChainForkConfig,
   emitter: ChainEventEmitter,
   currentSlot: Slot,
@@ -82,7 +124,7 @@ export function initializeForkChoice(
         parentRoot: toRootHex(blockHeader.parentRoot),
         stateRoot: toRootHex(blockHeader.stateRoot),
         blockRoot: toRootHex(checkpoint.root),
-        timeliness: true, // Optimisitcally assume is timely
+        timeliness: true, // Optimistically assume is timely
 
         justifiedEpoch: justifiedCheckpoint.epoch,
         justifiedRoot: toRootHex(justifiedCheckpoint.root),
@@ -105,6 +147,140 @@ export function initializeForkChoice(
       },
       currentSlot
     ),
+    state.validators.length,
+    metrics,
+    opts,
+    logger
+  );
+}
+
+/**
+ * Initialize forkchoice from an unfinalized state.
+ */
+export function initializeForkChoiceFromUnfinalizedState(
+  config: ChainForkConfig,
+  emitter: ChainEventEmitter,
+  currentSlot: Slot,
+  unfinalizedState: CachedBeaconStateAllForks,
+  opts: ForkChoiceOpts,
+  justifiedBalancesGetter: JustifiedBalancesGetter,
+  metrics: Metrics | null,
+  logger?: Logger
+): ForkChoice {
+  const {blockHeader} = computeAnchorCheckpoint(config, unfinalizedState);
+  const finalizedCheckpoint = unfinalizedState.finalizedCheckpoint.toValue();
+  const justifiedCheckpoint = unfinalizedState.currentJustifiedCheckpoint.toValue();
+  const headRoot = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader));
+
+  const logCtx = {
+    currentSlot: currentSlot,
+    stateSlot: unfinalizedState.slot,
+    headSlot: blockHeader.slot,
+    headRoot: headRoot,
+    finalizedEpoch: finalizedCheckpoint.epoch,
+    finalizedRoot: toRootHex(finalizedCheckpoint.root),
+    justifiedEpoch: justifiedCheckpoint.epoch,
+    justifiedRoot: toRootHex(justifiedCheckpoint.root),
+  };
+  logger?.warn("Initializing fork choice from unfinalized state", logCtx);
+
+  // this is not the justified state, but there is no other ways to get justified balances
+  const justifiedBalances = getEffectiveBalanceIncrementsZeroInactive(unfinalizedState);
+  const store = new ForkChoiceStore(
+    currentSlot,
+    justifiedCheckpoint,
+    finalizedCheckpoint,
+    justifiedBalances,
+    justifiedBalancesGetter,
+    {
+      onJustified: (cp) => emitter.emit(ChainEvent.forkChoiceJustified, cp),
+      onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp),
+    }
+  );
+
+  // this is the same to the finalized state
+  const headBlock: ProtoBlock = {
+    slot: blockHeader.slot,
+    parentRoot: toRootHex(blockHeader.parentRoot),
+    stateRoot: toRootHex(blockHeader.stateRoot),
+    blockRoot: headRoot,
+    targetRoot: headRoot,
+    timeliness: true, // Optimistically assume is timely
+
+    justifiedEpoch: justifiedCheckpoint.epoch,
+    justifiedRoot: toRootHex(justifiedCheckpoint.root),
+    finalizedEpoch: finalizedCheckpoint.epoch,
+    finalizedRoot: toRootHex(finalizedCheckpoint.root),
+    unrealizedJustifiedEpoch: justifiedCheckpoint.epoch,
+    unrealizedJustifiedRoot: toRootHex(justifiedCheckpoint.root),
+    unrealizedFinalizedEpoch: finalizedCheckpoint.epoch,
+    unrealizedFinalizedRoot: toRootHex(finalizedCheckpoint.root),
+
+    ...(isExecutionStateType(unfinalizedState) && isMergeTransitionComplete(unfinalizedState)
+      ? {
+          executionPayloadBlockHash: toRootHex(unfinalizedState.latestExecutionPayloadHeader.blockHash),
+          executionPayloadNumber: unfinalizedState.latestExecutionPayloadHeader.blockNumber,
+          executionStatus: blockHeader.slot === GENESIS_SLOT ? ExecutionStatus.Valid : ExecutionStatus.Syncing,
+        }
+      : {executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}),
+
+    dataAvailabilityStatus: DataAvailabilityStatus.PreData,
+  };
+
+  const parentSlot = blockHeader.slot - 1;
+  const parentEpoch = computeEpochAtSlot(parentSlot);
+  // parent of head block
+  const parentBlock: ProtoBlock = {
+    ...headBlock,
+    slot: parentSlot,
+    // link this to the dummy justified block
+    parentRoot: toRootHex(justifiedCheckpoint.root),
+    // dummy data, we're not able to regen state before headBlock
+    stateRoot: ZERO_HASH_HEX,
+    blockRoot: headBlock.parentRoot,
+    targetRoot: toRootHex(getBlockRootAtSlot(unfinalizedState, computeStartSlotAtEpoch(parentEpoch))),
+  };
+
+  const justifiedBlock: ProtoBlock = {
+    ...headBlock,
+    slot: computeStartSlotAtEpoch(justifiedCheckpoint.epoch),
+    // link this to the finalized root so that getAncestors can find the finalized block
+    parentRoot: toRootHex(finalizedCheckpoint.root),
+    // dummy data, we're not able to regen state before headBlock
+    stateRoot: ZERO_HASH_HEX,
+    blockRoot: toRootHex(justifiedCheckpoint.root),
+    // same to blockRoot
+    targetRoot: toRootHex(justifiedCheckpoint.root),
+  };
+
+  const finalizedBlock: ProtoBlock = {
+    ...headBlock,
+    slot: computeStartSlotAtEpoch(finalizedCheckpoint.epoch),
+    // we don't care parent of finalized block
+    parentRoot: ZERO_HASH_HEX,
+    // dummy data, we're not able to regen state before headBlock
+    stateRoot: ZERO_HASH_HEX,
+    blockRoot: toRootHex(finalizedCheckpoint.root),
+    // same to blockRoot
+    targetRoot: toRootHex(finalizedCheckpoint.root),
+  };
+
+  const protoArray = ProtoArray.initialize(finalizedBlock, currentSlot);
+  protoArray.onBlock(justifiedBlock, currentSlot);
+  protoArray.onBlock(parentBlock, currentSlot);
+  protoArray.onBlock(headBlock, currentSlot);
+
+  logger?.verbose("Initialized protoArray successfully", {...logCtx, length: protoArray.length()});
+
+  // forkchoiceConstructor is only used for some test cases
+  // production code use ForkChoice constructor directly
+  const forkchoiceConstructor = opts.forkchoiceConstructor ?? ForkChoice;
+
+  return new forkchoiceConstructor(
+    config,
+    store,
+    protoArray,
+    unfinalizedState.validators.length,
     metrics,
     opts,
     logger

package/src/chain/interface.ts

@@ -178,6 +178,8 @@ export interface IBeaconChain {
     stateRoot: RootHex,
     opts?: StateGetOpts
   ): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null>;
+  /** Return serialized bytes of a persisted checkpoint state */
+  getPersistedCheckpointState(checkpoint?: phase0.Checkpoint): Promise<Uint8Array | null>;
   /** Returns a cached state by checkpoint */
   getStateByCheckpoint(
     checkpoint: CheckpointWithHex

package/src/chain/stateCache/datastore/db.ts

@@ -1,5 +1,11 @@
-import {phase0, ssz} from "@lodestar/types";
+import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import {Epoch, phase0, ssz} from "@lodestar/types";
+import {MapDef} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/interface.js";
+import {
+  getLastProcessedSlotFromBeaconStateSerialized,
+  getSlotFromBeaconStateSerialized,
+} from "../../../util/sszBytes.js";
 import {CPStateDatastore, DatastoreKey} from "./types.js";
 
 /**
@@ -22,6 +28,13 @@ export class DbCPStateDatastore implements CPStateDatastore {
     return this.db.checkpointState.getBinary(serializedCheckpoint);
   }
 
+  async readLatestSafe(): Promise<Uint8Array | null> {
+    const allKeys = await this.readKeys();
+    if (allKeys.length === 0) return null;
+
+    return getLatestSafeDatastoreKey(allKeys, this.read.bind(this));
+  }
+
   async readKeys(): Promise<DatastoreKey[]> {
     return this.db.checkpointState.keys();
   }
@@ -34,3 +47,78 @@ export function datastoreKeyToCheckpoint(key: DatastoreKey): phase0.Checkpoint {
 export function checkpointToDatastoreKey(cp: phase0.Checkpoint): DatastoreKey {
   return ssz.phase0.Checkpoint.serialize(cp);
 }
+
+/**
+ * Get the latest safe checkpoint state the node can use to boot from
+ *   - it should be the checkpoint state that's unique in its epoch
+ *   - its last processed block slot should be at epoch boundary or last slot of previous epoch
+ *   - state slot should be at epoch boundary
+ *   - state slot should be equal to epoch * SLOTS_PER_EPOCH
+ *
+ * return the serialized data of Current Root Checkpoint State (CRCS) or Previous Root Checkpoint State (PRCS)
+ *
+ */
+export async function getLatestSafeDatastoreKey(
+  allKeys: DatastoreKey[],
+  readFn: (key: DatastoreKey) => Promise<Uint8Array | null>
+): Promise<Uint8Array | null> {
+  const checkpointsByEpoch = new MapDef<Epoch, DatastoreKey[]>(() => []);
+  for (const key of allKeys) {
+    const cp = datastoreKeyToCheckpoint(key);
+    checkpointsByEpoch.getOrDefault(cp.epoch).push(key);
+  }
+
+  const dataStoreKeyByEpoch: Map<Epoch, DatastoreKey> = new Map();
+  for (const [epoch, keys] of checkpointsByEpoch.entries()) {
+    // only consider epochs with a single checkpoint to avoid ambiguity from forks
+    if (keys.length === 1) {
+      dataStoreKeyByEpoch.set(epoch, keys[0]);
+    }
+  }
+
+  const epochsDesc = Array.from(dataStoreKeyByEpoch.keys()).sort((a, b) => b - a);
+  for (const epoch of epochsDesc) {
+    const datastoreKey = dataStoreKeyByEpoch.get(epoch);
+    if (datastoreKey == null) {
+      // should not happen
+      continue;
+    }
+
+    const stateBytes = await readFn(datastoreKey);
+    if (stateBytes == null) {
+      // should not happen
+      continue;
+    }
+
+    const lastProcessedSlot = getLastProcessedSlotFromBeaconStateSerialized(stateBytes);
+    if (lastProcessedSlot == null) {
+      // cannot extract last processed slot from serialized state, skip
+      continue;
+    }
+
+    const stateSlot = getSlotFromBeaconStateSerialized(stateBytes);
+    if (stateSlot == null) {
+      // cannot extract slot from serialized state, skip
+      continue;
+    }
+
+    if (lastProcessedSlot !== stateSlot && lastProcessedSlot !== stateSlot - 1) {
+      // not CRCS or PRCS, skip
+      continue;
+    }
+
+    if (stateSlot % SLOTS_PER_EPOCH !== 0) {
+      // not at epoch boundary, skip
+      continue;
+    }
+
+    if (stateSlot !== SLOTS_PER_EPOCH * epoch) {
+      // should not happen after above checks, but just to be safe
+      continue;
+    }
+
+    return stateBytes;
+  }
+
+  return null;
+}

package/src/chain/stateCache/datastore/file.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 import {phase0, ssz} from "@lodestar/types";
 import {fromHex, toHex} from "@lodestar/utils";
 import {ensureDir, readFile, readFileNames, removeFile, writeIfNotExist} from "../../../util/file.js";
+import {getLatestSafeDatastoreKey} from "./db.js";
 import {CPStateDatastore, DatastoreKey} from "./types.js";
 
 const CHECKPOINT_STATES_FOLDER = "checkpoint_states";
@@ -44,6 +45,13 @@ export class FileCPStateDatastore implements CPStateDatastore {
     return readFile(filePath);
   }
 
+  async readLatestSafe(): Promise<Uint8Array | null> {
+    const allKeys = await this.readKeys();
+    if (allKeys.length === 0) return null;
+
+    return getLatestSafeDatastoreKey(allKeys, this.read.bind(this));
+  }
+
   async readKeys(): Promise<DatastoreKey[]> {
     const fileNames = await readFileNames(this.folderPath);
     return fileNames

package/src/chain/stateCache/datastore/types.ts

@@ -8,6 +8,7 @@ export interface CPStateDatastore {
   write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array) => Promise<DatastoreKey>;
   remove: (key: DatastoreKey) => Promise<void>;
   read: (key: DatastoreKey) => Promise<Uint8Array | null>;
+  readLatestSafe: () => Promise<Uint8Array | null>;
   readKeys: () => Promise<DatastoreKey[]>;
   init?: () => Promise<void>;
 }

package/src/index.ts

@@ -3,6 +3,8 @@
 export type {RestApiServerMetrics, RestApiServerModules, RestApiServerOpts} from "./api/rest/base.js";
 export {RestApiServer} from "./api/rest/base.js";
 export {checkAndPersistAnchorState, initStateFromDb, initStateFromEth1} from "./chain/index.js";
+export {DbCPStateDatastore} from "./chain/stateCache/datastore/db.js";
+export {FileCPStateDatastore} from "./chain/stateCache/datastore/file.js";
 export {BeaconDb, type IBeaconDb} from "./db/index.js";
 export {Eth1Provider, type IEth1Provider} from "./eth1/index.js";
 // Export metrics utilities to de-duplicate validator metrics

package/src/metrics/metrics/lodestar.ts

@@ -1878,7 +1878,7 @@ export function createLodestarMetrics(
       fetchKeys: register.histogram({
         name: "lodestar_prune_history_fetch_keys_time_seconds",
         help: "Time to fetch keys in seconds",
-        buckets: [0.001, 0.01, 0.1, 1],
+        buckets: [0.001, 0.01, 0.1, 0.3, 0.5, 1],
       }),
 
       pruneKeys: register.histogram({

package/src/network/core/networkCoreWorker.ts

@@ -1,5 +1,3 @@
-import fs from "node:fs";
-import path from "node:path";
 import worker from "node:worker_threads";
 import {privateKeyFromProtobuf} from "@libp2p/crypto/keys";
 import {peerIdFromPrivateKey} from "@libp2p/peer-id";
@@ -11,7 +9,7 @@ import {RegistryMetricCreator, collectNodeJSMetrics} from "../../metrics/index.j
 import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js";
 import {Clock} from "../../util/clock.js";
 import {peerIdToString} from "../../util/peerId.js";
-import {profileNodeJS, writeHeapSnapshot} from "../../util/profile.js";
+import {ProfileThread, profileThread, writeHeapSnapshot} from "../../util/profile.js";
 import {wireEventsOnWorkerThread} from "../../util/workerEvents.js";
 import {NetworkEventBus, NetworkEventData, networkEventDirection} from "../events.js";
 import {
@@ -157,10 +155,7 @@ const libp2pWorkerApi: NetworkWorkerApi = {
   dumpDiscv5KadValues: () => core.dumpDiscv5KadValues(),
   dumpMeshPeers: () => core.dumpMeshPeers(),
   writeProfile: async (durationMs: number, dirpath: string) => {
-    const profile = await profileNodeJS(durationMs);
-    const filePath = path.join(dirpath, `network_thread_${new Date().toISOString()}.cpuprofile`);
-    fs.writeFileSync(filePath, profile);
-    return filePath;
+    return profileThread(ProfileThread.NETWORK, durationMs, dirpath);
   },
   writeDiscv5Profile: async (durationMs: number, dirpath: string) => {
     return core.writeDiscv5Profile(durationMs, dirpath);