@lodestar/beacon-node 1.36.0-dev.d42eb95bf0 → 1.36.0-dev.d690a62b6c

package/src/chain/stateCache/datastore/db.ts

@@ -1,5 +1,11 @@
-import {phase0, ssz} from "@lodestar/types";
+import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import {Epoch, phase0, ssz} from "@lodestar/types";
+import {MapDef} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/interface.js";
+import {
+  getLastProcessedSlotFromBeaconStateSerialized,
+  getSlotFromBeaconStateSerialized,
+} from "../../../util/sszBytes.js";
 import {CPStateDatastore, DatastoreKey} from "./types.js";
 
 /**
@@ -22,6 +28,13 @@ export class DbCPStateDatastore implements CPStateDatastore {
     return this.db.checkpointState.getBinary(serializedCheckpoint);
   }
 
+  async readLatestSafe(): Promise<Uint8Array | null> {
+    const allKeys = await this.readKeys();
+    if (allKeys.length === 0) return null;
+
+    return getLatestSafeDatastoreKey(allKeys, this.read.bind(this));
+  }
+
   async readKeys(): Promise<DatastoreKey[]> {
     return this.db.checkpointState.keys();
   }
@@ -34,3 +47,78 @@ export function datastoreKeyToCheckpoint(key: DatastoreKey): phase0.Checkpoint {
 export function checkpointToDatastoreKey(cp: phase0.Checkpoint): DatastoreKey {
   return ssz.phase0.Checkpoint.serialize(cp);
 }
+
+/**
+ * Get the latest safe checkpoint state the node can use to boot from
+ *   - it should be the checkpoint state that's unique in its epoch
+ *   - its last processed block slot should be at epoch boundary or last slot of previous epoch
+ *   - state slot should be at epoch boundary
+ *   - state slot should be equal to epoch * SLOTS_PER_EPOCH
+ *
+ * return the serialized data of Current Root Checkpoint State (CRCS) or Previous Root Checkpoint State (PRCS)
+ *
+ */
+export async function getLatestSafeDatastoreKey(
+  allKeys: DatastoreKey[],
+  readFn: (key: DatastoreKey) => Promise<Uint8Array | null>
+): Promise<Uint8Array | null> {
+  const checkpointsByEpoch = new MapDef<Epoch, DatastoreKey[]>(() => []);
+  for (const key of allKeys) {
+    const cp = datastoreKeyToCheckpoint(key);
+    checkpointsByEpoch.getOrDefault(cp.epoch).push(key);
+  }
+
+  const dataStoreKeyByEpoch: Map<Epoch, DatastoreKey> = new Map();
+  for (const [epoch, keys] of checkpointsByEpoch.entries()) {
+    // only consider epochs with a single checkpoint to avoid ambiguity from forks
+    if (keys.length === 1) {
+      dataStoreKeyByEpoch.set(epoch, keys[0]);
+    }
+  }
+
+  const epochsDesc = Array.from(dataStoreKeyByEpoch.keys()).sort((a, b) => b - a);
+  for (const epoch of epochsDesc) {
+    const datastoreKey = dataStoreKeyByEpoch.get(epoch);
+    if (datastoreKey == null) {
+      // should not happen
+      continue;
+    }
+
+    const stateBytes = await readFn(datastoreKey);
+    if (stateBytes == null) {
+      // should not happen
+      continue;
+    }
+
+    const lastProcessedSlot = getLastProcessedSlotFromBeaconStateSerialized(stateBytes);
+    if (lastProcessedSlot == null) {
+      // cannot extract last processed slot from serialized state, skip
+      continue;
+    }
+
+    const stateSlot = getSlotFromBeaconStateSerialized(stateBytes);
+    if (stateSlot == null) {
+      // cannot extract slot from serialized state, skip
+      continue;
+    }
+
+    if (lastProcessedSlot !== stateSlot && lastProcessedSlot !== stateSlot - 1) {
+      // not CRCS or PRCS, skip
+      continue;
+    }
+
+    if (stateSlot % SLOTS_PER_EPOCH !== 0) {
+      // not at epoch boundary, skip
+      continue;
+    }
+
+    if (stateSlot !== SLOTS_PER_EPOCH * epoch) {
+      // should not happen after above checks, but just to be safe
+      continue;
+    }
+
+    return stateBytes;
+  }
+
+  return null;
+}

package/src/chain/stateCache/datastore/file.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 import {phase0, ssz} from "@lodestar/types";
 import {fromHex, toHex} from "@lodestar/utils";
 import {ensureDir, readFile, readFileNames, removeFile, writeIfNotExist} from "../../../util/file.js";
+import {getLatestSafeDatastoreKey} from "./db.js";
 import {CPStateDatastore, DatastoreKey} from "./types.js";
 
 const CHECKPOINT_STATES_FOLDER = "checkpoint_states";
@@ -44,6 +45,13 @@ export class FileCPStateDatastore implements CPStateDatastore {
     return readFile(filePath);
   }
 
+  async readLatestSafe(): Promise<Uint8Array | null> {
+    const allKeys = await this.readKeys();
+    if (allKeys.length === 0) return null;
+
+    return getLatestSafeDatastoreKey(allKeys, this.read.bind(this));
+  }
+
   async readKeys(): Promise<DatastoreKey[]> {
     const fileNames = await readFileNames(this.folderPath);
     return fileNames

package/src/chain/stateCache/datastore/types.ts

@@ -8,6 +8,7 @@ export interface CPStateDatastore {
   write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array) => Promise<DatastoreKey>;
   remove: (key: DatastoreKey) => Promise<void>;
   read: (key: DatastoreKey) => Promise<Uint8Array | null>;
+  readLatestSafe: () => Promise<Uint8Array | null>;
   readKeys: () => Promise<DatastoreKey[]>;
   init?: () => Promise<void>;
 }

package/src/index.ts

@@ -3,6 +3,8 @@
 export type {RestApiServerMetrics, RestApiServerModules, RestApiServerOpts} from "./api/rest/base.js";
 export {RestApiServer} from "./api/rest/base.js";
 export {checkAndPersistAnchorState, initStateFromDb, initStateFromEth1} from "./chain/index.js";
+export {DbCPStateDatastore} from "./chain/stateCache/datastore/db.js";
+export {FileCPStateDatastore} from "./chain/stateCache/datastore/file.js";
 export {BeaconDb, type IBeaconDb} from "./db/index.js";
 export {Eth1Provider, type IEth1Provider} from "./eth1/index.js";
 // Export metrics utilities to de-duplicate validator metrics

package/src/metrics/metrics/lodestar.ts

@@ -1878,7 +1878,7 @@ export function createLodestarMetrics(
       fetchKeys: register.histogram({
         name: "lodestar_prune_history_fetch_keys_time_seconds",
         help: "Time to fetch keys in seconds",
-        buckets: [0.001, 0.01, 0.1, 1],
+        buckets: [0.001, 0.01, 0.1, 0.3, 0.5, 1],
       }),
 
       pruneKeys: register.histogram({

package/src/network/core/networkCore.ts

@@ -553,7 +553,11 @@ export class NetworkCore implements INetworkCore {
           // On fork boundary transition
           if (epoch === nextBoundaryEpoch) {
             // updateEth2Field() MUST be called with clock epoch, onEpoch event is emitted in response to clock events
-            this.metadata.updateEth2Field(epoch);
+            const {forkDigest} = this.metadata.updateEth2Field(epoch);
+            // Update local status to reflect the new fork digest, otherwise we will disconnect peers that re-status us
+            // right after the fork transition due to incompatible forks as our fork digest is stale since we only
+            // update it once we import a new head or when emitting update status event.
+            this.statusCache.update({...this.statusCache.get(), forkDigest});
             this.reqResp.registerProtocolsAtBoundary(nextBoundary);
           }
 

package/src/network/core/networkCoreWorker.ts

@@ -1,5 +1,3 @@
-import fs from "node:fs";
-import path from "node:path";
 import worker from "node:worker_threads";
 import {privateKeyFromProtobuf} from "@libp2p/crypto/keys";
 import {peerIdFromPrivateKey} from "@libp2p/peer-id";
@@ -11,7 +9,7 @@ import {RegistryMetricCreator, collectNodeJSMetrics} from "../../metrics/index.j
 import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js";
 import {Clock} from "../../util/clock.js";
 import {peerIdToString} from "../../util/peerId.js";
-import {profileNodeJS, writeHeapSnapshot} from "../../util/profile.js";
+import {ProfileThread, profileThread, writeHeapSnapshot} from "../../util/profile.js";
 import {wireEventsOnWorkerThread} from "../../util/workerEvents.js";
 import {NetworkEventBus, NetworkEventData, networkEventDirection} from "../events.js";
 import {
@@ -157,10 +155,7 @@ const libp2pWorkerApi: NetworkWorkerApi = {
   dumpDiscv5KadValues: () => core.dumpDiscv5KadValues(),
   dumpMeshPeers: () => core.dumpMeshPeers(),
   writeProfile: async (durationMs: number, dirpath: string) => {
-    const profile = await profileNodeJS(durationMs);
-    const filePath = path.join(dirpath, `network_thread_${new Date().toISOString()}.cpuprofile`);
-    fs.writeFileSync(filePath, profile);
-    return filePath;
+    return profileThread(ProfileThread.NETWORK, durationMs, dirpath);
   },
   writeDiscv5Profile: async (durationMs: number, dirpath: string) => {
     return core.writeDiscv5Profile(durationMs, dirpath);

package/src/network/discv5/worker.ts

@@ -1,5 +1,3 @@
-import fs from "node:fs";
-import path from "node:path";
 import worker from "node:worker_threads";
 import {privateKeyFromProtobuf} from "@libp2p/crypto/keys";
 import {peerIdFromPrivateKey} from "@libp2p/peer-id";
@@ -14,7 +12,7 @@ import {Gauge} from "@lodestar/utils";
 import {RegistryMetricCreator} from "../../metrics/index.js";
 import {collectNodeJSMetrics} from "../../metrics/nodeJsMetrics.js";
 import {Clock} from "../../util/clock.js";
-import {profileNodeJS, writeHeapSnapshot} from "../../util/profile.js";
+import {ProfileThread, profileThread, writeHeapSnapshot} from "../../util/profile.js";
 import {Discv5WorkerApi, Discv5WorkerData} from "./types.js";
 import {ENRRelevance, enrRelevance} from "./utils.js";
 
@@ -108,10 +106,7 @@ const module: Discv5WorkerApi = {
     return (await metricsRegistry?.metrics()) ?? "";
   },
   writeProfile: async (durationMs: number, dirpath: string) => {
-    const profile = await profileNodeJS(durationMs);
-    const filePath = path.join(dirpath, `discv5_thread_${new Date().toISOString()}.cpuprofile`);
-    fs.writeFileSync(filePath, profile);
-    return filePath;
+    return profileThread(ProfileThread.DISC5, durationMs, dirpath);
   },
   writeHeapSnapshot: async (prefix: string, dirpath: string) => {
     return writeHeapSnapshot(prefix, dirpath);

package/src/network/gossip/encoding.ts

@@ -1,6 +1,6 @@
 import {Message} from "@libp2p/interface";
-import {compress, uncompress} from "snappyjs";
 import xxhashFactory from "xxhash-wasm";
+import {compress, uncompress} from "#snappy";
 import {digest} from "@chainsafe/as-sha256";
 import {RPC} from "@chainsafe/libp2p-gossipsub/message";
 import {DataTransform} from "@chainsafe/libp2p-gossipsub/types";

package/src/network/gossip/snappy_bun.ts

@@ -0,0 +1,2 @@
+import {snappy} from "@lodestar/bun";
+export const {compress, uncompress} = snappy;

package/src/network/metadata.ts

@@ -126,7 +126,7 @@ export class MetadataController {
    * 2. Network MUST call this method on fork transition.
    *    Current Clock implementation ensures no race conditions, epoch is correct if re-fetched
    */
-  updateEth2Field(epoch: Epoch): void {
+  updateEth2Field(epoch: Epoch): phase0.ENRForkID {
     const config = this.networkConfig.config;
     const enrForkId = getENRForkID(config, epoch);
     const {forkDigest, nextForkVersion, nextForkEpoch} = enrForkId;
@@ -143,6 +143,8 @@ export class MetadataController {
         : ssz.ForkDigest.defaultValue();
     this.onSetValue(ENRKey.nfd, nextForkDigest);
     this.logger.debug("Updated nfd field in ENR", {nextForkDigest: toHex(nextForkDigest)});
+
+    return enrForkId;
   }
 }
 

package/src/network/options.ts

@@ -23,7 +23,6 @@ export interface NetworkOptions
   useWorker?: boolean;
   maxYoungGenerationSizeMb?: number;
   disableLightClientServer?: boolean;
-  supernode?: boolean;
 
   /**
    * During E2E tests observe a lot of following `missing stream`:

package/src/network/processor/gossipHandlers.ts

@@ -39,6 +39,7 @@ import {
   BlockError,
   BlockErrorCode,
   BlockGossipError,
+  DataColumnSidecarErrorCode,
   DataColumnSidecarGossipError,
   GossipAction,
   GossipActionError,
@@ -304,7 +305,11 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
           ...blockInput.getLogMeta(),
           index: dataColumnSidecar.index,
         });
-        return blockInput;
+        throw new DataColumnSidecarGossipError(GossipAction.IGNORE, {
+          code: DataColumnSidecarErrorCode.ALREADY_KNOWN,
+          columnIdx: dataColumnSidecar.index,
+          slot,
+        });
       }
     }
 
@@ -556,6 +561,16 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
           metrics?.dataColumns.elapsedTimeTillReceived.observe({receivedOrder: receivedColumns}, delaySec);
           break;
       }
+
+      if (!blockInput.hasAllData()) {
+        // immediately attempt fetch of data columns from execution engine
+        chain.getBlobsTracker.triggerGetBlobs(blockInput);
+        // if we've received at least half of the columns, trigger reconstruction of the rest
+        if (blockInput.columnCount >= NUMBER_OF_COLUMNS / 2) {
+          chain.columnReconstructionTracker.triggerColumnReconstruction(blockInput);
+        }
+      }
+
       if (!blockInput.hasBlockAndAllData()) {
         const cutoffTimeMs = getCutoffTimeMs(chain, dataColumnSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
         chain.logger.debug("Received gossip data column, waiting for full data availability", {
@@ -578,12 +593,6 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
             source: BlockInputSource.gossip,
           });
         });
-        // immediately attempt fetch of data columns from execution engine
-        chain.getBlobsTracker.triggerGetBlobs(blockInput);
-        // if we've received at least half of the columns, trigger reconstruction of the rest
-        if (blockInput.columnCount >= NUMBER_OF_COLUMNS / 2) {
-          chain.columnReconstructionTracker.triggerColumnReconstruction(blockInput);
-        }
       }
     },
 

package/src/network/reqresp/handlers/beaconBlocksByRange.ts

@@ -19,7 +19,9 @@ export async function* onBeaconBlocksByRange(
 
   const finalized = db.blockArchive;
   const unfinalized = db.block;
-  const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
+  // in the case of initializing from a non-finalized state, we don't have the finalized block so this api does not work
+  // chain.forkChoice.getFinalizeBlock().slot
+  const finalizedSlot = chain.forkChoice.getFinalizedCheckpointSlot();
 
   // Finalized range of blocks
   if (startSlot <= finalizedSlot) {

package/src/node/nodejs.ts

@@ -53,6 +53,7 @@ export type BeaconNodeInitModules = {
   dataDir: string;
   peerStoreDir?: string;
   anchorState: BeaconStateAllForks;
+  isAnchorStateFinalized: boolean;
   wsCheckpoint?: phase0.Checkpoint;
   metricsRegistries?: Registry[];
 };
@@ -154,6 +155,7 @@ export class BeaconNode {
     dataDir,
     peerStoreDir,
     anchorState,
+    isAnchorStateFinalized,
     wsCheckpoint,
     metricsRegistries = [],
   }: BeaconNodeInitModules): Promise<T> {
@@ -217,6 +219,7 @@ export class BeaconNode {
       metrics,
       validatorMonitor,
       anchorState,
+      isAnchorStateFinalized,
       eth1: initializeEth1ForBlockProduction(opts.eth1, {
         config,
         db,

package/src/sync/range/range.ts

@@ -114,13 +114,14 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
    */
   addPeer(peerId: PeerIdStr, localStatus: Status, peerStatus: Status): void {
     // Compute if we should do a Finalized or Head sync with this peer
-    const {syncType, startEpoch, target} = getRangeSyncTarget(localStatus, peerStatus, this.chain.forkChoice);
+    const {syncType, startEpoch, target} = getRangeSyncTarget(localStatus, peerStatus, this.chain);
     this.logger.debug("Sync peer joined", {
       peer: peerId,
       syncType,
       startEpoch,
       targetSlot: target.slot,
       targetRoot: toRootHex(target.root),
+      localHeadSlot: localStatus.headSlot,
       earliestAvailableSlot: (peerStatus as fulu.Status).earliestAvailableSlot ?? Infinity,
     });
 

package/src/sync/utils/remoteSyncType.ts

@@ -1,6 +1,7 @@
 import {IForkChoice} from "@lodestar/fork-choice";
 import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Slot, Status} from "@lodestar/types";
+import {IBeaconChain} from "../../chain/interface.ts";
 import {ChainTarget} from "../range/utils/index.js";
 
 /** The type of peer relative to our current state */
@@ -103,8 +104,11 @@ export function getRangeSyncType(local: Status, remote: Status, forkChoice: IFor
 export function getRangeSyncTarget(
   local: Status,
   remote: Status,
-  forkChoice: IForkChoice
+  chain: IBeaconChain
 ): {syncType: RangeSyncType; startEpoch: Slot; target: ChainTarget} {
+  const forkChoice = chain.forkChoice;
+
+  // finalized sync
   if (remote.finalizedEpoch > local.finalizedEpoch && !forkChoice.hasBlock(remote.finalizedRoot)) {
     return {
       // If  RangeSyncType.Finalized, the range of blocks fetchable from startEpoch and target must allow to switch
@@ -131,11 +135,26 @@ export function getRangeSyncTarget(
       },
     };
   }
+
+  // we don't want to sync from epoch < minEpoch
+  // if we boot from an unfinalized checkpoint state, we don't want to sync before anchorStateLatestBlockSlot
+  // if we boot from a finalized checkpoint state, anchorStateLatestBlockSlot is trusted and we also don't want to sync before it
+  const minEpoch = Math.max(remote.finalizedEpoch, computeEpochAtSlot(chain.anchorStateLatestBlockSlot));
+
+  // head sync
   return {
     syncType: RangeSyncType.Head,
-    // The new peer has the same finalized (earlier filters should prevent a peer with an
-    // earlier finalized chain from reaching here) and local head will always be >= local finalized.
-    startEpoch: computeEpochAtSlot(local.headSlot),
+    // The new peer has the same finalized `remote.finalizedEpoch == local.finalizedEpoch` since
+    // previous filters should prevent a peer with an earlier finalized chain from reaching here.
+    //
+    // By default and during stable network conditions, the head sync always starts from
+    // the finalized epoch (even though it's the head sync) because finalized epoch is < local head.
+    // This is to prevent the issue noted here https://github.com/ChainSafe/lodestar/pull/7509#discussion_r1984353063.
+    //
+    // During non-finality of the network, when starting from an unfinalized checkpoint state, we don't want
+    // to sync before anchorStateLatestBlockSlot as finalized epoch is too far away. Local head will also be
+    // the same to that value at startup, the head sync always starts from anchorStateLatestBlockSlot in this case.
+    startEpoch: Math.min(computeEpochAtSlot(local.headSlot), minEpoch),
     target: {
       slot: remote.headSlot,
       root: remote.headRoot,

package/src/util/blobs.ts

@@ -149,41 +149,85 @@ export async function dataColumnMatrixRecovery(
  * Reconstruct blobs from a set of data columns, at least 50%+ of all the columns
  * must be provided to allow to reconstruct the full data matrix
  */
-export async function reconstructBlobs(sidecars: fulu.DataColumnSidecars): Promise<deneb.Blobs> {
+export async function reconstructBlobs(sidecars: fulu.DataColumnSidecars, indices?: number[]): Promise<deneb.Blobs> {
   if (sidecars.length < NUMBER_OF_COLUMNS / 2) {
     throw Error(
       `Expected at least ${NUMBER_OF_COLUMNS / 2} data columns to reconstruct blobs, received ${sidecars.length}`
     );
   }
+  const blobCount = sidecars[0].column.length;
 
-  let fullSidecars: fulu.DataColumnSidecars;
-
-  if (sidecars.length === NUMBER_OF_COLUMNS) {
-    // Full columns, no need to recover
-    fullSidecars = sidecars;
-  } else {
-    const sidecarsByIndex = new Map<number, fulu.DataColumnSidecar>(sidecars.map((sc) => [sc.index, sc]));
-    const recoveredSidecars = await dataColumnMatrixRecovery(sidecarsByIndex);
-    if (recoveredSidecars === null) {
-      // Should not happen because we check the column count above
-      throw Error("Failed to reconstruct the full data matrix");
+  for (const index of indices ?? []) {
+    if (index < 0 || index >= blobCount) {
+      throw Error(`Invalid blob index ${index}, must be between 0 and ${blobCount - 1}`);
     }
-    fullSidecars = recoveredSidecars;
+  }
+  const indicesToReconstruct = indices ?? Array.from({length: blobCount}, (_, i) => i);
+
+  const recoveredCells = await recoverBlobCells(sidecars, indicesToReconstruct);
+  if (recoveredCells === null) {
+    // Should not happen because we check the column count above
+    throw Error("Failed to recover cells to reconstruct blobs");
   }
 
-  const blobCount = fullSidecars[0].column.length;
-  const blobs: deneb.Blobs = new Array(blobCount);
+  const blobs: deneb.Blobs = new Array(indicesToReconstruct.length);
 
-  const ordered = fullSidecars.slice().sort((a, b) => a.index - b.index);
-  for (let row = 0; row < blobCount; row++) {
-    // 128 cells that make up one "extended blob" row
-    const cells = ordered.map((col) => col.column[row]);
-    blobs[row] = cellsToBlob(cells);
+  for (let i = 0; i < indicesToReconstruct.length; i++) {
+    const blobIndex = indicesToReconstruct[i];
+    const cells = recoveredCells.get(blobIndex);
+    if (!cells) {
+      throw Error(`Failed to get recovered cells for blob index ${blobIndex}`);
+    }
+    blobs[i] = cellsToBlob(cells);
   }
 
   return blobs;
 }
 
+/**
+ * Recover cells for specific blob indices from a set of data columns
+ */
+async function recoverBlobCells(
+  partialSidecars: fulu.DataColumnSidecar[],
+  blobIndices: number[]
+): Promise<Map<number, fulu.Cell[]> | null> {
+  const columnCount = partialSidecars.length;
+  if (columnCount < NUMBER_OF_COLUMNS / 2) {
+    // We don't have enough columns to recover
+    return null;
+  }
+
+  const recoveredCells = new Map<number, fulu.Cell[]>();
+  // Sort data columns by index in ascending order
+  const partialSidecarsSorted = partialSidecars.slice().sort((a, b) => a.index - b.index);
+
+  if (columnCount === NUMBER_OF_COLUMNS) {
+    // Full columns, no need to recover
+    for (const blobIndex of blobIndices) {
+      // 128 cells that make up one "extended blob" row
+      const cells = partialSidecarsSorted.map((col) => col.column[blobIndex]);
+      recoveredCells.set(blobIndex, cells);
+    }
+    return recoveredCells;
+  }
+
+  await Promise.all(
+    blobIndices.map(async (blobIndex) => {
+      const cellIndices: number[] = [];
+      const cells: fulu.Cell[] = [];
+      for (const dataColumn of partialSidecarsSorted) {
+        cellIndices.push(dataColumn.index);
+        cells.push(dataColumn.column[blobIndex]);
+      }
+      // Recover cells for this specific blob row
+      const recovered = await kzg.asyncRecoverCellsAndKzgProofs(cellIndices, cells);
+      recoveredCells.set(blobIndex, recovered.cells);
+    })
+  );
+
+  return recoveredCells;
+}
+
 /**
  * Concatenate the systematic half (columns 0‑63) of a row of cells into
  * the original 131072 byte blob. The parity half (64‑127) is ignored as

package/src/util/profile.ts

@@ -1,13 +1,32 @@
+import fs from "node:fs";
+import path from "node:path";
 import {sleep} from "@lodestar/utils";
 
+export enum ProfileThread {
+  MAIN = "main",
+  NETWORK = "network",
+  DISC5 = "discv5",
+}
+
+/**
+ * The time to take a Bun profile.
+ * If we increase this time it'll potentiall cause the app to crash.
+ * If we decrease this time, profile recorded will be fragmented and hard to analyze.
+ */
+const BUN_PROFILE_MS = 3 * 1000;
+
+export async function profileThread(thread: ProfileThread, durationMs: number, dirpath: string): Promise<string> {
+  return globalThis.Bun ? profileBun(thread, durationMs) : profileNodeJS(thread, durationMs, dirpath);
+}
+
 /**
- * Take 10m profile of the current thread without promise tracking.
+ * Take `durationMs` profile of the current thread and return the persisted file path.
  */
-export async function profileNodeJS(durationMs: number): Promise<string> {
+async function profileNodeJS(thread: ProfileThread, durationMs: number, dirpath: string): Promise<string> {
   const inspector = await import("node:inspector");
 
   // due to some typing issues, not able to use promisify here
-  return new Promise<string>((resolve, reject) => {
+  const profile = await new Promise<string>((resolve, reject) => {
     // Start the inspector and connect to it
     const session = new inspector.Session();
     session.connect();
@@ -29,6 +48,29 @@ export async function profileNodeJS(durationMs: number): Promise<string> {
       });
     });
   });
+
+  const filePath = path.join(dirpath, `${thread}_thread_${new Date().toISOString()}.cpuprofile`);
+  fs.writeFileSync(filePath, profile);
+  return filePath;
+}
+
+/**
+ * Unlike NodeJS, Bun console.profile() api flush data to the inspector,
+ * so this api returns ms taken of this profile instead of file path.
+ */
+async function profileBun(thread: ProfileThread, durationMs: number): Promise<string> {
+  const start = Date.now();
+  let now = Date.now();
+  while (now - start < durationMs) {
+    // biome-ignore lint/suspicious/noConsole: need to use console api to profile in Bun
+    console.profile(String(now));
+    await sleep(BUN_PROFILE_MS);
+    // biome-ignore lint/suspicious/noConsole: need to use console api to profile in Bun
+    console.profileEnd(String(now));
+    now = Date.now();
+  }
+
+  return `Successfully take Bun ${thread} thread profile in ${now - start}ms. Check your inspector to see the profile.`;
 }
 
 /**

package/src/util/sszBytes.ts

@@ -417,6 +417,36 @@ export function getSlotFromDataColumnSidecarSerialized(data: Uint8Array): Slot |
   return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR);
 }
 
+/**
+ * BeaconState of all forks (up until Electra, check with new forks)
+ * class BeaconState(Container):
+ *   genesis_time: uint64                    - 8 bytes
+ *   genesis_validators_root: Root           - 32 bytes
+ *   slot: Slot                              - 8 bytes
+ *   fork: Fork                              - 16 bytes
+ *   latest_block_header: BeaconBlockHeader  - fixed size
+ *     slot: Slot                            - 8 bytes
+ *
+ */
+
+const BLOCK_HEADER_SLOT_BYTES_POSITION_IN_BEACON_STATE = 8 + 32 + 8 + 16;
+export function getLastProcessedSlotFromBeaconStateSerialized(data: Uint8Array): Slot | null {
+  if (data.length < BLOCK_HEADER_SLOT_BYTES_POSITION_IN_BEACON_STATE + SLOT_SIZE) {
+    return null;
+  }
+
+  return getSlotFromOffset(data, BLOCK_HEADER_SLOT_BYTES_POSITION_IN_BEACON_STATE);
+}
+
+const SLOT_BYTES_POSITION_IN_BEACON_STATE = 8 + 32;
+export function getSlotFromBeaconStateSerialized(data: Uint8Array): Slot | null {
+  if (data.length < SLOT_BYTES_POSITION_IN_BEACON_STATE) {
+    return null;
+  }
+
+  return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_BEACON_STATE);
+}
+
 /**
  * Read only the first 4 bytes of Slot, max value is 4,294,967,295 will be reached 1634 years after genesis
  *