@lodestar/beacon-node 1.41.0 → 1.42.0-dev.04e52d162f

package/src/chain/stateCache/persistentCheckpointsCache.ts

@@ -1,11 +1,7 @@
 import {routes} from "@lodestar/api";
 import {BeaconConfig} from "@lodestar/config";
-import {
-  CachedBeaconStateAllForks,
-  computeStartSlotAtEpoch,
-  getBlockRootAtSlot,
-  loadCachedBeaconState,
-} from "@lodestar/state-transition";
+import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice";
+import {IBeaconStateView, computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Epoch, RootHex, phase0} from "@lodestar/types";
 import {Logger, MapDef, fromHex, sleep, toHex, toRootHex} from "@lodestar/utils";
 import {Metrics} from "../../metrics/index.js";
@@ -14,7 +10,7 @@ import {IClock} from "../../util/clock.js";
 import {serializeState} from "../serializeState.js";
 import {CPStateDatastore, DatastoreKey} from "./datastore/index.js";
 import {MapTracker} from "./mapMetrics.js";
-import {BlockStateCache, CacheItemType, CheckpointHex, CheckpointStateCache} from "./types.js";
+import {BlockStateCache, CacheItemType, CheckpointHexPayload, CheckpointStateCache} from "./types.js";
 
 export type PersistentCheckpointStateCacheOpts = {
   /** Keep max n state epochs in memory, persist the rest to disk */
@@ -39,7 +35,7 @@ type CacheKey = string;
 
 type InMemoryCacheItem = {
   type: CacheItemType.inMemory;
-  state: CachedBeaconStateAllForks;
+  state: IBeaconStateView;
   // if a cp state is reloaded from disk, it'll keep track of persistedKey to allow us to remove it from disk later
   // it also helps not to persist it again
   persistedKey?: DatastoreKey;
@@ -54,6 +50,22 @@ type CacheItem = InMemoryCacheItem | PersistedCacheItem;
 
 type LoadedStateBytesData = {persistedKey: DatastoreKey; stateBytes: Uint8Array};
 
+/** Bitmask for tracking which payload variants exist per root in the epochIndex */
+enum PayloadAvailability {
+  NOT_PRESENT = 1,
+  PRESENT = 2,
+}
+
+const PAYLOAD_AVAILABILITY_ALL = [PayloadAvailability.NOT_PRESENT, PayloadAvailability.PRESENT] as const;
+
+function toPayloadAvailability(payloadPresent: boolean): PayloadAvailability {
+  return payloadPresent ? PayloadAvailability.PRESENT : PayloadAvailability.NOT_PRESENT;
+}
+
+function fromPayloadAvailability(flag: PayloadAvailability): boolean {
+  return flag === PayloadAvailability.PRESENT;
+}
+
 /**
  * Before n-historical states, lodestar keeps all checkpoint states since finalized
  * Since Sep 2024, lodestar stores 3 most recent checkpoint states in memory and the rest on disk. The finalized state
@@ -106,8 +118,8 @@ const PROCESS_CHECKPOINT_STATES_BPS = 6667;
  */
 export class PersistentCheckpointStateCache implements CheckpointStateCache {
   private readonly cache: MapTracker<CacheKey, CacheItem>;
-  /** Epoch -> Set<blockRoot> */
-  private readonly epochIndex = new MapDef<Epoch, Set<RootHex>>(() => new Set<string>());
+  /** Epoch -> Map<blockRoot, PayloadAvailability bitmask> */
+  private readonly epochIndex = new MapDef<Epoch, Map<RootHex, number>>(() => new Map());
   private readonly config: BeaconConfig;
   private readonly metrics: Metrics | null | undefined;
   private readonly logger: Logger;
@@ -203,19 +215,24 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    * - Get block for processing
    * - Regen head state
    */
-  async getOrReload(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | null> {
+  async getOrReload(cp: CheckpointHexPayload): Promise<IBeaconStateView | null> {
     const stateOrStateBytesData = await this.getStateOrLoadDb(cp);
-    if (stateOrStateBytesData === null || isCachedBeaconState(stateOrStateBytesData)) {
+    if (stateOrStateBytesData === null || isBeaconStateView(stateOrStateBytesData)) {
       return stateOrStateBytesData ?? null;
     }
     const {persistedKey, stateBytes} = stateOrStateBytesData;
-    const logMeta = {persistedKey: toHex(persistedKey)};
+    const logMeta = {
+      epoch: cp.epoch,
+      rootHex: cp.rootHex,
+      payloadPresent: cp.payloadPresent,
+      persistedKey: toHex(persistedKey),
+    };
     this.logger.debug("Reload: read state successful", logMeta);
     this.metrics?.cpStateCache.stateReloadSecFromSlot.observe(
       this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0
     );
     const seedState = this.findSeedStateToReload(cp);
-    this.metrics?.cpStateCache.stateReloadEpochDiff.observe(Math.abs(seedState.epochCtx.epoch - cp.epoch));
+    this.metrics?.cpStateCache.stateReloadEpochDiff.observe(Math.abs(seedState.epoch - cp.epoch));
     this.logger.debug("Reload: found seed state", {...logMeta, seedSlot: seedState.slot});
 
     try {
@@ -227,19 +244,16 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
       if (validatorsBytes == null) {
         // fallback logic in case we can't use the buffer pool
         this.metrics?.cpStateCache.stateReloadValidatorsSerializeAllocCount.inc();
-        validatorsBytes = seedState.validators.serialize();
+        validatorsBytes = seedState.serializeValidators();
       }
       sszTimer?.();
       const timer = this.metrics?.cpStateCache.stateReloadDuration.startTimer();
-      const newCachedState = loadCachedBeaconState(seedState, stateBytes, {}, validatorsBytes);
+      const newCachedState = seedState.loadOtherState(stateBytes, validatorsBytes);
       // hashTreeRoot() calls the commit() inside
       // there is no modification inside the state, it's just that we want to compute and cache all roots
       const stateRoot = toRootHex(newCachedState.hashTreeRoot());
       timer?.();
 
-      // load all cache in order for consumers (usually regen.getState()) to process blocks faster
-      newCachedState.validators.getAllReadonlyValues();
-      newCachedState.balances.getAll();
       this.logger.debug("Reload: cached state load successful", {
         ...logMeta,
         stateSlot: newCachedState.slot,
@@ -250,7 +264,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
       // only remove persisted state once we reload successfully
       const cpKey = toCacheKey(cp);
       this.cache.set(cpKey, {type: CacheItemType.inMemory, state: newCachedState, persistedKey});
-      this.epochIndex.getOrDefault(cp.epoch).add(cp.rootHex);
+      this.addToEpochIndex(cp.epoch, cp.rootHex, cp.payloadPresent);
       // don't prune from memory here, call it at the last 1/3 of slot 0 of an epoch
       return newCachedState;
     } catch (e) {
@@ -262,9 +276,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   /**
    * Return either state or state bytes loaded from db.
    */
-  async getStateOrBytes(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | Uint8Array | null> {
+  async getStateOrBytes(cp: CheckpointHexPayload): Promise<IBeaconStateView | Uint8Array | null> {
     const stateOrLoadedState = await this.getStateOrLoadDb(cp);
-    if (stateOrLoadedState === null || isCachedBeaconState(stateOrLoadedState)) {
+    if (stateOrLoadedState === null || isBeaconStateView(stateOrLoadedState)) {
       return stateOrLoadedState;
     }
     return stateOrLoadedState.stateBytes;
@@ -273,7 +287,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   /**
    * Return either state or state bytes with persisted key loaded from db.
    */
-  async getStateOrLoadDb(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | LoadedStateBytesData | null> {
+  async getStateOrLoadDb(cp: CheckpointHexPayload): Promise<IBeaconStateView | LoadedStateBytesData | null> {
     const cpKey = toCacheKey(cp);
     const inMemoryState = this.get(cpKey);
     if (inMemoryState) {
@@ -304,7 +318,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   /**
    * Similar to get() api without reloading from disk
    */
-  get(cpOrKey: CheckpointHex | string): CachedBeaconStateAllForks | null {
+  get(cpOrKey: CheckpointHexPayload | CacheKey): IBeaconStateView | null {
     this.metrics?.cpStateCache.lookups.inc();
     const cpKey = typeof cpOrKey === "string" ? cpOrKey : toCacheKey(cpOrKey);
     const cacheItem = this.cache.get(cpKey);
@@ -330,9 +344,11 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
 
   /**
    * Add a state of a checkpoint to this cache, prune from memory if necessary.
+   * @param payloadPresent - For Gloas: true if this is payload state, false if block state.
+   *                         Always true for pre-Gloas.
    */
-  add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks): void {
-    const cpHex = toCheckpointHex(cp);
+  add(cp: phase0.Checkpoint, state: IBeaconStateView, payloadPresent: boolean): void {
+    const cpHex = toCheckpointHexPayload(cp, payloadPresent);
     const key = toCacheKey(cpHex);
     const cacheItem = this.cache.get(key);
     this.metrics?.cpStateCache.adds.inc();
@@ -343,27 +359,32 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
       this.logger.verbose("Added checkpoint state to memory but a persisted key existed", {
         epoch: cp.epoch,
         rootHex: cpHex.rootHex,
+        payloadPresent,
         persistedKey: toHex(persistedKey),
       });
     } else {
       this.cache.set(key, {type: CacheItemType.inMemory, state});
-      this.logger.verbose("Added checkpoint state to memory", {epoch: cp.epoch, rootHex: cpHex.rootHex});
+      this.logger.verbose("Added checkpoint state to memory", {
+        epoch: cp.epoch,
+        rootHex: cpHex.rootHex,
+        payloadPresent,
+      });
     }
-    this.epochIndex.getOrDefault(cp.epoch).add(cpHex.rootHex);
+    this.addToEpochIndex(cp.epoch, cpHex.rootHex, cpHex.payloadPresent);
     this.prunePersistedStates();
   }
 
   /**
    * Searches in-memory state for the latest cached state with a `root` without reload, starting with `epoch` and descending
    */
-  getLatest(rootHex: RootHex, maxEpoch: Epoch): CachedBeaconStateAllForks | null {
+  getLatest(rootHex: RootHex, maxEpoch: Epoch, payloadPresent: boolean): IBeaconStateView | null {
     // sort epochs in descending order, only consider epochs lte `epoch`
     const epochs = Array.from(this.epochIndex.keys())
       .sort((a, b) => b - a)
       .filter((e) => e <= maxEpoch);
     for (const epoch of epochs) {
-      if (this.epochIndex.get(epoch)?.has(rootHex)) {
-        const inMemoryClonedState = this.get({rootHex, epoch});
+      if (this.hasPayloadVariant(epoch, rootHex, payloadPresent)) {
+        const inMemoryClonedState = this.get({rootHex, epoch, payloadPresent});
         if (inMemoryClonedState) {
           return inMemoryClonedState;
         }
@@ -379,20 +400,24 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    * - Get block for processing
    * - Regen head state
    */
-  async getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise<CachedBeaconStateAllForks | null> {
+  async getOrReloadLatest(
+    rootHex: RootHex,
+    maxEpoch: Epoch,
+    payloadPresent: boolean
+  ): Promise<IBeaconStateView | null> {
     // sort epochs in descending order, only consider epochs lte `epoch`
     const epochs = Array.from(this.epochIndex.keys())
       .sort((a, b) => b - a)
       .filter((e) => e <= maxEpoch);
     for (const epoch of epochs) {
-      if (this.epochIndex.get(epoch)?.has(rootHex)) {
+      if (this.hasPayloadVariant(epoch, rootHex, payloadPresent)) {
         try {
-          const state = await this.getOrReload({rootHex, epoch});
+          const state = await this.getOrReload({rootHex, epoch, payloadPresent});
           if (state) {
             return state;
           }
         } catch (e) {
-          this.logger.debug("Error get or reload state", {epoch, rootHex}, e as Error);
+          this.logger.debug("Error get or reload state", {epoch, rootHex, payloadPresent}, e as Error);
         }
       }
     }
@@ -400,12 +425,14 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   }
 
   /**
-   * Update the precomputed checkpoint and return the number of his for the
+   * Update the precomputed checkpoint and return the number of hits for the
    * previous one (if any).
+   * @param payloadPresent - For Gloas: true if head block has FULL payload, false if EMPTY.
+   *                         Always true for pre-Gloas.
    */
-  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null {
+  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null {
     const previousHits = this.preComputedCheckpointHits;
-    this.preComputedCheckpoint = toCacheKey({rootHex, epoch});
+    this.preComputedCheckpoint = toCacheKey({rootHex, epoch, payloadPresent});
     this.preComputedCheckpointHits = 0;
     return previousHits;
   }
@@ -479,8 +506,11 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    * - 2 then we'll persist {root: b2, epoch n-2} checkpoint state to disk, there are also 2 checkpoint states in memory at epoch n, same to the above (maxEpochsInMemory=1)
    *
    * As of Mar 2024, it takes <=350ms to persist a holesky state on fast server
+   *
+   * For Gloas: Processes both block state and payload state variants together. The decision of which roots to persist/prune
+   * is based on root canonicality (from state's view), not payload presence. Both variants are managed as a unit.
    */
-  async processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise<number> {
+  async processState(blockRootHex: RootHex, state: IBeaconStateView): Promise<number> {
     let persistCount = 0;
     // it's important to sort the epochs in ascending order, in case of big reorg we always want to keep the most recent checkpoint states
     const sortedEpochs = Array.from(this.epochIndex.keys()).sort((a, b) => a - b);
@@ -549,10 +579,10 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    *
    * Use seed state from the block cache if cannot find any seed states within this cache.
    */
-  findSeedStateToReload(reloadedCp: CheckpointHex): CachedBeaconStateAllForks {
+  findSeedStateToReload(reloadedCp: CheckpointHexPayload): IBeaconStateView {
     const maxEpoch = Math.max(...Array.from(this.epochIndex.keys()));
     const reloadedCpSlot = computeStartSlotAtEpoch(reloadedCp.epoch);
-    let firstState: CachedBeaconStateAllForks | null = null;
+    let firstState: IBeaconStateView | null = null;
     const logCtx = {reloadedCpEpoch: reloadedCp.epoch, reloadedCpRoot: reloadedCp.rootHex};
 
     // no need to check epochs before `maxEpoch - this.maxEpochsInMemory + 1` before they are all persisted
@@ -562,32 +592,35 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
         return firstState;
       }
 
-      for (const rootHex of this.epochIndex.get(epoch) || []) {
-        const cpKey = toCacheKey({rootHex, epoch});
-        const cacheItem = this.cache.get(cpKey);
-        if (cacheItem === undefined) {
-          // should not happen
-          continue;
-        }
-        if (isInMemoryCacheItem(cacheItem)) {
-          const {state} = cacheItem;
-          if (firstState === null) {
-            firstState = state;
+      for (const [rootHex, bitmask] of this.epochIndex.get(epoch) || []) {
+        for (const flag of PAYLOAD_AVAILABILITY_ALL) {
+          if (!(bitmask & flag)) continue;
+          const payloadPresent = fromPayloadAvailability(flag);
+          const cpKey = toCacheKey({rootHex, epoch, payloadPresent});
+          const cacheItem = this.cache.get(cpKey);
+          if (cacheItem === undefined) {
+            continue;
           }
-          const cpLog = {cpEpoch: epoch, cpRoot: rootHex};
-
-          try {
-            // amongst states of the same epoch, choose the one with the same view of reloadedCp
-            if (
-              reloadedCpSlot < state.slot &&
-              toRootHex(getBlockRootAtSlot(state, reloadedCpSlot)) === reloadedCp.rootHex
-            ) {
-              this.logger.verbose("Reload: use checkpoint state as seed state", {...cpLog, ...logCtx});
-              return state;
+          if (isInMemoryCacheItem(cacheItem)) {
+            const {state} = cacheItem;
+            if (firstState === null) {
+              firstState = state;
+            }
+            const cpLog = {cpEpoch: epoch, cpRoot: rootHex, payloadPresent};
+
+            try {
+              // amongst states of the same epoch, choose the one with the same view of reloadedCp
+              if (
+                reloadedCpSlot < state.slot &&
+                toRootHex(state.getBlockRootAtSlot(reloadedCpSlot)) === reloadedCp.rootHex
+              ) {
+                this.logger.verbose("Reload: use checkpoint state as seed state", {...cpLog, ...logCtx});
+                return state;
+              }
+            } catch (e) {
+              // getBlockRootAtSlot may throw error
+              this.logger.debug("Error finding checkpoint state to reload", {...cpLog, ...logCtx}, e as Error);
             }
-          } catch (e) {
-            // getBlockRootAtSlot may throw error
-            this.logger.debug("Error finding checkpoint state to reload", {...cpLog, ...logCtx}, e as Error);
           }
         }
       }
@@ -604,6 +637,31 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
     this.epochIndex.clear();
   }
 
+  private addToEpochIndex(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): void {
+    const rootMap = this.epochIndex.getOrDefault(epoch);
+    rootMap.set(rootHex, (rootMap.get(rootHex) ?? 0) | toPayloadAvailability(payloadPresent));
+  }
+
+  private removeFromEpochIndex(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): void {
+    const rootMap = this.epochIndex.get(epoch);
+    if (rootMap === undefined) return;
+    const existing = rootMap.get(rootHex);
+    if (existing === undefined) return;
+    const updated = existing & ~toPayloadAvailability(payloadPresent);
+    if (updated === 0) {
+      rootMap.delete(rootHex);
+      if (rootMap.size === 0) {
+        this.epochIndex.delete(epoch);
+      }
+    } else {
+      rootMap.set(rootHex, updated);
+    }
+  }
+
+  private hasPayloadVariant(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): boolean {
+    return Boolean((this.epochIndex.get(epoch)?.get(rootHex) ?? 0) & toPayloadAvailability(payloadPresent));
+  }
+
   /** ONLY FOR DEBUGGING PURPOSES. For lodestar debug API */
   dumpSummary(): routes.lodestar.StateCacheItem[] {
     return Array.from(this.cache.keys()).map((key) => {
@@ -619,7 +677,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
     });
   }
 
-  getStates(): IterableIterator<CachedBeaconStateAllForks> {
+  getStates(): IterableIterator<IBeaconStateView> {
     const items = Array.from(this.cache.values())
       .filter(isInMemoryCacheItem)
       .map((item) => item.state);
@@ -669,20 +727,16 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    *   - In normal condition, we persist 1 checkpoint state per epoch.
    *   - In reorged condition, we may persist multiple (most likely 2) checkpoint states per epoch.
    */
-  private async processPastEpoch(
-    blockRootHex: RootHex,
-    state: CachedBeaconStateAllForks,
-    epoch: Epoch
-  ): Promise<number> {
+  private async processPastEpoch(blockRootHex: RootHex, state: IBeaconStateView, epoch: Epoch): Promise<number> {
     let persistCount = 0;
     const epochBoundarySlot = computeStartSlotAtEpoch(epoch);
     const epochBoundaryRoot =
-      epochBoundarySlot === state.slot ? fromHex(blockRootHex) : getBlockRootAtSlot(state, epochBoundarySlot);
+      epochBoundarySlot === state.slot ? fromHex(blockRootHex) : state.getBlockRootAtSlot(epochBoundarySlot);
     const epochBoundaryHex = toRootHex(epochBoundaryRoot);
-    const prevEpochRoot = toRootHex(getBlockRootAtSlot(state, epochBoundarySlot - 1));
+    const prevEpochRoot = toRootHex(state.getBlockRootAtSlot(epochBoundarySlot - 1));
 
     // for each epoch, usually there are 2 rootHexes respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State
-    const cpRootHexes = this.epochIndex.get(epoch) ?? [];
+    const cpRootHexMap = this.epochIndex.get(epoch) ?? new Map<RootHex, number>();
     const persistedRootHexes = new Set<RootHex>();
 
     // 1) if there is no CRCS, persist PRCS (block 0 of epoch is skipped). In this case prevEpochRoot === epochBoundaryHex
@@ -691,76 +745,81 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
     persistedRootHexes.add(epochBoundaryHex);
 
     // 3) persist any states with unknown roots to this state
-    for (const rootHex of cpRootHexes) {
+    for (const rootHex of cpRootHexMap.keys()) {
       if (rootHex !== epochBoundaryHex && rootHex !== prevEpochRoot) {
         persistedRootHexes.add(rootHex);
       }
     }
 
-    for (const rootHex of cpRootHexes) {
-      const cpKey = toCacheKey({epoch: epoch, rootHex});
-      const cacheItem = this.cache.get(cpKey);
-
-      if (cacheItem !== undefined && isInMemoryCacheItem(cacheItem)) {
-        let {persistedKey} = cacheItem;
-        const {state} = cacheItem;
-        const logMeta = {
-          stateSlot: state.slot,
-          rootHex,
-          epochBoundaryHex,
-          persistedKey: persistedKey ? toHex(persistedKey) : "",
-        };
-
-        if (persistedRootHexes.has(rootHex)) {
-          if (persistedKey) {
-            // we don't care if the checkpoint state is already persisted
-            this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta);
-          } else {
-            // persist and do not update epochIndex
-            this.metrics?.cpStateCache.statePersistSecFromSlot.observe(
-              this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0
-            );
-            const cpPersist = {epoch: epoch, root: fromHex(rootHex)};
-            // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory.
-            // As monitored on holesky as of Jan 2024:
-            //   - This does not increase heap allocation while gc time is the same
-            //   - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s)
-            //   - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s)
-            //   - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization
-            const timer = this.metrics?.stateSerializeDuration.startTimer({
-              source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE,
-            });
-            persistedKey = await serializeState(
-              state,
-              AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE,
-              (stateBytes) => {
-                timer?.();
-                return this.datastore.write(cpPersist, stateBytes);
-              },
-              this.bufferPool
-            );
+    for (const [rootHex, bitmask] of cpRootHexMap) {
+      for (const flag of PAYLOAD_AVAILABILITY_ALL) {
+        if (!(bitmask & flag)) continue;
+        const payloadPresent = fromPayloadAvailability(flag);
+        const cpKey = toCacheKey({epoch: epoch, rootHex, payloadPresent});
+        const cacheItem = this.cache.get(cpKey);
 
-            persistCount++;
-            this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", {
-              ...logMeta,
-              persistedKey: toHex(persistedKey),
-            });
-          }
-          // overwrite cpKey, this means the state is deleted from memory
-          this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey});
-        } else {
-          if (persistedKey) {
-            // persisted file will be eventually deleted by the archive task
-            // this also means the state is deleted from memory
+        if (cacheItem !== undefined && isInMemoryCacheItem(cacheItem)) {
+          let {persistedKey} = cacheItem;
+          const {state} = cacheItem;
+          const logMeta = {
+            stateSlot: state.slot,
+            rootHex,
+            payloadPresent,
+            epochBoundaryHex,
+            persistedKey: persistedKey ? toHex(persistedKey) : "",
+          };
+
+          if (persistedRootHexes.has(rootHex)) {
+            if (persistedKey) {
+              // we don't care if the checkpoint state is already persisted
+              this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta);
+            } else {
+              // persist and do not update epochIndex
+              this.metrics?.cpStateCache.statePersistSecFromSlot.observe(
+                this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0
+              );
+              const cpPersist = {epoch: epoch, root: fromHex(rootHex)};
+              // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory.
+              // As monitored on holesky as of Jan 2024:
+              //   - This does not increase heap allocation while gc time is the same
+              //   - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s)
+              //   - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s)
+              //   - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization
+              const timer = this.metrics?.stateSerializeDuration.startTimer({
+                source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE,
+              });
+              persistedKey = await serializeState(
+                state,
+                AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE,
+                (stateBytes) => {
+                  timer?.();
+                  return this.datastore.write(cpPersist, stateBytes, payloadPresent);
+                },
+                this.bufferPool
+              );
+
+              persistCount++;
+              this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", {
+                ...logMeta,
+                persistedKey: toHex(persistedKey),
+              });
+            }
+            // overwrite cpKey, this means the state is deleted from memory
             this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey});
-            // do not update epochIndex
           } else {
-            // delete the state from memory
-            this.cache.delete(cpKey);
-            this.epochIndex.get(epoch)?.delete(rootHex);
+            if (persistedKey) {
+              // persisted file will be eventually deleted by the archive task
+              // this also means the state is deleted from memory
+              this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey});
+              // do not update epochIndex
+            } else {
+              // delete the state from memory
+              this.cache.delete(cpKey);
+              this.removeFromEpochIndex(epoch, rootHex, payloadPresent);
+            }
+            this.metrics?.cpStateCache.statePruneFromMemoryCount.inc();
+            this.logger.verbose("Pruned checkpoint state from memory", logMeta);
           }
-          this.metrics?.cpStateCache.statePruneFromMemoryCount.inc();
-          this.logger.verbose("Pruned checkpoint state from memory", logMeta);
         }
       }
     }
@@ -773,26 +832,40 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    */
   private async deleteAllEpochItems(epoch: Epoch): Promise<void> {
     let persistCount = 0;
-    const rootHexes = this.epochIndex.get(epoch) || [];
-    for (const rootHex of rootHexes) {
-      const key = toCacheKey({rootHex, epoch});
-      const cacheItem = this.cache.get(key);
-
-      if (cacheItem) {
-        const persistedKey = isPersistedCacheItem(cacheItem) ? cacheItem.value : cacheItem.persistedKey;
-        if (persistedKey) {
-          await this.datastore.remove(persistedKey);
-          persistCount++;
-          this.metrics?.cpStateCache.persistedStateRemoveCount.inc();
+    const rootHexMap = this.epochIndex.get(epoch) || new Map<RootHex, number>();
+    for (const [rootHex, bitmask] of rootHexMap) {
+      for (const flag of PAYLOAD_AVAILABILITY_ALL) {
+        if (!(bitmask & flag)) continue;
+        const payloadPresent = fromPayloadAvailability(flag);
+        const key = toCacheKey({rootHex, epoch, payloadPresent});
+        const cacheItem = this.cache.get(key);
+
+        if (cacheItem) {
+          const persistedKey = isPersistedCacheItem(cacheItem) ? cacheItem.value : cacheItem.persistedKey;
+          if (persistedKey) {
+            await this.datastore.remove(persistedKey);
+            persistCount++;
+            this.metrics?.cpStateCache.persistedStateRemoveCount.inc();
+          }
         }
+        this.cache.delete(key);
+        this.logger.verbose("Pruned checkpoint state", {
+          epoch,
+          rootHex,
+          payloadPresent,
+          type: cacheItem ? (isPersistedCacheItem(cacheItem) ? "persisted" : "in-memory") : "missing",
+        });
       }
-      this.cache.delete(key);
     }
     this.epochIndex.delete(epoch);
-    this.logger.verbose("Pruned checkpoint states for epoch", {
+    this.logger.verbose("Pruned all checkpoint states for epoch", {
       epoch,
       persistCount,
-      rootHexes: Array.from(rootHexes).join(","),
+      items: Array.from(rootHexMap.entries())
+        .flatMap(([rootHex, bitmask]) =>
+          PAYLOAD_AVAILABILITY_ALL.filter((f) => bitmask & f).map((f) => `${rootHex}:${fromPayloadAvailability(f)}`)
+        )
+        .join(","),
     });
   }
 
@@ -827,15 +900,14 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    *   - Also `serializeState.test.ts` perf test shows a lot of differences allocating validators bytes once vs every time,
    * This is 2x - 3x faster than allocating memory every time.
    */
-  private serializeStateValidators(state: CachedBeaconStateAllForks): BufferWithKey | null {
-    const type = state.type.fields.validators;
-    const size = type.tree_serializedSize(state.validators.node);
+  private serializeStateValidators(state: IBeaconStateView): BufferWithKey | null {
+    const size = state.serializedValidatorsSize();
     if (this.bufferPool) {
       const bufferWithKey = this.bufferPool.alloc(size, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_VALIDATORS);
       if (bufferWithKey) {
         const validatorsBytes = bufferWithKey.buffer;
         const dataView = new DataView(validatorsBytes.buffer, validatorsBytes.byteOffset, validatorsBytes.byteLength);
-        state.validators.serializeToBytes({uint8Array: validatorsBytes, dataView}, 0);
+        state.serializeValidatorsToBytes({uint8Array: validatorsBytes, dataView}, 0);
         return bufferWithKey;
       }
     }
@@ -844,36 +916,62 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   }
 }
 
-export function toCheckpointHex(checkpoint: phase0.Checkpoint): CheckpointHex {
+export function toCheckpointHexPayload(checkpoint: phase0.Checkpoint, payloadPresent: boolean): CheckpointHexPayload {
   return {
     epoch: checkpoint.epoch,
     rootHex: toRootHex(checkpoint.root),
+    payloadPresent,
   };
 }
 
-export function toCheckpointKey(cp: CheckpointHex): string {
-  return `${cp.rootHex}:${cp.epoch}`;
-}
+/**
+ * Convert fork-choice CheckpointWithPayloadStatus to beacon-node CheckpointHexPayload.
+ * Maps PayloadStatus enum to boolean payloadPresent.
+ * @throws Error if checkpoint has PENDING payload status (ambiguous which variant to use)
+ */
+export function fcCheckpointToHexPayload(checkpoint: CheckpointWithPayloadStatus): CheckpointHexPayload {
+  const PayloadStatus = {PENDING: 0, EMPTY: 1, FULL: 2} as const;
 
-function toCacheKey(cp: CheckpointHex | phase0.Checkpoint): CacheKey {
-  if (isCheckpointHex(cp)) {
-    return `${cp.rootHex}_${cp.epoch}`;
+  if (checkpoint.payloadStatus === PayloadStatus.PENDING) {
+    throw Error(
+      `Cannot convert checkpoint with PENDING payload status at epoch ${checkpoint.epoch} root ${checkpoint.rootHex}`
+    );
   }
-  return `${toRootHex(cp.root)}_${cp.epoch}`;
+
+  return {
+    epoch: checkpoint.epoch,
+    rootHex: checkpoint.rootHex,
+    payloadPresent: checkpoint.payloadStatus === PayloadStatus.FULL,
+  };
+}
+
+export function toCheckpointKey(cp: CheckpointHexPayload): string {
+  return `${cp.rootHex}:${cp.epoch}:${cp.payloadPresent}`;
 }
 
-function fromCacheKey(key: CacheKey): CheckpointHex {
-  const [rootHex, epoch] = key.split("_");
+/**
+ * Convert checkpoint to cache key string.
+ * Format: `{rootHex}_{epoch}_{payloadPresent}`
+ */
+function toCacheKey(cp: CheckpointHexPayload): CacheKey {
+  return `${cp.rootHex}_${cp.epoch}_${cp.payloadPresent}`;
+}
+
+function fromCacheKey(key: CacheKey): CheckpointHexPayload {
+  const parts = key.split("_");
+  const rootHex = parts[0];
+  const epoch = Number(parts[1]);
+  // For backward compatibility with old format (rootHex_epoch), default to true
+  const payloadPresent = parts.length > 2 ? parts[2] === "true" : true;
   return {
     rootHex,
-    epoch: Number(epoch),
+    epoch,
+    payloadPresent,
   };
 }
 
-function isCachedBeaconState(
-  stateOrBytes: CachedBeaconStateAllForks | LoadedStateBytesData
-): stateOrBytes is CachedBeaconStateAllForks {
-  return (stateOrBytes as CachedBeaconStateAllForks).slot !== undefined;
+function isBeaconStateView(stateOrBytes: IBeaconStateView | LoadedStateBytesData): stateOrBytes is IBeaconStateView {
+  return (stateOrBytes as IBeaconStateView).slot !== undefined;
 }
 
 function isInMemoryCacheItem(cacheItem: CacheItem): cacheItem is InMemoryCacheItem {
@@ -883,7 +981,3 @@ function isInMemoryCacheItem(cacheItem: CacheItem): cacheItem is InMemoryCacheIt
 function isPersistedCacheItem(cacheItem: CacheItem): cacheItem is PersistedCacheItem {
   return cacheItem.type === CacheItemType.persisted;
 }
-
-function isCheckpointHex(cp: CheckpointHex | phase0.Checkpoint): cp is CheckpointHex {
-  return (cp as CheckpointHex).rootHex !== undefined;
-}