@lodestar/beacon-node 1.41.0 → 1.42.0-dev.5f2fffc2ce

package/src/chain/stateCache/persistentCheckpointsCache.ts

@@ -1,5 +1,6 @@
 import {routes} from "@lodestar/api";
 import {BeaconConfig} from "@lodestar/config";
+import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice";
 import {
   CachedBeaconStateAllForks,
   computeStartSlotAtEpoch,
@@ -14,7 +15,7 @@ import {IClock} from "../../util/clock.js";
 import {serializeState} from "../serializeState.js";
 import {CPStateDatastore, DatastoreKey} from "./datastore/index.js";
 import {MapTracker} from "./mapMetrics.js";
-import {BlockStateCache, CacheItemType, CheckpointHex, CheckpointStateCache} from "./types.js";
+import {BlockStateCache, CacheItemType, CheckpointHexPayload, CheckpointStateCache} from "./types.js";
 
 export type PersistentCheckpointStateCacheOpts = {
   /** Keep max n state epochs in memory, persist the rest to disk */
@@ -54,6 +55,22 @@ type CacheItem = InMemoryCacheItem | PersistedCacheItem;
 
 type LoadedStateBytesData = {persistedKey: DatastoreKey; stateBytes: Uint8Array};
 
+/** Bitmask for tracking which payload variants exist per root in the epochIndex */
+enum PayloadAvailability {
+  NOT_PRESENT = 1,
+  PRESENT = 2,
+}
+
+const PAYLOAD_AVAILABILITY_ALL = [PayloadAvailability.NOT_PRESENT, PayloadAvailability.PRESENT] as const;
+
+function toPayloadAvailability(payloadPresent: boolean): PayloadAvailability {
+  return payloadPresent ? PayloadAvailability.PRESENT : PayloadAvailability.NOT_PRESENT;
+}
+
+function fromPayloadAvailability(flag: PayloadAvailability): boolean {
+  return flag === PayloadAvailability.PRESENT;
+}
+
 /**
  * Before n-historical states, lodestar keeps all checkpoint states since finalized
  * Since Sep 2024, lodestar stores 3 most recent checkpoint states in memory and the rest on disk. The finalized state
@@ -106,8 +123,8 @@ const PROCESS_CHECKPOINT_STATES_BPS = 6667;
  */
 export class PersistentCheckpointStateCache implements CheckpointStateCache {
   private readonly cache: MapTracker<CacheKey, CacheItem>;
-  /** Epoch -> Set<blockRoot> */
-  private readonly epochIndex = new MapDef<Epoch, Set<RootHex>>(() => new Set<string>());
+  /** Epoch -> Map<blockRoot, PayloadAvailability bitmask> */
+  private readonly epochIndex = new MapDef<Epoch, Map<RootHex, number>>(() => new Map());
   private readonly config: BeaconConfig;
   private readonly metrics: Metrics | null | undefined;
   private readonly logger: Logger;
@@ -203,13 +220,18 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    * - Get block for processing
    * - Regen head state
    */
-  async getOrReload(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | null> {
+  async getOrReload(cp: CheckpointHexPayload): Promise<CachedBeaconStateAllForks | null> {
     const stateOrStateBytesData = await this.getStateOrLoadDb(cp);
     if (stateOrStateBytesData === null || isCachedBeaconState(stateOrStateBytesData)) {
       return stateOrStateBytesData ?? null;
     }
     const {persistedKey, stateBytes} = stateOrStateBytesData;
-    const logMeta = {persistedKey: toHex(persistedKey)};
+    const logMeta = {
+      epoch: cp.epoch,
+      rootHex: cp.rootHex,
+      payloadPresent: cp.payloadPresent,
+      persistedKey: toHex(persistedKey),
+    };
     this.logger.debug("Reload: read state successful", logMeta);
     this.metrics?.cpStateCache.stateReloadSecFromSlot.observe(
       this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0
@@ -250,7 +272,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
       // only remove persisted state once we reload successfully
       const cpKey = toCacheKey(cp);
       this.cache.set(cpKey, {type: CacheItemType.inMemory, state: newCachedState, persistedKey});
-      this.epochIndex.getOrDefault(cp.epoch).add(cp.rootHex);
+      this.addToEpochIndex(cp.epoch, cp.rootHex, cp.payloadPresent);
       // don't prune from memory here, call it at the last 1/3 of slot 0 of an epoch
       return newCachedState;
     } catch (e) {
@@ -262,7 +284,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   /**
    * Return either state or state bytes loaded from db.
    */
-  async getStateOrBytes(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | Uint8Array | null> {
+  async getStateOrBytes(cp: CheckpointHexPayload): Promise<CachedBeaconStateAllForks | Uint8Array | null> {
     const stateOrLoadedState = await this.getStateOrLoadDb(cp);
     if (stateOrLoadedState === null || isCachedBeaconState(stateOrLoadedState)) {
       return stateOrLoadedState;
@@ -273,7 +295,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   /**
    * Return either state or state bytes with persisted key loaded from db.
    */
-  async getStateOrLoadDb(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | LoadedStateBytesData | null> {
+  async getStateOrLoadDb(cp: CheckpointHexPayload): Promise<CachedBeaconStateAllForks | LoadedStateBytesData | null> {
     const cpKey = toCacheKey(cp);
     const inMemoryState = this.get(cpKey);
     if (inMemoryState) {
@@ -304,7 +326,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   /**
    * Similar to get() api without reloading from disk
    */
-  get(cpOrKey: CheckpointHex | string): CachedBeaconStateAllForks | null {
+  get(cpOrKey: CheckpointHexPayload | CacheKey): CachedBeaconStateAllForks | null {
     this.metrics?.cpStateCache.lookups.inc();
     const cpKey = typeof cpOrKey === "string" ? cpOrKey : toCacheKey(cpOrKey);
     const cacheItem = this.cache.get(cpKey);
@@ -330,9 +352,11 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
 
   /**
    * Add a state of a checkpoint to this cache, prune from memory if necessary.
+   * @param payloadPresent - For Gloas: true if this is payload state, false if block state.
+   *                         Always true for pre-Gloas.
    */
-  add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks): void {
-    const cpHex = toCheckpointHex(cp);
+  add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks, payloadPresent: boolean): void {
+    const cpHex = toCheckpointHexPayload(cp, payloadPresent);
     const key = toCacheKey(cpHex);
     const cacheItem = this.cache.get(key);
     this.metrics?.cpStateCache.adds.inc();
@@ -343,27 +367,32 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
       this.logger.verbose("Added checkpoint state to memory but a persisted key existed", {
         epoch: cp.epoch,
         rootHex: cpHex.rootHex,
+        payloadPresent,
         persistedKey: toHex(persistedKey),
       });
     } else {
       this.cache.set(key, {type: CacheItemType.inMemory, state});
-      this.logger.verbose("Added checkpoint state to memory", {epoch: cp.epoch, rootHex: cpHex.rootHex});
+      this.logger.verbose("Added checkpoint state to memory", {
+        epoch: cp.epoch,
+        rootHex: cpHex.rootHex,
+        payloadPresent,
+      });
     }
-    this.epochIndex.getOrDefault(cp.epoch).add(cpHex.rootHex);
+    this.addToEpochIndex(cp.epoch, cpHex.rootHex, cpHex.payloadPresent);
     this.prunePersistedStates();
   }
 
   /**
    * Searches in-memory state for the latest cached state with a `root` without reload, starting with `epoch` and descending
    */
-  getLatest(rootHex: RootHex, maxEpoch: Epoch): CachedBeaconStateAllForks | null {
+  getLatest(rootHex: RootHex, maxEpoch: Epoch, payloadPresent: boolean): CachedBeaconStateAllForks | null {
     // sort epochs in descending order, only consider epochs lte `epoch`
     const epochs = Array.from(this.epochIndex.keys())
       .sort((a, b) => b - a)
       .filter((e) => e <= maxEpoch);
     for (const epoch of epochs) {
-      if (this.epochIndex.get(epoch)?.has(rootHex)) {
-        const inMemoryClonedState = this.get({rootHex, epoch});
+      if (this.hasPayloadVariant(epoch, rootHex, payloadPresent)) {
+        const inMemoryClonedState = this.get({rootHex, epoch, payloadPresent});
         if (inMemoryClonedState) {
           return inMemoryClonedState;
         }
@@ -379,20 +408,24 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    * - Get block for processing
    * - Regen head state
    */
-  async getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise<CachedBeaconStateAllForks | null> {
+  async getOrReloadLatest(
+    rootHex: RootHex,
+    maxEpoch: Epoch,
+    payloadPresent: boolean
+  ): Promise<CachedBeaconStateAllForks | null> {
     // sort epochs in descending order, only consider epochs lte `epoch`
     const epochs = Array.from(this.epochIndex.keys())
       .sort((a, b) => b - a)
       .filter((e) => e <= maxEpoch);
     for (const epoch of epochs) {
-      if (this.epochIndex.get(epoch)?.has(rootHex)) {
+      if (this.hasPayloadVariant(epoch, rootHex, payloadPresent)) {
         try {
-          const state = await this.getOrReload({rootHex, epoch});
+          const state = await this.getOrReload({rootHex, epoch, payloadPresent});
           if (state) {
             return state;
           }
         } catch (e) {
-          this.logger.debug("Error get or reload state", {epoch, rootHex}, e as Error);
+          this.logger.debug("Error get or reload state", {epoch, rootHex, payloadPresent}, e as Error);
         }
       }
     }
@@ -400,12 +433,14 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   }
 
   /**
-   * Update the precomputed checkpoint and return the number of his for the
+   * Update the precomputed checkpoint and return the number of hits for the
    * previous one (if any).
+   * @param payloadPresent - For Gloas: true if head block has FULL payload, false if EMPTY.
+   *                         Always true for pre-Gloas.
    */
-  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null {
+  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null {
     const previousHits = this.preComputedCheckpointHits;
-    this.preComputedCheckpoint = toCacheKey({rootHex, epoch});
+    this.preComputedCheckpoint = toCacheKey({rootHex, epoch, payloadPresent});
     this.preComputedCheckpointHits = 0;
     return previousHits;
   }
@@ -479,6 +514,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    * - 2 then we'll persist {root: b2, epoch n-2} checkpoint state to disk, there are also 2 checkpoint states in memory at epoch n, same to the above (maxEpochsInMemory=1)
    *
    * As of Mar 2024, it takes <=350ms to persist a holesky state on fast server
+   *
+   * For Gloas: Processes both block state and payload state variants together. The decision of which roots to persist/prune
+   * is based on root canonicality (from state's view), not payload presence. Both variants are managed as a unit.
    */
   async processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise<number> {
     let persistCount = 0;
@@ -549,7 +587,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    *
    * Use seed state from the block cache if cannot find any seed states within this cache.
    */
-  findSeedStateToReload(reloadedCp: CheckpointHex): CachedBeaconStateAllForks {
+  findSeedStateToReload(reloadedCp: CheckpointHexPayload): CachedBeaconStateAllForks {
     const maxEpoch = Math.max(...Array.from(this.epochIndex.keys()));
     const reloadedCpSlot = computeStartSlotAtEpoch(reloadedCp.epoch);
     let firstState: CachedBeaconStateAllForks | null = null;
@@ -562,32 +600,35 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
         return firstState;
       }
 
-      for (const rootHex of this.epochIndex.get(epoch) || []) {
-        const cpKey = toCacheKey({rootHex, epoch});
-        const cacheItem = this.cache.get(cpKey);
-        if (cacheItem === undefined) {
-          // should not happen
-          continue;
-        }
-        if (isInMemoryCacheItem(cacheItem)) {
-          const {state} = cacheItem;
-          if (firstState === null) {
-            firstState = state;
+      for (const [rootHex, bitmask] of this.epochIndex.get(epoch) || []) {
+        for (const flag of PAYLOAD_AVAILABILITY_ALL) {
+          if (!(bitmask & flag)) continue;
+          const payloadPresent = fromPayloadAvailability(flag);
+          const cpKey = toCacheKey({rootHex, epoch, payloadPresent});
+          const cacheItem = this.cache.get(cpKey);
+          if (cacheItem === undefined) {
+            continue;
           }
-          const cpLog = {cpEpoch: epoch, cpRoot: rootHex};
-
-          try {
-            // amongst states of the same epoch, choose the one with the same view of reloadedCp
-            if (
-              reloadedCpSlot < state.slot &&
-              toRootHex(getBlockRootAtSlot(state, reloadedCpSlot)) === reloadedCp.rootHex
-            ) {
-              this.logger.verbose("Reload: use checkpoint state as seed state", {...cpLog, ...logCtx});
-              return state;
+          if (isInMemoryCacheItem(cacheItem)) {
+            const {state} = cacheItem;
+            if (firstState === null) {
+              firstState = state;
+            }
+            const cpLog = {cpEpoch: epoch, cpRoot: rootHex, payloadPresent};
+
+            try {
+              // amongst states of the same epoch, choose the one with the same view of reloadedCp
+              if (
+                reloadedCpSlot < state.slot &&
+                toRootHex(getBlockRootAtSlot(state, reloadedCpSlot)) === reloadedCp.rootHex
+              ) {
+                this.logger.verbose("Reload: use checkpoint state as seed state", {...cpLog, ...logCtx});
+                return state;
+              }
+            } catch (e) {
+              // getBlockRootAtSlot may throw error
+              this.logger.debug("Error finding checkpoint state to reload", {...cpLog, ...logCtx}, e as Error);
             }
-          } catch (e) {
-            // getBlockRootAtSlot may throw error
-            this.logger.debug("Error finding checkpoint state to reload", {...cpLog, ...logCtx}, e as Error);
           }
         }
       }
@@ -604,6 +645,31 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
     this.epochIndex.clear();
   }
 
+  private addToEpochIndex(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): void {
+    const rootMap = this.epochIndex.getOrDefault(epoch);
+    rootMap.set(rootHex, (rootMap.get(rootHex) ?? 0) | toPayloadAvailability(payloadPresent));
+  }
+
+  private removeFromEpochIndex(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): void {
+    const rootMap = this.epochIndex.get(epoch);
+    if (rootMap === undefined) return;
+    const existing = rootMap.get(rootHex);
+    if (existing === undefined) return;
+    const updated = existing & ~toPayloadAvailability(payloadPresent);
+    if (updated === 0) {
+      rootMap.delete(rootHex);
+      if (rootMap.size === 0) {
+        this.epochIndex.delete(epoch);
+      }
+    } else {
+      rootMap.set(rootHex, updated);
+    }
+  }
+
+  private hasPayloadVariant(epoch: Epoch, rootHex: RootHex, payloadPresent: boolean): boolean {
+    return Boolean((this.epochIndex.get(epoch)?.get(rootHex) ?? 0) & toPayloadAvailability(payloadPresent));
+  }
+
   /** ONLY FOR DEBUGGING PURPOSES. For lodestar debug API */
   dumpSummary(): routes.lodestar.StateCacheItem[] {
     return Array.from(this.cache.keys()).map((key) => {
@@ -682,7 +748,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
     const prevEpochRoot = toRootHex(getBlockRootAtSlot(state, epochBoundarySlot - 1));
 
     // for each epoch, usually there are 2 rootHexes respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State
-    const cpRootHexes = this.epochIndex.get(epoch) ?? [];
+    const cpRootHexMap = this.epochIndex.get(epoch) ?? new Map<RootHex, number>();
     const persistedRootHexes = new Set<RootHex>();
 
     // 1) if there is no CRCS, persist PRCS (block 0 of epoch is skipped). In this case prevEpochRoot === epochBoundaryHex
@@ -691,76 +757,81 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
     persistedRootHexes.add(epochBoundaryHex);
 
     // 3) persist any states with unknown roots to this state
-    for (const rootHex of cpRootHexes) {
+    for (const rootHex of cpRootHexMap.keys()) {
       if (rootHex !== epochBoundaryHex && rootHex !== prevEpochRoot) {
         persistedRootHexes.add(rootHex);
       }
     }
 
-    for (const rootHex of cpRootHexes) {
-      const cpKey = toCacheKey({epoch: epoch, rootHex});
-      const cacheItem = this.cache.get(cpKey);
-
-      if (cacheItem !== undefined && isInMemoryCacheItem(cacheItem)) {
-        let {persistedKey} = cacheItem;
-        const {state} = cacheItem;
-        const logMeta = {
-          stateSlot: state.slot,
-          rootHex,
-          epochBoundaryHex,
-          persistedKey: persistedKey ? toHex(persistedKey) : "",
-        };
-
-        if (persistedRootHexes.has(rootHex)) {
-          if (persistedKey) {
-            // we don't care if the checkpoint state is already persisted
-            this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta);
-          } else {
-            // persist and do not update epochIndex
-            this.metrics?.cpStateCache.statePersistSecFromSlot.observe(
-              this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0
-            );
-            const cpPersist = {epoch: epoch, root: fromHex(rootHex)};
-            // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory.
-            // As monitored on holesky as of Jan 2024:
-            //   - This does not increase heap allocation while gc time is the same
-            //   - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s)
-            //   - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s)
-            //   - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization
-            const timer = this.metrics?.stateSerializeDuration.startTimer({
-              source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE,
-            });
-            persistedKey = await serializeState(
-              state,
-              AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE,
-              (stateBytes) => {
-                timer?.();
-                return this.datastore.write(cpPersist, stateBytes);
-              },
-              this.bufferPool
-            );
+    for (const [rootHex, bitmask] of cpRootHexMap) {
+      for (const flag of PAYLOAD_AVAILABILITY_ALL) {
+        if (!(bitmask & flag)) continue;
+        const payloadPresent = fromPayloadAvailability(flag);
+        const cpKey = toCacheKey({epoch: epoch, rootHex, payloadPresent});
+        const cacheItem = this.cache.get(cpKey);
 
-            persistCount++;
-            this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", {
-              ...logMeta,
-              persistedKey: toHex(persistedKey),
-            });
-          }
-          // overwrite cpKey, this means the state is deleted from memory
-          this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey});
-        } else {
-          if (persistedKey) {
-            // persisted file will be eventually deleted by the archive task
-            // this also means the state is deleted from memory
+        if (cacheItem !== undefined && isInMemoryCacheItem(cacheItem)) {
+          let {persistedKey} = cacheItem;
+          const {state} = cacheItem;
+          const logMeta = {
+            stateSlot: state.slot,
+            rootHex,
+            payloadPresent,
+            epochBoundaryHex,
+            persistedKey: persistedKey ? toHex(persistedKey) : "",
+          };
+
+          if (persistedRootHexes.has(rootHex)) {
+            if (persistedKey) {
+              // we don't care if the checkpoint state is already persisted
+              this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta);
+            } else {
+              // persist and do not update epochIndex
+              this.metrics?.cpStateCache.statePersistSecFromSlot.observe(
+                this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0
+              );
+              const cpPersist = {epoch: epoch, root: fromHex(rootHex)};
+              // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory.
+              // As monitored on holesky as of Jan 2024:
+              //   - This does not increase heap allocation while gc time is the same
+              //   - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s)
+              //   - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s)
+              //   - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization
+              const timer = this.metrics?.stateSerializeDuration.startTimer({
+                source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE,
+              });
+              persistedKey = await serializeState(
+                state,
+                AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE,
+                (stateBytes) => {
+                  timer?.();
+                  return this.datastore.write(cpPersist, stateBytes, payloadPresent);
+                },
+                this.bufferPool
+              );
+
+              persistCount++;
+              this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", {
+                ...logMeta,
+                persistedKey: toHex(persistedKey),
+              });
+            }
+            // overwrite cpKey, this means the state is deleted from memory
             this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey});
-            // do not update epochIndex
           } else {
-            // delete the state from memory
-            this.cache.delete(cpKey);
-            this.epochIndex.get(epoch)?.delete(rootHex);
+            if (persistedKey) {
+              // persisted file will be eventually deleted by the archive task
+              // this also means the state is deleted from memory
+              this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey});
+              // do not update epochIndex
+            } else {
+              // delete the state from memory
+              this.cache.delete(cpKey);
+              this.removeFromEpochIndex(epoch, rootHex, payloadPresent);
+            }
+            this.metrics?.cpStateCache.statePruneFromMemoryCount.inc();
+            this.logger.verbose("Pruned checkpoint state from memory", logMeta);
           }
-          this.metrics?.cpStateCache.statePruneFromMemoryCount.inc();
-          this.logger.verbose("Pruned checkpoint state from memory", logMeta);
         }
       }
     }
@@ -773,26 +844,40 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    */
   private async deleteAllEpochItems(epoch: Epoch): Promise<void> {
     let persistCount = 0;
-    const rootHexes = this.epochIndex.get(epoch) || [];
-    for (const rootHex of rootHexes) {
-      const key = toCacheKey({rootHex, epoch});
-      const cacheItem = this.cache.get(key);
-
-      if (cacheItem) {
-        const persistedKey = isPersistedCacheItem(cacheItem) ? cacheItem.value : cacheItem.persistedKey;
-        if (persistedKey) {
-          await this.datastore.remove(persistedKey);
-          persistCount++;
-          this.metrics?.cpStateCache.persistedStateRemoveCount.inc();
+    const rootHexMap = this.epochIndex.get(epoch) || new Map<RootHex, number>();
+    for (const [rootHex, bitmask] of rootHexMap) {
+      for (const flag of PAYLOAD_AVAILABILITY_ALL) {
+        if (!(bitmask & flag)) continue;
+        const payloadPresent = fromPayloadAvailability(flag);
+        const key = toCacheKey({rootHex, epoch, payloadPresent});
+        const cacheItem = this.cache.get(key);
+
+        if (cacheItem) {
+          const persistedKey = isPersistedCacheItem(cacheItem) ? cacheItem.value : cacheItem.persistedKey;
+          if (persistedKey) {
+            await this.datastore.remove(persistedKey);
+            persistCount++;
+            this.metrics?.cpStateCache.persistedStateRemoveCount.inc();
+          }
         }
+        this.cache.delete(key);
+        this.logger.verbose("Pruned checkpoint state", {
+          epoch,
+          rootHex,
+          payloadPresent,
+          type: cacheItem ? (isPersistedCacheItem(cacheItem) ? "persisted" : "in-memory") : "missing",
+        });
       }
-      this.cache.delete(key);
     }
     this.epochIndex.delete(epoch);
-    this.logger.verbose("Pruned checkpoint states for epoch", {
+    this.logger.verbose("Pruned all checkpoint states for epoch", {
       epoch,
       persistCount,
-      rootHexes: Array.from(rootHexes).join(","),
+      items: Array.from(rootHexMap.entries())
+        .flatMap(([rootHex, bitmask]) =>
+          PAYLOAD_AVAILABILITY_ALL.filter((f) => bitmask & f).map((f) => `${rootHex}:${fromPayloadAvailability(f)}`)
+        )
+        .join(","),
     });
   }
 
@@ -844,29 +929,57 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   }
 }
 
-export function toCheckpointHex(checkpoint: phase0.Checkpoint): CheckpointHex {
+export function toCheckpointHexPayload(checkpoint: phase0.Checkpoint, payloadPresent: boolean): CheckpointHexPayload {
   return {
     epoch: checkpoint.epoch,
     rootHex: toRootHex(checkpoint.root),
+    payloadPresent,
   };
 }
 
-export function toCheckpointKey(cp: CheckpointHex): string {
-  return `${cp.rootHex}:${cp.epoch}`;
-}
+/**
+ * Convert fork-choice CheckpointWithPayloadStatus to beacon-node CheckpointHexPayload.
+ * Maps PayloadStatus enum to boolean payloadPresent.
+ * @throws Error if checkpoint has PENDING payload status (ambiguous which variant to use)
+ */
+export function fcCheckpointToHexPayload(checkpoint: CheckpointWithPayloadStatus): CheckpointHexPayload {
+  const PayloadStatus = {PENDING: 0, EMPTY: 1, FULL: 2} as const;
 
-function toCacheKey(cp: CheckpointHex | phase0.Checkpoint): CacheKey {
-  if (isCheckpointHex(cp)) {
-    return `${cp.rootHex}_${cp.epoch}`;
+  if (checkpoint.payloadStatus === PayloadStatus.PENDING) {
+    throw Error(
+      `Cannot convert checkpoint with PENDING payload status at epoch ${checkpoint.epoch} root ${checkpoint.rootHex}`
+    );
   }
-  return `${toRootHex(cp.root)}_${cp.epoch}`;
+
+  return {
+    epoch: checkpoint.epoch,
+    rootHex: checkpoint.rootHex,
+    payloadPresent: checkpoint.payloadStatus === PayloadStatus.FULL,
+  };
+}
+
+export function toCheckpointKey(cp: CheckpointHexPayload): string {
+  return `${cp.rootHex}:${cp.epoch}:${cp.payloadPresent}`;
 }
 
-function fromCacheKey(key: CacheKey): CheckpointHex {
-  const [rootHex, epoch] = key.split("_");
+/**
+ * Convert checkpoint to cache key string.
+ * Format: `{rootHex}_{epoch}_{payloadPresent}`
+ */
+function toCacheKey(cp: CheckpointHexPayload): CacheKey {
+  return `${cp.rootHex}_${cp.epoch}_${cp.payloadPresent}`;
+}
+
+function fromCacheKey(key: CacheKey): CheckpointHexPayload {
+  const parts = key.split("_");
+  const rootHex = parts[0];
+  const epoch = Number(parts[1]);
+  // For backward compatibility with old format (rootHex_epoch), default to true
+  const payloadPresent = parts.length > 2 ? parts[2] === "true" : true;
   return {
     rootHex,
-    epoch: Number(epoch),
+    epoch,
+    payloadPresent,
   };
 }
 
@@ -883,7 +996,3 @@ function isInMemoryCacheItem(cacheItem: CacheItem): cacheItem is InMemoryCacheIt
 function isPersistedCacheItem(cacheItem: CacheItem): cacheItem is PersistedCacheItem {
   return cacheItem.type === CacheItemType.persisted;
 }
-
-function isCheckpointHex(cp: CheckpointHex | phase0.Checkpoint): cp is CheckpointHex {
-  return (cp as CheckpointHex).rootHex !== undefined;
-}

package/src/chain/stateCache/types.ts

@@ -2,7 +2,11 @@ import {routes} from "@lodestar/api";
 import {CachedBeaconStateAllForks} from "@lodestar/state-transition";
 import {Epoch, RootHex, phase0} from "@lodestar/types";
 
-export type CheckpointHex = {epoch: Epoch; rootHex: RootHex};
+/**
+ * Checkpoint hex representation for state cache keys.
+ * Extends CheckpointWithHex (from fork-choice) with payloadPresent.
+ */
+export type CheckpointHexPayload = {epoch: Epoch; rootHex: RootHex; payloadPresent: boolean};
 
 /**
  * Lodestar currently keeps two state caches around.
@@ -31,6 +35,8 @@ export interface BlockStateCache {
   size: number;
   prune(headStateRootHex: RootHex): void;
   deleteAllBeforeEpoch(finalizedEpoch: Epoch): void;
+  /** Upgrade cache capacity for Gloas fork (2x states for block + payload states) */
+  upgradeToGloas(): void;
   dumpSummary(): routes.lodestar.StateCacheItem[];
   /** Expose beacon states stored in cache. Use with caution */
   getStates(): IterableIterator<CachedBeaconStateAllForks>;
@@ -59,13 +65,17 @@ export interface BlockStateCache {
  */
 export interface CheckpointStateCache {
   init?: () => Promise<void>;
-  getOrReload(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | null>;
-  getStateOrBytes(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | Uint8Array | null>;
-  get(cpOrKey: CheckpointHex | string): CachedBeaconStateAllForks | null;
-  add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks): void;
-  getLatest(rootHex: RootHex, maxEpoch: Epoch): CachedBeaconStateAllForks | null;
-  getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise<CachedBeaconStateAllForks | null>;
-  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null;
+  getOrReload(cp: CheckpointHexPayload): Promise<CachedBeaconStateAllForks | null>;
+  getStateOrBytes(cp: CheckpointHexPayload): Promise<CachedBeaconStateAllForks | Uint8Array | null>;
+  get(cpOrKey: CheckpointHexPayload | string): CachedBeaconStateAllForks | null;
+  add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks, payloadPresent: boolean): void;
+  getLatest(rootHex: RootHex, maxEpoch: Epoch, payloadPresent: boolean): CachedBeaconStateAllForks | null;
+  getOrReloadLatest(
+    rootHex: RootHex,
+    maxEpoch: Epoch,
+    payloadPresent: boolean
+  ): Promise<CachedBeaconStateAllForks | null>;
+  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null;
   prune(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void;
   pruneFinalized(finalizedEpoch: Epoch): void;
   processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise<number>;

package/src/chain/validation/voluntaryExit.ts

@@ -1,4 +1,5 @@
 import {
+  BeaconStateView,
   VoluntaryExitValidity,
   getVoluntaryExitSignatureSet,
   getVoluntaryExitValidity,
@@ -59,7 +60,7 @@ async function validateVoluntaryExit(
     });
   }
 
-  const signatureSet = getVoluntaryExitSignatureSet(chain.config, state.slot, voluntaryExit);
+  const signatureSet = getVoluntaryExitSignatureSet(chain.config, new BeaconStateView(state), voluntaryExit);
   if (!(await chain.bls.verifySignatureSets([signatureSet], {batchable: true, priority: prioritizeBls}))) {
     throw new VoluntaryExitError(GossipAction.REJECT, {
       code: VoluntaryExitErrorCode.INVALID_SIGNATURE,

package/lib/chain/archiveStore/utils/archivePayloads.d.ts

@@ -1,7 +0,0 @@
-import { CheckpointWithHex } from "@lodestar/fork-choice";
-import { IBeaconChain } from "../../interface.js";
-/**
- * Archives execution payload envelopes from hot DB to archive DB after finalization.
- */
-export declare function archiveExecutionPayloadEnvelopes(chain: IBeaconChain, _finalized: CheckpointWithHex): Promise<void>;
-//# sourceMappingURL=archivePayloads.d.ts.map