@lodestar/beacon-node 1.41.0 → 1.42.0-dev.04e52d162f

package/lib/chain/stateCache/persistentCheckpointsCache.js

@@ -50,12 +50,25 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
     var e = new Error(message);
     return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
 });
-import { computeStartSlotAtEpoch, getBlockRootAtSlot, loadCachedBeaconState, } from "@lodestar/state-transition";
+import { computeStartSlotAtEpoch } from "@lodestar/state-transition";
 import { MapDef, fromHex, sleep, toHex, toRootHex } from "@lodestar/utils";
 import { AllocSource } from "../../util/bufferPool.js";
 import { serializeState } from "../serializeState.js";
 import { MapTracker } from "./mapMetrics.js";
 import { CacheItemType } from "./types.js";
+/** Bitmask for tracking which payload variants exist per root in the epochIndex */
+var PayloadAvailability;
+(function (PayloadAvailability) {
+    PayloadAvailability[PayloadAvailability["NOT_PRESENT"] = 1] = "NOT_PRESENT";
+    PayloadAvailability[PayloadAvailability["PRESENT"] = 2] = "PRESENT";
+})(PayloadAvailability || (PayloadAvailability = {}));
+const PAYLOAD_AVAILABILITY_ALL = [PayloadAvailability.NOT_PRESENT, PayloadAvailability.PRESENT];
+function toPayloadAvailability(payloadPresent) {
+    return payloadPresent ? PayloadAvailability.PRESENT : PayloadAvailability.NOT_PRESENT;
+}
+function fromPayloadAvailability(flag) {
+    return flag === PayloadAvailability.PRESENT;
+}
 /**
  * Before n-historical states, lodestar keeps all checkpoint states since finalized
  * Since Sep 2024, lodestar stores 3 most recent checkpoint states in memory and the rest on disk. The finalized state
@@ -105,8 +118,8 @@ const PROCESS_CHECKPOINT_STATES_BPS = 6667;
  */
 export class PersistentCheckpointStateCache {
     cache;
-    /** Epoch -> Set<blockRoot> */
-    epochIndex = new MapDef(() => new Set());
+    /** Epoch -> Map<blockRoot, PayloadAvailability bitmask> */
+    epochIndex = new MapDef(() => new Map());
     config;
     metrics;
     logger;
@@ -188,15 +201,20 @@ export class PersistentCheckpointStateCache {
      */
     async getOrReload(cp) {
         const stateOrStateBytesData = await this.getStateOrLoadDb(cp);
-        if (stateOrStateBytesData === null || isCachedBeaconState(stateOrStateBytesData)) {
+        if (stateOrStateBytesData === null || isBeaconStateView(stateOrStateBytesData)) {
             return stateOrStateBytesData ?? null;
         }
         const { persistedKey, stateBytes } = stateOrStateBytesData;
-        const logMeta = { persistedKey: toHex(persistedKey) };
+        const logMeta = {
+            epoch: cp.epoch,
+            rootHex: cp.rootHex,
+            payloadPresent: cp.payloadPresent,
+            persistedKey: toHex(persistedKey),
+        };
         this.logger.debug("Reload: read state successful", logMeta);
         this.metrics?.cpStateCache.stateReloadSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0);
         const seedState = this.findSeedStateToReload(cp);
-        this.metrics?.cpStateCache.stateReloadEpochDiff.observe(Math.abs(seedState.epochCtx.epoch - cp.epoch));
+        this.metrics?.cpStateCache.stateReloadEpochDiff.observe(Math.abs(seedState.epoch - cp.epoch));
         this.logger.debug("Reload: found seed state", { ...logMeta, seedSlot: seedState.slot });
         try {
             const env_1 = { stack: [], error: void 0, hasError: false };
@@ -209,18 +227,15 @@ export class PersistentCheckpointStateCache {
                 if (validatorsBytes == null) {
                     // fallback logic in case we can't use the buffer pool
                     this.metrics?.cpStateCache.stateReloadValidatorsSerializeAllocCount.inc();
-                    validatorsBytes = seedState.validators.serialize();
+                    validatorsBytes = seedState.serializeValidators();
                 }
                 sszTimer?.();
                 const timer = this.metrics?.cpStateCache.stateReloadDuration.startTimer();
-                const newCachedState = loadCachedBeaconState(seedState, stateBytes, {}, validatorsBytes);
+                const newCachedState = seedState.loadOtherState(stateBytes, validatorsBytes);
                 // hashTreeRoot() calls the commit() inside
                 // there is no modification inside the state, it's just that we want to compute and cache all roots
                 const stateRoot = toRootHex(newCachedState.hashTreeRoot());
                 timer?.();
-                // load all cache in order for consumers (usually regen.getState()) to process blocks faster
-                newCachedState.validators.getAllReadonlyValues();
-                newCachedState.balances.getAll();
                 this.logger.debug("Reload: cached state load successful", {
                     ...logMeta,
                     stateSlot: newCachedState.slot,
@@ -230,7 +245,7 @@ export class PersistentCheckpointStateCache {
                 // only remove persisted state once we reload successfully
                 const cpKey = toCacheKey(cp);
                 this.cache.set(cpKey, { type: CacheItemType.inMemory, state: newCachedState, persistedKey });
-                this.epochIndex.getOrDefault(cp.epoch).add(cp.rootHex);
+                this.addToEpochIndex(cp.epoch, cp.rootHex, cp.payloadPresent);
                 // don't prune from memory here, call it at the last 1/3 of slot 0 of an epoch
                 return newCachedState;
             }
@@ -252,7 +267,7 @@ export class PersistentCheckpointStateCache {
      */
     async getStateOrBytes(cp) {
         const stateOrLoadedState = await this.getStateOrLoadDb(cp);
-        if (stateOrLoadedState === null || isCachedBeaconState(stateOrLoadedState)) {
+        if (stateOrLoadedState === null || isBeaconStateView(stateOrLoadedState)) {
             return stateOrLoadedState;
         }
         return stateOrLoadedState.stateBytes;
@@ -306,9 +321,11 @@ export class PersistentCheckpointStateCache {
     }
     /**
      * Add a state of a checkpoint to this cache, prune from memory if necessary.
+     * @param payloadPresent - For Gloas: true if this is payload state, false if block state.
+     *                         Always true for pre-Gloas.
      */
-    add(cp, state) {
-        const cpHex = toCheckpointHex(cp);
+    add(cp, state, payloadPresent) {
+        const cpHex = toCheckpointHexPayload(cp, payloadPresent);
         const key = toCacheKey(cpHex);
         const cacheItem = this.cache.get(key);
         this.metrics?.cpStateCache.adds.inc();
@@ -319,27 +336,32 @@ export class PersistentCheckpointStateCache {
             this.logger.verbose("Added checkpoint state to memory but a persisted key existed", {
                 epoch: cp.epoch,
                 rootHex: cpHex.rootHex,
+                payloadPresent,
                 persistedKey: toHex(persistedKey),
             });
         }
         else {
             this.cache.set(key, { type: CacheItemType.inMemory, state });
-            this.logger.verbose("Added checkpoint state to memory", { epoch: cp.epoch, rootHex: cpHex.rootHex });
+            this.logger.verbose("Added checkpoint state to memory", {
+                epoch: cp.epoch,
+                rootHex: cpHex.rootHex,
+                payloadPresent,
+            });
         }
-        this.epochIndex.getOrDefault(cp.epoch).add(cpHex.rootHex);
+        this.addToEpochIndex(cp.epoch, cpHex.rootHex, cpHex.payloadPresent);
         this.prunePersistedStates();
     }
     /**
      * Searches in-memory state for the latest cached state with a `root` without reload, starting with `epoch` and descending
      */
-    getLatest(rootHex, maxEpoch) {
+    getLatest(rootHex, maxEpoch, payloadPresent) {
         // sort epochs in descending order, only consider epochs lte `epoch`
         const epochs = Array.from(this.epochIndex.keys())
             .sort((a, b) => b - a)
             .filter((e) => e <= maxEpoch);
         for (const epoch of epochs) {
-            if (this.epochIndex.get(epoch)?.has(rootHex)) {
-                const inMemoryClonedState = this.get({ rootHex, epoch });
+            if (this.hasPayloadVariant(epoch, rootHex, payloadPresent)) {
+                const inMemoryClonedState = this.get({ rootHex, epoch, payloadPresent });
                 if (inMemoryClonedState) {
                     return inMemoryClonedState;
                 }
@@ -354,33 +376,35 @@ export class PersistentCheckpointStateCache {
      * - Get block for processing
      * - Regen head state
      */
-    async getOrReloadLatest(rootHex, maxEpoch) {
+    async getOrReloadLatest(rootHex, maxEpoch, payloadPresent) {
         // sort epochs in descending order, only consider epochs lte `epoch`
         const epochs = Array.from(this.epochIndex.keys())
             .sort((a, b) => b - a)
             .filter((e) => e <= maxEpoch);
         for (const epoch of epochs) {
-            if (this.epochIndex.get(epoch)?.has(rootHex)) {
+            if (this.hasPayloadVariant(epoch, rootHex, payloadPresent)) {
                 try {
-                    const state = await this.getOrReload({ rootHex, epoch });
+                    const state = await this.getOrReload({ rootHex, epoch, payloadPresent });
                     if (state) {
                         return state;
                     }
                 }
                 catch (e) {
-                    this.logger.debug("Error get or reload state", { epoch, rootHex }, e);
+                    this.logger.debug("Error get or reload state", { epoch, rootHex, payloadPresent }, e);
                 }
             }
         }
         return null;
     }
     /**
-     * Update the precomputed checkpoint and return the number of his for the
+     * Update the precomputed checkpoint and return the number of hits for the
      * previous one (if any).
+     * @param payloadPresent - For Gloas: true if head block has FULL payload, false if EMPTY.
+     *                         Always true for pre-Gloas.
      */
-    updatePreComputedCheckpoint(rootHex, epoch) {
+    updatePreComputedCheckpoint(rootHex, epoch, payloadPresent) {
         const previousHits = this.preComputedCheckpointHits;
-        this.preComputedCheckpoint = toCacheKey({ rootHex, epoch });
+        this.preComputedCheckpoint = toCacheKey({ rootHex, epoch, payloadPresent });
         this.preComputedCheckpointHits = 0;
         return previousHits;
     }
@@ -449,6 +473,9 @@ export class PersistentCheckpointStateCache {
      * - 2 then we'll persist {root: b2, epoch n-2} checkpoint state to disk, there are also 2 checkpoint states in memory at epoch n, same to the above (maxEpochsInMemory=1)
      *
      * As of Mar 2024, it takes <=350ms to persist a holesky state on fast server
+     *
+     * For Gloas: Processes both block state and payload state variants together. The decision of which roots to persist/prune
+     * is based on root canonicality (from state's view), not payload presence. Both variants are managed as a unit.
      */
     async processState(blockRootHex, state) {
         let persistCount = 0;
@@ -523,30 +550,34 @@ export class PersistentCheckpointStateCache {
             if (firstState !== null) {
                 return firstState;
             }
-            for (const rootHex of this.epochIndex.get(epoch) || []) {
-                const cpKey = toCacheKey({ rootHex, epoch });
-                const cacheItem = this.cache.get(cpKey);
-                if (cacheItem === undefined) {
-                    // should not happen
-                    continue;
-                }
-                if (isInMemoryCacheItem(cacheItem)) {
-                    const { state } = cacheItem;
-                    if (firstState === null) {
-                        firstState = state;
+            for (const [rootHex, bitmask] of this.epochIndex.get(epoch) || []) {
+                for (const flag of PAYLOAD_AVAILABILITY_ALL) {
+                    if (!(bitmask & flag))
+                        continue;
+                    const payloadPresent = fromPayloadAvailability(flag);
+                    const cpKey = toCacheKey({ rootHex, epoch, payloadPresent });
+                    const cacheItem = this.cache.get(cpKey);
+                    if (cacheItem === undefined) {
+                        continue;
                     }
-                    const cpLog = { cpEpoch: epoch, cpRoot: rootHex };
-                    try {
-                        // amongst states of the same epoch, choose the one with the same view of reloadedCp
-                        if (reloadedCpSlot < state.slot &&
-                            toRootHex(getBlockRootAtSlot(state, reloadedCpSlot)) === reloadedCp.rootHex) {
-                            this.logger.verbose("Reload: use checkpoint state as seed state", { ...cpLog, ...logCtx });
-                            return state;
+                    if (isInMemoryCacheItem(cacheItem)) {
+                        const { state } = cacheItem;
+                        if (firstState === null) {
+                            firstState = state;
+                        }
+                        const cpLog = { cpEpoch: epoch, cpRoot: rootHex, payloadPresent };
+                        try {
+                            // amongst states of the same epoch, choose the one with the same view of reloadedCp
+                            if (reloadedCpSlot < state.slot &&
+                                toRootHex(state.getBlockRootAtSlot(reloadedCpSlot)) === reloadedCp.rootHex) {
+                                this.logger.verbose("Reload: use checkpoint state as seed state", { ...cpLog, ...logCtx });
+                                return state;
+                            }
+                        }
+                        catch (e) {
+                            // getBlockRootAtSlot may throw error
+                            this.logger.debug("Error finding checkpoint state to reload", { ...cpLog, ...logCtx }, e);
                         }
-                    }
-                    catch (e) {
-                        // getBlockRootAtSlot may throw error
-                        this.logger.debug("Error finding checkpoint state to reload", { ...cpLog, ...logCtx }, e);
                     }
                 }
             }
@@ -560,6 +591,31 @@ export class PersistentCheckpointStateCache {
         this.cache.clear();
         this.epochIndex.clear();
     }
+    addToEpochIndex(epoch, rootHex, payloadPresent) {
+        const rootMap = this.epochIndex.getOrDefault(epoch);
+        rootMap.set(rootHex, (rootMap.get(rootHex) ?? 0) | toPayloadAvailability(payloadPresent));
+    }
+    removeFromEpochIndex(epoch, rootHex, payloadPresent) {
+        const rootMap = this.epochIndex.get(epoch);
+        if (rootMap === undefined)
+            return;
+        const existing = rootMap.get(rootHex);
+        if (existing === undefined)
+            return;
+        const updated = existing & ~toPayloadAvailability(payloadPresent);
+        if (updated === 0) {
+            rootMap.delete(rootHex);
+            if (rootMap.size === 0) {
+                this.epochIndex.delete(epoch);
+            }
+        }
+        else {
+            rootMap.set(rootHex, updated);
+        }
+    }
+    hasPayloadVariant(epoch, rootHex, payloadPresent) {
+        return Boolean((this.epochIndex.get(epoch)?.get(rootHex) ?? 0) & toPayloadAvailability(payloadPresent));
+    }
     /** ONLY FOR DEBUGGING PURPOSES. For lodestar debug API */
     dumpSummary() {
         return Array.from(this.cache.keys()).map((key) => {
@@ -624,79 +680,85 @@ export class PersistentCheckpointStateCache {
     async processPastEpoch(blockRootHex, state, epoch) {
         let persistCount = 0;
         const epochBoundarySlot = computeStartSlotAtEpoch(epoch);
-        const epochBoundaryRoot = epochBoundarySlot === state.slot ? fromHex(blockRootHex) : getBlockRootAtSlot(state, epochBoundarySlot);
+        const epochBoundaryRoot = epochBoundarySlot === state.slot ? fromHex(blockRootHex) : state.getBlockRootAtSlot(epochBoundarySlot);
         const epochBoundaryHex = toRootHex(epochBoundaryRoot);
-        const prevEpochRoot = toRootHex(getBlockRootAtSlot(state, epochBoundarySlot - 1));
+        const prevEpochRoot = toRootHex(state.getBlockRootAtSlot(epochBoundarySlot - 1));
         // for each epoch, usually there are 2 rootHexes respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State
-        const cpRootHexes = this.epochIndex.get(epoch) ?? [];
+        const cpRootHexMap = this.epochIndex.get(epoch) ?? new Map();
         const persistedRootHexes = new Set();
         // 1) if there is no CRCS, persist PRCS (block 0 of epoch is skipped). In this case prevEpochRoot === epochBoundaryHex
         // 2) if there are PRCS and CRCS, persist CRCS => persist CRCS
         // => this is simplified to always persist epochBoundaryHex
         persistedRootHexes.add(epochBoundaryHex);
         // 3) persist any states with unknown roots to this state
-        for (const rootHex of cpRootHexes) {
+        for (const rootHex of cpRootHexMap.keys()) {
             if (rootHex !== epochBoundaryHex && rootHex !== prevEpochRoot) {
                 persistedRootHexes.add(rootHex);
             }
         }
-        for (const rootHex of cpRootHexes) {
-            const cpKey = toCacheKey({ epoch: epoch, rootHex });
-            const cacheItem = this.cache.get(cpKey);
-            if (cacheItem !== undefined && isInMemoryCacheItem(cacheItem)) {
-                let { persistedKey } = cacheItem;
-                const { state } = cacheItem;
-                const logMeta = {
-                    stateSlot: state.slot,
-                    rootHex,
-                    epochBoundaryHex,
-                    persistedKey: persistedKey ? toHex(persistedKey) : "",
-                };
-                if (persistedRootHexes.has(rootHex)) {
-                    if (persistedKey) {
-                        // we don't care if the checkpoint state is already persisted
-                        this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta);
-                    }
-                    else {
-                        // persist and do not update epochIndex
-                        this.metrics?.cpStateCache.statePersistSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0);
-                        const cpPersist = { epoch: epoch, root: fromHex(rootHex) };
-                        // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory.
-                        // As monitored on holesky as of Jan 2024:
-                        //   - This does not increase heap allocation while gc time is the same
-                        //   - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s)
-                        //   - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s)
-                        //   - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization
-                        const timer = this.metrics?.stateSerializeDuration.startTimer({
-                            source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE,
-                        });
-                        persistedKey = await serializeState(state, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, (stateBytes) => {
-                            timer?.();
-                            return this.datastore.write(cpPersist, stateBytes);
-                        }, this.bufferPool);
-                        persistCount++;
-                        this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", {
-                            ...logMeta,
-                            persistedKey: toHex(persistedKey),
-                        });
-                    }
-                    // overwrite cpKey, this means the state is deleted from memory
-                    this.cache.set(cpKey, { type: CacheItemType.persisted, value: persistedKey });
-                }
-                else {
-                    if (persistedKey) {
-                        // persisted file will be eventually deleted by the archive task
-                        // this also means the state is deleted from memory
+        for (const [rootHex, bitmask] of cpRootHexMap) {
+            for (const flag of PAYLOAD_AVAILABILITY_ALL) {
+                if (!(bitmask & flag))
+                    continue;
+                const payloadPresent = fromPayloadAvailability(flag);
+                const cpKey = toCacheKey({ epoch: epoch, rootHex, payloadPresent });
+                const cacheItem = this.cache.get(cpKey);
+                if (cacheItem !== undefined && isInMemoryCacheItem(cacheItem)) {
+                    let { persistedKey } = cacheItem;
+                    const { state } = cacheItem;
+                    const logMeta = {
+                        stateSlot: state.slot,
+                        rootHex,
+                        payloadPresent,
+                        epochBoundaryHex,
+                        persistedKey: persistedKey ? toHex(persistedKey) : "",
+                    };
+                    if (persistedRootHexes.has(rootHex)) {
+                        if (persistedKey) {
+                            // we don't care if the checkpoint state is already persisted
+                            this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta);
+                        }
+                        else {
+                            // persist and do not update epochIndex
+                            this.metrics?.cpStateCache.statePersistSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0);
+                            const cpPersist = { epoch: epoch, root: fromHex(rootHex) };
+                            // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory.
+                            // As monitored on holesky as of Jan 2024:
+                            //   - This does not increase heap allocation while gc time is the same
+                            //   - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s)
+                            //   - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s)
+                            //   - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization
+                            const timer = this.metrics?.stateSerializeDuration.startTimer({
+                                source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE,
+                            });
+                            persistedKey = await serializeState(state, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, (stateBytes) => {
+                                timer?.();
+                                return this.datastore.write(cpPersist, stateBytes, payloadPresent);
+                            }, this.bufferPool);
+                            persistCount++;
+                            this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", {
+                                ...logMeta,
+                                persistedKey: toHex(persistedKey),
+                            });
+                        }
+                        // overwrite cpKey, this means the state is deleted from memory
                         this.cache.set(cpKey, { type: CacheItemType.persisted, value: persistedKey });
-                        // do not update epochIndex
                     }
                     else {
-                        // delete the state from memory
-                        this.cache.delete(cpKey);
-                        this.epochIndex.get(epoch)?.delete(rootHex);
+                        if (persistedKey) {
+                            // persisted file will be eventually deleted by the archive task
+                            // this also means the state is deleted from memory
+                            this.cache.set(cpKey, { type: CacheItemType.persisted, value: persistedKey });
+                            // do not update epochIndex
+                        }
+                        else {
+                            // delete the state from memory
+                            this.cache.delete(cpKey);
+                            this.removeFromEpochIndex(epoch, rootHex, payloadPresent);
+                        }
+                        this.metrics?.cpStateCache.statePruneFromMemoryCount.inc();
+                        this.logger.verbose("Pruned checkpoint state from memory", logMeta);
                     }
-                    this.metrics?.cpStateCache.statePruneFromMemoryCount.inc();
-                    this.logger.verbose("Pruned checkpoint state from memory", logMeta);
                 }
             }
         }
@@ -707,25 +769,38 @@ export class PersistentCheckpointStateCache {
      */
     async deleteAllEpochItems(epoch) {
         let persistCount = 0;
-        const rootHexes = this.epochIndex.get(epoch) || [];
-        for (const rootHex of rootHexes) {
-            const key = toCacheKey({ rootHex, epoch });
-            const cacheItem = this.cache.get(key);
-            if (cacheItem) {
-                const persistedKey = isPersistedCacheItem(cacheItem) ? cacheItem.value : cacheItem.persistedKey;
-                if (persistedKey) {
-                    await this.datastore.remove(persistedKey);
-                    persistCount++;
-                    this.metrics?.cpStateCache.persistedStateRemoveCount.inc();
+        const rootHexMap = this.epochIndex.get(epoch) || new Map();
+        for (const [rootHex, bitmask] of rootHexMap) {
+            for (const flag of PAYLOAD_AVAILABILITY_ALL) {
+                if (!(bitmask & flag))
+                    continue;
+                const payloadPresent = fromPayloadAvailability(flag);
+                const key = toCacheKey({ rootHex, epoch, payloadPresent });
+                const cacheItem = this.cache.get(key);
+                if (cacheItem) {
+                    const persistedKey = isPersistedCacheItem(cacheItem) ? cacheItem.value : cacheItem.persistedKey;
+                    if (persistedKey) {
+                        await this.datastore.remove(persistedKey);
+                        persistCount++;
+                        this.metrics?.cpStateCache.persistedStateRemoveCount.inc();
+                    }
                 }
+                this.cache.delete(key);
+                this.logger.verbose("Pruned checkpoint state", {
+                    epoch,
+                    rootHex,
+                    payloadPresent,
+                    type: cacheItem ? (isPersistedCacheItem(cacheItem) ? "persisted" : "in-memory") : "missing",
+                });
             }
-            this.cache.delete(key);
         }
         this.epochIndex.delete(epoch);
-        this.logger.verbose("Pruned checkpoint states for epoch", {
+        this.logger.verbose("Pruned all checkpoint states for epoch", {
             epoch,
             persistCount,
-            rootHexes: Array.from(rootHexes).join(","),
+            items: Array.from(rootHexMap.entries())
+                .flatMap(([rootHex, bitmask]) => PAYLOAD_AVAILABILITY_ALL.filter((f) => bitmask & f).map((f) => `${rootHex}:${fromPayloadAvailability(f)}`))
+                .join(","),
         });
     }
     /**
@@ -752,43 +827,65 @@ export class PersistentCheckpointStateCache {
      * This is 2x - 3x faster than allocating memory every time.
      */
     serializeStateValidators(state) {
-        const type = state.type.fields.validators;
-        const size = type.tree_serializedSize(state.validators.node);
+        const size = state.serializedValidatorsSize();
         if (this.bufferPool) {
             const bufferWithKey = this.bufferPool.alloc(size, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_VALIDATORS);
             if (bufferWithKey) {
                 const validatorsBytes = bufferWithKey.buffer;
                 const dataView = new DataView(validatorsBytes.buffer, validatorsBytes.byteOffset, validatorsBytes.byteLength);
-                state.validators.serializeToBytes({ uint8Array: validatorsBytes, dataView }, 0);
+                state.serializeValidatorsToBytes({ uint8Array: validatorsBytes, dataView }, 0);
                 return bufferWithKey;
             }
         }
         return null;
     }
 }
-export function toCheckpointHex(checkpoint) {
+export function toCheckpointHexPayload(checkpoint, payloadPresent) {
     return {
         epoch: checkpoint.epoch,
         rootHex: toRootHex(checkpoint.root),
+        payloadPresent,
+    };
+}
+/**
+ * Convert fork-choice CheckpointWithPayloadStatus to beacon-node CheckpointHexPayload.
+ * Maps PayloadStatus enum to boolean payloadPresent.
+ * @throws Error if checkpoint has PENDING payload status (ambiguous which variant to use)
+ */
+export function fcCheckpointToHexPayload(checkpoint) {
+    const PayloadStatus = { PENDING: 0, EMPTY: 1, FULL: 2 };
+    if (checkpoint.payloadStatus === PayloadStatus.PENDING) {
+        throw Error(`Cannot convert checkpoint with PENDING payload status at epoch ${checkpoint.epoch} root ${checkpoint.rootHex}`);
+    }
+    return {
+        epoch: checkpoint.epoch,
+        rootHex: checkpoint.rootHex,
+        payloadPresent: checkpoint.payloadStatus === PayloadStatus.FULL,
     };
 }
 export function toCheckpointKey(cp) {
-    return `${cp.rootHex}:${cp.epoch}`;
+    return `${cp.rootHex}:${cp.epoch}:${cp.payloadPresent}`;
 }
+/**
+ * Convert checkpoint to cache key string.
+ * Format: `{rootHex}_{epoch}_{payloadPresent}`
+ */
 function toCacheKey(cp) {
-    if (isCheckpointHex(cp)) {
-        return `${cp.rootHex}_${cp.epoch}`;
-    }
-    return `${toRootHex(cp.root)}_${cp.epoch}`;
+    return `${cp.rootHex}_${cp.epoch}_${cp.payloadPresent}`;
 }
 function fromCacheKey(key) {
-    const [rootHex, epoch] = key.split("_");
+    const parts = key.split("_");
+    const rootHex = parts[0];
+    const epoch = Number(parts[1]);
+    // For backward compatibility with old format (rootHex_epoch), default to true
+    const payloadPresent = parts.length > 2 ? parts[2] === "true" : true;
     return {
         rootHex,
-        epoch: Number(epoch),
+        epoch,
+        payloadPresent,
     };
 }
-function isCachedBeaconState(stateOrBytes) {
+function isBeaconStateView(stateOrBytes) {
     return stateOrBytes.slot !== undefined;
 }
 function isInMemoryCacheItem(cacheItem) {
@@ -797,7 +894,4 @@ function isInMemoryCacheItem(cacheItem) {
 function isPersistedCacheItem(cacheItem) {
     return cacheItem.type === CacheItemType.persisted;
 }
-function isCheckpointHex(cp) {
-    return cp.rootHex !== undefined;
-}
 //# sourceMappingURL=persistentCheckpointsCache.js.map