@lodestar/beacon-node 1.41.0-dev.253ed58dd1 → 1.41.0-dev.274a9918a8

package/src/chain/produceBlock/produceBlockBody.ts

@@ -4,6 +4,7 @@ import {
   BUILDER_INDEX_SELF_BUILD,
   ForkName,
   ForkPostBellatrix,
+  ForkPostCapella,
   ForkPostDeneb,
   ForkPostFulu,
   ForkPostGloas,
@@ -510,9 +511,16 @@ export async function produceBlockBody<T extends BlockType>(
           // NOTE: Even though the fulu.BlobsBundle type is superficially the same as deneb.BlobsBundle, it is NOT.
           // In fulu, proofs are _cell_ proofs, vs in deneb they are _blob_ proofs.
 
+          const timer = this?.metrics?.peerDas.dataColumnSidecarComputationTime.startTimer();
           const cells = blobsBundle.blobs.map((blob) => kzg.computeCells(blob));
+          timer?.();
           if (this.opts.sanityCheckExecutionEngineBlobs) {
-            await validateCellsAndKzgCommitments(blobsBundle.commitments, blobsBundle.proofs, cells);
+            const validationTimer = this.metrics?.peerDas.kzgVerificationDataColumnBatchTime.startTimer();
+            try {
+              await validateCellsAndKzgCommitments(blobsBundle.commitments, blobsBundle.proofs, cells);
+            } finally {
+              validationTimer?.();
+            }
           }
 
           (blockBody as deneb.BeaconBlockBody).blobKzgCommitments = blobsBundle.commitments;
@@ -567,8 +575,14 @@ export async function produceBlockBody<T extends BlockType>(
     });
   }
 
-  if (ForkSeq[fork] >= ForkSeq.capella) {
-    const {blsToExecutionChanges, executionPayload} = blockBody as capella.BeaconBlockBody;
+  if (ForkSeq[fork] >= ForkSeq.gloas) {
+    const {blsToExecutionChanges, payloadAttestations} = blockBody as BeaconBlockBody<ForkPostGloas>;
+    Object.assign(logMeta, {
+      blsToExecutionChanges: blsToExecutionChanges.length,
+      payloadAttestations: payloadAttestations.length,
+    });
+  } else if (ForkSeq[fork] >= ForkSeq.capella) {
+    const {blsToExecutionChanges, executionPayload} = blockBody as BeaconBlockBody<ForkPostCapella & ForkPreGloas>;
     Object.assign(logMeta, {
       blsToExecutionChanges: blsToExecutionChanges.length,
     });
@@ -716,8 +730,7 @@ export function getPayloadAttributesForSSE(
 
   let parentBlockNumber: number;
   if (isForkPostGloas(fork)) {
-    // TODO GLOAS: revisit this after fork choice changes are merged
-    const parentBlock = chain.forkChoice.getBlock(parentBlockRoot);
+    const parentBlock = chain.forkChoice.getBlockHexAndBlockHash(toRootHex(parentBlockRoot), toRootHex(parentHash));
     if (parentBlock?.executionPayloadBlockHash == null) {
       throw Error(`Parent block not found in fork choice root=${toRootHex(parentBlockRoot)}`);
     }

package/src/chain/regen/errors.ts

@@ -1,3 +1,4 @@
+import {PayloadStatus} from "@lodestar/fork-choice";
 import {Root, RootHex, Slot} from "@lodestar/types";
 
 export enum RegenErrorCode {
@@ -9,6 +10,8 @@ export enum RegenErrorCode {
   BLOCK_NOT_IN_DB = "REGEN_ERROR_BLOCK_NOT_IN_DB",
   STATE_TRANSITION_ERROR = "REGEN_ERROR_STATE_TRANSITION_ERROR",
   INVALID_STATE_ROOT = "REGEN_ERROR_INVALID_STATE_ROOT",
+  UNEXPECTED_PAYLOAD_STATUS = "REGEN_ERROR_UNEXPECTED_PAYLOAD_STATUS",
+  INTERNAL_ERROR = "REGEN_ERROR_INTERNAL_ERROR",
 }
 
 export type RegenErrorType =
@@ -19,7 +22,9 @@ export type RegenErrorType =
   | {code: RegenErrorCode.TOO_MANY_BLOCK_PROCESSED; stateRoot: RootHex | Root}
   | {code: RegenErrorCode.BLOCK_NOT_IN_DB; blockRoot: RootHex | Root}
   | {code: RegenErrorCode.STATE_TRANSITION_ERROR; error: Error}
-  | {code: RegenErrorCode.INVALID_STATE_ROOT; slot: Slot; expected: RootHex; actual: RootHex};
+  | {code: RegenErrorCode.INVALID_STATE_ROOT; slot: Slot; expected: RootHex; actual: RootHex}
+  | {code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS; blockRoot: RootHex | Root; payloadStatus: PayloadStatus}
+  | {code: RegenErrorCode.INTERNAL_ERROR; message: string};
 
 export class RegenError extends Error {
   type: RegenErrorType;

package/src/chain/regen/interface.ts

@@ -2,7 +2,7 @@ import {routes} from "@lodestar/api";
 import {ProtoBlock} from "@lodestar/fork-choice";
 import {CachedBeaconStateAllForks} from "@lodestar/state-transition";
 import {BeaconBlock, Epoch, RootHex, Slot, phase0} from "@lodestar/types";
-import {CheckpointHex} from "../stateCache/types.js";
+import {CheckpointHexPayload} from "../stateCache/types.js";
 
 export enum RegenCaller {
   getDuties = "getDuties",
@@ -38,15 +38,21 @@ export interface IStateRegenerator extends IStateRegeneratorInternal {
   dumpCacheSummary(): routes.lodestar.StateCacheItem[];
   getStateSync(stateRoot: RootHex): CachedBeaconStateAllForks | null;
   getPreStateSync(block: BeaconBlock): CachedBeaconStateAllForks | null;
-  getCheckpointStateOrBytes(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | Uint8Array | null>;
-  getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null;
+  getCheckpointStateOrBytes(cp: CheckpointHexPayload): Promise<CachedBeaconStateAllForks | Uint8Array | null>;
+  getCheckpointStateSync(cp: CheckpointHexPayload): CachedBeaconStateAllForks | null;
   getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null;
   pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void;
   pruneOnFinalized(finalizedEpoch: Epoch): void;
-  processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void;
-  addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void;
+  processBlockState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void;
+  processPayloadState(payloadState: CachedBeaconStateAllForks): void;
+  /**
+   * payloadPresent is true if this is payload state, false if block state.
+   * payloadPresent is always true for pre-gloas.
+   */
+  addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks, payloadPresent: boolean): void;
   updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void;
-  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null;
+  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null;
+  upgradeForGloas(epoch: Epoch): void;
 }
 
 /**

package/src/chain/regen/queued.ts

@@ -1,11 +1,11 @@
 import {routes} from "@lodestar/api";
-import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
+import {IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
 import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition";
-import {BeaconBlock, Epoch, RootHex, Slot, phase0} from "@lodestar/types";
-import {Logger, toRootHex} from "@lodestar/utils";
+import {BeaconBlock, Epoch, RootHex, Slot, isGloasBeaconBlock, phase0} from "@lodestar/types";
+import {Logger, fromHex, toRootHex} from "@lodestar/utils";
 import {Metrics} from "../../metrics/index.js";
 import {JobItemQueue} from "../../util/queue/index.js";
-import {BlockStateCache, CheckpointHex, CheckpointStateCache} from "../stateCache/types.js";
+import {BlockStateCache, CheckpointHexPayload, CheckpointStateCache} from "../stateCache/types.js";
 import {RegenError, RegenErrorCode} from "./errors.js";
 import {
   IStateRegenerator,
@@ -88,7 +88,12 @@ export class QueuedStateRegenerator implements IStateRegenerator {
    */
   getPreStateSync(block: BeaconBlock): CachedBeaconStateAllForks | null {
     const parentRoot = toRootHex(block.parentRoot);
-    const parentBlock = this.forkChoice.getBlockHex(parentRoot);
+    const parentBlock = isGloasBeaconBlock(block)
+      ? this.forkChoice.getBlockHexAndBlockHash(
+          parentRoot,
+          toRootHex(block.body.signedExecutionPayloadBid.message.parentBlockHash)
+        )
+      : this.forkChoice.getBlockHexDefaultStatus(parentRoot);
     if (!parentBlock) {
       throw new RegenError({
         code: RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE,
@@ -99,9 +104,19 @@ export class QueuedStateRegenerator implements IStateRegenerator {
     const parentEpoch = computeEpochAtSlot(parentBlock.slot);
     const blockEpoch = computeEpochAtSlot(block.slot);
 
+    // Convert PayloadStatus to payloadPresent boolean
+    if (parentBlock.payloadStatus === PayloadStatus.PENDING) {
+      throw new RegenError({
+        code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS,
+        blockRoot: block.parentRoot,
+        payloadStatus: parentBlock.payloadStatus,
+      });
+    }
+    const payloadPresent = parentBlock.payloadStatus === PayloadStatus.FULL;
+
     // Check the checkpoint cache (if the pre-state is a checkpoint state)
     if (parentEpoch < blockEpoch) {
-      const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch);
+      const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch, payloadPresent);
       if (checkpointState && computeEpochAtSlot(checkpointState.slot) === blockEpoch) {
         return checkpointState;
       }
@@ -120,14 +135,14 @@ export class QueuedStateRegenerator implements IStateRegenerator {
     return null;
   }
 
-  async getCheckpointStateOrBytes(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | Uint8Array | null> {
+  async getCheckpointStateOrBytes(cp: CheckpointHexPayload): Promise<CachedBeaconStateAllForks | Uint8Array | null> {
     return this.checkpointStateCache.getStateOrBytes(cp);
   }
 
   /**
    * Get checkpoint state from cache
    */
-  getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null {
+  getCheckpointStateSync(cp: CheckpointHexPayload): CachedBeaconStateAllForks | null {
     return this.checkpointStateCache.get(cp);
   }
 
@@ -135,7 +150,19 @@ export class QueuedStateRegenerator implements IStateRegenerator {
    * Get state closest to head
    */
   getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null {
-    return this.checkpointStateCache.getLatest(head.blockRoot, Infinity) || this.blockStateCache.get(head.stateRoot);
+    // Convert PayloadStatus to payloadPresent boolean
+    if (head.payloadStatus === PayloadStatus.PENDING) {
+      throw new RegenError({
+        code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS,
+        blockRoot: fromHex(head.blockRoot),
+        payloadStatus: head.payloadStatus,
+      });
+    }
+    const payloadPresent = head.payloadStatus === PayloadStatus.FULL;
+    return (
+      this.checkpointStateCache.getLatest(head.blockRoot, Infinity, payloadPresent) ||
+      this.blockStateCache.get(head.stateRoot)
+    );
   }
 
   pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void {
@@ -148,15 +175,24 @@ export class QueuedStateRegenerator implements IStateRegenerator {
     this.blockStateCache.deleteAllBeforeEpoch(finalizedEpoch);
   }
 
-  processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void {
+  processBlockState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void {
     this.blockStateCache.add(postState);
     this.checkpointStateCache.processState(blockRootHex, postState).catch((e) => {
       this.logger.debug("Error processing block state", {blockRootHex, slot: postState.slot}, e);
     });
   }
 
-  addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void {
-    this.checkpointStateCache.add(cp, item);
+  /**
+   * Process payload state for caching after importing execution payload.
+   */
+  processPayloadState(payloadState: CachedBeaconStateAllForks): void {
+    // Add payload state to block state cache (keyed by payload state root)
+    this.blockStateCache.add(payloadState);
+  }
+
+  // TODO GLOAS: This should also be called when importing execution payload after we implement it
+  addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks, payloadPresent: boolean): void {
+    this.checkpointStateCache.add(cp, item, payloadPresent);
   }
 
   updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void {
@@ -192,8 +228,13 @@ export class QueuedStateRegenerator implements IStateRegenerator {
     }
   }
 
-  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null {
-    return this.checkpointStateCache.updatePreComputedCheckpoint(rootHex, epoch);
+  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch, payloadPresent: boolean): number | null {
+    return this.checkpointStateCache.updatePreComputedCheckpoint(rootHex, epoch, payloadPresent);
+  }
+
+  upgradeForGloas(epoch: Epoch): void {
+    this.logger.verbose("Upgrading block state cache for Gloas fork", {epoch});
+    this.blockStateCache.upgradeToGloas();
   }
 
   /**

package/src/chain/regen/regen.ts

@@ -1,6 +1,6 @@
 import {ChainForkConfig} from "@lodestar/config";
-import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
-import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import {IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
+import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {
   CachedBeaconStateAllForks,
   DataAvailabilityStatus,
@@ -11,7 +11,7 @@ import {
   processSlots,
   stateTransition,
 } from "@lodestar/state-transition";
-import {BeaconBlock, RootHex, SignedBeaconBlock, Slot} from "@lodestar/types";
+import {BeaconBlock, RootHex, SignedBeaconBlock, Slot, isGloasBeaconBlock} from "@lodestar/types";
 import {Logger, fromHex, toRootHex} from "@lodestar/utils";
 import {IBeaconDb} from "../../db/index.js";
 import {Metrics} from "../../metrics/index.js";
@@ -58,7 +58,13 @@ export class StateRegenerator implements IStateRegeneratorInternal {
     opts: StateRegenerationOpts,
     regenCaller: RegenCaller
   ): Promise<CachedBeaconStateAllForks> {
-    const parentBlock = this.modules.forkChoice.getBlock(block.parentRoot);
+    const parentRoot = toRootHex(block.parentRoot);
+    const parentBlock = isGloasBeaconBlock(block)
+      ? this.modules.forkChoice.getBlockHexAndBlockHash(
+          parentRoot,
+          toRootHex(block.body.signedExecutionPayloadBid.message.parentBlockHash)
+        )
+      : this.modules.forkChoice.getBlockHexDefaultStatus(parentRoot);
     if (!parentBlock) {
       throw new RegenError({
         code: RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE,
@@ -105,9 +111,20 @@ export class StateRegenerator implements IStateRegeneratorInternal {
     const {blockRoot} = block;
     const {checkpointStateCache} = this.modules;
     const epoch = computeEpochAtSlot(slot);
+
+    // Convert PayloadStatus to payloadPresent boolean
+    if (block.payloadStatus === PayloadStatus.PENDING) {
+      throw new RegenError({
+        code: RegenErrorCode.UNEXPECTED_PAYLOAD_STATUS,
+        blockRoot: fromHex(blockRoot),
+        payloadStatus: block.payloadStatus,
+      });
+    }
+    const payloadPresent = block.payloadStatus === PayloadStatus.FULL;
+
     const latestCheckpointStateCtx = allowDiskReload
-      ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch)
-      : checkpointStateCache.getLatest(blockRoot, epoch);
+      ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch, payloadPresent)
+      : checkpointStateCache.getLatest(blockRoot, epoch, payloadPresent);
 
     // If a checkpoint state exists with the given checkpoint root, it either is in requested epoch
     // or needs to have empty slots processed until the requested epoch
@@ -152,7 +169,7 @@ export class StateRegenerator implements IStateRegeneratorInternal {
 
     const getSeedStateTimer = this.modules.metrics?.regenGetState.getSeedState.startTimer({caller});
     // iterateAncestorBlocks only returns ancestor blocks, not the block itself
-    for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot)) {
+    for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot, block.payloadStatus)) {
       state = this.modules.blockStateCache.get(b.stateRoot);
       if (state) {
         break;
@@ -160,9 +177,19 @@ export class StateRegenerator implements IStateRegeneratorInternal {
       const lastBlockToReplay = blocksToReplay.at(-1);
       if (!lastBlockToReplay) continue;
       const epoch = computeEpochAtSlot(lastBlockToReplay.slot - 1);
+
+      // Convert PayloadStatus to payloadPresent boolean
+      if (b.payloadStatus === PayloadStatus.PENDING) {
+        throw new RegenError({
+          code: RegenErrorCode.INTERNAL_ERROR,
+          message: `Unexpected PENDING payloadStatus for ancestor block ${b.blockRoot} at slot ${b.slot}`,
+        });
+      }
+      const payloadPresent = b.payloadStatus === PayloadStatus.FULL;
+
       state = allowDiskReload
-        ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch)
-        : checkpointStateCache.getLatest(b.blockRoot, epoch);
+        ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch, payloadPresent)
+        : checkpointStateCache.getLatest(b.blockRoot, epoch, payloadPresent);
       if (state) {
         break;
       }
@@ -326,6 +353,11 @@ async function processSlotsByCheckpoint(
  * emitting "checkpoint" events after every epoch processed.
  *
  * Stops processing after no more full epochs can be processed.
+ *
+ * Output state variant:
+ * - Post-Gloas: If slots are processed, returns block state (payloadPresent=false).
+ *               If no slots processed, returns preState as-is (preserves variant).
+ * - Pre-Gloas: Always payloadPresent=true (no block/payload distinction).
  */
 export async function processSlotsToNearestCheckpoint(
   modules: {
@@ -368,7 +400,11 @@ export async function processSlotsToNearestCheckpoint(
     // This may becomes the "official" checkpoint state if the 1st block of epoch is skipped
     const checkpointState = postState;
     const cp = getCheckpointFromState(checkpointState);
-    checkpointStateCache.add(cp, checkpointState);
+    // processSlots() only does epoch transitions, never processes payloads
+    // Pre-Gloas: payloadPresent is always true (execution payload embedded in block)
+    // Post-Gloas: result is a block state (payloadPresent=false)
+    const isPayloadPresent = checkpointState.config.getForkSeq(checkpointState.slot) < ForkSeq.gloas;
+    checkpointStateCache.add(cp, checkpointState, isPayloadPresent);
     // consumers should not mutate state ever
     emitter?.emit(ChainEvent.checkpoint, cp, checkpointState);
 

package/src/chain/seenCache/seenGossipBlockInput.ts

@@ -3,6 +3,7 @@ import {CheckpointWithHex} from "@lodestar/fork-choice";
 import {
   ForkName,
   ForkPostFulu,
+  ForkPostGloas,
   ForkPreGloas,
   SLOTS_PER_EPOCH,
   isForkPostDeneb,
@@ -16,10 +17,12 @@ import {Metrics} from "../../metrics/metrics.js";
 import {MAX_LOOK_AHEAD_EPOCHS} from "../../sync/constants.js";
 import {IClock} from "../../util/clock.js";
 import {CustodyConfig} from "../../util/dataColumns.js";
+import {SerializedCache} from "../../util/serializedCache.js";
 import {
   BlockInput,
   BlockInputBlobs,
   BlockInputColumns,
+  BlockInputNoData,
   BlockInputPreData,
   BlockWithSource,
   DAType,
@@ -53,6 +56,7 @@ export type SeenBlockInputCacheModules = {
   chainEvents: ChainEventEmitter;
   signal: AbortSignal;
   custodyConfig: CustodyConfig;
+  serializedCache: SerializedCache;
   metrics: Metrics | null;
   logger?: Logger;
 };
@@ -99,6 +103,7 @@ export class SeenBlockInput {
   private readonly clock: IClock;
   private readonly chainEvents: ChainEventEmitter;
   private readonly signal: AbortSignal;
+  private readonly serializedCache: SerializedCache;
   private readonly metrics: Metrics | null;
   private readonly logger?: Logger;
   private blockInputs = new Map<RootHex, IBlockInput>();
@@ -107,19 +112,35 @@ export class SeenBlockInput {
   // and the signature to ensure we only skip verification if both match
   private verifiedProposerSignatures = new Map<Slot, Map<RootHex, BLSSignature>>();
 
-  constructor({config, custodyConfig, clock, chainEvents, signal, metrics, logger}: SeenBlockInputCacheModules) {
+  constructor({
+    config,
+    custodyConfig,
+    clock,
+    chainEvents,
+    signal,
+    serializedCache,
+    metrics,
+    logger,
+  }: SeenBlockInputCacheModules) {
     this.config = config;
     this.custodyConfig = custodyConfig;
     this.clock = clock;
     this.chainEvents = chainEvents;
     this.signal = signal;
+    this.serializedCache = serializedCache;
     this.metrics = metrics;
     this.logger = logger;
 
     if (metrics) {
-      metrics.seenCache.blockInput.blockInputCount.addCollect(() =>
-        metrics.seenCache.blockInput.blockInputCount.set(this.blockInputs.size)
-      );
+      metrics.seenCache.blockInput.blockInputCount.addCollect(() => {
+        metrics.seenCache.blockInput.blockInputCount.set(this.blockInputs.size);
+        metrics.seenCache.blockInput.serializedObjectRefs.set(
+          Array.from(this.blockInputs.values()).reduce(
+            (count, blockInput) => count + blockInput.getSerializedCacheKeys().length,
+            0
+          )
+        );
+      });
     }
 
     this.chainEvents.on(ChainEvent.forkChoiceFinalized, this.onFinalized);
@@ -140,7 +161,10 @@ export class SeenBlockInput {
    * Removes the single BlockInput from the cache
    */
   remove(rootHex: RootHex): void {
-    this.blockInputs.delete(rootHex);
+    const blockInput = this.blockInputs.get(rootHex);
+    if (blockInput) {
+      this.evictBlockInput(blockInput);
+    }
   }
 
   /**
@@ -152,7 +176,7 @@ export class SeenBlockInput {
     let deletedCount = 0;
     while (blockInput) {
       deletedCount++;
-      this.blockInputs.delete(blockInput.blockRootHex);
+      this.evictBlockInput(blockInput);
       blockInput = this.blockInputs.get(parentRootHex ?? "");
       parentRootHex = blockInput?.parentRootHex;
     }
@@ -163,10 +187,10 @@ export class SeenBlockInput {
   onFinalized = (checkpoint: CheckpointWithHex) => {
     let deletedCount = 0;
     const cutoffSlot = computeStartSlotAtEpoch(checkpoint.epoch);
-    for (const [rootHex, blockInput] of this.blockInputs) {
+    for (const [, blockInput] of this.blockInputs) {
       if (blockInput.slot < cutoffSlot) {
         deletedCount++;
-        this.blockInputs.delete(rootHex);
+        this.evictBlockInput(blockInput);
       }
     }
     this.logger?.debug(`BlockInputCache.onFinalized deleted ${deletedCount} cached BlockInputs`);
@@ -179,12 +203,19 @@ export class SeenBlockInput {
     if (!blockInput) {
       const {forkName, daOutOfRange} = this.buildCommonProps(block.message.slot);
 
-      // TODO GLOAS: Implement
       if (isForkPostGloas(forkName)) {
-        throw Error("Not implemented");
-      }
-      // Pre-deneb
-      if (!isForkPostDeneb(forkName)) {
+        // Post-gloas
+        blockInput = BlockInputNoData.createFromBlock({
+          block: block as SignedBeaconBlock<ForkPostGloas>,
+          blockRootHex,
+          daOutOfRange,
+          forkName,
+          source,
+          seenTimestampSec,
+          peerIdStr,
+        });
+      } else if (!isForkPostDeneb(forkName)) {
+        // Pre-deneb
         blockInput = BlockInputPreData.createFromBlock({
           block,
           blockRootHex,
@@ -194,8 +225,8 @@ export class SeenBlockInput {
           seenTimestampSec,
           peerIdStr,
         });
-        // Fulu Only
       } else if (isForkPostFulu(forkName)) {
+        // Fulu Only
         blockInput = BlockInputColumns.createFromBlock({
           block: block as SignedBeaconBlock<ForkPostFulu & ForkPreGloas>,
           blockRootHex,
@@ -207,8 +238,8 @@ export class SeenBlockInput {
           seenTimestampSec,
           peerIdStr,
         });
-        // Deneb and Electra
       } else {
+        // Deneb and Electra
         blockInput = BlockInputBlobs.createFromBlock({
           block: block as SignedBeaconBlock<ForkBlobsDA>,
           blockRootHex,
@@ -219,6 +250,7 @@ export class SeenBlockInput {
           peerIdStr,
         });
       }
+      this.metrics?.seenCache.blockInput.createdByBlock.inc();
       this.blockInputs.set(blockInput.blockRootHex, blockInput);
     }
 
@@ -318,7 +350,7 @@ export class SeenBlockInput {
         custodyColumns: this.custodyConfig.custodyColumns,
         sampledColumns: this.custodyConfig.sampledColumns,
       });
-      this.metrics?.seenCache.blockInput.createdByBlob.inc();
+      this.metrics?.seenCache.blockInput.createdByColumn.inc();
       this.blockInputs.set(blockRootHex, blockInput);
     }
 
@@ -399,14 +431,20 @@ export class SeenBlockInput {
 
     if (itemsToDelete > 0) {
       const sorted = [...this.blockInputs.entries()].sort((a, b) => a[1].slot - b[1].slot);
-      for (const [rootHex] of sorted) {
-        this.blockInputs.delete(rootHex);
+      for (const [, blockInput] of sorted) {
+        this.evictBlockInput(blockInput);
         itemsToDelete--;
         if (itemsToDelete <= 0) return;
       }
     }
     pruneSetToMax(this.verifiedProposerSignatures, MAX_BLOCK_INPUT_CACHE_SIZE);
   }
+
+  private evictBlockInput(blockInput: IBlockInput): void {
+    // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable
+    this.serializedCache.delete(blockInput.getSerializedCacheKeys());
+    this.blockInputs.delete(blockInput.blockRootHex);
+  }
 }
 
 enum SeenBlockInputCacheErrorCode {

package/src/chain/stateCache/datastore/db.ts

@@ -1,6 +1,6 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {Epoch, phase0, ssz} from "@lodestar/types";
-import {MapDef} from "@lodestar/utils";
+import {MapDef, byteArrayEquals} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/interface.js";
 import {
   getLastProcessedSlotFromBeaconStateSerialized,
@@ -14,8 +14,8 @@ import {CPStateDatastore, DatastoreKey} from "./types.js";
 export class DbCPStateDatastore implements CPStateDatastore {
   constructor(private readonly db: IBeaconDb) {}
 
-  async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise<DatastoreKey> {
-    const serializedCheckpoint = checkpointToDatastoreKey(cpKey);
+  async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean): Promise<DatastoreKey> {
+    const serializedCheckpoint = checkpointToDatastoreKey(cpKey, payloadPresent);
     await this.db.checkpointState.putBinary(serializedCheckpoint, stateBytes);
     return serializedCheckpoint;
   }
@@ -40,18 +40,30 @@ export class DbCPStateDatastore implements CPStateDatastore {
   }
 }
 
+function extractCheckpointBytes(key: DatastoreKey): Uint8Array {
+  const fixedSize = ssz.phase0.Checkpoint.minSize;
+  return key.subarray(0, fixedSize);
+}
+
 export function datastoreKeyToCheckpoint(key: DatastoreKey): phase0.Checkpoint {
-  return ssz.phase0.Checkpoint.deserialize(key);
+  return ssz.phase0.Checkpoint.deserialize(extractCheckpointBytes(key));
+}
+
+export function checkpointToDatastoreKey(cp: phase0.Checkpoint, payloadPresent: boolean): DatastoreKey {
+  const cpBytes = ssz.phase0.Checkpoint.serialize(cp);
+  const key = new Uint8Array(cpBytes.length + 1);
+  key.set(cpBytes);
+  key[cpBytes.length] = payloadPresent ? 1 : 0;
+  return key;
 }
 
-export function checkpointToDatastoreKey(cp: phase0.Checkpoint): DatastoreKey {
-  return ssz.phase0.Checkpoint.serialize(cp);
+function isPayloadCheckpointState(key: DatastoreKey): boolean {
+  return key.at(-1) === 1;
 }
 
 /**
- * Get the latest safe checkpoint state the node can use to boot from
- *   - it should be the checkpoint state that's unique in its epoch
- *   - its last processed block slot should be at epoch boundary or last slot of previous epoch
+ * Get the latest "safe" checkpoint state the node can use to boot from
+ *   - its last processed block slot should be at epoch boundary (CRCS) or last slot of previous epoch (PRCS)
  *   - state slot should be at epoch boundary
  *   - state slot should be equal to epoch * SLOTS_PER_EPOCH
  *
@@ -70,9 +82,20 @@ export async function getLatestSafeDatastoreKey(
 
   const dataStoreKeyByEpoch: Map<Epoch, DatastoreKey> = new Map();
   for (const [epoch, keys] of checkpointsByEpoch.entries()) {
-    // only consider epochs with a single checkpoint to avoid ambiguity from forks
     if (keys.length === 1) {
+      // PRCS (skipped slot) or CRCS and no payloadPresent
+      // Pre-gloas always fall into this case
       dataStoreKeyByEpoch.set(epoch, keys[0]);
+    } else if (keys.length === 2) {
+      // CRCS without payload and CRCS with payload
+      // ie Two keys for the same checkpoint with different payloadPresent suffix (FULL/EMPTY)
+      // TODO GLOAS: Here we pick FULL key, there is a chance that payload is orphaned hence we not be able to sync
+      const cp0 = extractCheckpointBytes(keys[0]);
+      const cp1 = extractCheckpointBytes(keys[1]);
+      if (byteArrayEquals(cp0, cp1)) {
+        const fullKey = isPayloadCheckpointState(keys[0]) ? keys[0] : keys[1];
+        dataStoreKeyByEpoch.set(epoch, fullKey);
+      }
     }
   }
 

package/src/chain/stateCache/datastore/file.ts

@@ -1,12 +1,13 @@
 import path from "node:path";
-import {phase0, ssz} from "@lodestar/types";
+import {phase0} from "@lodestar/types";
 import {fromHex, toHex} from "@lodestar/utils";
 import {ensureDir, readFile, readFileNames, removeFile, writeIfNotExist} from "../../../util/file.js";
-import {getLatestSafeDatastoreKey} from "./db.js";
+import {checkpointToDatastoreKey, getLatestSafeDatastoreKey} from "./db.js";
 import {CPStateDatastore, DatastoreKey} from "./types.js";
 
 const CHECKPOINT_STATES_FOLDER = "checkpoint_states";
-const CHECKPOINT_FILE_NAME_LENGTH = 82;
+/** 41 bytes (40 checkpoint + 1 payloadPresent) = 82 hex chars + "0x" prefix = 84 */
+const CHECKPOINT_FILE_NAME_LENGTH = 84;
 
 /**
  * Implementation of CPStateDatastore using file system, this is beneficial for debugging.
@@ -28,8 +29,8 @@ export class FileCPStateDatastore implements CPStateDatastore {
     }
   }
 
-  async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise<DatastoreKey> {
-    const serializedCheckpoint = ssz.phase0.Checkpoint.serialize(cpKey);
+  async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean): Promise<DatastoreKey> {
+    const serializedCheckpoint = checkpointToDatastoreKey(cpKey, payloadPresent);
     const filePath = path.join(this.folderPath, toHex(serializedCheckpoint));
     await writeIfNotExist(filePath, stateBytes);
     return serializedCheckpoint;

package/src/chain/stateCache/datastore/types.ts

@@ -1,11 +1,12 @@
 import {phase0} from "@lodestar/types";
 
-// With db implementation, persistedKey is serialized data of a checkpoint
+// With db implementation, persistedKey is serialized data of a checkpoint + 1
+// ie a fixed size of `ssz.phase0.Checkpoint.minSize + 1`
 export type DatastoreKey = Uint8Array;
 
 // Make this generic to support testing
 export interface CPStateDatastore {
-  write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array) => Promise<DatastoreKey>;
+  write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array, payloadPresent: boolean) => Promise<DatastoreKey>;
   remove: (key: DatastoreKey) => Promise<void>;
   read: (key: DatastoreKey) => Promise<Uint8Array | null>;
   readLatestSafe: () => Promise<Uint8Array | null>;