@lodestar/beacon-node 1.47.0-dev.268215c3da → 1.47.0-dev.2fba076c28

package/src/chain/opPools/deferredVoluntaryExitPool.ts

@@ -0,0 +1,65 @@
+import {Logger} from "@lodestar/logger";
+import {IBeaconStateView, VoluntaryExitValidity} from "@lodestar/state-transition";
+import {Epoch, ValidatorIndex} from "@lodestar/types";
+import {SignedVoluntaryExit} from "@lodestar/types/phase0";
+import {isTransientExitValidity} from "../validation/voluntaryExit.js";
+
+type DeferredEntry = {
+  exit: SignedVoluntaryExit;
+  validity: VoluntaryExitValidity;
+  insertedAtEpoch: Epoch;
+};
+
+export class DeferredVoluntaryExitPool {
+  private pool = new Map<ValidatorIndex, DeferredEntry>();
+
+  constructor(
+    private readonly logger: Logger,
+    private readonly maxSize = 1024,
+    private readonly maxDeferEpochs = 4096
+  ) {}
+
+  insert(exit: SignedVoluntaryExit, validity: VoluntaryExitValidity, currentEpoch: Epoch): boolean {
+    if (!isTransientExitValidity(validity)) return false;
+    if (this.pool.size === this.maxSize) return false;
+    if (this.pool.has(exit.message.validatorIndex)) return false;
+
+    this.pool.set(exit.message.validatorIndex, {exit, validity, insertedAtEpoch: currentEpoch});
+
+    return true;
+  }
+
+  drainProcessableExits(state: IBeaconStateView): SignedVoluntaryExit[] {
+    const epoch = state.epoch;
+    const validExits: SignedVoluntaryExit[] = [];
+    for (const [validatorIndex, entry] of this.pool) {
+      try {
+        if (epoch - entry.insertedAtEpoch > this.maxDeferEpochs) {
+          this.pool.delete(validatorIndex);
+          this.logger.debug("Deferred voluntary exit expired", {validatorIndex, lastValidity: entry.validity});
+          continue;
+        }
+        const validity = state.getVoluntaryExitValidity(entry.exit, true);
+        if (validity === VoluntaryExitValidity.valid) {
+          validExits.push(entry.exit);
+          this.pool.delete(validatorIndex);
+        } else if (!isTransientExitValidity(validity)) {
+          this.logger.debug("Deferred voluntary exit became permanently invalid", {validatorIndex, validity});
+          this.pool.delete(validatorIndex);
+        }
+        // Else if still transient - keep
+      } catch (e) {
+        this.logger.debug("Processing deferred voluntary exit failed", {validatorIndex}, e as Error);
+      }
+    }
+    return validExits;
+  }
+
+  has(validatorIndex: ValidatorIndex): boolean {
+    return this.pool.has(validatorIndex);
+  }
+
+  size(): number {
+    return this.pool.size;
+  }
+}

package/src/chain/opPools/index.ts

@@ -1,5 +1,6 @@
 export {AggregatedAttestationPool} from "./aggregatedAttestationPool.js";
 export {AttestationPool} from "./attestationPool.js";
+export {DeferredVoluntaryExitPool} from "./deferredVoluntaryExitPool.js";
 export {ExecutionPayloadBidPool} from "./executionPayloadBidPool.js";
 export {OpPool} from "./opPool.js";
 export {PayloadAttestationPool} from "./payloadAttestationPool.js";

package/src/chain/opPools/payloadAttestationPool.ts

@@ -1,4 +1,4 @@
-import {Signature, aggregateSignatures} from "@chainsafe/blst";
+import {Signature, aggregateSignatures} from "@chainsafe/lodestar-z/blst";
 import {BitArray} from "@chainsafe/ssz";
 import {ChainForkConfig} from "@lodestar/config";
 import {MAX_COMMITTEES_PER_SLOT, MAX_PAYLOAD_ATTESTATIONS, PTC_SIZE} from "@lodestar/params";
@@ -178,8 +178,8 @@ function aggregateMessageInto(
   validatorCommitteeIndices: number[],
   aggregate: AggregateFast
 ): InsertOutcome {
-  // Gossip dedup via `seenPayloadAttesters` is keyed by (epoch, validatorIndex), so the same
-  // validator's message is never processed twice — all of its bits are set together or none.
+  // Gossip dedup via `seenPayloadAttesters` is keyed by (slot, validatorIndex), so the same
+  // validator's message is never processed twice for this aggregate's slot. All of its bits are set together or none.
   // Checking the first index is sufficient.
   if (aggregate.aggregationBits.get(validatorCommitteeIndices[0]) === true) {
     return InsertOutcome.AlreadyKnown;

package/src/chain/opPools/syncCommitteeMessagePool.ts

@@ -1,4 +1,4 @@
-import {Signature, aggregateSignatures} from "@chainsafe/blst";
+import {Signature, aggregateSignatures} from "@chainsafe/lodestar-z/blst";
 import {BitArray} from "@chainsafe/ssz";
 import {ChainForkConfig} from "@lodestar/config";
 import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params";

package/src/chain/opPools/syncContributionAndProofPool.ts

@@ -1,4 +1,4 @@
-import {Signature, aggregateSignatures} from "@chainsafe/blst";
+import {Signature, aggregateSignatures} from "@chainsafe/lodestar-z/blst";
 import {BitArray} from "@chainsafe/ssz";
 import {ChainForkConfig} from "@lodestar/config";
 import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params";

package/src/chain/opPools/utils.ts

@@ -1,4 +1,4 @@
-import {Signature} from "@chainsafe/blst";
+import {Signature} from "@chainsafe/lodestar-z/blst";
 import {BLS_WITHDRAWAL_PREFIX} from "@lodestar/params";
 import {IBeaconStateView} from "@lodestar/state-transition";
 import {Slot, capella} from "@lodestar/types";

package/src/chain/options.ts

@@ -53,7 +53,7 @@ export type IChainOptions = BlockProcessOpts &
     nativeStateView?: boolean;
     /** Builder circuit breaker fault inspection window in slots */
     faultInspectionWindow?: number;
-    /** Allowed unrevealed payloads within the fault inspection window */
+    /** Canonical EMPTY blocks allowed per `faultInspectionWindow` observed blocks */
     allowedFaults?: number;
   };
 

package/src/chain/prepareNextSlot.ts

@@ -151,25 +151,40 @@ export class PrepareNextSlotScheduler {
         const proposerIndex = await getProposerIndex();
         feeRecipient = this.chain.beaconProposerCache.get(proposerIndex);
 
-        if (feeRecipient) {
-          // If we are proposing next slot, we need to predict if we can proposer-boost-reorg or not
+        // Predict the proposer head of the next slot when either:
+        //  - we are proposing the next slot (single-slot proposer-boost-reorg), or
+        //  - this is an epoch transition: post-Fulu the next proposer may reorg a weak
+        //    last-block-of-epoch and build slot 0 on the strong parent (epoch-boundary reorg),
+        //    so we should dial the strong parent through the boundary instead of the weak head.
+        if (feeRecipient || isEpochTransition) {
           const proposerHead = this.chain.predictProposerHead(clockSlot);
           const {slot: proposerHeadSlot, blockRoot: proposerHeadRoot} = proposerHead;
 
-          // If we predict we can reorg, we build on the proposer head (parent) block instead
+          // If we predict a reorg, we build the epoch transition on the proposer head (parent) instead
           if (proposerHeadRoot !== headRoot || proposerHeadSlot !== headSlot) {
-            this.logger.verbose("Weak head detected. May build on parent block instead", {
+            // feeRecipient set => WE propose the next slot and may reorg it out ourselves;
+            // otherwise it's a predicted reorg by another proposer (epoch-boundary reorg).
+            const logMessage = feeRecipient
+              ? "Weak head detected. We may build on parent block instead"
+              : "Weak head detected. Another proposer may build on parent block instead";
+            this.logger.verbose(logMessage, {
               proposerHeadSlot,
               proposerHeadRoot,
               headSlot,
               headRoot,
+              isEpochTransition,
             });
             this.metrics?.weakHeadDetected.inc();
+            if (isEpochTransition) {
+              this.metrics?.precomputeNextEpochTransition.predictedReorg.inc();
+            }
             updatedHead = proposerHead;
           }
+        }
 
+        if (feeRecipient) {
           if (isForkPostGloas(fork)) {
-            this.chain.builderCircuitBreaker.update(clockSlot);
+            this.chain.builderCircuitBreaker.update(clockSlot, updatedHead);
           } else {
             // Update the builder status, if enabled shoot an api call to check status
             this.chain.updateBuilderStatus(clockSlot);
@@ -182,6 +197,7 @@ export class PrepareNextSlotScheduler {
         }
 
         // post-fulu we'll always reach here because preparedState is undefined
+        // we may run epoch transition not with headBlock if it's weak
         if (preparedState === undefined || updatedHead !== headBlock) {
           preparedState = await this.chain.regen.getBlockSlotState(
             updatedHead,
@@ -280,7 +296,7 @@ export class PrepareNextSlotScheduler {
       //  + if next slot is a skipped slot, it'd help getting target checkpoint state faster to validate attestations
       if (isEpochTransition) {
         this.metrics?.precomputeNextEpochTransition.count.inc({result: "success"}, 1);
-        const previousHits = this.chain.regen.updatePreComputedCheckpoint(headRoot, nextEpoch);
+        const previousHits = this.chain.regen.updatePreComputedCheckpoint(updatedHead.blockRoot, nextEpoch);
         if (previousHits === 0) {
           this.metrics?.precomputeNextEpochTransition.waste.inc();
         }
@@ -289,6 +305,8 @@ export class PrepareNextSlotScheduler {
         this.logger.verbose("Completed PrepareNextSlotScheduler epoch transition", {
           nextEpoch,
           headSlot,
+          headRoot,
+          updatedHeadRoot: updatedHead.blockRoot,
           prepareSlot,
           previousHits,
           durationMs: Date.now() - start,

package/src/chain/produceBlock/produceBlockBody.ts

@@ -100,6 +100,8 @@ export type BlockAttributes = {
   slot: Slot;
   parentBlock: ProtoBlock;
   feeRecipient?: string;
+  /** Verify that a locally produced execution payload uses `feeRecipient`. */
+  strictFeeRecipientCheck?: boolean;
   /** When provided, build block with this builder bid instead of a self-build bid */
   builderBid?: gloas.SignedExecutionPayloadBid;
 };
@@ -201,6 +203,7 @@ export async function produceBlockBody<T extends BlockType>(
   const {
     slot: blockSlot,
     feeRecipient: requestedFeeRecipient,
+    strictFeeRecipientCheck,
     parentBlock,
     proposerIndex,
     proposerPubKey,
@@ -334,6 +337,16 @@ export async function produceBlockBody<T extends BlockType>(
     executionPayloadValue = payloadRes.executionPayloadValue;
     shouldOverrideBuilder = payloadRes.shouldOverrideBuilder;
 
+    if (
+      strictFeeRecipientCheck &&
+      requestedFeeRecipient &&
+      !byteArrayEquals(executionPayload.feeRecipient, fromHex(requestedFeeRecipient))
+    ) {
+      throw Error(
+        `Invalid feeRecipient set in engine payload expected=${requestedFeeRecipient} actual=${toHex(executionPayload.feeRecipient)}`
+      );
+    }
+
     if (blobsBundle === undefined) {
       throw Error(`Missing blobsBundle response from getPayload at fork=${fork}`);
     }

package/src/chain/regen/interface.ts

@@ -21,6 +21,8 @@ export enum RegenCaller {
   validateGossipAggregateAndProof = "validateGossipAggregateAndProof",
   validateGossipAttestation = "validateGossipAttestation",
   validateGossipVoluntaryExit = "validateGossipVoluntaryExit",
+  validateApiVoluntaryExit = "validateApiVoluntaryExit",
+  publishDeferredVoluntaryExits = "publishDeferredVoluntaryExits",
   validateGossipExecutionPayloadBid = "validateGossipExecutionPayloadBid",
   validateGossipPayloadAttestationMessage = "validateGossipPayloadAttestationMessage",
   validateGossipProposerPreferences = "validateGossipProposerPreferences",
@@ -39,14 +41,14 @@ export type StateRegenerationOpts = {
 };
 
 export interface IStateRegenerator extends IStateRegeneratorInternal {
-  dropCache(): void;
+  dropCache(): Promise<void>;
   dumpCacheSummary(): routes.lodestar.StateCacheItem[];
   getStateSync(stateRoot: RootHex): IBeaconStateView | null;
   getPreStateSync(block: BeaconBlock): IBeaconStateView | null;
   getCheckpointStateOrBytes(cp: CheckpointHex): Promise<IBeaconStateView | Uint8Array | null>;
   getCheckpointStateSync(cp: CheckpointHex): IBeaconStateView | null;
   getClosestHeadState(head: ProtoBlock): IBeaconStateView | null;
-  pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void;
+  onCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void;
   processState(blockRootHex: RootHex, postState: IBeaconStateView): void;
   addCheckpointState(cp: phase0.Checkpoint, item: IBeaconStateView): void;
   updateHeadState(newHead: ProtoBlock, maybeHeadState: IBeaconStateView): void;

package/src/chain/regen/queued.ts

@@ -67,9 +67,9 @@ export class QueuedStateRegenerator implements IStateRegenerator {
     return this.jobQueue.jobLen < REGEN_CAN_ACCEPT_WORK_THRESHOLD;
   }
 
-  dropCache(): void {
+  async dropCache(): Promise<void> {
     this.blockStateCache.clear();
-    this.checkpointStateCache.clear();
+    await this.checkpointStateCache.clear();
   }
 
   dumpCacheSummary(): routes.lodestar.StateCacheItem[] {
@@ -138,9 +138,9 @@ export class QueuedStateRegenerator implements IStateRegenerator {
     return this.checkpointStateCache.getLatest(head.blockRoot, Infinity) || this.blockStateCache.get(head.stateRoot);
   }
 
-  pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void {
-    this.checkpointStateCache.prune(finalizedEpoch, justifiedEpoch);
-    this.blockStateCache.prune(headStateRoot);
+  onCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void {
+    this.checkpointStateCache.onCheckpoint(finalizedEpoch, justifiedEpoch);
+    this.blockStateCache.onCheckpoint(headStateRoot);
   }
 
   processState(blockRootHex: RootHex, postState: IBeaconStateView): void {

package/src/chain/seenCache/seenAttesters.ts

@@ -1,4 +1,5 @@
-import {Epoch, ValidatorIndex} from "@lodestar/types";
+import {computeEpochAtSlot} from "@lodestar/state-transition";
+import {Epoch, Slot, ValidatorIndex} from "@lodestar/types";
 import {MapDef} from "@lodestar/utils";
 
 // How many *non future* epochs we intend to keep for SeenAttesters.
@@ -57,7 +58,32 @@ export class SeenAttesters {
  */
 export class SeenAggregators extends SeenAttesters {}
 
-/**
- * Keeps a cache to filter payload attestations from the same attesters in the same epoch
- */
-export class SeenPayloadAttesters extends SeenAttesters {}
+/** Keeps a cache to filter payload attestations from the same validator in the same slot. */
+export class SeenPayloadAttesters {
+  private readonly validatorIndexesBySlot = new MapDef<Slot, Set<ValidatorIndex>>(() => new Set<ValidatorIndex>());
+  private readonly validatorIndexesByEpoch = new SeenAttesters();
+
+  isKnown(slot: Slot, validatorIndex: ValidatorIndex): boolean {
+    return this.validatorIndexesBySlot.get(slot)?.has(validatorIndex) === true;
+  }
+
+  add(slot: Slot, validatorIndex: ValidatorIndex): void {
+    const epoch = computeEpochAtSlot(slot);
+    this.validatorIndexesByEpoch.add(epoch, validatorIndex);
+    this.validatorIndexesBySlot.getOrDefault(slot).add(validatorIndex);
+  }
+
+  seenAtEpoch(epoch: Epoch, validatorIndex: ValidatorIndex): boolean {
+    return this.validatorIndexesByEpoch.isKnown(epoch, validatorIndex);
+  }
+
+  prune(currentEpoch: Epoch): void {
+    const lowestPermissibleEpoch = Math.max(currentEpoch - EPOCH_LOOKBACK_LIMIT, 0);
+    for (const slot of this.validatorIndexesBySlot.keys()) {
+      if (computeEpochAtSlot(slot) < lowestPermissibleEpoch) {
+        this.validatorIndexesBySlot.delete(slot);
+      }
+    }
+    this.validatorIndexesByEpoch.prune(currentEpoch);
+  }
+}

package/src/chain/seenCache/seenPayloadEnvelopeInput.ts

@@ -1,18 +1,32 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {CheckpointWithHex, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
+import {ForkPostGloas, SLOTS_PER_EPOCH, isForkPostGloas} from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
-import {RootHex} from "@lodestar/types";
-import {Logger} from "@lodestar/utils";
+import {RootHex, SignedBeaconBlock} from "@lodestar/types";
+import {Logger, fromHex} from "@lodestar/utils";
+import {IBeaconDb} from "../../db/index.js";
 import {Metrics} from "../../metrics/metrics.js";
+import {MAX_LOOK_AHEAD_EPOCHS} from "../../sync/constants.js";
 import {IClock} from "../../util/clock.js";
+import {CustodyConfig} from "../../util/dataColumns.js";
 import {SerializedCache} from "../../util/serializedCache.js";
 import {isDaOutOfRange} from "../blocks/blockInput/index.js";
-import {CreateFromBidProps, CreateFromBlockProps, PayloadEnvelopeInput} from "../blocks/payloadEnvelopeInput/index.js";
+import {
+  CreateFromBidProps,
+  CreateFromBlockProps,
+  PayloadEnvelopeInput,
+  PayloadEnvelopeInputPruneReason,
+  PayloadEnvelopeInputSource,
+} from "../blocks/payloadEnvelopeInput/index.js";
 import {ChainEvent, ChainEventEmitter} from "../emitter.js";
+import {SeenBlockInput} from "./seenGossipBlockInput.js";
 
 export type {PayloadEnvelopeInputState} from "../blocks/payloadEnvelopeInput/index.js";
 export {PayloadEnvelopeInput} from "../blocks/payloadEnvelopeInput/index.js";
 
+// Bound this cache by this max size, this is the same to SeenBlockInput
+const MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE = (MAX_LOOK_AHEAD_EPOCHS + 1) * SLOTS_PER_EPOCH;
+
 export type SeenPayloadEnvelopeInputModules = {
   config: ChainForkConfig;
   clock: IClock;
@@ -20,6 +34,9 @@ export type SeenPayloadEnvelopeInputModules = {
   chainEvents: ChainEventEmitter;
   signal: AbortSignal;
   serializedCache: SerializedCache;
+  db: IBeaconDb;
+  seenBlockInputCache: SeenBlockInput;
+  custodyConfig: CustodyConfig;
   metrics: Metrics | null;
   logger?: Logger;
 };
@@ -40,9 +57,14 @@ export class SeenPayloadEnvelopeInput {
   private readonly chainEvents: ChainEventEmitter;
   private readonly signal: AbortSignal;
   private readonly serializedCache: SerializedCache;
+  private readonly db: IBeaconDb;
+  private readonly seenBlockInputCache: SeenBlockInput;
+  private readonly custodyConfig: CustodyConfig;
   private readonly metrics: Metrics | null;
   private readonly logger?: Logger;
   private payloadInputs = new Map<RootHex, PayloadEnvelopeInput>();
+  // Dedup concurrent DB reloads of the same root so callers share one reconstructed object.
+  private readonly reloading = new Map<RootHex, Promise<PayloadEnvelopeInput | undefined>>();
 
   constructor({
     config,
@@ -51,6 +73,9 @@ export class SeenPayloadEnvelopeInput {
     chainEvents,
     signal,
     serializedCache,
+    db,
+    seenBlockInputCache,
+    custodyConfig,
     metrics,
     logger,
   }: SeenPayloadEnvelopeInputModules) {
@@ -60,6 +85,9 @@ export class SeenPayloadEnvelopeInput {
     this.chainEvents = chainEvents;
     this.signal = signal;
     this.serializedCache = serializedCache;
+    this.db = db;
+    this.seenBlockInputCache = seenBlockInputCache;
+    this.custodyConfig = custodyConfig;
     this.metrics = metrics;
     this.logger = logger;
 
@@ -86,7 +114,7 @@ export class SeenPayloadEnvelopeInput {
     let deletedCount = 0;
     for (const [, input] of this.payloadInputs) {
       if (input.slot < finalizedSlot) {
-        this.evictPayloadInput(input);
+        this.evictPayloadInput(input, "finalized");
         deletedCount++;
       }
     }
@@ -110,12 +138,13 @@ export class SeenPayloadEnvelopeInput {
     const daOutOfRange = isDaOutOfRange(this.config, props.forkName, props.block.message.slot, this.clock.currentEpoch);
     const input = PayloadEnvelopeInput.createFromBlock({...props, daOutOfRange});
     this.payloadInputs.set(props.blockRootHex, input);
-    this.metrics?.seenCache.payloadEnvelopeInput.created.inc();
+    this.metrics?.seenCache.payloadEnvelopeInput.created.inc({source: props.source});
     this.logger?.verbose("SeenPayloadEnvelopeInput.add created new entry", {
       slot: input.slot,
       root: props.blockRootHex,
       daOutOfRange,
     });
+    this.pruneToMaxSize();
     return input;
   }
 
@@ -131,12 +160,13 @@ export class SeenPayloadEnvelopeInput {
     const daOutOfRange = isDaOutOfRange(this.config, props.forkName, props.slot, this.clock.currentEpoch);
     const input = PayloadEnvelopeInput.createFromBid({...props, daOutOfRange});
     this.payloadInputs.set(props.blockRootHex, input);
-    this.metrics?.seenCache.payloadEnvelopeInput.created.inc();
+    this.metrics?.seenCache.payloadEnvelopeInput.created.inc({source: props.source});
     this.logger?.verbose("SeenPayloadEnvelopeInput.addFromBid created new entry", {
       slot: input.slot,
       root: props.blockRootHex,
       daOutOfRange,
     });
+    this.pruneToMaxSize();
     return input;
   }
 
@@ -144,6 +174,80 @@ export class SeenPayloadEnvelopeInput {
     return this.payloadInputs.get(blockRootHex);
   }
 
+  /**
+   * Like `get()`, but on a cache miss reconstruct the shell (bid + versionedHashes) from the block
+   * in `seenBlockInputCache` or the hot DB.
+   * This api is meant for BlockInputSync when a late/weird payloads for old blocks
+   *
+   * NOTE: the reconstructed entry is always EMPTY even when the block is actually FULL (its
+   * payload envelope + columns persisted in the DB). The consumer should consult fork choice if it needs to.
+   */
+  async getOrReload(blockRootHex: RootHex): Promise<PayloadEnvelopeInput | undefined> {
+    const existing = this.payloadInputs.get(blockRootHex);
+    if (existing !== undefined) {
+      return existing;
+    }
+
+    // Without this dedup, two concurrent misses each db.block.get + createFromBlock a DIFFERENT shell
+    // for the same root; processPayloadEnvelopeJob's WeakMap (keyed by object) can't dedup the import.
+    const inflight = this.reloading.get(blockRootHex);
+    if (inflight !== undefined) {
+      return inflight;
+    }
+    const promise = this.reloadFromDb(blockRootHex);
+    this.reloading.set(blockRootHex, promise);
+    try {
+      return await promise;
+    } finally {
+      this.reloading.delete(blockRootHex);
+    }
+  }
+
+  private async reloadFromDb(blockRootHex: RootHex): Promise<PayloadEnvelopeInput | undefined> {
+    // Only recover unfinalized, fork-choice-known blocks. Do not read the finalized archive. The gate
+    // also filters out un-imported blocks that may sit in seenBlockInputCache before validation.
+    if (!this.forkChoice.hasBlockHex(blockRootHex)) {
+      return undefined;
+    }
+
+    // In-memory first: persistBlockInput writes db.block THEN prunes seenBlockInputCache, so a
+    // just-imported block whose async write (unfinalizedBlockWrites) has not flushed yet is still here;
+    // older blocks (already pruned from the cache) fall through to the hot db.
+    const cachedBlockInput = this.seenBlockInputCache.get(blockRootHex);
+    const block = cachedBlockInput?.hasBlock()
+      ? cachedBlockInput.getBlock()
+      : await this.db.block.get(fromHex(blockRootHex));
+    if (block == null) {
+      return undefined;
+    }
+
+    const forkName = this.config.getForkName(block.message.slot);
+    if (!isForkPostGloas(forkName)) {
+      return undefined;
+    }
+
+    const daOutOfRange = isDaOutOfRange(this.config, forkName, block.message.slot, this.clock.currentEpoch);
+    const input = PayloadEnvelopeInput.createFromBlock({
+      blockRootHex,
+      block: block as SignedBeaconBlock<ForkPostGloas>,
+      forkName,
+      sampledColumns: this.custodyConfig.sampledColumns,
+      custodyColumns: this.custodyConfig.custodyColumns,
+      seenTimestampSec: Date.now() / 1000,
+      source: PayloadEnvelopeInputSource.reload,
+      daOutOfRange,
+    });
+    this.payloadInputs.set(blockRootHex, input);
+    this.metrics?.seenCache.payloadEnvelopeInput.created.inc({source: PayloadEnvelopeInputSource.reload});
+    this.logger?.verbose("SeenPayloadEnvelopeInput.getOrReload reconstructed entry from db", {
+      slot: input.slot,
+      root: blockRootHex,
+    });
+    // Set above (entry is at the back of the insertion order), so the cap never evicts what we just reloaded.
+    this.pruneToMaxSize();
+    return input;
+  }
+
   hasPayload(blockRootHex: RootHex): boolean {
     return this.payloadInputs.get(blockRootHex)?.hasPayloadEnvelope() ?? false;
   }
@@ -155,8 +259,7 @@ export class SeenPayloadEnvelopeInput {
   prune(blockRootHex: RootHex): void {
     const input = this.payloadInputs.get(blockRootHex);
     if (input) {
-      this.evictPayloadInput(input);
-      this.logger?.verbose("SeenPayloadEnvelopeInput.prune deleted", {slot: input.slot, root: blockRootHex});
+      this.evictPayloadInput(input, "prune");
     }
   }
 
@@ -170,17 +273,43 @@ export class SeenPayloadEnvelopeInput {
         // ...and don't evict while columns are still being gathered: writeDataColumnsToDb awaits the
         // same hasComputedAllData() before persisting. Such entries are pruned by a later call.
         if (input?.hasComputedAllData()) {
-          this.evictPayloadInput(input);
-          this.logger?.verbose("SeenPayloadEnvelopeInput.pruneBelowParent deleted", {
-            slot: block.slot,
-            root: block.blockRoot,
-          });
+          this.evictPayloadInput(input, "belowParent");
         }
       }
     }
   }
 
-  private evictPayloadInput(payloadInput: PayloadEnvelopeInput): void {
+  /**
+   * Bound this cache to have at most MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE items.
+   * Evicts by insertion order so a just-reloaded old-slot entry is not evicted right after.
+   * On unstable network, we may need to query an evicted item, UnknownBlockInput will recover
+   * it from db via getOrReload() api. Runs after every single insert, so it evicts ~1 per call.
+   */
+  private pruneToMaxSize(): void {
+    let evicted = 0;
+    for (const input of this.payloadInputs.values()) {
+      if (this.payloadInputs.size <= MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE) {
+        break;
+      }
+      this.evictPayloadInput(input, "cap");
+      evicted++;
+    }
+    if (evicted > 0) {
+      this.logger?.debug("SeenPayloadEnvelopeInput.pruneToMaxSize evicted", {
+        evicted,
+        size: this.payloadInputs.size,
+        max: MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE,
+      });
+    }
+  }
+
+  private evictPayloadInput(payloadInput: PayloadEnvelopeInput, reason: PayloadEnvelopeInputPruneReason): void {
+    this.metrics?.seenCache.payloadEnvelopeInput.pruned.inc({reason});
+    this.logger?.debug("SeenPayloadEnvelopeInput evicted", {
+      slot: payloadInput.slot,
+      root: payloadInput.blockRootHex,
+      reason,
+    });
     this.serializedCache.delete(payloadInput.getSerializedCacheKeys());
     this.payloadInputs.delete(payloadInput.blockRootHex);
   }

package/src/chain/stateCache/fifoBlockStateCache.ts

@@ -144,13 +144,20 @@ export class FIFOBlockStateCache implements BlockStateCache {
     return this.cache.size;
   }
 
+  /**
+   * On a checkpoint event, prune the cache while keeping the head state.
+   */
+  onCheckpoint(headStateRootHex: string): void {
+    this.prune(headStateRootHex);
+  }
+
   /**
    * Prune the cache from tail to keep the most recent n states consistently.
    * The tail of the list is the oldest state, in case regen adds back the same state,
    * it should stay next to head so that it won't be pruned right away.
    * The FIFO cache helps with this.
    */
-  prune(lastAddedKey: string): void {
+  private prune(lastAddedKey: string): void {
     while (this.keyOrder.length > this.maxStates) {
       const key = this.keyOrder.last();
       // it does not make sense to prune the last added state