@lodestar/beacon-node 1.47.0-dev.0bcaaaebb5 → 1.47.0-dev.101e2c290d

package/src/chain/seenCache/seenPayloadEnvelopeInput.ts

@@ -1,18 +1,32 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {CheckpointWithHex, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
+import {ForkPostGloas, SLOTS_PER_EPOCH, isForkPostGloas} from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
-import {RootHex} from "@lodestar/types";
-import {Logger} from "@lodestar/utils";
+import {RootHex, SignedBeaconBlock} from "@lodestar/types";
+import {Logger, fromHex} from "@lodestar/utils";
+import {IBeaconDb} from "../../db/index.js";
 import {Metrics} from "../../metrics/metrics.js";
+import {MAX_LOOK_AHEAD_EPOCHS} from "../../sync/constants.js";
 import {IClock} from "../../util/clock.js";
+import {CustodyConfig} from "../../util/dataColumns.js";
 import {SerializedCache} from "../../util/serializedCache.js";
 import {isDaOutOfRange} from "../blocks/blockInput/index.js";
-import {CreateFromBidProps, CreateFromBlockProps, PayloadEnvelopeInput} from "../blocks/payloadEnvelopeInput/index.js";
+import {
+  CreateFromBidProps,
+  CreateFromBlockProps,
+  PayloadEnvelopeInput,
+  PayloadEnvelopeInputPruneReason,
+  PayloadEnvelopeInputSource,
+} from "../blocks/payloadEnvelopeInput/index.js";
 import {ChainEvent, ChainEventEmitter} from "../emitter.js";
+import {SeenBlockInput} from "./seenGossipBlockInput.js";
 
 export type {PayloadEnvelopeInputState} from "../blocks/payloadEnvelopeInput/index.js";
 export {PayloadEnvelopeInput} from "../blocks/payloadEnvelopeInput/index.js";
 
+// Bound this cache by this max size, this is the same to SeenBlockInput
+const MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE = (MAX_LOOK_AHEAD_EPOCHS + 1) * SLOTS_PER_EPOCH;
+
 export type SeenPayloadEnvelopeInputModules = {
   config: ChainForkConfig;
   clock: IClock;
@@ -20,6 +34,9 @@ export type SeenPayloadEnvelopeInputModules = {
   chainEvents: ChainEventEmitter;
   signal: AbortSignal;
   serializedCache: SerializedCache;
+  db: IBeaconDb;
+  seenBlockInputCache: SeenBlockInput;
+  custodyConfig: CustodyConfig;
   metrics: Metrics | null;
   logger?: Logger;
 };
@@ -40,9 +57,14 @@ export class SeenPayloadEnvelopeInput {
   private readonly chainEvents: ChainEventEmitter;
   private readonly signal: AbortSignal;
   private readonly serializedCache: SerializedCache;
+  private readonly db: IBeaconDb;
+  private readonly seenBlockInputCache: SeenBlockInput;
+  private readonly custodyConfig: CustodyConfig;
   private readonly metrics: Metrics | null;
   private readonly logger?: Logger;
   private payloadInputs = new Map<RootHex, PayloadEnvelopeInput>();
+  // Dedup concurrent DB reloads of the same root so callers share one reconstructed object.
+  private readonly reloading = new Map<RootHex, Promise<PayloadEnvelopeInput | undefined>>();
 
   constructor({
     config,
@@ -51,6 +73,9 @@ export class SeenPayloadEnvelopeInput {
     chainEvents,
     signal,
     serializedCache,
+    db,
+    seenBlockInputCache,
+    custodyConfig,
     metrics,
     logger,
   }: SeenPayloadEnvelopeInputModules) {
@@ -60,6 +85,9 @@ export class SeenPayloadEnvelopeInput {
     this.chainEvents = chainEvents;
     this.signal = signal;
     this.serializedCache = serializedCache;
+    this.db = db;
+    this.seenBlockInputCache = seenBlockInputCache;
+    this.custodyConfig = custodyConfig;
     this.metrics = metrics;
     this.logger = logger;
 
@@ -86,7 +114,7 @@ export class SeenPayloadEnvelopeInput {
     let deletedCount = 0;
     for (const [, input] of this.payloadInputs) {
       if (input.slot < finalizedSlot) {
-        this.evictPayloadInput(input);
+        this.evictPayloadInput(input, "finalized");
         deletedCount++;
       }
     }
@@ -110,12 +138,13 @@ export class SeenPayloadEnvelopeInput {
     const daOutOfRange = isDaOutOfRange(this.config, props.forkName, props.block.message.slot, this.clock.currentEpoch);
     const input = PayloadEnvelopeInput.createFromBlock({...props, daOutOfRange});
     this.payloadInputs.set(props.blockRootHex, input);
-    this.metrics?.seenCache.payloadEnvelopeInput.created.inc();
+    this.metrics?.seenCache.payloadEnvelopeInput.created.inc({source: props.source});
     this.logger?.verbose("SeenPayloadEnvelopeInput.add created new entry", {
       slot: input.slot,
       root: props.blockRootHex,
       daOutOfRange,
     });
+    this.pruneToMaxSize();
     return input;
   }
 
@@ -131,12 +160,13 @@ export class SeenPayloadEnvelopeInput {
     const daOutOfRange = isDaOutOfRange(this.config, props.forkName, props.slot, this.clock.currentEpoch);
     const input = PayloadEnvelopeInput.createFromBid({...props, daOutOfRange});
     this.payloadInputs.set(props.blockRootHex, input);
-    this.metrics?.seenCache.payloadEnvelopeInput.created.inc();
+    this.metrics?.seenCache.payloadEnvelopeInput.created.inc({source: props.source});
     this.logger?.verbose("SeenPayloadEnvelopeInput.addFromBid created new entry", {
       slot: input.slot,
       root: props.blockRootHex,
       daOutOfRange,
     });
+    this.pruneToMaxSize();
     return input;
   }
 
@@ -144,6 +174,80 @@ export class SeenPayloadEnvelopeInput {
     return this.payloadInputs.get(blockRootHex);
   }
 
+  /**
+   * Like `get()`, but on a cache miss reconstruct the shell (bid + versionedHashes) from the block
+   * in `seenBlockInputCache` or the hot DB.
+   * This api is meant for BlockInputSync when a late/weird payloads for old blocks
+   *
+   * NOTE: the reconstructed entry is always EMPTY even when the block is actually FULL (its
+   * payload envelope + columns persisted in the DB). The consumer should consult fork choice if it needs to.
+   */
+  async getOrReload(blockRootHex: RootHex): Promise<PayloadEnvelopeInput | undefined> {
+    const existing = this.payloadInputs.get(blockRootHex);
+    if (existing !== undefined) {
+      return existing;
+    }
+
+    // Without this dedup, two concurrent misses each db.block.get + createFromBlock a DIFFERENT shell
+    // for the same root; processPayloadEnvelopeJob's WeakMap (keyed by object) can't dedup the import.
+    const inflight = this.reloading.get(blockRootHex);
+    if (inflight !== undefined) {
+      return inflight;
+    }
+    const promise = this.reloadFromDb(blockRootHex);
+    this.reloading.set(blockRootHex, promise);
+    try {
+      return await promise;
+    } finally {
+      this.reloading.delete(blockRootHex);
+    }
+  }
+
+  private async reloadFromDb(blockRootHex: RootHex): Promise<PayloadEnvelopeInput | undefined> {
+    // Only recover unfinalized, fork-choice-known blocks. Do not read the finalized archive. The gate
+    // also filters out un-imported blocks that may sit in seenBlockInputCache before validation.
+    if (!this.forkChoice.hasBlockHex(blockRootHex)) {
+      return undefined;
+    }
+
+    // In-memory first: persistBlockInput writes db.block THEN prunes seenBlockInputCache, so a
+    // just-imported block whose async write (unfinalizedBlockWrites) has not flushed yet is still here;
+    // older blocks (already pruned from the cache) fall through to the hot db.
+    const cachedBlockInput = this.seenBlockInputCache.get(blockRootHex);
+    const block = cachedBlockInput?.hasBlock()
+      ? cachedBlockInput.getBlock()
+      : await this.db.block.get(fromHex(blockRootHex));
+    if (block == null) {
+      return undefined;
+    }
+
+    const forkName = this.config.getForkName(block.message.slot);
+    if (!isForkPostGloas(forkName)) {
+      return undefined;
+    }
+
+    const daOutOfRange = isDaOutOfRange(this.config, forkName, block.message.slot, this.clock.currentEpoch);
+    const input = PayloadEnvelopeInput.createFromBlock({
+      blockRootHex,
+      block: block as SignedBeaconBlock<ForkPostGloas>,
+      forkName,
+      sampledColumns: this.custodyConfig.sampledColumns,
+      custodyColumns: this.custodyConfig.custodyColumns,
+      seenTimestampSec: Date.now() / 1000,
+      source: PayloadEnvelopeInputSource.reload,
+      daOutOfRange,
+    });
+    this.payloadInputs.set(blockRootHex, input);
+    this.metrics?.seenCache.payloadEnvelopeInput.created.inc({source: PayloadEnvelopeInputSource.reload});
+    this.logger?.verbose("SeenPayloadEnvelopeInput.getOrReload reconstructed entry from db", {
+      slot: input.slot,
+      root: blockRootHex,
+    });
+    // Set above (entry is at the back of the insertion order), so the cap never evicts what we just reloaded.
+    this.pruneToMaxSize();
+    return input;
+  }
+
   hasPayload(blockRootHex: RootHex): boolean {
     return this.payloadInputs.get(blockRootHex)?.hasPayloadEnvelope() ?? false;
   }
@@ -155,8 +259,7 @@ export class SeenPayloadEnvelopeInput {
   prune(blockRootHex: RootHex): void {
     const input = this.payloadInputs.get(blockRootHex);
     if (input) {
-      this.evictPayloadInput(input);
-      this.logger?.verbose("SeenPayloadEnvelopeInput.prune deleted", {slot: input.slot, root: blockRootHex});
+      this.evictPayloadInput(input, "prune");
     }
   }
 
@@ -170,17 +273,43 @@ export class SeenPayloadEnvelopeInput {
         // ...and don't evict while columns are still being gathered: writeDataColumnsToDb awaits the
         // same hasComputedAllData() before persisting. Such entries are pruned by a later call.
         if (input?.hasComputedAllData()) {
-          this.evictPayloadInput(input);
-          this.logger?.verbose("SeenPayloadEnvelopeInput.pruneBelowParent deleted", {
-            slot: block.slot,
-            root: block.blockRoot,
-          });
+          this.evictPayloadInput(input, "belowParent");
         }
       }
     }
   }
 
-  private evictPayloadInput(payloadInput: PayloadEnvelopeInput): void {
+  /**
+   * Bound this cache to have at most MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE items.
+   * Evicts by insertion order so a just-reloaded old-slot entry is not evicted right after.
+   * On unstable network, we may need to query an evicted item, UnknownBlockInput will recover
+   * it from db via getOrReload() api. Runs after every single insert, so it evicts ~1 per call.
+   */
+  private pruneToMaxSize(): void {
+    let evicted = 0;
+    for (const input of this.payloadInputs.values()) {
+      if (this.payloadInputs.size <= MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE) {
+        break;
+      }
+      this.evictPayloadInput(input, "cap");
+      evicted++;
+    }
+    if (evicted > 0) {
+      this.logger?.debug("SeenPayloadEnvelopeInput.pruneToMaxSize evicted", {
+        evicted,
+        size: this.payloadInputs.size,
+        max: MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE,
+      });
+    }
+  }
+
+  private evictPayloadInput(payloadInput: PayloadEnvelopeInput, reason: PayloadEnvelopeInputPruneReason): void {
+    this.metrics?.seenCache.payloadEnvelopeInput.pruned.inc({reason});
+    this.logger?.debug("SeenPayloadEnvelopeInput evicted", {
+      slot: payloadInput.slot,
+      root: payloadInput.blockRootHex,
+      reason,
+    });
     this.serializedCache.delete(payloadInput.getSerializedCacheKeys());
     this.payloadInputs.delete(payloadInput.blockRootHex);
   }

package/src/chain/stateCache/fifoBlockStateCache.ts

@@ -144,13 +144,20 @@ export class FIFOBlockStateCache implements BlockStateCache {
     return this.cache.size;
   }
 
+  /**
+   * On a checkpoint event, prune the cache while keeping the head state.
+   */
+  onCheckpoint(headStateRootHex: string): void {
+    this.prune(headStateRootHex);
+  }
+
   /**
    * Prune the cache from tail to keep the most recent n states consistently.
    * The tail of the list is the oldest state, in case regen adds back the same state,
    * it should stay next to head so that it won't be pruned right away.
    * The FIFO cache helps with this.
    */
-  prune(lastAddedKey: string): void {
+  private prune(lastAddedKey: string): void {
     while (this.keyOrder.length > this.maxStates) {
       const key = this.keyOrder.last();
       // it does not make sense to prune the last added state

package/src/chain/stateCache/persistentCheckpointsCache.ts

@@ -14,7 +14,11 @@ import {BlockStateCache, CacheItemType, CheckpointHex, CheckpointStateCache} fro
 export type PersistentCheckpointStateCacheOpts = {
   /** Keep max n state epochs in memory, persist the rest to disk */
   maxCPStateEpochsInMemory?: number;
-  /** Keep max n state epochs on disk */
+  /**
+   * On-disk retention of persisted checkpoint states. Selects one of two modes:
+   * - `Infinity` (default) -> tiered retention mode
+   * - any finite `n` (>= 0) -> legacy flat retention (keep the last n dense epochs)
+   */
   maxCPStateEpochsOnDisk?: number;
 };
 
@@ -64,6 +68,18 @@ export const DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY = 3;
  */
 export const DEFAULT_MAX_CP_STATE_ON_DISK = Infinity;
 
+// CP_STATE_TIER_BASE = spacing multiplier per tier AND per-tier capacity. Deciding capacity == base
+// so that a full tier[n] always holds exactly ONE epoch aligned to tier[n+1]
+// given tier[n] has span = 16 ^ n and tier[n+1] has span = 16 ^ (n+1)
+// why 16?
+//   - tier 0 is the dense hot-regen window; it must exceed regen's MAX_EPOCH_TO_PROCESS (=5).
+//   - with a full 4 tiers = 64 epochs, we can cover ~ a year of continuous non-finality, and it'll grow very slowly after that
+//     - tier0 spans 16 epochs ≈ 1.7h, this is more than enough to handle reorg
+//     - tier1 spans 256 epochs ≈ 1.1d, ie we store 16 epochs of the previous day
+//     - tier2 spans 4096 epochs ≈ 18d, ie we store 16 epochs of the last ~2.5 weeks
+//     - tier3 spans 65536ep ≈ 291d, ie we store 16 epochs of the last 0.8 year
+export const CP_STATE_TIER_BASE = 16;
+
 // TODO GLOAS: re-evaluate this timing
 const PROCESS_CHECKPOINT_STATES_BPS = 6667;
 
@@ -112,6 +128,19 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   private preComputedCheckpointHits: number | null = null;
   private readonly maxEpochsInMemory: number;
   private readonly maxEpochsOnDisk: number;
+  /**
+   * Persist checkpoint states (epoch + root) by tiers
+   *
+   *                   ╔═════════ tier 2 ══════════╦═══════ tier 1 ═══════╦═══ tier 0 (dense) ══╗
+   * finalizedEpoch -> ║  keep 1 epoch per 256     ║  keep 1 epoch per 16 ║  keep every epoch   ║ -> head
+   *                   ║  x·······x·······x······  ║  x···x···x···x···    ║  xxxxxxxxxxxxxxxxx  ║
+   *                   ╚═══════════════════════════╩══════════════════════╩═════════════════════╝
+   *
+   * starting from tier 1, all epochs must be aligned to the finalized epoch
+   */
+  private persistentTiers: Map<Epoch, Set<RootHex>>[] = [new Map()];
+  private finalizedEpoch: Epoch = 0;
+  private justifiedEpoch: Epoch = 0;
   private readonly datastore: CPStateDatastore;
   private readonly blockStateCache: BlockStateCache;
   private readonly bufferPool?: BufferPool;
@@ -152,6 +181,13 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
         metrics.cpStateCache.size.set({type: CacheItemType.inMemory}, inMemoryCount);
         metrics.cpStateCache.epochSize.set({type: CacheItemType.persisted}, persistentEpochs.size);
         metrics.cpStateCache.epochSize.set({type: CacheItemType.inMemory}, memoryEpochs.size);
+
+        for (let tier = 0; tier < this.persistentTiers.length; tier++) {
+          let states = 0;
+          for (const roots of this.persistentTiers[tier].values()) states += roots.size;
+          metrics.cpStateCache.persistentTierEpochs.set({tier}, this.persistentTiers[tier].size);
+          metrics.cpStateCache.persistentTierStates.set({tier}, states);
+        }
       });
     }
     this.logger = logger;
@@ -407,10 +443,77 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   }
 
   /**
-   * This is just to conform to the old implementation
+   * This is called on every checkpoint event to refresh the pinned finalized/justified epochs.
    */
-  prune(): void {
-    // do nothing
+  onCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void {
+    this.finalizedEpoch = finalizedEpoch;
+    this.justifiedEpoch = justifiedEpoch;
+  }
+
+  /**
+   * This is called right after we persist a checkpoint state.
+   */
+  private async onPersist(epoch: Epoch, rootHex: RootHex): Promise<void> {
+    let roots = this.persistentTiers[0].get(epoch);
+    if (roots === undefined) {
+      roots = new Set<RootHex>();
+      this.persistentTiers[0].set(epoch, roots);
+    }
+    roots.add(rootHex);
+    await this.prunePersistentTier(0);
+  }
+
+  /**
+   * When tier `n` overflows (> CP_STATE_TIER_BASE epochs), evict its oldest epoch and either promote it to tier `n + 1` (if aligned there) or delete it.
+   *
+   *   before:  persistentTiers[n]   = { ..., e_old }
+   *            persistentTiers[n+1] = { ... }
+   *
+   *   after, if e_old aligns to n+1:           |  after, if e_old does NOT align to n+1:
+   *     persistentTiers[n]   = { ... }         |    persistentTiers[n]   = { ... }
+   *     persistentTiers[n+1] = { ..., e_old }  |    e_old and all its roots deleted from disk
+   *
+   * Since pruning a tier[n] may push an item at tier[n+1], we must call the function recursively.
+   */
+  private async prunePersistentTier(tier: number): Promise<void> {
+    const tierMap = this.persistentTiers[tier];
+    while (tierMap.size > CP_STATE_TIER_BASE) {
+      // Evict the oldest epoch, but never the pinned justified epoch
+      const oldestEpoch = Math.min(...Array.from(tierMap.keys()).filter((epoch) => epoch !== this.justifiedEpoch));
+      const roots = tierMap.get(oldestEpoch) as Set<RootHex>;
+      tierMap.delete(oldestEpoch);
+
+      const rootHexes = Array.from(roots).join(",");
+      // `>= finalizedEpoch` guards the transient window where an epoch below the (fork-choice)
+      // finalized epoch has not yet been removed by pruneFinalized: its offset would be negative
+      // and JS `%` could spuriously report it as aligned. Such epochs are stale -> delete.
+      if (oldestEpoch >= this.finalizedEpoch && this.alignedForTier(oldestEpoch, tier + 1)) {
+        if (this.persistentTiers[tier + 1] === undefined) {
+          this.persistentTiers[tier + 1] = new Map<Epoch, Set<RootHex>>();
+        }
+        this.persistentTiers[tier + 1].set(oldestEpoch, roots);
+        this.logger.verbose("Migrated persistent checkpoint states to lower tier", {
+          epoch: oldestEpoch,
+          fromTier: tier,
+          toTier: tier + 1,
+          rootHexes,
+        });
+        await this.prunePersistentTier(tier + 1);
+      } else {
+        // the oldest epoch is not aligned to the next tier
+        await this.deleteAllEpochItems(oldestEpoch)
+          .then(() => this.logger.verbose("Pruned checkpoint states from disk", {epoch: oldestEpoch, tier, rootHexes}))
+          .catch((e) => this.logger.debug("Error delete all epoch items", {epoch: oldestEpoch, tier}, e as Error));
+      }
+    }
+  }
+
+  /**
+   * Starting from tier[1], it must be aligned to the finalized epoch
+   * ie given finalized epoch F, tier[n] only epochs of F + n*16, n = 1,2,3...
+   */
+  private alignedForTier(epoch: Epoch, tier: number): boolean {
+    return (epoch - this.finalizedEpoch) % CP_STATE_TIER_BASE ** tier === 0;
   }
 
   /**
@@ -598,9 +701,16 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
     return seedBlockState;
   }
 
-  clear(): void {
-    this.cache.clear();
-    this.epochIndex.clear();
+  async clear(): Promise<void> {
+    // deleteAllEpochItems removes every root of an epoch from disk and clears the cache
+    await Promise.all(
+      Array.from(this.epochIndex.keys()).map((epoch) =>
+        this.deleteAllEpochItems(epoch).catch((e) =>
+          this.logger.debug("Error delete all epoch items on clear", {epoch}, e as Error)
+        )
+      )
+    );
+    this.persistentTiers = [new Map()];
   }
 
   /** ONLY FOR DEBUGGING PURPOSES. For lodestar debug API */
@@ -743,12 +853,18 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
           }
           // overwrite cpKey, this means the state is deleted from memory
           this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey});
+          if (this.isTieredMode) {
+            await this.onPersist(epoch, rootHex);
+          }
         } else {
           if (persistedKey) {
             // persisted file will be eventually deleted by the archive task
             // this also means the state is deleted from memory
             this.cache.set(cpKey, {type: CacheItemType.persisted, value: persistedKey});
             // do not update epochIndex
+            if (this.isTieredMode) {
+              await this.onPersist(epoch, rootHex);
+            }
           } else {
             // delete the state from memory
             this.cache.delete(cpKey);
@@ -790,6 +906,10 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
       this.cache.delete(key);
     }
     this.epochIndex.delete(epoch);
+    // keep the on-disk retention tiers in sync so they never drift from epochIndex
+    for (const tierMap of this.persistentTiers) {
+      tierMap.delete(epoch);
+    }
     this.logger.verbose("Pruned checkpoint states for epoch", {
       epoch,
       persistCount,
@@ -798,7 +918,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
   }
 
   /**
-   * Prune persisted checkpoint states from disk.
+   * Legacy flat-window on-disk pruning, used only when a `maxEpochsOnDisk` is configured.
    * Note that this should handle all possible errors and not throw.
    */
   private prunePersistedStates(): void {
@@ -842,6 +962,11 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
 
     return null;
   }
+
+  /** Tiered pruning is active with the default `Infinity` on-disk cap; a finite cap uses the flat path */
+  private get isTieredMode(): boolean {
+    return this.maxEpochsOnDisk === Infinity;
+  }
 }
 
 export function toCheckpointHex(checkpoint: phase0.Checkpoint): CheckpointHex {

package/src/chain/stateCache/types.ts

@@ -29,7 +29,7 @@ export interface BlockStateCache {
   getSeedState(): IBeaconStateView;
   clear(): void;
   size: number;
-  prune(headStateRootHex: RootHex): void;
+  onCheckpoint(headStateRootHex: RootHex): void;
   dumpSummary(): routes.lodestar.StateCacheItem[];
   /** Expose beacon states stored in cache. Use with caution */
   getStates(): IterableIterator<IBeaconStateView>;
@@ -65,9 +65,9 @@ export interface CheckpointStateCache {
   getLatest(rootHex: RootHex, maxEpoch: Epoch): IBeaconStateView | null;
   getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise<IBeaconStateView | null>;
   updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null;
-  prune(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void;
+  onCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void;
   processState(blockRootHex: RootHex, state: IBeaconStateView): Promise<number>;
-  clear(): void;
+  clear(): Promise<void>;
   dumpSummary(): routes.lodestar.StateCacheItem[];
   /** Expose beacon states stored in cache. Use with caution */
   getStates(): IterableIterator<IBeaconStateView>;

package/src/chain/validation/attestation.ts

@@ -142,13 +142,7 @@ export async function validateGossipAttestationsSameAttData(
   const batchableBls = signatureSets.length >= chain.opts.minSameMessageSignatureSetsToBatch;
   if (batchableBls) {
     // all signature sets should have same signing root since we filtered in network processor
-    signatureValids = await chain.bls.verifySignatureSetsSameMessage(
-      signatureSets.map((set) => {
-        const publicKey = chain.pubkeyCache.getOrThrow(set.index);
-        return {publicKey, signature: set.signature};
-      }),
-      signatureSets[0].signingRoot
-    );
+    signatureValids = await chain.bls.verifySignatureSetsSameMessage(signatureSets, signatureSets[0].signingRoot);
   } else {
     // don't want to block the main thread if there are too few signatures
     signatureValids = await Promise.all(

package/src/chain/validation/attesterSlashing.ts

@@ -44,14 +44,7 @@ export async function validateAttesterSlashing(
   // [REJECT] All of the conditions within process_attester_slashing pass validation.
   try {
     // verifySignature = false, verified in batch below
-    assertValidAttesterSlashing(
-      chain.config,
-      chain.pubkeyCache,
-      state.slot,
-      state.validatorCount,
-      attesterSlashing,
-      false
-    );
+    assertValidAttesterSlashing(chain.config, state.slot, state.validatorCount, attesterSlashing, false);
   } catch (e) {
     throw new AttesterSlashingError(GossipAction.REJECT, {
       code: AttesterSlashingErrorCode.INVALID,

package/src/chain/validation/executionPayloadBid.ts

@@ -1,4 +1,3 @@
-import {PublicKey} from "@chainsafe/blst";
 import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {PAYLOAD_BUILDER_VERSION} from "@lodestar/params";
 import {
@@ -345,7 +344,7 @@ async function validateExecutionPayloadBid(
 
   // [REJECT] `signed_execution_payload_bid.signature` is valid with respect to the `bid.builder_index`.
   const signatureSet = createSingleSignatureSetFromComponents(
-    PublicKey.fromBytes(builder.pubkey),
+    builder.pubkey,
     getExecutionPayloadBidSigningRoot(chain.config, bid),
     signedExecutionPayloadBid.signature
   );

package/src/chain/validation/executionPayloadEnvelope.ts

@@ -170,7 +170,6 @@ async function validateExecutionPayloadEnvelope(
   // by `verify_execution_payload_envelope_signature`.
   const signatureSet = getExecutionPayloadEnvelopeSignatureSet(
     chain.config,
-    chain.pubkeyCache,
     blockState,
     executionPayloadEnvelope,
     payloadInput.proposerIndex

package/src/chain/validation/payloadAttestationMessage.ts

@@ -1,6 +1,5 @@
 import {
-  computeEpochAtSlot,
-  createSingleSignatureSetFromComponents,
+  createIndexedSignatureSetFromComponents,
   getPayloadAttestationDataSigningRoot,
   isStatePostGloas,
 } from "@lodestar/state-transition";
@@ -36,7 +35,6 @@ async function validatePayloadAttestationMessage(
   prioritizeBls = false
 ): Promise<PayloadAttestationValidationResult> {
   const {data, validatorIndex} = payloadAttestationMessage;
-  const epoch = computeEpochAtSlot(data.slot);
 
   // [IGNORE] The message's slot is for the current slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance), i.e. `data.slot == current_slot`.
   if (!chain.clock.isCurrentSlotGivenGossipDisparity(data.slot)) {
@@ -48,9 +46,8 @@ async function validatePayloadAttestationMessage(
   }
 
   // [IGNORE] The `payload_attestation_message` is the first valid message received
-  // from the validator with index `payload_attestation_message.validator_index`.
-  // A single validator can participate PTC at most once per epoch
-  if (chain.seenPayloadAttesters.isKnown(epoch, validatorIndex)) {
+  // for the slot from the validator with index `payload_attestation_message.validator_index`.
+  if (chain.seenPayloadAttesters.isKnown(data.slot, validatorIndex)) {
     throw new PayloadAttestationError(GossipAction.IGNORE, {
       code: PayloadAttestationErrorCode.PAYLOAD_ATTESTATION_ALREADY_KNOWN,
       validatorIndex,
@@ -114,16 +111,8 @@ async function validatePayloadAttestationMessage(
   }
 
   // [REJECT] `payload_attestation_message.signature` is valid with respect to the validator's public key.
-  const validatorPubkey = chain.pubkeyCache.get(validatorIndex);
-  if (!validatorPubkey) {
-    throw new PayloadAttestationError(GossipAction.REJECT, {
-      code: PayloadAttestationErrorCode.INVALID_ATTESTER,
-      attesterIndex: validatorIndex,
-    });
-  }
-
-  const signatureSet = createSingleSignatureSetFromComponents(
-    validatorPubkey,
+  const signatureSet = createIndexedSignatureSetFromComponents(
+    validatorIndex,
     getPayloadAttestationDataSigningRoot(chain.config, data),
     payloadAttestationMessage.signature
   );
@@ -135,7 +124,7 @@ async function validatePayloadAttestationMessage(
   }
 
   // Valid
-  chain.seenPayloadAttesters.add(epoch, validatorIndex);
+  chain.seenPayloadAttesters.add(data.slot, validatorIndex);
 
   return {
     attDataRootHex: toRootHex(ssz.gloas.PayloadAttestationData.hashTreeRoot(data)),