@lodestar/beacon-node 1.41.0-dev.f36ec31497 → 1.41.0-dev.f7a5f4ddda

package/src/chain/blocks/importBlock.ts

@@ -95,7 +95,7 @@ export async function importBlock(
   // Without this, a supernode syncing from behind can accumulate many blocks worth of column
   // data in memory (up to 128 columns per block) causing OOM before persistence catches up.
   await this.unfinalizedBlockWrites.waitForSpace();
-  this.unfinalizedBlockWrites.push([blockInput]).catch((e) => {
+  this.unfinalizedBlockWrites.push(blockInput).catch((e) => {
     if (!isQueueErrorAborted(e)) {
       this.logger.error("Error pushing block to unfinalized write queue", {slot: blockSlot}, e as Error);
     }

package/src/chain/blocks/writeBlockInputToDb.ts

@@ -1,7 +1,9 @@
-import {fulu} from "@lodestar/types";
-import {prettyPrintIndices, toRootHex} from "@lodestar/utils";
+import {ForkPostDeneb, isForkPostDeneb} from "@lodestar/params";
+import {SignedBeaconBlock} from "@lodestar/types";
+import {fromHex, toRootHex} from "@lodestar/utils";
+import {getBlobKzgCommitments} from "../../util/dataColumns.js";
 import {BeaconChain} from "../chain.js";
-import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blockInput/index.js";
+import {IBlockInput, IDataColumnsInput, isBlockInputBlobs, isBlockInputColumns} from "./blockInput/index.js";
 import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js";
 
 /**
@@ -10,129 +12,128 @@ import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js";
  *
  * This operation may be performed before, during or after importing to the fork-choice. As long as errors
  * are handled properly for eventual consistency.
+ *
+ * Block+blobs (pre-fulu) and data columns (fulu+) are written in parallel.
  */
-export async function writeBlockInputToDb(this: BeaconChain, blocksInputs: IBlockInput[]): Promise<void> {
-  const fnPromises: Promise<void>[] = [];
-  // track slots for logging
-  const slots: number[] = [];
-
-  for (const blockInput of blocksInputs) {
-    const block = blockInput.getBlock();
-    const slot = block.message.slot;
-    slots.push(slot);
-    const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
-    const blockRootHex = toRootHex(blockRoot);
-    const blockBytes = this.serializedCache.get(block);
-    if (blockBytes) {
-      // skip serializing data if we already have it
-      this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc();
-      fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes));
-    } else {
-      this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc();
-      fnPromises.push(this.db.block.add(block));
-    }
+export async function writeBlockInputToDb(this: BeaconChain, blockInput: IBlockInput): Promise<void> {
+  const promises: Promise<void>[] = [writeBlockAndBlobsToDb.call(this, blockInput)];
 
-    this.logger.debug("Persist block to hot DB", {
-      slot: block.message.slot,
-      root: blockRootHex,
-      inputType: blockInput.type,
-    });
+  if (isBlockInputColumns(blockInput)) {
+    promises.push(writeDataColumnsToDb.call(this, blockInput));
+  }
 
-    if (!blockInput.hasAllData()) {
-      await blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT);
-    }
+  await Promise.all(promises);
+  this.logger.debug("Persisted blockInput to db", {slot: blockInput.slot, root: blockInput.blockRootHex});
+}
 
-    // NOTE: Old data is pruned on archive
-    if (isBlockInputColumns(blockInput)) {
-      if (!blockInput.hasComputedAllData()) {
-        // Supernodes may only have a subset of the data columns by the time the block begins to be imported
-        // because full data availability can be assumed after NUMBER_OF_COLUMNS / 2 columns are available.
-        // Here, however, all data columns must be fully available/reconstructed before persisting to the DB.
-        await blockInput.waitForComputedAllData(BLOB_AVAILABILITY_TIMEOUT).catch(() => {
-          this.logger.debug("Failed to wait for computed all data", {slot, blockRoot: blockRootHex});
-        });
-      }
+async function writeBlockAndBlobsToDb(this: BeaconChain, blockInput: IBlockInput): Promise<void> {
+  const block = blockInput.getBlock();
+  const slot = block.message.slot;
+  const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message);
+  const blockRootHex = toRootHex(blockRoot);
+  const numBlobs = isForkPostDeneb(blockInput.forkName)
+    ? getBlobKzgCommitments(blockInput.forkName, block as SignedBeaconBlock<ForkPostDeneb>).length
+    : undefined;
+  const fnPromises: Promise<void>[] = [];
 
-      const {custodyColumns} = this.custodyConfig;
-      const blobsLen = (block.message as fulu.BeaconBlock).body.blobKzgCommitments.length;
-      let dataColumnsLen: number;
-      if (blobsLen === 0) {
-        dataColumnsLen = 0;
-      } else {
-        dataColumnsLen = custodyColumns.length;
-      }
+  const blockBytes = this.serializedCache.get(block);
+  if (blockBytes) {
+    // skip serializing data if we already have it
+    this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc();
+    fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes));
+  } else {
+    this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc();
+    fnPromises.push(this.db.block.add(block));
+  }
 
-      const dataColumnSidecars = blockInput.getCustodyColumns();
-      if (dataColumnSidecars.length !== dataColumnsLen) {
-        this.logger.debug(
-          `Invalid dataColumnSidecars=${dataColumnSidecars.length} for custody expected custodyColumnsLen=${dataColumnsLen}`
-        );
-      }
+  this.logger.debug("Persist block to hot DB", {slot, root: blockRootHex, inputType: blockInput.type, numBlobs});
 
-      const binaryPuts = [];
-      const nonbinaryPuts = [];
-      for (const dataColumnSidecar of dataColumnSidecars) {
-        // skip reserializing column if we already have it
-        const serialized = this.serializedCache.get(dataColumnSidecar);
-        if (serialized) {
-          binaryPuts.push({key: dataColumnSidecar.index, value: serialized});
-        } else {
-          nonbinaryPuts.push(dataColumnSidecar);
+  if (isBlockInputBlobs(blockInput)) {
+    fnPromises.push(
+      (async () => {
+        if (!blockInput.hasAllData()) {
+          await blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT);
         }
-      }
-      fnPromises.push(this.db.dataColumnSidecar.putManyBinary(blockRoot, binaryPuts));
-      fnPromises.push(this.db.dataColumnSidecar.putMany(blockRoot, nonbinaryPuts));
-      this.logger.debug("Persisted dataColumnSidecars to hot DB", {
-        slot: block.message.slot,
-        root: blockRootHex,
-        dataColumnSidecars: dataColumnSidecars.length,
-        numBlobs: blobsLen,
-        custodyColumns: custodyColumns.length,
-      });
-    } else if (isBlockInputBlobs(blockInput)) {
-      const blobSidecars = blockInput.getBlobs();
-      fnPromises.push(this.db.blobSidecars.add({blockRoot, slot: block.message.slot, blobSidecars}));
-      this.logger.debug("Persisted blobSidecars to hot DB", {
-        blobsLen: blobSidecars.length,
-        slot: block.message.slot,
-        root: blockRootHex,
-      });
-    }
+        const blobSidecars = blockInput.getBlobs();
+        await this.db.blobSidecars.add({blockRoot, slot, blobSidecars});
+        this.logger.debug("Persisted blobSidecars to hot DB", {
+          slot,
+          root: blockRootHex,
+          numBlobs: blobSidecars.length,
+        });
+      })()
+    );
+  }
+
+  await Promise.all(fnPromises);
+}
+
+/**
+ * Persists data columns to DB for a given block. Accepts a narrow sub-interface of IBlockInput
+ * so it can be reused across forks (e.g. Fulu, Gloas).
+ *
+ * NOTE: Old data is pruned on archive.
+ */
+export async function writeDataColumnsToDb(this: BeaconChain, blockInput: IDataColumnsInput): Promise<void> {
+  const {slot, blockRootHex} = blockInput;
+  const blockRoot = fromHex(blockRootHex);
 
-    await Promise.all(fnPromises);
-    this.logger.debug("Persisted blocksInput to db", {
-      blocksInput: blocksInputs.length,
-      slots: prettyPrintIndices(slots),
+  if (!blockInput.hasComputedAllData()) {
+    // Supernodes may only have a subset of the data columns by the time the block begins to be imported
+    // because full data availability can be assumed after NUMBER_OF_COLUMNS / 2 columns are available.
+    // Here, however, all data columns must be fully available/reconstructed before persisting to the DB.
+    await blockInput.waitForComputedAllData(BLOB_AVAILABILITY_TIMEOUT).catch(() => {
+      this.logger.debug("Failed to wait for computed all data", {slot, blockRoot: blockRootHex});
     });
   }
+
+  const {custodyColumns} = this.custodyConfig;
+  const dataColumnSidecars = blockInput.getCustodyColumns();
+
+  const binaryPuts: {key: number; value: Uint8Array}[] = [];
+  const nonbinaryPuts = [];
+  for (const dataColumnSidecar of dataColumnSidecars) {
+    // skip reserializing column if we already have it
+    const serialized = this.serializedCache.get(dataColumnSidecar);
+    if (serialized) {
+      binaryPuts.push({key: dataColumnSidecar.index, value: serialized});
+    } else {
+      nonbinaryPuts.push(dataColumnSidecar);
+    }
+  }
+
+  await Promise.all([
+    this.db.dataColumnSidecar.putManyBinary(blockRoot, binaryPuts),
+    this.db.dataColumnSidecar.putMany(blockRoot, nonbinaryPuts),
+  ]);
+
+  this.logger.debug("Persisted dataColumnSidecars to hot DB", {
+    slot,
+    root: blockRootHex,
+    dataColumnSidecars: dataColumnSidecars.length,
+    custodyColumns: custodyColumns.length,
+    numBlobs: dataColumnSidecars[0]?.column.length,
+  });
 }
 
-export async function persistBlockInputs(this: BeaconChain, blockInputs: IBlockInput[]): Promise<void> {
+export async function persistBlockInput(this: BeaconChain, blockInput: IBlockInput): Promise<void> {
   await writeBlockInputToDb
-    .call(this, blockInputs)
+    .call(this, blockInput)
     .catch((e) => {
       this.logger.debug(
         "Error persisting block input in hot db",
         {
-          count: blockInputs.length,
-          slot: blockInputs[0].slot,
-          root: blockInputs[0].blockRootHex,
+          slot: blockInput.slot,
+          root: blockInput.blockRootHex,
         },
         e
       );
     })
     .finally(() => {
-      for (const blockInput of blockInputs) {
-        this.seenBlockInputCache.prune(blockInput.blockRootHex);
-      }
-      // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable.
-      // Clear here (after the DB write) so that writeBlockInputToDb can still use the cached serialized bytes.
-      this.serializedCache.clear();
-      if (blockInputs.length === 1) {
-        this.logger.debug("Pruned block input", {
-          slot: blockInputs[0].slot,
-          root: blockInputs[0].blockRootHex,
-        });
-      }
+      this.seenBlockInputCache.prune(blockInput.blockRootHex);
+      this.logger.debug("Pruned block input", {
+        slot: blockInput.slot,
+        root: blockInput.blockRootHex,
+      });
     });
 }

package/src/chain/chain.ts

@@ -2,7 +2,13 @@ import path from "node:path";
 import {PrivateKey} from "@libp2p/interface";
 import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz";
 import {BeaconConfig} from "@lodestar/config";
-import {CheckpointWithHex, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice";
+import {
+  CheckpointWithHex,
+  CheckpointWithPayloadStatus,
+  IForkChoice,
+  ProtoBlock,
+  UpdateHeadOpt,
+} from "@lodestar/fork-choice";
 import {LoggerNode} from "@lodestar/logger/node";
 import {
   BUILDER_INDEX_SELF_BUILD,
@@ -58,7 +64,7 @@ import {Logger, fromHex, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRoot
 import {ProcessShutdownCallback} from "@lodestar/validator";
 import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js";
 import {IBeaconDb} from "../db/index.js";
-import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../db/repositories/blobSidecars.ts";
+import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../db/repositories/blobSidecars.js";
 import {BuilderStatus} from "../execution/builder/http.js";
 import {IExecutionBuilder, IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/index.js";
@@ -69,15 +75,15 @@ import {CustodyConfig, getValidatorsCustodyRequirement} from "../util/dataColumn
 import {callInNextEventLoop} from "../util/eventLoop.js";
 import {ensureDir, writeIfNotExist} from "../util/file.js";
 import {isOptimisticBlock} from "../util/forkChoice.js";
-import {JobItemQueue} from "../util/queue/itemQueue.ts";
+import {JobItemQueue} from "../util/queue/itemQueue.js";
 import {SerializedCache} from "../util/serializedCache.js";
-import {getSlotFromSignedBeaconBlockSerialized} from "../util/sszBytes.ts";
+import {getSlotFromSignedBeaconBlockSerialized} from "../util/sszBytes.js";
 import {ArchiveStore} from "./archiveStore/archiveStore.js";
 import {CheckpointBalancesCache} from "./balancesCache.js";
 import {BeaconProposerCache} from "./beaconProposerCache.js";
 import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blocks/blockInput/index.js";
 import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js";
-import {persistBlockInputs} from "./blocks/writeBlockInputToDb.ts";
+import {persistBlockInput} from "./blocks/writeBlockInputToDb.js";
 import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js";
 import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js";
 import {ChainEvent, ChainEventEmitter} from "./emitter.js";
@@ -164,7 +170,7 @@ export class BeaconChain implements IBeaconChain {
   readonly lightClientServer?: LightClientServer;
   readonly reprocessController: ReprocessController;
   readonly archiveStore: ArchiveStore;
-  readonly unfinalizedBlockWrites: JobItemQueue<[IBlockInput[]], void>;
+  readonly unfinalizedBlockWrites: JobItemQueue<[IBlockInput], void>;
 
   // Ops pool
   readonly attestationPool: AttestationPool;
@@ -314,12 +320,14 @@ export class BeaconChain implements IBeaconChain {
 
     this.beaconProposerCache = new BeaconProposerCache(opts);
     this.checkpointBalancesCache = new CheckpointBalancesCache();
+    this.serializedCache = new SerializedCache();
     this.seenBlockInputCache = new SeenBlockInput({
       config,
       custodyConfig: this.custodyConfig,
       clock,
       chainEvents: emitter,
       signal,
+      serializedCache: this.serializedCache,
       metrics,
       logger,
     });
@@ -406,8 +414,6 @@ export class BeaconChain implements IBeaconChain {
     this.bls = bls;
     this.emitter = emitter;
 
-    this.serializedCache = new SerializedCache();
-
     this.getBlobsTracker = new GetBlobsTracker({
       logger,
       executionEngine: this.executionEngine,
@@ -429,7 +435,7 @@ export class BeaconChain implements IBeaconChain {
     );
 
     this.unfinalizedBlockWrites = new JobItemQueue(
-      persistBlockInputs.bind(this),
+      persistBlockInput.bind(this),
       {
         maxLength: DEFAULT_MAX_PENDING_UNFINALIZED_BLOCK_WRITES,
         signal,
@@ -1186,7 +1192,7 @@ export class BeaconChain implements IBeaconChain {
    * @param blockState state that declares justified checkpoint `checkpoint`
    */
   private justifiedBalancesGetter(
-    checkpoint: CheckpointWithHex,
+    checkpoint: CheckpointWithPayloadStatus,
     blockState: CachedBeaconStateAllForks
   ): EffectiveBalanceIncrements {
     this.metrics?.balancesCache.requests.inc();
@@ -1225,7 +1231,7 @@ export class BeaconChain implements IBeaconChain {
    * @param blockState state that declares justified checkpoint `checkpoint`
    */
   private closestJustifiedBalancesStateToCheckpoint(
-    checkpoint: CheckpointWithHex,
+    checkpoint: CheckpointWithPayloadStatus,
     blockState: CachedBeaconStateAllForks
   ): {state: CachedBeaconStateAllForks; stateId: string; shouldWarn: boolean} {
     const state = this.regen.getCheckpointStateSync(checkpoint);
@@ -1239,7 +1245,10 @@ export class BeaconChain implements IBeaconChain {
     }
 
     // Find a state in the same branch of checkpoint at same epoch. Balances should exactly the same
-    for (const descendantBlock of this.forkChoice.forwardIterateDescendants(checkpoint.rootHex)) {
+    for (const descendantBlock of this.forkChoice.forwardIterateDescendants(
+      checkpoint.rootHex,
+      checkpoint.payloadStatus
+    )) {
       if (computeEpochAtSlot(descendantBlock.slot) === checkpoint.epoch) {
         const descendantBlockState = this.regen.getStateSync(descendantBlock.stateRoot);
         if (descendantBlockState) {
@@ -1255,7 +1264,10 @@ export class BeaconChain implements IBeaconChain {
 
     // Find a state in the same branch of checkpoint at a latter epoch. Balances are not the same, but should be close
     // Note: must call .forwardIterateDescendants() again since nodes are not sorted
-    for (const descendantBlock of this.forkChoice.forwardIterateDescendants(checkpoint.rootHex)) {
+    for (const descendantBlock of this.forkChoice.forwardIterateDescendants(
+      checkpoint.rootHex,
+      checkpoint.payloadStatus
+    )) {
       if (computeEpochAtSlot(descendantBlock.slot) > checkpoint.epoch) {
         const descendantBlockState = this.regen.getStateSync(descendantBlock.stateRoot);
         if (descendantBlockState) {

package/src/chain/emitter.ts

@@ -1,7 +1,7 @@
 import {EventEmitter} from "node:events";
 import {StrictEventEmitter} from "strict-event-emitter-types";
 import {routes} from "@lodestar/api";
-import {CheckpointWithPayload} from "@lodestar/fork-choice";
+import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice";
 import {CachedBeaconStateAllForks} from "@lodestar/state-transition";
 import {DataColumnSidecars, RootHex, deneb, phase0} from "@lodestar/types";
 import {PeerIdStr} from "../util/peerId.js";
@@ -83,8 +83,8 @@ export type ChainEventData = {
 export type IChainEvents = ApiEvents & {
   [ChainEvent.checkpoint]: (checkpoint: phase0.Checkpoint, state: CachedBeaconStateAllForks) => void;
 
-  [ChainEvent.forkChoiceJustified]: (checkpoint: CheckpointWithPayload) => void;
-  [ChainEvent.forkChoiceFinalized]: (checkpoint: CheckpointWithPayload) => void;
+  [ChainEvent.forkChoiceJustified]: (checkpoint: CheckpointWithPayloadStatus) => void;
+  [ChainEvent.forkChoiceFinalized]: (checkpoint: CheckpointWithPayloadStatus) => void;
 
   [ChainEvent.updateTargetCustodyGroupCount]: (targetGroupCount: number) => void;
 

package/src/chain/errors/executionPayloadBid.ts

@@ -1,5 +1,5 @@
 import {BuilderIndex, RootHex, Slot} from "@lodestar/types";
-import {GossipActionError} from "./gossipValidation.ts";
+import {GossipActionError} from "./gossipValidation.js";
 
 export enum ExecutionPayloadBidErrorCode {
   BUILDER_NOT_ELIGIBLE = "EXECUTION_PAYLOAD_BID_ERROR_BUILDER_NOT_ELIGIBLE",

package/src/chain/errors/payloadAttestation.ts

@@ -1,5 +1,5 @@
 import {RootHex, Slot, ValidatorIndex} from "@lodestar/types";
-import {GossipActionError} from "./gossipValidation.ts";
+import {GossipActionError} from "./gossipValidation.js";
 
 export enum PayloadAttestationErrorCode {
   NOT_CURRENT_SLOT = "PAYLOAD_ATTESTATION_ERROR_NOT_CURRENT_SLOT",

package/src/chain/produceBlock/computeNewStateRoot.ts

@@ -61,7 +61,6 @@ export function computeNewStateRoot(
  * Compute the state root after processing an execution payload envelope.
  * Similar to `computeNewStateRoot` but for payload envelope processing.
  *
- * The `postBlockState` is mutated in place, callers must ensure it is not needed afterward.
  */
 export function computeEnvelopeStateRoot(
   metrics: Metrics | null,
@@ -74,13 +73,15 @@ export function computeEnvelopeStateRoot(
   };
 
   const processEnvelopeTimer = metrics?.blockPayload.executionPayloadEnvelopeProcessingTime.startTimer();
-  processExecutionPayloadEnvelope(postBlockState, signedEnvelope, false);
+  const postEnvelopeState = processExecutionPayloadEnvelope(postBlockState, signedEnvelope, false, {
+    dontTransferCache: true,
+  });
   processEnvelopeTimer?.();
 
   const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({
     source: StateHashTreeRootSource.computeEnvelopeStateRoot,
   });
-  const stateRoot = postBlockState.hashTreeRoot();
+  const stateRoot = postEnvelopeState.hashTreeRoot();
   hashTreeRootTimer?.();
 
   return stateRoot;

package/src/chain/regen/regen.ts

@@ -158,7 +158,7 @@ export class StateRegenerator implements IStateRegeneratorInternal {
 
     const getSeedStateTimer = this.modules.metrics?.regenGetState.getSeedState.startTimer({caller});
     // iterateAncestorBlocks only returns ancestor blocks, not the block itself
-    for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot)) {
+    for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot, block.payloadStatus)) {
       state = this.modules.blockStateCache.get(b.stateRoot);
       if (state) {
         break;

package/src/chain/seenCache/seenGossipBlockInput.ts

@@ -17,6 +17,7 @@ import {Metrics} from "../../metrics/metrics.js";
 import {MAX_LOOK_AHEAD_EPOCHS} from "../../sync/constants.js";
 import {IClock} from "../../util/clock.js";
 import {CustodyConfig} from "../../util/dataColumns.js";
+import {SerializedCache} from "../../util/serializedCache.js";
 import {
   BlockInput,
   BlockInputBlobs,
@@ -55,6 +56,7 @@ export type SeenBlockInputCacheModules = {
   chainEvents: ChainEventEmitter;
   signal: AbortSignal;
   custodyConfig: CustodyConfig;
+  serializedCache: SerializedCache;
   metrics: Metrics | null;
   logger?: Logger;
 };
@@ -101,6 +103,7 @@ export class SeenBlockInput {
   private readonly clock: IClock;
   private readonly chainEvents: ChainEventEmitter;
   private readonly signal: AbortSignal;
+  private readonly serializedCache: SerializedCache;
   private readonly metrics: Metrics | null;
   private readonly logger?: Logger;
   private blockInputs = new Map<RootHex, IBlockInput>();
@@ -109,19 +112,35 @@ export class SeenBlockInput {
   // and the signature to ensure we only skip verification if both match
   private verifiedProposerSignatures = new Map<Slot, Map<RootHex, BLSSignature>>();
 
-  constructor({config, custodyConfig, clock, chainEvents, signal, metrics, logger}: SeenBlockInputCacheModules) {
+  constructor({
+    config,
+    custodyConfig,
+    clock,
+    chainEvents,
+    signal,
+    serializedCache,
+    metrics,
+    logger,
+  }: SeenBlockInputCacheModules) {
     this.config = config;
     this.custodyConfig = custodyConfig;
     this.clock = clock;
     this.chainEvents = chainEvents;
     this.signal = signal;
+    this.serializedCache = serializedCache;
     this.metrics = metrics;
     this.logger = logger;
 
     if (metrics) {
-      metrics.seenCache.blockInput.blockInputCount.addCollect(() =>
-        metrics.seenCache.blockInput.blockInputCount.set(this.blockInputs.size)
-      );
+      metrics.seenCache.blockInput.blockInputCount.addCollect(() => {
+        metrics.seenCache.blockInput.blockInputCount.set(this.blockInputs.size);
+        metrics.seenCache.blockInput.serializedObjectCount.set(
+          Array.from(this.blockInputs.values()).reduce(
+            (count, blockInput) => count + blockInput.getSerializedCacheKeys().length,
+            0
+          )
+        );
+      });
     }
 
     this.chainEvents.on(ChainEvent.forkChoiceFinalized, this.onFinalized);
@@ -142,7 +161,10 @@ export class SeenBlockInput {
    * Removes the single BlockInput from the cache
    */
   remove(rootHex: RootHex): void {
-    this.blockInputs.delete(rootHex);
+    const blockInput = this.blockInputs.get(rootHex);
+    if (blockInput) {
+      this.evictBlockInput(blockInput);
+    }
   }
 
   /**
@@ -154,7 +176,7 @@ export class SeenBlockInput {
     let deletedCount = 0;
     while (blockInput) {
       deletedCount++;
-      this.blockInputs.delete(blockInput.blockRootHex);
+      this.evictBlockInput(blockInput);
       blockInput = this.blockInputs.get(parentRootHex ?? "");
       parentRootHex = blockInput?.parentRootHex;
     }
@@ -165,10 +187,10 @@ export class SeenBlockInput {
   onFinalized = (checkpoint: CheckpointWithHex) => {
     let deletedCount = 0;
     const cutoffSlot = computeStartSlotAtEpoch(checkpoint.epoch);
-    for (const [rootHex, blockInput] of this.blockInputs) {
+    for (const [, blockInput] of this.blockInputs) {
       if (blockInput.slot < cutoffSlot) {
         deletedCount++;
-        this.blockInputs.delete(rootHex);
+        this.evictBlockInput(blockInput);
       }
     }
     this.logger?.debug(`BlockInputCache.onFinalized deleted ${deletedCount} cached BlockInputs`);
@@ -408,14 +430,20 @@ export class SeenBlockInput {
 
     if (itemsToDelete > 0) {
       const sorted = [...this.blockInputs.entries()].sort((a, b) => a[1].slot - b[1].slot);
-      for (const [rootHex] of sorted) {
-        this.blockInputs.delete(rootHex);
+      for (const [, blockInput] of sorted) {
+        this.evictBlockInput(blockInput);
         itemsToDelete--;
         if (itemsToDelete <= 0) return;
       }
     }
     pruneSetToMax(this.verifiedProposerSignatures, MAX_BLOCK_INPUT_CACHE_SIZE);
   }
+
+  private evictBlockInput(blockInput: IBlockInput): void {
+    // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable
+    this.serializedCache.delete(blockInput.getSerializedCacheKeys());
+    this.blockInputs.delete(blockInput.blockRootHex);
+  }
 }
 
 enum SeenBlockInputCacheErrorCode {

package/src/db/index.ts

@@ -1,2 +1,3 @@
 export {BeaconDb} from "./beacon.js";
+export {Bucket} from "./buckets.js";
 export type {IBeaconDb} from "./interface.js";

package/src/metrics/metrics/lodestar.ts

@@ -1461,6 +1461,10 @@ export function createLodestarMetrics(
           name: "lodestar_seen_block_input_cache_size",
           help: "Number of cached BlockInputs",
         }),
+        serializedObjectCount: register.gauge({
+          name: "lodestar_seen_block_input_cache_serialized_object_count",
+          help: "Number of serialized objects retained by cached BlockInputs",
+        }),
         duplicateBlockCount: register.gauge<{source: BlockInputSource}>({
           name: "lodestar_seen_block_input_cache_duplicate_block_count",
           help: "Total number of duplicate blocks that pass validation and attempt to be cached but are known",

package/src/network/network.ts

@@ -539,7 +539,8 @@ export class Network implements INetwork {
         this.config.getForkSeq(this.clock.currentSlot) >= ForkSeq.altair ? [Version.V2] : [Version.V2, Version.V1],
         request
       ),
-      request
+      request,
+      this.chain.serializedCache
     );
   }
 

package/src/network/processor/gossipHandlers.ts

@@ -71,7 +71,7 @@ import {validateGossipPayloadAttestationMessage} from "../../chain/validation/pa
 import {OpSource} from "../../chain/validatorMonitor.js";
 import {Metrics} from "../../metrics/index.js";
 import {kzgCommitmentToVersionedHash} from "../../util/blobs.js";
-import {getBlobKzgCommitments} from "../../util/dataColumns.ts";
+import {getBlobKzgCommitments} from "../../util/dataColumns.js";
 import {INetworkCore} from "../core/index.js";
 import {NetworkEventBus} from "../events.js";
 import {

package/src/network/processor/gossipValidatorFn.ts

@@ -11,7 +11,7 @@ import {
   GossipValidatorBatchFn,
   GossipValidatorFn,
 } from "../gossip/interface.js";
-import {prettyPrintPeerIdStr} from "../util.ts";
+import {prettyPrintPeerIdStr} from "../util.js";
 
 export type ValidatorFnModules = {
   config: ChainForkConfig;

package/src/network/reqresp/ReqRespBeaconNode.ts

@@ -19,7 +19,7 @@ import {callInNextEventLoop} from "../../util/eventLoop.js";
 import {NetworkCoreMetrics} from "../core/metrics.js";
 import {INetworkEventBus, NetworkEvent} from "../events.js";
 import {MetadataController} from "../metadata.js";
-import {ClientKind} from "../peers/client.ts";
+import {ClientKind} from "../peers/client.js";
 import {PeersData} from "../peers/peersData.js";
 import {IPeerRpcScoreStore, PeerAction} from "../peers/score/index.js";
 import {StatusCache} from "../statusCache.js";

package/src/network/reqresp/handlers/beaconBlocksByRange.ts

@@ -6,7 +6,7 @@ import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {deneb, phase0} from "@lodestar/types";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
-import {prettyPrintPeerId} from "../../util.ts";
+import {prettyPrintPeerId} from "../../util.js";
 
 // TODO: Unit test
 
@@ -47,9 +47,10 @@ export async function* onBeaconBlocksByRange(
 
   // Non-finalized range of blocks
   if (endSlot > finalizedSlot) {
-    const headRoot = chain.forkChoice.getHeadRoot();
+    const headBlock = chain.forkChoice.getHead();
+    const headRoot = headBlock.blockRoot;
     // TODO DENEB: forkChoice should mantain an array of canonical blocks, and change only on reorg
-    const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot);
+    const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
     // getAllAncestorBlocks response includes the head node, so it's the full chain.
 
     // Iterate head chain with ascending block numbers