@lodestar/beacon-node 1.41.0-dev.0df187678b → 1.41.0-dev.165a02f873

package/lib/util/serializedCache.d.ts

@@ -4,14 +4,14 @@
  * This is a thin wrapper around WeakMap
  */
 export declare class SerializedCache {
-    map: WeakMap<object, Uint8Array>;
+    private map;
     get(obj: object): Uint8Array | undefined;
     set(obj: object, serialized: Uint8Array): void;
     /**
-     * Replace the internal WeakMap to force GC of all cached entries.
-     * Must only be called after all DB writes that may read from this cache have completed,
+     * Delete cached serialized entries for the provided object references.
+     * Must only be called after all DB writes that read from this cache for these objects have completed,
      * otherwise cached serialized bytes will be unavailable and data will be re-serialized unnecessarily.
      */
-    clear(): void;
+    delete(objs: object[]): void;
 }
 //# sourceMappingURL=serializedCache.d.ts.map

package/lib/util/serializedCache.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"serializedCache.d.ts","sourceRoot":"","sources":["../../src/util/serializedCache.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,qBAAa,eAAe;IAC1B,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAiB;IAEjD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAIxC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,GAAG,IAAI;IAI9C;;;;OAIG;IACH,KAAK,IAAI,IAAI;CAGd"}
+{"version":3,"file":"serializedCache.d.ts","sourceRoot":"","sources":["../../src/util/serializedCache.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,GAAG,CAA8C;IAEzD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAIxC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,GAAG,IAAI;IAI9C;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI;CAK7B"}

package/lib/util/serializedCache.js

@@ -12,12 +12,14 @@ export class SerializedCache {
         this.map.set(obj, serialized);
     }
     /**
-     * Replace the internal WeakMap to force GC of all cached entries.
-     * Must only be called after all DB writes that may read from this cache have completed,
+     * Delete cached serialized entries for the provided object references.
+     * Must only be called after all DB writes that read from this cache for these objects have completed,
      * otherwise cached serialized bytes will be unavailable and data will be re-serialized unnecessarily.
      */
-    clear() {
-        this.map = new WeakMap();
+    delete(objs) {
+        for (const obj of objs) {
+            this.map.delete(obj);
+        }
     }
 }
 //# sourceMappingURL=serializedCache.js.map

package/lib/util/serializedCache.js.map

@@ -1 +1 @@
-{"version":3,"file":"serializedCache.js","sourceRoot":"","sources":["../../src/util/serializedCache.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,OAAO,eAAe;IAC1B,GAAG,GAAgC,IAAI,OAAO,EAAE,CAAC;IAEjD,GAAG,CAAC,GAAW;QACb,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,GAAG,CAAC,GAAW,EAAE,UAAsB;QACrC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,KAAK;QACH,IAAI,CAAC,GAAG,GAAG,IAAI,OAAO,EAAE,CAAC;IAC3B,CAAC;CACF"}
+{"version":3,"file":"serializedCache.js","sourceRoot":"","sources":["../../src/util/serializedCache.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,OAAO,eAAe;IAClB,GAAG,GAAgC,IAAI,OAAO,EAAE,CAAC;IAEzD,GAAG,CAAC,GAAW;QACb,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,GAAG,CAAC,GAAW,EAAE,UAAsB;QACrC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,IAAc;QACnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;CACF"}

package/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.41.0-dev.0df187678b",
+  "version": "1.41.0-dev.165a02f873",
   "type": "module",
   "exports": {
     ".": {
@@ -134,18 +134,18 @@
     "@libp2p/peer-id": "^6.0.4",
     "@libp2p/prometheus-metrics": "^5.0.13",
     "@libp2p/tcp": "^11.0.12",
-    "@lodestar/api": "^1.41.0-dev.0df187678b",
-    "@lodestar/config": "^1.41.0-dev.0df187678b",
-    "@lodestar/db": "^1.41.0-dev.0df187678b",
-    "@lodestar/fork-choice": "^1.41.0-dev.0df187678b",
-    "@lodestar/light-client": "^1.41.0-dev.0df187678b",
-    "@lodestar/logger": "^1.41.0-dev.0df187678b",
-    "@lodestar/params": "^1.41.0-dev.0df187678b",
-    "@lodestar/reqresp": "^1.41.0-dev.0df187678b",
-    "@lodestar/state-transition": "^1.41.0-dev.0df187678b",
-    "@lodestar/types": "^1.41.0-dev.0df187678b",
-    "@lodestar/utils": "^1.41.0-dev.0df187678b",
-    "@lodestar/validator": "^1.41.0-dev.0df187678b",
+    "@lodestar/api": "^1.41.0-dev.165a02f873",
+    "@lodestar/config": "^1.41.0-dev.165a02f873",
+    "@lodestar/db": "^1.41.0-dev.165a02f873",
+    "@lodestar/fork-choice": "^1.41.0-dev.165a02f873",
+    "@lodestar/light-client": "^1.41.0-dev.165a02f873",
+    "@lodestar/logger": "^1.41.0-dev.165a02f873",
+    "@lodestar/params": "^1.41.0-dev.165a02f873",
+    "@lodestar/reqresp": "^1.41.0-dev.165a02f873",
+    "@lodestar/state-transition": "^1.41.0-dev.165a02f873",
+    "@lodestar/types": "^1.41.0-dev.165a02f873",
+    "@lodestar/utils": "^1.41.0-dev.165a02f873",
+    "@lodestar/validator": "^1.41.0-dev.165a02f873",
     "@multiformats/multiaddr": "^13.0.1",
     "datastore-core": "^11.0.2",
     "datastore-fs": "^11.0.2",
@@ -168,7 +168,7 @@
     "@libp2p/interface-internal": "^3.0.12",
     "@libp2p/logger": "^6.2.2",
     "@libp2p/utils": "^7.0.12",
-    "@lodestar/spec-test-util": "^1.41.0-dev.0df187678b",
+    "@lodestar/spec-test-util": "^1.41.0-dev.165a02f873",
     "@types/js-yaml": "^4.0.5",
     "@types/qs": "^6.9.7",
     "@types/tmp": "^0.2.3",
@@ -185,5 +185,5 @@
     "beacon",
     "blockchain"
   ],
-  "gitHead": "322d242cb2c57638744d875afb2ea30d96394f02"
+  "gitHead": "08a97ee173fe65d3c757a279a740741ca8bf1502"
 }

package/src/chain/blocks/blockInput/blockInput.ts

@@ -105,6 +105,7 @@ abstract class AbstractBlockInput<F extends ForkName = ForkName, TData extends D
   }
 
   abstract addBlock(props: AddBlock<F>): void;
+  abstract getSerializedCacheKeys(): object[];
 
   hasBlock(): boolean {
     return this.state.hasBlock;
@@ -243,6 +244,10 @@ export class BlockInputPreData extends AbstractBlockInput<ForkPreDeneb, null> {
       );
     }
   }
+
+  getSerializedCacheKeys(): object[] {
+    return [this.state.block];
+  }
 }
 
 // Blobs DA
@@ -526,6 +531,20 @@ export class BlockInputBlobs extends AbstractBlockInput<ForkBlobsDA, deneb.BlobS
   getBlobs(): deneb.BlobSidecars {
     return this.getAllBlobsWithSource().map(({blobSidecar}) => blobSidecar);
   }
+
+  getSerializedCacheKeys(): object[] {
+    const objects: object[] = [];
+
+    if (this.state.hasBlock) {
+      objects.push(this.state.block);
+    }
+
+    for (const {blobSidecar} of this.blobsCache.values()) {
+      objects.push(blobSidecar);
+    }
+
+    return objects;
+  }
 }
 
 function blockAndBlobArePaired(block: SignedBeaconBlock<ForkBlobsDA>, blobSidecar: deneb.BlobSidecar): boolean {
@@ -906,6 +925,18 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     }
     return Promise.resolve(this.getSampledColumns());
   }
+
+  getSerializedCacheKeys(): object[] {
+    const objects: object[] = [];
+
+    if (this.state.hasBlock) {
+      objects.push(this.state.block);
+    }
+
+    objects.push(...this.getAllColumns());
+
+    return objects;
+  }
 }
 
 type BlockInputNoDataState = {
@@ -967,4 +998,8 @@ export class BlockInputNoData extends AbstractBlockInput<ForkPostGloas, null> {
     return (this.state.block.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message
       .blobKzgCommitments;
   }
+
+  getSerializedCacheKeys(): object[] {
+    return [this.state.block];
+  }
 }

package/src/chain/blocks/blockInput/types.ts

@@ -151,6 +151,11 @@ export interface IBlockInput<F extends ForkName = ForkName, TData extends DAData
 
   /** Only safe to call when `hasBlockAndAllData` is true */
   getTimeComplete(): number;
+  /**
+   * Return object references used as keys in `SerializedCache` that can be safely removed
+   * once this block input lifecycle has completed.
+   */
+  getSerializedCacheKeys(): object[];
 
   waitForBlock(timeout: number, signal?: AbortSignal): Promise<SignedBeaconBlock<F>>;
   waitForAllData(timeout: number, signal?: AbortSignal): Promise<TData>;

package/src/chain/blocks/writeBlockInputToDb.ts

@@ -3,13 +3,7 @@ import {SignedBeaconBlock} from "@lodestar/types";
 import {fromHex, toRootHex} from "@lodestar/utils";
 import {getBlobKzgCommitments} from "../../util/dataColumns.js";
 import {BeaconChain} from "../chain.js";
-import {
-  IBlockInput,
-  IDataColumnsInput,
-  isBlockInputBlobs,
-  isBlockInputColumns,
-  isBlockInputNoData,
-} from "./blockInput/index.js";
+import {IBlockInput, IDataColumnsInput, isBlockInputBlobs, isBlockInputColumns} from "./blockInput/index.js";
 import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js";
 
 /**
@@ -137,17 +131,6 @@ export async function persistBlockInput(this: BeaconChain, blockInput: IBlockInp
     })
     .finally(() => {
       this.seenBlockInputCache.prune(blockInput.blockRootHex);
-      // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable.
-      // Clear here (after the DB write) so that writeBlockInputToDb can still use the cached serialized bytes.
-      //
-      // For Gloas (BlockInputNoData), the execution payload and columns arrive separately after the beacon block.
-      // Do NOT clear the cache here — it must remain available for writeDataColumnsToDb when the payload arrives.
-      // The cache is cleared in the Gloas payload persistence path instead.
-      if (!isBlockInputNoData(blockInput)) {
-        // TODO: enhance this SerializedCache for Gloas because payload may not come
-        // see https://github.com/ChainSafe/lodestar/pull/8974#discussion_r2885598229
-        this.serializedCache.clear();
-      }
       this.logger.debug("Pruned block input", {
         slot: blockInput.slot,
         root: blockInput.blockRootHex,

package/src/chain/chain.ts

@@ -320,12 +320,14 @@ export class BeaconChain implements IBeaconChain {
 
     this.beaconProposerCache = new BeaconProposerCache(opts);
     this.checkpointBalancesCache = new CheckpointBalancesCache();
+    this.serializedCache = new SerializedCache();
     this.seenBlockInputCache = new SeenBlockInput({
       config,
       custodyConfig: this.custodyConfig,
       clock,
       chainEvents: emitter,
       signal,
+      serializedCache: this.serializedCache,
       metrics,
       logger,
     });
@@ -412,8 +414,6 @@ export class BeaconChain implements IBeaconChain {
     this.bls = bls;
     this.emitter = emitter;
 
-    this.serializedCache = new SerializedCache();
-
     this.getBlobsTracker = new GetBlobsTracker({
       logger,
       executionEngine: this.executionEngine,

package/src/chain/produceBlock/computeNewStateRoot.ts

@@ -61,7 +61,6 @@ export function computeNewStateRoot(
  * Compute the state root after processing an execution payload envelope.
  * Similar to `computeNewStateRoot` but for payload envelope processing.
  *
- * The `postBlockState` is mutated in place, callers must ensure it is not needed afterward.
  */
 export function computeEnvelopeStateRoot(
   metrics: Metrics | null,
@@ -74,13 +73,15 @@ export function computeEnvelopeStateRoot(
   };
 
   const processEnvelopeTimer = metrics?.blockPayload.executionPayloadEnvelopeProcessingTime.startTimer();
-  processExecutionPayloadEnvelope(postBlockState, signedEnvelope, false);
+  const postEnvelopeState = processExecutionPayloadEnvelope(postBlockState, signedEnvelope, false, {
+    dontTransferCache: true,
+  });
   processEnvelopeTimer?.();
 
   const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({
     source: StateHashTreeRootSource.computeEnvelopeStateRoot,
   });
-  const stateRoot = postBlockState.hashTreeRoot();
+  const stateRoot = postEnvelopeState.hashTreeRoot();
   hashTreeRootTimer?.();
 
   return stateRoot;

package/src/chain/seenCache/seenGossipBlockInput.ts

@@ -17,6 +17,7 @@ import {Metrics} from "../../metrics/metrics.js";
 import {MAX_LOOK_AHEAD_EPOCHS} from "../../sync/constants.js";
 import {IClock} from "../../util/clock.js";
 import {CustodyConfig} from "../../util/dataColumns.js";
+import {SerializedCache} from "../../util/serializedCache.js";
 import {
   BlockInput,
   BlockInputBlobs,
@@ -55,6 +56,7 @@ export type SeenBlockInputCacheModules = {
   chainEvents: ChainEventEmitter;
   signal: AbortSignal;
   custodyConfig: CustodyConfig;
+  serializedCache: SerializedCache;
   metrics: Metrics | null;
   logger?: Logger;
 };
@@ -101,6 +103,7 @@ export class SeenBlockInput {
   private readonly clock: IClock;
   private readonly chainEvents: ChainEventEmitter;
   private readonly signal: AbortSignal;
+  private readonly serializedCache: SerializedCache;
   private readonly metrics: Metrics | null;
   private readonly logger?: Logger;
   private blockInputs = new Map<RootHex, IBlockInput>();
@@ -109,19 +112,35 @@ export class SeenBlockInput {
   // and the signature to ensure we only skip verification if both match
   private verifiedProposerSignatures = new Map<Slot, Map<RootHex, BLSSignature>>();
 
-  constructor({config, custodyConfig, clock, chainEvents, signal, metrics, logger}: SeenBlockInputCacheModules) {
+  constructor({
+    config,
+    custodyConfig,
+    clock,
+    chainEvents,
+    signal,
+    serializedCache,
+    metrics,
+    logger,
+  }: SeenBlockInputCacheModules) {
     this.config = config;
     this.custodyConfig = custodyConfig;
     this.clock = clock;
     this.chainEvents = chainEvents;
     this.signal = signal;
+    this.serializedCache = serializedCache;
     this.metrics = metrics;
     this.logger = logger;
 
     if (metrics) {
-      metrics.seenCache.blockInput.blockInputCount.addCollect(() =>
-        metrics.seenCache.blockInput.blockInputCount.set(this.blockInputs.size)
-      );
+      metrics.seenCache.blockInput.blockInputCount.addCollect(() => {
+        metrics.seenCache.blockInput.blockInputCount.set(this.blockInputs.size);
+        metrics.seenCache.blockInput.serializedObjectCount.set(
+          Array.from(this.blockInputs.values()).reduce(
+            (count, blockInput) => count + blockInput.getSerializedCacheKeys().length,
+            0
+          )
+        );
+      });
     }
 
     this.chainEvents.on(ChainEvent.forkChoiceFinalized, this.onFinalized);
@@ -142,7 +161,10 @@ export class SeenBlockInput {
    * Removes the single BlockInput from the cache
    */
   remove(rootHex: RootHex): void {
-    this.blockInputs.delete(rootHex);
+    const blockInput = this.blockInputs.get(rootHex);
+    if (blockInput) {
+      this.evictBlockInput(blockInput);
+    }
   }
 
   /**
@@ -154,7 +176,7 @@ export class SeenBlockInput {
     let deletedCount = 0;
     while (blockInput) {
       deletedCount++;
-      this.blockInputs.delete(blockInput.blockRootHex);
+      this.evictBlockInput(blockInput);
       blockInput = this.blockInputs.get(parentRootHex ?? "");
       parentRootHex = blockInput?.parentRootHex;
     }
@@ -165,10 +187,10 @@ export class SeenBlockInput {
   onFinalized = (checkpoint: CheckpointWithHex) => {
     let deletedCount = 0;
     const cutoffSlot = computeStartSlotAtEpoch(checkpoint.epoch);
-    for (const [rootHex, blockInput] of this.blockInputs) {
+    for (const [, blockInput] of this.blockInputs) {
       if (blockInput.slot < cutoffSlot) {
         deletedCount++;
-        this.blockInputs.delete(rootHex);
+        this.evictBlockInput(blockInput);
       }
     }
     this.logger?.debug(`BlockInputCache.onFinalized deleted ${deletedCount} cached BlockInputs`);
@@ -408,14 +430,20 @@ export class SeenBlockInput {
 
     if (itemsToDelete > 0) {
       const sorted = [...this.blockInputs.entries()].sort((a, b) => a[1].slot - b[1].slot);
-      for (const [rootHex] of sorted) {
-        this.blockInputs.delete(rootHex);
+      for (const [, blockInput] of sorted) {
+        this.evictBlockInput(blockInput);
         itemsToDelete--;
         if (itemsToDelete <= 0) return;
       }
     }
     pruneSetToMax(this.verifiedProposerSignatures, MAX_BLOCK_INPUT_CACHE_SIZE);
   }
+
+  private evictBlockInput(blockInput: IBlockInput): void {
+    // Without forcefully clearing this cache, we would rely on WeakMap to evict memory which is not reliable
+    this.serializedCache.delete(blockInput.getSerializedCacheKeys());
+    this.blockInputs.delete(blockInput.blockRootHex);
+  }
 }
 
 enum SeenBlockInputCacheErrorCode {

package/src/metrics/metrics/lodestar.ts

@@ -1461,6 +1461,10 @@ export function createLodestarMetrics(
           name: "lodestar_seen_block_input_cache_size",
           help: "Number of cached BlockInputs",
         }),
+        serializedObjectCount: register.gauge({
+          name: "lodestar_seen_block_input_cache_serialized_object_count",
+          help: "Number of serialized objects retained by cached BlockInputs",
+        }),
         duplicateBlockCount: register.gauge<{source: BlockInputSource}>({
           name: "lodestar_seen_block_input_cache_duplicate_block_count",
           help: "Total number of duplicate blocks that pass validation and attempt to be cached but are known",

package/src/network/network.ts

@@ -539,7 +539,8 @@ export class Network implements INetwork {
         this.config.getForkSeq(this.clock.currentSlot) >= ForkSeq.altair ? [Version.V2] : [Version.V2, Version.V1],
         request
       ),
-      request
+      request,
+      this.chain.serializedCache
     );
   }
 

package/src/util/serializedCache.ts

@@ -4,7 +4,7 @@
  * This is a thin wrapper around WeakMap
  */
 export class SerializedCache {
-  map: WeakMap<object, Uint8Array> = new WeakMap();
+  private map: WeakMap<object, Uint8Array> = new WeakMap();
 
   get(obj: object): Uint8Array | undefined {
     return this.map.get(obj);
@@ -15,11 +15,13 @@ export class SerializedCache {
   }
 
   /**
-   * Replace the internal WeakMap to force GC of all cached entries.
-   * Must only be called after all DB writes that may read from this cache have completed,
+   * Delete cached serialized entries for the provided object references.
+   * Must only be called after all DB writes that read from this cache for these objects have completed,
    * otherwise cached serialized bytes will be unavailable and data will be re-serialized unnecessarily.
    */
-  clear(): void {
-    this.map = new WeakMap();
+  delete(objs: object[]): void {
+    for (const obj of objs) {
+      this.map.delete(obj);
+    }
   }
 }