@lodestar/beacon-node 1.50.0-dev.db953a4f93 → 1.50.0-dev.dff4197219

package/src/chain/options.ts

@@ -27,8 +27,8 @@ export type IChainOptions = BlockProcessOpts &
     blsVerifyAllMainThread?: boolean;
     blsVerifyAllMultiThread?: boolean;
     blacklistedBlocks?: string[];
-    // TODO GLOAS: add similar option for execution payload envelopes?
     persistProducedBlocks?: boolean;
+    persistProducedPayloadEnvelopes?: boolean;
     /** Archive finalized Gloas payload envelopes as headers (bodies reconstructed from the EL on read) */
     dedupePayloads?: boolean;
     persistInvalidSszObjects?: boolean;
@@ -52,7 +52,7 @@ export type IChainOptions = BlockProcessOpts &
     minSameMessageSignatureSetsToBatch: number;
     archiveDateEpochs?: number;
     nHistoricalStatesFileDataStore?: boolean;
-    nativeStateView?: boolean;
+    nativeStateTransition?: boolean;
     /** Builder circuit breaker fault inspection window in slots */
     faultInspectionWindow?: number;
     /** Canonical EMPTY blocks allowed per `faultInspectionWindow` observed blocks */
@@ -131,7 +131,7 @@ export const defaultChainOptions: IChainOptions = {
   //   - users can prune the persisted checkpoint state files manually to save disc space
   //   - it helps debug easier when network is unfinalized
   nHistoricalStatesFileDataStore: true,
-  nativeStateView: false,
+  nativeStateTransition: false,
   maxBlockStates: DEFAULT_MAX_BLOCK_STATES,
   maxCPStateEpochsInMemory: DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY,
   maxCPStateEpochsOnDisk: DEFAULT_MAX_CP_STATE_ON_DISK,

package/src/chain/prepareNextSlot.ts

@@ -244,6 +244,10 @@ export class PrepareNextSlotScheduler {
           parentBlockHash = preparedState.latestExecutionPayloadHeader.blockHash;
         }
 
+        // The payload_attributes event must carry the same hashes we send to the EL
+        const safeBlockHash = getSafeExecutionBlockHash(this.chain.forkChoice, this.logger);
+        const finalizedBlockHash = getFinalizedExecutionBlockHash(this.chain.forkChoice);
+
         let payloadAttributes: PayloadAttributes | undefined;
         // If emitPayloadAttributes is true emit a SSE payloadAttributes event for
         // every slot. Without the flag, only emit the event if we are proposing in the next slot.
@@ -256,6 +260,8 @@ export class PrepareNextSlotScheduler {
             prepareSlot,
             parentBlockRoot: fromHex(updatedHead.blockRoot),
             parentBlockHash,
+            safeBlockHash,
+            finalizedBlockHash,
             feeRecipient: feeRecipient ?? "0x0000000000000000000000000000000000000000",
           });
           this.chain.emitter.emit(routes.events.EventType.payloadAttributes, {data, version: fork});
@@ -267,9 +273,6 @@ export class PrepareNextSlotScheduler {
             computeTimeAtSlot(this.config, prepareSlot, this.chain.genesisTime) - Date.now() / 1000;
           this.metrics?.blockPayload.payloadAdvancePrepTime.observe(preparationTime);
 
-          const safeBlockHash = getSafeExecutionBlockHash(this.chain.forkChoice, this.logger);
-          const finalizedBlockHash = getFinalizedExecutionBlockHash(this.chain.forkChoice);
-
           // awaiting here instead of throwing an async call because there is no other task
           // left for scheduler and this gives nice semantics to catch and log errors in the
           // try/catch wrapper here.

package/src/chain/produceBlock/computeNewStateRoot.ts

@@ -16,7 +16,7 @@ export function computeNewStateRoot(
   const signedBlock = {message: block, signature: EMPTY_SIGNATURE};
   const {newStateRoot, proposerReward, postState, hashTreeRootTime} = state.computeNewStateRoot(
     {block: signedBlock},
-    {metrics}
+    {metrics: metrics?.stateTransition}
   );
 
   metrics?.stateHashTreeRootTime.observe({source: StateHashTreeRootSource.computeNewStateRoot}, hashTreeRootTime);

package/src/chain/produceBlock/produceBlockBody.ts

@@ -845,6 +845,8 @@ export function getPayloadAttributesForSSE(
     prepareSlot,
     parentBlockRoot,
     parentBlockHash,
+    safeBlockHash,
+    finalizedBlockHash,
     feeRecipient,
   }: {
     /**
@@ -855,6 +857,8 @@ export function getPayloadAttributesForSSE(
     prepareSlot: Slot;
     parentBlockRoot: Root;
     parentBlockHash: Bytes32;
+    safeBlockHash: RootHex;
+    finalizedBlockHash: RootHex;
     feeRecipient: string;
   }
 ): SSEPayloadAttributes {
@@ -874,7 +878,10 @@ export function getPayloadAttributesForSSE(
     payloadAttributes,
   } as SSEPayloadAttributes;
 
-  if (!isForkPostGloas(fork)) {
+  if (isForkPostGloas(fork)) {
+    (ssePayloadAttributes as gloas.SSEPayloadAttributes).safeBlockHash = fromHex(safeBlockHash);
+    (ssePayloadAttributes as gloas.SSEPayloadAttributes).finalizedBlockHash = fromHex(finalizedBlockHash);
+  } else {
     // Removed in Gloas, builders can get the block number from the EL via the block hash if required
     (ssePayloadAttributes as bellatrix.SSEPayloadAttributes).parentBlockNumber = prepareState.payloadBlockNumber;
   }

package/src/chain/regen/regen.ts

@@ -14,6 +14,7 @@ import {Logger, fromHex, toRootHex} from "@lodestar/utils";
 import {IBeaconDb} from "../../db/index.js";
 import {Metrics} from "../../metrics/index.js";
 import {nextEventLoop} from "../../util/eventLoop.js";
+import {SerializedCache} from "../../util/serializedCache.js";
 import {getCheckpointFromState} from "../blocks/utils/checkpoint.js";
 import {ChainEvent, ChainEventEmitter} from "../emitter.js";
 import {SeenBlockInput} from "../seenCache/seenGossipBlockInput.js";
@@ -28,6 +29,7 @@ export type RegenModules = {
   blockStateCache: BlockStateCache;
   checkpointStateCache: CheckpointStateCache;
   seenBlockInputCache: SeenBlockInput;
+  serializedCache: SerializedCache;
   config: ChainForkConfig;
   emitter: ChainEventEmitter;
   logger: Logger;
@@ -235,7 +237,7 @@ export class StateRegenerator implements IStateRegeneratorInternal {
         // We are only running the state transition to get a specific state's data.
         // stateTransition() does the clone() inside, transfer cache to make the regen faster
         state = state.stateTransition(
-          block,
+          {block, ssz: this.modules.serializedCache.get(block)},
           {
             // Replay previously imported blocks, assume valid and available
             executionPayloadStatus: ExecutionPayloadStatus.valid,
@@ -245,7 +247,7 @@ export class StateRegenerator implements IStateRegeneratorInternal {
             verifySignatures: false,
             dontTransferCache: false,
           },
-          this.modules
+          {metrics: this.modules.metrics?.stateTransition, validatorMonitor: this.modules.validatorMonitor}
         );
 
         const hashTreeRootTimer = this.modules.metrics?.stateHashTreeRootTime.startTimer({
@@ -316,7 +318,10 @@ async function processSlotsByCheckpoint(
 ): Promise<IBeaconStateView> {
   let postState = await processSlotsToNearestCheckpoint(modules, preState, slot, regenCaller, opts);
   if (postState.slot < slot) {
-    postState = postState.processSlots(slot, opts, modules);
+    postState = postState.processSlots(slot, opts, {
+      metrics: modules.metrics?.stateTransition,
+      validatorMonitor: modules.validatorMonitor,
+    });
   }
   return postState;
 }
@@ -361,7 +366,10 @@ export async function processSlotsToNearestCheckpoint(
       caller: regenCaller,
     });
     // processSlots calls .clone() before mutating
-    postState = postState.processSlots(nextEpochSlot, opts, modules);
+    postState = postState.processSlots(nextEpochSlot, opts, {
+      metrics: metrics?.stateTransition,
+      validatorMonitor: modules.validatorMonitor,
+    });
     metrics?.epochTransitionByCaller.inc({caller: regenCaller});
 
     // this is usually added when we prepare for next slot or validate gossip block

package/src/chain/seenCache/seenGossipBlockInput.ts

@@ -111,6 +111,9 @@ export class SeenBlockInput {
   // there should only 1 block root per slot but we need to always compare against rootHex
   // and the signature to ensure we only skip verification if both match
   private verifiedProposerSignatures = new Map<Slot, Map<RootHex, BLSSignature>>();
+  // Roots of gossip blocks under validation, counted because blocks with the same root but different signatures
+  // can be validated concurrently
+  private validatingBlockRoots = new Map<RootHex, number>();
 
   constructor({
     config,
@@ -153,6 +156,26 @@ export class SeenBlockInput {
     return this.blockInputs.get(rootHex)?.hasBlock() ?? false;
   }
 
+  /**
+   * Mark a gossip block root as under validation, every call MUST be paired with `unmarkValidatingBlock()`
+   */
+  markValidatingBlock(rootHex: RootHex): void {
+    this.validatingBlockRoots.set(rootHex, (this.validatingBlockRoots.get(rootHex) ?? 0) + 1);
+  }
+
+  unmarkValidatingBlock(rootHex: RootHex): void {
+    const count = this.validatingBlockRoots.get(rootHex) ?? 0;
+    if (count <= 1) {
+      this.validatingBlockRoots.delete(rootHex);
+    } else {
+      this.validatingBlockRoots.set(rootHex, count - 1);
+    }
+  }
+
+  isValidatingBlock(rootHex: RootHex): boolean {
+    return this.validatingBlockRoots.has(rootHex);
+  }
+
   get(rootHex: RootHex): IBlockInput | undefined {
     return this.blockInputs.get(rootHex);
   }

package/src/chain/seenCache/seenPayloadEnvelopeInput.ts

@@ -47,8 +47,9 @@ export type SeenPayloadEnvelopeInputModules = {
  * Created whenever we have a block because it needs block bid.
  * Steady state (linear chain, healthy progression): the cache holds ~2 entries — the head
  * (parent for next-slot production) and its parent (proposer-boost-reorg fallback). It can
- * transiently hold more during forks, range-sync bursts, or when `prepareNextSlot` skips
- * ticks; subsequent ticks settle it back.
+ * hold more during forks and range sync, or while an entry's payload is not FULL on the head's
+ * branch or its columns are still being gathered. Such entries are evicted on a later block
+ * import, on finalization or by the size cap.
  */
 export class SeenPayloadEnvelopeInput {
   private readonly config: ChainForkConfig;
@@ -264,18 +265,35 @@ export class SeenPayloadEnvelopeInput {
     }
   }
 
+  /**
+   * Removes a PayloadEnvelopeInput whose envelope failed import with a definitive invalid verdict. The envelope
+   * cannot be replaced in place, sync recreates the entry via `getOrReload()` or `add()` to fetch another one.
+   * No-op if the cached entry is a different object, it may already be the recreated one.
+   */
+  removeInvalid(payloadInput: PayloadEnvelopeInput): void {
+    if (this.payloadInputs.get(payloadInput.blockRootHex) === payloadInput) {
+      this.evictPayloadInput(payloadInput, "invalid");
+    }
+  }
+
   pruneBelowParent(parentBlock: ProtoBlock): void {
-    for (const block of this.forkChoice.getAllAncestorBlocks(parentBlock.blockRoot, parentBlock.payloadStatus)) {
-      // Only evict once the payload is FULL (revealed/imported) — on an EMPTY/PENDING branch we may
-      // still need to download the FULL envelope (see #9475), and evicting would make payload-by-root
-      // sync throw "Missing PayloadEnvelopeInput for known block".
-      if (block.slot < parentBlock.slot && block.payloadStatus === PayloadStatus.FULL) {
-        const input = this.payloadInputs.get(block.blockRoot);
-        // ...and don't evict while columns are still being gathered: writeDataColumnsToDb awaits the
-        // same hasComputedAllData() before persisting. Such entries are pruned by a later call.
-        if (input?.hasComputedAllData()) {
-          this.evictPayloadInput(input, "belowParent");
-        }
+    for (const input of this.payloadInputs.values()) {
+      if (
+        input.slot < parentBlock.slot &&
+        // Don't evict while columns are still being gathered: writeDataColumnsToDb awaits the same
+        // hasComputedAllData() before persisting. Such entries are pruned by a later call.
+        input.hasComputedAllData() &&
+        // Only evict once the payload is FULL (revealed/imported) on the parent's branch. On an EMPTY/PENDING
+        // branch we may still need to download the FULL envelope (see #9475), and without the entry gossip
+        // for the block is ignored until by-root sync reloads it.
+        this.forkChoice.isDescendant(
+          input.blockRootHex,
+          PayloadStatus.FULL,
+          parentBlock.blockRoot,
+          parentBlock.payloadStatus
+        )
+      ) {
+        this.evictPayloadInput(input, "belowParent");
       }
     }
   }

package/src/chain/shufflingCache.ts

@@ -7,14 +7,12 @@ import {
   getIndexedAttestation,
 } from "@lodestar/state-transition";
 import {Attestation, CommitteeIndex, Epoch, IndexedAttestation, RootHex, Slot} from "@lodestar/types";
-import {LodestarError, Logger, MapDef, pruneSetToMax} from "@lodestar/utils";
+import {LodestarError, Logger, MapDef} from "@lodestar/utils";
 import {Metrics} from "../metrics/metrics.js";
 
 /**
- * Same value to CheckpointBalancesCache, with the assumption that we don't have to use it for old epochs. In the worse case:
- * - when loading state bytes from disk, we need to compute shuffling for all epochs (~1s as of Sep 2023)
- * - don't have shuffling to verify attestations, need to do 1 epoch transition to add shuffling to this cache. This never happens
- * with default chain option of maxSkipSlots = 32
+ * Lookahead duties, gossip validation, and block production need epochs n-1, n, and n+1 for current epoch n.
+ * Keep the 4 highest epochs so that it works for epoch rotation.
  **/
 const MAX_EPOCHS = 4;
 
@@ -37,24 +35,29 @@ type ShufflingCacheItem = {
 type PromiseCacheItem = {
   type: CacheItemType.promise;
   timeInsertedMs: number;
-  promise: Promise<EpochShuffling>;
-  resolveFn: (shuffling: EpochShuffling) => void;
+  /** Resolves to null if the promise is cancelled */
+  promise: Promise<EpochShuffling | null>;
+  resolveFn: (shuffling: EpochShuffling | null) => void;
 };
 
 type CacheItem = ShufflingCacheItem | PromiseCacheItem;
 
+export enum ShufflingPromiseCancelReason {
+  regenError = "regen_error",
+  decisionRootMismatch = "decision_root_mismatch",
+  pruned = "pruned",
+}
+
 export type ShufflingCacheOpts = {
   maxShufflingCacheEpochs?: number;
 };
 
 /**
- * A shuffling cache to help:
- * - get committee quickly for attestation verification
- * - if a shuffling is not available (which does not happen with default chain option of maxSkipSlots = 32), track a promise to make sure we don't compute the same shuffling twice
- * - skip computing shuffling when loading state bytes from disk
+ * Cache recent epoch shufflings for lookahead duties, gossip validation, and block production.
+ * Pending promises deduplicate computation of missing shufflings.
  */
 export class ShufflingCache {
-  /** LRU cache implemented as a map, pruned every time we add an item */
+  /** Pruned to the `maxEpochs` highest epochs every time we add a shuffling */
   private readonly itemsByDecisionRootByEpoch: MapDef<Epoch, Map<RootHex, CacheItem>> = new MapDef(
     () => new Map<RootHex, CacheItem>()
   );
@@ -73,6 +76,9 @@ export class ShufflingCache {
           Array.from(this.itemsByDecisionRootByEpoch.values()).reduce((total, innerMap) => total + innerMap.size, 0)
         )
       );
+      metrics.shufflingCache.epochs.addCollect(() =>
+        metrics.shufflingCache.epochs.set(this.itemsByDecisionRootByEpoch.size)
+      );
     }
 
     this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS;
@@ -97,8 +103,8 @@ export class ShufflingCache {
         `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${epoch}, decisionRootHex: ${decisionRoot}`
       );
     }
-    let resolveFn: ((shuffling: EpochShuffling) => void) | null = null;
-    const promise = new Promise<EpochShuffling>((resolve) => {
+    let resolveFn: ((shuffling: EpochShuffling | null) => void) | null = null;
+    const promise = new Promise<EpochShuffling | null>((resolve) => {
       resolveFn = resolve;
     });
     if (resolveFn === null) {
@@ -121,7 +127,7 @@ export class ShufflingCache {
    * Return null if we don't have a shuffling for this epoch and dependentRootHex.
    */
   async get(epoch: Epoch, decisionRoot: RootHex): Promise<EpochShuffling | null> {
-    const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
+    const cacheItem = this.itemsByDecisionRootByEpoch.get(epoch)?.get(decisionRoot);
     if (cacheItem === undefined) {
       this.metrics?.shufflingCache.miss.inc();
       return null;
@@ -136,12 +142,10 @@ export class ShufflingCache {
   }
 
   /**
-   * Get a shuffling synchronously, return null if not present.
-   * The only time we have a promise cache item is when we regen shuffling for attestation, which never happens
-   * with default chain option.
+   * Get a shuffling synchronously, return null if not present or if it's still being computed.
    */
   getSync(epoch: Epoch, decisionRoot: RootHex): EpochShuffling | null {
-    const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
+    const cacheItem = this.itemsByDecisionRootByEpoch.get(epoch)?.get(decisionRoot);
     if (cacheItem === undefined) {
       this.metrics?.shufflingCache.miss.inc();
       return null;
@@ -161,7 +165,7 @@ export class ShufflingCache {
    * `NativeBeaconStateView`.
    */
   has(epoch: Epoch, decisionRoot: RootHex): boolean {
-    const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
+    const cacheItem = this.itemsByDecisionRootByEpoch.get(epoch)?.get(decisionRoot);
     return cacheItem !== undefined && isShufflingCacheItem(cacheItem);
   }
 
@@ -186,6 +190,25 @@ export class ShufflingCache {
     }
   }
 
+  /**
+   * Resolve a pending promise with null so that waiters don't hang, then remove it.
+   */
+  cancelPromise(epoch: Epoch, decisionRoot: RootHex, reason: ShufflingPromiseCancelReason): void {
+    const itemsByDecisionRoot = this.itemsByDecisionRootByEpoch.get(epoch);
+    const cacheItem = itemsByDecisionRoot?.get(decisionRoot);
+    if (itemsByDecisionRoot === undefined || cacheItem === undefined || !isPromiseCacheItem(cacheItem)) {
+      return;
+    }
+
+    cacheItem.resolveFn(null);
+    itemsByDecisionRoot.delete(decisionRoot);
+    if (itemsByDecisionRoot.size === 0) {
+      this.itemsByDecisionRootByEpoch.delete(epoch);
+    }
+    this.metrics?.shufflingCache.cancelledPromises.inc({reason});
+    this.logger?.debug("Cancelled shuffling promise", {epoch, decisionRoot, reason});
+  }
+
   getIndexedAttestation(
     epoch: number,
     decisionRoot: string,
@@ -205,6 +228,11 @@ export class ShufflingCache {
     return this.getBeaconCommittees(epoch, decisionRoot, slot, [index])[0];
   }
 
+  getBeaconCommitteeOrNull(epoch: number, decisionRoot: string, slot: Slot, index: CommitteeIndex): Uint32Array | null {
+    const shuffling = this.getSync(epoch, decisionRoot);
+    return shuffling === null ? null : getBeaconCommittees(shuffling, slot, [index])[0];
+  }
+
   getBeaconCommittees(epoch: number, decisionRoot: string, slot: Slot, indices: CommitteeIndex[]): Uint32Array[] {
     const shuffling = this.getShufflingOrThrow(epoch, decisionRoot);
     return getBeaconCommittees(shuffling, slot, indices);
@@ -243,8 +271,42 @@ export class ShufflingCache {
     }
     // set the shuffling
     shufflingAtEpoch.set(decisionRoot, {type: CacheItemType.shuffling, shuffling});
-    // prune the cache
-    pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs);
+    this.prune();
+  }
+
+  /**
+   * Keep the `maxEpochs` highest epochs, regardless of insertion order.
+   */
+  private prune(): void {
+    const toDelete = this.itemsByDecisionRootByEpoch.size - this.maxEpochs;
+    if (toDelete <= 0) {
+      return;
+    }
+
+    const prunedEpochs = Array.from(this.itemsByDecisionRootByEpoch.keys())
+      .sort((a, b) => a - b)
+      .slice(0, toDelete);
+    let prunedShufflings = 0;
+    for (const epoch of prunedEpochs) {
+      for (const cacheItem of this.itemsByDecisionRootByEpoch.get(epoch)?.values() ?? []) {
+        if (isPromiseCacheItem(cacheItem)) {
+          cacheItem.resolveFn(null);
+          this.metrics?.shufflingCache.cancelledPromises.inc({reason: ShufflingPromiseCancelReason.pruned});
+        } else {
+          prunedShufflings++;
+        }
+      }
+      this.itemsByDecisionRootByEpoch.delete(epoch);
+    }
+    this.metrics?.shufflingCache.prunedShufflings.inc(prunedShufflings);
+
+    this.logger?.verbose("Pruned shuffling cache", {
+      prunedEpochs: prunedEpochs.join(","),
+      prunedShufflings,
+      cachedEpochs: Array.from(this.itemsByDecisionRootByEpoch.keys())
+        .sort((a, b) => a - b)
+        .join(","),
+    });
   }
 }