@lodestar/beacon-node 1.51.0-dev.f8eff664d2 → 1.51.0-dev.f921fbf6b7

package/src/chain/produceBlock/produceBlockBody.ts

@@ -1,4 +1,3 @@
-import {BitArray} from "@chainsafe/ssz";
 import {ChainForkConfig} from "@lodestar/config";
 import {
   IForkChoice,
@@ -16,7 +15,6 @@ import {
   ForkPostGloas,
   ForkPreGloas,
   ForkSeq,
-  INCLUSION_LIST_COMMITTEE_SIZE,
   isForkPostAltair,
   isForkPostBellatrix,
   isForkPostGloas,
@@ -74,14 +72,15 @@ import {
 } from "@lodestar/utils";
 import {ZERO_HASH_HEX} from "../../constants/index.js";
 import {numToQuantity} from "../../execution/engine/utils.js";
-import {IExecutionBuilder, IExecutionEngine, PayloadAttributes, PayloadId} from "../../execution/index.js";
+import {IExecutionBuilder, PayloadAttributes, PayloadId} from "../../execution/index.js";
 import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
 import {fromGraffitiBytes} from "../../util/graffiti.js";
 import {kzg} from "../../util/kzg.js";
 import {BeaconProposerCache} from "../beaconProposerCache.js";
 import type {BeaconChain} from "../chain.js";
+import {ForkChoiceExecutionEngine} from "../forkChoiceExecutionEngine.js";
 import {CommonBlockBody} from "../interface.js";
-import {ProposerPreferencesPool} from "../opPools/index.js";
+import {InclusionListStore, ProposerPreferencesPool} from "../opPools/index.js";
 import {validateBlobsAndKzgCommitments, validateCellsAndKzgCommitments} from "./validateBlobsAndKzgCommitments.js";
 
 // Time to provide the EL to generate a payload from new payload id
@@ -391,8 +390,19 @@ export async function produceBlockBody<T extends BlockType>(
       executionRequestsRoot: ssz.gloas.ExecutionRequests.hashTreeRoot(executionRequests as gloas.ExecutionRequests),
     };
     if (ForkSeq[fork] >= ForkSeq.heze) {
-      // TODO HEZE: populate from inclusion list pool once IL aggregation is wired up.
-      (bid as heze.ExecutionPayloadBid).inclusionListBits = BitArray.fromBitLen(INCLUSION_LIST_COMMITTEE_SIZE);
+      // Built from the full view (only_timely=False), a superset of the receiver's timely-only
+      // is_inclusion_list_bits_inclusive check
+      const inclusionListSlot = blockSlot - 1;
+      (bid as heze.ExecutionPayloadBid).inclusionListBits = this.inclusionListStore.getInclusionListBits(
+        inclusionListSlot,
+        getShufflingDependentRoot(
+          this.forkChoice,
+          computeEpochAtSlot(inclusionListSlot),
+          computeEpochAtSlot(parentBlock.slot),
+          parentBlock
+        ),
+        false
+      );
     }
     const signedBid: gloas.SignedExecutionPayloadBid = {
       message: bid,
@@ -733,10 +743,11 @@ export async function produceBlockBody<T extends BlockType>(
  */
 export async function prepareExecutionPayload(
   chain: {
-    executionEngine: IExecutionEngine;
+    executionEngine: ForkChoiceExecutionEngine;
     config: ChainForkConfig;
     forkChoice: IForkChoice;
     proposerPreferencesPool: ProposerPreferencesPool;
+    inclusionListStore: InclusionListStore;
   },
   logger: Logger,
   fork: ForkPostBellatrix,
@@ -756,12 +767,25 @@ export async function prepareExecutionPayload(
   const timestamp = computeTimeAtSlot(chain.config, state.slot, state.genesisTime);
   const prevRandao = state.getRandaoMix(state.epoch);
 
+  // Post-heze the inclusion lists are part of the cache key, so a list set that changed since the
+  // payload was prepared triggers a new forkchoiceUpdated instead of reusing a payload built without them
+  let attributes = payloadAttributes;
+  if (attributes === undefined && ForkSeq[fork] >= ForkSeq.heze) {
+    attributes = preparePayloadAttributes(fork, chain, {
+      prepareState: state,
+      prepareSlot: state.slot,
+      parentBlockRoot,
+      parentBlockHash,
+      feeRecipient: suggestedFeeRecipient,
+    });
+  }
+
   const payloadIdCached = chain.executionEngine.payloadIdCache.get({
     headBlockHash: toRootHex(parentBlockHash),
-    finalizedBlockHash,
     timestamp: numToQuantity(timestamp),
     prevRandao: toHex(prevRandao),
     suggestedFeeRecipient,
+    inclusionListTransactions: attributes?.inclusionListTransactions,
   });
 
   // prepareExecutionPayload will throw error via notifyForkchoiceUpdate if
@@ -782,22 +806,20 @@ export async function prepareExecutionPayload(
       prepType = PayloadPreparationType.Fresh;
     }
 
-    const attributes: PayloadAttributes =
-      payloadAttributes ??
-      preparePayloadAttributes(fork, chain, {
-        prepareState: state,
-        prepareSlot: state.slot,
-        parentBlockRoot,
-        parentBlockHash,
-        feeRecipient: suggestedFeeRecipient,
-      });
-
     payloadId = await chain.executionEngine.notifyForkchoiceUpdate(
       fork,
+      toRootHex(parentBlockRoot),
       toRootHex(parentBlockHash),
       safeBlockHash,
       finalizedBlockHash,
-      attributes
+      attributes ??
+        preparePayloadAttributes(fork, chain, {
+          prepareState: state,
+          prepareSlot: state.slot,
+          parentBlockRoot,
+          parentBlockHash,
+          feeRecipient: suggestedFeeRecipient,
+        })
     );
     logger.verbose("Prepared payload id from execution engine", {payloadId});
   }
@@ -842,6 +864,7 @@ export function getPayloadAttributesForSSE(
     config: ChainForkConfig;
     forkChoice: IForkChoice;
     proposerPreferencesPool: ProposerPreferencesPool;
+    inclusionListStore: InclusionListStore;
   },
   {
     prepareState,
@@ -898,6 +921,7 @@ function preparePayloadAttributes(
     config: ChainForkConfig;
     forkChoice: IForkChoice;
     proposerPreferencesPool: ProposerPreferencesPool;
+    inclusionListStore: InclusionListStore;
   },
   {
     prepareState,
@@ -966,8 +990,27 @@ function preparePayloadAttributes(
   }
 
   if (ForkSeq[fork] >= ForkSeq.heze) {
-    // TODO HEZE: populate from inclusion list pool once IL aggregation is wired up.
-    (payloadAttributes as heze.SSEPayloadAttributes["payloadAttributes"]).inclusionListTransactions = [];
+    // Built from the full view including untimely lists (only_timely=False in prepare_execution_payload),
+    // a superset of the timely-only set every validator enforces
+    const parentBlockRootHex = toRootHex(parentBlockRoot);
+    const parentBlock = chain.forkChoice.getBlockHexAndBlockHash(parentBlockRootHex, toRootHex(parentBlockHash));
+    if (parentBlock === null) {
+      throw new Error(
+        `Parent block not in fork choice for Heze payload attributes parentBlockRoot=${parentBlockRootHex} parentBlockHash=${toRootHex(parentBlockHash)}`
+      );
+    }
+    const inclusionListSlot = prepareSlot - 1;
+    (payloadAttributes as heze.SSEPayloadAttributes["payloadAttributes"]).inclusionListTransactions =
+      chain.inclusionListStore.getInclusionListTransactions(
+        inclusionListSlot,
+        getShufflingDependentRoot(
+          chain.forkChoice,
+          computeEpochAtSlot(inclusionListSlot),
+          computeEpochAtSlot(parentBlock.slot),
+          parentBlock
+        ),
+        false
+      );
   }
 
   return payloadAttributes;
@@ -1084,11 +1127,13 @@ export async function produceCommonBlockBody<T extends BlockType>(
     this.opPool.getSlashingsAndExits(currentState, blockType, this.metrics);
 
   const endAttestations = stepsMetrics?.startTimer();
+  // TODO: extract to a separate BeaconChain method to coordinate the 2 op-pools
   const attestations = this.aggregatedAttestationPool.getAttestationsForBlock(
     fork,
     this.forkChoice,
     this.shufflingCache,
-    currentState
+    currentState,
+    this.singleAttestationPool
   );
   endAttestations?.({
     step: BlockProductionStep.attestations,

package/src/chain/proposerTracker.ts

@@ -0,0 +1,94 @@
+import {MIN_ATTESTATION_INCLUSION_DELAY} from "@lodestar/params";
+import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {Slot} from "@lodestar/types";
+import {Metrics} from "../metrics/metrics.js";
+
+/**
+ * Track upcoming block proposals of validators connected to this node, and the vote slots those blocks can include,
+ * to only then insert single attestations into the SingleAttestationPool
+ */
+export class ProposerTracker {
+  /** Local proposal slots that have not passed yet */
+  private readonly proposalSlots = new Set<Slot>();
+  /** Vote slots of tracked proposals, see `getFirstIncludableVoteSlot` */
+  private readonly collectableSlots = new Set<Slot>();
+  private clockSlot = 0;
+
+  /**
+   * @param ignoreProposalDuty dev only, collect every vote slot a block at the clock slot can include regardless of
+   * local proposals
+   */
+  constructor(
+    private readonly ignoreProposalDuty = false,
+    metrics: Metrics | null = null
+  ) {
+    metrics?.proposerTracker.proposalSlots.addCollect(() =>
+      metrics.proposerTracker.proposalSlots.set(this.proposalSlots.size)
+    );
+    metrics?.proposerTracker.collectableSlot.addCollect(() =>
+      metrics.proposerTracker.collectableSlot.set(this.shouldCollect(this.clockSlot) ? 1 : 0)
+    );
+  }
+
+  /**
+   * Idempotent, a known proposal is ignored
+   * @returns true if the proposal was not tracked before
+   */
+  addProposal(proposalSlot: Slot): boolean {
+    if (this.proposalSlots.has(proposalSlot)) {
+      return false;
+    }
+
+    this.proposalSlots.add(proposalSlot);
+    for (
+      let voteSlot = getFirstIncludableVoteSlot(proposalSlot);
+      voteSlot <= proposalSlot - MIN_ATTESTATION_INCLUSION_DELAY;
+      voteSlot++
+    ) {
+      this.collectableSlots.add(voteSlot);
+    }
+    return true;
+  }
+
+  shouldCollect(attestationSlot: Slot): boolean {
+    if (this.ignoreProposalDuty) {
+      // to test block production without proposal duties
+      return attestationSlot >= getFirstIncludableVoteSlot(this.clockSlot);
+    }
+    return this.collectableSlots.has(attestationSlot);
+  }
+
+  /**
+   * Drop proposals that have passed. A proposal is kept during its own slot so that its single attestations stay
+   * available until the block is produced. Vote slots are dropped once no remaining proposal can include them.
+   */
+  prune(clockSlot: Slot): void {
+    this.clockSlot = clockSlot;
+    for (const proposalSlot of this.proposalSlots) {
+      if (proposalSlot < clockSlot) {
+        this.proposalSlots.delete(proposalSlot);
+      }
+    }
+
+    if (this.proposalSlots.size === 0) {
+      this.collectableSlots.clear();
+      return;
+    }
+
+    // Each remaining proposal covers its own vote slots, so a vote slot below the lowest one is covered by none
+    const lowestCollectableVoteSlot = getFirstIncludableVoteSlot(Math.min(...this.proposalSlots));
+    for (const voteSlot of this.collectableSlots) {
+      if (voteSlot < lowestCollectableVoteSlot) {
+        this.collectableSlots.delete(voteSlot);
+      }
+    }
+  }
+}
+
+/**
+ * Since deneb (EIP-7045) a block includes attestations of its current and previous epoch, so the first vote slot it can
+ * include is the start of the previous epoch, 32 to 63 slots before it on mainnet
+ */
+function getFirstIncludableVoteSlot(blockSlot: Slot): Slot {
+  return computeStartSlotAtEpoch(Math.max(computeEpochAtSlot(blockSlot) - 1, 0));
+}

package/src/chain/regen/interface.ts

@@ -24,7 +24,7 @@ export enum RegenCaller {
   validateApiVoluntaryExit = "validateApiVoluntaryExit",
   publishDeferredVoluntaryExits = "publishDeferredVoluntaryExits",
   validateGossipExecutionPayloadBid = "validateGossipExecutionPayloadBid",
-  validateApiExecutionPayloadBid = "validateApiExecutionPayloadBid",
+  validateGossipInclusionList = "validateGossipInclusionList",
   validateGossipPayloadAttestationMessage = "validateGossipPayloadAttestationMessage",
   validateGossipProposerPreferences = "validateGossipProposerPreferences",
   onForkChoiceFinalized = "onForkChoiceFinalized",
@@ -52,7 +52,7 @@ export interface IStateRegenerator extends IStateRegeneratorInternal {
   onCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void;
   processState(blockRootHex: RootHex, postState: IBeaconStateView): void;
   addCheckpointState(cp: phase0.Checkpoint, item: IBeaconStateView): void;
-  updateHeadState(newHead: ProtoBlock, maybeHeadState: IBeaconStateView): void;
+  updateHeadState(newHead: ProtoBlock, maybeHeadState?: IBeaconStateView): void;
   updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null;
 }
 

package/src/chain/regen/queued.ts

@@ -154,18 +154,20 @@ export class QueuedStateRegenerator implements IStateRegenerator {
     this.checkpointStateCache.add(cp, item);
   }
 
-  updateHeadState(newHead: ProtoBlock, maybeHeadState: IBeaconStateView): void {
+  updateHeadState(newHead: ProtoBlock, maybeHeadState?: IBeaconStateView): void {
     const {stateRoot: newHeadStateRoot, blockRoot: newHeadBlockRoot, slot: newHeadSlot} = newHead;
-    const maybeHeadStateRoot = toRootHex(maybeHeadState.hashTreeRoot());
+    const maybeHeadStateRoot = maybeHeadState ? toRootHex(maybeHeadState.hashTreeRoot()) : undefined;
     const logCtx = {
       newHeadSlot,
       newHeadBlockRoot,
       newHeadStateRoot,
-      maybeHeadSlot: maybeHeadState.slot,
+      maybeHeadSlot: maybeHeadState?.slot,
       maybeHeadStateRoot,
     };
     const headState =
-      newHeadStateRoot === maybeHeadStateRoot ? maybeHeadState : this.blockStateCache.get(newHeadStateRoot);
+      maybeHeadState && newHeadStateRoot === maybeHeadStateRoot
+        ? maybeHeadState
+        : this.blockStateCache.get(newHeadStateRoot);
 
     if (headState) {
       this.blockStateCache.setHeadState(headState);

package/src/chain/seenCache/seenGossipBlockInput.ts

@@ -4,15 +4,14 @@ import {
   ForkName,
   ForkPostFulu,
   ForkPostGloas,
-  ForkPreDeneb,
+  ForkPreFulu,
   ForkPreGloas,
   SLOTS_PER_EPOCH,
-  isForkPostDeneb,
   isForkPostFulu,
   isForkPostGloas,
 } from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
-import {BLSSignature, RootHex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types";
+import {BLSSignature, RootHex, SignedBeaconBlock, Slot, fulu} from "@lodestar/types";
 import {LodestarError, Logger, byteArrayEquals, pruneSetToMax} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
 import {MAX_LOOK_AHEAD_EPOCHS} from "../../sync/constants.js";
@@ -21,19 +20,15 @@ import {CustodyConfig} from "../../util/dataColumns.js";
 import {SerializedCache} from "../../util/serializedCache.js";
 import {
   BlockInput,
-  BlockInputBlobs,
   BlockInputColumns,
   BlockInputNoData,
   BlockInputPreData,
   BlockWithSource,
   DAType,
-  ForkBlobsDA,
   IBlockInput,
   LogMetaBasic,
-  LogMetaBlobs,
   LogMetaColumns,
   SourceMeta,
-  isBlockInputBlobs,
   isBlockInputColumns,
   isDaOutOfRange,
 } from "../blocks/blockInput/index.js";
@@ -62,20 +57,17 @@ export type SeenBlockInputCacheModules = {
   logger?: Logger;
 };
 
-export type GetByBlobOptions = {
+export type GetByColumnOptions = {
   throwErrorIfAlreadyKnown?: boolean;
 };
 
 /**
  * Consumers that create BlockInputs or change types of old BlockInputs
  *
- * - gossipHandlers (block and blob)
- * - beaconBlocksMaybeBlobsByRange
- * - unavailableBeaconBlobsByRoot (beaconBlocksMaybeBlobsByRoot)
+ * - gossipHandlers (block and data column)
+ * - cacheByRangeResponses in sync/utils/downloadByRange.ts
+ * - downloadByRoot in sync/utils/downloadByRoot.ts
  * - publishBlock in the beacon/blocks/index.ts API
- *   https://github.com/ChainSafe/lodestar/blob/unstable/packages/beacon-node/src/api/impl/beacon/blocks/index.ts#L62
- * - maybeValidateBlobs in verifyBlocksDataAvailability (is_data_available spec function)
- *   https://github.com/ChainSafe/lodestar/blob/unstable/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts#L111
  *
  *
  * Pruning management for SeenBlockInputCache
@@ -238,17 +230,6 @@ export class SeenBlockInput {
           seenTimestampSec,
           peerIdStr,
         });
-      } else if (!isForkPostDeneb(forkName)) {
-        // Pre-deneb
-        blockInput = BlockInputPreData.createFromBlock({
-          block: block as SignedBeaconBlock<ForkPreDeneb>,
-          blockRootHex,
-          daOutOfRange,
-          forkName,
-          source,
-          seenTimestampSec,
-          peerIdStr,
-        });
       } else if (isForkPostFulu(forkName)) {
         // Fulu Only
         blockInput = BlockInputColumns.createFromBlock({
@@ -263,9 +244,8 @@ export class SeenBlockInput {
           peerIdStr,
         });
       } else {
-        // Deneb and Electra
-        blockInput = BlockInputBlobs.createFromBlock({
-          block: block as SignedBeaconBlock<ForkBlobsDA>,
+        blockInput = BlockInputPreData.createFromBlock({
+          block: block as SignedBeaconBlock<ForkPreFulu>,
           blockRootHex,
           daOutOfRange,
           forkName,
@@ -288,66 +268,6 @@ export class SeenBlockInput {
     return blockInput as BlockInput;
   }
 
-  getByBlob(
-    {
-      blockRootHex,
-      blobSidecar,
-      source,
-      seenTimestampSec,
-      peerIdStr,
-    }: SourceMeta & {blockRootHex: RootHex; blobSidecar: deneb.BlobSidecar},
-    opts: GetByBlobOptions = {}
-  ): BlockInputBlobs {
-    // TODO(peerDAS): Why is it necessary to static cast this here. All conditional paths result in a valid value so should be defined correctly below
-    let blockInput = this.blockInputs.get(blockRootHex) as IBlockInput;
-    let created = false;
-    if (!blockInput) {
-      created = true;
-      const {forkName, daOutOfRange} = this.buildCommonProps(blobSidecar.signedBlockHeader.message.slot);
-      blockInput = BlockInputBlobs.createFromBlob({
-        blobSidecar,
-        blockRootHex,
-        daOutOfRange,
-        forkName,
-        source,
-        seenTimestampSec,
-        peerIdStr,
-      });
-      this.metrics?.seenCache.blockInput.createdByBlob.inc();
-      this.blockInputs.set(blockRootHex, blockInput);
-    }
-
-    if (!isBlockInputBlobs(blockInput)) {
-      throw new SeenBlockInputCacheError(
-        {
-          code: SeenBlockInputCacheErrorCode.WRONG_BLOCK_INPUT_TYPE,
-          cachedType: blockInput.type,
-          requestedType: DAType.Blobs,
-          ...blockInput.getLogMeta(),
-        },
-        `BlockInputType mismatch adding blobIndex=${blobSidecar.index}`
-      );
-    }
-
-    if (!blockInput.hasBlob(blobSidecar.index)) {
-      blockInput.addBlob({blobSidecar, blockRootHex, source, seenTimestampSec, peerIdStr});
-    } else if (!created) {
-      this.logger?.debug(
-        `Attempt to cache blob index #${blobSidecar.index} but is already cached on BlockInput`,
-        blockInput.getLogMeta()
-      );
-      this.metrics?.seenCache.blockInput.duplicateBlobCount.inc({source});
-      if (opts.throwErrorIfAlreadyKnown) {
-        throw new SeenBlockInputCacheError({
-          code: SeenBlockInputCacheErrorCode.GOSSIP_BLOB_ALREADY_KNOWN,
-          ...blockInput.getLogMeta(),
-        });
-      }
-    }
-
-    return blockInput;
-  }
-
   getByColumn(
     {
       blockRootHex,
@@ -356,7 +276,7 @@ export class SeenBlockInput {
       source,
       peerIdStr,
     }: SourceMeta & {blockRootHex: RootHex; columnSidecar: fulu.DataColumnSidecar},
-    opts: GetByBlobOptions = {}
+    opts: GetByColumnOptions = {}
   ): BlockInputColumns {
     let blockInput = this.blockInputs.get(blockRootHex);
     let created = false;
@@ -473,7 +393,6 @@ export class SeenBlockInput {
 
 enum SeenBlockInputCacheErrorCode {
   WRONG_BLOCK_INPUT_TYPE = "BLOCK_INPUT_CACHE_ERROR_WRONG_BLOCK_INPUT_TYPE",
-  GOSSIP_BLOB_ALREADY_KNOWN = "BLOCK_INPUT_CACHE_ERROR_GOSSIP_BLOB_ALREADY_KNOWN",
   GOSSIP_COLUMN_ALREADY_KNOWN = "BLOCK_INPUT_CACHE_ERROR_GOSSIP_COLUMN_ALREADY_KNOWN",
 }
 
@@ -483,9 +402,6 @@ type SeenBlockInputCacheErrorType =
       cachedType: DAType;
       requestedType: DAType;
     })
-  | (LogMetaBlobs & {
-      code: SeenBlockInputCacheErrorCode.GOSSIP_BLOB_ALREADY_KNOWN;
-    })
   | (LogMetaColumns & {
       code: SeenBlockInputCacheErrorCode.GOSSIP_COLUMN_ALREADY_KNOWN;
     });