@lodestar/beacon-node 1.46.0-dev.281a8ee194 → 1.46.0-dev.3e8651f2dc

package/src/chain/chain.ts

@@ -130,6 +130,7 @@ import {CPStateDatastore} from "./stateCache/datastore/types.js";
 import {FIFOBlockStateCache} from "./stateCache/fifoBlockStateCache.js";
 import {PersistentCheckpointStateCache} from "./stateCache/persistentCheckpointsCache.js";
 import {CheckpointStateCache} from "./stateCache/types.js";
+import {validateApiProposerSlashing} from "./validation/proposerSlashing.js";
 import {ValidatorMonitor} from "./validatorMonitor.js";
 
 /**
@@ -200,6 +201,8 @@ export class BeaconChain implements IBeaconChain {
   readonly seenAggregatedAttestations: SeenAggregatedAttestations;
   readonly seenExecutionPayloadBids = new SeenExecutionPayloadBids();
   readonly seenBlockProposers = new SeenBlockProposers();
+  /** Proposer indexes with an in-flight proposer slashing production */
+  private readonly producingProposerSlashing = new Set<ValidatorIndex>();
   readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages();
   readonly seenContributionAndProof: SeenContributionAndProof;
   readonly seenAttestationDatas: SeenAttestationDatas;
@@ -1163,6 +1166,63 @@ export class BeaconChain implements IBeaconChain {
     return this.payloadEnvelopeProcessor.processPayloadEnvelopeJob(payloadInput, opts);
   }
 
+  processProposerEquivocation(blockSlot: Slot, proposerIndex: ValidatorIndex): void {
+    this.produceProposerSlashing(blockSlot, proposerIndex).catch((e) => {
+      this.logger.debug("Error producing proposer slashing", {slot: blockSlot, proposerIndex}, e as Error);
+    });
+  }
+
+  /** Produce a proposer slashing from two conflicting signed block headers observed for the same slot and proposer */
+  private async produceProposerSlashing(blockSlot: Slot, proposerIndex: ValidatorIndex): Promise<void> {
+    if (this.opts.disableProposerSlashings === true || this.producingProposerSlashing.has(proposerIndex)) {
+      return;
+    }
+
+    if (this.opPool.hasSeenProposerSlashing(proposerIndex)) {
+      this.logger.debug("Not producing proposer slashing, already in the op pool", {
+        slot: blockSlot,
+        proposerIndex,
+      });
+      return;
+    }
+
+    const headers = this.seenBlockProposers.getEquivocationHeaders(blockSlot, proposerIndex);
+    if (headers === null) {
+      return;
+    }
+
+    this.producingProposerSlashing.add(proposerIndex);
+    try {
+      const [header1, header2] = headers;
+      // ProposerSlashing uses the bigint variant of the signed block header, see types package for details
+      const proposerSlashing: phase0.ProposerSlashing = {
+        signedHeader1: {
+          message: {...header1.message, slot: BigInt(header1.message.slot)},
+          signature: header1.signature,
+        },
+        signedHeader2: {
+          message: {...header2.message, slot: BigInt(header2.message.slot)},
+          signature: header2.signature,
+        },
+      };
+
+      try {
+        await validateApiProposerSlashing(this, proposerSlashing);
+      } catch (e) {
+        this.logger.debug("Produced proposer slashing is not valid", {slot: blockSlot, proposerIndex}, e as Error);
+        return;
+      }
+
+      this.opPool.insertProposerSlashing(proposerSlashing);
+      this.emitter.emit(routes.events.EventType.proposerSlashing, proposerSlashing);
+      this.emitter.emit(ChainEvent.publishProposerSlashing, proposerSlashing);
+      this.metrics?.opPool.proposerSlashingsProduced.inc();
+      this.logger.info("Produced proposer slashing from observed equivocation", {slot: blockSlot, proposerIndex});
+    } finally {
+      this.producingProposerSlashing.delete(proposerIndex);
+    }
+  }
+
   getStatus(): Status {
     const head = this.forkChoice.getHead();
     const finalizedCheckpoint = this.forkChoice.getFinalizedCheckpoint();

package/src/chain/emitter.ts

@@ -50,6 +50,11 @@ export enum ChainEvent {
    * and are ready to be published.
    */
   publishBlobSidecars = "publishBlobSidecars",
+  /**
+   * This event signals that a proposer slashing has been produced from an observed equivocation
+   * and is ready to be published.
+   */
+  publishProposerSlashing = "publishProposerSlashing",
   /**
    * Trigger an update of status so reqresp by peers have current earliestAvailableSlot
    */
@@ -110,6 +115,8 @@ export type IChainEvents = ApiEvents & {
 
   [ChainEvent.publishBlobSidecars]: (sidecars: deneb.BlobSidecar[]) => void;
 
+  [ChainEvent.publishProposerSlashing]: (proposerSlashing: phase0.ProposerSlashing) => void;
+
   [ChainEvent.updateStatus]: () => void;
 
   // Sync events that are chain->chain. Initiated from network requests but do not cross the network

package/src/chain/interface.ts

@@ -262,6 +262,9 @@ export interface IBeaconChain {
   /** Process execution payload envelope: verify, import to fork choice, and persist to DB */
   processExecutionPayload(payloadInput: PayloadEnvelopeInput, opts?: ImportPayloadOpts): Promise<void>;
 
+  /** Produce and publish a proposer slashing from an observed equivocation. Does not throw, only logs errors */
+  processProposerEquivocation(blockSlot: Slot, proposerIndex: ValidatorIndex): void;
+
   getStatus(): Status;
 
   recomputeForkChoiceHead(caller: ForkchoiceCaller): ProtoBlock;

package/src/chain/options.ts

@@ -37,6 +37,8 @@ export type IChainOptions = BlockProcessOpts &
     suggestedFeeRecipient: string;
     graffitiAppend?: boolean;
     maxSkipSlots?: number;
+    /** Do not produce proposer slashings from observed equivocations and do not include proposer slashings in produced blocks */
+    disableProposerSlashings?: boolean;
     /** Ensure blobs returned by the execution engine are valid */
     sanityCheckExecutionEngineBlobs?: boolean;
     /** Max number of produced blobs by local validators to cache */
@@ -122,6 +124,7 @@ export const defaultChainOptions: IChainOptions = {
   // for gossip block validation, it's unlikely we see a reorg with 32 slots
   // for attestation validation, having this value ensures we don't have to regen states most of the time
   maxSkipSlots: 32,
+  disableProposerSlashings: false,
   broadcastValidationStrictness: "warn",
   // should be less than or equal to MIN_SIGNATURE_SETS_TO_BATCH_VERIFY
   // batching too much may block the I/O thread so if useWorker=false, suggest this value to be 32

package/src/chain/prepareNextSlot.ts

@@ -1,6 +1,6 @@
 import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
-import {getSafeExecutionBlockHash} from "@lodestar/fork-choice";
+import {getFinalizedExecutionBlockHash, getSafeExecutionBlockHash} from "@lodestar/fork-choice";
 import {
   ForkPostBellatrix,
   ForkSeq,
@@ -10,17 +10,20 @@ import {
   isForkPostGloas,
 } from "@lodestar/params";
 import {
+  GLOAS_PREVERIFY_WINDOW_EPOCHS,
   IBeaconStateView,
   IBeaconStateViewBellatrix,
+  MAX_BUILDER_DEPOSITS_PER_SLOT,
   StateHashTreeRootSource,
   computeEpochAtSlot,
   computeTimeAtSlot,
   isStatePostBellatrix,
+  isStatePostFulu,
   isStatePostGloas,
 } from "@lodestar/state-transition";
 import {Bytes32, Slot, ValidatorIndex} from "@lodestar/types";
 import {Logger, fromHex, isErrorAborted, sleep} from "@lodestar/utils";
-import {GENESIS_SLOT, ZERO_HASH_HEX} from "../constants/constants.js";
+import {GENESIS_SLOT} from "../constants/constants.js";
 import {BuilderStatus} from "../execution/builder/http.js";
 import {Metrics} from "../metrics/index.js";
 import {ClockEvent} from "../util/clock.js";
@@ -37,6 +40,11 @@ export const PREPARE_NEXT_SLOT_BPS = 6667;
 /* We don't want to do more epoch transition than this */
 const PREPARE_EPOCH_LIMIT = 1;
 
+/* Fraction of a slot (out of 10_000) the pre-Gloas builder-deposit pre-verify may spend per tick.
+ * 2000 = 20% of slot (2.4s on mainnet), within the spare window left after PREPARE_NEXT_SLOT_BPS.
+ * Expressed in bps so it scales with slot duration (mainnet vs minimal/devnet). */
+const BUILDER_PREVERIFY_LIMIT_BPS = 2000;
+
 /**
  * At Bellatrix, if we are responsible for proposing in next slot, we want to prepare payload
  * 4s before the start of next slot at PREPARE_NEXT_SLOT_BPS of the current slot.
@@ -117,6 +125,7 @@ export class PrepareNextSlotScheduler {
         ? this.metrics?.precomputeNextEpochTransition.duration.startTimer()
         : null;
       const start = Date.now();
+      let feeRecipient: string | undefined;
       // No need to wait for this or the clock drift
       if (isForkPostBellatrix(fork)) {
         let preparedState: IBeaconStateView | undefined;
@@ -140,7 +149,7 @@ export class PrepareNextSlotScheduler {
         };
 
         const proposerIndex = await getProposerIndex();
-        const feeRecipient = this.chain.beaconProposerCache.get(proposerIndex);
+        feeRecipient = this.chain.beaconProposerCache.get(proposerIndex);
 
         if (feeRecipient) {
           // If we are proposing next slot, we need to predict if we can proposer-boost-reorg or not
@@ -214,9 +223,8 @@ export class PrepareNextSlotScheduler {
             computeTimeAtSlot(this.config, prepareSlot, this.chain.genesisTime) - Date.now() / 1000;
           this.metrics?.blockPayload.payloadAdvancePrepTime.observe(preparationTime);
 
-          const safeBlockHash = getSafeExecutionBlockHash(this.chain.forkChoice);
-          const finalizedBlockHash =
-            this.chain.forkChoice.getFinalizedBlock().executionPayloadBlockHash ?? ZERO_HASH_HEX;
+          const safeBlockHash = getSafeExecutionBlockHash(this.chain.forkChoice, this.logger);
+          const finalizedBlockHash = getFinalizedExecutionBlockHash(this.chain.forkChoice, this.logger);
 
           // awaiting here instead of throwing an async call because there is no other task
           // left for scheduler and this gives nice semantics to catch and log errors in the
@@ -287,6 +295,74 @@ export class PrepareNextSlotScheduler {
         });
 
         precomputeEpochTransitionTimer?.();
+
+        // Nothing more to do on an epoch-transition slot: that precompute already ran a full (>1s)
+        // epoch transition, so we don't pile the builder-deposit pre-verify (below) on top of it.
+        return;
+      }
+
+      // Pre-verify builder-deposit signatures in the GLOAS_PREVERIFY_WINDOW_EPOCHS epochs leading
+      // up to the Gloas fork, off the block-import hot path, so onboardBuildersFromPendingDeposits
+      // at the transition is mostly cache hits. Reached only on non-epoch-transition slots (the
+      // epoch-transition branch returned above). Runs on the head state — its pendingDeposits are
+      // what the fork transition derives from, and its epochCtx holds the shared, app-wide
+      // builderDepositSignatureCache.
+      const headState = this.chain.getHeadState();
+      if (isStatePostFulu(headState)) {
+        const gloasEpoch = this.config.GLOAS_FORK_EPOCH;
+        const finalizedEpoch = this.chain.forkChoice.getFinalizedCheckpoint().epoch;
+        if (finalizedEpoch >= gloasEpoch) {
+          // Gloas is finalized — the pre-verify cache has served its purpose; release its memory.
+          headState.clearPreGloasBuilderDepositCache();
+        } else {
+          const clockEpoch = computeEpochAtSlot(clockSlot);
+          if (
+            ForkSeq[fork] < ForkSeq.gloas && // only before the fork
+            clockEpoch >= gloasEpoch - GLOAS_PREVERIFY_WINDOW_EPOCHS && // Infinity when Gloas unscheduled → skip
+            clockEpoch < gloasEpoch &&
+            // skip when we're the next-slot proposer — don't compete with block-production prep.
+            feeRecipient === undefined
+          ) {
+            const maxDurationMs = this.config.getSlotComponentDurationMs(BUILDER_PREVERIFY_LIMIT_BPS);
+            const preVerifyTimer = this.metrics?.builderDepositPreVerify.duration.startTimer();
+            const result = headState.preVerifyBuilderDepositsPreGloas(MAX_BUILDER_DEPOSITS_PER_SLOT, maxDurationMs);
+            preVerifyTimer?.();
+
+            const preVerifyMetrics = this.metrics?.builderDepositPreVerify;
+            if (preVerifyMetrics) {
+              preVerifyMetrics.pendingDeposits.set(result.pendingDepositsCount);
+              preVerifyMetrics.cachedDeposits.set(result.totalCachedDeposits);
+              preVerifyMetrics.scannedDeposits.set(result.scannedPendingDeposits);
+              preVerifyMetrics.builderPubkeys.set(result.totalBuilderPubkeys);
+              if (result.validBuilderSignaturesCount > 0) {
+                preVerifyMetrics.validSignatures.inc({type: "builder"}, result.validBuilderSignaturesCount);
+              }
+              if (result.validValidatorSignaturesCount > 0) {
+                preVerifyMetrics.validSignatures.inc({type: "validator"}, result.validValidatorSignaturesCount);
+              }
+              if (result.invalidBuilderSignaturesCount > 0) {
+                preVerifyMetrics.invalidSignatures.inc({type: "builder"}, result.invalidBuilderSignaturesCount);
+              }
+              if (result.invalidValidatorSignaturesCount > 0) {
+                preVerifyMetrics.invalidSignatures.inc({type: "validator"}, result.invalidValidatorSignaturesCount);
+              }
+            }
+
+            // Always log (once per slot, only within the pre-fork window) so devnets show the
+            // pre-verify draining even on all-cache-hit ticks.
+            this.logger.verbose("PrepareNextSlotScheduler pre-verified deposit signatures", {
+              clockSlot,
+              validBuilderSignaturesCount: result.validBuilderSignaturesCount,
+              invalidBuilderSignaturesCount: result.invalidBuilderSignaturesCount,
+              validValidatorSignaturesCount: result.validValidatorSignaturesCount,
+              invalidValidatorSignaturesCount: result.invalidValidatorSignaturesCount,
+              scannedPendingDeposits: result.scannedPendingDeposits,
+              totalCachedDeposits: result.totalCachedDeposits,
+              totalBuilderPubkeys: result.totalBuilderPubkeys,
+              pendingDepositsCount: result.pendingDepositsCount,
+            });
+          }
+        }
       }
     } catch (e) {
       if (!isErrorAborted(e) && !isQueueErrorAborted(e)) {

package/src/chain/produceBlock/produceBlockBody.ts

@@ -1,6 +1,11 @@
 import {BitArray} from "@chainsafe/ssz";
 import {ChainForkConfig} from "@lodestar/config";
-import {IForkChoice, ProtoBlock, getSafeExecutionBlockHash} from "@lodestar/fork-choice";
+import {
+  IForkChoice,
+  ProtoBlock,
+  getFinalizedExecutionBlockHash,
+  getSafeExecutionBlockHash,
+} from "@lodestar/fork-choice";
 import {
   BUILDER_INDEX_SELF_BUILD,
   ForkName,
@@ -265,8 +270,8 @@ export async function produceBlockBody<T extends BlockType>(
     // TODO GLOAS: support non self-building here, the block type differentiation between
     // full and blinded no longer makes sense in gloas, it might be a good idea to move
     // this into a completely separate function and have pre/post gloas more separated
-    const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice);
-    const finalizedBlockHash = this.forkChoice.getFinalizedBlock().executionPayloadBlockHash ?? ZERO_HASH_HEX;
+    const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice, this.logger);
+    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice, this.logger);
     // TODO GLOAS: post-Gloas, proposer feeRecipient is also carried (signed) in
     // ProposerPreferencesPool. Consider using this unified cache instead
     // see https://github.com/ChainSafe/lodestar/issues/9379
@@ -411,8 +416,8 @@ export async function produceBlockBody<T extends BlockType>(
       throw new Error("Expected Bellatrix state for execution block production");
     }
 
-    const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice);
-    const finalizedBlockHash = this.forkChoice.getFinalizedBlock().executionPayloadBlockHash ?? ZERO_HASH_HEX;
+    const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice, this.logger);
+    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice, this.logger);
     const feeRecipient = requestedFeeRecipient ?? this.beaconProposerCache.getOrDefault(proposerIndex);
     const feeRecipientType = requestedFeeRecipient
       ? "requested"
@@ -1037,7 +1042,7 @@ export async function produceCommonBlockBody<T extends BlockType>(
     graffiti,
     // Eth1 data voting is no longer required since electra
     eth1Data: currentState.eth1Data,
-    proposerSlashings,
+    proposerSlashings: this.opts.disableProposerSlashings === true ? [] : proposerSlashings,
     attesterSlashings,
     attestations,
     // Since electra, deposits are processed by the execution layer,

package/src/chain/seenCache/seenBlockProposers.ts

@@ -1,21 +1,83 @@
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
-import {Epoch, Slot, ValidatorIndex} from "@lodestar/types";
+import {Epoch, RootHex, Slot, ValidatorIndex, phase0} from "@lodestar/types";
 import {MapDef} from "@lodestar/utils";
 
+/** Two distinct block roots signed by the same proposer for the same slot are sufficient to establish an equivocation */
+const MAX_BLOCK_ROOTS_PER_PROPOSAL = 2;
+
 /**
  * Keeps a cache to filter block proposals from the same validator in the same slot.
  *
- * This cache is not bounded and for extremely long periods of non-finality it can grow a lot. However it's practically
- * limited by the possible shufflings in those epochs, and the stored data is very cheap
+ * Block roots with a signature verified against the block's proposer index are tracked separately from proposals
+ * accepted by gossip validation or block import. A root from a block signed by its declared proposer is potential
+ * equivocation evidence, but the block may still fail other validation. Such a block must not mark the proposal as
+ * known, since that would cause a later valid block for the same slot and proposer to be ignored as a repeat proposal.
+ *
+ * The signed block header of each observed root is kept so a proposer slashing can be produced from the two
+ * conflicting headers once an equivocation is established.
+ *
+ * The cache is pruned on finalization and bounds the number of roots stored per proposer and slot
  */
 export class SeenBlockProposers {
   private readonly proposerIndexesBySlot = new MapDef<Slot, Set<ValidatorIndex>>(() => new Set<ValidatorIndex>());
-  private finalizedSlot: Epoch = 0;
+  private readonly signedBlockHeadersBySlot = new MapDef<
+    Slot,
+    MapDef<ValidatorIndex, Map<RootHex, phase0.SignedBeaconBlockHeader>>
+  >(() => new MapDef<ValidatorIndex, Map<RootHex, phase0.SignedBeaconBlockHeader>>(() => new Map()));
+  private finalizedSlot: Slot = 0;
 
   isKnown(blockSlot: Slot, proposerIndex: ValidatorIndex): boolean {
     return this.proposerIndexesBySlot.get(blockSlot)?.has(proposerIndex) === true;
   }
 
+  hasBlockRoot(blockSlot: Slot, proposerIndex: ValidatorIndex, blockRoot: RootHex): boolean {
+    return this.signedBlockHeadersBySlot.get(blockSlot)?.get(proposerIndex)?.has(blockRoot) === true;
+  }
+
+  isEquivocating(blockSlot: Slot, proposerIndex: ValidatorIndex): boolean {
+    return (
+      (this.signedBlockHeadersBySlot.get(blockSlot)?.get(proposerIndex)?.size ?? 0) >= MAX_BLOCK_ROOTS_PER_PROPOSAL
+    );
+  }
+
+  getConflictingBlockRoots(blockSlot: Slot, proposerIndex: ValidatorIndex, blockRoot: RootHex): RootHex[] {
+    const signedBlockHeaderByRoot = this.signedBlockHeadersBySlot.get(blockSlot)?.get(proposerIndex);
+    return signedBlockHeaderByRoot === undefined
+      ? []
+      : Array.from(signedBlockHeaderByRoot.keys()).filter((root) => root !== blockRoot);
+  }
+
+  /** Return the two signed block headers that establish an equivocation, or null if there is none */
+  getEquivocationHeaders(
+    blockSlot: Slot,
+    proposerIndex: ValidatorIndex
+  ): [phase0.SignedBeaconBlockHeader, phase0.SignedBeaconBlockHeader] | null {
+    const signedBlockHeaderByRoot = this.signedBlockHeadersBySlot.get(blockSlot)?.get(proposerIndex);
+    if (signedBlockHeaderByRoot === undefined || signedBlockHeaderByRoot.size < MAX_BLOCK_ROOTS_PER_PROPOSAL) {
+      return null;
+    }
+    const [signedHeader1, signedHeader2] = signedBlockHeaderByRoot.values();
+    return [signedHeader1, signedHeader2];
+  }
+
+  /** Record a block only after its proposer signature has been verified */
+  observeBlockRoot(
+    blockSlot: Slot,
+    proposerIndex: ValidatorIndex,
+    blockRoot: RootHex,
+    signedBlockHeader: phase0.SignedBeaconBlockHeader
+  ): void {
+    if (blockSlot < this.finalizedSlot) {
+      throw Error(`blockSlot ${blockSlot} < finalizedSlot ${this.finalizedSlot}`);
+    }
+
+    const signedBlockHeaderByRoot = this.signedBlockHeadersBySlot.getOrDefault(blockSlot).getOrDefault(proposerIndex);
+    if (signedBlockHeaderByRoot.size < MAX_BLOCK_ROOTS_PER_PROPOSAL && !signedBlockHeaderByRoot.has(blockRoot)) {
+      signedBlockHeaderByRoot.set(blockRoot, signedBlockHeader);
+    }
+  }
+
+  /** Mark a block as known from gossip or another block import path */
   add(blockSlot: Slot, proposerIndex: ValidatorIndex): void {
     if (blockSlot < this.finalizedSlot) {
       throw Error(`blockSlot ${blockSlot} < finalizedSlot ${this.finalizedSlot}`);
@@ -31,9 +93,14 @@ export class SeenBlockProposers {
         this.proposerIndexesBySlot.delete(slot);
       }
     }
+    for (const slot of this.signedBlockHeadersBySlot.keys()) {
+      if (slot < finalizedSlot) {
+        this.signedBlockHeadersBySlot.delete(slot);
+      }
+    }
   }
 
-  seenAtEpoch(epoch: Slot, index: ValidatorIndex): boolean {
+  seenAtEpoch(epoch: Epoch, index: ValidatorIndex): boolean {
     const fromSlot = computeStartSlotAtEpoch(epoch);
     const toSlot = computeStartSlotAtEpoch(epoch + 1);
 

package/src/chain/validation/block.ts

@@ -23,8 +23,9 @@ import {
   getBlockProposerSignatureSet,
   isExecutionBlockBodyType,
   isStatePostBellatrix,
+  signedBlockToSignedHeader,
 } from "@lodestar/state-transition";
-import {SignedBeaconBlock, deneb, gloas, isGloasBeaconBlock} from "@lodestar/types";
+import {RootHex, SignedBeaconBlock, deneb, gloas, isGloasBeaconBlock, ssz} from "@lodestar/types";
 import {byteArrayEquals, sleep, toRootHex} from "@lodestar/utils";
 import {BlockErrorCode, BlockGossipError, GossipAction} from "../errors/index.js";
 import {IBeaconChain} from "../interface.js";
@@ -75,7 +76,10 @@ export async function validateGossipBlock(
   // reboot if the `observed_block_producers` cache is empty. In that case, without this
   // check, we will load the parent and state from disk only to find out later that we
   // already know this block.
-  const blockRoot = toRootHex(config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block));
+  // A block's hash tree root is identical to its header's, so the root is derived from the header
+  // which is also used as potential equivocation evidence
+  const signedBlockHeader = signedBlockToSignedHeader(config, signedBlock);
+  const blockRoot = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(signedBlockHeader.message));
   if (chain.forkChoice.getBlockHexDefaultStatus(blockRoot) !== null) {
     throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.ALREADY_KNOWN, root: blockRoot});
   }
@@ -85,7 +89,12 @@ export async function validateGossipBlock(
 
   // [IGNORE] The block is the first block with valid signature received for the proposer for the slot, signed_beacon_block.message.slot.
   const proposerIndex = block.proposerIndex;
+  const hasBlockRoot = chain.seenBlockProposers.hasBlockRoot(blockSlot, proposerIndex, blockRoot);
   if (chain.seenBlockProposers.isKnown(blockSlot, proposerIndex)) {
+    if (!hasBlockRoot && !chain.seenBlockProposers.isEquivocating(blockSlot, proposerIndex)) {
+      await verifyBlockProposerSignature(chain, signedBlock, blockRoot, {verifyOnMainThread: false});
+      chain.seenBlockProposers.observeBlockRoot(blockSlot, proposerIndex, blockRoot, signedBlockHeader);
+    }
     throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.REPEAT_PROPOSAL, proposerIndex});
   }
 
@@ -270,18 +279,8 @@ export async function validateGossipBlock(
   }
 
   // [REJECT] The proposer signature, signed_beacon_block.signature, is valid with respect to the proposer_index pubkey.
-  if (!chain.seenBlockInputCache.isVerifiedProposerSignature(blockSlot, blockRoot, signedBlock.signature)) {
-    const signatureSet = getBlockProposerSignatureSet(chain.config, signedBlock);
-    // Don't batch so verification is not delayed
-    if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) {
-      throw new BlockGossipError(GossipAction.REJECT, {
-        code: BlockErrorCode.PROPOSAL_SIGNATURE_INVALID,
-        blockSlot,
-      });
-    }
-
-    chain.seenBlockInputCache.markVerifiedProposerSignature(blockSlot, blockRoot, signedBlock.signature);
-  }
+  await verifyBlockProposerSignature(chain, signedBlock, blockRoot);
+  chain.seenBlockProposers.observeBlockRoot(blockSlot, proposerIndex, blockRoot, signedBlockHeader);
 
   // [REJECT] The block is proposed by the expected proposer_index for the block's slot in the context of the current
   // shuffling (defined by parent_root/slot). If the proposer_index cannot immediately be verified against the expected
@@ -291,11 +290,6 @@ export async function validateGossipBlock(
     throw new BlockGossipError(GossipAction.REJECT, {code: BlockErrorCode.INCORRECT_PROPOSER, proposerIndex});
   }
 
-  // Check again in case there two blocks are processed concurrently
-  if (chain.seenBlockProposers.isKnown(blockSlot, proposerIndex)) {
-    throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.REPEAT_PROPOSAL, proposerIndex});
-  }
-
   // Simple implementation of a pending block queue. Keeping the block here recycles the queue logic, and keeps the
   // gossip validation promise without any extra infrastructure.
   // Do the sleep at the end, since regen and signature validation can already take longer than `msToBlockSlot`.
@@ -305,7 +299,35 @@ export async function validateGossipBlock(
     await sleep(msToBlockSlot);
   }
 
+  // Check again after all async validation and the early-block delay so concurrent proposals cannot both pass
+  if (chain.seenBlockProposers.isKnown(blockSlot, proposerIndex)) {
+    throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.REPEAT_PROPOSAL, proposerIndex});
+  }
+
   chain.seenBlockProposers.add(blockSlot, proposerIndex);
 
   return {skippedSlots};
 }
+
+export async function verifyBlockProposerSignature(
+  chain: IBeaconChain,
+  signedBlock: SignedBeaconBlock,
+  blockRoot: RootHex,
+  opts: {verifyOnMainThread?: boolean} = {}
+): Promise<void> {
+  const blockSlot = signedBlock.message.slot;
+  if (chain.seenBlockInputCache.isVerifiedProposerSignature(blockSlot, blockRoot, signedBlock.signature)) {
+    return;
+  }
+
+  const signatureSet = getBlockProposerSignatureSet(chain.config, signedBlock);
+  // Don't batch so verification is not delayed
+  if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: opts.verifyOnMainThread ?? true}))) {
+    throw new BlockGossipError(GossipAction.REJECT, {
+      code: BlockErrorCode.PROPOSAL_SIGNATURE_INVALID,
+      blockSlot,
+    });
+  }
+
+  chain.seenBlockInputCache.markVerifiedProposerSignature(blockSlot, blockRoot, signedBlock.signature);
+}

package/src/metrics/metrics/lodestar.ts

@@ -1179,6 +1179,10 @@ export function createLodestarMetrics(
         name: "lodestar_oppool_proposer_slashing_pool_size",
         help: "Current size of the ProposerSlashingPool",
       }),
+      proposerSlashingsProduced: register.counter({
+        name: "lodestar_oppool_proposer_slashings_produced_total",
+        help: "Total number of proposer slashings produced from observed equivocations",
+      }),
       voluntaryExitPoolSize: register.gauge({
         name: "lodestar_oppool_voluntary_exit_pool_size",
         help: "Current size of the VoluntaryExitPool",
@@ -1708,6 +1712,41 @@ export function createLodestarMetrics(
       }),
     },
 
+    // Pre-verify builder-deposit signatures in the epochs before the Gloas fork (prepareNextSlot).
+    builderDepositPreVerify: {
+      duration: register.histogram({
+        name: "lodestar_builder_deposit_preverify_duration_seconds",
+        help: "Duration of one preVerifyBuilderDepositsPreGloas call (BLS batch verify + scan)",
+        buckets: [1, 2, 3, 4],
+      }),
+      pendingDeposits: register.gauge({
+        name: "lodestar_builder_deposit_preverify_pending_deposits",
+        help: "Current state.pendingDeposits length (backlog to drain before the fork)",
+      }),
+      cachedDeposits: register.gauge({
+        name: "lodestar_builder_deposit_preverify_cached_deposits",
+        help: "Deposit signatures verified & cached so far this pre-fork window (valid + invalid)",
+      }),
+      scannedDeposits: register.gauge({
+        name: "lodestar_builder_deposit_preverify_scanned_deposits",
+        help: "Pending deposits examined by the last pre-verify call (< pending_deposits ⇒ cap hit)",
+      }),
+      builderPubkeys: register.gauge({
+        name: "lodestar_builder_deposit_preverify_builder_pubkeys",
+        help: "Distinct builder pubkeys tracked this window (validator deposits for these are pre-verified)",
+      }),
+      validSignatures: register.counter<{type: "builder" | "validator"}>({
+        name: "lodestar_builder_deposit_preverify_valid_signatures_total",
+        help: "Cumulative deposit signatures whose BLS verification passed (VALID), by credential type",
+        labelNames: ["type"],
+      }),
+      invalidSignatures: register.counter<{type: "builder" | "validator"}>({
+        name: "lodestar_builder_deposit_preverify_invalid_signatures_total",
+        help: "Cumulative deposit signatures whose BLS verification failed (INVALID), by credential type",
+        labelNames: ["type"],
+      }),
+    },
+
     // reprocess attestations
     reprocessApiAttestations: {
       total: register.gauge({

package/src/network/core/networkCore.ts

@@ -70,6 +70,15 @@ export type BaseNetworkInit = {
   initialCustodyGroupCount: number;
 };
 
+/** Counts of what is still keeping this thread's event loop alive, e.g. `TCPWrap=3,Timeout=1` */
+function formatActiveResources(): string {
+  const counts = new Map<string, number>();
+  for (const resource of process.getActiveResourcesInfo()) {
+    counts.set(resource, (counts.get(resource) ?? 0) + 1);
+  }
+  return Array.from(counts, ([name, count]) => `${name}=${count}`).join(",");
+}
+
 /**
  * This class is meant to work both:
  * - In a libp2p worker
@@ -289,7 +298,10 @@ export class NetworkCore implements INetworkCore {
     this.attnetsService.close();
     this.syncnetsService.close();
     await this.libp2p.stop();
-    this.logger.debug("network lib2p closed");
+    // Diagnostic for the shutdown hang, this thread can spin in `Environment::CleanupHandles()` on
+    // a handle that never closes and `Worker.terminate()` then never resolves. Diffing this list
+    // between a clean and a stuck shutdown should narrow down which handle it is
+    this.logger.debug("network lib2p closed", {activeResources: formatActiveResources()});
 
     this.closed = true;
   }

package/src/network/core/networkCoreWorkerHandler.ts

@@ -10,6 +10,7 @@ import {BeaconConfig, chainConfigToJson} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ResponseIncoming, ResponseOutgoing} from "@lodestar/reqresp";
 import {Status} from "@lodestar/types";
+import {withTimeout} from "@lodestar/utils";
 import {Metrics} from "../../metrics/index.js";
 import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js";
 import {PeerIdStr, peerIdFromString} from "../../util/peerId.js";
@@ -58,6 +59,10 @@ type WorkerNetworkCoreModules = WorkerNetworkCoreInitModules & {
 
 const NETWORK_WORKER_EXIT_TIMEOUT_MS = 1000;
 const NETWORK_WORKER_EXIT_RETRY_COUNT = 3;
+/** `getApi().close()` is an unbounded RPC into the worker and runs before `BeaconNode.close()`
+ * archives chain state to disk, so it is bounded here. Set well above the ~2-4s a close takes on
+ * mainnet, tripping it leaves libp2p half torn down */
+const NETWORK_CORE_CLOSE_TIMEOUT_MS = 5000;
 
 /**
  * NetworkCore implementation using a Worker thread
@@ -165,15 +170,25 @@ export class WorkerNetworkCore implements INetworkCore {
 
   async close(): Promise<void> {
     this.modules.logger.debug("closing network core running in network worker");
-    await this.getApi().close();
+    try {
+      await withTimeout(async () => this.getApi().close(), NETWORK_CORE_CLOSE_TIMEOUT_MS);
+    } catch (e) {
+      this.modules.logger.debug("Error closing network core", {}, e as Error);
+    }
     this.modules.logger.debug("terminating network worker");
-    await terminateWorkerThread({
+    const terminated = await terminateWorkerThread({
       worker: this.getApi(),
       retryCount: NETWORK_WORKER_EXIT_RETRY_COUNT,
       retryMs: NETWORK_WORKER_EXIT_TIMEOUT_MS,
       logger: this.modules.logger,
     });
-    this.modules.logger.debug("terminated network worker");
+    if (terminated) {
+      this.modules.logger.debug("terminated network worker");
+    } else {
+      // Nothing more to do, `process.exit()` joins the thread regardless so the process still waits
+      // on it. What matters is returning, so the caller can archive state and close the db
+      this.modules.logger.debug("Network worker did not terminate in time, continuing shutdown without it");
+    }
   }
 
   async test(): Promise<void> {