@lodestar/beacon-node 1.47.0-dev.4546ac1229 → 1.47.0-dev.56250f4ab6

package/src/execution/builder/validateBid.ts

@@ -0,0 +1,166 @@
+import {routes} from "@lodestar/api";
+import {ProtoBlock} from "@lodestar/fork-choice";
+import {MAX_EXECUTION_PAYMENT, PAYLOAD_BUILDER_VERSION} from "@lodestar/params";
+import {
+  computeEpochAtSlot,
+  createSingleSignatureSetFromComponents,
+  getExecutionPayloadBidSigningRoot,
+  isActiveBuilder,
+  isGasLimitTargetCompatible,
+  isStatePostGloas,
+} from "@lodestar/state-transition";
+import {RootHex, Slot, gloas} from "@lodestar/types";
+import {bigIntMin, byteArrayEquals, prettyGweiToEth, toHex, toRootHex} from "@lodestar/utils";
+import {IBeaconChain} from "../../chain/index.js";
+import {RegenCaller} from "../../chain/regen/index.js";
+import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
+
+/** Return the counted bid payment in Gwei, saturated at uint64 max. */
+export function getBuilderBidTotalGwei(bid: gloas.ExecutionPayloadBid, maxExecutionPayment: bigint): bigint {
+  return bigIntMin(MAX_EXECUTION_PAYMENT, BigInt(bid.value) + bigIntMin(bid.executionPayment, maxExecutionPayment));
+}
+
+/**
+ * Validate a bid received from a builder over the builder API in response to a bid request
+ * made during block production. Unlike gossip validation, the bid must match the requested
+ * slot and parent exactly, may carry a non-zero `executionPayment` which is counted at most at
+ * the entry's `maxExecutionPayment`, and is not subject to gossip anti-spam rules.
+ *
+ * Throws with a description of the failure, the caller drops the bid.
+ */
+export async function validateBuilderApiExecutionPayloadBid(
+  chain: IBeaconChain,
+  signedExecutionPayloadBid: gloas.SignedExecutionPayloadBid,
+  request: {
+    slot: Slot;
+    parentBlock: ProtoBlock;
+    parentBlockHash: RootHex;
+    parentBlockRoot: RootHex;
+    entry: routes.validator.BuilderEntry;
+  }
+): Promise<void> {
+  const bid = signedExecutionPayloadBid.message;
+  const {slot, parentBlock, parentBlockHash, parentBlockRoot, entry} = request;
+
+  if (bid.slot !== slot) {
+    throw Error(`Bid slot=${bid.slot} does not match requested slot=${slot}`);
+  }
+
+  const bidParentBlockHash = toRootHex(bid.parentBlockHash);
+  const bidParentBlockRoot = toRootHex(bid.parentBlockRoot);
+  if (bidParentBlockHash !== parentBlockHash || bidParentBlockRoot !== parentBlockRoot) {
+    throw Error(
+      `Bid parent parentBlockHash=${bidParentBlockHash} parentBlockRoot=${bidParentBlockRoot} does not match ` +
+        `requested parentBlockHash=${parentBlockHash} parentBlockRoot=${parentBlockRoot}`
+    );
+  }
+
+  const totalPayment = getBuilderBidTotalGwei(bid, entry.maxExecutionPayment);
+  if (totalPayment < entry.minBid) {
+    throw Error(
+      `Bid total payment=${prettyGweiToEth(totalPayment)} ` +
+        `(value=${prettyGweiToEth(bid.value)} ` +
+        `executionPayment=${prettyGweiToEth(bid.executionPayment)}) ` +
+        `is below minBid=${prettyGweiToEth(entry.minBid)}`
+    );
+  }
+
+  const state = await chain.regen
+    .getBlockSlotState(parentBlock, slot, {dontTransferCache: true}, RegenCaller.validateGossipExecutionPayloadBid)
+    .catch((e: Error) => {
+      throw Error(`Unable to regenerate state to validate bid: ${e.message}`);
+    });
+
+  if (!isStatePostGloas(state)) {
+    throw Error(`Expected gloas+ state for execution payload bid validation, got fork=${state.forkName}`);
+  }
+
+  if (bid.builderIndex >= state.getBuildersLength()) {
+    throw Error(`Bid builderIndex=${bid.builderIndex} is out of bounds`);
+  }
+
+  const builder = state.getBuilder(bid.builderIndex);
+  if (!isActiveBuilder(builder, state.finalizedCheckpoint.epoch)) {
+    throw Error(`Bid builderIndex=${bid.builderIndex} is not an active builder`);
+  }
+
+  if (builder.version !== PAYLOAD_BUILDER_VERSION) {
+    throw Error(`Invalid builder version=${builder.version} expected=${PAYLOAD_BUILDER_VERSION}`);
+  }
+
+  // A bid not signed by one of the builder pubkeys the entry accepts bids from must not be accepted
+  if (
+    entry.builderPubkeys.length > 0 &&
+    !entry.builderPubkeys.some((pubkey) => byteArrayEquals(pubkey, builder.pubkey))
+  ) {
+    throw Error(
+      `Bid builder pubkey=${toHex(builder.pubkey)} is not in the entry's ` +
+        `builderPubkeys=${entry.builderPubkeys.map(toHex).join(",")}`
+    );
+  }
+
+  const blobKzgCommitmentsLen = bid.blobKzgCommitments.length;
+  const maxBlobsPerBlock = chain.config.getMaxBlobsPerBlock(computeEpochAtSlot(bid.slot));
+  if (blobKzgCommitmentsLen > maxBlobsPerBlock) {
+    throw Error(`Bid has too many KZG commitments len=${blobKzgCommitmentsLen} limit=${maxBlobsPerBlock}`);
+  }
+
+  // The coverage check only applies to the staked collateral payment, a pure execution
+  // layer payment bid has nothing to cover on-chain
+  if (bid.value > 0 && !state.canBuilderCoverBid(bid.builderIndex, bid.value)) {
+    throw Error(
+      `Builder cannot cover bid value=${prettyGweiToEth(bid.value)} ` + `balance=${prettyGweiToEth(builder.balance)}`
+    );
+  }
+
+  const randaoMix = state.getRandaoMix(computeEpochAtSlot(state.slot));
+  if (!byteArrayEquals(bid.prevRandao, randaoMix)) {
+    throw Error(`Invalid bid prevRandao=${toHex(bid.prevRandao)} expected=${toHex(randaoMix)}`);
+  }
+
+  // The builder must honor the proposer preferences it learned over gossip
+  const bidEpoch = computeEpochAtSlot(bid.slot);
+  const dependentRootHex = (() => {
+    try {
+      return getShufflingDependentRoot(chain.forkChoice, bidEpoch, computeEpochAtSlot(parentBlock.slot), parentBlock);
+    } catch {
+      return null;
+    }
+  })();
+  if (dependentRootHex === null) {
+    throw Error(`Unable to resolve proposer preferences dependent root for bid slot=${bid.slot}`);
+  }
+  const proposerPreferences = chain.proposerPreferencesPool.get(bid.slot, dependentRootHex);
+  if (proposerPreferences === null) {
+    throw Error(`No proposer preferences found for bid slot=${bid.slot} dependentRoot=${dependentRootHex}`);
+  }
+  if (!byteArrayEquals(bid.feeRecipient, proposerPreferences.message.feeRecipient)) {
+    throw Error(
+      `Bid feeRecipient=${toHex(bid.feeRecipient)} does not match ` +
+        `proposer preferences feeRecipient=${toHex(proposerPreferences.message.feeRecipient)}`
+    );
+  }
+
+  const parentPayloadVariant = chain.forkChoice.getBlockHexAndBlockHash(bidParentBlockRoot, bidParentBlockHash);
+  if (parentPayloadVariant === null || parentPayloadVariant.executionPayloadBlockHash === null) {
+    throw Error(`Unable to resolve parent payload gas limit for bid parentBlockHash=${bidParentBlockHash}`);
+  }
+  const parentGasLimit = BigInt(parentPayloadVariant.executionPayloadGasLimit);
+  const targetGasLimit = proposerPreferences.message.targetGasLimit;
+  if (!isGasLimitTargetCompatible(parentGasLimit, bid.gasLimit, targetGasLimit)) {
+    throw Error(
+      `Bid gasLimit=${bid.gasLimit} is not compatible with ` +
+        `parentGasLimit=${parentGasLimit} targetGasLimit=${targetGasLimit}`
+    );
+  }
+
+  const signatureSet = createSingleSignatureSetFromComponents(
+    builder.pubkey,
+    getExecutionPayloadBidSigningRoot(chain.config, bid),
+    signedExecutionPayloadBid.signature
+  );
+
+  if (!(await chain.bls.verifySignatureSets([signatureSet]))) {
+    throw Error(`Invalid bid signature builderIndex=${bid.builderIndex}`);
+  }
+}

package/src/metrics/metrics/lodestar.ts

@@ -4,7 +4,10 @@ import {ArchiveStoreTask} from "../../chain/archiveStore/archiveStore.js";
 import {FrequencyStateArchiveStep} from "../../chain/archiveStore/strategies/frequencyStateArchiveStrategy.js";
 import {BlockInputSource} from "../../chain/blocks/blockInput/index.js";
 import {PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
-import {PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js";
+import {
+  PayloadEnvelopeInputPruneReason,
+  PayloadEnvelopeInputSource,
+} from "../../chain/blocks/payloadEnvelopeInput/index.js";
 import {JobQueueItemType} from "../../chain/bls/index.js";
 import {AttestationErrorCode, BlockErrorCode} from "../../chain/errors/index.js";
 import {
@@ -21,7 +24,7 @@ import {ExecutionPayloadStatus} from "../../execution/index.js";
 import {GossipType} from "../../network/index.js";
 import {CannotAcceptWorkReason, ReprocessRejectReason} from "../../network/processor/index.js";
 import {BackfillSyncMethod} from "../../sync/backfill/backfill.js";
-import {PendingBlockType} from "../../sync/types.js";
+import {DroppedItemReason, PendingBlockType} from "../../sync/types.js";
 import {PeerSyncType, RangeSyncType} from "../../sync/utils/remoteSyncType.js";
 import {AllocSource} from "../../util/bufferPool.js";
 import {DataColumnReconstructionCode} from "../../util/dataColumns.js";
@@ -101,9 +104,10 @@ export function createLodestarMetrics(
         help: "Age of the first item of each key in the indexed queues in seconds",
         buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5],
       }),
-      queueTime: register.histogram({
+      queueTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_time_seconds",
         help: "Total time an item stays in queue until it is processed in seconds",
+        labelNames: ["topic"],
         buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5],
       }),
     },
@@ -677,9 +681,15 @@ export function createLodestarMetrics(
         name: "lodestar_sync_unknown_block_downloaded_blocks_error_total",
         help: "Total number of downloaded blocks errors in UnknownBlockSync",
       }),
-      removedBlocks: register.gauge({
+      removedBlocks: register.gauge<{reason: DroppedItemReason}>({
         name: "lodestar_sync_unknown_block_removed_blocks_total",
-        help: "Total number of removed bad blocks in UnknownBlockSync",
+        help: "Total pending blocks dropped from BlockInputSync without completing, by reason",
+        labelNames: ["reason"],
+      }),
+      removedPayloads: register.gauge<{reason: DroppedItemReason}>({
+        name: "lodestar_sync_unknown_block_removed_payloads_total",
+        help: "Total pending payloads dropped from BlockInputSync without completing, by reason",
+        labelNames: ["reason"],
       }),
       elapsedTimeTillReceived: register.histogram({
         name: "lodestar_sync_unknown_block_elapsed_time_till_received",
@@ -1225,6 +1235,12 @@ export function createLodestarMetrics(
         name: "lodestar_oppool_attester_slashing_pool_size",
         help: "Current size of the AttesterSlashingPool",
       }),
+      deferredVoluntaryExitPool: {
+        size: register.gauge({
+          name: "lodestar_oppool_deferred_voluntary_exit_pool_size",
+          help: "Current size of the DeferredVoluntaryExitPool",
+        }),
+      },
       proposerSlashingPoolSize: register.gauge({
         name: "lodestar_oppool_proposer_slashing_pool_size",
         help: "Current size of the ProposerSlashingPool",
@@ -1429,6 +1445,16 @@ export function createLodestarMetrics(
         help: "Checkpoint state cache size",
         labelNames: ["type"],
       }),
+      persistentTierEpochs: register.gauge<{tier: number}>({
+        name: "lodestar_cp_state_cache_persistent_tier_epochs",
+        help: "Number of epoch keys retained in each on-disk checkpoint state retention tier",
+        labelNames: ["tier"],
+      }),
+      persistentTierStates: register.gauge<{tier: number}>({
+        name: "lodestar_cp_state_cache_persistent_tier_states",
+        help: "Number of persisted checkpoint states retained in each on-disk retention tier",
+        labelNames: ["tier"],
+      }),
       reads: register.avgMinMax({
         name: "lodestar_cp_state_epoch_reads",
         help: "Avg min max of all state cache items total read count",
@@ -1643,9 +1669,15 @@ export function createLodestarMetrics(
           name: "lodestar_seen_payload_envelope_input_cache_serialized_object_refs",
           help: "Number of serialized-cache object refs retained by cached PayloadEnvelopeInputs",
         }),
-        created: register.counter({
+        created: register.counter<{source: PayloadEnvelopeInputSource}>({
           name: "lodestar_seen_payload_envelope_input_cache_items_created_total",
-          help: "Number of PayloadEnvelopeInputs created",
+          help: "Number of PayloadEnvelopeInputs created by source",
+          labelNames: ["source"],
+        }),
+        pruned: register.counter<{reason: PayloadEnvelopeInputPruneReason}>({
+          name: "lodestar_seen_payload_envelope_input_cache_items_pruned_total",
+          help: "Number of PayloadEnvelopeInputs evicted by reason",
+          labelNames: ["reason"],
         }),
       },
     },
@@ -1755,6 +1787,10 @@ export function createLodestarMetrics(
         name: "lodestar_precompute_next_epoch_transition_waste_total",
         help: "Total number of precomputing next epoch transition wasted",
       }),
+      predictedReorg: register.counter({
+        name: "lodestar_precompute_next_epoch_transition_predicted_reorg_total",
+        help: "Predicted epoch-boundary reorgs where the strong parent was dialed instead of the weak head",
+      }),
       duration: register.histogram({
         name: "lodestar_precompute_next_epoch_transition_duration_seconds",
         help: "Duration of precomputeNextEpochTransition, including epoch transition and hashTreeRoot",
@@ -1994,6 +2030,35 @@ export function createLodestarMetrics(
       }),
     },
 
+    builderApi: {
+      bidRequests: register.counter({
+        name: "lodestar_builder_api_bid_requests_total",
+        help: "Total count of execution payload bid requests sent to external builders",
+      }),
+      bidRequestErrors: register.counter({
+        name: "lodestar_builder_api_bid_request_errors_total",
+        help: "Total count of failed execution payload bid requests to external builders",
+      }),
+      bidsReceived: register.counter({
+        name: "lodestar_builder_api_bids_received_total",
+        help: "Total count of execution payload bids received from external builders",
+      }),
+      bidsDiscarded: register.counter({
+        name: "lodestar_builder_api_bids_discarded_total",
+        help: "Total count of execution payload bids from external builders discarded due to failed validation",
+      }),
+      blockSubmissions: register.counter<{status: "success" | "error"}>({
+        name: "lodestar_builder_api_block_submissions_total",
+        help: "Total count of signed beacon blocks submitted to external builders",
+        labelNames: ["status"],
+      }),
+      preferencesForwarded: register.counter<{status: "success" | "error"}>({
+        name: "lodestar_builder_api_preferences_forwarded_total",
+        help: "Total count of builder preferences forwarded to external builders",
+        labelNames: ["status"],
+      }),
+    },
+
     db: {
       dbReadReq: register.gauge<{bucket: string}>({
         name: "lodestar_db_read_req_total",

package/src/network/processor/constants.ts

@@ -0,0 +1,6 @@
+/**
+ * Max distinct peers tracked per unknown block/payload root.
+ * Normally we have 8 mesh peers per topic, and multi topics may point to the same root.
+ * Having 32 peers per root seems enough to search for missing blocks/payloads.
+ **/
+export const MAX_PEERS_PER_ROOT = 32;

package/src/network/processor/gossipHandlers.ts

@@ -196,7 +196,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         forkName: fork,
         sampledColumns: chain.custodyConfig.sampledColumns,
         custodyColumns: chain.custodyConfig.custodyColumns,
-        timeCreatedSec: seenTimestampSec,
+        seenTimestampSec,
+        source: PayloadEnvelopeInputSource.gossip,
       });
     }
 
@@ -640,6 +641,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         metrics?.gossipBlock.elapsedTimeTillProcessed.observe(delaySec);
 
         if (isForkPostGloas(blockInput.forkName)) {
+          // we should have the payloadInput in the seen cache so no need getOrReload() here
           const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockInput.blockRootHex);
           // This payloadInput should have been created just after gossip validation
           if (!payloadInput) {
@@ -1189,6 +1191,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       chain.validatorMonitor?.registerExecutionPayloadEnvelope(OpSource.gossip, delaySec, signedEnvelope);
 
       const blockRootHex = toRootHex(envelope.beaconBlockRoot);
+      // a gossip payload cannot be `MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE` slots after the block
+      // otherwise it'll get to UnknownBlockInput flow
       const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
 
       if (!payloadInput) {
@@ -1302,7 +1306,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
       // Handle valid payload bid by storing in a bid pool
       try {
-        const insertOutcome = chain.executionPayloadBidPool.add(executionPayloadBid);
+        const insertOutcome = chain.executionPayloadBidPool.add(executionPayloadBid, Math.floor(elapsedSec * 1000));
         metrics?.opPool.executionPayloadBidPool.gossipInsertOutcome.inc({insertOutcome});
       } catch (e) {
         logger.error("Error adding to executionPayloadBid pool", {}, e as Error);

package/src/network/processor/gossipQueues/indexed.ts

@@ -86,7 +86,6 @@ export class IndexedGossipQueueMinSize<T extends {indexed?: string; queueAddedMs
     const now = Date.now();
     // here we mutate item, which is used for gossip validation later
     item.indexed = key;
-    item.queueAddedMs = now;
     let queueItem = this.indexedItems.get(key);
     if (queueItem == null) {
       queueItem = {firstSeenMs: now, listItems: new LinkedList<T>()};

package/src/network/processor/index.ts

@@ -30,6 +30,7 @@ import {
   GossipValidatorBatchFn,
   GossipValidatorFn,
 } from "../gossip/interface.js";
+import {MAX_PEERS_PER_ROOT} from "./constants.js";
 import {createExtractBlockSlotRootFns} from "./extractSlotRootFns.js";
 import {GossipHandlerOpts, ValidatorFnsModules, getGossipHandlers} from "./gossipHandlers.js";
 import {createGossipQueues} from "./gossipQueues/index.js";
@@ -191,8 +192,12 @@ export class NetworkProcessor {
   // in that case PendingGossipsubMessage needs to be stored in this Map and reprocessed once the payload comes
   private readonly awaitingMessagesByPayloadBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
   private awaitingPayloadMessageCount = 0;
-  private unknownBlocksBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
-  private unknownEnvelopesBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
+  private unknownBlocksBySlot = new MapDef<Slot, MapDef<RootHex, Set<PeerIdStr>>>(
+    () => new MapDef<RootHex, Set<PeerIdStr>>(() => new Set())
+  );
+  private unknownEnvelopesBySlot = new MapDef<Slot, MapDef<RootHex, Set<PeerIdStr>>>(
+    () => new MapDef<RootHex, Set<PeerIdStr>>(() => new Set())
+  );
 
   constructor(
     modules: NetworkProcessorModules,
@@ -271,35 +276,53 @@ export class NetworkProcessor {
    * Search block via `ChainEvent.unknownBlockRoot` event
    * Slot is the message slot, which is not necessarily the same as the block's slot, but it can be used for a good prune strategy.
    * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+   * We forward every distinct peer that gossiped this root (not just the first) so BlockInputSync can
+   * prefer them as by-root download candidates.
    */
   searchUnknownBlock({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
-    if (
-      this.chain.seenBlock(root) ||
-      this.awaitingMessagesByBlockRoot.has(root) ||
-      this.unknownBlocksBySlot.getOrDefault(slot).has(root)
-    ) {
+    if (this.chain.seenBlock(root)) {
       return;
     }
-    // Search for the unknown block
-    this.unknownBlocksBySlot.getOrDefault(slot).add(root);
-    this.chain.emitter.emit(ChainEvent.unknownBlockRoot, {rootHex: root, peer, source});
+    const peersForRoot = this.unknownBlocksBySlot.getOrDefault(slot);
+    const alreadySearching = peersForRoot.has(root) || this.awaitingMessagesByBlockRoot.has(root);
+    const forwardedPeers = peersForRoot.getOrDefault(root);
+
+    // brand-new search always emits (peer may be undefined)
+    let shouldEmit = !alreadySearching;
+    if (peer !== undefined && !forwardedPeers.has(peer) && forwardedPeers.size < MAX_PEERS_PER_ROOT) {
+      // First time we forward THIS peer for THIS root: emit so BlockInputSync accumulates it.
+      forwardedPeers.add(peer);
+      shouldEmit = true;
+    }
+    if (shouldEmit) {
+      this.chain.emitter.emit(ChainEvent.unknownBlockRoot, {rootHex: root, peer, source});
+    }
   }
 
   /**
    * Search envelope via `ChainEvent.unknownEnvelopeBlockRoot` event
    * Slot is the message slot, which is not necessarily the same as the envelope's slot, but it can be used for a good prune strategy.
    * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+   * Same peer-forwarding + dedup behavior as searchUnknownBlock.
    */
   searchUnknownEnvelope({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
-    if (
-      this.chain.seenPayloadEnvelope(root) ||
-      this.awaitingMessagesByPayloadBlockRoot.has(root) ||
-      this.unknownEnvelopesBySlot.getOrDefault(slot).has(root)
-    ) {
+    if (this.chain.seenPayloadEnvelope(root)) {
       return;
     }
-    this.unknownEnvelopesBySlot.getOrDefault(slot).add(root);
-    this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, slot, peer, source});
+    const peersForRoot = this.unknownEnvelopesBySlot.getOrDefault(slot);
+    // capture "already searching" BEFORE getOrDefault(root) creates the entry
+    const alreadySearching = peersForRoot.has(root) || this.awaitingMessagesByPayloadBlockRoot.has(root);
+    const forwardedPeers = peersForRoot.getOrDefault(root);
+
+    let shouldEmit = !alreadySearching; // brand-new search always emits (peer may be undefined)
+    if (peer !== undefined && !forwardedPeers.has(peer) && forwardedPeers.size < MAX_PEERS_PER_ROOT) {
+      // First time we forward THIS peer for THIS root: emit so BlockInputSync accumulates it.
+      forwardedPeers.add(peer);
+      shouldEmit = true;
+    }
+    if (shouldEmit) {
+      this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, slot, peer, source});
+    }
   }
 
   private onPendingGossipsubMessage = (message: PendingGossipsubMessage): void => {
@@ -537,6 +560,7 @@ export class NetworkProcessor {
 
   private pushPendingGossipsubMessageToQueue(message: PendingGossipsubMessage): void {
     const topicType = message.topic.type;
+    message.queueAddedMs = Date.now();
     const droppedCount = this.gossipQueues[topicType].add(message);
     if (droppedCount) {
       // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
@@ -616,7 +640,7 @@ export class NetworkProcessor {
 
     for (const [slot, roots] of this.unknownBlocksBySlot) {
       if (slot > minSlot) continue;
-      for (const rootHex of roots) {
+      for (const rootHex of roots.keys()) {
         const gossipMessages = this.awaitingMessagesByBlockRoot.get(rootHex);
         if (gossipMessages !== undefined) {
           for (const message of gossipMessages) {
@@ -641,7 +665,7 @@ export class NetworkProcessor {
 
     for (const [slot, roots] of this.unknownEnvelopesBySlot) {
       if (slot > minSlot) continue;
-      for (const rootHex of roots) {
+      for (const rootHex of roots.keys()) {
         const gossipMessages = this.awaitingMessagesByPayloadBlockRoot.get(rootHex);
         if (gossipMessages !== undefined) {
           for (const message of gossipMessages) {
@@ -723,12 +747,20 @@ export class NetworkProcessor {
       for (const msg of messageOrArray) {
         msg.startProcessUnixSec = nowSec;
         if (msg.queueAddedMs !== undefined) {
-          this.metrics?.gossipValidationQueue.queueTime.observe(nowSec - msg.queueAddedMs / 1000);
+          this.metrics?.gossipValidationQueue.queueTime.observe(
+            {topic: msg.topic.type},
+            nowSec - msg.queueAddedMs / 1000
+          );
         }
       }
     } else {
-      // indexed queue is not used here
       messageOrArray.startProcessUnixSec = nowSec;
+      if (messageOrArray.queueAddedMs !== undefined) {
+        this.metrics?.gossipValidationQueue.queueTime.observe(
+          {topic: messageOrArray.topic.type},
+          nowSec - messageOrArray.queueAddedMs / 1000
+        );
+      }
     }
 
     const acceptanceArr = Array.isArray(messageOrArray)

package/src/node/deferredVoluntaryExitPublisher.ts

@@ -0,0 +1,54 @@
+import {routes} from "@lodestar/api";
+import {Logger} from "@lodestar/utils";
+import {IBeaconChain} from "../chain/index.js";
+import {INetwork} from "../network/index.js";
+import {ClockEvent} from "../util/clock.js";
+
+/**
+ * On every epoch, drain the deferred voluntary exit pool and publish the exits that have become
+ * processable. Lives at the node layer rather than inside `BeaconChain` because publishing needs
+ * `network`, which the chain does not (and should not) depend on.
+ */
+export function startDeferredVoluntaryExitPublisher({
+  chain,
+  network,
+  logger,
+  signal,
+}: {
+  chain: IBeaconChain;
+  network: INetwork;
+  logger: Logger;
+  signal: AbortSignal;
+}): void {
+  const onEpoch = async (): Promise<void> => {
+    try {
+      // Use the cached head state directly rather than getHeadStateAtCurrentEpoch, which would regen
+      // the head forward to the current epoch. At init from a stale db-finalized state that regen can
+      // be many epoch transitions. The head state is fine here: this is a best-effort drain that
+      // re-runs every epoch, so a just-processable exit is republished next tick.
+      const state = chain.getHeadState();
+      const exits = chain.deferredVoluntaryExitPool.drainProcessableExits(state);
+      for (const exit of exits) {
+        try {
+          chain.opPool.insertVoluntaryExit(exit);
+          chain.emitter.emit(routes.events.EventType.voluntaryExit, exit);
+          await network.publishVoluntaryExit(exit);
+          logger.info("Voluntary exit successfully published for validator", {
+            validatorIndex: exit.message.validatorIndex,
+          });
+        } catch (e) {
+          logger.warn(
+            "Failed to publish deferred voluntary exit",
+            {validatorIndex: exit.message.validatorIndex},
+            e as Error
+          );
+        }
+      }
+    } catch (e) {
+      logger.warn("Failed to drain deferred voluntary exit pool", {}, e as Error);
+    }
+  };
+
+  chain.clock.addListener(ClockEvent.epoch, onEpoch);
+  signal.addEventListener("abort", () => chain.clock.removeListener(ClockEvent.epoch, onEpoch), {once: true});
+}

package/src/node/nodejs.ts

@@ -22,6 +22,7 @@ import {Network, getReqRespHandlers} from "../network/index.js";
 import {BackfillSync} from "../sync/backfill/index.js";
 import {BeaconSync, IBeaconSync} from "../sync/index.js";
 import {Clock} from "../util/clock.js";
+import {startDeferredVoluntaryExitPublisher} from "./deferredVoluntaryExitPublisher.js";
 import {runNodeNotifier} from "./notifier.js";
 import {IBeaconNodeOptions} from "./options.js";
 
@@ -259,6 +260,11 @@ export class BeaconNode {
       executionBuilder: opts.executionBuilder.enabled
         ? initializeExecutionBuilder(opts.executionBuilder, config, metrics, logger)
         : undefined,
+      builderApiClientOpts: {
+        timeout: opts.executionBuilder.timeout,
+        // Sent with all builder api requests, unless the node runs in private mode
+        userAgent: opts.executionBuilder.userAgent,
+      },
     });
 
     // Load persisted data from disk to in-memory caches
@@ -335,6 +341,8 @@ export class BeaconNode {
 
     void runNodeNotifier({network, chain, sync, config, logger, signal});
 
+    startDeferredVoluntaryExitPublisher({chain, network, logger, signal});
+
     return new BeaconNode({
       opts,
       config,

package/src/sync/types.ts

@@ -36,6 +36,39 @@ export enum PendingBlockInputStatus {
   processing = "processing",
 }
 
+/**
+ * Why a pending item was dropped from BlockInputSync WITHOUT completing.
+ * The happy path — a successfully imported/processed block or payload — is removed by the import
+ * flow (onPayloadImported, processReadyBlock/processPayload success) and is deliberately NOT tracked
+ * here
+ */
+export enum DroppedItemReason {
+  /** prune: item's own slot is below the finalized slot (routine cleanup), or a downloaded block landed at/below finality */
+  belowFinalized = "below_finalized",
+  /** prune: slot-less, unresolvable payload evicted after PRUNE_UNRESOLVED_SLOT_EPOCHS */
+  agedOut = "aged_out",
+  /** pruneSetToMax evicted the oldest entry because the cache hit its capacity (DoS guard) */
+  capacity = "capacity",
+  /** descendant removed because an ancestor was invalid/removed */
+  invalidParent = "invalid_parent",
+  /** transient execution-engine error removal (EXECUTION_ENGINE_ERROR) */
+  elError = "el_error",
+  /** the execution engine declared the item invalid (EXECUTION_ENGINE_INVALID) */
+  elInvalid = "el_invalid",
+  /** a block failed our own checks (not correct w.r.t. our chain) */
+  invalidBlock = "invalid_block",
+  /** a block declares a parent payload hash that conflicts with the parent's actual payload (gloas) */
+  invalidParentPayload = "invalid_parent_payload",
+  /** a payload envelope failed gossip verification (ENVELOPE_VERIFICATION_ERROR) */
+  invalidEnvelope = "invalid_envelope",
+  /** a payload had an invalid signature (INVALID_SIGNATURE) */
+  invalidSignature = "invalid_signature",
+  /** a block root was given up on after exhausting download attempts (data could not be fetched) */
+  unavailable = "unavailable",
+  /** an unexpected / unhandled error code led to removal */
+  unknown = "unknown",
+}
+
 export enum PendingPayloadInputStatus {
   pending = "pending",
   fetching = "fetching",