@lodestar/beacon-node 1.47.0-dev.9572bcc698 → 1.47.0-dev.9ba9a5ce85

package/src/execution/builder/validateBid.ts

@@ -0,0 +1,181 @@
+import {routes} from "@lodestar/api";
+import {ProtoBlock} from "@lodestar/fork-choice";
+import {MAX_EXECUTION_PAYMENT, PAYLOAD_BUILDER_VERSION} from "@lodestar/params";
+import {
+  computeEpochAtSlot,
+  createSingleSignatureSetFromComponents,
+  getExecutionPayloadBidSigningRoot,
+  isActiveBuilder,
+  isGasLimitTargetCompatible,
+  isStatePostGloas,
+} from "@lodestar/state-transition";
+import {RootHex, Slot, gloas} from "@lodestar/types";
+import {bigIntMin, byteArrayEquals, prettyGweiToEth, toHex, toRootHex} from "@lodestar/utils";
+import {IBeaconChain} from "../../chain/index.js";
+import {RegenCaller} from "../../chain/regen/index.js";
+import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
+
+/** Return the counted bid payment in Gwei, saturated at uint64 max. */
+export function getBuilderBidTotalGwei(bid: gloas.ExecutionPayloadBid, maxExecutionPayment: bigint): bigint {
+  return bigIntMin(MAX_EXECUTION_PAYMENT, BigInt(bid.value) + bigIntMin(bid.executionPayment, maxExecutionPayment));
+}
+
+/**
+ * Validate a bid received from a builder over the builder API in response to a bid request
+ * made during block production. Unlike gossip validation, the bid must match the requested
+ * slot and parent exactly, may carry a non-zero `executionPayment` which is counted at most at
+ * the entry's `maxExecutionPayment`, and is not subject to gossip anti-spam rules.
+ *
+ * Throws with a description of the failure, the caller drops the bid.
+ */
+export async function validateBuilderApiExecutionPayloadBid(
+  chain: IBeaconChain,
+  signedExecutionPayloadBid: gloas.SignedExecutionPayloadBid,
+  request: {
+    slot: Slot;
+    parentBlock: ProtoBlock;
+    parentBlockHash: RootHex;
+    parentBlockRoot: RootHex;
+    entry: routes.validator.BuilderEntry;
+    getParentExecutionRequests: () => Promise<gloas.ExecutionRequests>;
+  }
+): Promise<void> {
+  const bid = signedExecutionPayloadBid.message;
+  const {slot, parentBlock, parentBlockHash, parentBlockRoot, entry, getParentExecutionRequests} = request;
+
+  if (bid.slot !== slot) {
+    throw Error(`Bid slot=${bid.slot} does not match requested slot=${slot}`);
+  }
+
+  const bidParentBlockHash = toRootHex(bid.parentBlockHash);
+  const bidParentBlockRoot = toRootHex(bid.parentBlockRoot);
+  if (bidParentBlockHash !== parentBlockHash || bidParentBlockRoot !== parentBlockRoot) {
+    throw Error(
+      `Bid parent parentBlockHash=${bidParentBlockHash} parentBlockRoot=${bidParentBlockRoot} does not match ` +
+        `requested parentBlockHash=${parentBlockHash} parentBlockRoot=${parentBlockRoot}`
+    );
+  }
+
+  const totalPayment = getBuilderBidTotalGwei(bid, entry.maxExecutionPayment);
+  if (totalPayment < entry.minBid) {
+    throw Error(
+      `Bid total payment=${prettyGweiToEth(totalPayment)} ` +
+        `(value=${prettyGweiToEth(bid.value)} ` +
+        `executionPayment=${prettyGweiToEth(bid.executionPayment)}) ` +
+        `is below minBid=${prettyGweiToEth(entry.minBid)}`
+    );
+  }
+
+  const state = await chain.regen
+    .getBlockSlotState(parentBlock, slot, {dontTransferCache: true}, RegenCaller.validateGossipExecutionPayloadBid)
+    .catch((e: Error) => {
+      throw Error(`Unable to regenerate state to validate bid: ${e.message}`);
+    });
+
+  if (!isStatePostGloas(state)) {
+    throw Error(`Expected gloas+ state for execution payload bid validation, got fork=${state.forkName}`);
+  }
+
+  if (bid.builderIndex >= state.getBuildersLength()) {
+    throw Error(`Bid builderIndex=${bid.builderIndex} is out of bounds`);
+  }
+
+  const builder = state.getBuilder(bid.builderIndex);
+  if (!isActiveBuilder(builder, state.finalizedCheckpoint.epoch)) {
+    throw Error(`Bid builderIndex=${bid.builderIndex} is not an active builder`);
+  }
+
+  if (builder.version !== PAYLOAD_BUILDER_VERSION) {
+    throw Error(`Invalid builder version=${builder.version} expected=${PAYLOAD_BUILDER_VERSION}`);
+  }
+
+  // A bid not signed by one of the builder pubkeys the entry accepts bids from must not be accepted
+  if (
+    entry.builderPubkeys.length > 0 &&
+    !entry.builderPubkeys.some((pubkey) => byteArrayEquals(pubkey, builder.pubkey))
+  ) {
+    throw Error(
+      `Bid builder pubkey=${toHex(builder.pubkey)} is not in the entry's ` +
+        `builderPubkeys=${entry.builderPubkeys.map(toHex).join(",")}`
+    );
+  }
+
+  const blobKzgCommitmentsLen = bid.blobKzgCommitments.length;
+  const maxBlobsPerBlock = chain.config.getMaxBlobsPerBlock(computeEpochAtSlot(bid.slot));
+  if (blobKzgCommitmentsLen > maxBlobsPerBlock) {
+    throw Error(`Bid has too many KZG commitments len=${blobKzgCommitmentsLen} limit=${maxBlobsPerBlock}`);
+  }
+
+  // The coverage check only applies to the staked collateral payment, a pure execution
+  // layer payment bid has nothing to cover on-chain
+  if (bid.value > 0 && !state.canBuilderCoverBid(bid.builderIndex, bid.value)) {
+    throw Error(
+      `Builder cannot cover bid value=${prettyGweiToEth(bid.value)} ` + `balance=${prettyGweiToEth(builder.balance)}`
+    );
+  }
+
+  const randaoMix = state.getRandaoMix(computeEpochAtSlot(state.slot));
+  if (!byteArrayEquals(bid.prevRandao, randaoMix)) {
+    throw Error(`Invalid bid prevRandao=${toHex(bid.prevRandao)} expected=${toHex(randaoMix)}`);
+  }
+
+  // The parent's payload does not try to exit the builder
+  if (byteArrayEquals(bid.parentBlockHash, state.latestExecutionPayloadBid.blockHash)) {
+    const requests = await getParentExecutionRequests();
+    if (
+      requests.builderExits.some(
+        (request) =>
+          byteArrayEquals(request.pubkey, builder.pubkey) &&
+          byteArrayEquals(request.sourceAddress, builder.executionAddress)
+      )
+    ) {
+      throw Error(`Bid builderIndex=${bid.builderIndex} may exit in the parent payload`);
+    }
+  }
+
+  // The builder must honor the proposer preferences it learned over gossip
+  const bidEpoch = computeEpochAtSlot(bid.slot);
+  const dependentRootHex = (() => {
+    try {
+      return getShufflingDependentRoot(chain.forkChoice, bidEpoch, computeEpochAtSlot(parentBlock.slot), parentBlock);
+    } catch {
+      return null;
+    }
+  })();
+  if (dependentRootHex === null) {
+    throw Error(`Unable to resolve proposer preferences dependent root for bid slot=${bid.slot}`);
+  }
+  const proposerPreferences = chain.proposerPreferencesPool.get(bid.slot, dependentRootHex);
+  if (proposerPreferences === null) {
+    throw Error(`No proposer preferences found for bid slot=${bid.slot} dependentRoot=${dependentRootHex}`);
+  }
+  if (!byteArrayEquals(bid.feeRecipient, proposerPreferences.message.feeRecipient)) {
+    throw Error(
+      `Bid feeRecipient=${toHex(bid.feeRecipient)} does not match ` +
+        `proposer preferences feeRecipient=${toHex(proposerPreferences.message.feeRecipient)}`
+    );
+  }
+
+  const parentPayloadVariant = chain.forkChoice.getBlockHexAndBlockHash(bidParentBlockRoot, bidParentBlockHash);
+  if (parentPayloadVariant === null || parentPayloadVariant.executionPayloadBlockHash === null) {
+    throw Error(`Unable to resolve parent payload gas limit for bid parentBlockHash=${bidParentBlockHash}`);
+  }
+  const parentGasLimit = BigInt(parentPayloadVariant.executionPayloadGasLimit);
+  const targetGasLimit = proposerPreferences.message.targetGasLimit;
+  if (!isGasLimitTargetCompatible(parentGasLimit, bid.gasLimit, targetGasLimit)) {
+    throw Error(
+      `Bid gasLimit=${bid.gasLimit} is not compatible with ` +
+        `parentGasLimit=${parentGasLimit} targetGasLimit=${targetGasLimit}`
+    );
+  }
+
+  const signatureSet = createSingleSignatureSetFromComponents(
+    builder.pubkey,
+    getExecutionPayloadBidSigningRoot(chain.config, bid),
+    signedExecutionPayloadBid.signature
+  );
+
+  if (!(await chain.bls.verifySignatureSets([signatureSet]))) {
+    throw Error(`Invalid bid signature builderIndex=${bid.builderIndex}`);
+  }
+}

package/src/metrics/metrics/lodestar.ts

@@ -83,15 +83,15 @@ export function createLodestarMetrics(
       }),
       jobTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_job_time_seconds",
-        help: "Time to process gossip validation queue job in seconds",
+        help: "Time to process gossip validation queue job in seconds (accepted messages only)",
         labelNames: ["topic"],
-        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10],
+        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2],
       }),
       jobWaitTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_job_wait_time_seconds",
         help: "Time from job added to the queue to starting the job in seconds",
         labelNames: ["topic"],
-        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10],
+        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2],
       }),
       concurrency: register.gauge<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_concurrency",
@@ -104,9 +104,10 @@ export function createLodestarMetrics(
         help: "Age of the first item of each key in the indexed queues in seconds",
         buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5],
       }),
-      queueTime: register.histogram({
+      queueTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_time_seconds",
         help: "Total time an item stays in queue until it is processed in seconds",
+        labelNames: ["topic"],
         buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5],
       }),
     },
@@ -920,6 +921,12 @@ export function createLodestarMetrics(
         help: "Count of errors, by error type, while processing blocks",
         labelNames: ["error"],
       }),
+
+      preStateSource: register.counter<{source: "parentState" | "fallbackPreState" | "preState"}>({
+        name: "lodestar_gossip_block_validation_pre_state_source_total",
+        help: "Source of the pre-state used for gossip block validation",
+        labelNames: ["source"],
+      }),
     },
     gossipBlob: {
       recvToValidation: register.histogram({
@@ -1350,6 +1357,11 @@ export function createLodestarMetrics(
           help: "Total number of InsertOutcome as a result of adding an execution payload bid from api to the pool",
           labelNames: ["insertOutcome"],
         }),
+        apiValidationTime: register.histogram({
+          name: "lodestar_api_execution_payload_bid_validation_time_seconds",
+          help: "Time elapsed for signed execution payload bid validation - api path",
+          buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.5],
+        }),
       },
     },
 
@@ -1444,6 +1456,16 @@ export function createLodestarMetrics(
         help: "Checkpoint state cache size",
         labelNames: ["type"],
       }),
+      persistentTierEpochs: register.gauge<{tier: number}>({
+        name: "lodestar_cp_state_cache_persistent_tier_epochs",
+        help: "Number of epoch keys retained in each on-disk checkpoint state retention tier",
+        labelNames: ["tier"],
+      }),
+      persistentTierStates: register.gauge<{tier: number}>({
+        name: "lodestar_cp_state_cache_persistent_tier_states",
+        help: "Number of persisted checkpoint states retained in each on-disk retention tier",
+        labelNames: ["tier"],
+      }),
       reads: register.avgMinMax({
         name: "lodestar_cp_state_epoch_reads",
         help: "Avg min max of all state cache items total read count",
@@ -1776,6 +1798,10 @@ export function createLodestarMetrics(
         name: "lodestar_precompute_next_epoch_transition_waste_total",
         help: "Total number of precomputing next epoch transition wasted",
       }),
+      predictedReorg: register.counter({
+        name: "lodestar_precompute_next_epoch_transition_predicted_reorg_total",
+        help: "Predicted epoch-boundary reorgs where the strong parent was dialed instead of the weak head",
+      }),
       duration: register.histogram({
         name: "lodestar_precompute_next_epoch_transition_duration_seconds",
         help: "Duration of precomputeNextEpochTransition, including epoch transition and hashTreeRoot",
@@ -2015,6 +2041,40 @@ export function createLodestarMetrics(
       }),
     },
 
+    builderApi: {
+      statusChecks: register.counter<{status: "success" | "error"}>({
+        name: "lodestar_builder_api_status_checks_total",
+        help: "Total count of status checks sent to external builders ahead of a proposal",
+        labelNames: ["status"],
+      }),
+      bidRequests: register.counter({
+        name: "lodestar_builder_api_bid_requests_total",
+        help: "Total count of execution payload bid requests sent to external builders",
+      }),
+      bidRequestErrors: register.counter({
+        name: "lodestar_builder_api_bid_request_errors_total",
+        help: "Total count of failed execution payload bid requests to external builders",
+      }),
+      bidsReceived: register.counter({
+        name: "lodestar_builder_api_bids_received_total",
+        help: "Total count of execution payload bids received from external builders",
+      }),
+      bidsDiscarded: register.counter({
+        name: "lodestar_builder_api_bids_discarded_total",
+        help: "Total count of execution payload bids from external builders discarded due to failed validation",
+      }),
+      blockSubmissions: register.counter<{status: "success" | "error"}>({
+        name: "lodestar_builder_api_block_submissions_total",
+        help: "Total count of signed beacon blocks submitted to external builders",
+        labelNames: ["status"],
+      }),
+      preferencesForwarded: register.counter<{status: "success" | "error"}>({
+        name: "lodestar_builder_api_preferences_forwarded_total",
+        help: "Total count of builder preferences forwarded to external builders",
+        labelNames: ["status"],
+      }),
+    },
+
     db: {
       dbReadReq: register.gauge<{bucket: string}>({
         name: "lodestar_db_read_req_total",

package/src/network/network.ts

@@ -516,7 +516,8 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.execution_payload_bid>(
       {type: GossipType.execution_payload_bid, boundary},
-      signedBid
+      signedBid,
+      {floodPublish: true}
     );
   }
 
@@ -542,17 +543,19 @@ export class Network implements INetwork {
 
   private async publishGossip<K extends GossipType>(
     topic: GossipTopicMap[K],
-    object: GossipTypeMap[K]
+    object: GossipTypeMap[K],
+    opts?: PublishOpts
   ): Promise<number> {
     const topicStr = stringifyGossipTopic(this.config, topic);
     const sszType = getGossipSSZType(topic);
     const messageData = (sszType.serialize as (object: GossipTypeMap[GossipType]) => Uint8Array)(object);
-    const opts: PublishOpts = {
+    const publishOpts: PublishOpts = {
       ignoreDuplicatePublishError: gossipTopicIgnoreDuplicatePublishError[topic.type],
       // Leave undefined unless the topic opts out, so `--network.allowPublishToZeroPeers` still applies
       allowPublishToZeroTopicPeers: gossipTopicAllowPublishToZeroPeers[topic.type] ? true : undefined,
+      ...opts,
     };
-    const sentPeers = await this.core.publishGossip(topicStr, messageData, opts);
+    const sentPeers = await this.core.publishGossip(topicStr, messageData, publishOpts);
 
     this.logger.verbose("Publish to topic", {topic: topicStr, sentPeers, currentSlot: this.clock.currentSlot});
     return sentPeers;

package/src/network/processor/constants.ts

@@ -0,0 +1,6 @@
+/**
+ * Max distinct peers tracked per unknown block/payload root.
+ * Normally we have 8 mesh peers per topic, and multi topics may point to the same root.
+ * Having 32 peers per root seems enough to search for missing blocks/payloads.
+ **/
+export const MAX_PEERS_PER_ROOT = 32;

package/src/network/processor/gossipHandlers.ts

@@ -1306,7 +1306,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
       // Handle valid payload bid by storing in a bid pool
       try {
-        const insertOutcome = chain.executionPayloadBidPool.add(executionPayloadBid);
+        const insertOutcome = chain.executionPayloadBidPool.add(executionPayloadBid, Math.floor(elapsedSec * 1000));
         metrics?.opPool.executionPayloadBidPool.gossipInsertOutcome.inc({insertOutcome});
       } catch (e) {
         logger.error("Error adding to executionPayloadBid pool", {}, e as Error);

package/src/network/processor/gossipQueues/indexed.ts

@@ -86,7 +86,6 @@ export class IndexedGossipQueueMinSize<T extends {indexed?: string; queueAddedMs
     const now = Date.now();
     // here we mutate item, which is used for gossip validation later
     item.indexed = key;
-    item.queueAddedMs = now;
     let queueItem = this.indexedItems.get(key);
     if (queueItem == null) {
       queueItem = {firstSeenMs: now, listItems: new LinkedList<T>()};

package/src/network/processor/index.ts

@@ -1,3 +1,4 @@
+import {TopicValidatorResult} from "@libp2p/gossipsub";
 import {routes} from "@lodestar/api";
 import {ForkSeq} from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
@@ -30,6 +31,7 @@ import {
   GossipValidatorBatchFn,
   GossipValidatorFn,
 } from "../gossip/interface.js";
+import {MAX_PEERS_PER_ROOT} from "./constants.js";
 import {createExtractBlockSlotRootFns} from "./extractSlotRootFns.js";
 import {GossipHandlerOpts, ValidatorFnsModules, getGossipHandlers} from "./gossipHandlers.js";
 import {createGossipQueues} from "./gossipQueues/index.js";
@@ -191,8 +193,12 @@ export class NetworkProcessor {
   // in that case PendingGossipsubMessage needs to be stored in this Map and reprocessed once the payload comes
   private readonly awaitingMessagesByPayloadBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
   private awaitingPayloadMessageCount = 0;
-  private unknownBlocksBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
-  private unknownEnvelopesBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
+  private unknownBlocksBySlot = new MapDef<Slot, MapDef<RootHex, Set<PeerIdStr>>>(
+    () => new MapDef<RootHex, Set<PeerIdStr>>(() => new Set())
+  );
+  private unknownEnvelopesBySlot = new MapDef<Slot, MapDef<RootHex, Set<PeerIdStr>>>(
+    () => new MapDef<RootHex, Set<PeerIdStr>>(() => new Set())
+  );
 
   constructor(
     modules: NetworkProcessorModules,
@@ -271,35 +277,53 @@ export class NetworkProcessor {
    * Search block via `ChainEvent.unknownBlockRoot` event
    * Slot is the message slot, which is not necessarily the same as the block's slot, but it can be used for a good prune strategy.
    * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+   * We forward every distinct peer that gossiped this root (not just the first) so BlockInputSync can
+   * prefer them as by-root download candidates.
    */
   searchUnknownBlock({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
-    if (
-      this.chain.seenBlock(root) ||
-      this.awaitingMessagesByBlockRoot.has(root) ||
-      this.unknownBlocksBySlot.getOrDefault(slot).has(root)
-    ) {
+    if (this.chain.seenBlock(root)) {
       return;
     }
-    // Search for the unknown block
-    this.unknownBlocksBySlot.getOrDefault(slot).add(root);
-    this.chain.emitter.emit(ChainEvent.unknownBlockRoot, {rootHex: root, peer, source});
+    const peersForRoot = this.unknownBlocksBySlot.getOrDefault(slot);
+    const alreadySearching = peersForRoot.has(root) || this.awaitingMessagesByBlockRoot.has(root);
+    const forwardedPeers = peersForRoot.getOrDefault(root);
+
+    // brand-new search always emits (peer may be undefined)
+    let shouldEmit = !alreadySearching;
+    if (peer !== undefined && !forwardedPeers.has(peer) && forwardedPeers.size < MAX_PEERS_PER_ROOT) {
+      // First time we forward THIS peer for THIS root: emit so BlockInputSync accumulates it.
+      forwardedPeers.add(peer);
+      shouldEmit = true;
+    }
+    if (shouldEmit) {
+      this.chain.emitter.emit(ChainEvent.unknownBlockRoot, {rootHex: root, peer, source});
+    }
   }
 
   /**
    * Search envelope via `ChainEvent.unknownEnvelopeBlockRoot` event
    * Slot is the message slot, which is not necessarily the same as the envelope's slot, but it can be used for a good prune strategy.
    * In the rare case, if 2 messages on 2 slots search for the same root (for example beacon_attestation) we may emit the same root twice but BlockInputSync should handle it well.
+   * Same peer-forwarding + dedup behavior as searchUnknownBlock.
    */
   searchUnknownEnvelope({slot, root}: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
-    if (
-      this.chain.seenPayloadEnvelope(root) ||
-      this.awaitingMessagesByPayloadBlockRoot.has(root) ||
-      this.unknownEnvelopesBySlot.getOrDefault(slot).has(root)
-    ) {
+    if (this.chain.seenPayloadEnvelope(root)) {
       return;
     }
-    this.unknownEnvelopesBySlot.getOrDefault(slot).add(root);
-    this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, slot, peer, source});
+    const peersForRoot = this.unknownEnvelopesBySlot.getOrDefault(slot);
+    // capture "already searching" BEFORE getOrDefault(root) creates the entry
+    const alreadySearching = peersForRoot.has(root) || this.awaitingMessagesByPayloadBlockRoot.has(root);
+    const forwardedPeers = peersForRoot.getOrDefault(root);
+
+    let shouldEmit = !alreadySearching; // brand-new search always emits (peer may be undefined)
+    if (peer !== undefined && !forwardedPeers.has(peer) && forwardedPeers.size < MAX_PEERS_PER_ROOT) {
+      // First time we forward THIS peer for THIS root: emit so BlockInputSync accumulates it.
+      forwardedPeers.add(peer);
+      shouldEmit = true;
+    }
+    if (shouldEmit) {
+      this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, slot, peer, source});
+    }
   }
 
   private onPendingGossipsubMessage = (message: PendingGossipsubMessage): void => {
@@ -537,6 +561,7 @@ export class NetworkProcessor {
 
   private pushPendingGossipsubMessageToQueue(message: PendingGossipsubMessage): void {
     const topicType = message.topic.type;
+    message.queueAddedMs = Date.now();
     const droppedCount = this.gossipQueues[topicType].add(message);
     if (droppedCount) {
       // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
@@ -616,7 +641,7 @@ export class NetworkProcessor {
 
     for (const [slot, roots] of this.unknownBlocksBySlot) {
       if (slot > minSlot) continue;
-      for (const rootHex of roots) {
+      for (const rootHex of roots.keys()) {
         const gossipMessages = this.awaitingMessagesByBlockRoot.get(rootHex);
         if (gossipMessages !== undefined) {
           for (const message of gossipMessages) {
@@ -641,7 +666,7 @@ export class NetworkProcessor {
 
     for (const [slot, roots] of this.unknownEnvelopesBySlot) {
       if (slot > minSlot) continue;
-      for (const rootHex of roots) {
+      for (const rootHex of roots.keys()) {
         const gossipMessages = this.awaitingMessagesByPayloadBlockRoot.get(rootHex);
         if (gossipMessages !== undefined) {
           for (const message of gossipMessages) {
@@ -723,12 +748,20 @@ export class NetworkProcessor {
       for (const msg of messageOrArray) {
         msg.startProcessUnixSec = nowSec;
         if (msg.queueAddedMs !== undefined) {
-          this.metrics?.gossipValidationQueue.queueTime.observe(nowSec - msg.queueAddedMs / 1000);
+          this.metrics?.gossipValidationQueue.queueTime.observe(
+            {topic: msg.topic.type},
+            nowSec - msg.queueAddedMs / 1000
+          );
         }
       }
     } else {
-      // indexed queue is not used here
       messageOrArray.startProcessUnixSec = nowSec;
+      if (messageOrArray.queueAddedMs !== undefined) {
+        this.metrics?.gossipValidationQueue.queueTime.observe(
+          {topic: messageOrArray.topic.type},
+          nowSec - messageOrArray.queueAddedMs / 1000
+        );
+      }
     }
 
     const acceptanceArr = Array.isArray(messageOrArray)
@@ -741,11 +774,11 @@ export class NetworkProcessor {
         ];
 
     if (Array.isArray(messageOrArray)) {
-      for (const msg of messageOrArray) {
-        this.trackJobTime(msg, messageOrArray.length);
+      for (const [i, msg] of messageOrArray.entries()) {
+        this.trackJobTime(msg, messageOrArray.length, acceptanceArr[i]);
       }
     } else {
-      this.trackJobTime(messageOrArray, 1);
+      this.trackJobTime(messageOrArray, 1, acceptanceArr[0]);
     }
 
     // Use setTimeout to yield to the macro queue
@@ -773,17 +806,21 @@ export class NetworkProcessor {
     }
   }
 
-  private trackJobTime(message: PendingGossipsubMessage, numJob: number): void {
+  private trackJobTime(message: PendingGossipsubMessage, numJob: number, acceptance: TopicValidatorResult): void {
     if (message.startProcessUnixSec !== null) {
+      // record job wait time for all messages
       this.metrics?.gossipValidationQueue.jobWaitTime.observe(
         {topic: message.topic.type},
         message.startProcessUnixSec - message.seenTimestampSec
       );
-      // if it takes 64ms to process 64 jobs, the average job time is 1ms
-      this.metrics?.gossipValidationQueue.jobTime.observe(
-        {topic: message.topic.type},
-        (Date.now() / 1000 - message.startProcessUnixSec) / numJob
-      );
+      // but only record job time for ACCEPT messages
+      if (acceptance === TopicValidatorResult.Accept) {
+        // if it takes 64ms to process 64 jobs, the average job time is 1ms
+        this.metrics?.gossipValidationQueue.jobTime.observe(
+          {topic: message.topic.type},
+          (Date.now() / 1000 - message.startProcessUnixSec) / numJob
+        );
+      }
     }
   }
 

package/src/node/nodejs.ts

@@ -260,6 +260,11 @@ export class BeaconNode {
       executionBuilder: opts.executionBuilder.enabled
         ? initializeExecutionBuilder(opts.executionBuilder, config, metrics, logger)
         : undefined,
+      builderApiClientOpts: {
+        timeout: opts.executionBuilder.timeout,
+        // Sent with all builder api requests, unless the node runs in private mode
+        userAgent: opts.executionBuilder.userAgent,
+      },
     });
 
     // Load persisted data from disk to in-memory caches

package/src/sync/range/chain.ts

@@ -652,7 +652,9 @@ export class SyncChain {
 
         const blockSlots = downloadSuccessOutput.blocks.map((b) => b.slot);
         const envelopeSlots = downloadSuccessOutput.payloadEnvelopes
-          ? Array.from(downloadSuccessOutput.payloadEnvelopes.keys())
+          ? Array.from(downloadSuccessOutput.payloadEnvelopes.entries())
+              .filter(([, payloadInput]) => payloadInput.hasPayloadEnvelope())
+              .map(([slot]) => slot)
           : null;
 
         this.logger.debug(logMessage, {
@@ -684,13 +686,18 @@ export class SyncChain {
    */
   private async processBatch(batch: Batch): Promise<void> {
     const {blocks, payloadEnvelopes, peers} = batch.startProcessing();
+    const envelopeSlots = payloadEnvelopes
+      ? Array.from(payloadEnvelopes.entries())
+          .filter(([, payloadInput]) => payloadInput.hasPayloadEnvelope())
+          .map(([slot]) => slot)
+      : null;
 
     const logCtx = {
       id: this.logId,
       ...batch.getMetadata(),
       blockCount: blocks.length,
       blockSlots: prettyPrintIndices(blocks.map((b) => b.slot)),
-      ...(payloadEnvelopes ? {envelopeSlots: prettyPrintIndices(Array.from(payloadEnvelopes.keys()))} : {}),
+      ...(envelopeSlots ? {envelopeSlots: prettyPrintIndices(envelopeSlots)} : {}),
       peers: peers.map(prettyPrintPeerIdStr).join(","),
     };
     this.logger.verbose("Processing batch", logCtx);

package/src/sync/range/range.ts

@@ -212,7 +212,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
       for (const block of blocks) {
         await this.chain.processBlock(block, flags);
         const payloadEnvelope = payloadEnvelopes?.get(block.slot);
-        if (payloadEnvelope) {
+        if (payloadEnvelope?.hasPayloadEnvelope()) {
           await this.chain.processExecutionPayload(payloadEnvelope);
         }
       }