@lodestar/beacon-node 1.47.0-rc.0 → 1.48.0-dev.7d85330f92

package/src/chain/validation/executionPayloadBid.ts

@@ -1,4 +1,4 @@
-import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
+import {ProtoBlock} from "@lodestar/fork-choice";
 import {PAYLOAD_BUILDER_VERSION} from "@lodestar/params";
 import {
   computeEpochAtSlot,
@@ -46,11 +46,12 @@ function getMinBidValue(currentHighestBid: number): number {
 /**
  * Check whether a bid builds on one of the paths compatible with the local head branch.
  *
- * Building directly on the parent is allowed for proposer-boost reorgs outside epoch boundaries.
+ * Building directly on the parent is allowed for proposer-boost reorgs, including at epoch
+ * boundaries when the head is weak enough that a reorg onto the strong parent is predicted.
  * Otherwise the bid must build on the local head's full or empty payload variant, as selected for its slot.
  */
 function isBidCompatibleWithHead(
-  forkChoice: IForkChoice,
+  chain: IBeaconChain,
   head: ProtoBlock,
   bidSlot: Slot,
   bidParentBlockRoot: RootHex,
@@ -60,9 +61,13 @@ function isBidCompatibleWithHead(
   const buildsOnParentPayload = bidParentBlockHash === head.parentBlockHash;
 
   if (buildsOnParentBlock && buildsOnParentPayload) {
-    // The spec allows this at epoch boundaries, but Lodestar does not propagate these bids because validating
-    // them requires an epoch transition for a parent state that cannot be used for proposer-boost reorgs.
-    return !isStartSlotOfEpoch(bidSlot);
+    if (!isStartSlotOfEpoch(bidSlot)) {
+      return true;
+    }
+    // Epoch-boundary reorgs are allowed, but only propagate the bid if its parent matches the block
+    // we predict a proposer would reorg onto — predictProposerHead returns the strong parent only
+    // when the head is weak.
+    return chain.predictProposerHead().blockRoot === bidParentBlockRoot;
   }
 
   if (bidParentBlockRoot !== head.blockRoot) {
@@ -71,18 +76,170 @@ function isBidCompatibleWithHead(
 
   const buildsOnHeadPayload = bidParentBlockHash === head.executionPayloadBlockHash;
 
-  if (forkChoice.shouldBuildOnFull(head, bidSlot)) {
+  if (chain.forkChoice.shouldBuildOnFull(head, bidSlot)) {
     return buildsOnHeadPayload;
   }
 
   return buildsOnParentPayload;
 }
 
+/**
+ * Validation for bids submitted via the API by the operator's own builder.
+ *
+ * Only non-transient checks are applied: slot later than parent, zero execution payment, blob
+ * commitment count, builder eligibility, version and balance coverage, prev_randao, and signature.
+ * Transient gossip rules (head compatibility, first bid per tuple, value increment, proposer
+ * preferences) are not applied, since those only limit forwarding of peers' messages and the
+ * builder may legitimately bid on a branch this node does not consider head.
+ * The parent-payload builder-exit rule is omitted because this endpoint only accepts bids from
+ * the operator's own builder.
+ *
+ * Throws on any failed check. Also throws if the bid's parent block is unknown or its state is
+ * unavailable, since the checks cannot be evaluated against the parent branch.
+ */
 export async function validateApiExecutionPayloadBid(
   chain: IBeaconChain,
   signedExecutionPayloadBid: gloas.SignedExecutionPayloadBid
-): Promise<{proposerIndex: ValidatorIndex}> {
-  return validateExecutionPayloadBid(chain, signedExecutionPayloadBid);
+): Promise<void> {
+  const bid = signedExecutionPayloadBid.message;
+  const bidParentBlockRoot = toRootHex(bid.parentBlockRoot);
+
+  // [IGNORE] `bid.slot` is the current slot, or the next slot (`bid.slot - 1` is current), allowing for `MAXIMUM_GOSSIP_CLOCK_DISPARITY`.
+  if (
+    !chain.clock.isCurrentSlotGivenGossipDisparity(bid.slot) &&
+    !chain.clock.isCurrentSlotGivenGossipDisparity(bid.slot - 1)
+  ) {
+    throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
+      code: ExecutionPayloadBidErrorCode.INVALID_SLOT,
+      builderIndex: bid.builderIndex,
+      slot: bid.slot,
+    });
+  }
+
+  // [IGNORE] `bid.parent_block_root` is the hash tree root of a known beacon block in fork choice.
+  // Moved earlier than the spec ordering so we can derive the proposer dependent root for the
+  // proposer-preferences lookup below from a known fork-choice block.
+  const parentBlock = chain.forkChoice.getBlockHexDefaultStatus(bidParentBlockRoot);
+  if (parentBlock === null) {
+    throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
+      code: ExecutionPayloadBidErrorCode.UNKNOWN_BLOCK_ROOT,
+      parentBlockRoot: bidParentBlockRoot,
+    });
+  }
+
+  // [REJECT] The bid is for a higher slot than its parent block -- i.e.
+  // validate that `bid.slot` is greater than the slot of the block with root
+  // `bid.parent_block_root`.
+  if (bid.slot <= parentBlock.slot) {
+    throw new ExecutionPayloadBidError(GossipAction.REJECT, {
+      code: ExecutionPayloadBidErrorCode.NOT_LATER_THAN_PARENT,
+      parentSlot: parentBlock.slot,
+      slot: bid.slot,
+    });
+  }
+
+  // [REJECT] `bid.execution_payment` is zero.
+  if (bid.executionPayment !== 0n) {
+    throw new ExecutionPayloadBidError(GossipAction.REJECT, {
+      code: ExecutionPayloadBidErrorCode.NON_ZERO_EXECUTION_PAYMENT,
+      builderIndex: bid.builderIndex,
+      executionPayment: bid.executionPayment,
+    });
+  }
+
+  // [REJECT] The length of KZG commitments is less than or equal to the limitation defined in the
+  // consensus layer -- i.e. validate that
+  // `len(bid.blob_kzg_commitments) <= get_blob_parameters(compute_epoch_at_slot(bid.slot)).max_blobs_per_block`.
+  const blobKzgCommitmentsLen = bid.blobKzgCommitments.length;
+  const maxBlobsPerBlock = chain.config.getMaxBlobsPerBlock(computeEpochAtSlot(bid.slot));
+  if (blobKzgCommitmentsLen > maxBlobsPerBlock) {
+    throw new ExecutionPayloadBidError(GossipAction.REJECT, {
+      code: ExecutionPayloadBidErrorCode.TOO_MANY_KZG_COMMITMENTS,
+      blobKzgCommitmentsLen,
+      commitmentLimit: maxBlobsPerBlock,
+    });
+  }
+
+  const state = await chain.regen
+    .getBlockSlotState(parentBlock, bid.slot, {dontTransferCache: true}, RegenCaller.validateApiExecutionPayloadBid)
+    .catch(() => {
+      throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
+        code: ExecutionPayloadBidErrorCode.UNKNOWN_BLOCK_ROOT,
+        parentBlockRoot: bidParentBlockRoot,
+      });
+    });
+
+  if (!isStatePostGloas(state)) {
+    throw new Error(`Expected gloas+ state for execution payload bid validation, got fork=${state.forkName}`);
+  }
+
+  // [REJECT] `bid.builder_index` is within bounds -- i.e. `bid.builder_index < len(state.builders)`.
+  // `state.getBuilder` returns a lazy SSZ `getReadonly` view that is not bounds-checked eagerly; an
+  // out-of-range index only throws (`LeafNode has no right node`) on deferred field access (e.g. inside
+  // `isActiveBuilder`), escaping a try/catch around `getBuilder`. Check the length explicitly instead.
+  if (bid.builderIndex >= state.getBuildersLength()) {
+    throw new ExecutionPayloadBidError(GossipAction.REJECT, {
+      code: ExecutionPayloadBidErrorCode.BUILDER_NOT_ELIGIBLE,
+      builderIndex: bid.builderIndex,
+    });
+  }
+
+  // [REJECT] `bid.builder_index` is a valid/active builder index -- i.e.
+  // `is_active_builder(state, bid.builder_index)` returns `True`.
+  const builder = state.getBuilder(bid.builderIndex);
+  if (!isActiveBuilder(builder, state.finalizedCheckpoint.epoch)) {
+    throw new ExecutionPayloadBidError(GossipAction.REJECT, {
+      code: ExecutionPayloadBidErrorCode.BUILDER_NOT_ELIGIBLE,
+      builderIndex: bid.builderIndex,
+    });
+  }
+
+  // [REJECT] The builder version is `PAYLOAD_BUILDER_VERSION` -- i.e.
+  // `state.builders[bid.builder_index].version == PAYLOAD_BUILDER_VERSION`.
+  if (builder.version !== PAYLOAD_BUILDER_VERSION) {
+    throw new ExecutionPayloadBidError(GossipAction.REJECT, {
+      code: ExecutionPayloadBidErrorCode.INVALID_BUILDER_VERSION,
+      builderIndex: bid.builderIndex,
+      version: builder.version,
+      expectedVersion: PAYLOAD_BUILDER_VERSION,
+    });
+  }
+
+  // [IGNORE] `bid.value` is less or equal than the builder's excess balance --
+  // i.e. `can_builder_cover_bid(state, builder_index, amount)` returns `True`.
+  if (!state.canBuilderCoverBid(bid.builderIndex, bid.value)) {
+    throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
+      code: ExecutionPayloadBidErrorCode.BID_TOO_HIGH,
+      bidValue: bid.value,
+      builderBalance: builder.balance,
+    });
+  }
+
+  // [REJECT] `bid.prev_randao` is the correct RANDAO mix -- i.e. validate that
+  // `bid.prev_randao == get_randao_mix(parent_state, get_current_epoch(parent_state))`.
+  const randaoMix = state.getRandaoMix(computeEpochAtSlot(state.slot));
+  if (!byteArrayEquals(bid.prevRandao, randaoMix)) {
+    throw new ExecutionPayloadBidError(GossipAction.REJECT, {
+      code: ExecutionPayloadBidErrorCode.INVALID_PREV_RANDAO,
+      builderIndex: bid.builderIndex,
+      bidPrevRandao: toHex(bid.prevRandao),
+      expectedPrevRandao: toHex(randaoMix),
+    });
+  }
+
+  // [REJECT] `signed_execution_payload_bid.signature` is valid with respect to the `bid.builder_index`.
+  const signatureSet = createSingleSignatureSetFromComponents(
+    builder.pubkey,
+    getExecutionPayloadBidSigningRoot(chain.config, bid),
+    signedExecutionPayloadBid.signature
+  );
+  if (!(await chain.bls.verifySignatureSets([signatureSet]))) {
+    throw new ExecutionPayloadBidError(GossipAction.REJECT, {
+      code: ExecutionPayloadBidErrorCode.INVALID_SIGNATURE,
+      builderIndex: bid.builderIndex,
+      slot: bid.slot,
+    });
+  }
 }
 
 export async function validateGossipExecutionPayloadBid(
@@ -114,7 +271,7 @@ async function validateExecutionPayloadBid(
 
   // [IGNORE] The bid is compatible with the current head branch.
   const head = chain.forkChoice.getHead();
-  if (!isBidCompatibleWithHead(chain.forkChoice, head, bid.slot, bidParentBlockRoot, bidParentBlockHash)) {
+  if (!isBidCompatibleWithHead(chain, head, bid.slot, bidParentBlockRoot, bidParentBlockHash)) {
     throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
       code: ExecutionPayloadBidErrorCode.INCOMPATIBLE_WITH_HEAD,
       slot: bid.slot,
@@ -342,6 +499,29 @@ async function validateExecutionPayloadBid(
     });
   }
 
+  // [IGNORE] The parent's payload does not try to exit the builder
+  if (byteArrayEquals(bid.parentBlockHash, state.latestExecutionPayloadBid.blockHash)) {
+    const requests = await chain.getParentExecutionRequests(parentBlock.slot, parentBlock.blockRoot).catch(() => {
+      throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
+        code: ExecutionPayloadBidErrorCode.UNKNOWN_PARENT_BLOCK_HASH,
+        parentBlockHash: bidParentBlockHash,
+      });
+    });
+    if (
+      requests.builderExits.some(
+        (request) =>
+          byteArrayEquals(request.pubkey, builder.pubkey) &&
+          byteArrayEquals(request.sourceAddress, builder.executionAddress)
+      )
+    ) {
+      throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
+        code: ExecutionPayloadBidErrorCode.BUILDER_MAY_EXIT,
+        builderIndex: bid.builderIndex,
+        parentBlockRoot: bidParentBlockRoot,
+      });
+    }
+  }
+
   // [REJECT] `signed_execution_payload_bid.signature` is valid with respect to the `bid.builder_index`.
   const signatureSet = createSingleSignatureSetFromComponents(
     builder.pubkey,

package/src/execution/builder/apiClient.ts

@@ -201,6 +201,26 @@ export class BuilderApiClient {
     }
   }
 
+  /**
+   * Establish the connection to every known builder outside the bid deadline, a cold TCP/TLS handshake
+   * to a distant builder can exhaust `BUILDER_BID_DEADLINE_MS` on its own, and a long idle connection
+   * may be half-open and only fail once written to.
+   */
+  async checkStatus(): Promise<void> {
+    await Promise.all(
+      Array.from(this.clients.entries()).map(async ([url, client]) => {
+        try {
+          (await client.status()).assertOk();
+          this.metrics?.builderApi.statusChecks.inc({status: "success"});
+          this.logger?.debug("Builder is ready", {builder: toPrintableUrl(url)});
+        } catch (e) {
+          this.metrics?.builderApi.statusChecks.inc({status: "error"});
+          this.logger?.warn("Builder status check failed", {builder: toPrintableUrl(url)}, e as Error);
+        }
+      })
+    );
+  }
+
   private async getOrCreateClient(
     url: BuilderUrl,
     proposerPubkey: BLSPubkey,
@@ -237,7 +257,7 @@ export class BuilderApiClient {
         {config: this.config, metrics: this.metrics?.builderHttpClient, logger: this.logger}
       );
       this.clients.set(url, client);
-      this.logger?.info("External builder registered", {url: toPrintableUrl(url)});
+      this.logger?.debug("External builder registered", {url: toPrintableUrl(url)});
     }
     return client;
   }

package/src/execution/builder/validateBid.ts

@@ -37,10 +37,11 @@ export async function validateBuilderApiExecutionPayloadBid(
     parentBlockHash: RootHex;
     parentBlockRoot: RootHex;
     entry: routes.validator.BuilderEntry;
+    getParentExecutionRequests: () => Promise<gloas.ExecutionRequests>;
   }
 ): Promise<void> {
   const bid = signedExecutionPayloadBid.message;
-  const {slot, parentBlock, parentBlockHash, parentBlockRoot, entry} = request;
+  const {slot, parentBlock, parentBlockHash, parentBlockRoot, entry, getParentExecutionRequests} = request;
 
   if (bid.slot !== slot) {
     throw Error(`Bid slot=${bid.slot} does not match requested slot=${slot}`);
@@ -118,6 +119,20 @@ export async function validateBuilderApiExecutionPayloadBid(
     throw Error(`Invalid bid prevRandao=${toHex(bid.prevRandao)} expected=${toHex(randaoMix)}`);
   }
 
+  // The parent's payload does not try to exit the builder
+  if (byteArrayEquals(bid.parentBlockHash, state.latestExecutionPayloadBid.blockHash)) {
+    const requests = await getParentExecutionRequests();
+    if (
+      requests.builderExits.some(
+        (request) =>
+          byteArrayEquals(request.pubkey, builder.pubkey) &&
+          byteArrayEquals(request.sourceAddress, builder.executionAddress)
+      )
+    ) {
+      throw Error(`Bid builderIndex=${bid.builderIndex} may exit in the parent payload`);
+    }
+  }
+
   // The builder must honor the proposer preferences it learned over gossip
   const bidEpoch = computeEpochAtSlot(bid.slot);
   const dependentRootHex = (() => {

package/src/metrics/metrics/lodestar.ts

@@ -83,15 +83,15 @@ export function createLodestarMetrics(
       }),
       jobTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_job_time_seconds",
-        help: "Time to process gossip validation queue job in seconds",
+        help: "Time to process gossip validation queue job in seconds (accepted messages only)",
         labelNames: ["topic"],
-        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10],
+        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2],
       }),
       jobWaitTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_job_wait_time_seconds",
         help: "Time from job added to the queue to starting the job in seconds",
         labelNames: ["topic"],
-        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10],
+        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2],
       }),
       concurrency: register.gauge<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_concurrency",
@@ -104,9 +104,10 @@ export function createLodestarMetrics(
         help: "Age of the first item of each key in the indexed queues in seconds",
         buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5],
       }),
-      queueTime: register.histogram({
+      queueTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_time_seconds",
         help: "Total time an item stays in queue until it is processed in seconds",
+        labelNames: ["topic"],
         buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5],
       }),
     },
@@ -920,6 +921,12 @@ export function createLodestarMetrics(
         help: "Count of errors, by error type, while processing blocks",
         labelNames: ["error"],
       }),
+
+      preStateSource: register.counter<{source: "parentState" | "fallbackPreState" | "preState"}>({
+        name: "lodestar_gossip_block_validation_pre_state_source_total",
+        help: "Source of the pre-state used for gossip block validation",
+        labelNames: ["source"],
+      }),
     },
     gossipBlob: {
       recvToValidation: register.histogram({
@@ -1350,6 +1357,11 @@ export function createLodestarMetrics(
           help: "Total number of InsertOutcome as a result of adding an execution payload bid from api to the pool",
           labelNames: ["insertOutcome"],
         }),
+        apiValidationTime: register.histogram({
+          name: "lodestar_api_execution_payload_bid_validation_time_seconds",
+          help: "Time elapsed for signed execution payload bid validation - api path",
+          buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.5],
+        }),
       },
     },
 
@@ -2030,6 +2042,11 @@ export function createLodestarMetrics(
     },
 
     builderApi: {
+      statusChecks: register.counter<{status: "success" | "error"}>({
+        name: "lodestar_builder_api_status_checks_total",
+        help: "Total count of status checks sent to external builders ahead of a proposal",
+        labelNames: ["status"],
+      }),
       bidRequests: register.counter({
         name: "lodestar_builder_api_bid_requests_total",
         help: "Total count of execution payload bid requests sent to external builders",

package/src/network/network.ts

@@ -516,7 +516,8 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.execution_payload_bid>(
       {type: GossipType.execution_payload_bid, boundary},
-      signedBid
+      signedBid,
+      {floodPublish: true}
     );
   }
 
@@ -542,17 +543,19 @@ export class Network implements INetwork {
 
   private async publishGossip<K extends GossipType>(
     topic: GossipTopicMap[K],
-    object: GossipTypeMap[K]
+    object: GossipTypeMap[K],
+    opts?: PublishOpts
   ): Promise<number> {
     const topicStr = stringifyGossipTopic(this.config, topic);
     const sszType = getGossipSSZType(topic);
     const messageData = (sszType.serialize as (object: GossipTypeMap[GossipType]) => Uint8Array)(object);
-    const opts: PublishOpts = {
+    const publishOpts: PublishOpts = {
       ignoreDuplicatePublishError: gossipTopicIgnoreDuplicatePublishError[topic.type],
       // Leave undefined unless the topic opts out, so `--network.allowPublishToZeroPeers` still applies
       allowPublishToZeroTopicPeers: gossipTopicAllowPublishToZeroPeers[topic.type] ? true : undefined,
+      ...opts,
     };
-    const sentPeers = await this.core.publishGossip(topicStr, messageData, opts);
+    const sentPeers = await this.core.publishGossip(topicStr, messageData, publishOpts);
 
     this.logger.verbose("Publish to topic", {topic: topicStr, sentPeers, currentSlot: this.clock.currentSlot});
     return sentPeers;

package/src/network/processor/gossipQueues/indexed.ts

@@ -86,7 +86,6 @@ export class IndexedGossipQueueMinSize<T extends {indexed?: string; queueAddedMs
     const now = Date.now();
     // here we mutate item, which is used for gossip validation later
     item.indexed = key;
-    item.queueAddedMs = now;
     let queueItem = this.indexedItems.get(key);
     if (queueItem == null) {
       queueItem = {firstSeenMs: now, listItems: new LinkedList<T>()};

package/src/network/processor/index.ts

@@ -1,3 +1,4 @@
+import {TopicValidatorResult} from "@libp2p/gossipsub";
 import {routes} from "@lodestar/api";
 import {ForkSeq} from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
@@ -560,6 +561,7 @@ export class NetworkProcessor {
 
   private pushPendingGossipsubMessageToQueue(message: PendingGossipsubMessage): void {
     const topicType = message.topic.type;
+    message.queueAddedMs = Date.now();
     const droppedCount = this.gossipQueues[topicType].add(message);
     if (droppedCount) {
       // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
@@ -746,12 +748,20 @@ export class NetworkProcessor {
       for (const msg of messageOrArray) {
         msg.startProcessUnixSec = nowSec;
         if (msg.queueAddedMs !== undefined) {
-          this.metrics?.gossipValidationQueue.queueTime.observe(nowSec - msg.queueAddedMs / 1000);
+          this.metrics?.gossipValidationQueue.queueTime.observe(
+            {topic: msg.topic.type},
+            nowSec - msg.queueAddedMs / 1000
+          );
         }
       }
     } else {
-      // indexed queue is not used here
       messageOrArray.startProcessUnixSec = nowSec;
+      if (messageOrArray.queueAddedMs !== undefined) {
+        this.metrics?.gossipValidationQueue.queueTime.observe(
+          {topic: messageOrArray.topic.type},
+          nowSec - messageOrArray.queueAddedMs / 1000
+        );
+      }
     }
 
     const acceptanceArr = Array.isArray(messageOrArray)
@@ -764,11 +774,11 @@ export class NetworkProcessor {
         ];
 
     if (Array.isArray(messageOrArray)) {
-      for (const msg of messageOrArray) {
-        this.trackJobTime(msg, messageOrArray.length);
+      for (const [i, msg] of messageOrArray.entries()) {
+        this.trackJobTime(msg, messageOrArray.length, acceptanceArr[i]);
       }
     } else {
-      this.trackJobTime(messageOrArray, 1);
+      this.trackJobTime(messageOrArray, 1, acceptanceArr[0]);
     }
 
     // Use setTimeout to yield to the macro queue
@@ -796,17 +806,21 @@ export class NetworkProcessor {
     }
   }
 
-  private trackJobTime(message: PendingGossipsubMessage, numJob: number): void {
+  private trackJobTime(message: PendingGossipsubMessage, numJob: number, acceptance: TopicValidatorResult): void {
     if (message.startProcessUnixSec !== null) {
+      // record job wait time for all messages
       this.metrics?.gossipValidationQueue.jobWaitTime.observe(
         {topic: message.topic.type},
         message.startProcessUnixSec - message.seenTimestampSec
       );
-      // if it takes 64ms to process 64 jobs, the average job time is 1ms
-      this.metrics?.gossipValidationQueue.jobTime.observe(
-        {topic: message.topic.type},
-        (Date.now() / 1000 - message.startProcessUnixSec) / numJob
-      );
+      // but only record job time for ACCEPT messages
+      if (acceptance === TopicValidatorResult.Accept) {
+        // if it takes 64ms to process 64 jobs, the average job time is 1ms
+        this.metrics?.gossipValidationQueue.jobTime.observe(
+          {topic: message.topic.type},
+          (Date.now() / 1000 - message.startProcessUnixSec) / numJob
+        );
+      }
     }
   }
 

package/src/sync/range/chain.ts

@@ -652,7 +652,9 @@ export class SyncChain {
 
         const blockSlots = downloadSuccessOutput.blocks.map((b) => b.slot);
         const envelopeSlots = downloadSuccessOutput.payloadEnvelopes
-          ? Array.from(downloadSuccessOutput.payloadEnvelopes.keys())
+          ? Array.from(downloadSuccessOutput.payloadEnvelopes.entries())
+              .filter(([, payloadInput]) => payloadInput.hasPayloadEnvelope())
+              .map(([slot]) => slot)
           : null;
 
         this.logger.debug(logMessage, {
@@ -684,13 +686,18 @@ export class SyncChain {
    */
   private async processBatch(batch: Batch): Promise<void> {
     const {blocks, payloadEnvelopes, peers} = batch.startProcessing();
+    const envelopeSlots = payloadEnvelopes
+      ? Array.from(payloadEnvelopes.entries())
+          .filter(([, payloadInput]) => payloadInput.hasPayloadEnvelope())
+          .map(([slot]) => slot)
+      : null;
 
     const logCtx = {
       id: this.logId,
       ...batch.getMetadata(),
       blockCount: blocks.length,
       blockSlots: prettyPrintIndices(blocks.map((b) => b.slot)),
-      ...(payloadEnvelopes ? {envelopeSlots: prettyPrintIndices(Array.from(payloadEnvelopes.keys()))} : {}),
+      ...(envelopeSlots ? {envelopeSlots: prettyPrintIndices(envelopeSlots)} : {}),
       peers: peers.map(prettyPrintPeerIdStr).join(","),
     };
     this.logger.verbose("Processing batch", logCtx);

package/src/sync/range/range.ts

@@ -212,7 +212,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
       for (const block of blocks) {
         await this.chain.processBlock(block, flags);
         const payloadEnvelope = payloadEnvelopes?.get(block.slot);
-        if (payloadEnvelope) {
+        if (payloadEnvelope?.hasPayloadEnvelope()) {
           await this.chain.processExecutionPayload(payloadEnvelope);
         }
       }