@lodestar/beacon-node 1.51.0-dev.09763aa8d2 → 1.51.0-dev.0bb5c87a7e

package/src/chain/blocks/verifyBlocksSignatures.ts

@@ -36,10 +36,9 @@ export async function verifyBlocksSignatures(
     ? preState0.currentSyncCommitteeIndexed
     : new SyncCommitteeCacheEmpty();
 
-  // Verifies signatures after running state transition, so all SyncCommittee signed roots are known at this point.
-  // We must ensure block.slot <= state.slot before running getAllBlockSignatureSets().
-  // NOTE: If in the future multiple blocks signatures are verified at once, all blocks must be in the same epoch
-  // so the attester and proposer shufflings are correct.
+  // Runs concurrently with the state transition. Signature sets are built from preState0, which
+  // verifyBlocksInEpoch() guarantees is in the same epoch as every block, so the proposer and attester shufflings
+  // and the sync committee are the ones the blocks were signed under.
   for (const [i, block] of blocks.entries()) {
     // Use [i] to make clear that the index has to be correct to blame the right block below on BlockError()
     isValidPromises[i] = opts.validSignatures
@@ -50,7 +49,10 @@ export async function verifyBlocksSignatures(
         bls.verifySignatureSets(
           getBlockSignatureSets(config, currentSyncCommitteeIndexed, preState0, block, indexedAttestationsByBlock[i], {
             skipProposerSignature: opts.validProposerSignature,
-          })
+          }),
+          // Prioritize block verification over attestations, since
+          // attestations already buffer up to `MAX_BUFFER_WAIT_MS`
+          {priority: true}
         );
 
     // getBlockSignatureSets() takes 45ms in benchmarks for 2022Q2 mainnet blocks (100 sigs). When syncing a 32 blocks

package/src/chain/blocks/verifyPayloadsDataAvailability.ts

@@ -22,9 +22,7 @@ export async function verifyPayloadsDataAvailability(
   const promises: Promise<gloas.DataColumnSidecar[]>[] = [];
   for (const payloadInput of payloadInputs) {
     if (!payloadInput.hasAllData()) {
-      // Sync and reconstruction keep pulling the data columns from the network while we wait. The wait is
-      // capped at one slot because the payload queue is serial: a longer wait lets consecutive unavailable
-      // payloads push every later payload import further behind with each slot.
+      // Sync and reconstruction keep pulling the data columns from the network while we wait
       promises.push(
         payloadInput.waitForAllData(config.getSlotDurationMs(payloadInput.forkName), signal).catch((e) => {
           if (e instanceof TimeoutError) {

package/src/chain/bls/interface.ts

@@ -22,6 +22,8 @@ export type VerifySignatureOpts = {
   verifyOnMainThread?: boolean;
   /**
    * Some signature sets are more important than others, and should be verified first.
+   * Non-batchable priority jobs are posted to an idle worker before this call returns, so the caller
+   * can run synchronous work right after without delaying the verification.
    */
   priority?: boolean;
 };

package/src/chain/bls/multithread/index.ts

@@ -333,15 +333,19 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
       }
     }
 
+    // First hand work to an idle worker before returning to the caller.
+    // Deferring the post to the next macrotask would make the worker
+    // sit idle while waiting for the next synchronous work
+    else if (job.opts.priority) {
+      this.jobs.unshift(job);
+      void this.runJob();
+    }
+
     // Push job and schedule to call `runJob` in the next macro event loop cycle.
     // This is useful to allow batching job submitted from a synchronous for loop,
     // and to prevent large stacks since runJob may be called recursively.
     else {
-      if (job.opts.priority) {
-        this.jobs.unshift(job);
-      } else {
-        this.jobs.push(job);
-      }
+      this.jobs.push(job);
       callInNextEventLoop(() => void this.runJob());
     }
   }

package/src/chain/opPools/aggregatedAttestationPool.ts

@@ -20,6 +20,7 @@ import {
 } from "@lodestar/params";
 import {
   EffectiveBalanceIncrements,
+  G2_POINT_AT_INFINITY,
   IBeaconStateView,
   RootCache,
   computeEpochAtSlot,
@@ -30,7 +31,7 @@ import {
   isStatePostGloas,
 } from "@lodestar/state-transition";
 import {Attestation, Epoch, RootHex, Slot, electra, isElectraAttestation, phase0, ssz} from "@lodestar/types";
-import {Logger, MapDef, assert, toRootHex} from "@lodestar/utils";
+import {Logger, MapDef, assert, byteArrayEquals, toRootHex} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
 import {IntersectResult, intersectUint8Arrays} from "../../util/bitArray.js";
 import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
@@ -409,20 +410,34 @@ export class AggregatedAttestationPool {
 
     const sortedConsolidationsByScore = Array.from(consolidations.entries())
       .sort((a, b) => b[1] - a[1])
-      .map(([consolidation, _]) => consolidation)
-      .slice(0, MAX_ATTESTATIONS_ELECTRA);
+      .map(([consolidation, _]) => consolidation);
 
     // on chain aggregation is expensive, only do it after all
     const packedAttestationsMetrics = this.metrics?.opPool.aggregatedAttestationPool.packedAttestations;
-    const packedAttestations: electra.Attestation[] = new Array(sortedConsolidationsByScore.length);
-    for (const [i, consolidation] of sortedConsolidationsByScore.entries()) {
-      packedAttestations[i] = aggregateConsolidation(consolidation);
+    const packedAttestations: electra.Attestation[] = [];
+    for (const consolidation of sortedConsolidationsByScore) {
+      if (packedAttestations.length >= MAX_ATTESTATIONS_ELECTRA) {
+        break;
+      }
+
+      const attestation = aggregateConsolidation(consolidation);
+      // The point at infinity is not a valid aggregate signature, exclude it from the block
+      if (byteArrayEquals(attestation.signature, G2_POINT_AT_INFINITY)) {
+        this.logger?.debug("Excluding aggregate with infinity signature", {
+          slot: stateSlot,
+          attestationSlot: attestation.data.slot,
+          committees: consolidation.byCommittee.size,
+        });
+        continue;
+      }
+
+      const i = packedAttestations.push(attestation) - 1;
 
       // record metrics of packed attestations
       packedAttestationsMetrics?.committeeCount.set({index: i}, consolidation.byCommittee.size);
       packedAttestationsMetrics?.totalAttesters.set({index: i}, consolidation.totalAttesters);
       packedAttestationsMetrics?.nonParticipation.set({index: i}, consolidation.notSeenAttesters);
-      packedAttestationsMetrics?.inclusionDistance.set({index: i}, stateSlot - packedAttestations[i].data.slot);
+      packedAttestationsMetrics?.inclusionDistance.set({index: i}, stateSlot - attestation.data.slot);
       packedAttestationsMetrics?.newSeenAttesters.set({index: i}, consolidation.newSeenAttesters);
       packedAttestationsMetrics?.totalEffectiveBalance.set({index: i}, consolidation.totalNewSeenEffectiveBalance);
     }
@@ -462,7 +477,8 @@ export class AggregatedAttestationPool {
         attestations.push(...attestationGroup.getAttestations());
       }
     }
-    return attestations;
+    // The point at infinity is not a valid aggregate signature, do not serve such an aggregate
+    return attestations.filter((attestation) => !byteArrayEquals(attestation.signature, G2_POINT_AT_INFINITY));
   }
 
   private onScrapeMetrics(metrics: Metrics): void {

package/src/chain/opPools/attestationPool.ts

@@ -2,8 +2,9 @@ import {BitArray} from "@chainsafe/ssz";
 import {ChainForkConfig} from "@lodestar/config";
 import {Signature, aggregateSignatures} from "@lodestar/native/blst";
 import {MAX_COMMITTEES_PER_SLOT, isForkPostElectra} from "@lodestar/params";
+import {G2_POINT_AT_INFINITY} from "@lodestar/state-transition";
 import {Attestation, RootHex, SingleAttestation, Slot, isElectraSingleAttestation} from "@lodestar/types";
-import {MapDef, assert} from "@lodestar/utils";
+import {MapDef, assert, byteArrayEquals} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
 import {IClock} from "../../util/clock.js";
 import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js";
@@ -180,7 +181,13 @@ export class AttestationPool {
       assert.true(!isElectraAggregate(aggregate), "Aggregate should be type AggregateFastPhase0");
     }
 
-    return fastToAttestation(aggregate);
+    const attestation = fastToAttestation(aggregate);
+    // The point at infinity is not a valid aggregate signature, do not serve such an aggregate
+    if (byteArrayEquals(attestation.signature, G2_POINT_AT_INFINITY)) {
+      return null;
+    }
+
+    return attestation;
   }
 
   /**
@@ -209,7 +216,12 @@ export class AttestationPool {
       if (aggregateByRoot) {
         for (const aggFastByIndex of aggregateByRoot.values()) {
           for (const aggFast of aggFastByIndex.values()) {
-            attestations.push(fastToAttestation(aggFast));
+            const attestation = fastToAttestation(aggFast);
+            // The point at infinity is not a valid aggregate signature, do not serve such an aggregate
+            if (byteArrayEquals(attestation.signature, G2_POINT_AT_INFINITY)) {
+              continue;
+            }
+            attestations.push(attestation);
           }
         }
       }

package/src/chain/opPools/payloadAttestationPool.ts

@@ -2,8 +2,9 @@ import {BitArray} from "@chainsafe/ssz";
 import {ChainForkConfig} from "@lodestar/config";
 import {Signature, aggregateSignatures} from "@lodestar/native/blst";
 import {MAX_COMMITTEES_PER_SLOT, MAX_PAYLOAD_ATTESTATIONS, PTC_SIZE} from "@lodestar/params";
+import {G2_POINT_AT_INFINITY} from "@lodestar/state-transition";
 import {RootHex, Slot, gloas} from "@lodestar/types";
-import {MapDef, toRootHex} from "@lodestar/utils";
+import {MapDef, byteArrayEquals, toRootHex} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
 import {IClock} from "../../util/clock.js";
 import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js";
@@ -112,11 +113,19 @@ export class PayloadAttestationPool {
       return [];
     }
 
-    return Array.from(aggregateByDataRoot.values())
-      .slice()
+    const payloadAttestations: gloas.PayloadAttestation[] = [];
+    for (const aggregate of aggregateByDataRoot.values()) {
+      const payloadAttestation = fastToPayloadAttestation(aggregate);
+      // The point at infinity is not a valid aggregate signature, exclude it from the block
+      if (byteArrayEquals(payloadAttestation.signature, G2_POINT_AT_INFINITY)) {
+        continue;
+      }
+      payloadAttestations.push(payloadAttestation);
+    }
+
+    return payloadAttestations
       .sort((a, b) => b.aggregationBits.getTrueBitIndexes().length - a.aggregationBits.getTrueBitIndexes().length)
-      .slice(0, MAX_PAYLOAD_ATTESTATIONS)
-      .map(fastToPayloadAttestation);
+      .slice(0, MAX_PAYLOAD_ATTESTATIONS);
   }
 
   getAll(slot?: Slot): gloas.PayloadAttestation[] {
@@ -139,9 +148,11 @@ export class PayloadAttestationPool {
       }
     }
 
+    // The point at infinity is not a valid aggregate signature, do not serve such an aggregate
     return aggregates
       .sort((a, b) => b.aggregationBits.getTrueBitIndexes().length - a.aggregationBits.getTrueBitIndexes().length)
-      .map(fastToPayloadAttestation);
+      .map(fastToPayloadAttestation)
+      .filter((attestation) => !byteArrayEquals(attestation.signature, G2_POINT_AT_INFINITY));
   }
 
   prune(clockSlot: Slot): void {

package/src/chain/opPools/syncCommitteeMessagePool.ts

@@ -2,8 +2,9 @@ import {BitArray} from "@chainsafe/ssz";
 import {ChainForkConfig} from "@lodestar/config";
 import {Signature, aggregateSignatures} from "@lodestar/native/blst";
 import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params";
+import {G2_POINT_AT_INFINITY} from "@lodestar/state-transition";
 import {Root, Slot, SubcommitteeIndex, SubnetID, altair} from "@lodestar/types";
-import {MapDef, toRootHex} from "@lodestar/utils";
+import {MapDef, byteArrayEquals, toRootHex} from "@lodestar/utils";
 import {IClock} from "../../util/clock.js";
 import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js";
 import {pruneBySlot, signatureFromBytesNoCheck} from "./utils.js";
@@ -110,10 +111,16 @@ export class SyncCommitteeMessagePool {
       return null;
     }
 
+    const signature = contribution.signature.toBytes();
+    // The point at infinity is not a valid contribution signature, do not serve such a contribution
+    if (byteArrayEquals(signature, G2_POINT_AT_INFINITY)) {
+      return null;
+    }
+
     return {
       ...contribution,
       aggregationBits: contribution.aggregationBits,
-      signature: contribution.signature.toBytes(),
+      signature,
     };
   }
 

package/src/chain/opPools/syncContributionAndProofPool.ts

@@ -4,7 +4,7 @@ import {Signature, aggregateSignatures} from "@lodestar/native/blst";
 import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params";
 import {G2_POINT_AT_INFINITY} from "@lodestar/state-transition";
 import {Root, Slot, SubnetID, altair, ssz} from "@lodestar/types";
-import {Logger, MapDef, toRootHex} from "@lodestar/utils";
+import {Logger, MapDef, byteArrayEquals, toRootHex} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
 import {IClock} from "../../util/clock.js";
 import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js";
@@ -234,8 +234,18 @@ export function aggregate(
   }
 
   metrics?.opPool.syncContributionAndProofPool.getAggregateParticipants.set(participationCount);
+
+  const syncCommitteeSignature = aggregateSignatures(signatures).toBytes();
+  // The point at infinity is only a valid sync aggregate signature with empty participation bits
+  if (byteArrayEquals(syncCommitteeSignature, G2_POINT_AT_INFINITY)) {
+    return {
+      syncCommitteeBits: BitArray.fromBitLen(SYNC_COMMITTEE_SIZE),
+      syncCommitteeSignature,
+    };
+  }
+
   return {
     syncCommitteeBits,
-    syncCommitteeSignature: aggregateSignatures(signatures).toBytes(),
+    syncCommitteeSignature,
   };
 }

package/src/chain/validation/executionPayloadBid.ts

@@ -179,6 +179,20 @@ async function validateExecutionPayloadBid(
     });
   }
 
+  // [IGNORE] this bid is the highest value bid seen for the tuple
+  // `(bid.slot, bid.parent_block_hash, bid.parent_block_root)`.
+  // As a DoS prevention measure, the bid must also exceed the current highest bid by a minimum
+  // increment, see https://github.com/ethereum/consensus-specs/pull/4831. This prevents spam
+  // from builders submitting numerous bids with minimal value increments.
+  const bestBid = chain.executionPayloadBidPool.getBestBid(bid.slot, bidParentBlockHash, bidParentBlockRoot);
+  if (bestBid !== null && bid.value < getMinBidValue(bestBid.signedBid.message.value)) {
+    throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
+      code: ExecutionPayloadBidErrorCode.BID_TOO_LOW,
+      bidValue: bid.value,
+      currentHighestBid: bestBid.signedBid.message.value,
+    });
+  }
+
   // [IGNORE] `bid.parent_block_root` is the hash tree root of a known beacon block in fork choice.
   // Moved earlier than the spec ordering so we can derive the proposer dependent root for the
   // proposer-preferences lookup below from a known fork-choice block.
@@ -352,19 +366,6 @@ async function validateExecutionPayloadBid(
     });
   }
 
-  // [IGNORE] this bid is the highest value bid seen for the tuple
-  // `(bid.slot, bid.parent_block_hash, bid.parent_block_root)`.
-  // As a DoS prevention measure, the bid must also exceed the current highest bid by a minimum
-  // increment, see https://github.com/ethereum/consensus-specs/pull/4831. This prevents spam
-  // from builders submitting numerous bids with minimal value increments.
-  const bestBid = chain.executionPayloadBidPool.getBestBid(bid.slot, bidParentBlockHash, bidParentBlockRoot);
-  if (bestBid !== null && bid.value < getMinBidValue(bestBid.signedBid.message.value)) {
-    throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
-      code: ExecutionPayloadBidErrorCode.BID_TOO_LOW,
-      bidValue: bid.value,
-      currentHighestBid: bestBid.signedBid.message.value,
-    });
-  }
   // [IGNORE] `bid.value` is less or equal than the builder's excess balance --
   // i.e. `can_builder_cover_bid(state, builder_index, amount)` returns `True`.
   if (!state.canBuilderCoverBid(bid.builderIndex, bid.value)) {

package/src/db/flatFileStore/flatFileStore.ts

@@ -1,4 +1,4 @@
-import fs from "node:fs";
+import {type FileHandle, open, readFile, rm} from "node:fs/promises";
 import path from "node:path";
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkPostFulu} from "@lodestar/params";
@@ -172,7 +172,7 @@ export class FlatFileStore implements IFlatFileStore {
       for (const slot of slotsToPrune) {
         const release = await this.acquireMutationLock(slot);
         try {
-          await fs.promises.rm(path.join(this.dataColumnDir, padSlot(slot)), {recursive: true, force: true});
+          await rm(path.join(this.dataColumnDir, padSlot(slot)), {recursive: true, force: true});
           this.slotIndex.remove(slot);
           this.metrics?.prunedDirectories.inc();
         } finally {
@@ -212,7 +212,7 @@ export class FlatFileStore implements IFlatFileStore {
 
   private async readFile(slot: Slot, rootHex: RootHex): Promise<Uint8Array | null> {
     try {
-      return await fs.promises.readFile(this.filePath(slot, rootHex));
+      return await readFile(this.filePath(slot, rootHex));
     } catch (e) {
       if (!isFsNotFoundError(e)) throw e;
       return null;
@@ -224,9 +224,9 @@ export class FlatFileStore implements IFlatFileStore {
     rootHex: RootHex,
     indices: number[]
   ): Promise<(Uint8Array | undefined)[] | null> {
-    let fd: fs.promises.FileHandle;
+    let fd: FileHandle;
     try {
-      fd = await fs.promises.open(this.filePath(slot, rootHex), "r");
+      fd = await open(this.filePath(slot, rootHex), "r");
     } catch (e) {
       if (!isFsNotFoundError(e)) throw e;
       return null;
@@ -270,7 +270,7 @@ export class FlatFileStore implements IFlatFileStore {
     await observeFlatFileStoreOperation(this.metrics, FlatFileStoreOperation.delete, async () => {
       const release = await this.acquireMutationLock(slot);
       try {
-        await fs.promises.rm(this.filePath(slot, rootHex), {force: true});
+        await rm(this.filePath(slot, rootHex), {force: true});
       } finally {
         release();
       }
@@ -294,7 +294,7 @@ function validateDcolHeader(header: DcolHeader, slot: Slot, rootHex: RootHex): v
   }
 }
 
-async function readExactly(fd: fs.promises.FileHandle, buffer: Uint8Array, position: number): Promise<void> {
+async function readExactly(fd: FileHandle, buffer: Uint8Array, position: number): Promise<void> {
   let totalRead = 0;
   while (totalRead < buffer.length) {
     const {bytesRead} = await fd.read(buffer, totalRead, buffer.length - totalRead, position + totalRead);

package/src/db/flatFileStore/slotIndex.ts

@@ -1,4 +1,5 @@
-import fs from "node:fs";
+import type {Dirent} from "node:fs";
+import {readdir} from "node:fs/promises";
 import {Slot} from "@lodestar/types";
 import {isFsNotFoundError} from "./errors.js";
 import {padSlot} from "./path.js";
@@ -33,9 +34,9 @@ export class SlotIndex {
 
   async rebuildFromDisk(columnsDir: string): Promise<SlotIndexRebuildStats> {
     this.slots.clear();
-    let entries: fs.Dirent[];
+    let entries: Dirent[];
     try {
-      entries = await fs.promises.readdir(columnsDir, {withFileTypes: true});
+      entries = await readdir(columnsDir, {withFileTypes: true});
     } catch (e) {
       if (!isFsNotFoundError(e)) throw e;
       return {slots: 0, ignoredEntries: 0};

package/src/execution/builder/apiClient.ts

@@ -19,21 +19,20 @@ import {IClock} from "../../util/clock.js";
  * Additional duration to account for potential event loop lag which causes
  * builder bids to be rejected even though the response was sent in time.
  */
-const EVENT_LOOP_LAG_BUFFER = 250;
+export const BUILDER_BID_LAG_BUFFER_MS = 250;
 
 /**
- * Duration given to a builder to provide a `SignedExecutionPayloadBid` before the deadline
- * is reached, advertised on each bid request via the `X-Timeout-Ms` header. The p2p bid is
- * selected after the same duration, only considering bids received up to that point.
+ * Default duration given to a builder to provide a `SignedExecutionPayloadBid` before the deadline
+ * is reached, advertised on each bid request via the `X-Timeout-Ms` header. Bids received via
+ * p2p are given the same time to bid up.
  */
-export const BUILDER_BID_DEADLINE_MS = 500;
-
-/** Local timeout for bid requests, event loop lag must not discard bids that arrived in time */
-export const BUILDER_BID_REQUEST_TIMEOUT_MS = BUILDER_BID_DEADLINE_MS + EVENT_LOOP_LAG_BUFFER;
+export const DEFAULT_BUILDER_BID_DEADLINE_MS = 500;
 
 export type BuilderApiClientOpts = {
-  /** Timeout for builder api requests, bid requests always use `BUILDER_BID_REQUEST_TIMEOUT_MS` */
+  /** Timeout for builder api requests, bid requests are bounded by `bidDeadline` instead */
   timeout?: number;
+  /** Time builders are given to provide a bid once block production starts, in milliseconds */
+  bidDeadline?: number;
   // Add User-Agent header to all requests
   userAgent?: string;
 };
@@ -69,6 +68,7 @@ export type BuilderApiBid = {
  * each block production request, clients are dialed on demand based on the entry `url`.
  */
 export class BuilderApiClient {
+  readonly bidDeadline: number;
   private readonly clients = new Map<BuilderUrl, BuilderApi>();
 
   constructor(
@@ -78,7 +78,9 @@ export class BuilderApiClient {
     private readonly bls: IBlsVerifier,
     private readonly metrics: Metrics | null = null,
     private readonly logger?: Logger
-  ) {}
+  ) {
+    this.bidDeadline = opts.bidDeadline ?? DEFAULT_BUILDER_BID_DEADLINE_MS;
+  }
 
   /**
    * Fan out bid requests to the builders named by the entries, one request per unique
@@ -139,9 +141,10 @@ export class BuilderApiClient {
               proposerPubkey,
               requestAuth: entry.auth,
               dateMilliseconds: Date.now(),
-              timeoutMs: BUILDER_BID_DEADLINE_MS,
+              timeoutMs: this.bidDeadline,
             },
-            {timeoutMs: BUILDER_BID_REQUEST_TIMEOUT_MS}
+            // Event loop lag must not discard bids that arrived in time
+            {timeoutMs: this.bidDeadline + BUILDER_BID_LAG_BUFFER_MS}
           );
           const signedBid = res.value();
           if (signedBid === undefined) {
@@ -203,7 +206,7 @@ export class BuilderApiClient {
 
   /**
    * Establish the connection to every known builder outside the bid deadline, a cold TCP/TLS handshake
-   * to a distant builder can exhaust `BUILDER_BID_DEADLINE_MS` on its own, and a long idle connection
+   * to a distant builder can exhaust the bid deadline on its own, and a long idle connection
    * may be half-open and only fail once written to.
    */
   async checkStatus(): Promise<void> {

package/src/execution/builder/http.ts

@@ -20,6 +20,7 @@ import {
 } from "@lodestar/types";
 import {toPrintableUrl} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
+import {DEFAULT_BUILDER_BID_DEADLINE_MS} from "./apiClient.js";
 import {ValidatorRegistration, ValidatorRegistrationCache} from "./cache.js";
 import {IExecutionBuilder} from "./interface.js";
 
@@ -29,6 +30,8 @@ export type ExecutionBuilderHttpOpts = {
   timeout?: number;
   faultInspectionWindow?: number;
   allowedFaults?: number;
+  /** Post-gloas only, time builders are given to provide a bid once block production starts, in milliseconds */
+  bidDeadline?: number;
 
   // Only required for merge-mock runs, no need to expose it to cli
   issueLocalFcUWithFeeRecipient?: string;
@@ -40,6 +43,7 @@ export const defaultExecutionBuilderHttpOpts: ExecutionBuilderHttpOpts = {
   enabled: false,
   url: "http://localhost:8661",
   timeout: 12000,
+  bidDeadline: DEFAULT_BUILDER_BID_DEADLINE_MS,
 };
 
 export enum BuilderStatus {

package/src/metrics/metrics/lodestar.ts

@@ -645,6 +645,11 @@ export function createLodestarMetrics(
       help: "Count of peers by sync type [FullySynced, Advanced, Behind]",
       labelNames: ["syncType"],
     }),
+    syncPeerFinalizedCheckpointMismatch: register.counter<{reason: string}>({
+      name: "lodestar_sync_peer_finalized_checkpoint_mismatch_total",
+      help: "Total behind peers whose Status finalized checkpoint is not in our chain, by reason",
+      labelNames: ["reason"],
+    }),
     syncSwitchGossipSubscriptions: register.gauge<{action: string}>({
       name: "lodestar_sync_switch_gossip_subscriptions",
       help: "Sync switched gossip subscriptions on/off",
@@ -1008,6 +1013,11 @@ export function createLodestarMetrics(
         help: "Time elapsed between block received and fully verified state, signatures and payload",
         buckets: [0.05, 0.1, 0.3, 0.5, 0.7, 1, 1.3, 1.6, 2, 2.5, 3, 3.5, 4],
       }),
+      receivedToConsensusVerifiedTime: register.histogram({
+        name: "lodestar_gossip_block_received_to_consensus_verified_seconds",
+        help: "Time from gossip block arrival until both the state transition and all block signature verifications are complete, excluding execution payload verification",
+        buckets: [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1],
+      }),
       verifiedToBlobsAvailabiltyTime: register.histogram<{numBlobs: number}>({
         name: "lodestar_gossip_block_verified_to_blobs_availability_time",
         help: "Time elapsed between block verified and blobs became available",

package/src/network/core/networkCore.ts

@@ -10,6 +10,7 @@ import {isForkPostFulu} from "@lodestar/params";
 import {ResponseIncoming} from "@lodestar/reqresp";
 import {Epoch, Status, fulu, sszTypesFor} from "@lodestar/types";
 import {formatNodePeer} from "../../api/impl/node/utils.js";
+import {GoodByeReasonCode} from "../../constants/index.js";
 import {RegistryMetricCreator} from "../../metrics/index.js";
 import {ClockEvent, IClock} from "../../util/clock.js";
 import {CustodyConfig} from "../../util/dataColumns.js";
@@ -332,6 +333,10 @@ export class NetworkCore implements INetworkCore {
     this.peerManager.reStatusPeers(peers);
   }
 
+  async goodbyeAndDisconnect(peer: PeerIdStr, reason: GoodByeReasonCode): Promise<void> {
+    await this.peerManager.goodbyeAndDisconnect(peerIdFromString(peer), reason);
+  }
+
   /**
    * Request att subnets up `toSlot`. Network will ensure to mantain some peers for each
    */

package/src/network/core/networkCoreWorker.ts

@@ -138,6 +138,7 @@ const libp2pWorkerApi: NetworkWorkerApi = {
   prepareSyncCommitteeSubnets: (subscriptions) => core.prepareSyncCommitteeSubnets(subscriptions),
   reportPeer: (peer, action, actionName) => core.reportPeer(peer, action, actionName),
   reStatusPeers: (peers) => core.reStatusPeers(peers),
+  goodbyeAndDisconnect: (peer, reason) => core.goodbyeAndDisconnect(peer, reason),
   subscribeGossipCoreTopics: () => core.subscribeGossipCoreTopics(),
   unsubscribeGossipCoreTopics: () => core.unsubscribeGossipCoreTopics(),
 

package/src/network/core/networkCoreWorkerHandler.ts

@@ -11,6 +11,7 @@ import type {LoggerNode} from "@lodestar/logger/node";
 import {ResponseIncoming, ResponseOutgoing} from "@lodestar/reqresp";
 import {Status} from "@lodestar/types";
 import {withTimeout} from "@lodestar/utils";
+import {GoodByeReasonCode} from "../../constants/index.js";
 import {Metrics} from "../../metrics/index.js";
 import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js";
 import {PeerIdStr, peerIdFromString} from "../../util/peerId.js";
@@ -205,6 +206,9 @@ export class WorkerNetworkCore implements INetworkCore {
   reStatusPeers(peers: PeerIdStr[]): Promise<void> {
     return this.getApi().reStatusPeers(peers);
   }
+  goodbyeAndDisconnect(peer: PeerIdStr, reason: GoodByeReasonCode): Promise<void> {
+    return this.getApi().goodbyeAndDisconnect(peer, reason);
+  }
   reportPeer(peer: PeerIdStr, action: PeerAction, actionName: string): Promise<void> {
     return this.getApi().reportPeer(peer, action, actionName);
   }

package/src/network/core/types.ts

@@ -5,6 +5,7 @@ import {SpecJson} from "@lodestar/config";
 import {LoggerNodeOpts} from "@lodestar/logger/node";
 import {ResponseIncoming} from "@lodestar/reqresp";
 import {Status} from "@lodestar/types";
+import {GoodByeReasonCode} from "../../constants/index.js";
 import {PeerIdStr} from "../../util/peerId.js";
 import {NetworkOptions} from "../options.js";
 import {PeerAction, PeerScoreStats} from "../peers/index.js";
@@ -53,6 +54,7 @@ export interface INetworkCore extends INetworkCorePublic {
   // Sync method
   reportPeer(peer: PeerIdStr, action: PeerAction, actionName: string): void;
   reStatusPeers(peers: PeerIdStr[]): Promise<void>;
+  goodbyeAndDisconnect(peer: PeerIdStr, reason: GoodByeReasonCode): Promise<void>;
 
   // TODO: Duplicated methods with INetwork interface
   getConnectedPeers(): Promise<PeerIdStr[]>;

package/src/network/interface.ts

@@ -37,6 +37,7 @@ import {
   phase0,
 } from "@lodestar/types";
 import {BlockInputSource} from "../chain/blocks/blockInput/types.js";
+import {GoodByeReasonCode} from "../constants/index.js";
 import {CustodyConfig} from "../util/dataColumns.js";
 import {PeerIdStr} from "../util/peerId.js";
 import {
@@ -74,6 +75,7 @@ export interface INetwork extends INetworkCorePublic {
   reportPeer(peer: PeerIdStr, action: PeerAction, actionName: string): void;
   shouldAggregate(subnet: SubnetID, slot: Slot): boolean;
   reStatusPeers(peers: PeerIdStr[]): Promise<void>;
+  goodbyeAndDisconnect(peer: PeerIdStr, reason: GoodByeReasonCode): Promise<void>;
   searchUnknownBlock(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void;
   searchUnknownEnvelope(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void;
   // ReqResp

package/src/network/libp2p/index.ts

@@ -150,7 +150,11 @@ export async function createNodeJsLibp2p(
     nodeInfo: {
       name: "lodestar",
       version: networkOpts.version ?? "unknown",
-      userAgent: networkOpts.private ? "" : networkOpts.version ? `lodestar/${networkOpts.version}` : "lodestar",
+      userAgent: networkOpts.private
+        ? ""
+        : networkOpts.advertiseVersion && networkOpts.version
+          ? `lodestar/${networkOpts.version}`
+          : "lodestar",
     },
     addresses: {
       listen: localMultiaddrs,

package/src/network/network.ts

@@ -34,6 +34,7 @@ import {prettyPrintIndices, sleep} from "@lodestar/utils";
 import {BlockInputSource} from "../chain/blocks/blockInput/types.js";
 import {ChainEvent, IBeaconChain} from "../chain/index.js";
 import {computeSubnetForDataColumnSidecar} from "../chain/validation/dataColumnSidecar.js";
+import {GoodByeReasonCode} from "../constants/index.js";
 import {IBeaconDb} from "../db/interface.js";
 import {Metrics, RegistryMetricCreator} from "../metrics/index.js";
 import {IClock} from "../util/clock.js";
@@ -289,6 +290,10 @@ export class Network implements INetwork {
     return this.core.reStatusPeers(peers);
   }
 
+  async goodbyeAndDisconnect(peer: PeerIdStr, reason: GoodByeReasonCode): Promise<void> {
+    return this.core.goodbyeAndDisconnect(peer, reason);
+  }
+
   searchUnknownBlock(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
     this.networkProcessor.searchUnknownBlock(slotRoot, source, peer);
   }