@lodestar/beacon-node 1.51.0-dev.ec493d509d → 1.51.0-dev.f28f5e623b

package/src/chain/opPools/syncContributionAndProofPool.ts

@@ -1,10 +1,10 @@
-import {Signature, aggregateSignatures} from "@chainsafe/lodestar-z/blst";
 import {BitArray} from "@chainsafe/ssz";
 import {ChainForkConfig} from "@lodestar/config";
+import {Signature, aggregateSignatures} from "@lodestar/native/blst";
 import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params";
 import {G2_POINT_AT_INFINITY} from "@lodestar/state-transition";
 import {Root, Slot, SubnetID, altair, ssz} from "@lodestar/types";
-import {Logger, MapDef, toRootHex} from "@lodestar/utils";
+import {Logger, MapDef, byteArrayEquals, toRootHex} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
 import {IClock} from "../../util/clock.js";
 import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js";
@@ -234,8 +234,18 @@ export function aggregate(
   }
 
   metrics?.opPool.syncContributionAndProofPool.getAggregateParticipants.set(participationCount);
+
+  const syncCommitteeSignature = aggregateSignatures(signatures).toBytes();
+  // The point at infinity is only a valid sync aggregate signature with empty participation bits
+  if (byteArrayEquals(syncCommitteeSignature, G2_POINT_AT_INFINITY)) {
+    return {
+      syncCommitteeBits: BitArray.fromBitLen(SYNC_COMMITTEE_SIZE),
+      syncCommitteeSignature,
+    };
+  }
+
   return {
     syncCommitteeBits,
-    syncCommitteeSignature: aggregateSignatures(signatures).toBytes(),
+    syncCommitteeSignature,
   };
 }

package/src/chain/opPools/utils.ts

@@ -1,6 +1,6 @@
-import {Signature} from "@chainsafe/lodestar-z/blst";
 import {BeaconConfig} from "@lodestar/config";
 import {DbBatch, Id, Repository} from "@lodestar/db";
+import {Signature} from "@lodestar/native/blst";
 import {BLS_WITHDRAWAL_PREFIX, DOMAIN_BEACON_ATTESTER, DOMAIN_BEACON_PROPOSER} from "@lodestar/params";
 import {IBeaconStateView, computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {AttesterSlashing, Domain, Slot, capella, phase0} from "@lodestar/types";

package/src/chain/options.ts

@@ -48,7 +48,6 @@ export type IChainOptions = BlockProcessOpts &
     /** Max number of produced block roots (blinded or full) cached for broadcast validations */
     maxCachedProducedRoots?: number;
     initialCustodyGroupCount?: number;
-    broadcastValidationStrictness?: string;
     minSameMessageSignatureSetsToBatch: number;
     archiveDateEpochs?: number;
     nHistoricalStatesFileDataStore?: boolean;
@@ -121,7 +120,6 @@ export const defaultChainOptions: IChainOptions = {
   // for attestation validation, having this value ensures we don't have to regen states most of the time
   maxSkipSlots: 32,
   disableProposerSlashings: false,
-  broadcastValidationStrictness: "warn",
   // should be less than or equal to MIN_SIGNATURE_SETS_TO_BATCH_VERIFY
   // batching too much may block the I/O thread so if useWorker=false, suggest this value to be 32
   // since this batch attestation work is designed to work with useWorker=true, make this the lowest value

package/src/chain/validation/executionPayloadBid.ts

@@ -179,6 +179,20 @@ async function validateExecutionPayloadBid(
     });
   }
 
+  // [IGNORE] this bid is the highest value bid seen for the tuple
+  // `(bid.slot, bid.parent_block_hash, bid.parent_block_root)`.
+  // As a DoS prevention measure, the bid must also exceed the current highest bid by a minimum
+  // increment, see https://github.com/ethereum/consensus-specs/pull/4831. This prevents spam
+  // from builders submitting numerous bids with minimal value increments.
+  const bestBid = chain.executionPayloadBidPool.getBestBid(bid.slot, bidParentBlockHash, bidParentBlockRoot);
+  if (bestBid !== null && bid.value < getMinBidValue(bestBid.signedBid.message.value)) {
+    throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
+      code: ExecutionPayloadBidErrorCode.BID_TOO_LOW,
+      bidValue: bid.value,
+      currentHighestBid: bestBid.signedBid.message.value,
+    });
+  }
+
   // [IGNORE] `bid.parent_block_root` is the hash tree root of a known beacon block in fork choice.
   // Moved earlier than the spec ordering so we can derive the proposer dependent root for the
   // proposer-preferences lookup below from a known fork-choice block.
@@ -352,19 +366,6 @@ async function validateExecutionPayloadBid(
     });
   }
 
-  // [IGNORE] this bid is the highest value bid seen for the tuple
-  // `(bid.slot, bid.parent_block_hash, bid.parent_block_root)`.
-  // As a DoS prevention measure, the bid must also exceed the current highest bid by a minimum
-  // increment, see https://github.com/ethereum/consensus-specs/pull/4831. This prevents spam
-  // from builders submitting numerous bids with minimal value increments.
-  const bestBid = chain.executionPayloadBidPool.getBestBid(bid.slot, bidParentBlockHash, bidParentBlockRoot);
-  if (bestBid !== null && bid.value < getMinBidValue(bestBid.signedBid.message.value)) {
-    throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
-      code: ExecutionPayloadBidErrorCode.BID_TOO_LOW,
-      bidValue: bid.value,
-      currentHighestBid: bestBid.signedBid.message.value,
-    });
-  }
   // [IGNORE] `bid.value` is less or equal than the builder's excess balance --
   // i.e. `can_builder_cover_bid(state, builder_index, amount)` returns `True`.
   if (!state.canBuilderCoverBid(bid.builderIndex, bid.value)) {

package/src/db/beacon.ts

@@ -30,6 +30,9 @@ import {
   VoluntaryExitRepository,
 } from "./repositories/index.js";
 
+/** Below this size, seeks over leftover legacy column data stay cheap */
+const LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES = 1024 ** 3;
+
 export type BeaconDbOpts = {
   dataColumnDir: string;
   logger: Logger;
@@ -67,6 +70,7 @@ export class BeaconDb implements IBeaconDb {
   readonly dataColumns: IDataColumnStore;
   lastLegacyArchiveSlot: Slot | null = null;
   private readonly flatFileStore: FlatFileStore;
+  private readonly logger: Logger;
 
   constructor(
     config: ChainForkConfig,
@@ -100,6 +104,7 @@ export class BeaconDb implements IBeaconDb {
     this.syncCommittee = new SyncCommitteeRepository(config, db);
     this.syncCommitteeWitness = new SyncCommitteeWitnessRepository(config, db);
 
+    this.logger = opts.logger;
     this.flatFileStore = new FlatFileStore(opts.dataColumnDir, config, opts.logger, opts.metrics);
     this.dataColumns = new LegacyDataColumnStore(
       this.flatFileStore,
@@ -125,6 +130,40 @@ export class BeaconDb implements IBeaconDb {
     this.flatFileStore.setMetrics(flatFileStoreMetrics);
   }
 
+  /**
+   * Deletes made while an iterator held a snapshot can stay in the bottom LevelDB level, which `compactRange` never
+   * rewrites, so every seek into the legacy column buckets walks them. Writing a key at each end of the pruned range
+   * makes the compaction overlap those files and rewrite them.
+   */
+  async compactLegacyDataColumns(
+    prunedBeforeSlot: Slot,
+    minSizeBytes = LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES
+  ): Promise<void> {
+    const archiveBucket = Bucket.allForks_dataColumnSidecarsArchive;
+    const start = encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(0));
+    const end = encodeKey(archiveBucket, this.dataColumnSidecarArchive.getMinKeyRaw(prunedBeforeSlot));
+    const sizeBytes = await this.db.approximateSize(start, end);
+    if (sizeBytes < minSizeBytes) return;
+
+    // Column indices stay below 0xffff and hot column keys are shorter, so no sentinel matches a real key
+    const upperSentinel =
+      prunedBeforeSlot > 0
+        ? encodeKey(archiveBucket, this.dataColumnSidecarArchive.encodeKeyRaw(prunedBeforeSlot - 1, 0xffff))
+        : encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(35, 0xff));
+    const sentinels = [start, upperSentinel];
+
+    this.logger.info("Compacting legacy data columns, this may take a while", {sizeBytes});
+    const startTime = Date.now();
+    await this.db.batchPut(sentinels.map((key) => ({key, value: new Uint8Array()})));
+    await this.db.compactRange(start, end);
+    await this.db.batchDelete(sentinels);
+    await this.db.compactRange(start, end);
+    this.logger.verbose("Compacted legacy data columns", {
+      durationMs: Date.now() - startTime,
+      sizeBytes: await this.db.approximateSize(start, end),
+    });
+  }
+
   async pruneHotDb(): Promise<void> {
     // Prune all hot blobs
     await this.blobSidecars.batchDelete(await this.blobSidecars.keys());

package/src/db/dataColumnStore.ts

@@ -13,7 +13,7 @@ export type IndexedDataColumnBytes = {index: ColumnIndex; data: Uint8Array};
 type LegacyHotDataColumnStore = Pick<DataColumnSidecarRepository, "values" | "getManyBinary" | "deleteMany">;
 type LegacyArchiveDataColumnStore = Pick<
   DataColumnSidecarArchiveRepository,
-  "values" | "getManyBinary" | "keys" | "deleteMany"
+  "values" | "getManyBinary" | "pruneBefore"
 >;
 type BlockRootIndex = Pick<BlockArchiveRepository, "getSlotByRoot">;
 
@@ -86,9 +86,7 @@ export class LegacyDataColumnStore implements IDataColumnStore {
 
   async pruneBefore(slot: Slot): Promise<Slot[]> {
     const flatFileSlots = await this.flatFiles.pruneBefore(slot);
-    const prefixedKeys = await this.legacyArchive.keys({lt: {prefix: slot, id: 0}});
-    const slots = [...new Set(prefixedKeys.map(({prefix}) => prefix))];
-    if (slots.length > 0) await this.legacyArchive.deleteMany(slots);
+    const slots = await this.legacyArchive.pruneBefore(slot);
     return [...new Set([...flatFileSlots, ...slots])].sort((a, b) => a - b);
   }
 }

package/src/db/flatFileStore/flatFileStore.ts

@@ -1,4 +1,4 @@
-import fs from "node:fs";
+import {type FileHandle, open, readFile, rm} from "node:fs/promises";
 import path from "node:path";
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkPostFulu} from "@lodestar/params";
@@ -172,7 +172,7 @@ export class FlatFileStore implements IFlatFileStore {
       for (const slot of slotsToPrune) {
         const release = await this.acquireMutationLock(slot);
         try {
-          await fs.promises.rm(path.join(this.dataColumnDir, padSlot(slot)), {recursive: true, force: true});
+          await rm(path.join(this.dataColumnDir, padSlot(slot)), {recursive: true, force: true});
           this.slotIndex.remove(slot);
           this.metrics?.prunedDirectories.inc();
         } finally {
@@ -212,7 +212,7 @@ export class FlatFileStore implements IFlatFileStore {
 
   private async readFile(slot: Slot, rootHex: RootHex): Promise<Uint8Array | null> {
     try {
-      return await fs.promises.readFile(this.filePath(slot, rootHex));
+      return await readFile(this.filePath(slot, rootHex));
     } catch (e) {
       if (!isFsNotFoundError(e)) throw e;
       return null;
@@ -224,9 +224,9 @@ export class FlatFileStore implements IFlatFileStore {
     rootHex: RootHex,
     indices: number[]
   ): Promise<(Uint8Array | undefined)[] | null> {
-    let fd: fs.promises.FileHandle;
+    let fd: FileHandle;
     try {
-      fd = await fs.promises.open(this.filePath(slot, rootHex), "r");
+      fd = await open(this.filePath(slot, rootHex), "r");
     } catch (e) {
       if (!isFsNotFoundError(e)) throw e;
       return null;
@@ -270,7 +270,7 @@ export class FlatFileStore implements IFlatFileStore {
     await observeFlatFileStoreOperation(this.metrics, FlatFileStoreOperation.delete, async () => {
       const release = await this.acquireMutationLock(slot);
       try {
-        await fs.promises.rm(this.filePath(slot, rootHex), {force: true});
+        await rm(this.filePath(slot, rootHex), {force: true});
       } finally {
         release();
       }
@@ -294,7 +294,7 @@ function validateDcolHeader(header: DcolHeader, slot: Slot, rootHex: RootHex): v
   }
 }
 
-async function readExactly(fd: fs.promises.FileHandle, buffer: Uint8Array, position: number): Promise<void> {
+async function readExactly(fd: FileHandle, buffer: Uint8Array, position: number): Promise<void> {
   let totalRead = 0;
   while (totalRead < buffer.length) {
     const {bytesRead} = await fd.read(buffer, totalRead, buffer.length - totalRead, position + totalRead);

package/src/db/flatFileStore/slotIndex.ts

@@ -1,4 +1,5 @@
-import fs from "node:fs";
+import type {Dirent} from "node:fs";
+import {readdir} from "node:fs/promises";
 import {Slot} from "@lodestar/types";
 import {isFsNotFoundError} from "./errors.js";
 import {padSlot} from "./path.js";
@@ -33,9 +34,9 @@ export class SlotIndex {
 
   async rebuildFromDisk(columnsDir: string): Promise<SlotIndexRebuildStats> {
     this.slots.clear();
-    let entries: fs.Dirent[];
+    let entries: Dirent[];
     try {
-      entries = await fs.promises.readdir(columnsDir, {withFileTypes: true});
+      entries = await readdir(columnsDir, {withFileTypes: true});
     } catch (e) {
       if (!isFsNotFoundError(e)) throw e;
       return {slots: 0, ignoredEntries: 0};

package/src/db/interface.ts

@@ -68,6 +68,8 @@ export interface IBeaconDb {
   readonly lastLegacyArchiveSlot: Slot | null;
   init(): Promise<void>;
 
+  compactLegacyDataColumns(prunedBeforeSlot: Slot, minSizeBytes?: number): Promise<void>;
+
   pruneHotDb(): Promise<void>;
 
   deleteDeprecatedEth1Data(): Promise<void>;

package/src/db/repositories/dataColumnSidecarArchive.ts

@@ -15,6 +15,8 @@ const SLOT_BYTE_SIZE = 8;
  * Indexed data by `slot` + `columnIndex`
  */
 export class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot, ColumnIndex, DataColumnSidecar> {
+  private prunedBeforeSlot: Slot | null = null;
+
   constructor(config: ChainForkConfig, db: Db) {
     const bucket = Bucket.allForks_dataColumnSidecarsArchive;
     // Type won't be used since we select it dynamically based on fork
@@ -70,4 +72,22 @@ export class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot,
       encodeNumberForDbKey(0, COLUMN_INDEX_BYTE_SIZE),
     ]);
   }
+
+  /**
+   * Deletes all entries before `slot`. Prunes only delete from the start of the bucket, so the first prune
+   * compacts the range to drop tombstones of earlier runs and later prunes resume where the previous one ended.
+   */
+  async pruneBefore(slot: Slot): Promise<Slot[]> {
+    if (this.prunedBeforeSlot === null) {
+      await this.db.compactRange(this.wrapKey(this.getMinKeyRaw(0)), this.wrapKey(this.getMinKeyRaw(slot)));
+    } else if (slot <= this.prunedBeforeSlot) {
+      return [];
+    }
+
+    const keys = await this.keys({gte: {prefix: this.prunedBeforeSlot ?? 0, id: 0}, lt: {prefix: slot, id: 0}});
+    const slots = [...new Set(keys.map(({prefix}) => prefix))];
+    if (slots.length > 0) await this.deleteMany(slots);
+    this.prunedBeforeSlot = slot;
+    return slots;
+  }
 }

package/src/execution/builder/apiClient.ts

@@ -19,21 +19,20 @@ import {IClock} from "../../util/clock.js";
  * Additional duration to account for potential event loop lag which causes
  * builder bids to be rejected even though the response was sent in time.
  */
-const EVENT_LOOP_LAG_BUFFER = 250;
+export const BUILDER_BID_LAG_BUFFER_MS = 250;
 
 /**
- * Duration given to a builder to provide a `SignedExecutionPayloadBid` before the deadline
- * is reached, advertised on each bid request via the `X-Timeout-Ms` header. The p2p bid is
- * selected after the same duration, only considering bids received up to that point.
+ * Default duration given to a builder to provide a `SignedExecutionPayloadBid` before the deadline
+ * is reached, advertised on each bid request via the `X-Timeout-Ms` header. Bids received via
+ * p2p are given the same time to bid up.
  */
-export const BUILDER_BID_DEADLINE_MS = 500;
-
-/** Local timeout for bid requests, event loop lag must not discard bids that arrived in time */
-export const BUILDER_BID_REQUEST_TIMEOUT_MS = BUILDER_BID_DEADLINE_MS + EVENT_LOOP_LAG_BUFFER;
+export const DEFAULT_BUILDER_BID_DEADLINE_MS = 500;
 
 export type BuilderApiClientOpts = {
-  /** Timeout for builder api requests, bid requests always use `BUILDER_BID_REQUEST_TIMEOUT_MS` */
+  /** Timeout for builder api requests, bid requests are bounded by `bidDeadline` instead */
   timeout?: number;
+  /** Time builders are given to provide a bid once block production starts, in milliseconds */
+  bidDeadline?: number;
   // Add User-Agent header to all requests
   userAgent?: string;
 };
@@ -69,6 +68,7 @@ export type BuilderApiBid = {
  * each block production request, clients are dialed on demand based on the entry `url`.
  */
 export class BuilderApiClient {
+  readonly bidDeadline: number;
   private readonly clients = new Map<BuilderUrl, BuilderApi>();
 
   constructor(
@@ -78,7 +78,9 @@ export class BuilderApiClient {
     private readonly bls: IBlsVerifier,
     private readonly metrics: Metrics | null = null,
     private readonly logger?: Logger
-  ) {}
+  ) {
+    this.bidDeadline = opts.bidDeadline ?? DEFAULT_BUILDER_BID_DEADLINE_MS;
+  }
 
   /**
    * Fan out bid requests to the builders named by the entries, one request per unique
@@ -139,9 +141,10 @@ export class BuilderApiClient {
               proposerPubkey,
               requestAuth: entry.auth,
               dateMilliseconds: Date.now(),
-              timeoutMs: BUILDER_BID_DEADLINE_MS,
+              timeoutMs: this.bidDeadline,
             },
-            {timeoutMs: BUILDER_BID_REQUEST_TIMEOUT_MS}
+            // Event loop lag must not discard bids that arrived in time
+            {timeoutMs: this.bidDeadline + BUILDER_BID_LAG_BUFFER_MS}
           );
           const signedBid = res.value();
           if (signedBid === undefined) {
@@ -203,7 +206,7 @@ export class BuilderApiClient {
 
   /**
    * Establish the connection to every known builder outside the bid deadline, a cold TCP/TLS handshake
-   * to a distant builder can exhaust `BUILDER_BID_DEADLINE_MS` on its own, and a long idle connection
+   * to a distant builder can exhaust the bid deadline on its own, and a long idle connection
    * may be half-open and only fail once written to.
    */
   async checkStatus(): Promise<void> {

package/src/execution/builder/http.ts

@@ -20,6 +20,7 @@ import {
 } from "@lodestar/types";
 import {toPrintableUrl} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
+import {DEFAULT_BUILDER_BID_DEADLINE_MS} from "./apiClient.js";
 import {ValidatorRegistration, ValidatorRegistrationCache} from "./cache.js";
 import {IExecutionBuilder} from "./interface.js";
 
@@ -29,6 +30,8 @@ export type ExecutionBuilderHttpOpts = {
   timeout?: number;
   faultInspectionWindow?: number;
   allowedFaults?: number;
+  /** Post-gloas only, time builders are given to provide a bid once block production starts, in milliseconds */
+  bidDeadline?: number;
 
   // Only required for merge-mock runs, no need to expose it to cli
   issueLocalFcUWithFeeRecipient?: string;
@@ -40,6 +43,7 @@ export const defaultExecutionBuilderHttpOpts: ExecutionBuilderHttpOpts = {
   enabled: false,
   url: "http://localhost:8661",
   timeout: 12000,
+  bidDeadline: DEFAULT_BUILDER_BID_DEADLINE_MS,
 };
 
 export enum BuilderStatus {

package/src/execution/engine/index.ts

@@ -17,8 +17,8 @@ import {EngineRestHttpClient} from "./restHttpClient.js";
 import {RestEngineTransport, formatClientVersionHeader} from "./restTransport.js";
 import {ExecutionEngineMockJsonRpcClient, JsonRpcBackend} from "./utils.js";
 
-export {ExecutionEngineHttp, ExecutionEngineDisabled, defaultExecutionEngineHttpOpts, engineApiModes};
 export type {EngineApiMode};
+export {ExecutionEngineDisabled, ExecutionEngineHttp, defaultExecutionEngineHttpOpts, engineApiModes};
 
 export type ExecutionEngineOpts =
   | ({mode?: "http"} & ExecutionEngineHttpOpts)

package/src/execution/engine/interface.ts

@@ -25,7 +25,7 @@ import {PayloadId, PayloadIdCache, WithdrawalV1} from "./payloadIdCache.js";
 import {ExecutionPayloadBodyV2} from "./types.js";
 import {DATA} from "./utils.js";
 
-export {PayloadIdCache, type PayloadId, type WithdrawalV1};
+export {type PayloadId, PayloadIdCache, type WithdrawalV1};
 export type ClientVersion = routes.node.ClientVersion;
 export const ClientCode = routes.node.ClientCode;
 

package/src/execution/engine/types.ts

@@ -9,6 +9,7 @@ import {
   DEPOSIT_REQUEST_TYPE,
   FIELD_ELEMENTS_PER_BLOB,
   ForkName,
+  ForkPostElectra,
   ForkSeq,
   WITHDRAWAL_REQUEST_TYPE,
 } from "@lodestar/params";
@@ -23,6 +24,7 @@ import {
   deneb,
   gloas,
   ssz,
+  sszTypesFor,
 } from "@lodestar/types";
 import {BlobAndProof} from "@lodestar/types/deneb";
 import {BlobAndProofV2} from "@lodestar/types/fulu";
@@ -628,7 +630,7 @@ export function deserializeExecutionRequestsFromBytes(fork: ForkName, serialized
 
     switch (currentRequestType) {
       case DEPOSIT_REQUEST_TYPE: {
-        result.deposits = ssz.electra.DepositRequests.deserialize(requests);
+        result.deposits = sszTypesFor(fork as ForkPostElectra).DepositRequests.deserialize(requests);
         break;
       }
       case WITHDRAWAL_REQUEST_TYPE: {

package/src/metrics/metrics/lodestar.ts

@@ -645,6 +645,11 @@ export function createLodestarMetrics(
       help: "Count of peers by sync type [FullySynced, Advanced, Behind]",
       labelNames: ["syncType"],
     }),
+    syncPeerFinalizedCheckpointMismatch: register.counter<{reason: string}>({
+      name: "lodestar_sync_peer_finalized_checkpoint_mismatch_total",
+      help: "Total behind peers whose Status finalized checkpoint is not in our chain, by reason",
+      labelNames: ["reason"],
+    }),
     syncSwitchGossipSubscriptions: register.gauge<{action: string}>({
       name: "lodestar_sync_switch_gossip_subscriptions",
       help: "Sync switched gossip subscriptions on/off",
@@ -1008,6 +1013,11 @@ export function createLodestarMetrics(
         help: "Time elapsed between block received and fully verified state, signatures and payload",
         buckets: [0.05, 0.1, 0.3, 0.5, 0.7, 1, 1.3, 1.6, 2, 2.5, 3, 3.5, 4],
       }),
+      receivedToConsensusVerifiedTime: register.histogram({
+        name: "lodestar_gossip_block_received_to_consensus_verified_seconds",
+        help: "Time from gossip block arrival until both the state transition and all block signature verifications are complete, excluding execution payload verification",
+        buckets: [0.01, 0.015, 0.02, 0.025, 0.03, 0.04, 0.5, 1],
+      }),
       verifiedToBlobsAvailabiltyTime: register.histogram<{numBlobs: number}>({
         name: "lodestar_gossip_block_verified_to_blobs_availability_time",
         help: "Time elapsed between block verified and blobs became available",

package/src/metrics/server/http.ts

@@ -44,16 +44,16 @@ export async function getHttpMetricsServer(
   ): Promise<void> {
     if (req.method === "GET" && req.url && req.url.includes("/metrics")) {
       const timer = scrapeTimeMetric.startTimer();
-      const metricsRes = await Promise.all([wrapError(register.metrics()), getOtherMetrics()]);
-      timer({status: metricsRes[0].err ? RequestStatus.error : RequestStatus.success});
+      const metricsRes = await wrapError(Promise.all([register.metrics(), getOtherMetrics()]));
+      timer({status: metricsRes.err ? RequestStatus.error : RequestStatus.success});
 
       // Ensure we only writeHead once
-      if (metricsRes[0].err) {
-        res.writeHead(500, {"content-type": "text/plain"}).end(metricsRes[0].err.stack);
+      if (metricsRes.err) {
+        res.writeHead(500, {"content-type": "text/plain"}).end(metricsRes.err.stack);
       } else {
         // Get scrape time metrics
         const httpServerMetrics = await httpServerRegister.metrics();
-        const metrics = [metricsRes[0].result, httpServerMetrics, ...metricsRes[1]];
+        const metrics = [metricsRes.result[0], httpServerMetrics, ...metricsRes.result[1]];
         const metricsStr = metrics.join("\n\n");
         res.writeHead(200, {"content-type": register.contentType}).end(metricsStr);
       }

package/src/monitoring/service.ts

@@ -101,7 +101,7 @@ export class MonitoringService {
     if (this.monitoringInterval) {
       clearTimeout(this.monitoringInterval);
     }
-    if (this.pendingRequest) {
+    if (this.pendingRequest !== undefined) {
       this.fetchAbortController?.abort(FetchAbortReason.Close);
       await this.pendingRequest;
     }

package/src/network/core/networkCore.ts

@@ -10,6 +10,7 @@ import {isForkPostFulu} from "@lodestar/params";
 import {ResponseIncoming} from "@lodestar/reqresp";
 import {Epoch, Status, fulu, sszTypesFor} from "@lodestar/types";
 import {formatNodePeer} from "../../api/impl/node/utils.js";
+import {GoodByeReasonCode} from "../../constants/index.js";
 import {RegistryMetricCreator} from "../../metrics/index.js";
 import {ClockEvent, IClock} from "../../util/clock.js";
 import {CustodyConfig} from "../../util/dataColumns.js";
@@ -332,6 +333,10 @@ export class NetworkCore implements INetworkCore {
     this.peerManager.reStatusPeers(peers);
   }
 
+  async goodbyeAndDisconnect(peer: PeerIdStr, reason: GoodByeReasonCode): Promise<void> {
+    await this.peerManager.goodbyeAndDisconnect(peerIdFromString(peer), reason);
+  }
+
   /**
    * Request att subnets up `toSlot`. Network will ensure to mantain some peers for each
    */

package/src/network/core/networkCoreWorker.ts

@@ -138,6 +138,7 @@ const libp2pWorkerApi: NetworkWorkerApi = {
   prepareSyncCommitteeSubnets: (subscriptions) => core.prepareSyncCommitteeSubnets(subscriptions),
   reportPeer: (peer, action, actionName) => core.reportPeer(peer, action, actionName),
   reStatusPeers: (peers) => core.reStatusPeers(peers),
+  goodbyeAndDisconnect: (peer, reason) => core.goodbyeAndDisconnect(peer, reason),
   subscribeGossipCoreTopics: () => core.subscribeGossipCoreTopics(),
   unsubscribeGossipCoreTopics: () => core.unsubscribeGossipCoreTopics(),
 

package/src/network/core/networkCoreWorkerHandler.ts

@@ -11,6 +11,7 @@ import type {LoggerNode} from "@lodestar/logger/node";
 import {ResponseIncoming, ResponseOutgoing} from "@lodestar/reqresp";
 import {Status} from "@lodestar/types";
 import {withTimeout} from "@lodestar/utils";
+import {GoodByeReasonCode} from "../../constants/index.js";
 import {Metrics} from "../../metrics/index.js";
 import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js";
 import {PeerIdStr, peerIdFromString} from "../../util/peerId.js";
@@ -205,6 +206,9 @@ export class WorkerNetworkCore implements INetworkCore {
   reStatusPeers(peers: PeerIdStr[]): Promise<void> {
     return this.getApi().reStatusPeers(peers);
   }
+  goodbyeAndDisconnect(peer: PeerIdStr, reason: GoodByeReasonCode): Promise<void> {
+    return this.getApi().goodbyeAndDisconnect(peer, reason);
+  }
   reportPeer(peer: PeerIdStr, action: PeerAction, actionName: string): Promise<void> {
     return this.getApi().reportPeer(peer, action, actionName);
   }

package/src/network/core/types.ts

@@ -5,6 +5,7 @@ import {SpecJson} from "@lodestar/config";
 import {LoggerNodeOpts} from "@lodestar/logger/node";
 import {ResponseIncoming} from "@lodestar/reqresp";
 import {Status} from "@lodestar/types";
+import {GoodByeReasonCode} from "../../constants/index.js";
 import {PeerIdStr} from "../../util/peerId.js";
 import {NetworkOptions} from "../options.js";
 import {PeerAction, PeerScoreStats} from "../peers/index.js";
@@ -53,6 +54,7 @@ export interface INetworkCore extends INetworkCorePublic {
   // Sync method
   reportPeer(peer: PeerIdStr, action: PeerAction, actionName: string): void;
   reStatusPeers(peers: PeerIdStr[]): Promise<void>;
+  goodbyeAndDisconnect(peer: PeerIdStr, reason: GoodByeReasonCode): Promise<void>;
 
   // TODO: Duplicated methods with INetwork interface
   getConnectedPeers(): Promise<PeerIdStr[]>;

package/src/network/interface.ts

@@ -37,6 +37,7 @@ import {
   phase0,
 } from "@lodestar/types";
 import {BlockInputSource} from "../chain/blocks/blockInput/types.js";
+import {GoodByeReasonCode} from "../constants/index.js";
 import {CustodyConfig} from "../util/dataColumns.js";
 import {PeerIdStr} from "../util/peerId.js";
 import {
@@ -74,6 +75,7 @@ export interface INetwork extends INetworkCorePublic {
   reportPeer(peer: PeerIdStr, action: PeerAction, actionName: string): void;
   shouldAggregate(subnet: SubnetID, slot: Slot): boolean;
   reStatusPeers(peers: PeerIdStr[]): Promise<void>;
+  goodbyeAndDisconnect(peer: PeerIdStr, reason: GoodByeReasonCode): Promise<void>;
   searchUnknownBlock(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void;
   searchUnknownEnvelope(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void;
   // ReqResp

package/src/network/libp2p/index.ts

@@ -150,7 +150,11 @@ export async function createNodeJsLibp2p(
     nodeInfo: {
       name: "lodestar",
       version: networkOpts.version ?? "unknown",
-      userAgent: networkOpts.private ? "" : networkOpts.version ? `lodestar/${networkOpts.version}` : "lodestar",
+      userAgent: networkOpts.private
+        ? ""
+        : networkOpts.advertiseVersion && networkOpts.version
+          ? `lodestar/${networkOpts.version}`
+          : "lodestar",
     },
     addresses: {
       listen: localMultiaddrs,

package/src/network/network.ts

@@ -34,6 +34,7 @@ import {prettyPrintIndices, sleep} from "@lodestar/utils";
 import {BlockInputSource} from "../chain/blocks/blockInput/types.js";
 import {ChainEvent, IBeaconChain} from "../chain/index.js";
 import {computeSubnetForDataColumnSidecar} from "../chain/validation/dataColumnSidecar.js";
+import {GoodByeReasonCode} from "../constants/index.js";
 import {IBeaconDb} from "../db/interface.js";
 import {Metrics, RegistryMetricCreator} from "../metrics/index.js";
 import {IClock} from "../util/clock.js";
@@ -289,6 +290,10 @@ export class Network implements INetwork {
     return this.core.reStatusPeers(peers);
   }
 
+  async goodbyeAndDisconnect(peer: PeerIdStr, reason: GoodByeReasonCode): Promise<void> {
+    return this.core.goodbyeAndDisconnect(peer, reason);
+  }
+
   searchUnknownBlock(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void {
     this.networkProcessor.searchUnknownBlock(slotRoot, source, peer);
   }
@@ -855,7 +860,9 @@ export class Network implements INetwork {
   };
 
   private onTargetGroupCountUpdated = (count: number): void => {
-    this.core.setTargetGroupCount(count);
+    this.core.setTargetGroupCount(count).catch((e) => {
+      this.logger.debug("Error setting target custody group count", {count}, e as Error);
+    });
   };
 
   private onPublishDataColumns = async (sidecars: DataColumnSidecar[]): Promise<void> => {

package/src/network/options.ts

@@ -25,6 +25,8 @@ export interface NetworkOptions
   mdns?: boolean;
   connectToDiscv5Bootnodes?: boolean;
   version?: string;
+  /** Include the version in libp2p Identify. Private mode takes precedence. */
+  advertiseVersion?: boolean;
   private?: boolean;
   useWorker?: boolean;
   maxYoungGenerationSizeMb?: number;

package/src/network/peers/peerManager.ts

@@ -893,7 +893,7 @@ export class PeerManager {
     }
   }
 
-  private async goodbyeAndDisconnect(peer: PeerId, goodbye: GoodByeReasonCode): Promise<void> {
+  async goodbyeAndDisconnect(peer: PeerId, goodbye: GoodByeReasonCode): Promise<void> {
     const reason = GOODBYE_KNOWN_CODES[goodbye.toString()] || "";
     const peerIdStr = peer.toString();
     try {