@lodestar/beacon-node 1.51.0-dev.596284206a → 1.51.0-dev.6484138c50

package/lib/node/options.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/node/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAoB,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAC,WAAW,EAAE,oBAAoB,EAAE,aAAa,EAAsB,MAAM,qBAAqB,CAAC;AAC1G,OAAO,EAAC,oBAAoB,EAA8B,MAAM,8BAA8B,CAAC;AAC/F,OAAO,EAAC,eAAe,EAAmB,MAAM,kBAAkB,CAAC;AACnE,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,mBAAmB,EACnB,+BAA+B,EAE/B,8BAA8B,EAE9B,cAAc,EACf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAC,cAAc,EAAwB,MAAM,uBAAuB,CAAC;AAC5E,OAAO,EAAC,iBAAiB,EAA2B,MAAM,0BAA0B,CAAC;AACrF,OAAO,EAAC,cAAc,EAAwB,MAAM,uBAAuB,CAAC;AAC5E,OAAO,EAAC,WAAW,EAAqB,MAAM,oBAAoB,CAAC;AAGnE,OAAO,EAAC,aAAa,EAAC,MAAM,sBAAsB,CAAC;AACnD,YAAY,EAAC,aAAa,EAAC,CAAC;AAE5B,OAAO,EACL,WAAW,EACX,oBAAoB,EACpB,+BAA+B,EAC/B,8BAA8B,EAC9B,cAAc,GACf,CAAC;AAEF,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,UAAU,CAAC;IAChB,KAAK,EAAE,aAAa,CAAC;IACrB,EAAE,EAAE,eAAe,CAAC;IACpB,eAAe,EAAE,mBAAmB,CAAC;IACrC,gBAAgB,EAAE,oBAAoB,CAAC;IACvC,OAAO,EAAE,cAAc,CAAC;IACxB,gBAAgB,EAAE,oBAAoB,CAAC;IACvC,UAAU,EAAE,iBAAiB,CAAC;IAC9B,OAAO,EAAE,cAAc,CAAC;IACxB,IAAI,EAAE,WAAW,CAAC;CACnB;AAED,eAAO,MAAM,cAAc,EAAE,kBAW5B,CAAC"}
+{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/node/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,sCAAsC,EAAC,MAAM,gCAAgC,CAAC;AACtF,OAAO,EAAC,UAAU,EAAoB,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAC,WAAW,EAAE,oBAAoB,EAAE,aAAa,EAAsB,MAAM,qBAAqB,CAAC;AAC1G,OAAO,EAAC,oBAAoB,EAA8B,MAAM,8BAA8B,CAAC;AAC/F,OAAO,EAAC,eAAe,EAAmB,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAC,yBAAyB,EAAC,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,mBAAmB,EACnB,+BAA+B,EAE/B,8BAA8B,EAE9B,cAAc,EACf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAC,cAAc,EAAwB,MAAM,uBAAuB,CAAC;AAC5E,OAAO,EAAC,iBAAiB,EAA2B,MAAM,0BAA0B,CAAC;AACrF,OAAO,EAAC,cAAc,EAAwB,MAAM,uBAAuB,CAAC;AAC5E,OAAO,EAAC,WAAW,EAAqB,MAAM,oBAAoB,CAAC;AAGnE,OAAO,EAAC,aAAa,EAAC,MAAM,sBAAsB,CAAC;AACnD,YAAY,EAAC,aAAa,EAAC,CAAC;AAE5B,OAAO,EACL,WAAW,EACX,sCAAsC,EACtC,yBAAyB,EACzB,oBAAoB,EACpB,+BAA+B,EAC/B,8BAA8B,EAC9B,cAAc,GACf,CAAC;AAEF,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,UAAU,CAAC;IAChB,KAAK,EAAE,aAAa,CAAC;IACrB,EAAE,EAAE,eAAe,CAAC;IACpB,eAAe,EAAE,mBAAmB,CAAC;IACrC,gBAAgB,EAAE,oBAAoB,CAAC;IACvC,OAAO,EAAE,cAAc,CAAC;IACxB,gBAAgB,EAAE,oBAAoB,CAAC;IACvC,UAAU,EAAE,iBAAiB,CAAC;IAC9B,OAAO,EAAE,cAAc,CAAC;IACxB,IAAI,EAAE,WAAW,CAAC;CACnB;AAED,eAAO,MAAM,cAAc,EAAE,kBAW5B,CAAC"}

package/lib/node/options.js

@@ -1,7 +1,9 @@
+import { BLOCK_PRODUCTION_RACE_CUTOFF_GLOAS_BPS } from "../api/impl/validator/index.js";
 import { defaultApiOptions } from "../api/options.js";
 import { ArchiveMode, DEFAULT_ARCHIVE_MODE, defaultChainOptions } from "../chain/options.js";
 import { defaultValidatorMonitorOpts } from "../chain/validatorMonitor.js";
 import { defaultDbOptions } from "../db/options.js";
+import { BUILDER_BID_LAG_BUFFER_MS } from "../execution/builder/apiClient.js";
 import { defaultExecutionBuilderHttpOpts, defaultExecutionBuilderOpts, defaultExecutionEngineHttpOpts, defaultExecutionEngineOpts, engineApiModes, } from "../execution/index.js";
 import { defaultMetricsOptions } from "../metrics/options.js";
 import { defaultMonitoringOptions } from "../monitoring/options.js";
@@ -9,8 +11,8 @@ import { defaultNetworkOptions } from "../network/options.js";
 import { defaultSyncOptions } from "../sync/options.js";
 // Re-export so the CLI doesn't need to depend on lodestar-api
 export { allNamespaces } from "../api/rest/index.js";
-// Re-export to use as default values in CLI args
-export { ArchiveMode, DEFAULT_ARCHIVE_MODE, defaultExecutionBuilderHttpOpts, defaultExecutionEngineHttpOpts, engineApiModes, };
+// Re-export to use as default values and for validation in CLI args
+export { ArchiveMode, BLOCK_PRODUCTION_RACE_CUTOFF_GLOAS_BPS, BUILDER_BID_LAG_BUFFER_MS, DEFAULT_ARCHIVE_MODE, defaultExecutionBuilderHttpOpts, defaultExecutionEngineHttpOpts, engineApiModes, };
 export const defaultOptions = {
     api: defaultApiOptions,
     chain: defaultChainOptions,

package/lib/node/options.js.map

@@ -1 +1 @@
-{"version":3,"file":"options.js","sourceRoot":"","sources":["../../src/node/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAC,WAAW,EAAE,oBAAoB,EAAiB,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAC1G,OAAO,EAAuB,2BAA2B,EAAC,MAAM,8BAA8B,CAAC;AAC/F,OAAO,EAAkB,gBAAgB,EAAC,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAIL,+BAA+B,EAC/B,2BAA2B,EAC3B,8BAA8B,EAC9B,0BAA0B,EAC1B,cAAc,GACf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAiB,qBAAqB,EAAC,MAAM,uBAAuB,CAAC;AAC5E,OAAO,EAAoB,wBAAwB,EAAC,MAAM,0BAA0B,CAAC;AACrF,OAAO,EAAiB,qBAAqB,EAAC,MAAM,uBAAuB,CAAC;AAC5E,OAAO,EAAc,kBAAkB,EAAC,MAAM,oBAAoB,CAAC;AAEnE,8DAA8D;AAC9D,OAAO,EAAC,aAAa,EAAC,MAAM,sBAAsB,CAAC;AAEnD,iDAAiD;AACjD,OAAO,EACL,WAAW,EACX,oBAAoB,EACpB,+BAA+B,EAC/B,8BAA8B,EAC9B,cAAc,GACf,CAAC;AAeF,MAAM,CAAC,MAAM,cAAc,GAAuB;IAChD,GAAG,EAAE,iBAAiB;IACtB,KAAK,EAAE,mBAAmB;IAC1B,EAAE,EAAE,gBAAgB;IACpB,eAAe,EAAE,0BAA0B;IAC3C,gBAAgB,EAAE,2BAA2B;IAC7C,OAAO,EAAE,qBAAqB;IAC9B,gBAAgB,EAAE,2BAA2B;IAC7C,UAAU,EAAE,wBAAwB;IACpC,OAAO,EAAE,qBAAqB;IAC9B,IAAI,EAAE,kBAAkB;CACzB,CAAC"}
+{"version":3,"file":"options.js","sourceRoot":"","sources":["../../src/node/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,sCAAsC,EAAC,MAAM,gCAAgC,CAAC;AACtF,OAAO,EAAa,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAC,WAAW,EAAE,oBAAoB,EAAiB,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAC1G,OAAO,EAAuB,2BAA2B,EAAC,MAAM,8BAA8B,CAAC;AAC/F,OAAO,EAAkB,gBAAgB,EAAC,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAC,yBAAyB,EAAC,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAIL,+BAA+B,EAC/B,2BAA2B,EAC3B,8BAA8B,EAC9B,0BAA0B,EAC1B,cAAc,GACf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAiB,qBAAqB,EAAC,MAAM,uBAAuB,CAAC;AAC5E,OAAO,EAAoB,wBAAwB,EAAC,MAAM,0BAA0B,CAAC;AACrF,OAAO,EAAiB,qBAAqB,EAAC,MAAM,uBAAuB,CAAC;AAC5E,OAAO,EAAc,kBAAkB,EAAC,MAAM,oBAAoB,CAAC;AAEnE,8DAA8D;AAC9D,OAAO,EAAC,aAAa,EAAC,MAAM,sBAAsB,CAAC;AAEnD,oEAAoE;AACpE,OAAO,EACL,WAAW,EACX,sCAAsC,EACtC,yBAAyB,EACzB,oBAAoB,EACpB,+BAA+B,EAC/B,8BAA8B,EAC9B,cAAc,GACf,CAAC;AAeF,MAAM,CAAC,MAAM,cAAc,GAAuB;IAChD,GAAG,EAAE,iBAAiB;IACtB,KAAK,EAAE,mBAAmB;IAC1B,EAAE,EAAE,gBAAgB;IACpB,eAAe,EAAE,0BAA0B;IAC3C,gBAAgB,EAAE,2BAA2B;IAC7C,OAAO,EAAE,qBAAqB;IAC9B,gBAAgB,EAAE,2BAA2B;IAC7C,UAAU,EAAE,wBAAwB;IACpC,OAAO,EAAE,qBAAqB;IAC9B,IAAI,EAAE,kBAAkB;CACzB,CAAC"}

package/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.51.0-dev.596284206a",
+  "version": "1.51.0-dev.6484138c50",
   "type": "module",
   "exports": {
     ".": {
@@ -126,18 +126,18 @@
     "@libp2p/prometheus-metrics": "^5.0.31",
     "@libp2p/tcp": "^11.0.30",
     "@libp2p/utils": "^7.4.3",
-    "@lodestar/api": "^1.51.0-dev.596284206a",
-    "@lodestar/config": "^1.51.0-dev.596284206a",
-    "@lodestar/db": "^1.51.0-dev.596284206a",
-    "@lodestar/fork-choice": "^1.51.0-dev.596284206a",
-    "@lodestar/logger": "^1.51.0-dev.596284206a",
-    "@lodestar/native": "^1.51.0-dev.596284206a",
-    "@lodestar/params": "^1.51.0-dev.596284206a",
-    "@lodestar/reqresp": "^1.51.0-dev.596284206a",
-    "@lodestar/state-transition": "^1.51.0-dev.596284206a",
-    "@lodestar/types": "^1.51.0-dev.596284206a",
-    "@lodestar/utils": "^1.51.0-dev.596284206a",
-    "@lodestar/validator": "^1.51.0-dev.596284206a",
+    "@lodestar/api": "^1.51.0-dev.6484138c50",
+    "@lodestar/config": "^1.51.0-dev.6484138c50",
+    "@lodestar/db": "^1.51.0-dev.6484138c50",
+    "@lodestar/fork-choice": "^1.51.0-dev.6484138c50",
+    "@lodestar/logger": "^1.51.0-dev.6484138c50",
+    "@lodestar/native": "^1.51.0-dev.6484138c50",
+    "@lodestar/params": "^1.51.0-dev.6484138c50",
+    "@lodestar/reqresp": "^1.51.0-dev.6484138c50",
+    "@lodestar/state-transition": "^1.51.0-dev.6484138c50",
+    "@lodestar/types": "^1.51.0-dev.6484138c50",
+    "@lodestar/utils": "^1.51.0-dev.6484138c50",
+    "@lodestar/validator": "^1.51.0-dev.6484138c50",
     "@multiformats/multiaddr": "^13.0.1",
     "datastore-core": "^12.0.1",
     "datastore-level": "^13.0.1",
@@ -159,8 +159,8 @@
     "@ethereumjs/rlp": "^4.0.1",
     "@libp2p/interface-internal": "^3.2.0",
     "@libp2p/logger": "^6.2.15",
-    "@lodestar/spec-test-util": "^1.51.0-dev.596284206a",
-    "@lodestar/test-utils": "^1.51.0-dev.596284206a",
+    "@lodestar/spec-test-util": "^1.51.0-dev.6484138c50",
+    "@lodestar/test-utils": "^1.51.0-dev.6484138c50",
     "@types/js-yaml": "^4.0.5",
     "@types/qs": "^6.9.7",
     "@types/tmp": "^0.2.3",
@@ -174,5 +174,5 @@
     "beacon",
     "blockchain"
   ],
-  "gitHead": "999601160563b6a81973479fa641229c71afba55"
+  "gitHead": "52d9f70effc2ef20ef5f1beb3a6a453528223d9c"
 }

package/src/api/impl/validator/index.ts

@@ -94,7 +94,7 @@ import {
 import {validateGossipProposerPreferences} from "../../../chain/validation/proposerPreferences.js";
 import {validateSyncCommitteeGossipContributionAndProof} from "../../../chain/validation/syncCommitteeContributionAndProof.js";
 import {ZERO_HASH} from "../../../constants/index.js";
-import {BUILDER_BID_DEADLINE_MS, BuilderApiBid, decodeBuilderUrl} from "../../../execution/builder/apiClient.js";
+import {BuilderApiBid, decodeBuilderUrl} from "../../../execution/builder/apiClient.js";
 import {BuilderStatus, NoBidReceived} from "../../../execution/builder/http.js";
 import {getBuilderBidTotalGwei, validateBuilderApiExecutionPayloadBid} from "../../../execution/builder/validateBid.js";
 import {validateGossipFnRetryUnknownRoot} from "../../../network/processor/gossipHandlers.js";
@@ -123,10 +123,14 @@ import {
  * A cutoff of 1667 BPS (2 seconds for 12s slots) gives enough time and if there are unexpected delays
  * it ensures we publish in time as proposals after the attestation deadline will likely be orphaned
  * due to proposer boost reorg.
- *
- * TODO GLOAS: re-evaluate cutoff timing due to attestation deadline changes in gloas
  */
 const BLOCK_PRODUCTION_RACE_CUTOFF_BPS = 1667;
+/**
+ * Post-gloas cutoff, kept at 1667 BPS (2 seconds for 12s slots) for now.
+ *
+ * TODO GLOAS: revisit, the block propagates faster as it only commits to a bid and the attestation deadline changed
+ */
+export const BLOCK_PRODUCTION_RACE_CUTOFF_GLOAS_BPS = 1667;
 /** Rejection message of the bid block branch when there is no viable bid to commit to */
 const NO_BID_AVAILABLE = "No builder bid available";
 
@@ -1005,33 +1009,50 @@ export function getValidatorApi(
         (!circuitBreakerActive &&
           chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex) !== null);
 
-      // Select the p2p bid once builders had time to bid up, matching the deadline advertised
-      // on builder API bid requests, unless the circuit breaker is active
-      const p2pBidPromise: Promise<PooledExecutionPayloadBid | null> = circuitBreakerActive
-        ? Promise.resolve(null)
-        : sleep(Math.max(0, BUILDER_BID_DEADLINE_MS - chain.clock.msFromSlot(slot))).then(() => {
-            const p2pBid = chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
-            // Discard p2p bids below the proposer's configured floor on the total payment.
-            // A p2p bid's total is just its value since gossip validation enforces executionPayment=0.
-            if (p2pBid !== null && BigInt(p2pBid.signedBid.message.value) < builderConfig.minBid) {
-              logger.info("Best p2p bid below configured minimum", {
-                slot,
-                bidValue: prettyGweiToEth(p2pBid.signedBid.message.value),
-                minBid: prettyGweiToEth(builderConfig.minBid),
-              });
-              return null;
-            }
-            return p2pBid;
-          });
+      // Calculate cutoff time based on start of the slot, ensures a slow local payload build does
+      // not delay the proposal when a builder bid block is available (and vice versa)
+      const cutoffMs = Math.max(
+        0,
+        config.getSlotComponentDurationMs(fork, BLOCK_PRODUCTION_RACE_CUTOFF_GLOAS_BPS) - chain.clock.msFromSlot(slot)
+      );
+
+      // Give p2p bids the same time to bid up as granted to builder API bid requests via the advertised
+      // deadline, the pool is read once the deadline passed and the bid requests settled. A late request
+      // does not wait past the race cutoff, a pooled bid is still the fastest fallback if the local block fails
+      const bidDeadlinePromise = circuitBreakerActive
+        ? Promise.resolve()
+        : sleep(Math.min(chain.builderApiClient.bidDeadline, cutoffMs));
 
       // Candidates are ranked by their boosted counted total payment, the p2p bid is governed
       // by the top-level factors and each builder API bid by its own entry. Ties prefer the
       // earliest received builder API bid, so the signed block can be routed back to its
       // builder directly.
       const bestBidPromise: Promise<BidCandidate | null> = (async () => {
-        const [builderApiBids, p2pBid] = await Promise.all([builderApiBidsPromise, p2pBidPromise]);
+        // Past the race cutoff a pooled p2p bid is used right away rather than waiting for pending
+        // builder API requests, the local block is selected at the cutoff anyway if it is ready
+        let [builderApiBids] = await Promise.all([
+          Promise.race([builderApiBidsPromise, sleep(cutoffMs).then(() => null)]),
+          bidDeadlinePromise,
+        ]);
         let parentExecutionRequestsPromise: Promise<gloas.ExecutionRequests> | null = null;
 
+        let p2pBid: PooledExecutionPayloadBid | null = circuitBreakerActive
+          ? null
+          : chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
+        // Discard p2p bids below the proposer's configured floor on the total payment.
+        // A p2p bid's total is just its value since gossip validation enforces executionPayment=0.
+        if (p2pBid !== null && BigInt(p2pBid.signedBid.message.value) < builderConfig.minBid) {
+          logger.info("Best p2p bid below configured minimum", {
+            slot,
+            bidValue: prettyGweiToEth(p2pBid.signedBid.message.value),
+            minBid: prettyGweiToEth(builderConfig.minBid),
+          });
+          p2pBid = null;
+        }
+        if (builderApiBids === null) {
+          builderApiBids = p2pBid !== null ? [] : await builderApiBidsPromise;
+        }
+
         const candidates = (
           await Promise.all(
             builderApiBids.map(async ({url, entry, signedBid, receivedMs}): Promise<BidCandidate | null> => {
@@ -1118,13 +1139,6 @@ export function getValidatorApi(
         return fn().finally(() => t?.({source}));
       };
 
-      // Calculate cutoff time based on start of the slot, ensures a slow local payload build does
-      // not delay the proposal when a builder bid block is available (and vice versa)
-      const cutoffMs = Math.max(
-        0,
-        config.getSlotComponentDurationMs(fork, BLOCK_PRODUCTION_RACE_CUTOFF_BPS) - chain.clock.msFromSlot(slot)
-      );
-
       // use abort controller to stop waiting for the bid block if the engine block will be selected
       const controller = new AbortController();
 

package/src/chain/blocks/types.ts

@@ -109,7 +109,10 @@ export type FullyVerifiedBlock = {
   parentBlockSlot: Slot;
   proposerBalanceDelta: number;
   dataAvailabilityStatus: DataAvailabilityStatus;
-  /** Pre-computed indexed attestations from signature verification to avoid duplicate work */
+  /**
+   * Pre-computed indexed attestations from signature verification to avoid duplicate work.
+   * Their `attestingIndices` are in committee order, not sorted, so they are not spec-valid `IndexedAttestation`s.
+   */
   indexedAttestations: IndexedAttestation[];
   /** Seen timestamp seconds */
   seenTimestampSec: number;

package/src/chain/blocks/verifyBlock.ts

@@ -4,7 +4,7 @@ import {
   DataAvailabilityStatus,
   IBeaconStateView,
   computeEpochAtSlot,
-  getIndexedAttestation,
+  getAttestingIndices,
   signedBlockToSignedHeader,
 } from "@lodestar/state-transition";
 import {IndexedAttestation, Slot, deneb} from "@lodestar/types";
@@ -119,7 +119,13 @@ export async function verifyBlocksInEpoch(
         const shuffling =
           this.shufflingCache.getSync(attEpoch, preState0.getShufflingDecisionRoot(attEpoch)) ??
           preState0.getShufflingAtEpoch(attEpoch);
-        return getIndexedAttestation(shuffling, fork, attestation);
+        // Unlike getIndexedAttestation, keep attestingIndices in committee order: sorting them dominates building
+        // these on mainnet blocks. See FullyVerifiedBlock.indexedAttestations.
+        return {
+          attestingIndices: getAttestingIndices(shuffling, fork, attestation),
+          data: attestation.data,
+          signature: attestation.signature,
+        };
       });
     }
 
@@ -175,16 +181,29 @@ export async function verifyBlocksInEpoch(
     const [
       segmentExecStatus,
       {blockDAStatuses, payloadDAStatuses, availableTime},
-      {postStates, proposerBalanceDeltas, verifyStateTime},
       {verifySignaturesTime},
+      {postStates, proposerBalanceDeltas, verifyStateTime},
     ] = await Promise.all([
       verifyExecutionPayloadsPromise,
 
       // data availability (fork-specific; see daAvailabilityPromise above)
       daAvailabilityPromise,
 
+      // All signatures at once. Must be called before verifyBlocksStateTransitionOnly()
+      // so we hand the BLS job to the worker pool before the main thread
+      // is busy with the state transition.
+      verifyBlocksSignatures(
+        this.config,
+        this.bls,
+        this.logger,
+        this.metrics,
+        preState0,
+        blocks,
+        indexedAttestationsByBlock,
+        opts
+      ),
+
       // Run state transition only
-      // TODO: Ensure it yields to allow flushing to workers and engine API
       verifyBlocksStateTransitionOnly(
         preState0,
         blockInputs,
@@ -198,22 +217,17 @@ export async function verifyBlocksInEpoch(
         opts
       ),
 
-      // All signatures at once
-      verifyBlocksSignatures(
-        this.config,
-        this.bls,
-        this.logger,
-        this.metrics,
-        preState0,
-        blocks,
-        indexedAttestationsByBlock,
-        opts
-      ),
-
       // TODO GLOAS: can verify payload signatures in batch too
       // maybe chain with the above verifyBlocksSignatures()
     ]);
 
+    if (blockInputs.length === 1 && opts.seenTimestampSec !== undefined) {
+      const consensusVerifiedTime = Math.max(verifyStateTime, verifySignaturesTime);
+      this.metrics?.gossipBlock.receivedToConsensusVerifiedTime.observe(
+        consensusVerifiedTime / 1000 - opts.seenTimestampSec
+      );
+    }
+
     for (const blockInput of blockInputs) {
       const block = blockInput.getBlock();
       const {slot, proposerIndex} = block.message;

package/src/chain/blocks/verifyBlocksSignatures.ts

@@ -36,10 +36,9 @@ export async function verifyBlocksSignatures(
     ? preState0.currentSyncCommitteeIndexed
     : new SyncCommitteeCacheEmpty();
 
-  // Verifies signatures after running state transition, so all SyncCommittee signed roots are known at this point.
-  // We must ensure block.slot <= state.slot before running getAllBlockSignatureSets().
-  // NOTE: If in the future multiple blocks signatures are verified at once, all blocks must be in the same epoch
-  // so the attester and proposer shufflings are correct.
+  // Runs concurrently with the state transition. Signature sets are built from preState0, which
+  // verifyBlocksInEpoch() guarantees is in the same epoch as every block, so the proposer and attester shufflings
+  // and the sync committee are the ones the blocks were signed under.
   for (const [i, block] of blocks.entries()) {
     // Use [i] to make clear that the index has to be correct to blame the right block below on BlockError()
     isValidPromises[i] = opts.validSignatures
@@ -50,7 +49,10 @@ export async function verifyBlocksSignatures(
         bls.verifySignatureSets(
           getBlockSignatureSets(config, currentSyncCommitteeIndexed, preState0, block, indexedAttestationsByBlock[i], {
             skipProposerSignature: opts.validProposerSignature,
-          })
+          }),
+          // Prioritize block verification over attestations, since
+          // attestations already buffer up to `MAX_BUFFER_WAIT_MS`
+          {priority: true}
         );
 
     // getBlockSignatureSets() takes 45ms in benchmarks for 2022Q2 mainnet blocks (100 sigs). When syncing a 32 blocks

package/src/chain/bls/interface.ts

@@ -22,6 +22,8 @@ export type VerifySignatureOpts = {
   verifyOnMainThread?: boolean;
   /**
    * Some signature sets are more important than others, and should be verified first.
+   * Non-batchable priority jobs are posted to an idle worker before this call returns, so the caller
+   * can run synchronous work right after without delaying the verification.
    */
   priority?: boolean;
 };

package/src/chain/bls/multithread/index.ts

@@ -333,15 +333,19 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
       }
     }
 
+    // First hand work to an idle worker before returning to the caller.
+    // Deferring the post to the next macrotask would make the worker
+    // sit idle while waiting for the next synchronous work
+    else if (job.opts.priority) {
+      this.jobs.unshift(job);
+      void this.runJob();
+    }
+
     // Push job and schedule to call `runJob` in the next macro event loop cycle.
     // This is useful to allow batching job submitted from a synchronous for loop,
     // and to prevent large stacks since runJob may be called recursively.
     else {
-      if (job.opts.priority) {
-        this.jobs.unshift(job);
-      } else {
-        this.jobs.push(job);
-      }
+      this.jobs.push(job);
       callInNextEventLoop(() => void this.runJob());
     }
   }

package/src/execution/builder/apiClient.ts

@@ -19,21 +19,20 @@ import {IClock} from "../../util/clock.js";
  * Additional duration to account for potential event loop lag which causes
  * builder bids to be rejected even though the response was sent in time.
  */
-const EVENT_LOOP_LAG_BUFFER = 250;
+export const BUILDER_BID_LAG_BUFFER_MS = 250;
 
 /**
- * Duration given to a builder to provide a `SignedExecutionPayloadBid` before the deadline
- * is reached, advertised on each bid request via the `X-Timeout-Ms` header. The p2p bid is
- * selected after the same duration, only considering bids received up to that point.
+ * Default duration given to a builder to provide a `SignedExecutionPayloadBid` before the deadline
+ * is reached, advertised on each bid request via the `X-Timeout-Ms` header. Bids received via
+ * p2p are given the same time to bid up.
  */
-export const BUILDER_BID_DEADLINE_MS = 500;
-
-/** Local timeout for bid requests, event loop lag must not discard bids that arrived in time */
-export const BUILDER_BID_REQUEST_TIMEOUT_MS = BUILDER_BID_DEADLINE_MS + EVENT_LOOP_LAG_BUFFER;
+export const DEFAULT_BUILDER_BID_DEADLINE_MS = 500;
 
 export type BuilderApiClientOpts = {
-  /** Timeout for builder api requests, bid requests always use `BUILDER_BID_REQUEST_TIMEOUT_MS` */
+  /** Timeout for builder api requests, bid requests are bounded by `bidDeadline` instead */
   timeout?: number;
+  /** Time builders are given to provide a bid once block production starts, in milliseconds */
+  bidDeadline?: number;
   // Add User-Agent header to all requests
   userAgent?: string;
 };
@@ -69,6 +68,7 @@ export type BuilderApiBid = {
  * each block production request, clients are dialed on demand based on the entry `url`.
  */
 export class BuilderApiClient {
+  readonly bidDeadline: number;
   private readonly clients = new Map<BuilderUrl, BuilderApi>();
 
   constructor(
@@ -78,7 +78,9 @@ export class BuilderApiClient {
     private readonly bls: IBlsVerifier,
     private readonly metrics: Metrics | null = null,
     private readonly logger?: Logger
-  ) {}
+  ) {
+    this.bidDeadline = opts.bidDeadline ?? DEFAULT_BUILDER_BID_DEADLINE_MS;
+  }
 
   /**
    * Fan out bid requests to the builders named by the entries, one request per unique
@@ -139,9 +141,10 @@ export class BuilderApiClient {
               proposerPubkey,
               requestAuth: entry.auth,
               dateMilliseconds: Date.now(),
-              timeoutMs: BUILDER_BID_DEADLINE_MS,
+              timeoutMs: this.bidDeadline,
             },
-            {timeoutMs: BUILDER_BID_REQUEST_TIMEOUT_MS}
+            // Event loop lag must not discard bids that arrived in time
+            {timeoutMs: this.bidDeadline + BUILDER_BID_LAG_BUFFER_MS}
           );
           const signedBid = res.value();
           if (signedBid === undefined) {
@@ -203,7 +206,7 @@ export class BuilderApiClient {
 
   /**
    * Establish the connection to every known builder outside the bid deadline, a cold TCP/TLS handshake
-   * to a distant builder can exhaust `BUILDER_BID_DEADLINE_MS` on its own, and a long idle connection
+   * to a distant builder can exhaust the bid deadline on its own, and a long idle connection
    * may be half-open and only fail once written to.
    */
   async checkStatus(): Promise<void> {

package/src/execution/builder/http.ts

@@ -20,6 +20,7 @@ import {
 } from "@lodestar/types";
 import {toPrintableUrl} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
+import {DEFAULT_BUILDER_BID_DEADLINE_MS} from "./apiClient.js";
 import {ValidatorRegistration, ValidatorRegistrationCache} from "./cache.js";
 import {IExecutionBuilder} from "./interface.js";
 
@@ -29,6 +30,8 @@ export type ExecutionBuilderHttpOpts = {
   timeout?: number;
   faultInspectionWindow?: number;
   allowedFaults?: number;
+  /** Post-gloas only, time builders are given to provide a bid once block production starts, in milliseconds */
+  bidDeadline?: number;
 
   // Only required for merge-mock runs, no need to expose it to cli
   issueLocalFcUWithFeeRecipient?: string;
@@ -40,6 +43,7 @@ export const defaultExecutionBuilderHttpOpts: ExecutionBuilderHttpOpts = {
   enabled: false,
   url: "http://localhost:8661",
   timeout: 12000,
+  bidDeadline: DEFAULT_BUILDER_BID_DEADLINE_MS,
 };
 
 export enum BuilderStatus {

package/src/execution/engine/utils.ts

@@ -169,14 +169,16 @@ function getExecutionEngineStateForPayloadStatus(payloadStatus: ExecutionPayload
       return ExecutionEngineState.SYNCED;
 
     case ExecutionPayloadStatus.ELERROR:
-    case ExecutionPayloadStatus.INVALID:
     case ExecutionPayloadStatus.SYNCING:
-    case ExecutionPayloadStatus.INVALID_BLOCK_HASH:
       return ExecutionEngineState.SYNCING;
 
     case ExecutionPayloadStatus.UNAVAILABLE:
       return ExecutionEngineState.OFFLINE;
 
+    case ExecutionPayloadStatus.INVALID:
+    case ExecutionPayloadStatus.INVALID_BLOCK_HASH:
+      return ExecutionEngineState.ONLINE;
+
     default:
       // In case we can't determine the state, we assume it stays in old state
       // This assumption is better than considering offline, because the offline state may trigger some notifications

package/src/metrics/metrics/lodestar.ts

@@ -1013,6 +1013,11 @@ export function createLodestarMetrics(
         help: "Time elapsed between block received and fully verified state, signatures and payload",
         buckets: [0.05, 0.1, 0.3, 0.5, 0.7, 1, 1.3, 1.6, 2, 2.5, 3, 3.5, 4],
       }),
+      receivedToConsensusVerifiedTime: register.histogram({
+        name: "lodestar_gossip_block_received_to_consensus_verified_seconds",
+        help: "Time from gossip block arrival until both the state transition and all block signature verifications are complete, excluding execution payload verification",
+        buckets: [0.01, 0.015, 0.02, 0.025, 0.03, 0.04, 0.5, 1],
+      }),
       verifiedToBlobsAvailabiltyTime: register.histogram<{numBlobs: number}>({
         name: "lodestar_gossip_block_verified_to_blobs_availability_time",
         help: "Time elapsed between block verified and blobs became available",

package/src/metrics/server/http.ts

@@ -44,16 +44,16 @@ export async function getHttpMetricsServer(
   ): Promise<void> {
     if (req.method === "GET" && req.url && req.url.includes("/metrics")) {
       const timer = scrapeTimeMetric.startTimer();
-      const metricsRes = await Promise.all([wrapError(register.metrics()), getOtherMetrics()]);
-      timer({status: metricsRes[0].err ? RequestStatus.error : RequestStatus.success});
+      const metricsRes = await wrapError(Promise.all([register.metrics(), getOtherMetrics()]));
+      timer({status: metricsRes.err ? RequestStatus.error : RequestStatus.success});
 
       // Ensure we only writeHead once
-      if (metricsRes[0].err) {
-        res.writeHead(500, {"content-type": "text/plain"}).end(metricsRes[0].err.stack);
+      if (metricsRes.err) {
+        res.writeHead(500, {"content-type": "text/plain"}).end(metricsRes.err.stack);
       } else {
         // Get scrape time metrics
         const httpServerMetrics = await httpServerRegister.metrics();
-        const metrics = [metricsRes[0].result, httpServerMetrics, ...metricsRes[1]];
+        const metrics = [metricsRes.result[0], httpServerMetrics, ...metricsRes.result[1]];
         const metricsStr = metrics.join("\n\n");
         res.writeHead(200, {"content-type": register.contentType}).end(metricsStr);
       }

package/src/network/libp2p/index.ts

@@ -150,7 +150,11 @@ export async function createNodeJsLibp2p(
     nodeInfo: {
       name: "lodestar",
       version: networkOpts.version ?? "unknown",
-      userAgent: networkOpts.private ? "" : networkOpts.version ? `lodestar/${networkOpts.version}` : "lodestar",
+      userAgent: networkOpts.private
+        ? ""
+        : networkOpts.advertiseVersion && networkOpts.version
+          ? `lodestar/${networkOpts.version}`
+          : "lodestar",
     },
     addresses: {
       listen: localMultiaddrs,

package/src/network/options.ts

@@ -25,6 +25,8 @@ export interface NetworkOptions
   mdns?: boolean;
   connectToDiscv5Bootnodes?: boolean;
   version?: string;
+  /** Include the version in libp2p Identify. Private mode takes precedence. */
+  advertiseVersion?: boolean;
   private?: boolean;
   useWorker?: boolean;
   maxYoungGenerationSizeMb?: number;

package/src/node/nodejs.ts

@@ -20,7 +20,7 @@ import {
   isStatePostGloas,
 } from "@lodestar/state-transition";
 import {Slot} from "@lodestar/types";
-import {sleep, toRootHex} from "@lodestar/utils";
+import {defer, sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
 import {BeaconRestApiServer, getApi} from "../api/index.js";
 import {getDataColumnSidecarsMinEpoch} from "../chain/archiveStore/utils/archiveBlocks.js";
@@ -117,6 +117,7 @@ export class BeaconNode {
 
   status: BeaconNodeStatus;
   private controller?: AbortController;
+  private closePromise: Promise<void> | undefined;
 
   constructor({
     opts,
@@ -297,6 +298,7 @@ export class BeaconNode {
         : undefined,
       builderApiClientOpts: {
         timeout: opts.executionBuilder.timeout,
+        bidDeadline: opts.executionBuilder.bidDeadline,
         // Sent with all builder api requests, unless the node runs in private mode
         userAgent: opts.executionBuilder.userAgent,
       },
@@ -392,23 +394,40 @@ export class BeaconNode {
   /**
    * Stop beacon node and its sub-components.
    */
-  async close(): Promise<void> {
-    if (this.status === BeaconNodeStatus.started) {
-      this.status = BeaconNodeStatus.closing;
-      this.sync.close();
-      if (this.restApi) await this.restApi.close();
-      await this.network.close();
-      if (this.metricsServer) await this.metricsServer.close();
-      if (this.monitoring) await this.monitoring.close();
-      await this.chain.persistToDisk();
-      await this.chain.close();
+  close(): Promise<void> {
+    if (this.closePromise !== undefined) return this.closePromise;
+    const completion = defer<void>();
+    this.closePromise = completion.promise;
+    this.status = BeaconNodeStatus.closing;
+    void this.closeInternal().then(completion.resolve, completion.reject);
+    return this.closePromise;
+  }
+
+  private async closeInternal(): Promise<void> {
+    const errors: unknown[] = [];
+    for (const cleanup of [
+      () => this.sync.close(),
+      () => this.restApi?.close(),
+      () => this.network.close(),
+      () => this.metricsServer?.close(),
+      () => this.monitoring?.close(),
+      () => this.chain.persistToDisk(),
+      () => this.chain.close(),
       // Abort signal last: close() calls above clear intervals/timeouts so no new
       // operations get scheduled. If we aborted first, a still-pending interval could
       // fire and schedule a new operation after abort, leaving it stuck and delaying shutdown.
-      if (this.controller) this.controller.abort();
-      await sleep(DELAY_BEFORE_CLOSING_DB_MS);
-      await this.db.close();
-      this.status = BeaconNodeStatus.closed;
+      () => this.controller?.abort(),
+      () => sleep(DELAY_BEFORE_CLOSING_DB_MS),
+      () => this.db.close(),
+    ]) {
+      try {
+        await cleanup();
+      } catch (error) {
+        errors.push(error);
+      }
     }
+    this.status = BeaconNodeStatus.closed;
+    if (errors.length === 1) throw errors[0];
+    if (errors.length > 1) throw new AggregateError(errors, "Beacon node cleanup failed");
   }
 }