@lodestar/beacon-node 1.47.0-dev.931ecbec99 → 1.47.0-dev.9ba9a5ce85

package/src/execution/builder/apiClient.ts

@@ -0,0 +1,264 @@
+import {routes} from "@lodestar/api";
+import {ApiClient as BuilderApi, getClient} from "@lodestar/api/builder";
+import {ChainForkConfig} from "@lodestar/config";
+import {Logger} from "@lodestar/logger";
+import {DOMAIN_BUILDER_REQUEST_AUTH, ForkPostGloas} from "@lodestar/params";
+import {
+  ZERO_HASH,
+  computeDomain,
+  computeSigningRoot,
+  createSingleSignatureSetFromComponents,
+} from "@lodestar/state-transition";
+import {BLSPubkey, Root, SignedBeaconBlock, Slot, WithOptionalBytes, gloas, ssz} from "@lodestar/types";
+import {isValidAsciiHttpUrl, toHex, toPrintableUrl} from "@lodestar/utils";
+import type {IBlsVerifier} from "../../chain/bls/index.js";
+import {Metrics} from "../../metrics/metrics.js";
+import {IClock} from "../../util/clock.js";
+
+/**
+ * Additional duration to account for potential event loop lag which causes
+ * builder bids to be rejected even though the response was sent in time.
+ */
+const EVENT_LOOP_LAG_BUFFER = 250;
+
+/**
+ * Duration given to a builder to provide a `SignedExecutionPayloadBid` before the deadline
+ * is reached, advertised on each bid request via the `X-Timeout-Ms` header. The p2p bid is
+ * selected after the same duration, only considering bids received up to that point.
+ */
+export const BUILDER_BID_DEADLINE_MS = 500;
+
+/** Local timeout for bid requests, event loop lag must not discard bids that arrived in time */
+export const BUILDER_BID_REQUEST_TIMEOUT_MS = BUILDER_BID_DEADLINE_MS + EVENT_LOOP_LAG_BUFFER;
+
+export type BuilderApiClientOpts = {
+  /** Timeout for builder api requests, bid requests always use `BUILDER_BID_REQUEST_TIMEOUT_MS` */
+  timeout?: number;
+  // Add User-Agent header to all requests
+  userAgent?: string;
+};
+
+type BuilderUrl = string;
+
+/** Decode the SSZ URL bytes without allowing replacement characters or unsafe header values. */
+export function decodeBuilderUrl(value: Uint8Array): BuilderUrl {
+  let url: string;
+  try {
+    url = new TextDecoder("utf8", {fatal: true}).decode(value);
+  } catch {
+    throw Error("Builder url must be valid UTF-8");
+  }
+  if (!isValidAsciiHttpUrl(url)) {
+    throw Error("Invalid builder url");
+  }
+  return url;
+}
+
+export type BuilderApiBid = {
+  url: BuilderUrl;
+  entry: routes.validator.BuilderEntry;
+  signedBid: gloas.SignedExecutionPayloadBid;
+  /** Time in milliseconds from the slot start when the bid was received */
+  receivedMs: number;
+};
+
+/**
+ * External builder integration post-gloas
+ *
+ * The builder set is driven by the resolved `BuilderConfig` the validator client supplies on
+ * each block production request, clients are dialed on demand based on the entry `url`.
+ */
+export class BuilderApiClient {
+  private readonly clients = new Map<BuilderUrl, BuilderApi>();
+
+  constructor(
+    private readonly opts: BuilderApiClientOpts,
+    private readonly config: ChainForkConfig,
+    private readonly clock: IClock,
+    private readonly bls: IBlsVerifier,
+    private readonly metrics: Metrics | null = null,
+    private readonly logger?: Logger
+  ) {}
+
+  /**
+   * Fan out bid requests to the builders named by the entries, one request per unique
+   * `(url, auth.message.data)` pair. Errors and empty responses (204) are logged and filtered
+   * out so a single unresponsive builder never fails block production.
+   */
+  async getExecutionPayloadBids(
+    entries: routes.validator.BuilderEntry[],
+    slot: Slot,
+    parentHash: Root,
+    parentRoot: Root,
+    proposerPubkey: BLSPubkey
+  ): Promise<BuilderApiBid[]> {
+    const seenRequests = new Set<string>();
+    const requests: {url: BuilderUrl; entry: routes.validator.BuilderEntry}[] = [];
+
+    for (const entry of entries) {
+      const urlForLog = Buffer.from(entry.url).toString("utf8");
+      let url: BuilderUrl;
+      try {
+        url = decodeBuilderUrl(entry.url);
+      } catch {
+        this.logger?.warn("Ignoring builder entry with invalid url", {slot, url: urlForLog});
+        continue;
+      }
+
+      const requestKey = `${url}-${toHex(entry.auth.message.data)}`;
+      if (seenRequests.has(requestKey)) {
+        continue;
+      }
+      seenRequests.add(requestKey);
+
+      // The builder rejects a mismatch, an entry naming a different slot is not used for a bid request
+      if (entry.auth.message.slot !== slot) {
+        this.logger?.warn("Ignoring builder entry with auth for different slot", {
+          slot,
+          authSlot: entry.auth.message.slot,
+          builder: toPrintableUrl(url),
+        });
+        continue;
+      }
+
+      requests.push({url, entry});
+    }
+
+    // Collected in arrival order so an earlier received bid wins ties during candidate ranking
+    const bids: BuilderApiBid[] = [];
+    await Promise.all(
+      requests.map(async ({url, entry}) => {
+        this.metrics?.builderApi.bidRequests.inc();
+        try {
+          const client = await this.getOrCreateClient(url, proposerPubkey, entry.auth);
+          const res = await client.getExecutionPayloadBid(
+            {
+              slot,
+              parentHash,
+              parentRoot,
+              proposerPubkey,
+              requestAuth: entry.auth,
+              dateMilliseconds: Date.now(),
+              timeoutMs: BUILDER_BID_DEADLINE_MS,
+            },
+            {timeoutMs: BUILDER_BID_REQUEST_TIMEOUT_MS}
+          );
+          const signedBid = res.value();
+          if (signedBid === undefined) {
+            this.logger?.debug("No bid received from builder", {slot, builder: toPrintableUrl(url)});
+            return;
+          }
+          this.metrics?.builderApi.bidsReceived.inc();
+          bids.push({url, entry, signedBid, receivedMs: this.clock.msFromSlot(slot)});
+        } catch (e) {
+          this.metrics?.builderApi.bidRequestErrors.inc();
+          this.logger?.warn("Failed to get bid from builder", {slot, builder: toPrintableUrl(url)}, e as Error);
+        }
+      })
+    );
+
+    return bids;
+  }
+
+  /** Forward a proposer's builder preferences to the builder at the given url */
+  async submitBuilderPreferences(
+    url: BuilderUrl,
+    proposerPubkey: BLSPubkey,
+    request: gloas.BuilderPreferencesRequest
+  ): Promise<void> {
+    try {
+      const client = await this.getOrCreateClient(url, proposerPubkey, request.auth);
+      (await client.submitBuilderPreferences({proposerPubkey, request})).assertOk();
+      this.metrics?.builderApi.preferencesForwarded.inc({status: "success"});
+    } catch (e) {
+      this.metrics?.builderApi.preferencesForwarded.inc({status: "error"});
+      throw e;
+    }
+  }
+
+  /**
+   * Submit the signed beacon block to the builder whose bid was included. The builder is
+   * then responsible for constructing and broadcasting the corresponding
+   * `SignedExecutionPayloadEnvelope`, no further action is required by the proposer.
+   */
+  async submitSignedBeaconBlock(
+    url: BuilderUrl,
+    signedBlock: WithOptionalBytes<SignedBeaconBlock<ForkPostGloas>>
+  ): Promise<void> {
+    const client = this.clients.get(url);
+    if (client === undefined) {
+      this.metrics?.builderApi.blockSubmissions.inc({status: "error"});
+      this.logger?.warn("Ignoring signed block submission to unauthenticated builder", {url});
+      return;
+    }
+
+    try {
+      (await client.submitSignedBeaconBlock({signedBlock}, {redirect: "manual"})).assertOk();
+      this.metrics?.builderApi.blockSubmissions.inc({status: "success"});
+    } catch (e) {
+      this.metrics?.builderApi.blockSubmissions.inc({status: "error"});
+      throw e;
+    }
+  }
+
+  /**
+   * Establish the connection to every known builder outside the bid deadline, a cold TCP/TLS handshake
+   * to a distant builder can exhaust `BUILDER_BID_DEADLINE_MS` on its own, and a long idle connection
+   * may be half-open and only fail once written to.
+   */
+  async checkStatus(): Promise<void> {
+    await Promise.all(
+      Array.from(this.clients.entries()).map(async ([url, client]) => {
+        try {
+          (await client.status()).assertOk();
+          this.metrics?.builderApi.statusChecks.inc({status: "success"});
+          this.logger?.debug("Builder is ready", {builder: toPrintableUrl(url)});
+        } catch (e) {
+          this.metrics?.builderApi.statusChecks.inc({status: "error"});
+          this.logger?.warn("Builder status check failed", {builder: toPrintableUrl(url)}, e as Error);
+        }
+      })
+    );
+  }
+
+  private async getOrCreateClient(
+    url: BuilderUrl,
+    proposerPubkey: BLSPubkey,
+    auth: gloas.SignedBuilderRequestAuth
+  ): Promise<BuilderApi> {
+    if (auth.message.data.length === 0) {
+      throw Error("Builder request auth data must not be empty");
+    }
+
+    let client = this.clients.get(url);
+    if (client === undefined) {
+      const domain = computeDomain(DOMAIN_BUILDER_REQUEST_AUTH, this.config.GENESIS_FORK_VERSION, ZERO_HASH);
+      const signingRoot = computeSigningRoot(ssz.gloas.BuilderRequestAuth, auth.message, domain);
+      const signatureSet = createSingleSignatureSetFromComponents(proposerPubkey, signingRoot, auth.signature);
+
+      let isValid = false;
+      try {
+        isValid = await this.bls.verifySignatureSets([signatureSet]);
+      } catch {
+        // Malformed signatures are invalid request authentication.
+      }
+      if (!isValid) {
+        throw Error("Invalid builder request auth");
+      }
+
+      client = getClient(
+        {
+          baseUrl: url,
+          globalInit: {
+            timeoutMs: this.opts.timeout,
+            headers: this.opts.userAgent ? {"User-Agent": this.opts.userAgent} : undefined,
+          },
+        },
+        {config: this.config, metrics: this.metrics?.builderHttpClient, logger: this.logger}
+      );
+      this.clients.set(url, client);
+      this.logger?.debug("External builder registered", {url: toPrintableUrl(url)});
+    }
+    return client;
+  }
+}

package/src/execution/builder/validateBid.ts

@@ -0,0 +1,181 @@
+import {routes} from "@lodestar/api";
+import {ProtoBlock} from "@lodestar/fork-choice";
+import {MAX_EXECUTION_PAYMENT, PAYLOAD_BUILDER_VERSION} from "@lodestar/params";
+import {
+  computeEpochAtSlot,
+  createSingleSignatureSetFromComponents,
+  getExecutionPayloadBidSigningRoot,
+  isActiveBuilder,
+  isGasLimitTargetCompatible,
+  isStatePostGloas,
+} from "@lodestar/state-transition";
+import {RootHex, Slot, gloas} from "@lodestar/types";
+import {bigIntMin, byteArrayEquals, prettyGweiToEth, toHex, toRootHex} from "@lodestar/utils";
+import {IBeaconChain} from "../../chain/index.js";
+import {RegenCaller} from "../../chain/regen/index.js";
+import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
+
+/** Return the counted bid payment in Gwei, saturated at uint64 max. */
+export function getBuilderBidTotalGwei(bid: gloas.ExecutionPayloadBid, maxExecutionPayment: bigint): bigint {
+  return bigIntMin(MAX_EXECUTION_PAYMENT, BigInt(bid.value) + bigIntMin(bid.executionPayment, maxExecutionPayment));
+}
+
+/**
+ * Validate a bid received from a builder over the builder API in response to a bid request
+ * made during block production. Unlike gossip validation, the bid must match the requested
+ * slot and parent exactly, may carry a non-zero `executionPayment` which is counted at most at
+ * the entry's `maxExecutionPayment`, and is not subject to gossip anti-spam rules.
+ *
+ * Throws with a description of the failure, the caller drops the bid.
+ */
+export async function validateBuilderApiExecutionPayloadBid(
+  chain: IBeaconChain,
+  signedExecutionPayloadBid: gloas.SignedExecutionPayloadBid,
+  request: {
+    slot: Slot;
+    parentBlock: ProtoBlock;
+    parentBlockHash: RootHex;
+    parentBlockRoot: RootHex;
+    entry: routes.validator.BuilderEntry;
+    getParentExecutionRequests: () => Promise<gloas.ExecutionRequests>;
+  }
+): Promise<void> {
+  const bid = signedExecutionPayloadBid.message;
+  const {slot, parentBlock, parentBlockHash, parentBlockRoot, entry, getParentExecutionRequests} = request;
+
+  if (bid.slot !== slot) {
+    throw Error(`Bid slot=${bid.slot} does not match requested slot=${slot}`);
+  }
+
+  const bidParentBlockHash = toRootHex(bid.parentBlockHash);
+  const bidParentBlockRoot = toRootHex(bid.parentBlockRoot);
+  if (bidParentBlockHash !== parentBlockHash || bidParentBlockRoot !== parentBlockRoot) {
+    throw Error(
+      `Bid parent parentBlockHash=${bidParentBlockHash} parentBlockRoot=${bidParentBlockRoot} does not match ` +
+        `requested parentBlockHash=${parentBlockHash} parentBlockRoot=${parentBlockRoot}`
+    );
+  }
+
+  const totalPayment = getBuilderBidTotalGwei(bid, entry.maxExecutionPayment);
+  if (totalPayment < entry.minBid) {
+    throw Error(
+      `Bid total payment=${prettyGweiToEth(totalPayment)} ` +
+        `(value=${prettyGweiToEth(bid.value)} ` +
+        `executionPayment=${prettyGweiToEth(bid.executionPayment)}) ` +
+        `is below minBid=${prettyGweiToEth(entry.minBid)}`
+    );
+  }
+
+  const state = await chain.regen
+    .getBlockSlotState(parentBlock, slot, {dontTransferCache: true}, RegenCaller.validateGossipExecutionPayloadBid)
+    .catch((e: Error) => {
+      throw Error(`Unable to regenerate state to validate bid: ${e.message}`);
+    });
+
+  if (!isStatePostGloas(state)) {
+    throw Error(`Expected gloas+ state for execution payload bid validation, got fork=${state.forkName}`);
+  }
+
+  if (bid.builderIndex >= state.getBuildersLength()) {
+    throw Error(`Bid builderIndex=${bid.builderIndex} is out of bounds`);
+  }
+
+  const builder = state.getBuilder(bid.builderIndex);
+  if (!isActiveBuilder(builder, state.finalizedCheckpoint.epoch)) {
+    throw Error(`Bid builderIndex=${bid.builderIndex} is not an active builder`);
+  }
+
+  if (builder.version !== PAYLOAD_BUILDER_VERSION) {
+    throw Error(`Invalid builder version=${builder.version} expected=${PAYLOAD_BUILDER_VERSION}`);
+  }
+
+  // A bid not signed by one of the builder pubkeys the entry accepts bids from must not be accepted
+  if (
+    entry.builderPubkeys.length > 0 &&
+    !entry.builderPubkeys.some((pubkey) => byteArrayEquals(pubkey, builder.pubkey))
+  ) {
+    throw Error(
+      `Bid builder pubkey=${toHex(builder.pubkey)} is not in the entry's ` +
+        `builderPubkeys=${entry.builderPubkeys.map(toHex).join(",")}`
+    );
+  }
+
+  const blobKzgCommitmentsLen = bid.blobKzgCommitments.length;
+  const maxBlobsPerBlock = chain.config.getMaxBlobsPerBlock(computeEpochAtSlot(bid.slot));
+  if (blobKzgCommitmentsLen > maxBlobsPerBlock) {
+    throw Error(`Bid has too many KZG commitments len=${blobKzgCommitmentsLen} limit=${maxBlobsPerBlock}`);
+  }
+
+  // The coverage check only applies to the staked collateral payment, a pure execution
+  // layer payment bid has nothing to cover on-chain
+  if (bid.value > 0 && !state.canBuilderCoverBid(bid.builderIndex, bid.value)) {
+    throw Error(
+      `Builder cannot cover bid value=${prettyGweiToEth(bid.value)} ` + `balance=${prettyGweiToEth(builder.balance)}`
+    );
+  }
+
+  const randaoMix = state.getRandaoMix(computeEpochAtSlot(state.slot));
+  if (!byteArrayEquals(bid.prevRandao, randaoMix)) {
+    throw Error(`Invalid bid prevRandao=${toHex(bid.prevRandao)} expected=${toHex(randaoMix)}`);
+  }
+
+  // The parent's payload does not try to exit the builder
+  if (byteArrayEquals(bid.parentBlockHash, state.latestExecutionPayloadBid.blockHash)) {
+    const requests = await getParentExecutionRequests();
+    if (
+      requests.builderExits.some(
+        (request) =>
+          byteArrayEquals(request.pubkey, builder.pubkey) &&
+          byteArrayEquals(request.sourceAddress, builder.executionAddress)
+      )
+    ) {
+      throw Error(`Bid builderIndex=${bid.builderIndex} may exit in the parent payload`);
+    }
+  }
+
+  // The builder must honor the proposer preferences it learned over gossip
+  const bidEpoch = computeEpochAtSlot(bid.slot);
+  const dependentRootHex = (() => {
+    try {
+      return getShufflingDependentRoot(chain.forkChoice, bidEpoch, computeEpochAtSlot(parentBlock.slot), parentBlock);
+    } catch {
+      return null;
+    }
+  })();
+  if (dependentRootHex === null) {
+    throw Error(`Unable to resolve proposer preferences dependent root for bid slot=${bid.slot}`);
+  }
+  const proposerPreferences = chain.proposerPreferencesPool.get(bid.slot, dependentRootHex);
+  if (proposerPreferences === null) {
+    throw Error(`No proposer preferences found for bid slot=${bid.slot} dependentRoot=${dependentRootHex}`);
+  }
+  if (!byteArrayEquals(bid.feeRecipient, proposerPreferences.message.feeRecipient)) {
+    throw Error(
+      `Bid feeRecipient=${toHex(bid.feeRecipient)} does not match ` +
+        `proposer preferences feeRecipient=${toHex(proposerPreferences.message.feeRecipient)}`
+    );
+  }
+
+  const parentPayloadVariant = chain.forkChoice.getBlockHexAndBlockHash(bidParentBlockRoot, bidParentBlockHash);
+  if (parentPayloadVariant === null || parentPayloadVariant.executionPayloadBlockHash === null) {
+    throw Error(`Unable to resolve parent payload gas limit for bid parentBlockHash=${bidParentBlockHash}`);
+  }
+  const parentGasLimit = BigInt(parentPayloadVariant.executionPayloadGasLimit);
+  const targetGasLimit = proposerPreferences.message.targetGasLimit;
+  if (!isGasLimitTargetCompatible(parentGasLimit, bid.gasLimit, targetGasLimit)) {
+    throw Error(
+      `Bid gasLimit=${bid.gasLimit} is not compatible with ` +
+        `parentGasLimit=${parentGasLimit} targetGasLimit=${targetGasLimit}`
+    );
+  }
+
+  const signatureSet = createSingleSignatureSetFromComponents(
+    builder.pubkey,
+    getExecutionPayloadBidSigningRoot(chain.config, bid),
+    signedExecutionPayloadBid.signature
+  );
+
+  if (!(await chain.bls.verifySignatureSets([signatureSet]))) {
+    throw Error(`Invalid bid signature builderIndex=${bid.builderIndex}`);
+  }
+}

package/src/metrics/metrics/lodestar.ts

@@ -4,7 +4,10 @@ import {ArchiveStoreTask} from "../../chain/archiveStore/archiveStore.js";
 import {FrequencyStateArchiveStep} from "../../chain/archiveStore/strategies/frequencyStateArchiveStrategy.js";
 import {BlockInputSource} from "../../chain/blocks/blockInput/index.js";
 import {PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
-import {PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js";
+import {
+  PayloadEnvelopeInputPruneReason,
+  PayloadEnvelopeInputSource,
+} from "../../chain/blocks/payloadEnvelopeInput/index.js";
 import {JobQueueItemType} from "../../chain/bls/index.js";
 import {AttestationErrorCode, BlockErrorCode} from "../../chain/errors/index.js";
 import {
@@ -21,7 +24,7 @@ import {ExecutionPayloadStatus} from "../../execution/index.js";
 import {GossipType} from "../../network/index.js";
 import {CannotAcceptWorkReason, ReprocessRejectReason} from "../../network/processor/index.js";
 import {BackfillSyncMethod} from "../../sync/backfill/backfill.js";
-import {PendingBlockType} from "../../sync/types.js";
+import {DroppedItemReason, PendingBlockType} from "../../sync/types.js";
 import {PeerSyncType, RangeSyncType} from "../../sync/utils/remoteSyncType.js";
 import {AllocSource} from "../../util/bufferPool.js";
 import {DataColumnReconstructionCode} from "../../util/dataColumns.js";
@@ -80,15 +83,15 @@ export function createLodestarMetrics(
       }),
       jobTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_job_time_seconds",
-        help: "Time to process gossip validation queue job in seconds",
+        help: "Time to process gossip validation queue job in seconds (accepted messages only)",
         labelNames: ["topic"],
-        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10],
+        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2],
       }),
       jobWaitTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_job_wait_time_seconds",
         help: "Time from job added to the queue to starting the job in seconds",
         labelNames: ["topic"],
-        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10],
+        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2],
       }),
       concurrency: register.gauge<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_concurrency",
@@ -101,9 +104,10 @@ export function createLodestarMetrics(
         help: "Age of the first item of each key in the indexed queues in seconds",
         buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5],
       }),
-      queueTime: register.histogram({
+      queueTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_time_seconds",
         help: "Total time an item stays in queue until it is processed in seconds",
+        labelNames: ["topic"],
         buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5],
       }),
     },
@@ -677,9 +681,15 @@ export function createLodestarMetrics(
         name: "lodestar_sync_unknown_block_downloaded_blocks_error_total",
         help: "Total number of downloaded blocks errors in UnknownBlockSync",
       }),
-      removedBlocks: register.gauge({
+      removedBlocks: register.gauge<{reason: DroppedItemReason}>({
         name: "lodestar_sync_unknown_block_removed_blocks_total",
-        help: "Total number of removed bad blocks in UnknownBlockSync",
+        help: "Total pending blocks dropped from BlockInputSync without completing, by reason",
+        labelNames: ["reason"],
+      }),
+      removedPayloads: register.gauge<{reason: DroppedItemReason}>({
+        name: "lodestar_sync_unknown_block_removed_payloads_total",
+        help: "Total pending payloads dropped from BlockInputSync without completing, by reason",
+        labelNames: ["reason"],
       }),
       elapsedTimeTillReceived: register.histogram({
         name: "lodestar_sync_unknown_block_elapsed_time_till_received",
@@ -911,6 +921,12 @@ export function createLodestarMetrics(
         help: "Count of errors, by error type, while processing blocks",
         labelNames: ["error"],
       }),
+
+      preStateSource: register.counter<{source: "parentState" | "fallbackPreState" | "preState"}>({
+        name: "lodestar_gossip_block_validation_pre_state_source_total",
+        help: "Source of the pre-state used for gossip block validation",
+        labelNames: ["source"],
+      }),
     },
     gossipBlob: {
       recvToValidation: register.histogram({
@@ -1225,6 +1241,12 @@ export function createLodestarMetrics(
         name: "lodestar_oppool_attester_slashing_pool_size",
         help: "Current size of the AttesterSlashingPool",
       }),
+      deferredVoluntaryExitPool: {
+        size: register.gauge({
+          name: "lodestar_oppool_deferred_voluntary_exit_pool_size",
+          help: "Current size of the DeferredVoluntaryExitPool",
+        }),
+      },
       proposerSlashingPoolSize: register.gauge({
         name: "lodestar_oppool_proposer_slashing_pool_size",
         help: "Current size of the ProposerSlashingPool",
@@ -1335,6 +1357,11 @@ export function createLodestarMetrics(
           help: "Total number of InsertOutcome as a result of adding an execution payload bid from api to the pool",
           labelNames: ["insertOutcome"],
         }),
+        apiValidationTime: register.histogram({
+          name: "lodestar_api_execution_payload_bid_validation_time_seconds",
+          help: "Time elapsed for signed execution payload bid validation - api path",
+          buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.5],
+        }),
       },
     },
 
@@ -1429,6 +1456,16 @@ export function createLodestarMetrics(
         help: "Checkpoint state cache size",
         labelNames: ["type"],
       }),
+      persistentTierEpochs: register.gauge<{tier: number}>({
+        name: "lodestar_cp_state_cache_persistent_tier_epochs",
+        help: "Number of epoch keys retained in each on-disk checkpoint state retention tier",
+        labelNames: ["tier"],
+      }),
+      persistentTierStates: register.gauge<{tier: number}>({
+        name: "lodestar_cp_state_cache_persistent_tier_states",
+        help: "Number of persisted checkpoint states retained in each on-disk retention tier",
+        labelNames: ["tier"],
+      }),
       reads: register.avgMinMax({
         name: "lodestar_cp_state_epoch_reads",
         help: "Avg min max of all state cache items total read count",
@@ -1643,9 +1680,15 @@ export function createLodestarMetrics(
           name: "lodestar_seen_payload_envelope_input_cache_serialized_object_refs",
           help: "Number of serialized-cache object refs retained by cached PayloadEnvelopeInputs",
         }),
-        created: register.counter({
+        created: register.counter<{source: PayloadEnvelopeInputSource}>({
           name: "lodestar_seen_payload_envelope_input_cache_items_created_total",
-          help: "Number of PayloadEnvelopeInputs created",
+          help: "Number of PayloadEnvelopeInputs created by source",
+          labelNames: ["source"],
+        }),
+        pruned: register.counter<{reason: PayloadEnvelopeInputPruneReason}>({
+          name: "lodestar_seen_payload_envelope_input_cache_items_pruned_total",
+          help: "Number of PayloadEnvelopeInputs evicted by reason",
+          labelNames: ["reason"],
         }),
       },
     },
@@ -1755,6 +1798,10 @@ export function createLodestarMetrics(
         name: "lodestar_precompute_next_epoch_transition_waste_total",
         help: "Total number of precomputing next epoch transition wasted",
       }),
+      predictedReorg: register.counter({
+        name: "lodestar_precompute_next_epoch_transition_predicted_reorg_total",
+        help: "Predicted epoch-boundary reorgs where the strong parent was dialed instead of the weak head",
+      }),
       duration: register.histogram({
         name: "lodestar_precompute_next_epoch_transition_duration_seconds",
         help: "Duration of precomputeNextEpochTransition, including epoch transition and hashTreeRoot",
@@ -1994,6 +2041,40 @@ export function createLodestarMetrics(
       }),
     },
 
+    builderApi: {
+      statusChecks: register.counter<{status: "success" | "error"}>({
+        name: "lodestar_builder_api_status_checks_total",
+        help: "Total count of status checks sent to external builders ahead of a proposal",
+        labelNames: ["status"],
+      }),
+      bidRequests: register.counter({
+        name: "lodestar_builder_api_bid_requests_total",
+        help: "Total count of execution payload bid requests sent to external builders",
+      }),
+      bidRequestErrors: register.counter({
+        name: "lodestar_builder_api_bid_request_errors_total",
+        help: "Total count of failed execution payload bid requests to external builders",
+      }),
+      bidsReceived: register.counter({
+        name: "lodestar_builder_api_bids_received_total",
+        help: "Total count of execution payload bids received from external builders",
+      }),
+      bidsDiscarded: register.counter({
+        name: "lodestar_builder_api_bids_discarded_total",
+        help: "Total count of execution payload bids from external builders discarded due to failed validation",
+      }),
+      blockSubmissions: register.counter<{status: "success" | "error"}>({
+        name: "lodestar_builder_api_block_submissions_total",
+        help: "Total count of signed beacon blocks submitted to external builders",
+        labelNames: ["status"],
+      }),
+      preferencesForwarded: register.counter<{status: "success" | "error"}>({
+        name: "lodestar_builder_api_preferences_forwarded_total",
+        help: "Total count of builder preferences forwarded to external builders",
+        labelNames: ["status"],
+      }),
+    },
+
     db: {
       dbReadReq: register.gauge<{bucket: string}>({
         name: "lodestar_db_read_req_total",

package/src/network/gossip/topic.ts

@@ -414,3 +414,41 @@ export const gossipTopicIgnoreDuplicatePublishError: Record<GossipType, boolean>
   [GossipType.execution_payload_bid]: true,
   [GossipType.proposer_preferences]: true,
 };
+
+/**
+ * Whether a publish that reached zero subscribed peers is an acceptable outcome for a topic.
+ *
+ * Publishing to a topic with no subscribed peers throws `PublishError.NoPeersSubscribedToTopic`, which
+ * propagates to the caller and, for messages submitted through the beacon API, is surfaced to the
+ * validator client. That is the right default: a message that reached nobody is a failed broadcast, and a
+ * VC configured with several beacon nodes can fall back to another one.
+ *
+ * Only topics that have a concrete reason to tolerate it opt out here, ie. are set to `true`, and the
+ * reason is named on the entry. Everything else is `false` and keeps raising. Note this is applied per
+ * publish, a topic set to `false` still honors the global `--network.allowPublishToZeroPeers` flag.
+ */
+export const gossipTopicAllowPublishToZeroPeers: Record<GossipType, boolean> = {
+  [GossipType.beacon_block]: false,
+  [GossipType.blob_sidecar]: false,
+  // data_column_sidecar: we ensure having all topic peers via prioritizePeers(). Even with 0 peers on the
+  // topic the overall publish can still succeed, because a supernode rebuilds and publishes the missing
+  // columns for us, so track sent peers as 0 instead of raising
+  [GossipType.data_column_sidecar]: true,
+  [GossipType.beacon_aggregate_and_proof]: false,
+  [GossipType.beacon_attestation]: false,
+  [GossipType.voluntary_exit]: false,
+  [GossipType.proposer_slashing]: false,
+  [GossipType.attester_slashing]: false,
+  [GossipType.sync_committee_contribution_and_proof]: false,
+  [GossipType.sync_committee]: false,
+  // light_client_finality_update and light_client_optimistic_update: non-mandatory route on most of the
+  // network as of Oct 2022, there may be no peer subscribed to the topic at all. Reason carried over from
+  // the handlers in network.ts, which used to swallow the error instead
+  [GossipType.light_client_finality_update]: true,
+  [GossipType.light_client_optimistic_update]: true,
+  [GossipType.bls_to_execution_change]: false,
+  [GossipType.execution_payload]: false,
+  [GossipType.payload_attestation_message]: false,
+  [GossipType.execution_payload_bid]: false,
+  [GossipType.proposer_preferences]: false,
+};