@lodestar/beacon-node 1.45.0-dev.f535421f29 → 1.45.0-rc.0

package/src/chain/builderCircuitBreaker.ts

@@ -0,0 +1,84 @@
+import {IForkChoice} from "@lodestar/fork-choice";
+import {Slot} from "@lodestar/types";
+import {Logger} from "@lodestar/utils";
+import {getFaultInspectionParams} from "../execution/builder/http.js";
+import {Metrics} from "../metrics/index.js";
+
+export type BuilderCircuitBreakerOpts = {
+  faultInspectionWindow?: number;
+  allowedFaults?: number;
+};
+
+export type BuilderCircuitBreakerModules = {
+  forkChoice: IForkChoice;
+  logger: Logger;
+  metrics: Metrics | null;
+};
+
+/**
+ * Post-gloas circuit breaker for builder bids. The beacon block is produced by the proposer
+ * regardless of bid source, so missed blocks are not a useful builder health signal. Instead
+ * count blocks whose payload was never revealed and stop selecting builder bids while the
+ * non-reveal rate in the fault inspection window is too high.
+ */
+export class BuilderCircuitBreaker {
+  readonly faultInspectionWindow: number;
+  readonly allowedFaults: number;
+
+  private active = false;
+  private lastUpdatedSlot = -1;
+
+  constructor(
+    opts: BuilderCircuitBreakerOpts,
+    private readonly modules: BuilderCircuitBreakerModules
+  ) {
+    const {faultInspectionWindow, allowedFaults} = getFaultInspectionParams(opts);
+    this.faultInspectionWindow = faultInspectionWindow;
+    this.allowedFaults = allowedFaults;
+  }
+
+  /** Whether builder bids must be ignored for a block produced at clockSlot */
+  isActive(clockSlot: Slot): boolean {
+    this.update(clockSlot);
+    return this.active;
+  }
+
+  update(clockSlot: Slot): void {
+    if (clockSlot <= this.lastUpdatedSlot) {
+      return;
+    }
+    this.lastUpdatedSlot = clockSlot;
+
+    // Exclude clockSlot itself, its payload reveal may still be in flight
+    const {blocksPresent, payloadsRevealed} = this.modules.forkChoice.getPayloadRevealCounts(
+      Math.max(clockSlot - this.faultInspectionWindow, 0),
+      clockSlot - 1
+    );
+    const faults = blocksPresent - payloadsRevealed;
+
+    const wasActive = this.active;
+    // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates
+    this.active = faults * this.faultInspectionWindow > this.allowedFaults * blocksPresent;
+
+    this.modules.metrics?.builderCircuitBreaker.active.set(this.active ? 1 : 0);
+    this.modules.metrics?.builderCircuitBreaker.faults.set(faults);
+    this.modules.metrics?.builderCircuitBreaker.blocksPresent.set(blocksPresent);
+    this.modules.metrics?.builderCircuitBreaker.payloadsRevealed.set(payloadsRevealed);
+
+    const logCtx = {
+      blocksPresent,
+      faults,
+      faultInspectionWindow: this.faultInspectionWindow,
+      allowedFaults: this.allowedFaults,
+    };
+    if (this.active !== wasActive) {
+      if (this.active) {
+        this.modules.logger.warn("Builder circuit breaker activated, ignoring builder bids", logCtx);
+      } else {
+        this.modules.logger.info("Builder circuit breaker deactivated", logCtx);
+      }
+    } else {
+      this.modules.logger.verbose("Builder circuit breaker status", {active: this.active, ...logCtx});
+    }
+  }
+}

package/src/chain/chain.ts

@@ -75,6 +75,7 @@ import {ImportPayloadOpts} from "./blocks/types.js";
 import {persistBlockInput} from "./blocks/writeBlockInputToDb.js";
 import {persistPayloadEnvelopeInput} from "./blocks/writePayloadEnvelopeInputToDb.js";
 import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js";
+import {BuilderCircuitBreaker} from "./builderCircuitBreaker.js";
 import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js";
 import {ChainEvent, ChainEventEmitter} from "./emitter.js";
 import {ForkchoiceCaller, initializeForkChoice} from "./forkChoice/index.js";
@@ -107,7 +108,6 @@ import {
   SeenExecutionPayloadBids,
   SeenPayloadAttesters,
   SeenPayloadEnvelopeInput,
-  SeenProposerPreferences,
   SeenSyncCommitteeMessages,
 } from "./seenCache/index.js";
 import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js";
@@ -152,6 +152,7 @@ export class BeaconChain implements IBeaconChain {
   readonly genesisValidatorsRoot: Root;
   readonly executionEngine: IExecutionEngine;
   readonly executionBuilder?: IExecutionBuilder;
+  readonly builderCircuitBreaker: BuilderCircuitBreaker;
   // Expose config for convenience in modularized functions
   readonly config: BeaconConfig;
   readonly custodyConfig: CustodyConfig;
@@ -189,7 +190,6 @@ export class BeaconChain implements IBeaconChain {
   readonly seenPayloadAttesters = new SeenPayloadAttesters();
   readonly seenAggregatedAttestations: SeenAggregatedAttestations;
   readonly seenExecutionPayloadBids = new SeenExecutionPayloadBids();
-  readonly seenProposerPreferences = new SeenProposerPreferences();
   readonly seenBlockProposers = new SeenBlockProposers();
   readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages();
   readonly seenContributionAndProof: SeenContributionAndProof;
@@ -427,6 +427,11 @@ export class BeaconChain implements IBeaconChain {
 
     this.forkChoice = forkChoice;
 
+    this.builderCircuitBreaker = new BuilderCircuitBreaker(
+      {faultInspectionWindow: opts.faultInspectionWindow, allowedFaults: opts.allowedFaults},
+      {forkChoice, logger, metrics}
+    );
+
     this.seenPayloadEnvelopeInputCache = new SeenPayloadEnvelopeInput({
       config,
       clock,
@@ -1473,7 +1478,6 @@ export class BeaconChain implements IBeaconChain {
     this.payloadAttestationPool.prune(slot);
     this.executionPayloadBidPool.prune(slot);
     this.seenExecutionPayloadBids.prune(slot);
-    this.seenProposerPreferences.prune(slot);
     this.proposerPreferencesPool.prune(slot);
     this.seenAttestationDatas.onSlot(slot);
     this.reprocessController.onSlot(slot);

package/src/chain/interface.ts

@@ -35,6 +35,7 @@ import {BeaconProposerCache, ProposerPreparationData} from "./beaconProposerCach
 import {IBlockInput} from "./blocks/blockInput/index.js";
 import {ImportBlockOpts, ImportPayloadOpts} from "./blocks/types.js";
 import {IBlsVerifier} from "./bls/index.js";
+import {BuilderCircuitBreaker} from "./builderCircuitBreaker.js";
 import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js";
 import {ChainEventEmitter} from "./emitter.js";
 import {ForkchoiceCaller} from "./forkChoice/index.js";
@@ -61,7 +62,6 @@ import {
   SeenContributionAndProof,
   SeenExecutionPayloadBids,
   SeenPayloadAttesters,
-  SeenProposerPreferences,
   SeenSyncCommitteeMessages,
 } from "./seenCache/index.js";
 import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js";
@@ -96,6 +96,7 @@ export interface IBeaconChain {
   readonly earliestAvailableSlot: Slot;
   readonly executionEngine: IExecutionEngine;
   readonly executionBuilder?: IExecutionBuilder;
+  readonly builderCircuitBreaker: BuilderCircuitBreaker;
   // Expose config for convenience in modularized functions
   readonly config: BeaconConfig;
   readonly custodyConfig: CustodyConfig;
@@ -133,7 +134,6 @@ export interface IBeaconChain {
   readonly seenPayloadAttesters: SeenPayloadAttesters;
   readonly seenAggregatedAttestations: SeenAggregatedAttestations;
   readonly seenExecutionPayloadBids: SeenExecutionPayloadBids;
-  readonly seenProposerPreferences: SeenProposerPreferences;
   readonly seenBlockProposers: SeenBlockProposers;
   readonly seenSyncCommitteeMessages: SeenSyncCommitteeMessages;
   readonly seenContributionAndProof: SeenContributionAndProof;

package/src/chain/opPools/proposerPreferencesPool.ts

@@ -1,4 +1,4 @@
-import {RootHex, Slot, gloas} from "@lodestar/types";
+import {RootHex, Slot, ValidatorIndex, gloas} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
 
 /**
@@ -22,6 +22,10 @@ export class ProposerPreferencesPool {
     return this.bySlot.get(slot)?.get(dependentRootHex) ?? null;
   }
 
+  isKnown(proposalSlot: Slot, dependentRoot: RootHex, validatorIndex: ValidatorIndex): boolean {
+    return this.get(proposalSlot, dependentRoot)?.message.validatorIndex === validatorIndex;
+  }
+
   add(signed: gloas.SignedProposerPreferences): void {
     const {proposalSlot, dependentRoot} = signed.message;
     const rootHex = toRootHex(dependentRoot);

package/src/chain/options.ts

@@ -35,6 +35,7 @@ export type IChainOptions = BlockProcessOpts &
     persistOrphanedBlocksDir?: string;
     skipCreateStateCacheIfAvailable?: boolean;
     suggestedFeeRecipient: string;
+    graffitiAppend?: boolean;
     maxSkipSlots?: number;
     /** Ensure blobs returned by the execution engine are valid */
     sanityCheckExecutionEngineBlobs?: boolean;
@@ -48,6 +49,10 @@ export type IChainOptions = BlockProcessOpts &
     archiveDateEpochs?: number;
     nHistoricalStatesFileDataStore?: boolean;
     nativeStateView?: boolean;
+    /** Builder circuit breaker fault inspection window in slots */
+    faultInspectionWindow?: number;
+    /** Allowed unrevealed payloads within the fault inspection window */
+    allowedFaults?: number;
   };
 
 export type BlockProcessOpts = {
@@ -107,6 +112,7 @@ export const defaultChainOptions: IChainOptions = {
   computeUnrealized: true,
   fastConfirmation: false,
   suggestedFeeRecipient: defaultValidatorOptions.suggestedFeeRecipient,
+  graffitiAppend: true,
   serveHistoricalState: false,
   assertCorrectProgressiveBalances: false,
   archiveStateEpochFrequency: 1024,

package/src/chain/prepareNextSlot.ts

@@ -1,7 +1,7 @@
 import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
 import {getSafeExecutionBlockHash} from "@lodestar/fork-choice";
-import {ForkPostBellatrix, ForkSeq, SLOTS_PER_EPOCH, isForkPostBellatrix} from "@lodestar/params";
+import {ForkPostBellatrix, ForkSeq, SLOTS_PER_EPOCH, isForkPostBellatrix, isForkPostGloas} from "@lodestar/params";
 import {
   IBeaconStateView,
   IBeaconStateViewBellatrix,
@@ -152,12 +152,16 @@ export class PrepareNextSlotScheduler {
             updatedHead = proposerHead;
           }
 
-          // Update the builder status, if enabled shoot an api call to check status
-          this.chain.updateBuilderStatus(clockSlot);
-          if (this.chain.executionBuilder?.status === BuilderStatus.enabled) {
-            this.chain.executionBuilder.checkStatus().catch((e) => {
-              this.logger.error("Builder disabled as the check status api failed", {prepareSlot}, e as Error);
-            });
+          if (isForkPostGloas(fork)) {
+            this.chain.builderCircuitBreaker.update(clockSlot);
+          } else {
+            // Update the builder status, if enabled shoot an api call to check status
+            this.chain.updateBuilderStatus(clockSlot);
+            if (this.chain.executionBuilder?.status === BuilderStatus.enabled) {
+              this.chain.executionBuilder.checkStatus().catch((e) => {
+                this.logger.error("Builder disabled as the check status api failed", {prepareSlot}, e as Error);
+              });
+            }
           }
         }
 

package/src/chain/produceBlock/produceBlockBody.ts

@@ -43,6 +43,7 @@ import {
   ValidatorIndex,
   Wei,
   altair,
+  bellatrix,
   capella,
   deneb,
   electra,
@@ -830,28 +831,19 @@ export function getPayloadAttributesForSSE(
     feeRecipient,
   });
 
-  let parentBlockNumber: number;
-  if (isForkPostGloas(fork)) {
-    const parentBlock = chain.forkChoice.getBlockHexAndBlockHash(
-      toRootHex(parentBlockRoot),
-      toRootHex(parentBlockHash)
-    );
-    if (parentBlock?.executionPayloadBlockHash == null) {
-      throw Error(`Parent block not found in fork choice root=${toRootHex(parentBlockRoot)}`);
-    }
-    parentBlockNumber = parentBlock.executionPayloadNumber;
-  } else {
-    parentBlockNumber = prepareState.payloadBlockNumber;
-  }
-
-  const ssePayloadAttributes: SSEPayloadAttributes = {
+  const ssePayloadAttributes = {
     proposerIndex: prepareState.getBeaconProposer(prepareSlot),
     proposalSlot: prepareSlot,
-    parentBlockNumber,
     parentBlockRoot,
     parentBlockHash,
     payloadAttributes,
-  };
+  } as SSEPayloadAttributes;
+
+  if (!isForkPostGloas(fork)) {
+    // Removed in Gloas, builders can get the block number from the EL via the block hash if required
+    (ssePayloadAttributes as bellatrix.SSEPayloadAttributes).parentBlockNumber = prepareState.payloadBlockNumber;
+  }
+
   return ssePayloadAttributes;
 }
 

package/src/chain/seenCache/index.ts

@@ -5,4 +5,3 @@ export {SeenContributionAndProof} from "./seenCommitteeContribution.js";
 export {SeenExecutionPayloadBids} from "./seenExecutionPayloadBids.js";
 export {SeenBlockInput} from "./seenGossipBlockInput.js";
 export {PayloadEnvelopeInput, SeenPayloadEnvelopeInput} from "./seenPayloadEnvelopeInput.js";
-export {SeenProposerPreferences} from "./seenProposerPreferences.js";

package/src/chain/validation/executionPayloadBid.ts

@@ -37,9 +37,11 @@ async function validateExecutionPayloadBid(
   const parentBlockRootHex = toRootHex(bid.parentBlockRoot);
   const parentBlockHashHex = toRootHex(bid.parentBlockHash);
 
-  // [IGNORE] `bid.slot` is the current slot or the next slot.
-  const currentSlot = chain.clock.currentSlot;
-  if (bid.slot !== currentSlot && bid.slot !== currentSlot + 1) {
+  // [IGNORE] `bid.slot` is the current slot, or the next slot (`bid.slot - 1` is current), allowing for `MAXIMUM_GOSSIP_CLOCK_DISPARITY`.
+  if (
+    !chain.clock.isCurrentSlotGivenGossipDisparity(bid.slot) &&
+    !chain.clock.isCurrentSlotGivenGossipDisparity(bid.slot - 1)
+  ) {
     throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
       code: ExecutionPayloadBidErrorCode.INVALID_SLOT,
       builderIndex: bid.builderIndex,
@@ -122,17 +124,20 @@ async function validateExecutionPayloadBid(
     throw new Error(`Expected gloas+ state for execution payload bid validation, got fork=${state.forkName}`);
   }
 
-  // [REJECT] `bid.builder_index` is a valid/active builder index -- i.e.
-  // `is_active_builder(state, bid.builder_index)` returns `True`.
-  let builder: gloas.Builder;
-  try {
-    builder = state.getBuilder(bid.builderIndex);
-  } catch {
+  // [REJECT] `bid.builder_index` is within bounds -- i.e. `bid.builder_index < len(state.builders)`.
+  // `state.getBuilder` returns a lazy SSZ `getReadonly` view that is not bounds-checked eagerly; an
+  // out-of-range index only throws (`LeafNode has no right node`) on deferred field access (e.g. inside
+  // `isActiveBuilder`), escaping a try/catch around `getBuilder`. Check the length explicitly instead.
+  if (bid.builderIndex >= state.getBuildersLength()) {
     throw new ExecutionPayloadBidError(GossipAction.REJECT, {
       code: ExecutionPayloadBidErrorCode.BUILDER_NOT_ELIGIBLE,
       builderIndex: bid.builderIndex,
     });
   }
+
+  // [REJECT] `bid.builder_index` is a valid/active builder index -- i.e.
+  // `is_active_builder(state, bid.builder_index)` returns `True`.
+  const builder = state.getBuilder(bid.builderIndex);
   if (!isActiveBuilder(builder, state.finalizedCheckpoint.epoch)) {
     throw new ExecutionPayloadBidError(GossipAction.REJECT, {
       code: ExecutionPayloadBidErrorCode.BUILDER_NOT_ELIGIBLE,

package/src/chain/validation/proposerPreferences.ts

@@ -32,8 +32,9 @@ export async function validateGossipProposerPreferences(
     });
   }
 
-  // [IGNORE] `preferences.proposal_slot` has not already passed.
-  const currentSlot = chain.clock.currentSlot;
+  // [IGNORE] `preferences.proposal_slot` has not already passed, i.e. `proposal_slot > current_slot`,
+  // allowing for `MAXIMUM_GOSSIP_CLOCK_DISPARITY`.
+  const currentSlot = chain.clock.currentSlotWithGossipDisparity;
   if (proposalSlot <= currentSlot) {
     throw new ProposerPreferencesError(GossipAction.IGNORE, {
       code: ProposerPreferencesErrorCode.PROPOSAL_SLOT_PASSED,
@@ -81,7 +82,7 @@ export async function validateGossipProposerPreferences(
   }
 
   // [IGNORE] First valid message for (dependent_root, proposal_slot, validator_index).
-  if (chain.seenProposerPreferences.isKnown(dependentRootHex, proposalSlot, validatorIndex)) {
+  if (chain.proposerPreferencesPool.isKnown(proposalSlot, dependentRootHex, validatorIndex)) {
     throw new ProposerPreferencesError(GossipAction.IGNORE, {
       code: ProposerPreferencesErrorCode.ALREADY_KNOWN,
       proposalSlot,
@@ -105,6 +106,15 @@ export async function validateGossipProposerPreferences(
     });
   }
 
-  // Valid
-  chain.seenProposerPreferences.add(dependentRootHex, proposalSlot, validatorIndex);
+  // Repeated check - deals with race-condition between preferences submissions
+  if (chain.proposerPreferencesPool.isKnown(proposalSlot, dependentRootHex, validatorIndex)) {
+    throw new ProposerPreferencesError(GossipAction.IGNORE, {
+      code: ProposerPreferencesErrorCode.ALREADY_KNOWN,
+      proposalSlot,
+      validatorIndex,
+      dependentRoot: dependentRootHex,
+    });
+  }
+
+  chain.proposerPreferencesPool.add(signedProposerPreferences);
 }

package/src/execution/builder/http.ts

@@ -70,6 +70,33 @@ export class NoBidReceived extends Error {
   }
 }
 
+/**
+ * Beacon clients select randomized values from the following ranges when initializing
+ * the circuit breaker (so at boot time and once for each unique boot).
+ *
+ * ALLOWED_FAULTS: between 1 and SLOTS_PER_EPOCH // 4
+ * FAULT_INSPECTION_WINDOW: between SLOTS_PER_EPOCH and 2 * SLOTS_PER_EPOCH
+ *
+ * The values are randomized per node so builders cannot predict when a given proposer will
+ * fall back to local blocks. With fixed thresholds a builder could withhold payloads right up
+ * to the limit without ever tripping the breaker.
+ */
+export function getFaultInspectionParams(opts: {faultInspectionWindow?: number; allowedFaults?: number}): {
+  faultInspectionWindow: number;
+  allowedFaults: number;
+} {
+  const faultInspectionWindow = Math.max(
+    opts.faultInspectionWindow ?? SLOTS_PER_EPOCH + Math.floor(Math.random() * SLOTS_PER_EPOCH),
+    SLOTS_PER_EPOCH
+  );
+  // allowedFaults should be < faultInspectionWindow, limiting them to faultInspectionWindow/4
+  const allowedFaults = Math.min(
+    opts.allowedFaults ?? Math.floor(faultInspectionWindow / 4),
+    Math.floor(faultInspectionWindow / 4)
+  );
+  return {faultInspectionWindow, allowedFaults};
+}
+
 /**
  * Additional duration to account for potential event loop lag which causes
  * builder blocks to be rejected even though the response was sent in time.
@@ -122,23 +149,9 @@ export class ExecutionBuilderHttp implements IExecutionBuilder {
     this.registrations = new ValidatorRegistrationCache();
     this.issueLocalFcUWithFeeRecipient = opts.issueLocalFcUWithFeeRecipient;
 
-    /**
-     * Beacon clients select randomized values from the following ranges when initializing
-     * the circuit breaker (so at boot time and once for each unique boot).
-     *
-     * ALLOWED_FAULTS: between 1 and SLOTS_PER_EPOCH // 4
-     * FAULT_INSPECTION_WINDOW: between SLOTS_PER_EPOCH and 2 * SLOTS_PER_EPOCH
-     *
-     */
-    this.faultInspectionWindow = Math.max(
-      opts.faultInspectionWindow ?? SLOTS_PER_EPOCH + Math.floor(Math.random() * SLOTS_PER_EPOCH),
-      SLOTS_PER_EPOCH
-    );
-    // allowedFaults should be < faultInspectionWindow, limiting them to faultInspectionWindow/4
-    this.allowedFaults = Math.min(
-      opts.allowedFaults ?? Math.floor(this.faultInspectionWindow / 4),
-      Math.floor(this.faultInspectionWindow / 4)
-    );
+    const {faultInspectionWindow, allowedFaults} = getFaultInspectionParams(opts);
+    this.faultInspectionWindow = faultInspectionWindow;
+    this.allowedFaults = allowedFaults;
   }
 
   updateStatus(status: BuilderStatus): void {

package/src/metrics/metrics/beacon.ts

@@ -138,6 +138,25 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
       help: "Count of cached produced results",
     }),
 
+    builderCircuitBreaker: {
+      active: register.gauge({
+        name: "beacon_builder_circuit_breaker_active",
+        help: "Whether the builder circuit breaker is active (1) causing builder bids to be ignored",
+      }),
+      faults: register.gauge({
+        name: "beacon_builder_circuit_breaker_faults",
+        help: "Count of blocks with unrevealed payloads in the fault inspection window",
+      }),
+      blocksPresent: register.gauge({
+        name: "beacon_builder_circuit_breaker_blocks_present",
+        help: "Count of blocks present in the fault inspection window",
+      }),
+      payloadsRevealed: register.gauge({
+        name: "beacon_builder_circuit_breaker_payloads_revealed",
+        help: "Count of blocks with revealed payloads in the fault inspection window",
+      }),
+    },
+
     blockPayload: {
       payloadAdvancePrepTime: register.histogram({
         name: "beacon_block_payload_prepare_time",

package/src/network/core/metrics.ts

@@ -127,6 +127,100 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) {
         name: "lodestar_peer_manager_starved_bool",
         help: "Whether lodestar is starved of data while syncing",
       }),
+      /**
+       * Core selection/pruning phase of the heartbeat, split out of heartbeatDuration so it
+       * can be attributed on its own. Divide by peersEvaluated for time-per-peer.
+       */
+      prioritizePeersDuration: register.histogram({
+        name: "lodestar_peer_manager_prioritize_peers_seconds",
+        help: "prioritizePeers function duration in seconds, the core peer selection/pruning algorithm",
+        buckets: [0.0005, 0.001, 0.005, 0.01, 0.05, 0.1],
+      }),
+      /**
+       * Score decay + map prune over every tracked peer, split out of heartbeatDuration.
+       * This loop iterates the whole score store, so divide by scoreMapSize (not
+       * peersEvaluated) for time-per-peer.
+       */
+      scoreUpdateDuration: register.histogram({
+        name: "lodestar_peer_score_update_seconds",
+        help: "Peer score store update (decay + prune over all peers) duration in seconds",
+        buckets: [0.0005, 0.001, 0.005, 0.01, 0.05, 0.1],
+      }),
+      /**
+       * Denominator for prioritizePeersDuration: it scales with the number of connected
+       * healthy peers, so time-per-peer (prioritizePeersDuration / this) is the comparable
+       * figure. For scoreUpdateDuration use scoreMapSize instead.
+       */
+      peersEvaluated: register.histogram({
+        name: "lodestar_peer_manager_peers_evaluated_count",
+        help: "Number of connected healthy peers evaluated by prioritizePeers per heartbeat, denominator for prioritize_peers_seconds",
+        buckets: [0, 25, 50, 75, 100, 150, 200],
+      }),
+      /**
+       * Every peer the manager intends to disconnect. Covers both the top-of-heartbeat
+       * bad-score disconnects ("banned"/"score_too_low") and the prioritization reasons;
+       * the older peersRequestedToDisconnect gauge only recorded the latter. Counts intent,
+       * not completion (goodbye is fire-and-forget).
+       */
+      peersPruned: register.counter<{reason: string}>({
+        name: "lodestar_peer_manager_peers_pruned_total",
+        help: "Total peers the peer manager intends to disconnect, labeled by reason (incl. bad-score and prioritization reasons)",
+        labelNames: ["reason"],
+      }),
+      /**
+       * Actual peer count per active subnet, to check the min-peers-per-subnet invariant.
+       * Buckets straddle the target (TARGET_SUBNET_PEERS = 6).
+       */
+      peersPerActiveSubnet: register.histogram<{type: SubnetType}>({
+        name: "lodestar_peer_manager_peers_per_active_subnet",
+        help: "Histogram of connected peer count per active subnet, labeled by subnet type",
+        labelNames: ["type"],
+        buckets: [0, 2, 4, 6, 8, 12],
+      }),
+      /**
+       * Live outbound ratio, to check the OUTBOUND_PEERS_RATIO (10%) invariant (issue #2215).
+       * Denominator is connected healthy peers (banned/disconnected already removed).
+       */
+      outboundPeersRatio: register.gauge({
+        name: "lodestar_peer_manager_outbound_peers_ratio",
+        help: "Ratio of outbound peers to total connected healthy peers, verifies the outbound peers invariant",
+      }),
+      /**
+       * Score state crossings (Healthy/Disconnected/Banned). Sensitive to the decay formula,
+       * thresholds, and gossip-score weighting.
+       */
+      scoreStateTransitions: register.counter<{from: string; to: string}>({
+        name: "lodestar_peer_score_state_transitions_total",
+        help: "Total peer score state transitions, labeled by from and to state (Healthy/Disconnected/Banned)",
+        labelNames: ["from", "to"],
+      }),
+      /**
+       * Entry count of the score store (not bytes). Reflects the prune-to-MAX_ENTRIES /
+       * SCORE_THRESHOLD retention logic. Also the denominator for scoreUpdateDuration
+       * time-per-peer (that loop iterates the whole store, not just connected peers).
+       */
+      scoreMapSize: register.gauge({
+        name: "lodestar_peer_manager_score_map_size",
+        help: "Current number of entries in the peer score store",
+      }),
+      /**
+       * Entry count of the connectedPeers map (not bytes). Should track libp2p_peers; a
+       * persistent gap indicates the connection leak guarded by leakedConnectionsCount.
+       */
+      connectedPeersMapSize: register.gauge({
+        name: "lodestar_peer_manager_connected_peers_map_size",
+        help: "Current number of entries in the peer manager connectedPeers map",
+      }),
+      /**
+       * assertPeerRelevance outcome per Status exchange: "relevant", an irrelevant reason
+       * code, or "error". Fires on every status evaluation (inbound requests and our own
+       * requestStatus responses both route through onStatus).
+       */
+      relevanceCheck: register.counter<{result: string}>({
+        name: "lodestar_peer_relevance_check_total",
+        help: "Total peer relevance checks on Status, labeled by result (relevant or irrelevant reason code)",
+        labelNames: ["result"],
+      }),
     },
     leakedConnectionsCount: register.gauge({
       name: "lodestar_peer_manager_leaked_connections_count",

package/src/network/discv5/utils.ts

@@ -13,9 +13,17 @@ export enum ENRRelevance {
 }
 
 export function enrRelevance(enr: ENR, config: BeaconConfig, clock: IClock): ENRRelevance {
-  // We are not interested in peers that don't advertise at least one transport (tcp or quic)
-  const multiaddrTCP = enr.getLocationMultiaddr(ENRKey.tcp);
-  const multiaddrQUIC = enr.getLocationMultiaddr(ENRKey.quic);
+  // We are not interested in peers that don't advertise at least one transport (tcp or quic).
+  let multiaddrTCP: ReturnType<ENR["getLocationMultiaddr"]>;
+  let multiaddrQUIC: ReturnType<ENR["getLocationMultiaddr"]>;
+  // Malformed ENR endpoint fields can make multiaddr conversion throw.
+  // Treat endpoints that cannot be converted as unusable.
+  try {
+    multiaddrTCP = enr.getLocationMultiaddr(ENRKey.tcp);
+    multiaddrQUIC = enr.getLocationMultiaddr(ENRKey.quic);
+  } catch {
+    return ENRRelevance.no_transport;
+  }
   if (!multiaddrTCP && !multiaddrQUIC) {
     return ENRRelevance.no_transport;
   }

package/src/network/gossip/encoding.ts

@@ -52,6 +52,13 @@ export function fastMsgIdFn(rpcMsg: RPC.Message): string {
 }
 
 export function msgIdToStrFn(msgId: Uint8Array): string {
+  // Spec mandates a 20-byte gossipsub message-id, but control-message IDs from peers
+  // can be any length. Reject non-20-byte IDs to avoid a Buffer.set() RangeError on
+  // longer IDs and stale shared-buffer bytes on shorter ones.
+  // Ref: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.10/specs/phase0/p2p-interface.md?plain=1#L486
+  if (msgId.length !== 20) {
+    throw Error(`Expect msgId to be 20 bytes, got ${msgId.length}`);
+  }
   // this is the same logic to `toHex(msgId)` with better performance
   sharedMsgIdBuf.set(msgId);
   return `0x${sharedMsgIdBuf.toString("hex")}`;

package/src/network/gossip/gossipsub.ts

@@ -35,7 +35,7 @@ import {
   computeGossipPeerScoreParams,
   gossipScoreThresholds,
 } from "./scoringParameters.js";
-import {GossipTopicCache, getCoreTopicsAtFork, stringifyGossipTopic} from "./topic.js";
+import {GossipTopicCache, getAllowedTopics, getCoreTopicsAtFork, stringifyGossipTopic} from "./topic.js";
 
 /** As specified in https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md */
 const GOSSIPSUB_HEARTBEAT_INTERVAL = 0.7 * 1000;
@@ -140,6 +140,7 @@ export class Eth2Gossipsub {
     const gossipsubInstance = gossipsub({
       globalSignaturePolicy: StrictNoSign,
       allowPublishToZeroTopicPeers: allowPublishToZeroPeers,
+      allowedTopics: getAllowedTopics(networkConfig),
       D: gossipsubD ?? GOSSIP_D,
       Dlo: gossipsubDLow ?? GOSSIP_D_LOW,
       Dhi: gossipsubDHigh ?? GOSSIP_D_HIGH,