@lodestar/beacon-node 1.47.0-dev.404f68958c → 1.47.0-dev.6325e4e1fe

package/src/api/impl/validator/index.ts

@@ -92,7 +92,12 @@ import {getStateResponseWithRegen} from "../beacon/state/utils.js";
 import {ApiError, FailureList, IndexedError, NodeIsSyncing, OnlySupportedByDVT} from "../errors.js";
 import {ApiModules} from "../types.js";
 import {notWhileSyncing} from "../utils.js";
-import {computeSubnetForCommitteesAtSlot, getPubkeysForIndices, selectBlockProductionSource} from "./utils.js";
+import {
+  computeSubnetForCommitteesAtSlot,
+  getPubkeysForIndices,
+  selectBlockProductionSource,
+  selectBlockProductionSourceByBoostFactor,
+} from "./utils.js";
 
 /**
  * Cutoff time to wait from start of the slot for execution and builder block production apis to resolve.
@@ -577,8 +582,7 @@ export function getValidatorApi(
       builderSelection,
       isBuilderEnabled,
       strictFeeRecipientCheck,
-      // winston logger doesn't like bigint
-      builderBoostFactor: `${builderBoostFactor}`,
+      builderBoostFactor,
     };
 
     logger.verbose("Assembling block with produceEngineOrBuilderBlock", loggerContext);
@@ -851,8 +855,8 @@ export function getValidatorApi(
       randaoReveal,
       graffiti,
       feeRecipient,
+      strictFeeRecipientCheck,
       includePayload,
-      builderSelection,
       builderBoostFactor,
     }) {
       const fork = config.getForkName(slot);
@@ -861,11 +865,6 @@ export function getValidatorApi(
         throw new ApiError(400, `produceBlockV4 not supported for pre-gloas fork=${fork}`);
       }
 
-      builderSelection = builderSelection ?? routes.validator.BuilderSelection.MaxProfit;
-      if (builderSelection === routes.validator.BuilderSelection.BuilderOnly) {
-        logger.warn("Builder selection builderonly is no longer supported, treating as builderalways");
-        builderSelection = routes.validator.BuilderSelection.BuilderAlways;
-      }
       builderBoostFactor = builderBoostFactor ?? BigInt(100);
       if (builderBoostFactor > MAX_BUILDER_BOOST_FACTOR) {
         throw new ApiError(400, `Invalid builderBoostFactor=${builderBoostFactor} > MAX_BUILDER_BOOST_FACTOR`);
@@ -896,14 +895,11 @@ export function getValidatorApi(
       // TODO GLOAS: add external builder api support when it is implemented
       const isBuildingOnFull = chain.forkChoice.shouldBuildOnFull(parentBlock, slot);
       const bidParentBlockHash = isBuildingOnFull ? parentBlock.executionPayloadBlockHash : parentBlock.parentBlockHash;
-      // Bids are only skipped entirely with executiononly or while the circuit breaker is active,
-      // other engine-preferring selections still build a block with the best bid as fallback in
-      // case local production fails
-      const circuitBreakerActive = chain.builderCircuitBreaker.isActive(slot);
-      const builderBid =
-        builderSelection === routes.validator.BuilderSelection.ExecutionOnly || circuitBreakerActive
-          ? null
-          : chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
+      const circuitBreakerActive = chain.builderCircuitBreaker.isActive(slot, parentBlock);
+      // Keep a builder bid as fallback unless the circuit breaker is active
+      const builderBid = circuitBreakerActive
+        ? null
+        : chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
 
       const logCtx = {
         slot,
@@ -911,8 +907,8 @@ export function getValidatorApi(
         parentBlockRoot: parentBlockRootHex,
         parentBlockHash: parentBlock.executionPayloadBlockHash,
         fork,
-        builderSelection,
         builderBoostFactor,
+        strictFeeRecipientCheck,
         circuitBreakerActive,
         ...(builderBid !== null
           ? {
@@ -936,6 +932,7 @@ export function getValidatorApi(
         randaoReveal,
         graffiti: graffitiBytes,
         feeRecipient,
+        strictFeeRecipientCheck,
         commonBlockBodyPromise,
       };
 
@@ -960,12 +957,8 @@ export function getValidatorApi(
         chain.produceBlock(baseAttrs)
       ).then((engineBlock) => {
         // No need to wait for the bid block if the engine block will always be selected due to
-        // suspected builder censorship, a builder boost factor of 0 or executionalways selection
-        if (
-          engineBlock.shouldOverrideBuilder ||
-          builderBoostFactor === BigInt(0) ||
-          builderSelection === routes.validator.BuilderSelection.ExecutionAlways
-        ) {
+        // suspected builder censorship or a builder boost factor of 0
+        if (engineBlock.shouldOverrideBuilder || builderBoostFactor === BigInt(0)) {
           controller.abort();
         }
         return engineBlock;
@@ -994,8 +987,7 @@ export function getValidatorApi(
         });
         logger.warn("Selected local block: censorship suspected in builder bid", logCtx);
       } else if (engineResult.status === "fulfilled" && bidResult.status === "fulfilled") {
-        const result = selectBlockProductionSource({
-          builderSelection,
+        const result = selectBlockProductionSourceByBoostFactor({
           builderBoostFactor,
           engineExecutionPayloadValue: engineResult.value.executionPayloadValue,
           // The bid value is the payment to the proposer, in Gwei
@@ -1061,7 +1053,10 @@ export function getValidatorApi(
         root: blockRoot,
       });
       if (chain.opts.persistProducedBlocks) {
-        void chain.persistBlock(block, "produced_engine_block");
+        void chain.persistBlock(
+          block,
+          source === ProducedBlockSource.builder ? "produced_builder_block" : "produced_engine_block"
+        );
       }
 
       // Include the payload for self-builds unless disabled (stateless flow)

package/src/api/impl/validator/utils.ts

@@ -59,24 +59,39 @@ export function selectBlockProductionSource({
       return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
 
     case routes.validator.BuilderSelection.Default:
-    case routes.validator.BuilderSelection.MaxProfit: {
-      if (builderBoostFactor === BigInt(0)) {
-        return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
-      }
-
-      if (builderBoostFactor === MAX_BUILDER_BOOST_FACTOR) {
-        return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
-      }
-
-      if (engineExecutionPayloadValue >= (builderExecutionPayloadValue * builderBoostFactor) / BigInt(100)) {
-        return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.BlockValue};
-      }
-
-      return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BlockValue};
-    }
+    case routes.validator.BuilderSelection.MaxProfit:
+      return selectBlockProductionSourceByBoostFactor({
+        engineExecutionPayloadValue,
+        builderExecutionPayloadValue,
+        builderBoostFactor,
+      });
 
     case routes.validator.BuilderSelection.BuilderAlways:
     case routes.validator.BuilderSelection.BuilderOnly:
       return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
   }
 }
+
+export function selectBlockProductionSourceByBoostFactor({
+  engineExecutionPayloadValue,
+  builderExecutionPayloadValue,
+  builderBoostFactor,
+}: {
+  engineExecutionPayloadValue: bigint;
+  builderExecutionPayloadValue: bigint;
+  builderBoostFactor: bigint;
+}): BlockSelectionResult {
+  if (builderBoostFactor === BigInt(0)) {
+    return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
+  }
+
+  if (builderBoostFactor === MAX_BUILDER_BOOST_FACTOR) {
+    return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
+  }
+
+  if (engineExecutionPayloadValue >= (builderExecutionPayloadValue * builderBoostFactor) / BigInt(100)) {
+    return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.BlockValue};
+  }
+
+  return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BlockValue};
+}

package/src/chain/archiveStore/historicalState/metrics.ts

@@ -1,3 +1,4 @@
+import {ForkName} from "@lodestar/params";
 import {
   BeaconStateTransitionMetrics,
   EpochTransitionStep,
@@ -45,6 +46,27 @@ export function createHistoricalStateTransitionMetrics(
       labelNames: ["step"],
       buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1],
     }),
+    forkUpgradeTime: metricsRegister.histogram<{fork: ForkName}>({
+      name: "lodestar_historical_state_stfn_fork_upgrade_seconds",
+      help: "Time to upgrade the state at a fork boundary in seconds",
+      labelNames: ["fork"],
+      buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30],
+    }),
+    onboardBuildersTime: metricsRegister.histogram({
+      name: "lodestar_historical_state_stfn_gloas_onboard_builders_seconds",
+      help: "Time spent in onboardBuildersFromPendingDeposits at the gloas fork transition",
+      buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30],
+    }),
+    onboardBuildersDeposits: metricsRegister.gauge<{outcome: "onboarded" | "topup" | "kept" | "dropped"}>({
+      name: "lodestar_historical_state_stfn_gloas_onboard_builders_deposits",
+      help: "Pending deposits handled at the gloas fork transition, by outcome",
+      labelNames: ["outcome"],
+    }),
+    onboardBuildersSignatureChecks: metricsRegister.gauge<{source: "cache" | "verified"}>({
+      name: "lodestar_historical_state_stfn_gloas_onboard_builders_signature_checks",
+      help: "Builder deposit signature checks at the gloas fork transition, by whether the pre-verify cache served them",
+      labelNames: ["source"],
+    }),
     processBlockTime: metricsRegister.histogram({
       name: "lodestar_historical_state_stfn_process_block_seconds",
       help: "Time to process a single block in seconds",

package/src/chain/blocks/index.ts

@@ -110,7 +110,7 @@ export async function processBlocks(
 
     for (const blockInput of relevantBlocks) {
       const block = blockInput.getBlock().message;
-      this.seenBlockProposers.add(block.slot, block.proposerIndex);
+      this.seenBlockProposers.add(block.slot, block.proposerIndex, blockInput.blockRootHex);
     }
 
     const {executionStatuses} = segmentExecStatus;

package/src/chain/builderCircuitBreaker.ts

@@ -1,4 +1,4 @@
-import {IForkChoice} from "@lodestar/fork-choice";
+import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {Slot} from "@lodestar/types";
 import {Logger} from "@lodestar/utils";
 import {getFaultInspectionParams} from "../execution/builder/http.js";
@@ -15,11 +15,15 @@ export type BuilderCircuitBreakerModules = {
   metrics: Metrics | null;
 };
 
+/** Four observations is the minimum useful recovery sample for the default ~25% fault budget */
+const MIN_BLOCKS_TO_DEACTIVATE = 4;
+
 /**
  * Post-gloas circuit breaker for builder bids. The beacon block is produced by the proposer
  * regardless of bid source, so missed blocks are not a useful builder health signal. Instead
- * count blocks whose payload was never revealed and stop selecting builder bids while the
- * non-reveal rate in the fault inspection window is too high.
+ * count canonical blocks selected as EMPTY. Activate when the proportion of EMPTY blocks exceeds
+ * the fault budget, and resume selecting builder bids only when the observed blocks are within
+ * budget and meet the minimum recovery sample size.
  */
 export class BuilderCircuitBreaker {
   readonly faultInspectionWindow: number;
@@ -35,39 +39,49 @@ export class BuilderCircuitBreaker {
     const {faultInspectionWindow, allowedFaults} = getFaultInspectionParams(opts);
     this.faultInspectionWindow = faultInspectionWindow;
     this.allowedFaults = allowedFaults;
+    this.modules.logger.info("Builder circuit breaker initialized", {faultInspectionWindow, allowedFaults});
   }
 
   /** Whether builder bids must be ignored for a block produced at clockSlot */
-  isActive(clockSlot: Slot): boolean {
-    this.update(clockSlot);
+  isActive(clockSlot: Slot, head: ProtoBlock): boolean {
+    this.update(clockSlot, head);
     return this.active;
   }
 
-  update(clockSlot: Slot): void {
+  update(clockSlot: Slot, head: ProtoBlock): void {
     if (clockSlot <= this.lastUpdatedSlot) {
       return;
     }
     this.lastUpdatedSlot = clockSlot;
 
-    // Exclude clockSlot itself, its payload reveal may still be in flight
-    const {blocksPresent, payloadsRevealed} = this.modules.forkChoice.getPayloadRevealCounts(
+    // Exclude clockSlot itself, its payload status may still be unresolved
+    const {full, empty} = this.modules.forkChoice.getCanonicalPayloadCounts(
       Math.max(clockSlot - this.faultInspectionWindow, 0),
-      clockSlot - 1
+      clockSlot - 1,
+      head
     );
-    const faults = blocksPresent - payloadsRevealed;
+    const canonicalBlocks = full + empty;
 
     const wasActive = this.active;
-    // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates
-    this.active = faults * this.faultInspectionWindow > this.allowedFaults * blocksPresent;
+    // Scale the fault budget by canonical blocks so sparse windows still trigger on high EMPTY rates
+    const exceedsFaultBudget = empty * this.faultInspectionWindow > this.allowedFaults * canonicalBlocks;
+    if (exceedsFaultBudget) {
+      this.active = true;
+    } else if (canonicalBlocks >= MIN_BLOCKS_TO_DEACTIVATE) {
+      // Require a minimum sample within the fault budget before accepting builder bids again
+      this.active = false;
+    }
 
     this.modules.metrics?.builderCircuitBreaker.active.set(this.active ? 1 : 0);
-    this.modules.metrics?.builderCircuitBreaker.faults.set(faults);
-    this.modules.metrics?.builderCircuitBreaker.blocksPresent.set(blocksPresent);
-    this.modules.metrics?.builderCircuitBreaker.payloadsRevealed.set(payloadsRevealed);
+    this.modules.metrics?.builderCircuitBreaker.canonicalBlocks.set(canonicalBlocks);
+    this.modules.metrics?.builderCircuitBreaker.fullBlocks.set(full);
+    this.modules.metrics?.builderCircuitBreaker.emptyBlocks.set(empty);
 
     const logCtx = {
-      blocksPresent,
-      faults,
+      clockSlot,
+      canonicalBlocks,
+      full,
+      empty,
       faultInspectionWindow: this.faultInspectionWindow,
       allowedFaults: this.allowedFaults,
     };

package/src/chain/chain.ts

@@ -1072,6 +1072,7 @@ export class BeaconChain implements IBeaconChain {
       graffiti,
       slot,
       feeRecipient,
+      strictFeeRecipientCheck,
       commonBlockBodyPromise,
       parentBlock,
       builderBid,
@@ -1100,6 +1101,7 @@ export class BeaconChain implements IBeaconChain {
         graffiti,
         slot,
         feeRecipient,
+        strictFeeRecipientCheck,
         parentBlock,
         proposerIndex,
         proposerPubKey,

package/src/chain/errors/blockError.ts

@@ -118,7 +118,7 @@ export type BlockErrorType =
   | {code: BlockErrorCode.GENESIS_BLOCK}
   | {code: BlockErrorCode.WOULD_REVERT_FINALIZED_SLOT; blockSlot: Slot; finalizedSlot: Slot}
   | {code: BlockErrorCode.ALREADY_KNOWN; root: RootHex}
-  | {code: BlockErrorCode.REPEAT_PROPOSAL; proposerIndex: ValidatorIndex}
+  | {code: BlockErrorCode.REPEAT_PROPOSAL; proposerIndex: ValidatorIndex; root: RootHex}
   | {code: BlockErrorCode.BLOCK_SLOT_LIMIT_REACHED}
   | {code: BlockErrorCode.INCORRECT_PROPOSER; proposerIndex: ValidatorIndex}
   | {code: BlockErrorCode.PROPOSAL_SIGNATURE_INVALID; blockSlot: Slot}

package/src/chain/options.ts

@@ -53,7 +53,7 @@ export type IChainOptions = BlockProcessOpts &
     nativeStateView?: boolean;
     /** Builder circuit breaker fault inspection window in slots */
     faultInspectionWindow?: number;
-    /** Allowed unrevealed payloads within the fault inspection window */
+    /** Canonical EMPTY blocks allowed per `faultInspectionWindow` observed blocks */
     allowedFaults?: number;
   };
 

package/src/chain/prepareNextSlot.ts

@@ -169,7 +169,7 @@ export class PrepareNextSlotScheduler {
           }
 
           if (isForkPostGloas(fork)) {
-            this.chain.builderCircuitBreaker.update(clockSlot);
+            this.chain.builderCircuitBreaker.update(clockSlot, updatedHead);
           } else {
             // Update the builder status, if enabled shoot an api call to check status
             this.chain.updateBuilderStatus(clockSlot);

package/src/chain/produceBlock/produceBlockBody.ts

@@ -100,6 +100,8 @@ export type BlockAttributes = {
   slot: Slot;
   parentBlock: ProtoBlock;
   feeRecipient?: string;
+  /** Verify that a locally produced execution payload uses `feeRecipient`. */
+  strictFeeRecipientCheck?: boolean;
   /** When provided, build block with this builder bid instead of a self-build bid */
   builderBid?: gloas.SignedExecutionPayloadBid;
 };
@@ -201,6 +203,7 @@ export async function produceBlockBody<T extends BlockType>(
   const {
     slot: blockSlot,
     feeRecipient: requestedFeeRecipient,
+    strictFeeRecipientCheck,
     parentBlock,
     proposerIndex,
     proposerPubKey,
@@ -334,6 +337,16 @@ export async function produceBlockBody<T extends BlockType>(
     executionPayloadValue = payloadRes.executionPayloadValue;
     shouldOverrideBuilder = payloadRes.shouldOverrideBuilder;
 
+    if (
+      strictFeeRecipientCheck &&
+      requestedFeeRecipient &&
+      !byteArrayEquals(executionPayload.feeRecipient, fromHex(requestedFeeRecipient))
+    ) {
+      throw Error(
+        `Invalid feeRecipient set in engine payload expected=${requestedFeeRecipient} actual=${toHex(executionPayload.feeRecipient)}`
+      );
+    }
+
     if (blobsBundle === undefined) {
       throw Error(`Missing blobsBundle response from getPayload at fork=${fork}`);
     }

package/src/chain/seenCache/seenBlockProposers.ts

@@ -19,7 +19,9 @@ const MIN_EQUIVOCATION_BLOCK_ROOTS_PER_PROPOSAL = 2;
  * The cache is pruned on finalization and bounds the number of roots stored per proposer and slot
  */
 export class SeenBlockProposers {
-  private readonly proposerIndexesBySlot = new MapDef<Slot, Set<ValidatorIndex>>(() => new Set<ValidatorIndex>());
+  private readonly proposerIndexesBySlot = new MapDef<Slot, Map<ValidatorIndex, RootHex>>(
+    () => new Map<ValidatorIndex, RootHex>()
+  );
   private readonly signedBlockHeadersBySlot = new MapDef<
     Slot,
     MapDef<ValidatorIndex, Map<RootHex, phase0.SignedBeaconBlockHeader>>
@@ -30,6 +32,14 @@ export class SeenBlockProposers {
     return this.proposerIndexesBySlot.get(blockSlot)?.has(proposerIndex) === true;
   }
 
+  /**
+   * The block proposer is known at slot with a different root.
+   */
+  isRepeatProposal(blockSlot: Slot, proposerIndex: ValidatorIndex, blockRoot: RootHex): boolean {
+    const knownRoot = this.proposerIndexesBySlot.get(blockSlot)?.get(proposerIndex);
+    return knownRoot !== undefined && knownRoot !== blockRoot;
+  }
+
   hasBlockRoot(blockSlot: Slot, proposerIndex: ValidatorIndex, blockRoot: RootHex): boolean {
     return this.signedBlockHeadersBySlot.get(blockSlot)?.get(proposerIndex)?.has(blockRoot) === true;
   }
@@ -85,12 +95,12 @@ export class SeenBlockProposers {
   }
 
   /** Mark a block as known from gossip or another block import path */
-  add(blockSlot: Slot, proposerIndex: ValidatorIndex): void {
+  add(blockSlot: Slot, proposerIndex: ValidatorIndex, blockRoot: RootHex): void {
     if (blockSlot < this.finalizedSlot) {
       throw Error(`blockSlot ${blockSlot} < finalizedSlot ${this.finalizedSlot}`);
     }
 
-    this.proposerIndexesBySlot.getOrDefault(blockSlot).add(proposerIndex);
+    this.proposerIndexesBySlot.getOrDefault(blockSlot).set(proposerIndex, blockRoot);
   }
 
   prune(finalizedSlot: Slot): void {

package/src/chain/validation/block.ts

@@ -89,13 +89,20 @@ export async function validateGossipBlock(
 
   // [IGNORE] The block is the first block with valid signature received for the proposer for the slot, signed_beacon_block.message.slot.
   const proposerIndex = block.proposerIndex;
-  const hasBlockRoot = chain.seenBlockProposers.hasBlockRoot(blockSlot, proposerIndex, blockRoot);
   if (chain.seenBlockProposers.isKnown(blockSlot, proposerIndex)) {
-    if (!hasBlockRoot && !chain.seenBlockProposers.isEquivocating(blockSlot, proposerIndex)) {
-      await verifyBlockProposerSignature(chain, signedBlock, blockRoot, {verifyOnMainThread: false});
-      chain.seenBlockProposers.observeBlockRoot(blockSlot, proposerIndex, blockRoot, signedBlockHeader);
+    if (chain.seenBlockProposers.isRepeatProposal(blockSlot, proposerIndex, blockRoot)) {
+      const hasBlockRoot = chain.seenBlockProposers.hasBlockRoot(blockSlot, proposerIndex, blockRoot);
+      if (!hasBlockRoot && !chain.seenBlockProposers.isEquivocating(blockSlot, proposerIndex)) {
+        await verifyBlockProposerSignature(chain, signedBlock, blockRoot, {verifyOnMainThread: false});
+        chain.seenBlockProposers.observeBlockRoot(blockSlot, proposerIndex, blockRoot, signedBlockHeader);
+      }
+      throw new BlockGossipError(GossipAction.IGNORE, {
+        code: BlockErrorCode.REPEAT_PROPOSAL,
+        proposerIndex,
+        root: blockRoot,
+      });
     }
-    throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.REPEAT_PROPOSAL, proposerIndex});
+    throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.ALREADY_KNOWN, root: blockRoot});
   }
 
   // [REJECT] The current finalized_checkpoint is an ancestor of block -- i.e.
@@ -301,10 +308,17 @@ export async function validateGossipBlock(
 
   // Check again after all async validation and the early-block delay so concurrent proposals cannot both pass
   if (chain.seenBlockProposers.isKnown(blockSlot, proposerIndex)) {
-    throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.REPEAT_PROPOSAL, proposerIndex});
+    if (chain.seenBlockProposers.isRepeatProposal(blockSlot, proposerIndex, blockRoot)) {
+      throw new BlockGossipError(GossipAction.IGNORE, {
+        code: BlockErrorCode.REPEAT_PROPOSAL,
+        proposerIndex,
+        root: blockRoot,
+      });
+    }
+    throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.ALREADY_KNOWN, root: blockRoot});
   }
 
-  chain.seenBlockProposers.add(blockSlot, proposerIndex);
+  chain.seenBlockProposers.add(blockSlot, proposerIndex, blockRoot);
 
   return {skippedSlots};
 }

package/src/execution/builder/http.ts

@@ -71,15 +71,18 @@ export class NoBidReceived extends Error {
 }
 
 /**
- * Beacon clients select randomized values from the following ranges when initializing
- * the circuit breaker (so at boot time and once for each unique boot).
+ * Default circuit breaker parameters:
  *
- * ALLOWED_FAULTS: between 1 and SLOTS_PER_EPOCH // 4
- * FAULT_INSPECTION_WINDOW: between SLOTS_PER_EPOCH and 2 * SLOTS_PER_EPOCH
+ * SLOTS_PER_EPOCH <= FAULT_INSPECTION_WINDOW < 2 * SLOTS_PER_EPOCH (randomized at initialization)
+ * ALLOWED_FAULTS: FAULT_INSPECTION_WINDOW // 4
  *
- * The values are randomized per node so builders cannot predict when a given proposer will
- * fall back to local blocks. With fixed thresholds a builder could withhold payloads right up
- * to the limit without ever tripping the breaker.
+ * e.g. on mainnet SLOTS_PER_EPOCH is 32, so FAULT_INSPECTION_WINDOW is between 32 and 63,
+ * and a window of 40 results in ALLOWED_FAULTS = 10.
+ *
+ * The randomized default window keeps the exact slots under inspection unpredictable per node,
+ * so a builder cannot tell when past faults age out of a given proposer's window and time
+ * withholding around trip or recovery points. Explicitly configured windows are clamped to at
+ * least SLOTS_PER_EPOCH, and configured allowedFaults is capped at the default ~25% budget.
  */
 export function getFaultInspectionParams(opts: {faultInspectionWindow?: number; allowedFaults?: number}): {
   faultInspectionWindow: number;
@@ -90,10 +93,7 @@ export function getFaultInspectionParams(opts: {faultInspectionWindow?: number;
     SLOTS_PER_EPOCH
   );
   // allowedFaults should be < faultInspectionWindow, limiting them to faultInspectionWindow/4
-  const allowedFaults = Math.min(
-    opts.allowedFaults ?? Math.floor(faultInspectionWindow / 4),
-    Math.floor(faultInspectionWindow / 4)
-  );
+  const allowedFaults = Math.min(opts.allowedFaults ?? Infinity, Math.floor(faultInspectionWindow / 4));
   return {faultInspectionWindow, allowedFaults};
 }
 

package/src/metrics/metrics/beacon.ts

@@ -143,17 +143,17 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
         name: "beacon_builder_circuit_breaker_active",
         help: "Whether the builder circuit breaker is active (1) causing builder bids to be ignored",
       }),
-      faults: register.gauge({
-        name: "beacon_builder_circuit_breaker_faults",
-        help: "Count of blocks with unrevealed payloads in the fault inspection window",
+      canonicalBlocks: register.gauge({
+        name: "beacon_builder_circuit_breaker_canonical_blocks",
+        help: "Count of canonical FULL and EMPTY blocks in the fault inspection window",
       }),
-      blocksPresent: register.gauge({
-        name: "beacon_builder_circuit_breaker_blocks_present",
-        help: "Count of blocks present in the fault inspection window",
+      fullBlocks: register.gauge({
+        name: "beacon_builder_circuit_breaker_full_blocks",
+        help: "Count of canonical FULL blocks in the fault inspection window",
       }),
-      payloadsRevealed: register.gauge({
-        name: "beacon_builder_circuit_breaker_payloads_revealed",
-        help: "Count of blocks with revealed payloads in the fault inspection window",
+      emptyBlocks: register.gauge({
+        name: "beacon_builder_circuit_breaker_empty_blocks",
+        help: "Count of canonical EMPTY blocks in the fault inspection window",
       }),
     },
 

package/src/network/processor/gossipHandlers.ts

@@ -713,9 +713,32 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       const {serializedData} = gossipData;
 
       const signedBlock = sszDeserialize(topic, serializedData);
-      const blockInput = await validateBeaconBlock(signedBlock, topic.boundary.fork, peerIdStr, seenTimestampSec);
-      chain.serializedCache.set(signedBlock, serializedData);
-      handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec);
+      try {
+        const blockInput = await validateBeaconBlock(signedBlock, topic.boundary.fork, peerIdStr, seenTimestampSec);
+        chain.serializedCache.set(signedBlock, serializedData);
+        handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec);
+      } catch (e) {
+        // Spec: IGNORE the block, ie not to re-publish to peers
+        // but we should still import an equivocating (REPEAT_PROPOSAL) block into fork choice because we don't
+        // know if this block (or the 1st known block with same slot) will become canonical yet
+        if (
+          e instanceof BlockGossipError &&
+          e.type.code === BlockErrorCode.REPEAT_PROPOSAL &&
+          // this is make sure the block's proposer signature was verified, it should be true anyway
+          chain.seenBlockProposers.hasBlockRoot(signedBlock.message.slot, e.type.proposerIndex, e.type.root)
+        ) {
+          // blockInput was optimistically seeded in validateBeaconBlock and retained on IGNORE
+          const blockInput = chain.seenBlockInputCache.get(e.type.root);
+          if (blockInput) {
+            chain.serializedCache.set(signedBlock, serializedData);
+            // this is technically not a valid gossip block but gossip validation is a cheap subset of checks
+            // this runs the full state transition, so importing an equivocating-but-valid block here is safe.
+            handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec);
+          }
+        }
+        // rethrow so gossipValidatorFn maps IGNORE -> TopicValidatorResult.Ignore (message not forwarded)
+        throw e;
+      }
     },
 
     [GossipType.blob_sidecar]: async ({

package/src/sync/unknownBlock.ts

@@ -1,6 +1,6 @@
 import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
-import {ForkSeq} from "@lodestar/params";
+import {ForkSeq, isForkPostGloas} from "@lodestar/params";
 import {RequestError, RequestErrorCode} from "@lodestar/reqresp";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
 import {RootHex, Slot, gloas} from "@lodestar/types";
@@ -764,9 +764,10 @@ export class BlockInputSync {
     // this prevents unbundling attack
     // see https://lighthouse-blog.sigmaprime.io/mev-unbundling-rpc.html
     const {slot: blockSlot, proposerIndex} = pendingBlock.blockInput.getBlock().message;
+    const blockRootHex = pendingBlock.blockInput.blockRootHex;
     const logCtx = {
       slot: blockSlot,
-      root: pendingBlock.blockInput.blockRootHex,
+      root: blockRootHex,
       delaySec: this.chain.clock.secFromSlot(blockSlot),
     };
     this.logger.verbose("Processing downloaded block", logCtx);
@@ -780,7 +781,7 @@ export class BlockInputSync {
       // eligible for proposer boost to prevent unbundling attack
       this.logger.verbose("Avoid proposer boost for this block of known proposer", {
         slot: blockSlot,
-        root: pendingBlock.blockInput.blockRootHex,
+        root: blockRootHex,
         proposerIndex,
       });
       await sleep(proposerBoostWindowMs);
@@ -807,12 +808,24 @@ export class BlockInputSync {
     if (!res.err) {
       this.logger.verbose("Processed block from unknown sync", logCtx);
       // no need to update status to "processed", delete anyway
-      this.pendingBlocks.delete(pendingBlock.blockInput.blockRootHex);
+      this.pendingBlocks.delete(blockRootHex);
+
+      if (isForkPostGloas(fork)) {
+        const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
+        if (!payloadInput) {
+          this.logger.warn("PayloadEnvelopeInput not seeded during unknown sync processReadyBlock()", logCtx);
+        } else {
+          // Similar to the gossip path: immediately attempt fetch of data columns from the execution engine.
+          // The bid already carries the kzg commitments, so there is no reason to wait for the payload to arrive.
+          this.chain.getBlobsTracker.triggerGetBlobs(payloadInput);
+        }
+      }
+
       // Re-enter the scheduler so descendants blocked on either parent blocks or parent payloads
       // are advanced through the same dependency checks as every other pending item.
       this.triggerUnknownBlockSearch();
     } else {
-      const errorData = {slot: pendingBlock.blockInput.slot, root: pendingBlock.blockInput.blockRootHex};
+      const errorData = {slot: pendingBlock.blockInput.slot, root: blockRootHex};
       if (res.err instanceof BlockError) {
         switch (res.err.type.code) {
           // This cases are already handled with `{ignoreIfKnown: true}`