@lodestar/beacon-node 1.46.0 → 1.47.0-dev.09607ce6d4

package/src/api/impl/beacon/blocks/index.ts

@@ -220,12 +220,6 @@ export function getBeaconBlockApi({
                   // Block has already been seen, e.g. via gossip racing the publish API. Benign.
                   chain.logger.debug("Ignoring already-known block during publishing", valLogMeta);
                   return;
-                case BlockErrorCode.REPEAT_PROPOSAL:
-                  // The proposer already produced a block for this slot. For a solo setup this is a
-                  // notable signal (duplicate-proposal attempt). For fallback / DVT setups it is
-                  // expected on every block where another node published first.
-                  chain.logger.warn("Ignoring repeat-proposal block during publishing", valLogMeta);
-                  return;
               }
             }
 
@@ -1032,6 +1026,7 @@ export function getBeaconBlockApi({
     },
 
     async publishExecutionPayloadBid({signedExecutionPayloadBid}) {
+      const seenTimestampSec = Date.now() / 1000;
       const bid = signedExecutionPayloadBid.message;
       const slot = bid.slot;
       const fork = config.getForkName(slot);
@@ -1042,6 +1037,9 @@ export function getBeaconBlockApi({
 
       await validateApiExecutionPayloadBid(chain, signedExecutionPayloadBid);
 
+      const elapsedSec = chain.clock.secFromSlot(slot, seenTimestampSec);
+      metrics?.gossipExecutionPayloadBid.elapsedTimeTillReceived.observe({source: OpSource.api}, elapsedSec);
+
       try {
         const insertOutcome = chain.executionPayloadBidPool.add(signedExecutionPayloadBid);
         metrics?.opPool.executionPayloadBidPool.apiInsertOutcome.inc({insertOutcome});

package/src/api/impl/beacon/pool/index.ts

@@ -25,6 +25,7 @@ import {validateApiPayloadAttestationMessage} from "../../../../chain/validation
 import {validateApiProposerSlashing} from "../../../../chain/validation/proposerSlashing.js";
 import {validateApiSyncCommittee} from "../../../../chain/validation/syncCommittee.js";
 import {validateApiVoluntaryExit} from "../../../../chain/validation/voluntaryExit.js";
+import {OpSource} from "../../../../chain/validatorMonitor.js";
 import {validateGossipFnRetryUnknownRoot} from "../../../../network/processor/gossipHandlers.js";
 import {ApiError, FailureList, IndexedError} from "../../errors.js";
 import {ApiModules} from "../../types.js";
@@ -211,6 +212,7 @@ export function getBeaconPoolApi({
     },
 
     async submitPayloadAttestationMessages({payloadAttestationMessages}) {
+      const seenTimestampSec = Date.now() / 1000;
       const failures: FailureList = [];
 
       await Promise.all(
@@ -226,6 +228,9 @@ export function getBeaconPoolApi({
               beaconBlockRoot
             );
 
+            const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
+            metrics?.gossipPayloadAttestationMessage.elapsedTimeTillReceived.observe({source: OpSource.api}, delaySec);
+
             const insertOutcome = chain.payloadAttestationPool.add(
               payloadAttestationMessage,
               attDataRootHex,

package/src/api/impl/lodestar/index.ts

@@ -280,7 +280,7 @@ export function getLodestarApi({
     },
 
     async getFastConfirmationInfo() {
-      const confirmedRoot = chain.forkChoice.getConfirmedRoot();
+      const fcrStore = chain.forkChoice.getFastConfirmationStore();
       const confirmedBlock = chain.forkChoice.getConfirmedBlock();
       const justifiedCheckpoint = chain.forkChoice.getJustifiedCheckpoint();
       const finalizedCheckpoint = chain.forkChoice.getFinalizedCheckpoint();
@@ -290,21 +290,20 @@ export function getLodestarApi({
       return {
         data: {
           confirmed: {
-            rootHex: confirmedRoot,
-            slot: confirmedBlock?.slot ?? null,
+            root: fromHex(fcrStore.confirmedRoot),
+            slot: confirmedBlock?.slot ?? 0,
           },
           head: {
-            rootHex: headRoot,
+            root: fromHex(headRoot),
             slot: head.slot,
           },
-          justifiedCheckpoint: {
-            rootHex: justifiedCheckpoint.rootHex,
-            epoch: justifiedCheckpoint.epoch,
-          },
-          finalizedCheckpoint: {
-            rootHex: finalizedCheckpoint.rootHex,
-            epoch: finalizedCheckpoint.epoch,
-          },
+          justifiedCheckpoint,
+          finalizedCheckpoint,
+          previousEpochObservedJustifiedCheckpoint: fcrStore.previousEpochObservedJustifiedCheckpoint,
+          currentEpochObservedJustifiedCheckpoint: fcrStore.currentEpochObservedJustifiedCheckpoint,
+          previousEpochGreatestUnrealizedCheckpoint: fcrStore.previousEpochGreatestUnrealizedCheckpoint,
+          previousSlotHead: fromHex(fcrStore.previousSlotHead),
+          currentSlotHead: fromHex(fcrStore.currentSlotHead),
         },
       };
     },

package/src/api/impl/validator/index.ts

@@ -92,7 +92,12 @@ import {getStateResponseWithRegen} from "../beacon/state/utils.js";
 import {ApiError, FailureList, IndexedError, NodeIsSyncing, OnlySupportedByDVT} from "../errors.js";
 import {ApiModules} from "../types.js";
 import {notWhileSyncing} from "../utils.js";
-import {computeSubnetForCommitteesAtSlot, getPubkeysForIndices, selectBlockProductionSource} from "./utils.js";
+import {
+  computeSubnetForCommitteesAtSlot,
+  getPubkeysForIndices,
+  selectBlockProductionSource,
+  selectBlockProductionSourceByBoostFactor,
+} from "./utils.js";
 
 /**
  * Cutoff time to wait from start of the slot for execution and builder block production apis to resolve.
@@ -577,8 +582,7 @@ export function getValidatorApi(
       builderSelection,
       isBuilderEnabled,
       strictFeeRecipientCheck,
-      // winston logger doesn't like bigint
-      builderBoostFactor: `${builderBoostFactor}`,
+      builderBoostFactor,
     };
 
     logger.verbose("Assembling block with produceEngineOrBuilderBlock", loggerContext);
@@ -851,8 +855,8 @@ export function getValidatorApi(
       randaoReveal,
       graffiti,
       feeRecipient,
+      strictFeeRecipientCheck,
       includePayload,
-      builderSelection,
       builderBoostFactor,
     }) {
       const fork = config.getForkName(slot);
@@ -861,11 +865,6 @@ export function getValidatorApi(
         throw new ApiError(400, `produceBlockV4 not supported for pre-gloas fork=${fork}`);
       }
 
-      builderSelection = builderSelection ?? routes.validator.BuilderSelection.MaxProfit;
-      if (builderSelection === routes.validator.BuilderSelection.BuilderOnly) {
-        logger.warn("Builder selection builderonly is no longer supported, treating as builderalways");
-        builderSelection = routes.validator.BuilderSelection.BuilderAlways;
-      }
       builderBoostFactor = builderBoostFactor ?? BigInt(100);
       if (builderBoostFactor > MAX_BUILDER_BOOST_FACTOR) {
         throw new ApiError(400, `Invalid builderBoostFactor=${builderBoostFactor} > MAX_BUILDER_BOOST_FACTOR`);
@@ -896,14 +895,11 @@ export function getValidatorApi(
       // TODO GLOAS: add external builder api support when it is implemented
       const isBuildingOnFull = chain.forkChoice.shouldBuildOnFull(parentBlock, slot);
       const bidParentBlockHash = isBuildingOnFull ? parentBlock.executionPayloadBlockHash : parentBlock.parentBlockHash;
-      // Bids are only skipped entirely with executiononly or while the circuit breaker is active,
-      // other engine-preferring selections still build a block with the best bid as fallback in
-      // case local production fails
-      const circuitBreakerActive = chain.builderCircuitBreaker.isActive(slot);
-      const builderBid =
-        builderSelection === routes.validator.BuilderSelection.ExecutionOnly || circuitBreakerActive
-          ? null
-          : chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
+      const circuitBreakerActive = chain.builderCircuitBreaker.isActive(slot, parentBlock);
+      // Keep a builder bid as fallback unless the circuit breaker is active
+      const builderBid = circuitBreakerActive
+        ? null
+        : chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
 
       const logCtx = {
         slot,
@@ -911,8 +907,8 @@ export function getValidatorApi(
         parentBlockRoot: parentBlockRootHex,
         parentBlockHash: parentBlock.executionPayloadBlockHash,
         fork,
-        builderSelection,
         builderBoostFactor,
+        strictFeeRecipientCheck,
         circuitBreakerActive,
         ...(builderBid !== null
           ? {
@@ -936,6 +932,7 @@ export function getValidatorApi(
         randaoReveal,
         graffiti: graffitiBytes,
         feeRecipient,
+        strictFeeRecipientCheck,
         commonBlockBodyPromise,
       };
 
@@ -960,12 +957,8 @@ export function getValidatorApi(
         chain.produceBlock(baseAttrs)
       ).then((engineBlock) => {
         // No need to wait for the bid block if the engine block will always be selected due to
-        // suspected builder censorship, a builder boost factor of 0 or executionalways selection
-        if (
-          engineBlock.shouldOverrideBuilder ||
-          builderBoostFactor === BigInt(0) ||
-          builderSelection === routes.validator.BuilderSelection.ExecutionAlways
-        ) {
+        // suspected builder censorship or a builder boost factor of 0
+        if (engineBlock.shouldOverrideBuilder || builderBoostFactor === BigInt(0)) {
           controller.abort();
         }
         return engineBlock;
@@ -994,8 +987,7 @@ export function getValidatorApi(
         });
         logger.warn("Selected local block: censorship suspected in builder bid", logCtx);
       } else if (engineResult.status === "fulfilled" && bidResult.status === "fulfilled") {
-        const result = selectBlockProductionSource({
-          builderSelection,
+        const result = selectBlockProductionSourceByBoostFactor({
           builderBoostFactor,
           engineExecutionPayloadValue: engineResult.value.executionPayloadValue,
           // The bid value is the payment to the proposer, in Gwei
@@ -1061,7 +1053,10 @@ export function getValidatorApi(
         root: blockRoot,
       });
       if (chain.opts.persistProducedBlocks) {
-        void chain.persistBlock(block, "produced_engine_block");
+        void chain.persistBlock(
+          block,
+          source === ProducedBlockSource.builder ? "produced_builder_block" : "produced_engine_block"
+        );
       }
 
       // Include the payload for self-builds unless disabled (stateless flow)

package/src/api/impl/validator/utils.ts

@@ -59,24 +59,39 @@ export function selectBlockProductionSource({
       return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
 
     case routes.validator.BuilderSelection.Default:
-    case routes.validator.BuilderSelection.MaxProfit: {
-      if (builderBoostFactor === BigInt(0)) {
-        return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
-      }
-
-      if (builderBoostFactor === MAX_BUILDER_BOOST_FACTOR) {
-        return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
-      }
-
-      if (engineExecutionPayloadValue >= (builderExecutionPayloadValue * builderBoostFactor) / BigInt(100)) {
-        return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.BlockValue};
-      }
-
-      return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BlockValue};
-    }
+    case routes.validator.BuilderSelection.MaxProfit:
+      return selectBlockProductionSourceByBoostFactor({
+        engineExecutionPayloadValue,
+        builderExecutionPayloadValue,
+        builderBoostFactor,
+      });
 
     case routes.validator.BuilderSelection.BuilderAlways:
     case routes.validator.BuilderSelection.BuilderOnly:
       return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
   }
 }
+
+export function selectBlockProductionSourceByBoostFactor({
+  engineExecutionPayloadValue,
+  builderExecutionPayloadValue,
+  builderBoostFactor,
+}: {
+  engineExecutionPayloadValue: bigint;
+  builderExecutionPayloadValue: bigint;
+  builderBoostFactor: bigint;
+}): BlockSelectionResult {
+  if (builderBoostFactor === BigInt(0)) {
+    return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
+  }
+
+  if (builderBoostFactor === MAX_BUILDER_BOOST_FACTOR) {
+    return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
+  }
+
+  if (engineExecutionPayloadValue >= (builderExecutionPayloadValue * builderBoostFactor) / BigInt(100)) {
+    return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.BlockValue};
+  }
+
+  return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BlockValue};
+}

package/src/chain/archiveStore/historicalState/metrics.ts

@@ -1,3 +1,4 @@
+import {ForkName} from "@lodestar/params";
 import {
   BeaconStateTransitionMetrics,
   EpochTransitionStep,
@@ -45,6 +46,27 @@ export function createHistoricalStateTransitionMetrics(
       labelNames: ["step"],
       buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1],
     }),
+    forkUpgradeTime: metricsRegister.histogram<{fork: ForkName}>({
+      name: "lodestar_historical_state_stfn_fork_upgrade_seconds",
+      help: "Time to upgrade the state at a fork boundary in seconds",
+      labelNames: ["fork"],
+      buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30],
+    }),
+    onboardBuildersTime: metricsRegister.histogram({
+      name: "lodestar_historical_state_stfn_gloas_onboard_builders_seconds",
+      help: "Time spent in onboardBuildersFromPendingDeposits at the gloas fork transition",
+      buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30],
+    }),
+    onboardBuildersDeposits: metricsRegister.gauge<{outcome: "onboarded" | "topup" | "kept" | "dropped"}>({
+      name: "lodestar_historical_state_stfn_gloas_onboard_builders_deposits",
+      help: "Pending deposits handled at the gloas fork transition, by outcome",
+      labelNames: ["outcome"],
+    }),
+    onboardBuildersSignatureChecks: metricsRegister.gauge<{source: "cache" | "verified"}>({
+      name: "lodestar_historical_state_stfn_gloas_onboard_builders_signature_checks",
+      help: "Builder deposit signature checks at the gloas fork transition, by whether the pre-verify cache served them",
+      labelNames: ["source"],
+    }),
     processBlockTime: metricsRegister.histogram({
       name: "lodestar_historical_state_stfn_process_block_seconds",
       help: "Time to process a single block in seconds",

package/src/chain/blocks/importBlock.ts

@@ -22,7 +22,6 @@ import {
   RootCache,
   computeEpochAtSlot,
   computeTimeAtSlot,
-  isStartSlotOfEpoch,
   isStatePostAltair,
   isStatePostBellatrix,
 } from "@lodestar/state-transition";
@@ -404,11 +403,7 @@ export async function importBlock(
         notOverrideFcuReason = NotReorgedReason.NotProposerOfNextSlot;
       }
     } catch (e) {
-      if (isStartSlotOfEpoch(proposalSlot)) {
-        notOverrideFcuReason = NotReorgedReason.AtEpochBoundary;
-      } else {
-        this.logger.warn("Unable to get beacon proposer. Do not override fcu.", {proposalSlot}, e as Error);
-      }
+      this.logger.warn("Unable to get beacon proposer. Do not override fcu.", {proposalSlot}, e as Error);
     }
 
     if (shouldOverrideFcu) {
@@ -445,7 +440,7 @@ export async function importBlock(
      * zero block hash (pre TTD)
      */
     const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice, this.logger);
-    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice, this.logger);
+    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice);
     if (headBlockHash !== ZERO_HASH_HEX) {
       this.executionEngine
         .notifyForkchoiceUpdate(

package/src/chain/blocks/importExecutionPayload.ts

@@ -250,7 +250,7 @@ export async function importExecutionPayload(
   const head = this.forkChoice.getHead();
   if (!this.opts.disableImportExecutionFcU && blockRootHex === head.blockRoot) {
     const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice, this.logger);
-    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice, this.logger);
+    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice);
     this.executionEngine.notifyForkchoiceUpdate(fork, blockHashHex, safeBlockHash, finalizedBlockHash).catch((e) => {
       if (!isErrorAborted(e) && !isQueueErrorAborted(e)) {
         this.logger.error("Error pushing notifyForkchoiceUpdate()", {blockHashHex, finalizedBlockHash}, e);

package/src/chain/blocks/index.ts

@@ -108,11 +108,9 @@ export async function processBlocks(
       throw segmentExecStatus.execAborted.execError;
     }
 
-    if (opts.skipVerifyBlockSignatures !== true) {
-      for (const blockInput of relevantBlocks) {
-        const block = blockInput.getBlock().message;
-        this.seenBlockProposers.add(block.slot, block.proposerIndex);
-      }
+    for (const blockInput of relevantBlocks) {
+      const block = blockInput.getBlock().message;
+      this.seenBlockProposers.add(block.slot, block.proposerIndex, blockInput.blockRootHex);
     }
 
     const {executionStatuses} = segmentExecStatus;

package/src/chain/blocks/verifyBlock.ts

@@ -6,9 +6,9 @@ import {
   computeEpochAtSlot,
   signedBlockToSignedHeader,
 } from "@lodestar/state-transition";
-import {IndexedAttestation, Slot, deneb, ssz} from "@lodestar/types";
-import {toRootHex} from "@lodestar/utils";
+import {IndexedAttestation, Slot, deneb} from "@lodestar/types";
 import {getBlobKzgCommitments} from "../../util/dataColumns.js";
+import {callInNextEventLoop} from "../../util/eventLoop.js";
 import type {BeaconChain} from "../chain.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {BlockProcessOpts} from "../options.js";
@@ -193,34 +193,30 @@ export async function verifyBlocksInEpoch(
       ),
 
       // All signatures at once
-      opts.skipVerifyBlockSignatures !== true
-        ? verifyBlocksSignatures(
-            this.config,
-            this.bls,
-            this.logger,
-            this.metrics,
-            preState0,
-            blocks,
-            indexedAttestationsByBlock,
-            opts
-          )
-        : Promise.resolve({verifySignaturesTime: Date.now()}),
+      verifyBlocksSignatures(
+        this.config,
+        this.bls,
+        this.logger,
+        this.metrics,
+        preState0,
+        blocks,
+        indexedAttestationsByBlock,
+        opts
+      ),
 
       // TODO GLOAS: can verify payload signatures in batch too
       // maybe chain with the above verifyBlocksSignatures()
     ]);
 
-    if (opts.skipVerifyBlockSignatures !== true) {
-      for (const block of blocks) {
-        const {slot, proposerIndex} = block.message;
-        const signedBlockHeader = signedBlockToSignedHeader(this.config, block);
-        const blockRoot = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(signedBlockHeader.message));
-        this.seenBlockProposers.observeBlockRoot(slot, proposerIndex, blockRoot, signedBlockHeader);
-        // Only produce a slashing while importing the block. A block that is verified before it is published
-        // must not be treated as equivocation evidence since it may never be seen by the network
-        if (opts.verifyOnly !== true && this.seenBlockProposers.isEquivocating(slot, proposerIndex)) {
-          this.processProposerEquivocation(slot, proposerIndex);
-        }
+    for (const blockInput of blockInputs) {
+      const block = blockInput.getBlock();
+      const {slot, proposerIndex} = block.message;
+      const signedBlockHeader = signedBlockToSignedHeader(this.config, block);
+      this.seenBlockProposers.observeBlockRoot(slot, proposerIndex, blockInput.blockRootHex, signedBlockHeader);
+      // Only produce a slashing while importing the block. A block that is verified before it is published
+      // must not be treated as equivocation evidence since it may never be seen by the network
+      if (opts.verifyOnly !== true && this.seenBlockProposers.isEquivocating(slot, proposerIndex)) {
+        callInNextEventLoop(() => this.processProposerEquivocation(slot, proposerIndex));
       }
     }
 

package/src/chain/builderCircuitBreaker.ts

@@ -1,4 +1,4 @@
-import {IForkChoice} from "@lodestar/fork-choice";
+import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {Slot} from "@lodestar/types";
 import {Logger} from "@lodestar/utils";
 import {getFaultInspectionParams} from "../execution/builder/http.js";
@@ -15,11 +15,15 @@ export type BuilderCircuitBreakerModules = {
   metrics: Metrics | null;
 };
 
+/** Four observations is the minimum useful recovery sample for the default ~25% fault budget */
+const MIN_BLOCKS_TO_DEACTIVATE = 4;
+
 /**
  * Post-gloas circuit breaker for builder bids. The beacon block is produced by the proposer
  * regardless of bid source, so missed blocks are not a useful builder health signal. Instead
- * count blocks whose payload was never revealed and stop selecting builder bids while the
- * non-reveal rate in the fault inspection window is too high.
+ * count canonical blocks selected as EMPTY. Activate when the proportion of EMPTY blocks exceeds
+ * the fault budget, and resume selecting builder bids only when the observed blocks are within
+ * budget and meet the minimum recovery sample size.
  */
 export class BuilderCircuitBreaker {
   readonly faultInspectionWindow: number;
@@ -35,39 +39,49 @@ export class BuilderCircuitBreaker {
     const {faultInspectionWindow, allowedFaults} = getFaultInspectionParams(opts);
     this.faultInspectionWindow = faultInspectionWindow;
     this.allowedFaults = allowedFaults;
+    this.modules.logger.info("Builder circuit breaker initialized", {faultInspectionWindow, allowedFaults});
   }
 
   /** Whether builder bids must be ignored for a block produced at clockSlot */
-  isActive(clockSlot: Slot): boolean {
-    this.update(clockSlot);
+  isActive(clockSlot: Slot, head: ProtoBlock): boolean {
+    this.update(clockSlot, head);
     return this.active;
   }
 
-  update(clockSlot: Slot): void {
+  update(clockSlot: Slot, head: ProtoBlock): void {
     if (clockSlot <= this.lastUpdatedSlot) {
       return;
     }
     this.lastUpdatedSlot = clockSlot;
 
-    // Exclude clockSlot itself, its payload reveal may still be in flight
-    const {blocksPresent, payloadsRevealed} = this.modules.forkChoice.getPayloadRevealCounts(
+    // Exclude clockSlot itself, its payload status may still be unresolved
+    const {full, empty} = this.modules.forkChoice.getCanonicalPayloadCounts(
       Math.max(clockSlot - this.faultInspectionWindow, 0),
-      clockSlot - 1
+      clockSlot - 1,
+      head
     );
-    const faults = blocksPresent - payloadsRevealed;
+    const canonicalBlocks = full + empty;
 
     const wasActive = this.active;
-    // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates
-    this.active = faults * this.faultInspectionWindow > this.allowedFaults * blocksPresent;
+    // Scale the fault budget by canonical blocks so sparse windows still trigger on high EMPTY rates
+    const exceedsFaultBudget = empty * this.faultInspectionWindow > this.allowedFaults * canonicalBlocks;
+    if (exceedsFaultBudget) {
+      this.active = true;
+    } else if (canonicalBlocks >= MIN_BLOCKS_TO_DEACTIVATE) {
+      // Require a minimum sample within the fault budget before accepting builder bids again
+      this.active = false;
+    }
 
     this.modules.metrics?.builderCircuitBreaker.active.set(this.active ? 1 : 0);
-    this.modules.metrics?.builderCircuitBreaker.faults.set(faults);
-    this.modules.metrics?.builderCircuitBreaker.blocksPresent.set(blocksPresent);
-    this.modules.metrics?.builderCircuitBreaker.payloadsRevealed.set(payloadsRevealed);
+    this.modules.metrics?.builderCircuitBreaker.canonicalBlocks.set(canonicalBlocks);
+    this.modules.metrics?.builderCircuitBreaker.fullBlocks.set(full);
+    this.modules.metrics?.builderCircuitBreaker.emptyBlocks.set(empty);
 
     const logCtx = {
-      blocksPresent,
-      faults,
+      clockSlot,
+      canonicalBlocks,
+      full,
+      empty,
       faultInspectionWindow: this.faultInspectionWindow,
       allowedFaults: this.allowedFaults,
     };

package/src/chain/chain.ts

@@ -1072,6 +1072,7 @@ export class BeaconChain implements IBeaconChain {
       graffiti,
       slot,
       feeRecipient,
+      strictFeeRecipientCheck,
       commonBlockBodyPromise,
       parentBlock,
       builderBid,
@@ -1100,6 +1101,7 @@ export class BeaconChain implements IBeaconChain {
         graffiti,
         slot,
         feeRecipient,
+        strictFeeRecipientCheck,
         parentBlock,
         proposerIndex,
         proposerPubKey,
@@ -1217,7 +1219,12 @@ export class BeaconChain implements IBeaconChain {
       this.emitter.emit(routes.events.EventType.proposerSlashing, proposerSlashing);
       this.emitter.emit(ChainEvent.publishProposerSlashing, proposerSlashing);
       this.metrics?.opPool.proposerSlashingsProduced.inc();
-      this.logger.info("Produced proposer slashing from observed equivocation", {slot: blockSlot, proposerIndex});
+      this.logger.info("Produced proposer slashing from observed equivocation", {
+        slot: blockSlot,
+        proposerIndex,
+        header1Root: toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(header1.message)),
+        header2Root: toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(header2.message)),
+      });
     } finally {
       this.producingProposerSlashing.delete(proposerIndex);
     }

package/src/chain/errors/blockError.ts

@@ -118,7 +118,7 @@ export type BlockErrorType =
   | {code: BlockErrorCode.GENESIS_BLOCK}
   | {code: BlockErrorCode.WOULD_REVERT_FINALIZED_SLOT; blockSlot: Slot; finalizedSlot: Slot}
   | {code: BlockErrorCode.ALREADY_KNOWN; root: RootHex}
-  | {code: BlockErrorCode.REPEAT_PROPOSAL; proposerIndex: ValidatorIndex}
+  | {code: BlockErrorCode.REPEAT_PROPOSAL; proposerIndex: ValidatorIndex; root: RootHex}
   | {code: BlockErrorCode.BLOCK_SLOT_LIMIT_REACHED}
   | {code: BlockErrorCode.INCORRECT_PROPOSER; proposerIndex: ValidatorIndex}
   | {code: BlockErrorCode.PROPOSAL_SIGNATURE_INVALID; blockSlot: Slot}

package/src/chain/options.ts

@@ -53,7 +53,7 @@ export type IChainOptions = BlockProcessOpts &
     nativeStateView?: boolean;
     /** Builder circuit breaker fault inspection window in slots */
     faultInspectionWindow?: number;
-    /** Allowed unrevealed payloads within the fault inspection window */
+    /** Canonical EMPTY blocks allowed per `faultInspectionWindow` observed blocks */
     allowedFaults?: number;
   };
 
@@ -86,8 +86,6 @@ export type BlockProcessOpts = {
   verifyOnly?: boolean;
   /** Used to specify to skip execution payload validation */
   skipVerifyExecutionPayload?: boolean;
-  /** Used to specify to skip block signatures validation */
-  skipVerifyBlockSignatures?: boolean;
 };
 
 export type PoolOpts = {

package/src/chain/prepareNextSlot.ts

@@ -169,7 +169,7 @@ export class PrepareNextSlotScheduler {
           }
 
           if (isForkPostGloas(fork)) {
-            this.chain.builderCircuitBreaker.update(clockSlot);
+            this.chain.builderCircuitBreaker.update(clockSlot, updatedHead);
           } else {
             // Update the builder status, if enabled shoot an api call to check status
             this.chain.updateBuilderStatus(clockSlot);
@@ -224,7 +224,7 @@ export class PrepareNextSlotScheduler {
           this.metrics?.blockPayload.payloadAdvancePrepTime.observe(preparationTime);
 
           const safeBlockHash = getSafeExecutionBlockHash(this.chain.forkChoice, this.logger);
-          const finalizedBlockHash = getFinalizedExecutionBlockHash(this.chain.forkChoice, this.logger);
+          const finalizedBlockHash = getFinalizedExecutionBlockHash(this.chain.forkChoice);
 
           // awaiting here instead of throwing an async call because there is no other task
           // left for scheduler and this gives nice semantics to catch and log errors in the

package/src/chain/produceBlock/produceBlockBody.ts

@@ -100,6 +100,8 @@ export type BlockAttributes = {
   slot: Slot;
   parentBlock: ProtoBlock;
   feeRecipient?: string;
+  /** Verify that a locally produced execution payload uses `feeRecipient`. */
+  strictFeeRecipientCheck?: boolean;
   /** When provided, build block with this builder bid instead of a self-build bid */
   builderBid?: gloas.SignedExecutionPayloadBid;
 };
@@ -201,6 +203,7 @@ export async function produceBlockBody<T extends BlockType>(
   const {
     slot: blockSlot,
     feeRecipient: requestedFeeRecipient,
+    strictFeeRecipientCheck,
     parentBlock,
     proposerIndex,
     proposerPubKey,
@@ -271,7 +274,7 @@ export async function produceBlockBody<T extends BlockType>(
     // full and blinded no longer makes sense in gloas, it might be a good idea to move
     // this into a completely separate function and have pre/post gloas more separated
     const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice, this.logger);
-    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice, this.logger);
+    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice);
     // TODO GLOAS: post-Gloas, proposer feeRecipient is also carried (signed) in
     // ProposerPreferencesPool. Consider using this unified cache instead
     // see https://github.com/ChainSafe/lodestar/issues/9379
@@ -334,6 +337,16 @@ export async function produceBlockBody<T extends BlockType>(
     executionPayloadValue = payloadRes.executionPayloadValue;
     shouldOverrideBuilder = payloadRes.shouldOverrideBuilder;
 
+    if (
+      strictFeeRecipientCheck &&
+      requestedFeeRecipient &&
+      !byteArrayEquals(executionPayload.feeRecipient, fromHex(requestedFeeRecipient))
+    ) {
+      throw Error(
+        `Invalid feeRecipient set in engine payload expected=${requestedFeeRecipient} actual=${toHex(executionPayload.feeRecipient)}`
+      );
+    }
+
     if (blobsBundle === undefined) {
       throw Error(`Missing blobsBundle response from getPayload at fork=${fork}`);
     }
@@ -417,7 +430,7 @@ export async function produceBlockBody<T extends BlockType>(
     }
 
     const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice, this.logger);
-    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice, this.logger);
+    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice);
     const feeRecipient = requestedFeeRecipient ?? this.beaconProposerCache.getOrDefault(proposerIndex);
     const feeRecipientType = requestedFeeRecipient
       ? "requested"