@lodestar/beacon-node 1.47.0-dev.36fc832983 → 1.47.0-dev.41653ebb93

package/src/chain/builderCircuitBreaker.ts

@@ -1,4 +1,4 @@
-import {IForkChoice} from "@lodestar/fork-choice";
+import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {Slot} from "@lodestar/types";
 import {Logger} from "@lodestar/utils";
 import {getFaultInspectionParams} from "../execution/builder/http.js";
@@ -15,11 +15,15 @@ export type BuilderCircuitBreakerModules = {
   metrics: Metrics | null;
 };
 
+/** Four observations is the minimum useful recovery sample for the default ~25% fault budget */
+const MIN_BLOCKS_TO_DEACTIVATE = 4;
+
 /**
  * Post-gloas circuit breaker for builder bids. The beacon block is produced by the proposer
  * regardless of bid source, so missed blocks are not a useful builder health signal. Instead
- * count blocks whose payload was never revealed and stop selecting builder bids while the
- * non-reveal rate in the fault inspection window is too high.
+ * count canonical blocks selected as EMPTY. Activate when the proportion of EMPTY blocks exceeds
+ * the fault budget, and resume selecting builder bids only when the observed blocks are within
+ * budget and meet the minimum recovery sample size.
  */
 export class BuilderCircuitBreaker {
   readonly faultInspectionWindow: number;
@@ -35,39 +39,49 @@ export class BuilderCircuitBreaker {
     const {faultInspectionWindow, allowedFaults} = getFaultInspectionParams(opts);
     this.faultInspectionWindow = faultInspectionWindow;
     this.allowedFaults = allowedFaults;
+    this.modules.logger.info("Builder circuit breaker initialized", {faultInspectionWindow, allowedFaults});
   }
 
   /** Whether builder bids must be ignored for a block produced at clockSlot */
-  isActive(clockSlot: Slot): boolean {
-    this.update(clockSlot);
+  isActive(clockSlot: Slot, head: ProtoBlock): boolean {
+    this.update(clockSlot, head);
     return this.active;
   }
 
-  update(clockSlot: Slot): void {
+  update(clockSlot: Slot, head: ProtoBlock): void {
     if (clockSlot <= this.lastUpdatedSlot) {
       return;
     }
     this.lastUpdatedSlot = clockSlot;
 
-    // Exclude clockSlot itself, its payload reveal may still be in flight
-    const {blocksPresent, payloadsRevealed} = this.modules.forkChoice.getPayloadRevealCounts(
+    // Exclude clockSlot itself, its payload status may still be unresolved
+    const {full, empty} = this.modules.forkChoice.getCanonicalPayloadCounts(
       Math.max(clockSlot - this.faultInspectionWindow, 0),
-      clockSlot - 1
+      clockSlot - 1,
+      head
     );
-    const faults = blocksPresent - payloadsRevealed;
+    const canonicalBlocks = full + empty;
 
     const wasActive = this.active;
-    // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates
-    this.active = faults * this.faultInspectionWindow > this.allowedFaults * blocksPresent;
+    // Scale the fault budget by canonical blocks so sparse windows still trigger on high EMPTY rates
+    const exceedsFaultBudget = empty * this.faultInspectionWindow > this.allowedFaults * canonicalBlocks;
+    if (exceedsFaultBudget) {
+      this.active = true;
+    } else if (canonicalBlocks >= MIN_BLOCKS_TO_DEACTIVATE) {
+      // Require a minimum sample within the fault budget before accepting builder bids again
+      this.active = false;
+    }
 
     this.modules.metrics?.builderCircuitBreaker.active.set(this.active ? 1 : 0);
-    this.modules.metrics?.builderCircuitBreaker.faults.set(faults);
-    this.modules.metrics?.builderCircuitBreaker.blocksPresent.set(blocksPresent);
-    this.modules.metrics?.builderCircuitBreaker.payloadsRevealed.set(payloadsRevealed);
+    this.modules.metrics?.builderCircuitBreaker.canonicalBlocks.set(canonicalBlocks);
+    this.modules.metrics?.builderCircuitBreaker.fullBlocks.set(full);
+    this.modules.metrics?.builderCircuitBreaker.emptyBlocks.set(empty);
 
     const logCtx = {
-      blocksPresent,
-      faults,
+      clockSlot,
+      canonicalBlocks,
+      full,
+      empty,
       faultInspectionWindow: this.faultInspectionWindow,
       allowedFaults: this.allowedFaults,
     };

package/src/chain/chain.ts

@@ -1072,6 +1072,7 @@ export class BeaconChain implements IBeaconChain {
       graffiti,
       slot,
       feeRecipient,
+      strictFeeRecipientCheck,
       commonBlockBodyPromise,
       parentBlock,
       builderBid,
@@ -1100,6 +1101,7 @@ export class BeaconChain implements IBeaconChain {
         graffiti,
         slot,
         feeRecipient,
+        strictFeeRecipientCheck,
         parentBlock,
         proposerIndex,
         proposerPubKey,

package/src/chain/options.ts

@@ -53,7 +53,7 @@ export type IChainOptions = BlockProcessOpts &
     nativeStateView?: boolean;
     /** Builder circuit breaker fault inspection window in slots */
     faultInspectionWindow?: number;
-    /** Allowed unrevealed payloads within the fault inspection window */
+    /** Canonical EMPTY blocks allowed per `faultInspectionWindow` observed blocks */
     allowedFaults?: number;
   };
 

package/src/chain/prepareNextSlot.ts

@@ -169,7 +169,7 @@ export class PrepareNextSlotScheduler {
           }
 
           if (isForkPostGloas(fork)) {
-            this.chain.builderCircuitBreaker.update(clockSlot);
+            this.chain.builderCircuitBreaker.update(clockSlot, updatedHead);
           } else {
             // Update the builder status, if enabled shoot an api call to check status
             this.chain.updateBuilderStatus(clockSlot);

package/src/chain/produceBlock/produceBlockBody.ts

@@ -100,6 +100,8 @@ export type BlockAttributes = {
   slot: Slot;
   parentBlock: ProtoBlock;
   feeRecipient?: string;
+  /** Verify that a locally produced execution payload uses `feeRecipient`. */
+  strictFeeRecipientCheck?: boolean;
   /** When provided, build block with this builder bid instead of a self-build bid */
   builderBid?: gloas.SignedExecutionPayloadBid;
 };
@@ -201,6 +203,7 @@ export async function produceBlockBody<T extends BlockType>(
   const {
     slot: blockSlot,
     feeRecipient: requestedFeeRecipient,
+    strictFeeRecipientCheck,
     parentBlock,
     proposerIndex,
     proposerPubKey,
@@ -334,6 +337,16 @@ export async function produceBlockBody<T extends BlockType>(
     executionPayloadValue = payloadRes.executionPayloadValue;
     shouldOverrideBuilder = payloadRes.shouldOverrideBuilder;
 
+    if (
+      strictFeeRecipientCheck &&
+      requestedFeeRecipient &&
+      !byteArrayEquals(executionPayload.feeRecipient, fromHex(requestedFeeRecipient))
+    ) {
+      throw Error(
+        `Invalid feeRecipient set in engine payload expected=${requestedFeeRecipient} actual=${toHex(executionPayload.feeRecipient)}`
+      );
+    }
+
     if (blobsBundle === undefined) {
       throw Error(`Missing blobsBundle response from getPayload at fork=${fork}`);
     }

package/src/chain/validation/proposerPreferences.ts

@@ -13,7 +13,7 @@ import {IBeaconChain} from "../index.js";
 
 /**
  * Validates a gossiped `SignedProposerPreferences` per
- * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.13/specs/gloas/p2p-interface.md#new-proposer_preferences
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.14/specs/gloas/p2p-interface.md#new-proposer_preferences
  */
 export async function validateGossipProposerPreferences(
   chain: IBeaconChain,

package/src/execution/builder/http.ts

@@ -71,15 +71,18 @@ export class NoBidReceived extends Error {
 }
 
 /**
- * Beacon clients select randomized values from the following ranges when initializing
- * the circuit breaker (so at boot time and once for each unique boot).
+ * Default circuit breaker parameters:
  *
- * ALLOWED_FAULTS: between 1 and SLOTS_PER_EPOCH // 4
- * FAULT_INSPECTION_WINDOW: between SLOTS_PER_EPOCH and 2 * SLOTS_PER_EPOCH
+ * SLOTS_PER_EPOCH <= FAULT_INSPECTION_WINDOW < 2 * SLOTS_PER_EPOCH (randomized at initialization)
+ * ALLOWED_FAULTS: FAULT_INSPECTION_WINDOW // 4
  *
- * The values are randomized per node so builders cannot predict when a given proposer will
- * fall back to local blocks. With fixed thresholds a builder could withhold payloads right up
- * to the limit without ever tripping the breaker.
+ * e.g. on mainnet SLOTS_PER_EPOCH is 32, so FAULT_INSPECTION_WINDOW is between 32 and 63,
+ * and a window of 40 results in ALLOWED_FAULTS = 10.
+ *
+ * The randomized default window keeps the exact slots under inspection unpredictable per node,
+ * so a builder cannot tell when past faults age out of a given proposer's window and time
+ * withholding around trip or recovery points. Explicitly configured windows are clamped to at
+ * least SLOTS_PER_EPOCH, and configured allowedFaults is capped at the default ~25% budget.
  */
 export function getFaultInspectionParams(opts: {faultInspectionWindow?: number; allowedFaults?: number}): {
   faultInspectionWindow: number;
@@ -90,10 +93,7 @@ export function getFaultInspectionParams(opts: {faultInspectionWindow?: number;
     SLOTS_PER_EPOCH
   );
   // allowedFaults should be < faultInspectionWindow, limiting them to faultInspectionWindow/4
-  const allowedFaults = Math.min(
-    opts.allowedFaults ?? Math.floor(faultInspectionWindow / 4),
-    Math.floor(faultInspectionWindow / 4)
-  );
+  const allowedFaults = Math.min(opts.allowedFaults ?? Infinity, Math.floor(faultInspectionWindow / 4));
   return {faultInspectionWindow, allowedFaults};
 }
 

package/src/metrics/metrics/beacon.ts

@@ -143,17 +143,17 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
         name: "beacon_builder_circuit_breaker_active",
         help: "Whether the builder circuit breaker is active (1) causing builder bids to be ignored",
       }),
-      faults: register.gauge({
-        name: "beacon_builder_circuit_breaker_faults",
-        help: "Count of blocks with unrevealed payloads in the fault inspection window",
+      canonicalBlocks: register.gauge({
+        name: "beacon_builder_circuit_breaker_canonical_blocks",
+        help: "Count of canonical FULL and EMPTY blocks in the fault inspection window",
       }),
-      blocksPresent: register.gauge({
-        name: "beacon_builder_circuit_breaker_blocks_present",
-        help: "Count of blocks present in the fault inspection window",
+      fullBlocks: register.gauge({
+        name: "beacon_builder_circuit_breaker_full_blocks",
+        help: "Count of canonical FULL blocks in the fault inspection window",
       }),
-      payloadsRevealed: register.gauge({
-        name: "beacon_builder_circuit_breaker_payloads_revealed",
-        help: "Count of blocks with revealed payloads in the fault inspection window",
+      emptyBlocks: register.gauge({
+        name: "beacon_builder_circuit_breaker_empty_blocks",
+        help: "Count of canonical EMPTY blocks in the fault inspection window",
       }),
     },
 

package/src/metrics/metrics/lodestar.ts

@@ -908,6 +908,22 @@ export function createLodestarMetrics(
         labelNames: ["error"],
       }),
     },
+    gossipExecutionPayloadBid: {
+      elapsedTimeTillReceived: register.histogram<{source: OpSource}>({
+        name: "lodestar_gossip_execution_payload_bid_elapsed_time_till_received_seconds",
+        help: "Time elapsed between the bid's slot start and the time the execution payload bid was received (negative = received before its slot, broadcast in the previous slot; positive = received during its own slot, late)",
+        labelNames: ["source"],
+        buckets: [-6, -3, -1, 0, 3],
+      }),
+    },
+    gossipPayloadAttestationMessage: {
+      elapsedTimeTillReceived: register.histogram<{source: OpSource}>({
+        name: "lodestar_gossip_payload_attestation_message_elapsed_time_till_received_seconds",
+        help: "Time elapsed between slot time and the time the payload attestation message was received",
+        labelNames: ["source"],
+        buckets: [3, 6, 9, 12],
+      }),
+    },
     // recovery in the case of specific blob rows required
     recoverBlobSidecars: {
       blobsReconstructed: register.counter({

package/src/network/processor/gossipHandlers.ts

@@ -1255,11 +1255,15 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     [GossipType.payload_attestation_message]: async ({
       gossipData,
       topic,
+      seenTimestampSec,
     }: GossipHandlerParamGeneric<GossipType.payload_attestation_message>) => {
       const {serializedData} = gossipData;
       const payloadAttestationMessage = sszDeserialize(topic, serializedData);
       const validationResult = await validateGossipPayloadAttestationMessage(chain, payloadAttestationMessage);
 
+      const delaySec = chain.clock.secFromSlot(payloadAttestationMessage.data.slot, seenTimestampSec);
+      metrics?.gossipPayloadAttestationMessage.elapsedTimeTillReceived.observe({source: OpSource.gossip}, delaySec);
+
       try {
         const insertOutcome = chain.payloadAttestationPool.add(
           payloadAttestationMessage,
@@ -1286,11 +1290,16 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     [GossipType.execution_payload_bid]: async ({
       gossipData,
       topic,
+      seenTimestampSec,
     }: GossipHandlerParamGeneric<GossipType.execution_payload_bid>) => {
       const {serializedData} = gossipData;
       const executionPayloadBid = sszDeserialize(topic, serializedData);
       const {proposerIndex} = await validateGossipExecutionPayloadBid(chain, executionPayloadBid);
 
+      // this could be negative, because it's most likely the bid of next slot comes at this clock slot
+      const elapsedSec = chain.clock.secFromSlot(executionPayloadBid.message.slot, seenTimestampSec);
+      metrics?.gossipExecutionPayloadBid.elapsedTimeTillReceived.observe({source: OpSource.gossip}, elapsedSec);
+
       // Handle valid payload bid by storing in a bid pool
       try {
         const insertOutcome = chain.executionPayloadBidPool.add(executionPayloadBid);

package/src/sync/unknownBlock.ts

@@ -1,6 +1,6 @@
 import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
-import {ForkSeq} from "@lodestar/params";
+import {ForkSeq, isForkPostGloas} from "@lodestar/params";
 import {RequestError, RequestErrorCode} from "@lodestar/reqresp";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
 import {RootHex, Slot, gloas} from "@lodestar/types";
@@ -764,9 +764,10 @@ export class BlockInputSync {
     // this prevents unbundling attack
     // see https://lighthouse-blog.sigmaprime.io/mev-unbundling-rpc.html
     const {slot: blockSlot, proposerIndex} = pendingBlock.blockInput.getBlock().message;
+    const blockRootHex = pendingBlock.blockInput.blockRootHex;
     const logCtx = {
       slot: blockSlot,
-      root: pendingBlock.blockInput.blockRootHex,
+      root: blockRootHex,
       delaySec: this.chain.clock.secFromSlot(blockSlot),
     };
     this.logger.verbose("Processing downloaded block", logCtx);
@@ -780,7 +781,7 @@ export class BlockInputSync {
       // eligible for proposer boost to prevent unbundling attack
       this.logger.verbose("Avoid proposer boost for this block of known proposer", {
         slot: blockSlot,
-        root: pendingBlock.blockInput.blockRootHex,
+        root: blockRootHex,
         proposerIndex,
       });
       await sleep(proposerBoostWindowMs);
@@ -807,12 +808,24 @@ export class BlockInputSync {
     if (!res.err) {
       this.logger.verbose("Processed block from unknown sync", logCtx);
       // no need to update status to "processed", delete anyway
-      this.pendingBlocks.delete(pendingBlock.blockInput.blockRootHex);
+      this.pendingBlocks.delete(blockRootHex);
+
+      if (isForkPostGloas(fork)) {
+        const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
+        if (!payloadInput) {
+          this.logger.warn("PayloadEnvelopeInput not seeded during unknown sync processReadyBlock()", logCtx);
+        } else {
+          // Similar to the gossip path: immediately attempt fetch of data columns from the execution engine.
+          // The bid already carries the kzg commitments, so there is no reason to wait for the payload to arrive.
+          this.chain.getBlobsTracker.triggerGetBlobs(payloadInput);
+        }
+      }
+
       // Re-enter the scheduler so descendants blocked on either parent blocks or parent payloads
       // are advanced through the same dependency checks as every other pending item.
       this.triggerUnknownBlockSearch();
     } else {
-      const errorData = {slot: pendingBlock.blockInput.slot, root: pendingBlock.blockInput.blockRootHex};
+      const errorData = {slot: pendingBlock.blockInput.slot, root: blockRootHex};
       if (res.err instanceof BlockError) {
         switch (res.err.type.code) {
           // This cases are already handled with `{ignoreIfKnown: true}`