@lodestar/beacon-node 1.45.0-dev.51a1c44b27 → 1.45.0-dev.6568180f96

package/src/api/rest/base.ts

@@ -3,7 +3,9 @@ import {fastifyCors} from "@fastify/cors";
 import {FastifyError, FastifyInstance, FastifyRequest, errorCodes, fastify} from "fastify";
 import {parse as parseQueryString} from "qs";
 import {addSszContentTypeParser} from "@lodestar/api/server";
+import {NUMBER_OF_COLUMNS} from "@lodestar/params";
 import {ErrorAborted, Gauge, Histogram, Logger} from "@lodestar/utils";
+import {GossipActionError} from "../../chain/errors/gossipValidation.js";
 import {isLocalhostIP} from "../../util/ip.js";
 import {ApiError, FailureList, IndexedError, NodeIsSyncing} from "../impl/errors.js";
 import {HttpActiveSocketsTracker, SocketMetrics} from "./activeSockets.js";
@@ -55,6 +57,13 @@ const INVALID_MEDIA_TYPE_CODE = errorCodes.FST_ERR_CTP_INVALID_MEDIA_TYPE().code
  */
 const SCHEMA_VALIDATION_ERROR_CODE = errorCodes.FST_ERR_VALIDATION().code;
 
+/**
+ * Cap for array query params, set to the largest array any beacon-API query can carry:
+ * a full data-column custody set (`getDebugDataColumnSidecars` `indices`). `qs` turns
+ * longer arrays into an object, which then fails schema validation.
+ */
+const QUERY_STRING_ARRAY_LIMIT = NUMBER_OF_COLUMNS;
+
 /**
  * REST API powered by `fastify` server.
  */
@@ -82,6 +91,7 @@ export class RestApiServer {
             // be OpenAPI spec compliant and results are inconsistent, see https://github.com/ljharb/qs/issues/331.
             // The schema validation will catch this and throw an error as parsed query string results in an object.
             parseArrays: false,
+            arrayLimit: QUERY_STRING_ARRAY_LIMIT,
           }),
       },
       bodyLimit: opts.bodyLimit,
@@ -112,8 +122,8 @@ export class RestApiServer {
         };
         void res.status(err.statusCode).send(payload);
       } else {
-        // Convert our custom ApiError into status code
-        const statusCode = err instanceof ApiError ? err.statusCode : 500;
+        // Convert known request errors into status codes
+        const statusCode = err instanceof ApiError ? err.statusCode : err instanceof GossipActionError ? 400 : 500;
         const payload: ErrorResponse = {code: statusCode, message: err.message, stacktraces};
         void res.status(statusCode).send(payload);
       }
@@ -161,7 +171,11 @@ export class RestApiServer {
 
       const operationId = getOperationId(req);
 
-      if (err instanceof ApiError || [INVALID_MEDIA_TYPE_CODE, SCHEMA_VALIDATION_ERROR_CODE].includes(err.code)) {
+      if (
+        err instanceof ApiError ||
+        err instanceof GossipActionError ||
+        [INVALID_MEDIA_TYPE_CODE, SCHEMA_VALIDATION_ERROR_CODE].includes(err.code)
+      ) {
         this.logger.warn(`Req ${req.id} ${operationId} failed`, {reason: err.message});
       } else {
         this.logger.error(`Req ${req.id} ${operationId} error`, {}, err);

package/src/chain/blocks/importBlock.ts

@@ -2,7 +2,6 @@ import {BitArray} from "@chainsafe/ssz";
 import {routes} from "@lodestar/api";
 import {
   AncestorStatus,
-  EpochDifference,
   ExecutionStatus,
   ForkChoiceError,
   ForkChoiceErrorCode,
@@ -21,7 +20,6 @@ import {
   IBeaconStateView,
   RootCache,
   computeEpochAtSlot,
-  computeStartSlotAtEpoch,
   computeTimeAtSlot,
   isStartSlotOfEpoch,
   isStatePostAltair,
@@ -124,18 +122,13 @@ export async function importBlock(
     executionStatus = parentBlock.executionStatus;
   }
 
-  // getBeaconProposerOrNull will return null if head state is more than one epoch away
-  // from block slot. We skip proposer boost canonical check as we cannot determine the canonical proposer
-  const expectedProposerIndex: number | null = this.getHeadState().getBeaconProposerOrNull(blockSlot);
-
   const blockSummary = this.forkChoice.onBlock(
     block.message,
     postState,
     blockDelaySec,
     currentSlot,
     executionStatus,
-    dataAvailabilityStatus,
-    expectedProposerIndex
+    dataAvailabilityStatus
   );
 
   // This adds the state necessary to process the next block
@@ -306,21 +299,6 @@ export async function importBlock(
     // Set head state as strong reference
     this.regen.updateHeadState(newHead, postState);
 
-    try {
-      this.emitter.emit(routes.events.EventType.head, {
-        block: newHead.blockRoot,
-        epochTransition: computeStartSlotAtEpoch(computeEpochAtSlot(newHead.slot)) === newHead.slot,
-        slot: newHead.slot,
-        state: newHead.stateRoot,
-        previousDutyDependentRoot: this.forkChoice.getDependentRoot(newHead, EpochDifference.previous),
-        currentDutyDependentRoot: this.forkChoice.getDependentRoot(newHead, EpochDifference.current),
-        executionOptimistic: isOptimisticBlock(newHead),
-      });
-    } catch (e) {
-      // getDependentRoot() may fail with error: "No block for root" as we can see in holesky non-finality issue
-      this.logger.debug("Error emitting head event", {slot: newHead.slot, root: newHead.blockRoot}, e as Error);
-    }
-
     const delaySec = this.clock.secFromSlot(newHead.slot);
     this.logger.verbose("New chain head", {
       slot: newHead.slot,
@@ -426,7 +404,7 @@ export async function importBlock(
       }
     } catch (e) {
       if (isStartSlotOfEpoch(proposalSlot)) {
-        notOverrideFcuReason = NotReorgedReason.NotShufflingStable;
+        notOverrideFcuReason = NotReorgedReason.AtEpochBoundary;
       } else {
         this.logger.warn("Unable to get beacon proposer. Do not override fcu.", {proposalSlot}, e as Error);
       }

package/src/chain/blocks/payloadEnvelopeProcessor.ts

@@ -8,28 +8,24 @@ import {ImportPayloadOpts} from "./types.js";
 // TODO GLOAS: Set to be equal to DEFAULT_MAX_PENDING_UNFINALIZED_PAYLOAD_ENVELOPE_WRITES for now
 const QUEUE_MAX_LENGTH = 16;
 
-enum PayloadEnvelopeImportStatus {
-  queued = "queued",
-  importing = "importing",
-  imported = "imported",
-}
-
 /**
  * PayloadEnvelopeProcessor processes payload envelope jobs in a queued fashion, one after the other.
  *
  * Jobs are enqueued only on envelope arrival (gossip or API). The envelope may reach us before
  * the sampled data columns; importExecutionPayload awaits `verifyPayloadsDataAvailability`
  * internally, so a queued job can pend for up to `PAYLOAD_DATA_AVAILABILITY_TIMEOUT` while
- * waiting for columns. Duplicate triggers for the same payloadInput are deduped via `importStatus`.
+ * waiting for columns. Duplicate triggers for the same payloadInput are deduped by sharing the
+ * in-flight import promise: every caller observes the real import outcome. Resolving duplicates
+ * early instead would let BlockInputSync mark a still-pending payload as processed, re-queue it,
+ * and spin re-processing the same root until OOM while the import is blocked (e.g. EL syncing).
  */
 export class PayloadEnvelopeProcessor {
   readonly jobQueue: JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>;
-  private readonly importStatus = new WeakMap<PayloadEnvelopeInput, PayloadEnvelopeImportStatus>();
+  private readonly imports = new WeakMap<PayloadEnvelopeInput, Promise<void>>();
 
   constructor(chain: BeaconChain, metrics: Metrics | null, signal: AbortSignal) {
     this.jobQueue = new JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>(
       (payloadInput, opts) => {
-        this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.importing);
         return processExecutionPayload.call(chain, payloadInput, signal, opts);
       },
       {maxLength: QUEUE_MAX_LENGTH, noYieldIfOneItem: true, signal},
@@ -38,24 +34,28 @@ export class PayloadEnvelopeProcessor {
   }
 
   async processPayloadEnvelopeJob(payloadInput: PayloadEnvelopeInput, opts: ImportPayloadOpts = {}): Promise<void> {
-    if (this.importStatus.get(payloadInput) !== undefined) {
-      return;
+    const existing = this.imports.get(payloadInput);
+    if (existing) {
+      return existing;
     }
 
     await this.jobQueue.waitForSpace();
 
     // Re-check after await, as another call may have queued this payload.
-    if (this.importStatus.get(payloadInput) !== undefined) {
-      return;
+    const queued = this.imports.get(payloadInput);
+    if (queued) {
+      return queued;
     }
 
-    this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.queued);
+    const importPromise = this.jobQueue.push(payloadInput, opts);
+    this.imports.set(payloadInput, importPromise);
 
     try {
-      await this.jobQueue.push(payloadInput, opts);
-      this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.imported);
+      await importPromise;
     } catch (e) {
-      this.importStatus.delete(payloadInput);
+      // Drop the failed import so a later attempt can retry once the failure cause is resolved
+      // (e.g. BLOCK_NOT_IN_FORK_CHOICE after the block lands in fork choice)
+      this.imports.delete(payloadInput);
       throw e;
     }
   }

package/src/chain/blocks/utils/chainSegment.ts

@@ -86,11 +86,22 @@ export function assertLinearChainSegment(
           // Maybe the previous slot's FULL envelope was orphaned — try falling back.
           // If even prevExecHash doesn't match, the segment is non-linear.
           if (bidParentHash !== prevExecHash) {
-            throw new BlockError(block, {
-              code: BlockErrorCode.PARENT_PAYLOAD_UNKNOWN,
-              parentRoot: toRootHex(block.message.parentRoot),
-              parentBlockHash: bidParentHash,
-            });
+            // i === 0 compares against the seeded fork-choice parent (segment boundary); i > 0
+            // compares against the in-segment predecessor's payload (broken link inside the segment).
+            throw new BlockError(
+              block,
+              i === 0
+                ? {
+                    code: BlockErrorCode.PARENT_PAYLOAD_UNKNOWN,
+                    parentRoot: toRootHex(block.message.parentRoot),
+                    parentBlockHash: bidParentHash,
+                  }
+                : {
+                    code: BlockErrorCode.NON_LINEAR_PAYLOAD_ROOTS,
+                    parentBlockHash: bidParentHash,
+                    expectedBlockHash: currentExecHash,
+                  }
+            );
           }
           if (lastFullSlot !== null && payloadEnvelopes !== null) {
             const orphanedInput = payloadEnvelopes.get(lastFullSlot);

package/src/chain/blocks/verifyBlocksExecutionPayloads.ts

@@ -201,7 +201,7 @@ export async function verifyBlockExecutionPayload(
         invalidateFromParentBlockHash: toRootHex(executionPayloadEnabled.parentHash),
       };
       const execError = new BlockError(block, {
-        code: BlockErrorCode.EXECUTION_ENGINE_ERROR,
+        code: BlockErrorCode.EXECUTION_ENGINE_INVALID,
         execStatus: execResult.status,
         errorMessage: execResult.validationError ?? "",
       });

package/src/chain/blocks/verifyBlocksSanityChecks.ts

@@ -94,7 +94,7 @@ export function verifyBlocksSanityChecks(
       const parentRoot = toRootHex(block.message.parentRoot);
       const parentBlockDefaultStatus = chain.forkChoice.getBlockHexDefaultStatus(parentRoot);
       if (!parentBlockDefaultStatus) {
-        throw new BlockError(block, {code: BlockErrorCode.PARENT_UNKNOWN, parentRoot});
+        throw new BlockError(block, {code: BlockErrorCode.PARENT_BLOCK_UNKNOWN, parentRoot});
       }
 
       parentBlock = parentBlockDefaultStatus;

package/src/chain/blocks/verifyExecutionPayloadEnvelope.ts

@@ -100,8 +100,8 @@ export function verifyExecutionPayloadEnvelope(
   }
 
   // Verify consistency with expected withdrawals
-  const payloadWithdrawalsRoot = ssz.capella.Withdrawals.hashTreeRoot(payload.withdrawals);
-  const expectedWithdrawalsRoot = ssz.capella.Withdrawals.hashTreeRoot(state.payloadExpectedWithdrawals);
+  const payloadWithdrawalsRoot = ssz.gloas.Withdrawals.hashTreeRoot(payload.withdrawals);
+  const expectedWithdrawalsRoot = ssz.gloas.Withdrawals.hashTreeRoot(state.payloadExpectedWithdrawals);
   if (!byteArrayEquals(payloadWithdrawalsRoot, expectedWithdrawalsRoot)) {
     throw new Error(
       `Withdrawals mismatch between payload and expected payload=${toRootHex(payloadWithdrawalsRoot)} expected=${toRootHex(expectedWithdrawalsRoot)}`

package/src/chain/builderCircuitBreaker.ts

@@ -0,0 +1,84 @@
+import {IForkChoice} from "@lodestar/fork-choice";
+import {Slot} from "@lodestar/types";
+import {Logger} from "@lodestar/utils";
+import {getFaultInspectionParams} from "../execution/builder/http.js";
+import {Metrics} from "../metrics/index.js";
+
+export type BuilderCircuitBreakerOpts = {
+  faultInspectionWindow?: number;
+  allowedFaults?: number;
+};
+
+export type BuilderCircuitBreakerModules = {
+  forkChoice: IForkChoice;
+  logger: Logger;
+  metrics: Metrics | null;
+};
+
+/**
+ * Post-gloas circuit breaker for builder bids. The beacon block is produced by the proposer
+ * regardless of bid source, so missed blocks are not a useful builder health signal. Instead
+ * count blocks whose payload was never revealed and stop selecting builder bids while the
+ * non-reveal rate in the fault inspection window is too high.
+ */
+export class BuilderCircuitBreaker {
+  readonly faultInspectionWindow: number;
+  readonly allowedFaults: number;
+
+  private active = false;
+  private lastUpdatedSlot = -1;
+
+  constructor(
+    opts: BuilderCircuitBreakerOpts,
+    private readonly modules: BuilderCircuitBreakerModules
+  ) {
+    const {faultInspectionWindow, allowedFaults} = getFaultInspectionParams(opts);
+    this.faultInspectionWindow = faultInspectionWindow;
+    this.allowedFaults = allowedFaults;
+  }
+
+  /** Whether builder bids must be ignored for a block produced at clockSlot */
+  isActive(clockSlot: Slot): boolean {
+    this.update(clockSlot);
+    return this.active;
+  }
+
+  update(clockSlot: Slot): void {
+    if (clockSlot <= this.lastUpdatedSlot) {
+      return;
+    }
+    this.lastUpdatedSlot = clockSlot;
+
+    // Exclude clockSlot itself, its payload reveal may still be in flight
+    const {blocksPresent, payloadsRevealed} = this.modules.forkChoice.getPayloadRevealCounts(
+      Math.max(clockSlot - this.faultInspectionWindow, 0),
+      clockSlot - 1
+    );
+    const faults = blocksPresent - payloadsRevealed;
+
+    const wasActive = this.active;
+    // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates
+    this.active = faults * this.faultInspectionWindow > this.allowedFaults * blocksPresent;
+
+    this.modules.metrics?.builderCircuitBreaker.active.set(this.active ? 1 : 0);
+    this.modules.metrics?.builderCircuitBreaker.faults.set(faults);
+    this.modules.metrics?.builderCircuitBreaker.blocksPresent.set(blocksPresent);
+    this.modules.metrics?.builderCircuitBreaker.payloadsRevealed.set(payloadsRevealed);
+
+    const logCtx = {
+      blocksPresent,
+      faults,
+      faultInspectionWindow: this.faultInspectionWindow,
+      allowedFaults: this.allowedFaults,
+    };
+    if (this.active !== wasActive) {
+      if (this.active) {
+        this.modules.logger.warn("Builder circuit breaker activated, ignoring builder bids", logCtx);
+      } else {
+        this.modules.logger.info("Builder circuit breaker deactivated", logCtx);
+      }
+    } else {
+      this.modules.logger.verbose("Builder circuit breaker status", {active: this.active, ...logCtx});
+    }
+  }
+}

package/src/chain/chain.ts

@@ -1,8 +1,16 @@
 import path from "node:path";
 import {PrivateKey} from "@libp2p/interface";
 import {Type} from "@chainsafe/ssz";
+import {routes} from "@lodestar/api";
 import {BeaconConfig} from "@lodestar/config";
-import {CheckpointWithHex, ForkChoiceStateGetter, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice";
+import {
+  CheckpointWithHex,
+  EpochDifference,
+  ForkChoiceStateGetter,
+  IForkChoice,
+  ProtoBlock,
+  UpdateHeadOpt,
+} from "@lodestar/fork-choice";
 import {LoggerNode} from "@lodestar/logger/node";
 import {
   EFFECTIVE_BALANCE_INCREMENT,
@@ -75,6 +83,7 @@ import {ImportPayloadOpts} from "./blocks/types.js";
 import {persistBlockInput} from "./blocks/writeBlockInputToDb.js";
 import {persistPayloadEnvelopeInput} from "./blocks/writePayloadEnvelopeInputToDb.js";
 import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js";
+import {BuilderCircuitBreaker} from "./builderCircuitBreaker.js";
 import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js";
 import {ChainEvent, ChainEventEmitter} from "./emitter.js";
 import {ForkchoiceCaller, initializeForkChoice} from "./forkChoice/index.js";
@@ -107,7 +116,6 @@ import {
   SeenExecutionPayloadBids,
   SeenPayloadAttesters,
   SeenPayloadEnvelopeInput,
-  SeenProposerPreferences,
   SeenSyncCommitteeMessages,
 } from "./seenCache/index.js";
 import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js";
@@ -152,6 +160,7 @@ export class BeaconChain implements IBeaconChain {
   readonly genesisValidatorsRoot: Root;
   readonly executionEngine: IExecutionEngine;
   readonly executionBuilder?: IExecutionBuilder;
+  readonly builderCircuitBreaker: BuilderCircuitBreaker;
   // Expose config for convenience in modularized functions
   readonly config: BeaconConfig;
   readonly custodyConfig: CustodyConfig;
@@ -189,7 +198,6 @@ export class BeaconChain implements IBeaconChain {
   readonly seenPayloadAttesters = new SeenPayloadAttesters();
   readonly seenAggregatedAttestations: SeenAggregatedAttestations;
   readonly seenExecutionPayloadBids = new SeenExecutionPayloadBids();
-  readonly seenProposerPreferences = new SeenProposerPreferences();
   readonly seenBlockProposers = new SeenBlockProposers();
   readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages();
   readonly seenContributionAndProof: SeenContributionAndProof;
@@ -427,6 +435,11 @@ export class BeaconChain implements IBeaconChain {
 
     this.forkChoice = forkChoice;
 
+    this.builderCircuitBreaker = new BuilderCircuitBreaker(
+      {faultInspectionWindow: opts.faultInspectionWindow, allowedFaults: opts.allowedFaults},
+      {forkChoice, logger, metrics}
+    );
+
     this.seenPayloadEnvelopeInputCache = new SeenPayloadEnvelopeInput({
       config,
       clock,
@@ -1174,7 +1187,27 @@ export class BeaconChain implements IBeaconChain {
     const timer = this.metrics?.forkChoice.findHead.startTimer({caller});
 
     try {
-      return this.forkChoice.updateAndGetHead({mode: UpdateHeadOpt.GetCanonicalHead}).head;
+      const prevHead = this.forkChoice.getHead();
+      const head = this.forkChoice.updateAndGetHead({mode: UpdateHeadOpt.GetCanonicalHead}).head;
+
+      if (head.blockRoot !== prevHead.blockRoot) {
+        try {
+          this.emitter.emit(routes.events.EventType.head, {
+            block: head.blockRoot,
+            epochTransition: computeStartSlotAtEpoch(computeEpochAtSlot(head.slot)) === head.slot,
+            slot: head.slot,
+            state: head.stateRoot,
+            previousDutyDependentRoot: this.forkChoice.getDependentRoot(head, EpochDifference.previous),
+            currentDutyDependentRoot: this.forkChoice.getDependentRoot(head, EpochDifference.current),
+            executionOptimistic: isOptimisticBlock(head),
+          });
+        } catch (e) {
+          // getDependentRoot() may fail with error: "No block for root" as we can see in holesky non-finality issue
+          this.logger.debug("Error emitting head event", {slot: head.slot, root: head.blockRoot}, e as Error);
+        }
+      }
+
+      return head;
     } catch (e) {
       this.metrics?.forkChoice.errors.inc({entrypoint: UpdateHeadOpt.GetCanonicalHead});
       throw e;
@@ -1204,6 +1237,7 @@ export class BeaconChain implements IBeaconChain {
     const secFromSlot = this.clock.secFromSlot(slot);
 
     try {
+      // Do not emit head event here, when proposing we rely on the one emitted when importing our own block
       const {head, isHeadTimely, notReorgedReason} = this.forkChoice.updateAndGetHead({
         mode: UpdateHeadOpt.GetProposerHead,
         secFromSlot,
@@ -1473,7 +1507,6 @@ export class BeaconChain implements IBeaconChain {
     this.payloadAttestationPool.prune(slot);
     this.executionPayloadBidPool.prune(slot);
     this.seenExecutionPayloadBids.prune(slot);
-    this.seenProposerPreferences.prune(slot);
     this.proposerPreferencesPool.prune(slot);
     this.seenAttestationDatas.onSlot(slot);
     this.reprocessController.onSlot(slot);

package/src/chain/errors/blockError.ts

@@ -8,8 +8,10 @@ import {GossipActionError} from "./gossipValidation.js";
 export enum BlockErrorCode {
   /** The prestate cannot be fetched */
   PRESTATE_MISSING = "BLOCK_ERROR_PRESTATE_MISSING",
-  /** The parent block was unknown. */
-  PARENT_UNKNOWN = "BLOCK_ERROR_PARENT_UNKNOWN",
+  /**
+   * The first block of a chain segment references a parentRoot unknown to fork-choice.
+   */
+  PARENT_BLOCK_UNKNOWN = "BLOCK_ERROR_PARENT_BLOCK_UNKNOWN",
   /** The block slot is greater than the present slot. */
   FUTURE_SLOT = "BLOCK_ERROR_FUTURE_SLOT",
   /** The block state_root does not match the generated state. */
@@ -38,7 +40,10 @@ export enum BlockErrorCode {
   NOT_FINALIZED_DESCENDANT = "BLOCK_ERROR_NOT_FINALIZED_DESCENDANT",
   /** The provided block is from an later slot than its parent. */
   NOT_LATER_THAN_PARENT = "BLOCK_ERROR_NOT_LATER_THAN_PARENT",
-  /** At least one block in the chain segment did not have it's parent root set to the root of the prior block. */
+  /**
+   * A block in the middle of a chain segment does not set its parentRoot to the previous block's root
+   * (broken parent-root link inside the segment).
+   */
   NON_LINEAR_PARENT_ROOTS = "BLOCK_ERROR_NON_LINEAR_PARENT_ROOTS",
   /** The slots of the blocks in the chain segment were not strictly increasing. */
   NON_LINEAR_SLOTS = "BLOCK_ERROR_NON_LINEAR_SLOTS",
@@ -61,16 +66,32 @@ export enum BlockErrorCode {
   TRANSACTIONS_TOO_BIG = "BLOCK_ERROR_TRANSACTIONS_TOO_BIG",
   /** Execution engine is unavailable, syncing, or api call errored. Peers must not be downscored on this code */
   EXECUTION_ENGINE_ERROR = "BLOCK_ERROR_EXECUTION_ERROR",
+  /**
+   * Execution engine returned a definitive INVALID verdict on the payload. Unlike
+   * EXECUTION_ENGINE_ERROR this is evidence about the data itself, not a local malfunction, so
+   * whoever served the block MAY be downscored. Mirrors PayloadErrorCode.EXECUTION_ENGINE_INVALID.
+   */
+  EXECUTION_ENGINE_INVALID = "BLOCK_ERROR_EXECUTION_INVALID",
   /** The blobs are unavailable */
   DATA_UNAVAILABLE = "BLOCK_ERROR_DATA_UNAVAILABLE",
   /** Block contains too many kzg commitments */
   TOO_MANY_KZG_COMMITMENTS = "BLOCK_ERROR_TOO_MANY_KZG_COMMITMENTS",
   /** Bid parent block root does not match block parent root */
   BID_PARENT_ROOT_MISMATCH = "BLOCK_ERROR_BID_PARENT_ROOT_MISMATCH",
+  /** A block body operation or parent execution request list exceeds its per-block limit */
+  TOO_MANY_BLOCK_OPERATIONS = "BLOCK_ERROR_TOO_MANY_BLOCK_OPERATIONS",
   /** The parent block's execution payload has been verified as invalid */
   PARENT_EXECUTION_INVALID = "BLOCK_ERROR_PARENT_EXECUTION_INVALID",
-  /** The block's parent execution payload (defined by bid.parent_block_hash) has not been seen */
+  /**
+   * [gloas] The first block of a chain segment references a parent execution payload
+   * (bid.parent_block_hash) unknown to fork-choice (segment boundary).
+   */
   PARENT_PAYLOAD_UNKNOWN = "BLOCK_ERROR_PARENT_PAYLOAD_UNKNOWN",
+  /**
+   * [gloas] A block in the middle of a chain segment has a bid.parent_block_hash that does not chain onto the
+   * previous block's execution payload (broken payload link inside the segment).
+   */
+  NON_LINEAR_PAYLOAD_ROOTS = "BLOCK_ERROR_NON_LINEAR_PAYLOAD_ROOTS",
   /** An execution payload envelope in the chain segment references a block root that does not match its slot's block */
   ENVELOPE_BLOCK_ROOT_MISMATCH = "BLOCK_ERROR_ENVELOPE_BLOCK_ROOT_MISMATCH",
 }
@@ -80,9 +101,18 @@ type ExecutionErrorStatus = Exclude<
   ExecutionPayloadStatus.VALID | ExecutionPayloadStatus.ACCEPTED | ExecutionPayloadStatus.SYNCING
 >;
 
+/**
+ * Statuses where the execution engine could NOT render a verdict on the payload: it is unreachable
+ * (UNAVAILABLE), errored (ELERROR), or returned a malformed/client-specific response
+ * (INVALID_BLOCK_HASH, dropped from engine_newPayloadV2+). None of these are evidence about the
+ * data, so peers must not be downscored on them. A definitive INVALID carries
+ * BlockErrorCode.EXECUTION_ENGINE_INVALID instead.
+ */
+type ExecutionEngineErrorStatus = Exclude<ExecutionErrorStatus, ExecutionPayloadStatus.INVALID>;
+
 export type BlockErrorType =
   | {code: BlockErrorCode.PRESTATE_MISSING; error: Error}
-  | {code: BlockErrorCode.PARENT_UNKNOWN; parentRoot: RootHex}
+  | {code: BlockErrorCode.PARENT_BLOCK_UNKNOWN; parentRoot: RootHex}
   | {code: BlockErrorCode.FUTURE_SLOT; blockSlot: Slot; currentSlot: Slot}
   | {code: BlockErrorCode.STATE_ROOT_MISMATCH}
   | {code: BlockErrorCode.GENESIS_BLOCK}
@@ -114,12 +144,19 @@ export type BlockErrorType =
   | {code: BlockErrorCode.TOO_MUCH_GAS_USED; gasUsed: number; gasLimit: number}
   | {code: BlockErrorCode.SAME_PARENT_HASH; blockHash: RootHex}
   | {code: BlockErrorCode.TRANSACTIONS_TOO_BIG; size: number; max: number}
-  | {code: BlockErrorCode.EXECUTION_ENGINE_ERROR; execStatus: ExecutionErrorStatus; errorMessage: string}
+  | {code: BlockErrorCode.EXECUTION_ENGINE_ERROR; execStatus: ExecutionEngineErrorStatus; errorMessage: string}
+  | {
+      code: BlockErrorCode.EXECUTION_ENGINE_INVALID;
+      execStatus: ExecutionPayloadStatus.INVALID;
+      errorMessage: string;
+    }
   | {code: BlockErrorCode.DATA_UNAVAILABLE}
   | {code: BlockErrorCode.TOO_MANY_KZG_COMMITMENTS; blobKzgCommitmentsLen: number; commitmentLimit: number}
   | {code: BlockErrorCode.BID_PARENT_ROOT_MISMATCH; bidParentRoot: RootHex; blockParentRoot: RootHex}
+  | {code: BlockErrorCode.TOO_MANY_BLOCK_OPERATIONS; name: string; count: number; limit: number}
   | {code: BlockErrorCode.PARENT_EXECUTION_INVALID; parentRoot: RootHex}
-  | {code: BlockErrorCode.PARENT_PAYLOAD_UNKNOWN; parentRoot: RootHex; parentBlockHash: RootHex};
+  | {code: BlockErrorCode.PARENT_PAYLOAD_UNKNOWN; parentRoot: RootHex; parentBlockHash: RootHex}
+  | {code: BlockErrorCode.NON_LINEAR_PAYLOAD_ROOTS; parentBlockHash: RootHex; expectedBlockHash: RootHex};
 
 export class BlockGossipError extends GossipActionError<BlockErrorType> {}
 

package/src/chain/errors/executionPayloadEnvelope.ts

@@ -12,6 +12,8 @@ export enum ExecutionPayloadEnvelopeErrorCode {
   BUILDER_INDEX_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BUILDER_INDEX_MISMATCH",
   BLOCK_HASH_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BLOCK_HASH_MISMATCH",
   EXECUTION_REQUESTS_ROOT_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_EXECUTION_REQUESTS_ROOT_MISMATCH",
+  EXECUTION_REQUESTS_COUNT_EXCEEDED = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_EXECUTION_REQUESTS_COUNT_EXCEEDED",
+  WITHDRAWALS_COUNT_EXCEEDED = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_WITHDRAWALS_COUNT_EXCEEDED",
   INVALID_SIGNATURE = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_INVALID_SIGNATURE",
   PAYLOAD_ENVELOPE_INPUT_MISSING = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_PAYLOAD_ENVELOPE_INPUT_MISSING",
 }
@@ -42,6 +44,13 @@ export type ExecutionPayloadEnvelopeErrorType =
       envelopeRequestsRoot: RootHex;
       bidRequestsRoot: RootHex;
     }
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.EXECUTION_REQUESTS_COUNT_EXCEEDED;
+      name: string;
+      count: number;
+      limit: number;
+    }
+  | {code: ExecutionPayloadEnvelopeErrorCode.WITHDRAWALS_COUNT_EXCEEDED; count: number; limit: number}
   | {code: ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE}
   | {code: ExecutionPayloadEnvelopeErrorCode.PAYLOAD_ENVELOPE_INPUT_MISSING; blockRoot: RootHex};
 

package/src/chain/errors/proposerPreferences.ts

@@ -1,10 +1,12 @@
-import {RootHex, Slot, ValidatorIndex} from "@lodestar/types";
+import {Epoch, RootHex, Slot, ValidatorIndex} from "@lodestar/types";
 import {GossipActionError} from "./gossipValidation.js";
 
 export enum ProposerPreferencesErrorCode {
   INVALID_EPOCH = "PROPOSER_PREFERENCES_ERROR_INVALID_EPOCH",
   PROPOSAL_SLOT_PASSED = "PROPOSER_PREFERENCES_ERROR_PROPOSAL_SLOT_PASSED",
   UNKNOWN_DEPENDENT_ROOT = "PROPOSER_PREFERENCES_ERROR_UNKNOWN_DEPENDENT_ROOT",
+  INVALID_DEPENDENT_ROOT_SLOT = "PROPOSER_PREFERENCES_ERROR_INVALID_DEPENDENT_ROOT_SLOT",
+  INVALID_DEPENDENT_ROOT = "PROPOSER_PREFERENCES_ERROR_INVALID_DEPENDENT_ROOT",
   INVALID_PROPOSER = "PROPOSER_PREFERENCES_ERROR_INVALID_PROPOSER",
   ALREADY_KNOWN = "PROPOSER_PREFERENCES_ERROR_ALREADY_KNOWN",
   INVALID_SIGNATURE = "PROPOSER_PREFERENCES_ERROR_INVALID_SIGNATURE",
@@ -26,6 +28,19 @@ export type ProposerPreferencesErrorType =
       proposalSlot: Slot;
       dependentRoot: RootHex;
     }
+  | {
+      code: ProposerPreferencesErrorCode.INVALID_DEPENDENT_ROOT_SLOT;
+      dependentRoot: RootHex;
+      dependentBlockSlot: Slot;
+      dependentRootSlot: Slot;
+      dependentEpoch: Epoch;
+    }
+  | {
+      code: ProposerPreferencesErrorCode.INVALID_DEPENDENT_ROOT;
+      dependentRoot: RootHex;
+      dependentBlockSlot: Slot;
+      dependentEpoch: Epoch;
+    }
   | {
       code: ProposerPreferencesErrorCode.INVALID_PROPOSER;
       proposalSlot: Slot;

package/src/chain/interface.ts

@@ -35,6 +35,7 @@ import {BeaconProposerCache, ProposerPreparationData} from "./beaconProposerCach
 import {IBlockInput} from "./blocks/blockInput/index.js";
 import {ImportBlockOpts, ImportPayloadOpts} from "./blocks/types.js";
 import {IBlsVerifier} from "./bls/index.js";
+import {BuilderCircuitBreaker} from "./builderCircuitBreaker.js";
 import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js";
 import {ChainEventEmitter} from "./emitter.js";
 import {ForkchoiceCaller} from "./forkChoice/index.js";
@@ -61,7 +62,6 @@ import {
   SeenContributionAndProof,
   SeenExecutionPayloadBids,
   SeenPayloadAttesters,
-  SeenProposerPreferences,
   SeenSyncCommitteeMessages,
 } from "./seenCache/index.js";
 import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js";
@@ -96,6 +96,7 @@ export interface IBeaconChain {
   readonly earliestAvailableSlot: Slot;
   readonly executionEngine: IExecutionEngine;
   readonly executionBuilder?: IExecutionBuilder;
+  readonly builderCircuitBreaker: BuilderCircuitBreaker;
   // Expose config for convenience in modularized functions
   readonly config: BeaconConfig;
   readonly custodyConfig: CustodyConfig;
@@ -133,7 +134,6 @@ export interface IBeaconChain {
   readonly seenPayloadAttesters: SeenPayloadAttesters;
   readonly seenAggregatedAttestations: SeenAggregatedAttestations;
   readonly seenExecutionPayloadBids: SeenExecutionPayloadBids;
-  readonly seenProposerPreferences: SeenProposerPreferences;
   readonly seenBlockProposers: SeenBlockProposers;
   readonly seenSyncCommitteeMessages: SeenSyncCommitteeMessages;
   readonly seenContributionAndProof: SeenContributionAndProof;