@lodestar/beacon-node 1.47.0-dev.931ecbec99 → 1.47.0-dev.9ba9a5ce85

package/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts

@@ -9,6 +9,7 @@ import {
   ColumnWithSource,
   CreateFromBidProps,
   CreateFromBlockProps,
+  PayloadEnvelopeInputSource,
   SourceMeta,
 } from "./types.js";
 
@@ -71,6 +72,7 @@ export class PayloadEnvelopeInput {
   readonly bid: gloas.ExecutionPayloadBid;
   readonly versionedHashes: VersionedHashes;
   readonly daOutOfRange: boolean;
+  readonly source: PayloadEnvelopeInputSource;
 
   private columnsCache = new Map<ColumnIndex, ColumnWithSource>();
 
@@ -95,6 +97,7 @@ export class PayloadEnvelopeInput {
     custodyColumns: ColumnIndex[];
     timeCreatedSec: number;
     daOutOfRange: boolean;
+    source: PayloadEnvelopeInputSource;
   }) {
     this.blockRootHex = props.blockRootHex;
     this.slot = props.slot;
@@ -106,6 +109,7 @@ export class PayloadEnvelopeInput {
     this.custodyColumns = props.custodyColumns;
     this.timeCreatedSec = props.timeCreatedSec;
     this.daOutOfRange = props.daOutOfRange;
+    this.source = props.source;
     this.payloadEnvelopeDataPromise = createPromise();
     this.allDataPromise = createPromise();
     this.columnsDataPromise = createPromise();
@@ -133,8 +137,9 @@ export class PayloadEnvelopeInput {
       bid,
       sampledColumns: props.sampledColumns,
       custodyColumns: props.custodyColumns,
-      timeCreatedSec: props.timeCreatedSec,
+      timeCreatedSec: props.seenTimestampSec,
       daOutOfRange: props.daOutOfRange,
+      source: props.source,
     });
   }
 
@@ -153,8 +158,9 @@ export class PayloadEnvelopeInput {
       bid: props.bid,
       sampledColumns: props.sampledColumns,
       custodyColumns: props.custodyColumns,
-      timeCreatedSec: props.timeCreatedSec,
+      timeCreatedSec: props.seenTimestampSec,
       daOutOfRange: props.daOutOfRange,
+      source: props.source,
     });
   }
 

package/src/chain/blocks/payloadEnvelopeInput/types.ts

@@ -7,9 +7,24 @@ export enum PayloadEnvelopeInputSource {
   engine = "engine",
   byRange = "req_resp_by_range",
   byRoot = "req_resp_by_root",
+  // Data-column reconstruction (KZG cell recovery), NOT a cache reload
   recovery = "recovery",
+  // Entry reconstructed from the hot DB by SeenPayloadEnvelopeInput.getOrReload
+  reload = "reload",
+  // Entry seeded from a checkpoint anchor state's latestExecutionPayloadBid at chain init
+  anchorState = "anchor_state",
 }
 
+/**
+ * Reason a PayloadEnvelopeInput is evicted from SeenPayloadEnvelopeInput. Used for the `pruned` metric
+ * label and the eviction log.
+ * - belowParent: pruned below the new head's parent (canonical, FULL, all-columns)
+ * - finalized: below the finalized slot
+ * - prune: explicit prune by root
+ * - cap: insertion-order backstop cap (MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE)
+ */
+export type PayloadEnvelopeInputPruneReason = "belowParent" | "finalized" | "prune" | "cap";
+
 export type SourceMeta = {
   source: PayloadEnvelopeInputSource;
   seenTimestampSec: number;
@@ -20,13 +35,12 @@ export type ColumnWithSource = SourceMeta & {
   columnSidecar: gloas.DataColumnSidecar;
 };
 
-export type CreateFromBlockProps = {
+export type CreateFromBlockProps = SourceMeta & {
   blockRootHex: RootHex;
   block: SignedBeaconBlock<ForkPostGloas>;
   forkName: ForkName;
   sampledColumns: ColumnIndex[];
   custodyColumns: ColumnIndex[];
-  timeCreatedSec: number;
   daOutOfRange: boolean;
 };
 
@@ -35,7 +49,7 @@ export type CreateFromBlockProps = {
  * the chain from a checkpoint anchor state — we have the bid via the state but not the
  * full SignedBeaconBlock).
  */
-export type CreateFromBidProps = {
+export type CreateFromBidProps = SourceMeta & {
   blockRootHex: RootHex;
   slot: number;
   forkName: ForkName;
@@ -43,7 +57,6 @@ export type CreateFromBidProps = {
   bid: gloas.ExecutionPayloadBid;
   sampledColumns: ColumnIndex[];
   custodyColumns: ColumnIndex[];
-  timeCreatedSec: number;
   daOutOfRange: boolean;
 };
 

package/src/chain/blocks/verifyBlock.ts

@@ -132,12 +132,15 @@ export async function verifyBlocksInEpoch(
     }> =
       fork >= ForkSeq.gloas
         ? (async () => {
-            // Validate DA for ALL payloads in the Map, not just those paired with blockInputs.
+            // Range sync creates a PayloadEnvelopeInput for every Gloas block, but an entry without
+            // an envelope represents a valid EMPTY slot and has no payload DA to verify.
             // A checkpoint-sync batch may include a payload for a slot whose block was filtered
             // out of relevantBlocks (e.g., the anchor at the finalized slot); that payload still
             // needs DA validation so it can be imported in processBlocks.
             const payloadInputsForDa: PayloadEnvelopeInput[] =
-              payloadEnvelopes !== null ? Array.from(payloadEnvelopes.values()) : [];
+              payloadEnvelopes !== null
+                ? Array.from(payloadEnvelopes.values()).filter((payloadInput) => payloadInput.hasPayloadEnvelope())
+                : [];
             const {dataAvailabilityStatuses, availableTime} = await verifyPayloadsDataAvailability(
               payloadInputsForDa,
               abortController.signal

package/src/chain/blocks/verifyBlocksStateTransitionOnly.ts

@@ -44,22 +44,31 @@ export async function verifyBlocksStateTransitionOnly(
     // STFN - per_slot_processing() + per_block_processing()
     // NOTE: `regen.getPreState()` should have dialed forward the state already caching checkpoint states
     const useBlsBatchVerify = !opts?.disableBlsBatchVerify;
-    const postState = preState.stateTransition(
-      block,
-      {
-        // NOTE: Assume valid for now while sending payload to execution engine in parallel
-        // Latter verifyBlocksInEpoch() will make sure that payload is indeed valid
-        executionPayloadStatus: ExecutionPayloadStatus.valid,
-        dataAvailabilityStatus,
-        // false because it's verified below with better error typing
-        verifyStateRoot: false,
-        // if block is trusted don't verify proposer or op signature
-        verifyProposer: !useBlsBatchVerify && !validSignatures && !validProposerSignature,
-        verifySignatures: !useBlsBatchVerify && !validSignatures,
-        dontTransferCache: false,
-      },
-      {metrics, validatorMonitor}
-    );
+    let postState: IBeaconStateView;
+    try {
+      postState = preState.stateTransition(
+        block,
+        {
+          // NOTE: Assume valid for now while sending payload to execution engine in parallel
+          // Latter verifyBlocksInEpoch() will make sure that payload is indeed valid
+          executionPayloadStatus: ExecutionPayloadStatus.valid,
+          dataAvailabilityStatus,
+          // false because it's verified below with better error typing
+          verifyStateRoot: false,
+          // if block is trusted don't verify proposer or op signature
+          verifyProposer: !useBlsBatchVerify && !validSignatures && !validProposerSignature,
+          verifySignatures: !useBlsBatchVerify && !validSignatures,
+          dontTransferCache: false,
+        },
+        {metrics, validatorMonitor}
+      );
+    } catch (e) {
+      throw new BlockError(block, {
+        code: BlockErrorCode.PER_BLOCK_PROCESSING_ERROR,
+        blockRoot: blocks[i].blockRootHex,
+        error: e as Error,
+      });
+    }
 
     const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({
       source: StateHashTreeRootSource.blockTransition,

package/src/chain/chain.ts

@@ -61,6 +61,7 @@ import {ProcessShutdownCallback} from "@lodestar/validator";
 import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js";
 import {IBeaconDb} from "../db/index.js";
 import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../db/repositories/blobSidecars.js";
+import {BuilderApiClient, BuilderApiClientOpts} from "../execution/builder/apiClient.js";
 import {BuilderStatus} from "../execution/builder/http.js";
 import {IExecutionBuilder, IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/index.js";
@@ -79,6 +80,7 @@ import {CheckpointBalancesCache} from "./balancesCache.js";
 import {BeaconProposerCache} from "./beaconProposerCache.js";
 import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blocks/blockInput/index.js";
 import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js";
+import {PayloadEnvelopeInputSource} from "./blocks/payloadEnvelopeInput/index.js";
 import {PayloadEnvelopeProcessor} from "./blocks/payloadEnvelopeProcessor.js";
 import {ImportPayloadOpts} from "./blocks/types.js";
 import {persistBlockInput} from "./blocks/writeBlockInputToDb.js";
@@ -94,6 +96,7 @@ import {LightClientServer} from "./lightClient/index.js";
 import {
   AggregatedAttestationPool,
   AttestationPool,
+  DeferredVoluntaryExitPool,
   ExecutionPayloadBidPool,
   OpPool,
   PayloadAttestationPool,
@@ -163,6 +166,7 @@ export class BeaconChain implements IBeaconChain {
   readonly executionEngine: IExecutionEngine;
   readonly executionBuilder?: IExecutionBuilder;
   readonly builderCircuitBreaker: BuilderCircuitBreaker;
+  readonly builderApiClient: BuilderApiClient;
   // Expose config for convenience in modularized functions
   readonly config: BeaconConfig;
   readonly custodyConfig: CustodyConfig;
@@ -193,6 +197,7 @@ export class BeaconChain implements IBeaconChain {
   readonly payloadAttestationPool: PayloadAttestationPool;
   readonly proposerPreferencesPool = new ProposerPreferencesPool();
   readonly opPool: OpPool;
+  readonly deferredVoluntaryExitPool: DeferredVoluntaryExitPool;
 
   // Gossip seen cache
   readonly seenAttesters = new SeenAttesters();
@@ -272,6 +277,7 @@ export class BeaconChain implements IBeaconChain {
       isAnchorStateFinalized,
       executionEngine,
       executionBuilder,
+      builderApiClientOpts,
     }: {
       privateKey: PrivateKey;
       config: BeaconConfig;
@@ -289,6 +295,7 @@ export class BeaconChain implements IBeaconChain {
       isAnchorStateFinalized: boolean;
       executionEngine: IExecutionEngine;
       executionBuilder?: IExecutionBuilder;
+      builderApiClientOpts?: BuilderApiClientOpts;
     }
   ) {
     this.opts = opts;
@@ -320,6 +327,7 @@ export class BeaconChain implements IBeaconChain {
     this.executionPayloadBidPool = new ExecutionPayloadBidPool();
     this.payloadAttestationPool = new PayloadAttestationPool(config, clock, metrics);
     this.opPool = new OpPool(config);
+    this.deferredVoluntaryExitPool = new DeferredVoluntaryExitPool(logger);
 
     this.seenAggregatedAttestations = new SeenAggregatedAttestations(metrics);
     this.seenContributionAndProof = new SeenContributionAndProof(metrics);
@@ -444,6 +452,8 @@ export class BeaconChain implements IBeaconChain {
       {forkChoice, logger, metrics}
     );
 
+    this.builderApiClient = new BuilderApiClient(builderApiClientOpts ?? {}, config, clock, bls, metrics, logger);
+
     this.seenPayloadEnvelopeInputCache = new SeenPayloadEnvelopeInput({
       config,
       clock,
@@ -451,6 +461,9 @@ export class BeaconChain implements IBeaconChain {
       chainEvents: emitter,
       signal,
       serializedCache: this.serializedCache,
+      db,
+      seenBlockInputCache: this.seenBlockInputCache,
+      custodyConfig: this.custodyConfig,
       metrics,
       logger,
     });
@@ -466,7 +479,8 @@ export class BeaconChain implements IBeaconChain {
         bid: anchorBid,
         sampledColumns: this.custodyConfig.sampledColumns,
         custodyColumns: this.custodyConfig.custodyColumns,
-        timeCreatedSec: Math.floor(Date.now() / 1000),
+        seenTimestampSec: Date.now() / 1000,
+        source: PayloadEnvelopeInputSource.anchorState,
       });
     }
 
@@ -1313,9 +1327,10 @@ export class BeaconChain implements IBeaconChain {
     }
   }
 
-  predictProposerHead(slot: Slot): ProtoBlock {
+  predictProposerHead(): ProtoBlock {
     this.metrics?.forkChoice.requests.inc();
     const timer = this.metrics?.forkChoice.findHead.startTimer({caller: FindHeadFnName.predictProposerHead});
+    const slot = this.clock.currentSlot;
     const secFromSlot = this.clock.secFromSlot(slot);
 
     try {
@@ -1569,6 +1584,7 @@ export class BeaconChain implements IBeaconChain {
     // aggregatedAttestationPool tracks metrics on its own
     metrics.opPool.attestationPool.size.set(this.attestationPool.getAttestationCount());
     metrics.opPool.attesterSlashingPoolSize.set(this.opPool.attesterSlashingsSize);
+    metrics.opPool.deferredVoluntaryExitPool.size.set(this.deferredVoluntaryExitPool.size());
     metrics.opPool.proposerSlashingPoolSize.set(this.opPool.proposerSlashingsSize);
     metrics.opPool.voluntaryExitPoolSize.set(this.opPool.voluntaryExitsSize);
     metrics.opPool.syncCommitteeMessagePoolSize.set(this.syncCommitteeMessagePool.size);

package/src/chain/errors/blockError.ts

@@ -136,7 +136,7 @@ export type BlockErrorType =
   | {code: BlockErrorCode.NON_LINEAR_PARENT_ROOTS}
   | {code: BlockErrorCode.NON_LINEAR_SLOTS}
   | {code: BlockErrorCode.ENVELOPE_BLOCK_ROOT_MISMATCH; envelopeBlockRoot: RootHex; blockRoot: RootHex}
-  | {code: BlockErrorCode.PER_BLOCK_PROCESSING_ERROR; error: Error}
+  | {code: BlockErrorCode.PER_BLOCK_PROCESSING_ERROR; blockRoot: RootHex; error: Error}
   | {code: BlockErrorCode.BEACON_CHAIN_ERROR; error: Error}
   | {code: BlockErrorCode.KNOWN_BAD_BLOCK}
   | {code: BlockErrorCode.BLACKLISTED_BLOCK}
@@ -184,13 +184,19 @@ export function isBlockErrorAborted(e: unknown): e is BlockError {
 export function renderBlockErrorType(type: BlockErrorType): Record<string, string | number | null> {
   switch (type.code) {
     case BlockErrorCode.PRESTATE_MISSING:
-    case BlockErrorCode.PER_BLOCK_PROCESSING_ERROR:
     case BlockErrorCode.BEACON_CHAIN_ERROR:
       return {
         code: type.code,
         error: type.error.message,
       };
 
+    case BlockErrorCode.PER_BLOCK_PROCESSING_ERROR:
+      return {
+        code: type.code,
+        blockRoot: type.blockRoot,
+        error: type.error.message,
+      };
+
     case BlockErrorCode.INVALID_SIGNATURE:
       return {
         code: type.code,

package/src/chain/errors/executionPayloadBid.ts

@@ -9,6 +9,7 @@ export enum ExecutionPayloadBidErrorCode {
   BID_ALREADY_KNOWN = "EXECUTION_PAYLOAD_BID_ERROR_BID_ALREADY_KNOWN",
   BID_TOO_LOW = "EXECUTION_PAYLOAD_BID_ERROR_BID_TOO_LOW",
   BID_TOO_HIGH = "EXECUTION_PAYLOAD_BID_ERROR_BID_TOO_HIGH",
+  BUILDER_MAY_EXIT = "EXECUTION_PAYLOAD_BID_ERROR_BUILDER_MAY_EXIT",
   TOO_MANY_KZG_COMMITMENTS = "EXECUTION_PAYLOAD_BID_ERROR_TOO_MANY_KZG_COMMITMENTS",
   UNKNOWN_BLOCK_ROOT = "EXECUTION_PAYLOAD_BID_ERROR_UNKNOWN_BLOCK_ROOT",
   UNKNOWN_PARENT_BLOCK_HASH = "EXECUTION_PAYLOAD_BID_ERROR_UNKNOWN_PARENT_BLOCK_HASH",
@@ -50,6 +51,11 @@ export type ExecutionPayloadBidErrorType =
     }
   | {code: ExecutionPayloadBidErrorCode.BID_TOO_LOW; bidValue: number; currentHighestBid: number}
   | {code: ExecutionPayloadBidErrorCode.BID_TOO_HIGH; bidValue: number; builderBalance: number}
+  | {
+      code: ExecutionPayloadBidErrorCode.BUILDER_MAY_EXIT;
+      builderIndex: BuilderIndex;
+      parentBlockRoot: RootHex;
+    }
   | {
       code: ExecutionPayloadBidErrorCode.TOO_MANY_KZG_COMMITMENTS;
       blobKzgCommitmentsLen: number;

package/src/chain/forkChoice/index.ts

@@ -134,6 +134,8 @@ export function initializeForkChoiceFromFinalizedState(
         stateRoot: toRootHex(blockHeader.stateRoot),
         blockRoot: toRootHex(checkpoint.root),
         timeliness: true, // Optimistically assume is timely
+        ptcTimeliness: true, // Spec: block_timeliness for anchor = [True, True]
+        proposerIndex: blockHeader.proposerIndex,
 
         justifiedEpoch: justifiedCheckpoint.epoch,
         justifiedRoot: toRootHex(justifiedCheckpoint.root),
@@ -230,6 +232,8 @@ export function initializeForkChoiceFromUnfinalizedState(
     blockRoot: headRoot,
     targetRoot: headRoot,
     timeliness: true, // Optimistically assume is timely
+    ptcTimeliness: true, // Spec: block_timeliness for anchor = [True, True]
+    proposerIndex: blockHeader.proposerIndex,
 
     justifiedEpoch: justifiedCheckpoint.epoch,
     justifiedRoot: toRootHex(justifiedCheckpoint.root),

package/src/chain/interface.ts

@@ -24,6 +24,7 @@ import {
   rewards,
 } from "@lodestar/types";
 import {Logger} from "@lodestar/utils";
+import {BuilderApiClient} from "../execution/builder/apiClient.js";
 import {IExecutionBuilder, IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/metrics.js";
 import {BufferPool} from "../util/bufferPool.js";
@@ -45,6 +46,7 @@ import {LightClientServer} from "./lightClient/index.js";
 import {AggregatedAttestationPool} from "./opPools/aggregatedAttestationPool.js";
 import {
   AttestationPool,
+  DeferredVoluntaryExitPool,
   ExecutionPayloadBidPool,
   OpPool,
   PayloadAttestationPool,
@@ -98,6 +100,7 @@ export interface IBeaconChain {
   readonly executionEngine: IExecutionEngine;
   readonly executionBuilder?: IExecutionBuilder;
   readonly builderCircuitBreaker: BuilderCircuitBreaker;
+  readonly builderApiClient: BuilderApiClient;
   // Expose config for convenience in modularized functions
   readonly config: BeaconConfig;
   readonly custodyConfig: CustodyConfig;
@@ -128,6 +131,7 @@ export interface IBeaconChain {
   readonly payloadAttestationPool: PayloadAttestationPool;
   readonly proposerPreferencesPool: ProposerPreferencesPool;
   readonly opPool: OpPool;
+  readonly deferredVoluntaryExitPool: DeferredVoluntaryExitPool;
 
   // Gossip seen cache
   readonly seenAttesters: SeenAttesters;
@@ -270,8 +274,8 @@ export interface IBeaconChain {
 
   recomputeForkChoiceHead(caller: ForkchoiceCaller): ProtoBlock;
 
-  /** When proposerBoostReorg is enabled, this is called at slot n-1 to predict the head block to build on if we are proposing at slot n */
-  predictProposerHead(slot: Slot): ProtoBlock;
+  /** When proposerBoostReorg is enabled, predict (as of the current slot) the head block a proposer would build on next slot */
+  predictProposerHead(): ProtoBlock;
 
   /** When proposerBoostReorg is enabled and we are proposing a block, this is called to determine which head block to build on */
   getProposerHead(slot: Slot): ProtoBlock;

package/src/chain/opPools/deferredVoluntaryExitPool.ts

@@ -0,0 +1,65 @@
+import {Logger} from "@lodestar/logger";
+import {IBeaconStateView, VoluntaryExitValidity} from "@lodestar/state-transition";
+import {Epoch, ValidatorIndex} from "@lodestar/types";
+import {SignedVoluntaryExit} from "@lodestar/types/phase0";
+import {isTransientExitValidity} from "../validation/voluntaryExit.js";
+
+type DeferredEntry = {
+  exit: SignedVoluntaryExit;
+  validity: VoluntaryExitValidity;
+  insertedAtEpoch: Epoch;
+};
+
+export class DeferredVoluntaryExitPool {
+  private pool = new Map<ValidatorIndex, DeferredEntry>();
+
+  constructor(
+    private readonly logger: Logger,
+    private readonly maxSize = 1024,
+    private readonly maxDeferEpochs = 4096
+  ) {}
+
+  insert(exit: SignedVoluntaryExit, validity: VoluntaryExitValidity, currentEpoch: Epoch): boolean {
+    if (!isTransientExitValidity(validity)) return false;
+    if (this.pool.size === this.maxSize) return false;
+    if (this.pool.has(exit.message.validatorIndex)) return false;
+
+    this.pool.set(exit.message.validatorIndex, {exit, validity, insertedAtEpoch: currentEpoch});
+
+    return true;
+  }
+
+  drainProcessableExits(state: IBeaconStateView): SignedVoluntaryExit[] {
+    const epoch = state.epoch;
+    const validExits: SignedVoluntaryExit[] = [];
+    for (const [validatorIndex, entry] of this.pool) {
+      try {
+        if (epoch - entry.insertedAtEpoch > this.maxDeferEpochs) {
+          this.pool.delete(validatorIndex);
+          this.logger.debug("Deferred voluntary exit expired", {validatorIndex, lastValidity: entry.validity});
+          continue;
+        }
+        const validity = state.getVoluntaryExitValidity(entry.exit, true);
+        if (validity === VoluntaryExitValidity.valid) {
+          validExits.push(entry.exit);
+          this.pool.delete(validatorIndex);
+        } else if (!isTransientExitValidity(validity)) {
+          this.logger.debug("Deferred voluntary exit became permanently invalid", {validatorIndex, validity});
+          this.pool.delete(validatorIndex);
+        }
+        // Else if still transient - keep
+      } catch (e) {
+        this.logger.debug("Processing deferred voluntary exit failed", {validatorIndex}, e as Error);
+      }
+    }
+    return validExits;
+  }
+
+  has(validatorIndex: ValidatorIndex): boolean {
+    return this.pool.has(validatorIndex);
+  }
+
+  size(): number {
+    return this.pool.size;
+  }
+}

package/src/chain/opPools/executionPayloadBidPool.ts

@@ -11,14 +11,20 @@ const SLOTS_RETAINED = 2;
 type BlockRootHex = string;
 type BlockHashHex = string;
 
+export type PooledExecutionPayloadBid = {
+  signedBid: gloas.SignedExecutionPayloadBid;
+  /** Time in milliseconds from the slot start when the bid was received */
+  receivedMs: number;
+};
+
 /**
  * Store the best signed execution payload bid per slot / (parent block root, parent block hash).
  */
 export class ExecutionPayloadBidPool {
   private readonly bidByParentHashByParentRootBySlot = new MapDef<
     Slot,
-    MapDef<BlockRootHex, Map<BlockHashHex, gloas.SignedExecutionPayloadBid>>
-  >(() => new MapDef<BlockRootHex, Map<BlockHashHex, gloas.SignedExecutionPayloadBid>>(() => new Map()));
+    MapDef<BlockRootHex, Map<BlockHashHex, PooledExecutionPayloadBid>>
+  >(() => new MapDef<BlockRootHex, Map<BlockHashHex, PooledExecutionPayloadBid>>(() => new Map()));
   private lowestPermissibleSlot = 0;
 
   get size(): number {
@@ -31,7 +37,7 @@ export class ExecutionPayloadBidPool {
     return count;
   }
 
-  add(bid: gloas.SignedExecutionPayloadBid): InsertOutcome {
+  add(bid: gloas.SignedExecutionPayloadBid, receivedMs: number): InsertOutcome {
     const {slot, parentBlockRoot, parentBlockHash, value} = bid.message;
     const lowestPermissibleSlot = this.lowestPermissibleSlot;
 
@@ -45,16 +51,16 @@ export class ExecutionPayloadBidPool {
     const existing = bidByParentHash.get(parentHashHex);
 
     if (existing) {
-      const existingValue = existing.message.value;
+      const existingValue = existing.signedBid.message.value;
       const newValue = value;
       if (newValue > existingValue) {
-        bidByParentHash.set(parentHashHex, bid);
+        bidByParentHash.set(parentHashHex, {signedBid: bid, receivedMs});
         return InsertOutcome.NewData;
       }
       return newValue === existingValue ? InsertOutcome.AlreadyKnown : InsertOutcome.NotBetterThan;
     }
 
-    bidByParentHash.set(parentHashHex, bid);
+    bidByParentHash.set(parentHashHex, {signedBid: bid, receivedMs});
     return InsertOutcome.NewData;
   }
 
@@ -66,7 +72,7 @@ export class ExecutionPayloadBidPool {
     slot: Slot,
     parentBlockHash: BlockHashHex | null,
     parentBlockRoot: BlockRootHex
-  ): gloas.SignedExecutionPayloadBid | null {
+  ): PooledExecutionPayloadBid | null {
     if (parentBlockHash === null) return null;
     const bidByParentHash = this.bidByParentHashByParentRootBySlot.get(slot)?.get(parentBlockRoot);
     return bidByParentHash?.get(parentBlockHash) ?? null;

package/src/chain/opPools/index.ts

@@ -1,6 +1,7 @@
 export {AggregatedAttestationPool} from "./aggregatedAttestationPool.js";
 export {AttestationPool} from "./attestationPool.js";
-export {ExecutionPayloadBidPool} from "./executionPayloadBidPool.js";
+export {DeferredVoluntaryExitPool} from "./deferredVoluntaryExitPool.js";
+export {ExecutionPayloadBidPool, type PooledExecutionPayloadBid} from "./executionPayloadBidPool.js";
 export {OpPool} from "./opPool.js";
 export {PayloadAttestationPool} from "./payloadAttestationPool.js";
 export {ProposerPreferencesPool} from "./proposerPreferencesPool.js";

package/src/chain/prepareNextSlot.ts

@@ -45,6 +45,14 @@ const PREPARE_EPOCH_LIMIT = 1;
  * Expressed in bps so it scales with slot duration (mainnet vs minimal/devnet). */
 const BUILDER_PREVERIFY_LIMIT_BPS = 2000;
 
+/**
+ * How far before the next slot builder connections are pre-established, see `checkBuilderStatusBeforeSlot`.
+ *
+ * 1667 = 16.67% of slot (2s on mainnet). Must stay under the few seconds an idle connection survives,
+ * while leaving room for a round trip to a distant builder to complete before the slot begins.
+ */
+const BUILDER_STATUS_CHECK_BEFORE_SLOT_BPS = 1667;
+
 /**
  * At Bellatrix, if we are responsible for proposing in next slot, we want to prepare payload
  * 4s before the start of next slot at PREPARE_NEXT_SLOT_BPS of the current slot.
@@ -151,25 +159,41 @@ export class PrepareNextSlotScheduler {
         const proposerIndex = await getProposerIndex();
         feeRecipient = this.chain.beaconProposerCache.get(proposerIndex);
 
-        if (feeRecipient) {
-          // If we are proposing next slot, we need to predict if we can proposer-boost-reorg or not
-          const proposerHead = this.chain.predictProposerHead(clockSlot);
+        // Predict the proposer head of the next slot when either:
+        //  - we are proposing the next slot (single-slot proposer-boost-reorg), or
+        //  - this is an epoch transition: post-Fulu the next proposer may reorg a weak
+        //    last-block-of-epoch and build slot 0 on the strong parent (epoch-boundary reorg),
+        //    so we should dial the strong parent through the boundary instead of the weak head.
+        if (feeRecipient || isEpochTransition) {
+          const proposerHead = this.chain.predictProposerHead();
           const {slot: proposerHeadSlot, blockRoot: proposerHeadRoot} = proposerHead;
 
-          // If we predict we can reorg, we build on the proposer head (parent) block instead
+          // If we predict a reorg, we build the epoch transition on the proposer head (parent) instead
           if (proposerHeadRoot !== headRoot || proposerHeadSlot !== headSlot) {
-            this.logger.verbose("Weak head detected. May build on parent block instead", {
+            // feeRecipient set => WE propose the next slot and may reorg it out ourselves;
+            // otherwise it's a predicted reorg by another proposer (epoch-boundary reorg).
+            const logMessage = feeRecipient
+              ? "Weak head detected. We may build on parent block instead"
+              : "Weak head detected. Another proposer may build on parent block instead";
+            this.logger.verbose(logMessage, {
               proposerHeadSlot,
               proposerHeadRoot,
               headSlot,
               headRoot,
+              isEpochTransition,
             });
             this.metrics?.weakHeadDetected.inc();
+            if (isEpochTransition) {
+              this.metrics?.precomputeNextEpochTransition.predictedReorg.inc();
+            }
             updatedHead = proposerHead;
           }
+        }
 
+        if (feeRecipient) {
           if (isForkPostGloas(fork)) {
             this.chain.builderCircuitBreaker.update(clockSlot, updatedHead);
+            this.checkBuilderStatusBeforeSlot(prepareSlot);
           } else {
             // Update the builder status, if enabled shoot an api call to check status
             this.chain.updateBuilderStatus(clockSlot);
@@ -182,6 +206,7 @@ export class PrepareNextSlotScheduler {
         }
 
         // post-fulu we'll always reach here because preparedState is undefined
+        // we may run epoch transition not with headBlock if it's weak
         if (preparedState === undefined || updatedHead !== headBlock) {
           preparedState = await this.chain.regen.getBlockSlotState(
             updatedHead,
@@ -280,7 +305,7 @@ export class PrepareNextSlotScheduler {
       //  + if next slot is a skipped slot, it'd help getting target checkpoint state faster to validate attestations
       if (isEpochTransition) {
         this.metrics?.precomputeNextEpochTransition.count.inc({result: "success"}, 1);
-        const previousHits = this.chain.regen.updatePreComputedCheckpoint(headRoot, nextEpoch);
+        const previousHits = this.chain.regen.updatePreComputedCheckpoint(updatedHead.blockRoot, nextEpoch);
         if (previousHits === 0) {
           this.metrics?.precomputeNextEpochTransition.waste.inc();
         }
@@ -289,6 +314,8 @@ export class PrepareNextSlotScheduler {
         this.logger.verbose("Completed PrepareNextSlotScheduler epoch transition", {
           nextEpoch,
           headSlot,
+          headRoot,
+          updatedHeadRoot: updatedHead.blockRoot,
           prepareSlot,
           previousHits,
           durationMs: Date.now() - start,
@@ -372,6 +399,22 @@ export class PrepareNextSlotScheduler {
     }
   };
 
+  /**
+   * Pre-establish the builder connections so the handshake lands outside the bid deadline. Scheduled
+   * later in the slot, connecting at PREPARE_NEXT_SLOT_BPS would go cold before bids are requested.
+   */
+  private checkBuilderStatusBeforeSlot(prepareSlot: Slot): void {
+    const msBeforeSlot = this.config.getSlotComponentDurationMs(BUILDER_STATUS_CHECK_BEFORE_SLOT_BPS);
+    const msUntilCheck = -this.chain.clock.msFromSlot(prepareSlot) - msBeforeSlot;
+    sleep(Math.max(0, msUntilCheck), this.signal)
+      .then(() => this.chain.builderApiClient.checkStatus())
+      .catch((e) => {
+        if (!isErrorAborted(e)) {
+          this.logger.debug("Failed to check builder status", {prepareSlot}, e as Error);
+        }
+      });
+  }
+
   computeStateHashTreeRoot(state: IBeaconStateView, isEpochTransition: boolean): void {
     // cache HashObjects for faster hashTreeRoot() later, especially for computeNewStateRoot() if we need to produce a block at slot 0 of epoch
     // see https://github.com/ChainSafe/lodestar/issues/6194