@lodestar/beacon-node 1.47.0-dev.d2f4b96574 → 1.47.0-dev.d477ac6ef7

package/src/chain/bls/multithread/utils.ts

@@ -17,3 +17,14 @@ export function chunkifyMaximizeChunkSize<T>(arr: T[], minPerChunk: number): T[]
 
   return arrArr;
 }
+
+/**
+ * Splits an array into chunks that do not exceed a maximum size.
+ */
+export function chunkifyMaxChunkSize<T>(arr: T[], maxPerChunk: number): T[][] {
+  const chunks: T[][] = [];
+  for (let i = 0; i < arr.length; i += maxPerChunk) {
+    chunks.push(arr.slice(i, i + maxPerChunk));
+  }
+  return chunks;
+}

package/src/chain/bls/multithread/worker.ts

@@ -1,8 +1,20 @@
 import worker from "node:worker_threads";
-import {PublicKey} from "@chainsafe/blst";
+import {
+  type BlsSignatureSet,
+  verifySignatureSets,
+  verifySignatureSetsSameMessage,
+} from "@chainsafe/lodestar-z/bls-verifier";
 import {expose} from "@chainsafe/threads/worker";
-import {SignatureSetDeserialized, verifySignatureSetsMaybeBatch} from "../maybeBatch.js";
-import {BlsWorkReq, BlsWorkResult, SerializedSet, WorkResult, WorkResultCode, WorkerData} from "./types.js";
+import {
+  BlsWorkReq,
+  BlsWorkResult,
+  JobQueueItemType,
+  VerificationCall,
+  VerificationCallOperation,
+  WorkResult,
+  WorkResultCode,
+  WorkerData,
+} from "./types.js";
 import {chunkifyMaximizeChunkSize} from "./utils.js";
 
 /**
@@ -27,23 +39,45 @@ expose({
 
 function verifyManySignatureSets(workReqArr: BlsWorkReq[]): BlsWorkResult {
   const [startSec, startNs] = process.hrtime();
-  const results: WorkResult<boolean>[] = [];
+  const results: WorkResult<boolean[]>[] = [];
+  const verificationCalls: VerificationCall[] = [];
   let batchRetries = 0;
   let batchSigsSuccess = 0;
 
   // If there are multiple batchable sets attempt batch verification with them
-  const batchableSets: {idx: number; sets: SignatureSetDeserialized[]}[] = [];
-  const nonBatchableSets: {idx: number; sets: SignatureSetDeserialized[]}[] = [];
+  const batchableSets: {idx: number; sets: BlsSignatureSet[]}[] = [];
+  const nonBatchableSets: {
+    idx: number;
+    sets: BlsSignatureSet[];
+    operation: VerificationCallOperation.single | VerificationCallOperation.fallback;
+  }[] = [];
 
   // Split sets between batchable and non-batchable preserving their original index in the req array
   for (let i = 0; i < workReqArr.length; i++) {
     const workReq = workReqArr[i];
-    const sets = workReq.sets.map(deserializeSet);
-
-    if (workReq.opts.batchable) {
-      batchableSets.push({idx: i, sets});
-    } else {
-      nonBatchableSets.push({idx: i, sets});
+    switch (workReq.type) {
+      case JobQueueItemType.default: {
+        const {sets} = workReq;
+        if (workReq.opts.batchable) {
+          batchableSets.push({idx: i, sets});
+        } else {
+          nonBatchableSets.push({idx: i, sets, operation: VerificationCallOperation.single});
+        }
+        break;
+      }
+      case JobQueueItemType.sameMessage:
+        try {
+          const result = recordVerificationCall(
+            verificationCalls,
+            VerificationCallOperation.sameMessage,
+            workReq.sets.length,
+            () => verifySignatureSetsSameMessage(workReq.sets, workReq.message)
+          );
+          results[i] = {code: WorkResultCode.success, result};
+        } catch (e) {
+          results[i] = {code: WorkResultCode.error, error: e as Error};
+        }
+        break;
     }
   }
 
@@ -52,7 +86,7 @@ function verifyManySignatureSets(workReqArr: BlsWorkReq[]): BlsWorkResult {
     const batchableChunks = chunkifyMaximizeChunkSize(batchableSets, BATCHABLE_MIN_PER_CHUNK);
 
     for (const batchableChunk of batchableChunks) {
-      const allSets: SignatureSetDeserialized[] = [];
+      const allSets: BlsSignatureSet[] = [];
       for (const {sets} of batchableChunk) {
         for (const set of sets) {
           allSets.push(set);
@@ -61,33 +95,49 @@ function verifyManySignatureSets(workReqArr: BlsWorkReq[]): BlsWorkResult {
 
       try {
         // Attempt to verify multiple sets at once
-        const isValid = verifySignatureSetsMaybeBatch(allSets);
+        const isValid = recordVerificationCall(verificationCalls, VerificationCallOperation.batch, allSets.length, () =>
+          verifySignatureSets(allSets)
+        );
 
         if (isValid) {
           // The entire batch is valid, return success to all
           for (const {idx, sets} of batchableChunk) {
             batchSigsSuccess += sets.length;
-            results[idx] = {code: WorkResultCode.success, result: isValid};
+            results[idx] = {code: WorkResultCode.success, result: [isValid]};
           }
         } else {
           batchRetries++;
           // Re-verify all sigs
-          nonBatchableSets.push(...batchableChunk);
+          nonBatchableSets.push(
+            ...batchableChunk.map(({idx, sets}) => ({
+              idx,
+              sets,
+              operation: VerificationCallOperation.fallback as const,
+            }))
+          );
         }
       } catch (_e) {
         // TODO: Ignore this error expecting that the same error will happen when re-verifying the set individually
-        //       It's not ideal but '@chainsafe/blst' may throw errors on some conditions
+        //       It's not ideal but the BLS implementation may throw errors on some conditions
         batchRetries++;
         // Re-verify all sigs
-        nonBatchableSets.push(...batchableChunk);
+        nonBatchableSets.push(
+          ...batchableChunk.map(({idx, sets}) => ({
+            idx,
+            sets,
+            operation: VerificationCallOperation.fallback as const,
+          }))
+        );
       }
     }
   }
 
-  for (const {idx, sets} of nonBatchableSets) {
+  for (const {idx, sets, operation} of nonBatchableSets) {
     try {
-      const isValid = verifySignatureSetsMaybeBatch(sets);
-      results[idx] = {code: WorkResultCode.success, result: isValid};
+      const isValid = recordVerificationCall(verificationCalls, operation, sets.length, () =>
+        verifySignatureSets(sets)
+      );
+      results[idx] = {code: WorkResultCode.success, result: [isValid]};
     } catch (e) {
       results[idx] = {code: WorkResultCode.error, error: e as Error};
     }
@@ -99,16 +149,24 @@ function verifyManySignatureSets(workReqArr: BlsWorkReq[]): BlsWorkResult {
     workerId,
     batchRetries,
     batchSigsSuccess,
+    verificationCalls,
     workerStartTime: [startSec, startNs],
     workerEndTime: [workerEndSec, workerEndNs],
     results,
   };
 }
 
-function deserializeSet(set: SerializedSet): SignatureSetDeserialized {
-  return {
-    publicKey: PublicKey.fromBytes(set.publicKey),
-    message: set.message,
-    signature: set.signature,
-  };
+function recordVerificationCall<T>(
+  calls: VerificationCall[],
+  operation: VerificationCallOperation,
+  signatureSets: number,
+  verify: () => T
+): T {
+  const startTime = process.hrtime();
+  try {
+    return verify();
+  } finally {
+    const [seconds, nanoseconds] = process.hrtime(startTime);
+    calls.push({operation, signatureSets, duration: seconds + nanoseconds / 1e9});
+  }
 }

package/src/chain/bls/singleThread.ts

@@ -1,31 +1,20 @@
-import {PublicKey, Signature, aggregatePublicKeys, aggregateSignatures, verify} from "@chainsafe/blst";
-import {ISignatureSet, PubkeyCache} from "@lodestar/state-transition";
+import {verifySignatureSetsSameMessage} from "@chainsafe/lodestar-z/bls-verifier";
+import {ISignatureSet} from "@lodestar/state-transition";
 import {Metrics} from "../../metrics/index.js";
-import {IBlsVerifier} from "./interface.js";
-import {verifySignatureSetsMaybeBatch} from "./maybeBatch.js";
-import {getAggregatedPubkey, getAggregatedPubkeysCount} from "./utils.js";
+import {IBlsVerifier, SameMessageSignatureSet} from "./interface.js";
+import {chunkSameMessageSignatureSets, getAggregatedPubkeysCount, verifySignatureSetsInBatches} from "./utils.js";
 
 export class BlsSingleThreadVerifier implements IBlsVerifier {
   private readonly metrics: Metrics | null;
-  private readonly pubkeyCache: PubkeyCache;
-
-  constructor({metrics = null, pubkeyCache}: {metrics: Metrics | null; pubkeyCache: PubkeyCache}) {
+  constructor({metrics = null}: {metrics: Metrics | null}) {
     this.metrics = metrics;
-    this.pubkeyCache = pubkeyCache;
   }
 
   async verifySignatureSets(sets: ISignatureSet[]): Promise<boolean> {
     this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets));
 
-    const setsAggregated = sets.map((set) => ({
-      publicKey: getAggregatedPubkey(set, this.pubkeyCache, this.metrics),
-      message: set.signingRoot,
-      signature: set.signature,
-    }));
-
-    // Count time after aggregating
     const timer = this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.startTimer();
-    const isValid = verifySignatureSetsMaybeBatch(setsAggregated);
+    const isValid = verifySignatureSetsInBatches(sets);
 
     // Don't use a try/catch, only count run without exceptions
     if (timer) {
@@ -35,46 +24,13 @@ export class BlsSingleThreadVerifier implements IBlsVerifier {
     return isValid;
   }
 
-  async verifySignatureSetsSameMessage(
-    sets: {publicKey: PublicKey; signature: Uint8Array}[],
-    message: Uint8Array
-  ): Promise<boolean[]> {
+  async verifySignatureSetsSameMessage(sets: SameMessageSignatureSet[], message: Uint8Array): Promise<boolean[]> {
     const timer = this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.startTimer();
-    const pubkey = aggregatePublicKeys(sets.map((set) => set.publicKey));
-    let isAllValid = true;
-    // validate signature = true
-    const signatures = sets.map((set) => {
-      try {
-        return Signature.fromBytes(set.signature, true);
-      } catch (_) {
-        // at least one set has malformed signature
-        isAllValid = false;
-        return null;
-      }
-    });
-
-    if (isAllValid) {
-      const signature = aggregateSignatures(signatures as Signature[]);
-      isAllValid = verify(message, pubkey, signature);
-    }
-
-    let result: boolean[];
-    if (isAllValid) {
-      result = sets.map(() => true);
-    } else {
-      result = sets.map((set, i) => {
-        const sig = signatures[i];
-        if (sig === null) {
-          return false;
-        }
-        return verify(message, set.publicKey, sig);
-      });
+    const result: boolean[] = [];
+    for (const setsChunk of chunkSameMessageSignatureSets(sets)) {
+      result.push(...verifySignatureSetsSameMessage(setsChunk, message));
     }
-
-    if (timer) {
-      timer();
-    }
-
+    timer?.();
     return result;
   }
 

package/src/chain/bls/utils.ts

@@ -1,33 +1,32 @@
-import {PublicKey, aggregatePublicKeys} from "@chainsafe/blst";
-import {ISignatureSet, PubkeyCache, SignatureSetType} from "@lodestar/state-transition";
-import {Metrics} from "../../metrics/metrics.js";
+import {
+  BLS_VERIFIER_MAX_BATCH_SIZE,
+  BLS_VERIFIER_MAX_SAME_MESSAGE_BATCH_SIZE,
+  verifySignatureSets,
+} from "@chainsafe/lodestar-z/bls-verifier";
+import {ISignatureSet, SignatureSetType, toBlsSignatureSet} from "@lodestar/state-transition";
+import {SameMessageSignatureSet} from "./interface.js";
 
-export function getAggregatedPubkey(
-  signatureSet: ISignatureSet,
-  pubkeyCache: PubkeyCache,
-  metrics: Metrics | null = null
-): PublicKey {
-  switch (signatureSet.type) {
-    case SignatureSetType.single:
-      return signatureSet.pubkey;
+export function verifySignatureSetsInBatches(signatureSets: ISignatureSet[]): boolean {
+  if (signatureSets.length === 0) {
+    return false;
+  }
 
-    case SignatureSetType.indexed: {
-      return pubkeyCache.getOrThrow(signatureSet.index);
+  for (let start = 0; start < signatureSets.length; start += BLS_VERIFIER_MAX_BATCH_SIZE) {
+    const sets = signatureSets.slice(start, start + BLS_VERIFIER_MAX_BATCH_SIZE).map(toBlsSignatureSet);
+    if (!verifySignatureSets(sets)) {
+      return false;
     }
+  }
 
-    case SignatureSetType.aggregate: {
-      const timer = metrics?.blsThreadPool.pubkeysAggregationMainThreadDuration.startTimer();
-      const pubkeys = signatureSet.indices.map((i) => {
-        return pubkeyCache.getOrThrow(i);
-      });
-      const aggregated = aggregatePublicKeys(pubkeys);
-      timer?.();
-      return aggregated;
-    }
+  return true;
+}
 
-    default:
-      throw Error("Unknown signature set type");
+export function chunkSameMessageSignatureSets(sets: SameMessageSignatureSet[]): SameMessageSignatureSet[][] {
+  const chunks: SameMessageSignatureSet[][] = [];
+  for (let start = 0; start < sets.length; start += BLS_VERIFIER_MAX_SAME_MESSAGE_BATCH_SIZE) {
+    chunks.push(sets.slice(start, start + BLS_VERIFIER_MAX_SAME_MESSAGE_BATCH_SIZE));
   }
+  return chunks;
 }
 
 export function getAggregatedPubkeysCount(signatureSets: ISignatureSet[]): number {

package/src/chain/builderCircuitBreaker.ts

@@ -1,4 +1,4 @@
-import {IForkChoice} from "@lodestar/fork-choice";
+import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {Slot} from "@lodestar/types";
 import {Logger} from "@lodestar/utils";
 import {getFaultInspectionParams} from "../execution/builder/http.js";
@@ -15,11 +15,15 @@ export type BuilderCircuitBreakerModules = {
   metrics: Metrics | null;
 };
 
+/** Four observations is the minimum useful recovery sample for the default ~25% fault budget */
+const MIN_BLOCKS_TO_DEACTIVATE = 4;
+
 /**
  * Post-gloas circuit breaker for builder bids. The beacon block is produced by the proposer
  * regardless of bid source, so missed blocks are not a useful builder health signal. Instead
- * count blocks whose payload was never revealed and stop selecting builder bids while the
- * non-reveal rate in the fault inspection window is too high.
+ * count canonical blocks selected as EMPTY. Activate when the proportion of EMPTY blocks exceeds
+ * the fault budget, and resume selecting builder bids only when the observed blocks are within
+ * budget and meet the minimum recovery sample size.
  */
 export class BuilderCircuitBreaker {
   readonly faultInspectionWindow: number;
@@ -35,39 +39,49 @@ export class BuilderCircuitBreaker {
     const {faultInspectionWindow, allowedFaults} = getFaultInspectionParams(opts);
     this.faultInspectionWindow = faultInspectionWindow;
     this.allowedFaults = allowedFaults;
+    this.modules.logger.info("Builder circuit breaker initialized", {faultInspectionWindow, allowedFaults});
   }
 
   /** Whether builder bids must be ignored for a block produced at clockSlot */
-  isActive(clockSlot: Slot): boolean {
-    this.update(clockSlot);
+  isActive(clockSlot: Slot, head: ProtoBlock): boolean {
+    this.update(clockSlot, head);
     return this.active;
   }
 
-  update(clockSlot: Slot): void {
+  update(clockSlot: Slot, head: ProtoBlock): void {
     if (clockSlot <= this.lastUpdatedSlot) {
       return;
     }
     this.lastUpdatedSlot = clockSlot;
 
-    // Exclude clockSlot itself, its payload reveal may still be in flight
-    const {blocksPresent, payloadsRevealed} = this.modules.forkChoice.getPayloadRevealCounts(
+    // Exclude clockSlot itself, its payload status may still be unresolved
+    const {full, empty} = this.modules.forkChoice.getCanonicalPayloadCounts(
       Math.max(clockSlot - this.faultInspectionWindow, 0),
-      clockSlot - 1
+      clockSlot - 1,
+      head
     );
-    const faults = blocksPresent - payloadsRevealed;
+    const canonicalBlocks = full + empty;
 
     const wasActive = this.active;
-    // Scale the fault budget by blocks present so sparse windows still trigger on high non-reveal rates
-    this.active = faults * this.faultInspectionWindow > this.allowedFaults * blocksPresent;
+    // Scale the fault budget by canonical blocks so sparse windows still trigger on high EMPTY rates
+    const exceedsFaultBudget = empty * this.faultInspectionWindow > this.allowedFaults * canonicalBlocks;
+    if (exceedsFaultBudget) {
+      this.active = true;
+    } else if (canonicalBlocks >= MIN_BLOCKS_TO_DEACTIVATE) {
+      // Require a minimum sample within the fault budget before accepting builder bids again
+      this.active = false;
+    }
 
     this.modules.metrics?.builderCircuitBreaker.active.set(this.active ? 1 : 0);
-    this.modules.metrics?.builderCircuitBreaker.faults.set(faults);
-    this.modules.metrics?.builderCircuitBreaker.blocksPresent.set(blocksPresent);
-    this.modules.metrics?.builderCircuitBreaker.payloadsRevealed.set(payloadsRevealed);
+    this.modules.metrics?.builderCircuitBreaker.canonicalBlocks.set(canonicalBlocks);
+    this.modules.metrics?.builderCircuitBreaker.fullBlocks.set(full);
+    this.modules.metrics?.builderCircuitBreaker.emptyBlocks.set(empty);
 
     const logCtx = {
-      blocksPresent,
-      faults,
+      clockSlot,
+      canonicalBlocks,
+      full,
+      empty,
       faultInspectionWindow: this.faultInspectionWindow,
       allowedFaults: this.allowedFaults,
     };

package/src/chain/chain.ts

@@ -1,5 +1,6 @@
 import path from "node:path";
 import {PrivateKey} from "@libp2p/interface";
+import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
 import {Type} from "@chainsafe/ssz";
 import {routes} from "@lodestar/api";
 import {BeaconConfig} from "@lodestar/config";
@@ -25,7 +26,6 @@ import {
   EffectiveBalanceIncrements,
   EpochShuffling,
   IBeaconStateView,
-  PubkeyCache,
   computeEndSlotAtEpoch,
   computeEpochAtSlot,
   computeStartSlotAtEpoch,
@@ -61,6 +61,7 @@ import {ProcessShutdownCallback} from "@lodestar/validator";
 import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js";
 import {IBeaconDb} from "../db/index.js";
 import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../db/repositories/blobSidecars.js";
+import {BuilderApiClient, BuilderApiClientOpts} from "../execution/builder/apiClient.js";
 import {BuilderStatus} from "../execution/builder/http.js";
 import {IExecutionBuilder, IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/index.js";
@@ -79,6 +80,7 @@ import {CheckpointBalancesCache} from "./balancesCache.js";
 import {BeaconProposerCache} from "./beaconProposerCache.js";
 import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blocks/blockInput/index.js";
 import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js";
+import {PayloadEnvelopeInputSource} from "./blocks/payloadEnvelopeInput/index.js";
 import {PayloadEnvelopeProcessor} from "./blocks/payloadEnvelopeProcessor.js";
 import {ImportPayloadOpts} from "./blocks/types.js";
 import {persistBlockInput} from "./blocks/writeBlockInputToDb.js";
@@ -94,6 +96,7 @@ import {LightClientServer} from "./lightClient/index.js";
 import {
   AggregatedAttestationPool,
   AttestationPool,
+  DeferredVoluntaryExitPool,
   ExecutionPayloadBidPool,
   OpPool,
   PayloadAttestationPool,
@@ -163,6 +166,7 @@ export class BeaconChain implements IBeaconChain {
   readonly executionEngine: IExecutionEngine;
   readonly executionBuilder?: IExecutionBuilder;
   readonly builderCircuitBreaker: BuilderCircuitBreaker;
+  readonly builderApiClient: BuilderApiClient;
   // Expose config for convenience in modularized functions
   readonly config: BeaconConfig;
   readonly custodyConfig: CustodyConfig;
@@ -193,6 +197,7 @@ export class BeaconChain implements IBeaconChain {
   readonly payloadAttestationPool: PayloadAttestationPool;
   readonly proposerPreferencesPool = new ProposerPreferencesPool();
   readonly opPool: OpPool;
+  readonly deferredVoluntaryExitPool: DeferredVoluntaryExitPool;
 
   // Gossip seen cache
   readonly seenAttesters = new SeenAttesters();
@@ -272,6 +277,7 @@ export class BeaconChain implements IBeaconChain {
       isAnchorStateFinalized,
       executionEngine,
       executionBuilder,
+      builderApiClientOpts,
     }: {
       privateKey: PrivateKey;
       config: BeaconConfig;
@@ -289,6 +295,7 @@ export class BeaconChain implements IBeaconChain {
       isAnchorStateFinalized: boolean;
       executionEngine: IExecutionEngine;
       executionBuilder?: IExecutionBuilder;
+      builderApiClientOpts?: BuilderApiClientOpts;
     }
   ) {
     this.opts = opts;
@@ -307,8 +314,8 @@ export class BeaconChain implements IBeaconChain {
     const emitter = new ChainEventEmitter();
     // by default, verify signatures on both main threads and worker threads
     const bls = opts.blsVerifyAllMainThread
-      ? new BlsSingleThreadVerifier({metrics, pubkeyCache})
-      : new BlsMultiThreadWorkerPool(opts, {logger, metrics, pubkeyCache});
+      ? new BlsSingleThreadVerifier({metrics})
+      : new BlsMultiThreadWorkerPool(opts, {logger, metrics});
 
     if (!clock) clock = new Clock({config, genesisTime: this.genesisTime, signal});
 
@@ -320,6 +327,7 @@ export class BeaconChain implements IBeaconChain {
     this.executionPayloadBidPool = new ExecutionPayloadBidPool();
     this.payloadAttestationPool = new PayloadAttestationPool(config, clock, metrics);
     this.opPool = new OpPool(config);
+    this.deferredVoluntaryExitPool = new DeferredVoluntaryExitPool(logger);
 
     this.seenAggregatedAttestations = new SeenAggregatedAttestations(metrics);
     this.seenContributionAndProof = new SeenContributionAndProof(metrics);
@@ -444,6 +452,8 @@ export class BeaconChain implements IBeaconChain {
       {forkChoice, logger, metrics}
     );
 
+    this.builderApiClient = new BuilderApiClient(builderApiClientOpts ?? {}, config, clock, bls, metrics, logger);
+
     this.seenPayloadEnvelopeInputCache = new SeenPayloadEnvelopeInput({
       config,
       clock,
@@ -451,6 +461,9 @@ export class BeaconChain implements IBeaconChain {
       chainEvents: emitter,
       signal,
       serializedCache: this.serializedCache,
+      db,
+      seenBlockInputCache: this.seenBlockInputCache,
+      custodyConfig: this.custodyConfig,
       metrics,
       logger,
     });
@@ -466,7 +479,8 @@ export class BeaconChain implements IBeaconChain {
         bid: anchorBid,
         sampledColumns: this.custodyConfig.sampledColumns,
         custodyColumns: this.custodyConfig.custodyColumns,
-        timeCreatedSec: Math.floor(Date.now() / 1000),
+        seenTimestampSec: Date.now() / 1000,
+        source: PayloadEnvelopeInputSource.anchorState,
       });
     }
 
@@ -579,7 +593,7 @@ export class BeaconChain implements IBeaconChain {
       //   seenAggregators = single aggregator index, not participants of the aggregate
       this.seenAggregators.isKnown(epoch, index) ||
       //   seenPayloadAttesters = single signer of payload attestation message
-      this.seenPayloadAttesters.isKnown(epoch, index) ||
+      this.seenPayloadAttesters.seenAtEpoch(epoch, index) ||
       //   seenBlockProposers = single block proposer
       this.seenBlockProposers.seenAtEpoch(epoch, index)
     );
@@ -1072,6 +1086,7 @@ export class BeaconChain implements IBeaconChain {
       graffiti,
       slot,
       feeRecipient,
+      strictFeeRecipientCheck,
       commonBlockBodyPromise,
       parentBlock,
       builderBid,
@@ -1089,7 +1104,7 @@ export class BeaconChain implements IBeaconChain {
       RegenCaller.produceBlock
     );
     const proposerIndex = state.getBeaconProposer(slot);
-    const proposerPubKey = this.pubkeyCache.getOrThrow(proposerIndex).toBytes();
+    const proposerPubKey = this.pubkeyCache.getPubkeyBytesOrThrow(proposerIndex);
 
     const {body, produceResult, executionPayloadValue, shouldOverrideBuilder} = await produceBlockBody.call(
       this,
@@ -1100,6 +1115,7 @@ export class BeaconChain implements IBeaconChain {
         graffiti,
         slot,
         feeRecipient,
+        strictFeeRecipientCheck,
         parentBlock,
         proposerIndex,
         proposerPubKey,
@@ -1311,9 +1327,10 @@ export class BeaconChain implements IBeaconChain {
     }
   }
 
-  predictProposerHead(slot: Slot): ProtoBlock {
+  predictProposerHead(): ProtoBlock {
     this.metrics?.forkChoice.requests.inc();
     const timer = this.metrics?.forkChoice.findHead.startTimer({caller: FindHeadFnName.predictProposerHead});
+    const slot = this.clock.currentSlot;
     const secFromSlot = this.clock.secFromSlot(slot);
 
     try {
@@ -1567,6 +1584,7 @@ export class BeaconChain implements IBeaconChain {
     // aggregatedAttestationPool tracks metrics on its own
     metrics.opPool.attestationPool.size.set(this.attestationPool.getAttestationCount());
     metrics.opPool.attesterSlashingPoolSize.set(this.opPool.attesterSlashingsSize);
+    metrics.opPool.deferredVoluntaryExitPool.size.set(this.deferredVoluntaryExitPool.size());
     metrics.opPool.proposerSlashingPoolSize.set(this.opPool.proposerSlashingsSize);
     metrics.opPool.voluntaryExitPoolSize.set(this.opPool.voluntaryExitsSize);
     metrics.opPool.syncCommitteeMessagePoolSize.set(this.syncCommitteeMessagePool.size);
@@ -1575,6 +1593,8 @@ export class BeaconChain implements IBeaconChain {
     // syncContributionAndProofPool tracks metrics on its own
     metrics.opPool.blsToExecutionChangePoolSize.set(this.opPool.blsToExecutionChangeSize);
     metrics.chain.blacklistedBlocks.set(this.blacklistedBlocks.size);
+    metrics.bls.pubkeyCacheSize.set(this.pubkeyCache.size);
+    metrics.bls.pubkeyCacheCapacity.set(this.pubkeyCache.capacity);
 
     const headState = this.getHeadState();
     if (isStatePostElectra(headState)) {

package/src/chain/errors/blockError.ts

@@ -118,7 +118,7 @@ export type BlockErrorType =
   | {code: BlockErrorCode.GENESIS_BLOCK}
   | {code: BlockErrorCode.WOULD_REVERT_FINALIZED_SLOT; blockSlot: Slot; finalizedSlot: Slot}
   | {code: BlockErrorCode.ALREADY_KNOWN; root: RootHex}
-  | {code: BlockErrorCode.REPEAT_PROPOSAL; proposerIndex: ValidatorIndex}
+  | {code: BlockErrorCode.REPEAT_PROPOSAL; proposerIndex: ValidatorIndex; root: RootHex}
   | {code: BlockErrorCode.BLOCK_SLOT_LIMIT_REACHED}
   | {code: BlockErrorCode.INCORRECT_PROPOSER; proposerIndex: ValidatorIndex}
   | {code: BlockErrorCode.PROPOSAL_SIGNATURE_INVALID; blockSlot: Slot}
@@ -136,7 +136,7 @@ export type BlockErrorType =
   | {code: BlockErrorCode.NON_LINEAR_PARENT_ROOTS}
   | {code: BlockErrorCode.NON_LINEAR_SLOTS}
   | {code: BlockErrorCode.ENVELOPE_BLOCK_ROOT_MISMATCH; envelopeBlockRoot: RootHex; blockRoot: RootHex}
-  | {code: BlockErrorCode.PER_BLOCK_PROCESSING_ERROR; error: Error}
+  | {code: BlockErrorCode.PER_BLOCK_PROCESSING_ERROR; blockRoot: RootHex; error: Error}
   | {code: BlockErrorCode.BEACON_CHAIN_ERROR; error: Error}
   | {code: BlockErrorCode.KNOWN_BAD_BLOCK}
   | {code: BlockErrorCode.BLACKLISTED_BLOCK}
@@ -184,13 +184,19 @@ export function isBlockErrorAborted(e: unknown): e is BlockError {
 export function renderBlockErrorType(type: BlockErrorType): Record<string, string | number | null> {
   switch (type.code) {
     case BlockErrorCode.PRESTATE_MISSING:
-    case BlockErrorCode.PER_BLOCK_PROCESSING_ERROR:
     case BlockErrorCode.BEACON_CHAIN_ERROR:
       return {
         code: type.code,
         error: type.error.message,
       };
 
+    case BlockErrorCode.PER_BLOCK_PROCESSING_ERROR:
+      return {
+        code: type.code,
+        blockRoot: type.blockRoot,
+        error: type.error.message,
+      };
+
     case BlockErrorCode.INVALID_SIGNATURE:
       return {
         code: type.code,

package/src/chain/errors/proposerPreferences.ts

@@ -1,7 +1,9 @@
+import {ForkName} from "@lodestar/params";
 import {Epoch, RootHex, Slot, ValidatorIndex} from "@lodestar/types";
 import {GossipActionError} from "./gossipValidation.js";
 
 export enum ProposerPreferencesErrorCode {
+  PRE_GLOAS_PROPOSAL_SLOT = "PROPOSER_PREFERENCES_ERROR_PRE_GLOAS_PROPOSAL_SLOT",
   INVALID_EPOCH = "PROPOSER_PREFERENCES_ERROR_INVALID_EPOCH",
   PROPOSAL_SLOT_PASSED = "PROPOSER_PREFERENCES_ERROR_PROPOSAL_SLOT_PASSED",
   UNKNOWN_DEPENDENT_ROOT = "PROPOSER_PREFERENCES_ERROR_UNKNOWN_DEPENDENT_ROOT",
@@ -13,6 +15,11 @@ export enum ProposerPreferencesErrorCode {
 }
 
 export type ProposerPreferencesErrorType =
+  | {
+      code: ProposerPreferencesErrorCode.PRE_GLOAS_PROPOSAL_SLOT;
+      proposalSlot: Slot;
+      proposalFork: ForkName;
+    }
   | {
       code: ProposerPreferencesErrorCode.INVALID_EPOCH;
       proposalSlot: Slot;