@lodestar/beacon-node 1.47.0-dev.be9d5b774d → 1.47.0-dev.beabd3a65c

package/src/chain/seenCache/seenPayloadEnvelopeInput.ts

@@ -1,18 +1,32 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {CheckpointWithHex, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
+import {ForkPostGloas, SLOTS_PER_EPOCH, isForkPostGloas} from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
-import {RootHex} from "@lodestar/types";
-import {Logger} from "@lodestar/utils";
+import {RootHex, SignedBeaconBlock} from "@lodestar/types";
+import {Logger, fromHex} from "@lodestar/utils";
+import {IBeaconDb} from "../../db/index.js";
 import {Metrics} from "../../metrics/metrics.js";
+import {MAX_LOOK_AHEAD_EPOCHS} from "../../sync/constants.js";
 import {IClock} from "../../util/clock.js";
+import {CustodyConfig} from "../../util/dataColumns.js";
 import {SerializedCache} from "../../util/serializedCache.js";
 import {isDaOutOfRange} from "../blocks/blockInput/index.js";
-import {CreateFromBidProps, CreateFromBlockProps, PayloadEnvelopeInput} from "../blocks/payloadEnvelopeInput/index.js";
+import {
+  CreateFromBidProps,
+  CreateFromBlockProps,
+  PayloadEnvelopeInput,
+  PayloadEnvelopeInputPruneReason,
+  PayloadEnvelopeInputSource,
+} from "../blocks/payloadEnvelopeInput/index.js";
 import {ChainEvent, ChainEventEmitter} from "../emitter.js";
+import {SeenBlockInput} from "./seenGossipBlockInput.js";
 
 export type {PayloadEnvelopeInputState} from "../blocks/payloadEnvelopeInput/index.js";
 export {PayloadEnvelopeInput} from "../blocks/payloadEnvelopeInput/index.js";
 
+// Bound this cache by this max size, this is the same to SeenBlockInput
+const MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE = (MAX_LOOK_AHEAD_EPOCHS + 1) * SLOTS_PER_EPOCH;
+
 export type SeenPayloadEnvelopeInputModules = {
   config: ChainForkConfig;
   clock: IClock;
@@ -20,6 +34,9 @@ export type SeenPayloadEnvelopeInputModules = {
   chainEvents: ChainEventEmitter;
   signal: AbortSignal;
   serializedCache: SerializedCache;
+  db: IBeaconDb;
+  seenBlockInputCache: SeenBlockInput;
+  custodyConfig: CustodyConfig;
   metrics: Metrics | null;
   logger?: Logger;
 };
@@ -40,9 +57,14 @@ export class SeenPayloadEnvelopeInput {
   private readonly chainEvents: ChainEventEmitter;
   private readonly signal: AbortSignal;
   private readonly serializedCache: SerializedCache;
+  private readonly db: IBeaconDb;
+  private readonly seenBlockInputCache: SeenBlockInput;
+  private readonly custodyConfig: CustodyConfig;
   private readonly metrics: Metrics | null;
   private readonly logger?: Logger;
   private payloadInputs = new Map<RootHex, PayloadEnvelopeInput>();
+  // Dedup concurrent DB reloads of the same root so callers share one reconstructed object.
+  private readonly reloading = new Map<RootHex, Promise<PayloadEnvelopeInput | undefined>>();
 
   constructor({
     config,
@@ -51,6 +73,9 @@ export class SeenPayloadEnvelopeInput {
     chainEvents,
     signal,
     serializedCache,
+    db,
+    seenBlockInputCache,
+    custodyConfig,
     metrics,
     logger,
   }: SeenPayloadEnvelopeInputModules) {
@@ -60,6 +85,9 @@ export class SeenPayloadEnvelopeInput {
     this.chainEvents = chainEvents;
     this.signal = signal;
     this.serializedCache = serializedCache;
+    this.db = db;
+    this.seenBlockInputCache = seenBlockInputCache;
+    this.custodyConfig = custodyConfig;
     this.metrics = metrics;
     this.logger = logger;
 
@@ -86,7 +114,7 @@ export class SeenPayloadEnvelopeInput {
     let deletedCount = 0;
     for (const [, input] of this.payloadInputs) {
       if (input.slot < finalizedSlot) {
-        this.evictPayloadInput(input);
+        this.evictPayloadInput(input, "finalized");
         deletedCount++;
       }
     }
@@ -110,12 +138,13 @@ export class SeenPayloadEnvelopeInput {
     const daOutOfRange = isDaOutOfRange(this.config, props.forkName, props.block.message.slot, this.clock.currentEpoch);
     const input = PayloadEnvelopeInput.createFromBlock({...props, daOutOfRange});
     this.payloadInputs.set(props.blockRootHex, input);
-    this.metrics?.seenCache.payloadEnvelopeInput.created.inc();
+    this.metrics?.seenCache.payloadEnvelopeInput.created.inc({source: props.source});
     this.logger?.verbose("SeenPayloadEnvelopeInput.add created new entry", {
       slot: input.slot,
       root: props.blockRootHex,
       daOutOfRange,
     });
+    this.pruneToMaxSize();
     return input;
   }
 
@@ -131,12 +160,13 @@ export class SeenPayloadEnvelopeInput {
     const daOutOfRange = isDaOutOfRange(this.config, props.forkName, props.slot, this.clock.currentEpoch);
     const input = PayloadEnvelopeInput.createFromBid({...props, daOutOfRange});
     this.payloadInputs.set(props.blockRootHex, input);
-    this.metrics?.seenCache.payloadEnvelopeInput.created.inc();
+    this.metrics?.seenCache.payloadEnvelopeInput.created.inc({source: props.source});
     this.logger?.verbose("SeenPayloadEnvelopeInput.addFromBid created new entry", {
       slot: input.slot,
       root: props.blockRootHex,
       daOutOfRange,
     });
+    this.pruneToMaxSize();
     return input;
   }
 
@@ -144,6 +174,80 @@ export class SeenPayloadEnvelopeInput {
     return this.payloadInputs.get(blockRootHex);
   }
 
+  /**
+   * Like `get()`, but on a cache miss reconstruct the shell (bid + versionedHashes) from the block
+   * in `seenBlockInputCache` or the hot DB.
+   * This api is meant for BlockInputSync when a late/weird payloads for old blocks
+   *
+   * NOTE: the reconstructed entry is always EMPTY even when the block is actually FULL (its
+   * payload envelope + columns persisted in the DB). The consumer should consult fork choice if it needs to.
+   */
+  async getOrReload(blockRootHex: RootHex): Promise<PayloadEnvelopeInput | undefined> {
+    const existing = this.payloadInputs.get(blockRootHex);
+    if (existing !== undefined) {
+      return existing;
+    }
+
+    // Without this dedup, two concurrent misses each db.block.get + createFromBlock a DIFFERENT shell
+    // for the same root; processPayloadEnvelopeJob's WeakMap (keyed by object) can't dedup the import.
+    const inflight = this.reloading.get(blockRootHex);
+    if (inflight !== undefined) {
+      return inflight;
+    }
+    const promise = this.reloadFromDb(blockRootHex);
+    this.reloading.set(blockRootHex, promise);
+    try {
+      return await promise;
+    } finally {
+      this.reloading.delete(blockRootHex);
+    }
+  }
+
+  private async reloadFromDb(blockRootHex: RootHex): Promise<PayloadEnvelopeInput | undefined> {
+    // Only recover unfinalized, fork-choice-known blocks. Do not read the finalized archive. The gate
+    // also filters out un-imported blocks that may sit in seenBlockInputCache before validation.
+    if (!this.forkChoice.hasBlockHex(blockRootHex)) {
+      return undefined;
+    }
+
+    // In-memory first: persistBlockInput writes db.block THEN prunes seenBlockInputCache, so a
+    // just-imported block whose async write (unfinalizedBlockWrites) has not flushed yet is still here;
+    // older blocks (already pruned from the cache) fall through to the hot db.
+    const cachedBlockInput = this.seenBlockInputCache.get(blockRootHex);
+    const block = cachedBlockInput?.hasBlock()
+      ? cachedBlockInput.getBlock()
+      : await this.db.block.get(fromHex(blockRootHex));
+    if (block == null) {
+      return undefined;
+    }
+
+    const forkName = this.config.getForkName(block.message.slot);
+    if (!isForkPostGloas(forkName)) {
+      return undefined;
+    }
+
+    const daOutOfRange = isDaOutOfRange(this.config, forkName, block.message.slot, this.clock.currentEpoch);
+    const input = PayloadEnvelopeInput.createFromBlock({
+      blockRootHex,
+      block: block as SignedBeaconBlock<ForkPostGloas>,
+      forkName,
+      sampledColumns: this.custodyConfig.sampledColumns,
+      custodyColumns: this.custodyConfig.custodyColumns,
+      seenTimestampSec: Date.now() / 1000,
+      source: PayloadEnvelopeInputSource.reload,
+      daOutOfRange,
+    });
+    this.payloadInputs.set(blockRootHex, input);
+    this.metrics?.seenCache.payloadEnvelopeInput.created.inc({source: PayloadEnvelopeInputSource.reload});
+    this.logger?.verbose("SeenPayloadEnvelopeInput.getOrReload reconstructed entry from db", {
+      slot: input.slot,
+      root: blockRootHex,
+    });
+    // Set above (entry is at the back of the insertion order), so the cap never evicts what we just reloaded.
+    this.pruneToMaxSize();
+    return input;
+  }
+
   hasPayload(blockRootHex: RootHex): boolean {
     return this.payloadInputs.get(blockRootHex)?.hasPayloadEnvelope() ?? false;
   }
@@ -155,8 +259,7 @@ export class SeenPayloadEnvelopeInput {
   prune(blockRootHex: RootHex): void {
     const input = this.payloadInputs.get(blockRootHex);
     if (input) {
-      this.evictPayloadInput(input);
-      this.logger?.verbose("SeenPayloadEnvelopeInput.prune deleted", {slot: input.slot, root: blockRootHex});
+      this.evictPayloadInput(input, "prune");
     }
   }
 
@@ -170,17 +273,43 @@ export class SeenPayloadEnvelopeInput {
         // ...and don't evict while columns are still being gathered: writeDataColumnsToDb awaits the
         // same hasComputedAllData() before persisting. Such entries are pruned by a later call.
         if (input?.hasComputedAllData()) {
-          this.evictPayloadInput(input);
-          this.logger?.verbose("SeenPayloadEnvelopeInput.pruneBelowParent deleted", {
-            slot: block.slot,
-            root: block.blockRoot,
-          });
+          this.evictPayloadInput(input, "belowParent");
         }
       }
     }
   }
 
-  private evictPayloadInput(payloadInput: PayloadEnvelopeInput): void {
+  /**
+   * Bound this cache to have at most MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE items.
+   * Evicts by insertion order so a just-reloaded old-slot entry is not evicted right after.
+   * On unstable network, we may need to query an evicted item, UnknownBlockInput will recover
+   * it from db via getOrReload() api. Runs after every single insert, so it evicts ~1 per call.
+   */
+  private pruneToMaxSize(): void {
+    let evicted = 0;
+    for (const input of this.payloadInputs.values()) {
+      if (this.payloadInputs.size <= MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE) {
+        break;
+      }
+      this.evictPayloadInput(input, "cap");
+      evicted++;
+    }
+    if (evicted > 0) {
+      this.logger?.debug("SeenPayloadEnvelopeInput.pruneToMaxSize evicted", {
+        evicted,
+        size: this.payloadInputs.size,
+        max: MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE,
+      });
+    }
+  }
+
+  private evictPayloadInput(payloadInput: PayloadEnvelopeInput, reason: PayloadEnvelopeInputPruneReason): void {
+    this.metrics?.seenCache.payloadEnvelopeInput.pruned.inc({reason});
+    this.logger?.debug("SeenPayloadEnvelopeInput evicted", {
+      slot: payloadInput.slot,
+      root: payloadInput.blockRootHex,
+      reason,
+    });
     this.serializedCache.delete(payloadInput.getSerializedCacheKeys());
     this.payloadInputs.delete(payloadInput.blockRootHex);
   }

package/src/chain/validation/attestation.ts

@@ -142,13 +142,7 @@ export async function validateGossipAttestationsSameAttData(
   const batchableBls = signatureSets.length >= chain.opts.minSameMessageSignatureSetsToBatch;
   if (batchableBls) {
     // all signature sets should have same signing root since we filtered in network processor
-    signatureValids = await chain.bls.verifySignatureSetsSameMessage(
-      signatureSets.map((set) => {
-        const publicKey = chain.pubkeyCache.getOrThrow(set.index);
-        return {publicKey, signature: set.signature};
-      }),
-      signatureSets[0].signingRoot
-    );
+    signatureValids = await chain.bls.verifySignatureSetsSameMessage(signatureSets, signatureSets[0].signingRoot);
   } else {
     // don't want to block the main thread if there are too few signatures
     signatureValids = await Promise.all(

package/src/chain/validation/attesterSlashing.ts

@@ -44,14 +44,7 @@ export async function validateAttesterSlashing(
   // [REJECT] All of the conditions within process_attester_slashing pass validation.
   try {
     // verifySignature = false, verified in batch below
-    assertValidAttesterSlashing(
-      chain.config,
-      chain.pubkeyCache,
-      state.slot,
-      state.validatorCount,
-      attesterSlashing,
-      false
-    );
+    assertValidAttesterSlashing(chain.config, state.slot, state.validatorCount, attesterSlashing, false);
   } catch (e) {
     throw new AttesterSlashingError(GossipAction.REJECT, {
       code: AttesterSlashingErrorCode.INVALID,

package/src/chain/validation/executionPayloadBid.ts

@@ -1,4 +1,3 @@
-import {PublicKey} from "@chainsafe/blst";
 import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {PAYLOAD_BUILDER_VERSION} from "@lodestar/params";
 import {
@@ -345,7 +344,7 @@ async function validateExecutionPayloadBid(
 
   // [REJECT] `signed_execution_payload_bid.signature` is valid with respect to the `bid.builder_index`.
   const signatureSet = createSingleSignatureSetFromComponents(
-    PublicKey.fromBytes(builder.pubkey),
+    builder.pubkey,
     getExecutionPayloadBidSigningRoot(chain.config, bid),
     signedExecutionPayloadBid.signature
   );

package/src/chain/validation/executionPayloadEnvelope.ts

@@ -170,7 +170,6 @@ async function validateExecutionPayloadEnvelope(
   // by `verify_execution_payload_envelope_signature`.
   const signatureSet = getExecutionPayloadEnvelopeSignatureSet(
     chain.config,
-    chain.pubkeyCache,
     blockState,
     executionPayloadEnvelope,
     payloadInput.proposerIndex

package/src/chain/validation/payloadAttestationMessage.ts

@@ -1,6 +1,5 @@
 import {
-  computeEpochAtSlot,
-  createSingleSignatureSetFromComponents,
+  createIndexedSignatureSetFromComponents,
   getPayloadAttestationDataSigningRoot,
   isStatePostGloas,
 } from "@lodestar/state-transition";
@@ -36,7 +35,6 @@ async function validatePayloadAttestationMessage(
   prioritizeBls = false
 ): Promise<PayloadAttestationValidationResult> {
   const {data, validatorIndex} = payloadAttestationMessage;
-  const epoch = computeEpochAtSlot(data.slot);
 
   // [IGNORE] The message's slot is for the current slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance), i.e. `data.slot == current_slot`.
   if (!chain.clock.isCurrentSlotGivenGossipDisparity(data.slot)) {
@@ -48,9 +46,8 @@ async function validatePayloadAttestationMessage(
   }
 
   // [IGNORE] The `payload_attestation_message` is the first valid message received
-  // from the validator with index `payload_attestation_message.validator_index`.
-  // A single validator can participate PTC at most once per epoch
-  if (chain.seenPayloadAttesters.isKnown(epoch, validatorIndex)) {
+  // for the slot from the validator with index `payload_attestation_message.validator_index`.
+  if (chain.seenPayloadAttesters.isKnown(data.slot, validatorIndex)) {
     throw new PayloadAttestationError(GossipAction.IGNORE, {
       code: PayloadAttestationErrorCode.PAYLOAD_ATTESTATION_ALREADY_KNOWN,
       validatorIndex,
@@ -114,16 +111,8 @@ async function validatePayloadAttestationMessage(
   }
 
   // [REJECT] `payload_attestation_message.signature` is valid with respect to the validator's public key.
-  const validatorPubkey = chain.pubkeyCache.get(validatorIndex);
-  if (!validatorPubkey) {
-    throw new PayloadAttestationError(GossipAction.REJECT, {
-      code: PayloadAttestationErrorCode.INVALID_ATTESTER,
-      attesterIndex: validatorIndex,
-    });
-  }
-
-  const signatureSet = createSingleSignatureSetFromComponents(
-    validatorPubkey,
+  const signatureSet = createIndexedSignatureSetFromComponents(
+    validatorIndex,
     getPayloadAttestationDataSigningRoot(chain.config, data),
     payloadAttestationMessage.signature
   );
@@ -135,7 +124,7 @@ async function validatePayloadAttestationMessage(
   }
 
   // Valid
-  chain.seenPayloadAttesters.add(epoch, validatorIndex);
+  chain.seenPayloadAttesters.add(data.slot, validatorIndex);
 
   return {
     attDataRootHex: toRootHex(ssz.gloas.PayloadAttestationData.hashTreeRoot(data)),

package/src/chain/validation/proposerPreferences.ts

@@ -3,7 +3,7 @@ import {GENESIS_EPOCH, GENESIS_SLOT, MIN_SEED_LOOKAHEAD, SLOTS_PER_EPOCH, isFork
 import {
   computeEpochAtSlot,
   computeStartSlotAtEpoch,
-  createSingleSignatureSetFromComponents,
+  createIndexedSignatureSetFromComponents,
   getProposerPreferencesSigningRoot,
 } from "@lodestar/state-transition";
 import {Epoch, Slot, ValidatorIndex, gloas} from "@lodestar/types";
@@ -139,8 +139,8 @@ export async function validateGossipProposerPreferences(
   }
 
   // [REJECT] `signed_proposer_preferences.signature` is valid with respect to the validator's public key.
-  const signatureSet = createSingleSignatureSetFromComponents(
-    chain.pubkeyCache.getOrThrow(validatorIndex),
+  const signatureSet = createIndexedSignatureSetFromComponents(
+    validatorIndex,
     getProposerPreferencesSigningRoot(chain.config, preferences),
     signedProposerPreferences.signature
   );

package/src/chain/validation/proposerSlashing.ts

@@ -37,7 +37,7 @@ async function validateProposerSlashing(
   try {
     const proposer = state.getValidator(proposerSlashing.signedHeader1.message.proposerIndex);
     // verifySignature = false, verified in batch below
-    assertValidProposerSlashing(chain.config, chain.pubkeyCache, state.slot, proposerSlashing, proposer, false);
+    assertValidProposerSlashing(chain.config, state.slot, proposerSlashing, proposer, false);
   } catch (e) {
     throw new ProposerSlashingError(GossipAction.REJECT, {
       code: ProposerSlashingErrorCode.INVALID,

package/src/metrics/metrics/lodestar.ts

@@ -4,7 +4,10 @@ import {ArchiveStoreTask} from "../../chain/archiveStore/archiveStore.js";
 import {FrequencyStateArchiveStep} from "../../chain/archiveStore/strategies/frequencyStateArchiveStrategy.js";
 import {BlockInputSource} from "../../chain/blocks/blockInput/index.js";
 import {PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
-import {PayloadEnvelopeInputSource} from "../../chain/blocks/payloadEnvelopeInput/index.js";
+import {
+  PayloadEnvelopeInputPruneReason,
+  PayloadEnvelopeInputSource,
+} from "../../chain/blocks/payloadEnvelopeInput/index.js";
 import {JobQueueItemType} from "../../chain/bls/index.js";
 import {AttestationErrorCode, BlockErrorCode} from "../../chain/errors/index.js";
 import {
@@ -31,6 +34,10 @@ import {RegistryMetricCreator} from "../utils/registryMetricCreator.js";
 
 export type LodestarMetrics = ReturnType<typeof createLodestarMetrics>;
 
+type BlsJobOutcome = "valid" | "invalid" | "prepError" | "verifyError" | "workerError";
+type BlsBufferFlushReason = "size" | "timeout";
+type BlsVerificationCallOperation = "batch" | "single" | "fallback" | "same_message";
+
 /**
  * Extra Lodestar custom metrics
  */
@@ -396,86 +403,108 @@ export function createLodestarMetrics(
     // BLS verifier thread pool and queue
 
     bls: {
-      aggregatedPubkeys: register.gauge({
+      aggregatedPubkeys: register.counter({
         name: "lodestar_bls_aggregated_pubkeys_total",
-        help: "Total aggregated pubkeys for BLS validation",
+        help: "Total validator public keys used as inputs to aggregate signature sets",
+      }),
+      pubkeyCacheSize: register.gauge({
+        name: "lodestar_bls_pubkey_cache_size",
+        help: "Current number of validator public keys in the native cache",
+      }),
+      pubkeyCacheCapacity: register.gauge({
+        name: "lodestar_bls_pubkey_cache_capacity",
+        help: "Number of validator public keys the native cache can hold without growing",
       }),
     },
 
     blsThreadPool: {
-      jobsWorkerTime: register.gauge<{workerId: number}>({
+      jobsWorkerTime: register.counter<{workerId: number}>({
         name: "lodestar_bls_thread_pool_time_seconds_sum",
-        help: "Total time spent verifying signature sets measured on the worker",
+        help: "Cumulative wall time spent processing BLS dispatch groups by worker",
         labelNames: ["workerId"],
       }),
-      successJobsSignatureSetsCount: register.gauge({
+      successJobsSignatureSetsCount: register.counter({
         name: "lodestar_bls_thread_pool_success_jobs_signature_sets_count",
-        help: "Count of total verified signature sets",
-      }),
-      errorAggregateSignatureSetsCount: register.gauge<{type: JobQueueItemType}>({
-        name: "lodestar_bls_thread_pool_error_aggregate_signature_sets_count",
-        help: "Count of error when aggregating pubkeys or signatures",
-        labelNames: ["type"],
+        help: "Count of signature sets completed without an operational error",
       }),
-      errorJobsSignatureSetsCount: register.gauge({
+      errorJobsSignatureSetsCount: register.counter({
         name: "lodestar_bls_thread_pool_error_jobs_signature_sets_count",
-        help: "Count of total error-ed signature sets",
+        help: "Count of signature sets that ended in an input, binding, or worker error",
       }),
-      jobWaitTime: register.histogram({
+      jobWaitTime: register.histogram<{type: JobQueueItemType}>({
         name: "lodestar_bls_thread_pool_queue_job_wait_time_seconds",
-        help: "Time from job added to the queue to starting the job in seconds",
-        buckets: [0.01, 0.02, 0.1, 0.3, 0.5, 1],
+        help: "Time from adding a BLS job until it is selected for a worker dispatch group",
+        labelNames: ["type"],
+        buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2],
+      }),
+      jobDuration: register.histogram<{type: JobQueueItemType}>({
+        name: "lodestar_bls_thread_pool_job_duration_seconds",
+        help: "End-to-end duration of a BLS job from submission through completion",
+        labelNames: ["type"],
+        buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2],
+      }),
+      jobResults: register.counter<{type: JobQueueItemType; outcome: BlsJobOutcome}>({
+        name: "lodestar_bls_thread_pool_job_results_total",
+        help: "Count of completed BLS jobs by cryptographic or operational outcome",
+        labelNames: ["type", "outcome"],
       }),
       queueLength: register.gauge({
         name: "lodestar_bls_thread_pool_queue_length",
-        help: "Count of total block processor queue length",
+        help: "Count of ready BLS jobs waiting for a worker",
+      }),
+      bufferedQueueLength: register.gauge({
+        name: "lodestar_bls_thread_pool_buffered_queue_length",
+        help: "Count of BLS jobs waiting in the intentional batching buffer",
       }),
       workersBusy: register.gauge({
         name: "lodestar_bls_thread_pool_workers_busy",
         help: "Count of current busy workers",
       }),
-      totalJobsGroupsStarted: register.gauge({
+      totalJobsGroupsStarted: register.counter({
         name: "lodestar_bls_thread_pool_job_groups_started_total",
-        help: "Count of total jobs groups started in bls thread pool, job groups include +1 jobs",
+        help: "Count of worker dispatch groups started by the BLS thread pool",
       }),
-      totalJobsStarted: register.gauge<{type: JobQueueItemType}>({
+      totalJobsStarted: register.counter<{type: JobQueueItemType}>({
         name: "lodestar_bls_thread_pool_jobs_started_total",
-        help: "Count of total jobs started in bls thread pool, jobs include +1 signature sets",
+        help: "Count of jobs started by the BLS thread pool",
         labelNames: ["type"],
       }),
-      totalSigSetsStarted: register.gauge<{type: JobQueueItemType}>({
+      totalSigSetsStarted: register.counter<{type: JobQueueItemType}>({
         name: "lodestar_bls_thread_pool_sig_sets_started_total",
-        help: "Count of total signature sets started in bls thread pool, sig sets include 1 pk, msg, sig",
+        help: "Count of signature sets started by the BLS thread pool",
         labelNames: ["type"],
       }),
       // Re-verifying a batch means doing double work. This number must be very low or it can be a waste of CPU resources
-      batchRetries: register.gauge({
+      batchRetries: register.counter({
         name: "lodestar_bls_thread_pool_batch_retries_total",
-        help: "Count of total batches that failed and had to be verified again.",
+        help: "Count of combined batches that failed and required per-job fallback",
       }),
       // To count how many sigs are being validated with the optimization of batching them
-      batchSigsSuccess: register.gauge({
+      batchSigsSuccess: register.counter({
         name: "lodestar_bls_thread_pool_batch_sigs_success_total",
-        help: "Count of total batches that failed and had to be verified again.",
+        help: "Count of signature sets accepted by combined batch verification without fallback",
       }),
-      sameMessageRetryJobs: register.gauge({
-        name: "lodestar_bls_thread_pool_same_message_jobs_retries_total",
-        help: "Count of total same message jobs that failed and had to be verified again.",
+      bufferFlushes: register.counter<{reason: BlsBufferFlushReason}>({
+        name: "lodestar_bls_thread_pool_buffer_flushes_total",
+        help: "Count of batching-buffer flushes by trigger",
+        labelNames: ["reason"],
       }),
-      sameMessageRetrySets: register.gauge({
-        name: "lodestar_bls_thread_pool_same_message_sets_retries_total",
-        help: "Count of total same message sets that failed and had to be verified again.",
+      bufferedSignatureSets: register.histogram({
+        name: "lodestar_bls_thread_pool_buffered_signature_sets",
+        help: "Count of signature sets included in each batching-buffer flush",
+        buckets: [1, 8, 16, 24, 28, 32, 36, 40, 44, 48, 64, 128],
       }),
       // To measure the time cost of main thread <-> worker message passing
       latencyToWorker: register.histogram({
         name: "lodestar_bls_thread_pool_latency_to_worker",
         help: "Time from sending the job to the worker and the worker receiving it",
-        buckets: [0.001, 0.003, 0.01, 0.03, 0.1],
+        // The dashboard treats 20ms as slow; values above 100ms only need severe-stall classification.
+        buckets: [0.00025, 0.001, 0.005, 0.02, 0.1],
       }),
       latencyFromWorker: register.histogram({
         name: "lodestar_bls_thread_pool_latency_from_worker",
         help: "Time from the worker sending the result and the main thread receiving it",
-        buckets: [0.001, 0.003, 0.01, 0.03, 0.1],
+        buckets: [0.00025, 0.001, 0.005, 0.02, 0.1],
       }),
       mainThreadDurationInThreadPool: register.histogram({
         name: "lodestar_bls_thread_pool_main_thread_time_seconds",
@@ -485,32 +514,40 @@ export function createLodestarMetrics(
       }),
       timePerSigSet: register.histogram({
         name: "lodestar_bls_worker_thread_time_per_sigset_seconds",
-        help: "Time to verify each sigset with worker thread mode",
+        help: "Average BLS worker dispatch wall time per original signature set",
         // Time per sig ~0.9ms on good machines
-        buckets: [0.5e-3, 0.75e-3, 1e-3, 1.5e-3, 2e-3, 5e-3],
-      }),
-      totalSigSets: register.gauge({
+        buckets: [0.5e-3, 0.75e-3, 1e-3, 1.5e-3, 2e-3, 5e-3, 10e-3, 20e-3, 50e-3, 100e-3],
+      }),
+      workRequestPreparationDuration: register.histogram({
+        name: "lodestar_bls_thread_pool_work_request_preparation_duration_seconds",
+        help: "Main-thread time spent converting one worker dispatch group to binding inputs",
+        // Normal requests take microseconds; the dashboard treats 1ms as a main-thread stall.
+        buckets: [0.000005, 0.000025, 0.0001, 0.0005, 0.001, 0.005, 0.025],
+      }),
+      verificationCallDuration: register.histogram<{operation: BlsVerificationCallOperation}>({
+        name: "lodestar_bls_thread_pool_verification_call_duration_seconds",
+        help: "Duration of one worker-side lodestar-z verification binding call, including input conversion",
+        labelNames: ["operation"],
+        // Verification ranges from sub-millisecond calls to large batches; above 500ms is uniformly severe.
+        buckets: [0.00025, 0.001, 0.005, 0.02, 0.1, 0.5],
+      }),
+      verificationCallSignatureSets: register.counter<{operation: BlsVerificationCallOperation}>({
+        name: "lodestar_bls_thread_pool_verification_call_signature_sets_total",
+        help: "Count of signature sets passed to worker-side lodestar-z verification binding calls",
+        labelNames: ["operation"],
+      }),
+      totalSigSets: register.counter({
         name: "lodestar_bls_thread_pool_sig_sets_total",
         help: "Count of total signature sets",
       }),
-      prioritizedSigSets: register.gauge({
+      prioritizedSigSets: register.counter({
         name: "lodestar_bls_thread_pool_prioritized_sig_sets_total",
         help: "Count of total prioritized signature sets",
       }),
-      batchableSigSets: register.gauge({
+      batchableSigSets: register.counter({
         name: "lodestar_bls_thread_pool_batchable_sig_sets_total",
         help: "Count of total batchable signature sets",
       }),
-      aggregateWithRandomnessAsyncDuration: register.histogram({
-        name: "lodestar_bls_thread_pool_aggregate_with_randomness_async_time_seconds",
-        help: "Total time performing aggregateWithRandomness async",
-        buckets: [0.001, 0.005, 0.01, 0.1, 0.3],
-      }),
-      pubkeysAggregationMainThreadDuration: register.histogram({
-        name: "lodestar_bls_thread_pool_pubkeys_aggregation_main_thread_time_seconds",
-        help: "Total time spent aggregating pubkeys on main thread",
-        buckets: [0.001, 0.005, 0.01, 0.1],
-      }),
     },
 
     // BLS time on single thread mode
@@ -1609,9 +1646,15 @@ export function createLodestarMetrics(
           name: "lodestar_seen_payload_envelope_input_cache_serialized_object_refs",
           help: "Number of serialized-cache object refs retained by cached PayloadEnvelopeInputs",
         }),
-        created: register.counter({
+        created: register.counter<{source: PayloadEnvelopeInputSource}>({
           name: "lodestar_seen_payload_envelope_input_cache_items_created_total",
-          help: "Number of PayloadEnvelopeInputs created",
+          help: "Number of PayloadEnvelopeInputs created by source",
+          labelNames: ["source"],
+        }),
+        pruned: register.counter<{reason: PayloadEnvelopeInputPruneReason}>({
+          name: "lodestar_seen_payload_envelope_input_cache_items_pruned_total",
+          help: "Number of PayloadEnvelopeInputs evicted by reason",
+          labelNames: ["reason"],
         }),
       },
     },