@lodestar/beacon-node 1.47.0-dev.92bd4efe9b → 1.47.0-dev.931ecbec99

package/src/chain/validation/proposerPreferences.ts

@@ -1,9 +1,9 @@
 import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
-import {GENESIS_EPOCH, GENESIS_SLOT, MIN_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params";
+import {GENESIS_EPOCH, GENESIS_SLOT, MIN_SEED_LOOKAHEAD, SLOTS_PER_EPOCH, isForkPostGloas} from "@lodestar/params";
 import {
   computeEpochAtSlot,
   computeStartSlotAtEpoch,
-  createSingleSignatureSetFromComponents,
+  createIndexedSignatureSetFromComponents,
   getProposerPreferencesSigningRoot,
 } from "@lodestar/state-transition";
 import {Epoch, Slot, ValidatorIndex, gloas} from "@lodestar/types";
@@ -13,7 +13,7 @@ import {IBeaconChain} from "../index.js";
 
 /**
  * Validates a gossiped `SignedProposerPreferences` per
- * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.13/specs/gloas/p2p-interface.md#new-proposer_preferences
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.14/specs/gloas/p2p-interface.md#new-proposer_preferences
  */
 export async function validateGossipProposerPreferences(
   chain: IBeaconChain,
@@ -24,6 +24,16 @@ export async function validateGossipProposerPreferences(
   const dependentRootHex = toRootHex(dependentRoot);
   const proposalEpoch = computeEpochAtSlot(proposalSlot);
 
+  // [IGNORE] The proposal epoch is after the Gloas upgrade
+  const proposalFork = chain.config.getForkName(proposalSlot);
+  if (!isForkPostGloas(proposalFork)) {
+    throw new ProposerPreferencesError(GossipAction.IGNORE, {
+      code: ProposerPreferencesErrorCode.PRE_GLOAS_PROPOSAL_SLOT,
+      proposalSlot,
+      proposalFork,
+    });
+  }
+
   // [IGNORE] `preferences.proposal_slot` is within the proposer lookahead,
   // allowing for `MAXIMUM_GOSSIP_CLOCK_DISPARITY`.
   const currentEpoch = computeEpochAtSlot(chain.clock.currentSlotWithGossipDisparity);
@@ -129,8 +139,8 @@ export async function validateGossipProposerPreferences(
   }
 
   // [REJECT] `signed_proposer_preferences.signature` is valid with respect to the validator's public key.
-  const signatureSet = createSingleSignatureSetFromComponents(
-    chain.pubkeyCache.getOrThrow(validatorIndex),
+  const signatureSet = createIndexedSignatureSetFromComponents(
+    validatorIndex,
     getProposerPreferencesSigningRoot(chain.config, preferences),
     signedProposerPreferences.signature
   );

package/src/chain/validation/proposerSlashing.ts

@@ -37,7 +37,7 @@ async function validateProposerSlashing(
   try {
     const proposer = state.getValidator(proposerSlashing.signedHeader1.message.proposerIndex);
     // verifySignature = false, verified in batch below
-    assertValidProposerSlashing(chain.config, chain.pubkeyCache, state.slot, proposerSlashing, proposer, false);
+    assertValidProposerSlashing(chain.config, state.slot, proposerSlashing, proposer, false);
   } catch (e) {
     throw new ProposerSlashingError(GossipAction.REJECT, {
       code: ProposerSlashingErrorCode.INVALID,

package/src/metrics/metrics/lodestar.ts

@@ -31,6 +31,10 @@ import {RegistryMetricCreator} from "../utils/registryMetricCreator.js";
 
 export type LodestarMetrics = ReturnType<typeof createLodestarMetrics>;
 
+type BlsJobOutcome = "valid" | "invalid" | "prepError" | "verifyError" | "workerError";
+type BlsBufferFlushReason = "size" | "timeout";
+type BlsVerificationCallOperation = "batch" | "single" | "fallback" | "same_message";
+
 /**
  * Extra Lodestar custom metrics
  */
@@ -396,86 +400,108 @@ export function createLodestarMetrics(
     // BLS verifier thread pool and queue
 
     bls: {
-      aggregatedPubkeys: register.gauge({
+      aggregatedPubkeys: register.counter({
         name: "lodestar_bls_aggregated_pubkeys_total",
-        help: "Total aggregated pubkeys for BLS validation",
+        help: "Total validator public keys used as inputs to aggregate signature sets",
+      }),
+      pubkeyCacheSize: register.gauge({
+        name: "lodestar_bls_pubkey_cache_size",
+        help: "Current number of validator public keys in the native cache",
+      }),
+      pubkeyCacheCapacity: register.gauge({
+        name: "lodestar_bls_pubkey_cache_capacity",
+        help: "Number of validator public keys the native cache can hold without growing",
       }),
     },
 
     blsThreadPool: {
-      jobsWorkerTime: register.gauge<{workerId: number}>({
+      jobsWorkerTime: register.counter<{workerId: number}>({
         name: "lodestar_bls_thread_pool_time_seconds_sum",
-        help: "Total time spent verifying signature sets measured on the worker",
+        help: "Cumulative wall time spent processing BLS dispatch groups by worker",
         labelNames: ["workerId"],
       }),
-      successJobsSignatureSetsCount: register.gauge({
+      successJobsSignatureSetsCount: register.counter({
         name: "lodestar_bls_thread_pool_success_jobs_signature_sets_count",
-        help: "Count of total verified signature sets",
+        help: "Count of signature sets completed without an operational error",
       }),
-      errorAggregateSignatureSetsCount: register.gauge<{type: JobQueueItemType}>({
-        name: "lodestar_bls_thread_pool_error_aggregate_signature_sets_count",
-        help: "Count of error when aggregating pubkeys or signatures",
-        labelNames: ["type"],
-      }),
-      errorJobsSignatureSetsCount: register.gauge({
+      errorJobsSignatureSetsCount: register.counter({
         name: "lodestar_bls_thread_pool_error_jobs_signature_sets_count",
-        help: "Count of total error-ed signature sets",
+        help: "Count of signature sets that ended in an input, binding, or worker error",
       }),
-      jobWaitTime: register.histogram({
+      jobWaitTime: register.histogram<{type: JobQueueItemType}>({
         name: "lodestar_bls_thread_pool_queue_job_wait_time_seconds",
-        help: "Time from job added to the queue to starting the job in seconds",
-        buckets: [0.01, 0.02, 0.1, 0.3, 0.5, 1],
+        help: "Time from adding a BLS job until it is selected for a worker dispatch group",
+        labelNames: ["type"],
+        buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2],
+      }),
+      jobDuration: register.histogram<{type: JobQueueItemType}>({
+        name: "lodestar_bls_thread_pool_job_duration_seconds",
+        help: "End-to-end duration of a BLS job from submission through completion",
+        labelNames: ["type"],
+        buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2],
+      }),
+      jobResults: register.counter<{type: JobQueueItemType; outcome: BlsJobOutcome}>({
+        name: "lodestar_bls_thread_pool_job_results_total",
+        help: "Count of completed BLS jobs by cryptographic or operational outcome",
+        labelNames: ["type", "outcome"],
       }),
       queueLength: register.gauge({
         name: "lodestar_bls_thread_pool_queue_length",
-        help: "Count of total block processor queue length",
+        help: "Count of ready BLS jobs waiting for a worker",
+      }),
+      bufferedQueueLength: register.gauge({
+        name: "lodestar_bls_thread_pool_buffered_queue_length",
+        help: "Count of BLS jobs waiting in the intentional batching buffer",
       }),
       workersBusy: register.gauge({
         name: "lodestar_bls_thread_pool_workers_busy",
         help: "Count of current busy workers",
       }),
-      totalJobsGroupsStarted: register.gauge({
+      totalJobsGroupsStarted: register.counter({
         name: "lodestar_bls_thread_pool_job_groups_started_total",
-        help: "Count of total jobs groups started in bls thread pool, job groups include +1 jobs",
+        help: "Count of worker dispatch groups started by the BLS thread pool",
       }),
-      totalJobsStarted: register.gauge<{type: JobQueueItemType}>({
+      totalJobsStarted: register.counter<{type: JobQueueItemType}>({
         name: "lodestar_bls_thread_pool_jobs_started_total",
-        help: "Count of total jobs started in bls thread pool, jobs include +1 signature sets",
+        help: "Count of jobs started by the BLS thread pool",
         labelNames: ["type"],
       }),
-      totalSigSetsStarted: register.gauge<{type: JobQueueItemType}>({
+      totalSigSetsStarted: register.counter<{type: JobQueueItemType}>({
         name: "lodestar_bls_thread_pool_sig_sets_started_total",
-        help: "Count of total signature sets started in bls thread pool, sig sets include 1 pk, msg, sig",
+        help: "Count of signature sets started by the BLS thread pool",
         labelNames: ["type"],
       }),
       // Re-verifying a batch means doing double work. This number must be very low or it can be a waste of CPU resources
-      batchRetries: register.gauge({
+      batchRetries: register.counter({
         name: "lodestar_bls_thread_pool_batch_retries_total",
-        help: "Count of total batches that failed and had to be verified again.",
+        help: "Count of combined batches that failed and required per-job fallback",
       }),
       // To count how many sigs are being validated with the optimization of batching them
-      batchSigsSuccess: register.gauge({
+      batchSigsSuccess: register.counter({
         name: "lodestar_bls_thread_pool_batch_sigs_success_total",
-        help: "Count of total batches that failed and had to be verified again.",
+        help: "Count of signature sets accepted by combined batch verification without fallback",
       }),
-      sameMessageRetryJobs: register.gauge({
-        name: "lodestar_bls_thread_pool_same_message_jobs_retries_total",
-        help: "Count of total same message jobs that failed and had to be verified again.",
+      bufferFlushes: register.counter<{reason: BlsBufferFlushReason}>({
+        name: "lodestar_bls_thread_pool_buffer_flushes_total",
+        help: "Count of batching-buffer flushes by trigger",
+        labelNames: ["reason"],
       }),
-      sameMessageRetrySets: register.gauge({
-        name: "lodestar_bls_thread_pool_same_message_sets_retries_total",
-        help: "Count of total same message sets that failed and had to be verified again.",
+      bufferedSignatureSets: register.histogram({
+        name: "lodestar_bls_thread_pool_buffered_signature_sets",
+        help: "Count of signature sets included in each batching-buffer flush",
+        buckets: [1, 8, 16, 24, 28, 32, 36, 40, 44, 48, 64, 128],
       }),
       // To measure the time cost of main thread <-> worker message passing
       latencyToWorker: register.histogram({
         name: "lodestar_bls_thread_pool_latency_to_worker",
         help: "Time from sending the job to the worker and the worker receiving it",
-        buckets: [0.001, 0.003, 0.01, 0.03, 0.1],
+        // The dashboard treats 20ms as slow; values above 100ms only need severe-stall classification.
+        buckets: [0.00025, 0.001, 0.005, 0.02, 0.1],
       }),
       latencyFromWorker: register.histogram({
         name: "lodestar_bls_thread_pool_latency_from_worker",
         help: "Time from the worker sending the result and the main thread receiving it",
-        buckets: [0.001, 0.003, 0.01, 0.03, 0.1],
+        buckets: [0.00025, 0.001, 0.005, 0.02, 0.1],
       }),
       mainThreadDurationInThreadPool: register.histogram({
         name: "lodestar_bls_thread_pool_main_thread_time_seconds",
@@ -485,32 +511,40 @@ export function createLodestarMetrics(
       }),
       timePerSigSet: register.histogram({
         name: "lodestar_bls_worker_thread_time_per_sigset_seconds",
-        help: "Time to verify each sigset with worker thread mode",
+        help: "Average BLS worker dispatch wall time per original signature set",
         // Time per sig ~0.9ms on good machines
-        buckets: [0.5e-3, 0.75e-3, 1e-3, 1.5e-3, 2e-3, 5e-3],
-      }),
-      totalSigSets: register.gauge({
+        buckets: [0.5e-3, 0.75e-3, 1e-3, 1.5e-3, 2e-3, 5e-3, 10e-3, 20e-3, 50e-3, 100e-3],
+      }),
+      workRequestPreparationDuration: register.histogram({
+        name: "lodestar_bls_thread_pool_work_request_preparation_duration_seconds",
+        help: "Main-thread time spent converting one worker dispatch group to binding inputs",
+        // Normal requests take microseconds; the dashboard treats 1ms as a main-thread stall.
+        buckets: [0.000005, 0.000025, 0.0001, 0.0005, 0.001, 0.005, 0.025],
+      }),
+      verificationCallDuration: register.histogram<{operation: BlsVerificationCallOperation}>({
+        name: "lodestar_bls_thread_pool_verification_call_duration_seconds",
+        help: "Duration of one worker-side lodestar-z verification binding call, including input conversion",
+        labelNames: ["operation"],
+        // Verification ranges from sub-millisecond calls to large batches; above 500ms is uniformly severe.
+        buckets: [0.00025, 0.001, 0.005, 0.02, 0.1, 0.5],
+      }),
+      verificationCallSignatureSets: register.counter<{operation: BlsVerificationCallOperation}>({
+        name: "lodestar_bls_thread_pool_verification_call_signature_sets_total",
+        help: "Count of signature sets passed to worker-side lodestar-z verification binding calls",
+        labelNames: ["operation"],
+      }),
+      totalSigSets: register.counter({
         name: "lodestar_bls_thread_pool_sig_sets_total",
         help: "Count of total signature sets",
       }),
-      prioritizedSigSets: register.gauge({
+      prioritizedSigSets: register.counter({
         name: "lodestar_bls_thread_pool_prioritized_sig_sets_total",
         help: "Count of total prioritized signature sets",
       }),
-      batchableSigSets: register.gauge({
+      batchableSigSets: register.counter({
         name: "lodestar_bls_thread_pool_batchable_sig_sets_total",
         help: "Count of total batchable signature sets",
       }),
-      aggregateWithRandomnessAsyncDuration: register.histogram({
-        name: "lodestar_bls_thread_pool_aggregate_with_randomness_async_time_seconds",
-        help: "Total time performing aggregateWithRandomness async",
-        buckets: [0.001, 0.005, 0.01, 0.1, 0.3],
-      }),
-      pubkeysAggregationMainThreadDuration: register.histogram({
-        name: "lodestar_bls_thread_pool_pubkeys_aggregation_main_thread_time_seconds",
-        help: "Total time spent aggregating pubkeys on main thread",
-        buckets: [0.001, 0.005, 0.01, 0.1],
-      }),
     },
 
     // BLS time on single thread mode
@@ -908,6 +942,22 @@ export function createLodestarMetrics(
         labelNames: ["error"],
       }),
     },
+    gossipExecutionPayloadBid: {
+      elapsedTimeTillReceived: register.histogram<{source: OpSource}>({
+        name: "lodestar_gossip_execution_payload_bid_elapsed_time_till_received_seconds",
+        help: "Time elapsed between the bid's slot start and the time the execution payload bid was received (negative = received before its slot, broadcast in the previous slot; positive = received during its own slot, late)",
+        labelNames: ["source"],
+        buckets: [-6, -3, -1, 0, 3],
+      }),
+    },
+    gossipPayloadAttestationMessage: {
+      elapsedTimeTillReceived: register.histogram<{source: OpSource}>({
+        name: "lodestar_gossip_payload_attestation_message_elapsed_time_till_received_seconds",
+        help: "Time elapsed between slot time and the time the payload attestation message was received",
+        labelNames: ["source"],
+        buckets: [3, 6, 9, 12],
+      }),
+    },
     // recovery in the case of specific blob rows required
     recoverBlobSidecars: {
       blobsReconstructed: register.counter({

package/src/network/libp2p/index.ts

@@ -1,3 +1,4 @@
+import {getCiphers} from "node:crypto";
 import {bootstrap} from "@libp2p/bootstrap";
 import {identify} from "@libp2p/identify";
 import type {PrivateKey} from "@libp2p/interface";
@@ -9,12 +10,20 @@ import {Libp2pInit, createLibp2p} from "libp2p";
 import {Registry} from "prom-client";
 import {ENR} from "@chainsafe/enr";
 import {noise} from "@chainsafe/libp2p-noise";
-import {defaultCrypto} from "@chainsafe/libp2p-noise/crypto";
+import {asCrypto, defaultCrypto} from "@chainsafe/libp2p-noise/crypto";
 import {quic} from "@chainsafe/libp2p-quic";
 import {Libp2p, LodestarComponents} from "../interface.js";
 import {NetworkOptions, defaultNetworkOptions} from "../options.js";
 import {Eth2PeerDataStore} from "../peers/datastore.js";
 
+const noiseCrypto = getCiphers().includes("chacha20-poly1305")
+  ? defaultCrypto
+  : {
+      ...defaultCrypto,
+      chaCha20Poly1305Encrypt: asCrypto.chaCha20Poly1305Encrypt,
+      chaCha20Poly1305Decrypt: asCrypto.chaCha20Poly1305Decrypt,
+    };
+
 export type NodeJsLibp2pOpts = {
   peerStoreDir?: string;
   disablePeerDiscovery?: boolean;
@@ -122,7 +131,7 @@ export async function createNodeJsLibp2p(
       listen: localMultiaddrs,
       announce: [],
     },
-    connectionEncrypters: [noise({crypto: defaultCrypto})],
+    connectionEncrypters: [noise({crypto: noiseCrypto})],
     transports,
     streamMuxers: [mplex({disconnectThreshold})],
     peerDiscovery,

package/src/network/processor/gossipHandlers.ts

@@ -1255,11 +1255,15 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     [GossipType.payload_attestation_message]: async ({
       gossipData,
       topic,
+      seenTimestampSec,
     }: GossipHandlerParamGeneric<GossipType.payload_attestation_message>) => {
       const {serializedData} = gossipData;
       const payloadAttestationMessage = sszDeserialize(topic, serializedData);
       const validationResult = await validateGossipPayloadAttestationMessage(chain, payloadAttestationMessage);
 
+      const delaySec = chain.clock.secFromSlot(payloadAttestationMessage.data.slot, seenTimestampSec);
+      metrics?.gossipPayloadAttestationMessage.elapsedTimeTillReceived.observe({source: OpSource.gossip}, delaySec);
+
       try {
         const insertOutcome = chain.payloadAttestationPool.add(
           payloadAttestationMessage,
@@ -1286,11 +1290,16 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     [GossipType.execution_payload_bid]: async ({
       gossipData,
       topic,
+      seenTimestampSec,
     }: GossipHandlerParamGeneric<GossipType.execution_payload_bid>) => {
       const {serializedData} = gossipData;
       const executionPayloadBid = sszDeserialize(topic, serializedData);
       const {proposerIndex} = await validateGossipExecutionPayloadBid(chain, executionPayloadBid);
 
+      // this could be negative, because it's most likely the bid of next slot comes at this clock slot
+      const elapsedSec = chain.clock.secFromSlot(executionPayloadBid.message.slot, seenTimestampSec);
+      metrics?.gossipExecutionPayloadBid.elapsedTimeTillReceived.observe({source: OpSource.gossip}, elapsedSec);
+
       // Handle valid payload bid by storing in a bid pool
       try {
         const insertOutcome = chain.executionPayloadBidPool.add(executionPayloadBid);

package/src/network/reqresp/score.ts

@@ -44,17 +44,11 @@ export function onOutgoingReqRespError(e: RequestError, method: ReqRespMethod):
         // failure, so keep the stronger penalty (Fatal for Ping, as before).
         return method === ReqRespMethod.Ping ? PeerAction.Fatal : PeerAction.LowToleranceError;
       }
-      switch (method) {
-        // Ping and Status are liveness probes; their dial timeouts are dominated by transient
-        // network congestion rather than peer misbehavior, so penalize leniently to avoid
-        // self-inflicted peer starvation (https://github.com/ChainSafe/lodestar/issues/9562),
-        // while still applying some penalty so genuinely dead peers eventually free the slot.
-        case ReqRespMethod.Ping:
-        case ReqRespMethod.Status:
-          return PeerAction.HighToleranceError;
-        default:
-          return PeerAction.LowToleranceError;
-      }
+      // A dial failure means we could not open a stream. That is evidence about the transport, not
+      // about the peer's behavior, and it is dominated by transient congestion on our side, so
+      // penalize leniently for every method to avoid self-inflicted peer starvation (#9562). A
+      // genuinely dead peer still accumulates penalty and is eventually evicted, freeing the slot.
+      return PeerAction.HighToleranceError;
     // TODO: Detect SSZDecodeError and return PeerAction.Fatal
 
     case RequestErrorCode.RESP_TIMEOUT:

package/src/node/nodejs.ts

@@ -1,12 +1,13 @@
 import {setMaxListeners} from "node:events";
 import {PrivateKey} from "@libp2p/interface";
 import {Registry} from "prom-client";
+import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
 import {hasher} from "@chainsafe/persistent-merkle-tree";
 import {BeaconApiMethods} from "@lodestar/api/beacon/server";
 import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ZERO_HASH_HEX} from "@lodestar/params";
-import {IBeaconStateView, PubkeyCache, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
+import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
 import {phase0} from "@lodestar/types";
 import {sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";

package/src/sync/types.ts

@@ -77,7 +77,9 @@ export type PendingPayloadInput = {
 export type PendingPayloadRootHex = {
   status: PendingPayloadInputStatus.pending | PendingPayloadInputStatus.fetching;
   rootHex: RootHex;
-  // message slot hint, may be missing when resolving a parent payload
+  // Trusted slot only (fork choice / validated data), may be missing until resolved. NOT the gossip
+  // message slot from ChainEvent.unknownEnvelopeBlockRoot, which is untrusted and not necessarily the
+  // payload/block slot. See BlockInputSync.resolvePayloadSlot.
   slot?: Slot;
   timeAddedSec: number;
   timeSyncedSec?: number;

package/src/sync/unknownBlock.ts

@@ -1,5 +1,6 @@
 import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
+import {CheckpointWithHex} from "@lodestar/fork-choice";
 import {ForkSeq, isForkPostGloas} from "@lodestar/params";
 import {RequestError, RequestErrorCode} from "@lodestar/reqresp";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
@@ -154,6 +155,7 @@ export class BlockInputSync {
       this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
       this.chain.emitter.on(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
       this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
+      this.chain.emitter.on(ChainEvent.forkChoiceFinalized, this.pruneFinalized);
       this.chain.emitter.on(routes.events.EventType.block, this.onBlockImported);
       this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadImported);
       this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
@@ -170,6 +172,7 @@ export class BlockInputSync {
     this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
     this.chain.emitter.off(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
     this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
+    this.chain.emitter.off(ChainEvent.forkChoiceFinalized, this.pruneFinalized);
     this.chain.emitter.off(routes.events.EventType.block, this.onBlockImported);
     this.chain.emitter.off(routes.events.EventType.executionPayload, this.onPayloadImported);
     this.network.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
@@ -215,7 +218,7 @@ export class BlockInputSync {
 
   private onUnknownEnvelopeBlockRoot = (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRoot]): void => {
     try {
-      this.addByPayloadRootHex(data.rootHex, data.peer, data.slot);
+      this.addByPayloadRootHex(data.rootHex, data.peer, this.resolvePayloadSlot(data.rootHex));
       this.triggerUnknownBlockSearch();
       this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_PAYLOAD_BLOCK_ROOT});
       this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
@@ -258,7 +261,7 @@ export class BlockInputSync {
       }
 
       if (missingDependency.kind === "parentPayload") {
-        this.addByPayloadRootHex(missingDependency.rootHex, data.peer);
+        this.addByPayloadRootHex(missingDependency.rootHex, data.peer, missingDependency.slot);
       } else if (missingDependency.kind === "parentBlock") {
         this.addByRootHex(missingDependency.rootHex, data.peer);
       }
@@ -284,6 +287,61 @@ export class BlockInputSync {
     this.triggerUnknownBlockSearch();
   };
 
+  /**
+   * Drop pending payloads and blocks that fell below the finalized slot.
+   * See https://github.com/ChainSafe/lodestar/issues/9800
+   */
+  private pruneFinalized = (checkpoint: CheckpointWithHex): void => {
+    // Use the actual finalized block slot, not computeStartSlotAtEpoch(checkpoint.epoch)
+    // because the payload at the finalized block is not finalized
+    const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
+
+    let deletedPayloads = 0;
+    let unresolvedSlotPayloads = 0;
+    for (const [rootHex, payload] of this.pendingPayloads) {
+      // Last-effort resolve for a slot-less PendingPayloadRootHex, and store it back for later passes.
+      if (!isPendingPayloadInput(payload) && !isPendingPayloadEnvelope(payload) && payload.slot === undefined) {
+        payload.slot = this.resolvePayloadSlot(rootHex);
+      }
+
+      const slot = getPayloadSyncCacheItemSlot(payload);
+      if (typeof slot !== "number") {
+        unresolvedSlotPayloads++;
+        this.logger.debug("BlockInputSync.pruneFinalized: pending payload with unresolved slot", {
+          root: rootHex,
+          finalizedSlot,
+        });
+        continue;
+      }
+
+      if (slot < finalizedSlot) {
+        this.pendingPayloads.delete(rootHex);
+        deletedPayloads++;
+      }
+    }
+
+    let deletedBlocks = 0;
+    for (const [rootHex, block] of this.pendingBlocks) {
+      const slot = getBlockInputSyncCacheItemSlot(block);
+      if (typeof slot === "number" && slot < finalizedSlot) {
+        this.pendingBlocks.delete(rootHex);
+        deletedBlocks++;
+      }
+    }
+
+    if (deletedPayloads > 0 || deletedBlocks > 0) {
+      this.metrics?.blockInputSync.removedBlocks.inc(deletedPayloads + deletedBlocks);
+    }
+
+    this.logger.debug("BlockInputSync.pruneFinalized dropped pre-finalized pending items", {
+      finalizedSlot,
+      finalizedRoot: checkpoint.rootHex,
+      deletedPayloads,
+      deletedBlocks,
+      unresolvedSlotPayloads,
+    });
+  };
+
   private addByRootHex = (rootHex: RootHex, peerIdStr?: PeerIdStr): boolean => {
     let pendingBlock = this.pendingBlocks.get(rootHex);
     let added = false;
@@ -382,6 +440,29 @@ export class BlockInputSync {
     return added;
   };
 
+  /**
+   * Resolve the block slot for a payload root.
+   */
+  private resolvePayloadSlot(rootHex: RootHex): Slot | undefined {
+    // a pending block of the same root
+    const pendingBlock = this.pendingBlocks.get(rootHex);
+    if (pendingBlock) {
+      const pendingBlockSlot = getBlockInputSyncCacheItemSlot(pendingBlock);
+      if (typeof pendingBlockSlot === "number") {
+        return pendingBlockSlot;
+      }
+    }
+
+    // the block is in fork choice
+    const forkChoiceSlot = this.chain.forkChoice.getBlockHexDefaultStatus(rootHex)?.slot;
+    if (forkChoiceSlot !== undefined) {
+      return forkChoiceSlot;
+    }
+
+    // a seen payload envelope input
+    return this.chain.seenPayloadEnvelopeInputCache.get(rootHex)?.slot;
+  }
+
   private addByPayloadInput = (
     payloadInput: PayloadEnvelopeInput,
     peerIdStr?: PeerIdStr,
@@ -436,10 +517,12 @@ export class BlockInputSync {
     blockInput: IBlockInput
   ):
     | {kind: "ready"}
-    | {kind: "block" | "parentBlock" | "parentPayload"; rootHex: RootHex}
+    | {kind: "block" | "parentBlock"; rootHex: RootHex}
+    | {kind: "parentPayload"; rootHex: RootHex; slot: Slot}
     | {kind: "invalidParentPayload"; parentRootHex: RootHex; parentBlockHashHex: RootHex} {
     const parentRootHex = blockInput.parentRootHex;
-    if (!this.chain.forkChoice.hasBlockHex(parentRootHex)) {
+    const parentBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentRootHex);
+    if (!parentBlock) {
       return {kind: "parentBlock", rootHex: parentRootHex};
     }
 
@@ -464,13 +547,13 @@ export class BlockInputSync {
     const parentPayloadInput = this.chain.seenPayloadEnvelopeInputCache.get(parentRootHex);
     if (parentPayloadInput) {
       if (parentPayloadInput.getBlockHashHex() === parentBlockHashHex) {
-        return {kind: "parentPayload", rootHex: parentRootHex};
+        return {kind: "parentPayload", rootHex: parentRootHex, slot: parentBlock.slot};
       }
 
       return {kind: "invalidParentPayload", parentRootHex, parentBlockHashHex};
     }
 
-    return {kind: "parentPayload", rootHex: parentRootHex};
+    return {kind: "parentPayload", rootHex: parentRootHex, slot: parentBlock.slot};
   }
 
   private advancePendingBlock(pendingBlock: PendingBlockInput): AdvancePendingBlockResult {
@@ -493,9 +576,9 @@ export class BlockInputSync {
       }
 
       case "parentPayload": {
-        let added = this.addByPayloadRootHex(missingDependency.rootHex);
+        let added = this.addByPayloadRootHex(missingDependency.rootHex, undefined, missingDependency.slot);
         for (const peerIdStr of pendingBlock.peerIdStrings) {
-          added = this.addByPayloadRootHex(missingDependency.rootHex, peerIdStr) || added;
+          added = this.addByPayloadRootHex(missingDependency.rootHex, peerIdStr, missingDependency.slot) || added;
         }
         return added ? "queued_parent_payload" : "blocked";
       }
@@ -928,6 +1011,7 @@ export class BlockInputSync {
         const pendingPayloadByRoot: PendingPayloadRootHex = {
           status: PendingPayloadInputStatus.pending,
           rootHex,
+          slot: pendingPayload.envelope.message.payload.slotNumber,
           timeAddedSec: pendingPayload.timeAddedSec,
           peerIdStrings: new Set(pendingPayload.peerIdStrings),
         };
@@ -967,6 +1051,10 @@ export class BlockInputSync {
       return;
     }
 
+    if (!isPendingPayloadInput(payload) && payload.slot === undefined) {
+      payload.slot = this.resolvePayloadSlot(rootHex);
+    }
+
     const payloadSlot = getPayloadSyncCacheItemSlot(payload);
     const logCtx = {
       slot: payloadSlot,
@@ -982,7 +1070,13 @@ export class BlockInputSync {
     const res = await wrapError(this.fetchPayloadInput(payload));
     if (!res.err) {
       const pendingPayload = res.result;
-      this.pendingPayloads.set(getPayloadSyncCacheItemRootHex(pendingPayload), pendingPayload);
+      const resultRootHex = getPayloadSyncCacheItemRootHex(pendingPayload);
+      // finalization may have deleted this entry while we were awaiting the network, do not resurrect it via the set below
+      if (!this.pendingPayloads.has(resultRootHex)) {
+        this.logger.verbose("Dropping downloaded payload, entry pruned during fetch", logCtx);
+        return;
+      }
+      this.pendingPayloads.set(resultRootHex, pendingPayload);
 
       if (isPendingPayloadEnvelope(pendingPayload)) {
         await this.reconcilePayloadEnvelope(pendingPayload);
@@ -1110,9 +1204,11 @@ export class BlockInputSync {
           );
         }
 
-        throw Error(
-          `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
-        );
+        const reason =
+          pendingColumns.size > 0
+            ? `cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
+            : "no peer available to download pending payload";
+        throw Error(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: ${reason}`);
       }
 
       const {peerId, client: peerClient} = peerMeta;

package/lib/chain/bls/maybeBatch.d.ts

@@ -1,12 +0,0 @@
-import { PublicKey } from "@chainsafe/blst";
-export type SignatureSetDeserialized = {
-    publicKey: PublicKey;
-    message: Uint8Array;
-    signature: Uint8Array;
-};
-/**
- * Verify signatures sets with batch verification or regular core verify depending on the set count.
- * Abstracted in a separate file to be consumed by the threaded pool and the main thread implementation.
- */
-export declare function verifySignatureSetsMaybeBatch(sets: SignatureSetDeserialized[]): boolean;
-//# sourceMappingURL=maybeBatch.d.ts.map