@lodestar/beacon-node 1.44.0-dev.c874e6b4ce → 1.44.0-dev.d01471ec84

package/src/api/impl/lodestar/index.ts

@@ -280,6 +280,36 @@ export function getLodestarApi({
       };
     },
 
+    async getFastConfirmationInfo() {
+      const confirmedRoot = chain.forkChoice.getConfirmedRoot();
+      const confirmedBlock = chain.forkChoice.getConfirmedBlock();
+      const justifiedCheckpoint = chain.forkChoice.getJustifiedCheckpoint();
+      const finalizedCheckpoint = chain.forkChoice.getFinalizedCheckpoint();
+      const headRoot = chain.forkChoice.getHeadRoot();
+      const head = chain.forkChoice.getHead();
+
+      return {
+        data: {
+          confirmed: {
+            rootHex: confirmedRoot,
+            slot: confirmedBlock?.slot ?? null,
+          },
+          head: {
+            rootHex: headRoot,
+            slot: head.slot,
+          },
+          justifiedCheckpoint: {
+            rootHex: justifiedCheckpoint.rootHex,
+            epoch: justifiedCheckpoint.epoch,
+          },
+          finalizedCheckpoint: {
+            rootHex: finalizedCheckpoint.rootHex,
+            epoch: finalizedCheckpoint.epoch,
+          },
+        },
+      };
+    },
+
     async getAttesterSlashingsFromBlocks({signedBlocks}) {
       const attestations = new Map<Epoch, Attestation[]>();
 

package/src/api/impl/validator/index.ts

@@ -927,11 +927,9 @@ export function getValidatorApi(
       // TODO GLOAS: respect builderSelection (MaxProfit, BuilderAlways, ExecutionAlways, etc.) to let
       // the user control bid source preferences and value comparison. Also add external builder api
       // support when it is implemented.
-      const builderBid = chain.executionPayloadBidPool.getBestBid(
-        slot,
-        parentBlock.executionPayloadBlockHash,
-        parentBlockRootHex
-      );
+      const isBuildingOnFull = chain.forkChoice.shouldBuildOnFull(parentBlock, slot);
+      const bidParentBlockHash = isBuildingOnFull ? parentBlock.executionPayloadBlockHash : parentBlock.parentBlockHash;
+      const builderBid = chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
 
       const logCtx = {
         slot,

package/src/chain/archiveStore/archiveStore.ts

@@ -30,7 +30,6 @@ export enum ArchiveStoreTask {
   PruneHistory = "prune_history",
   OnFinalizedCheckpoint = "on_finalized_checkpoint",
   MaybeArchiveState = "maybe_archive_state",
-  RegenPruneOnFinalized = "regen_prune_on_finalized",
   ForkchoicePrune = "forkchoice_prune",
   UpdateBackfillRange = "update_backfill_range",
 }
@@ -229,10 +228,6 @@ export class ArchiveStore {
       await this.statesArchiverStrategy.maybeArchiveState(finalized, this.metrics);
       timer?.({source: ArchiveStoreTask.MaybeArchiveState});
 
-      timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer();
-      this.chain.regen.pruneOnFinalized(finalizedEpoch);
-      timer?.({source: ArchiveStoreTask.RegenPruneOnFinalized});
-
       // tasks rely on extended fork choice
       timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer();
       const prunedBlocks = this.chain.forkChoice.prune(finalized.rootHex);

package/src/chain/chain.ts

@@ -2,7 +2,7 @@ import path from "node:path";
 import {PrivateKey} from "@libp2p/interface";
 import {Type} from "@chainsafe/ssz";
 import {BeaconConfig} from "@lodestar/config";
-import {CheckpointWithHex, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice";
+import {CheckpointWithHex, ForkChoiceStateGetter, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice";
 import {LoggerNode} from "@lodestar/logger/node";
 import {
   EFFECTIVE_BALANCE_INCREMENT,
@@ -382,6 +382,14 @@ export class BeaconChain implements IBeaconChain {
     blockStateCache.setHeadState(anchorState);
     checkpointStateCache.add(checkpoint, anchorState);
 
+    const forkChoiceStateGetter: ForkChoiceStateGetter = ({stateRoot, checkpoint}) => {
+      if (stateRoot) return blockStateCache.get(stateRoot);
+
+      if (checkpoint) return checkpointStateCache.get({epoch: checkpoint.epoch, rootHex: checkpoint.rootHex});
+
+      return null;
+    };
+
     const forkChoice = initializeForkChoice(
       config,
       emitter,
@@ -390,6 +398,7 @@ export class BeaconChain implements IBeaconChain {
       isAnchorStateFinalized,
       opts,
       this.justifiedBalancesGetter.bind(this),
+      forkChoiceStateGetter,
       metrics,
       logger
     );

package/src/chain/emitter.ts

@@ -3,7 +3,7 @@ import {StrictEventEmitter} from "strict-event-emitter-types";
 import {routes} from "@lodestar/api";
 import {CheckpointWithHex} from "@lodestar/fork-choice";
 import {IBeaconStateView} from "@lodestar/state-transition";
-import {DataColumnSidecar, RootHex, deneb, phase0} from "@lodestar/types";
+import {DataColumnSidecar, RootHex, Slot, deneb, phase0} from "@lodestar/types";
 import {PeerIdStr} from "../util/peerId.js";
 import {BlockInputSource, IBlockInput} from "./blocks/blockInput/types.js";
 import {PayloadEnvelopeInput} from "./blocks/payloadEnvelopeInput/payloadEnvelopeInput.js";
@@ -94,7 +94,8 @@ export type ChainEventData = {
     peer: PeerIdStr;
     source: BlockInputSource;
   };
-  [ChainEvent.unknownEnvelopeBlockRoot]: {rootHex: RootHex; peer?: PeerIdStr; source: BlockInputSource};
+  // slot is the message slot, not necessarily the envelope's slot, but useful as a logging/prune hint
+  [ChainEvent.unknownEnvelopeBlockRoot]: {rootHex: RootHex; slot: Slot; peer?: PeerIdStr; source: BlockInputSource};
 };
 
 export type IChainEvents = ApiEvents & {

package/src/chain/errors/blockError.ts

@@ -61,9 +61,6 @@ export enum BlockErrorCode {
   TRANSACTIONS_TOO_BIG = "BLOCK_ERROR_TRANSACTIONS_TOO_BIG",
   /** Execution engine is unavailable, syncing, or api call errored. Peers must not be downscored on this code */
   EXECUTION_ENGINE_ERROR = "BLOCK_ERROR_EXECUTION_ERROR",
-  /** The attestation head block is too far behind the attestation slot, causing many skip slots.
-  This is deemed a DoS risk */
-  TOO_MANY_SKIPPED_SLOTS = "TOO_MANY_SKIPPED_SLOTS",
   /** The blobs are unavailable */
   DATA_UNAVAILABLE = "BLOCK_ERROR_DATA_UNAVAILABLE",
   /** Block contains too many kzg commitments */
@@ -89,7 +86,6 @@ export type BlockErrorType =
   | {code: BlockErrorCode.FUTURE_SLOT; blockSlot: Slot; currentSlot: Slot}
   | {code: BlockErrorCode.STATE_ROOT_MISMATCH}
   | {code: BlockErrorCode.GENESIS_BLOCK}
-  | {code: BlockErrorCode.TOO_MANY_SKIPPED_SLOTS; parentSlot: Slot; blockSlot: Slot}
   | {code: BlockErrorCode.WOULD_REVERT_FINALIZED_SLOT; blockSlot: Slot; finalizedSlot: Slot}
   | {code: BlockErrorCode.ALREADY_KNOWN; root: RootHex}
   | {code: BlockErrorCode.REPEAT_PROPOSAL; proposerIndex: ValidatorIndex}

package/src/chain/forkChoice/index.ts

@@ -1,7 +1,9 @@
+import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
 import {
   ExecutionStatus,
   ForkChoice,
+  ForkChoiceStateGetter,
   ForkChoiceStore,
   JustifiedBalancesGetter,
   PayloadStatus,
@@ -45,6 +47,7 @@ export function initializeForkChoice(
   isFinalizedState: boolean,
   opts: ForkChoiceOpts,
   justifiedBalancesGetter: JustifiedBalancesGetter,
+  stateGetter: ForkChoiceStateGetter,
   metrics: Metrics | null,
   logger?: Logger
 ): ForkChoice {
@@ -56,6 +59,7 @@ export function initializeForkChoice(
         state,
         opts,
         justifiedBalancesGetter,
+        stateGetter,
         metrics,
         logger
       )
@@ -66,6 +70,7 @@ export function initializeForkChoice(
         state,
         opts,
         justifiedBalancesGetter,
+        stateGetter,
         metrics,
         logger
       );
@@ -81,6 +86,7 @@ export function initializeForkChoiceFromFinalizedState(
   state: IBeaconStateView,
   opts: ForkChoiceOpts,
   justifiedBalancesGetter: JustifiedBalancesGetter,
+  stateGetter: ForkChoiceStateGetter,
   metrics: Metrics | null,
   logger?: Logger
 ): ForkChoice {
@@ -112,9 +118,12 @@ export function initializeForkChoiceFromFinalizedState(
       finalizedCheckpoint,
       justifiedBalances,
       justifiedBalancesGetter,
+      stateGetter,
       {
         onJustified: (cp) => emitter.emit(ChainEvent.forkChoiceJustified, cp),
         onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp),
+        onFastConfirmation: ({block, slot, currentSlot}) =>
+          emitter.emit(routes.events.EventType.fastConfirmation, {block, slot, currentSlot}),
       }
     ),
 
@@ -172,6 +181,7 @@ export function initializeForkChoiceFromUnfinalizedState(
   unfinalizedState: IBeaconStateView,
   opts: ForkChoiceOpts,
   justifiedBalancesGetter: JustifiedBalancesGetter,
+  stateGetter: ForkChoiceStateGetter,
   metrics: Metrics | null,
   logger?: Logger
 ): ForkChoice {
@@ -203,9 +213,12 @@ export function initializeForkChoiceFromUnfinalizedState(
     finalizedCheckpoint,
     justifiedBalances,
     justifiedBalancesGetter,
+    stateGetter,
     {
       onJustified: (cp) => emitter.emit(ChainEvent.forkChoiceJustified, cp),
       onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp),
+      onFastConfirmation: ({block, slot, currentSlot}) =>
+        emitter.emit(routes.events.EventType.fastConfirmation, {block, slot, currentSlot}),
     }
   );
 

package/src/chain/options.ts

@@ -105,6 +105,7 @@ export const defaultChainOptions: IChainOptions = {
   proposerBoost: true,
   proposerBoostReorg: true,
   computeUnrealized: true,
+  fastConfirmation: false,
   suggestedFeeRecipient: defaultValidatorOptions.suggestedFeeRecipient,
   serveHistoricalState: false,
   assertCorrectProgressiveBalances: false,

package/src/chain/regen/interface.ts

@@ -47,7 +47,6 @@ export interface IStateRegenerator extends IStateRegeneratorInternal {
   getCheckpointStateSync(cp: CheckpointHex): IBeaconStateView | null;
   getClosestHeadState(head: ProtoBlock): IBeaconStateView | null;
   pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void;
-  pruneOnFinalized(finalizedEpoch: Epoch): void;
   processState(blockRootHex: RootHex, postState: IBeaconStateView): void;
   addCheckpointState(cp: phase0.Checkpoint, item: IBeaconStateView): void;
   updateHeadState(newHead: ProtoBlock, maybeHeadState: IBeaconStateView): void;

package/src/chain/regen/queued.ts

@@ -143,11 +143,6 @@ export class QueuedStateRegenerator implements IStateRegenerator {
     this.blockStateCache.prune(headStateRoot);
   }
 
-  pruneOnFinalized(finalizedEpoch: number): void {
-    this.checkpointStateCache.pruneFinalized(finalizedEpoch);
-    this.blockStateCache.deleteAllBeforeEpoch(finalizedEpoch);
-  }
-
   processState(blockRootHex: RootHex, postState: IBeaconStateView): void {
     this.blockStateCache.add(postState);
     this.checkpointStateCache.processState(blockRootHex, postState).catch((e) => {

package/src/chain/stateCache/fifoBlockStateCache.ts

@@ -167,12 +167,6 @@ export class FIFOBlockStateCache implements BlockStateCache {
     }
   }
 
-  /**
-   * No need for this implementation
-   * This is only to conform to the old api
-   */
-  deleteAllBeforeEpoch(): void {}
-
   /**
    * ONLY FOR DEBUGGING PURPOSES. For lodestar debug API.
    */

package/src/chain/stateCache/persistentCheckpointsCache.ts

@@ -414,11 +414,12 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
 
   /**
    * Prune all checkpoint states before the provided finalized epoch.
+   * Driven sequentially from processState() so it never interleaves with persist.
    */
-  pruneFinalized(finalizedEpoch: Epoch): void {
+  private async pruneFinalized(finalizedEpoch: Epoch): Promise<void> {
     for (const epoch of this.epochIndex.keys()) {
       if (epoch < finalizedEpoch) {
-        this.deleteAllEpochItems(epoch).catch((e) =>
+        await this.deleteAllEpochItems(epoch).catch((e) =>
           this.logger.debug("Error delete all epoch items", {epoch, finalizedEpoch}, e as Error)
         );
       }
@@ -476,6 +477,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
    * As of Mar 2024, it takes <=350ms to persist a holesky state on fast server
    */
   async processState(blockRootHex: RootHex, state: IBeaconStateView): Promise<number> {
+    // prune finalized in the same flow so a finalized cp state is pruned, never persisted
+    await this.pruneFinalized(state.finalizedCheckpoint.epoch);
+
     let persistCount = 0;
     // it's important to sort the epochs in ascending order, in case of big reorg we always want to keep the most recent checkpoint states
     const sortedEpochs = Array.from(this.epochIndex.keys()).sort((a, b) => a - b);

package/src/chain/stateCache/types.ts

@@ -30,7 +30,6 @@ export interface BlockStateCache {
   clear(): void;
   size: number;
   prune(headStateRootHex: RootHex): void;
-  deleteAllBeforeEpoch(finalizedEpoch: Epoch): void;
   dumpSummary(): routes.lodestar.StateCacheItem[];
   /** Expose beacon states stored in cache. Use with caution */
   getStates(): IterableIterator<IBeaconStateView>;
@@ -67,7 +66,6 @@ export interface CheckpointStateCache {
   getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise<IBeaconStateView | null>;
   updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null;
   prune(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void;
-  pruneFinalized(finalizedEpoch: Epoch): void;
   processState(blockRootHex: RootHex, state: IBeaconStateView): Promise<number>;
   clear(): void;
   dumpSummary(): routes.lodestar.StateCacheItem[];

package/src/chain/validation/block.ts

@@ -15,12 +15,17 @@ import {BlockErrorCode, BlockGossipError, GossipAction} from "../errors/index.js
 import {IBeaconChain} from "../interface.js";
 import {RegenCaller} from "../regen/index.js";
 
+export type GossipBlockValidationResult = {
+  /** Number of skipped slots between the block and its parent (blockSlot - parentSlot - 1) */
+  skippedSlots: number;
+};
+
 export async function validateGossipBlock(
   config: ChainForkConfig,
   chain: IBeaconChain,
   signedBlock: SignedBeaconBlock,
   fork: ForkName
-): Promise<void> {
+): Promise<GossipBlockValidationResult> {
   const block = signedBlock.message;
   const blockSlot = block.slot;
   const blockEpoch = computeEpochAtSlot(blockSlot);
@@ -109,21 +114,6 @@ export async function validateGossipBlock(
     }
   }
 
-  // [IGNORE] The attestation head block is too far behind the attestation slot, causing many skip slots.
-  // This is deemed a DoS risk because we need to get the proposerShuffling. To get the shuffling we have
-  // to do a bunch of epoch transitions, the longer the distance between the parent and block,
-  // the more we have to do. epochTransitions are expensive ~750ms, so we must limit how many a
-  // single bad block can trigger
-  // Note: Ensure this check is done before calling chain.regen.getBlockSlotStat as this is the function that does various epoch transitions.
-  // Note: This validation check is not part of the spec.
-  if (chain.opts.maxSkipSlots != null && parentBlock.slot + chain.opts.maxSkipSlots < blockSlot) {
-    throw new BlockGossipError(GossipAction.IGNORE, {
-      code: BlockErrorCode.TOO_MANY_SKIPPED_SLOTS,
-      parentSlot: parentBlock.slot,
-      blockSlot,
-    });
-  }
-
   // [REJECT] The block is from a higher slot than its parent.
   if (parentBlock.slot >= blockSlot) {
     throw new BlockGossipError(GossipAction.REJECT, {
@@ -133,6 +123,10 @@ export async function validateGossipBlock(
     });
   }
 
+  // Number of skipped slots between block and parent (non-spec). Previously this gated blocks via
+  // maxSkipSlots; now the caller only observes it so legitimate post-skip blocks are no longer ignored.
+  const skippedSlots = blockSlot - parentBlock.slot - 1;
+
   // [REJECT] The length of KZG commitments is less than or equal to the limitation defined in Consensus Layer -- i.e. validate that len(body.signed_beacon_block.message.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK
   if (isForkPostDeneb(fork) && !isForkPostGloas(fork)) {
     const blobKzgCommitmentsLen = (block as deneb.BeaconBlock).body.blobKzgCommitments.length;
@@ -247,4 +241,6 @@ export async function validateGossipBlock(
   }
 
   chain.seenBlockProposers.add(blockSlot, proposerIndex);
+
+  return {skippedSlots};
 }

package/src/chain/validation/executionPayloadEnvelope.ts

@@ -35,8 +35,6 @@ async function validateExecutionPayloadEnvelope(
   // [IGNORE] The envelope's block root `envelope.beacon_block_root` has been seen (via
   // gossip or non-gossip sources) (a client MAY queue payload for processing once
   // the block is retrieved).
-  // TODO GLOAS: Need to review this, we should queue the envelope for later
-  // processing if the block is not yet known, otherwise we would ignore it here
   const block = chain.forkChoice.getBlockDefaultStatus(envelope.beaconBlockRoot);
   if (block === null) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, {

package/src/metrics/metrics/lodestar.ts

@@ -610,6 +610,11 @@ export function createLodestarMetrics(
         help: "The origination source of one of the BlockInputSync triggers",
         labelNames: ["source"],
       }),
+      payloadSource: register.counter<{source: BlockInputSource}>({
+        name: "lodestar_payload_input_sync_source_total",
+        help: "Count of payload (execution payload envelope) sync triggers, labeled by their source",
+        labelNames: ["source"],
+      }),
       pendingBlocks: register.gauge({
         name: "lodestar_sync_unknown_block_pending_blocks_size",
         help: "Current size of UnknownBlockSync pending blocks cache",
@@ -861,6 +866,12 @@ export function createLodestarMetrics(
         labelNames: ["numBlobs"],
       }),
 
+      skippedSlots: register.histogram({
+        name: "lodestar_gossip_block_skipped_slots",
+        help: "Number of skipped slots between a gossip block and its parent (blockSlot - parentSlot - 1)",
+        buckets: [0, 1, 2, 4, 8, 16, 32],
+      }),
+
       processBlockErrors: register.gauge<{error: BlockErrorCode | "NOT_BLOCK_ERROR"}>({
         name: "lodestar_gossip_block_process_block_errors",
         help: "Count of errors, by error type, while processing blocks",

package/src/network/processor/gossipHandlers.ts

@@ -185,7 +185,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       peerIdStr,
     });
     try {
-      await validateGossipBlock(config, chain, signedBlock, fork);
+      const {skippedSlots} = await validateGossipBlock(config, chain, signedBlock, fork);
 
       if (isForkPostGloas(fork)) {
         chain.seenPayloadEnvelopeInputCache.add({
@@ -205,8 +205,15 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
       metrics?.gossipBlock.gossipValidation.recvToValidation.observe(recvToValidation);
       metrics?.gossipBlock.gossipValidation.validationTime.observe(validationTime);
+      metrics?.gossipBlock.skippedSlots.observe(skippedSlots);
 
-      logger.debug("Validated gossip block", {...blockInputMeta, ...logCtx, recvToValidation, validationTime});
+      logger.debug("Validated gossip block", {
+        ...blockInputMeta,
+        ...logCtx,
+        recvToValidation,
+        validationTime,
+        skippedSlots,
+      });
 
       chain.emitter.emit(routes.events.EventType.blockGossip, {slot, block: blockRootHex});
 

package/src/network/processor/index.ts

@@ -186,9 +186,11 @@ export class NetworkProcessor {
   // we may not receive the block for messages like Attestation and SignedAggregateAndProof messages, in that case PendingGossipsubMessage needs
   // to be stored in this Map and reprocessed once the block comes
   private readonly awaitingMessagesByBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
+  private awaitingBlockMessageCount = 0;
   // we may not receive the payload for messages that require the FULL payload variant to be processed,
   // in that case PendingGossipsubMessage needs to be stored in this Map and reprocessed once the payload comes
   private readonly awaitingMessagesByPayloadBlockRoot: MapDef<RootHex, Set<PendingGossipsubMessage>>;
+  private awaitingPayloadMessageCount = 0;
   private unknownBlocksBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
   private unknownEnvelopesBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
 
@@ -228,8 +230,8 @@ export class NetworkProcessor {
           metrics.gossipValidationQueue.keySize.set({topic}, this.gossipQueues[topic].keySize);
           metrics.gossipValidationQueue.concurrency.set({topic}, this.gossipTopicConcurrency[topic]);
         }
-        metrics.awaitingBlockGossipMessages.countPerSlot.set(this.unknownBlockGossipsubMessagesCount);
-        metrics.awaitingPayloadGossipMessages.countPerSlot.set(this.unknownPayloadGossipsubMessagesCount);
+        metrics.awaitingBlockGossipMessages.countPerSlot.set(this.awaitingBlockMessageCount);
+        metrics.awaitingPayloadGossipMessages.countPerSlot.set(this.awaitingPayloadMessageCount);
         // specific metric for beacon_attestation topic
         metrics.gossipValidationQueue.keyAge.reset();
         for (const ageMs of this.gossipQueues.beacon_attestation.getDataAgeMs()) {
@@ -297,7 +299,7 @@ export class NetworkProcessor {
       return;
     }
     this.unknownEnvelopesBySlot.getOrDefault(slot).add(root);
-    this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, peer, source});
+    this.chain.emitter.emit(ChainEvent.unknownEnvelopeBlockRoot, {rootHex: root, slot, peer, source});
   }
 
   private onPendingGossipsubMessage = (message: PendingGossipsubMessage): void => {
@@ -497,7 +499,7 @@ export class NetworkProcessor {
         this.pushPendingGossipsubMessageToQueue(message);
         break;
       case PreprocessAction.AwaitBlock: {
-        if (this.unknownBlockGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
+        if (this.awaitingBlockMessageCount > MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS) {
           // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           this.metrics?.awaitingBlockGossipMessages.reject.inc({
             reason: ReprocessRejectReason.reached_limit,
@@ -509,10 +511,11 @@ export class NetworkProcessor {
         this.metrics?.awaitingBlockGossipMessages.queue.inc({topic: topicType});
         const awaitingGossipsubMessages = this.awaitingMessagesByBlockRoot.getOrDefault(preprocessResult.root);
         awaitingGossipsubMessages.add(message);
+        this.awaitingBlockMessageCount++;
         break;
       }
       case PreprocessAction.AwaitEnvelope: {
-        if (this.unknownPayloadGossipsubMessagesCount > MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS) {
+        if (this.awaitingPayloadMessageCount > MAX_QUEUED_UNKNOWN_PAYLOAD_GOSSIP_OBJECTS) {
           this.metrics?.awaitingPayloadGossipMessages.reject.inc({
             reason: ReprocessRejectReason.reached_limit,
             topic: topicType,
@@ -525,6 +528,7 @@ export class NetworkProcessor {
           preprocessResult.root
         );
         awaitingPayloadGossipsubMessages.add(message);
+        this.awaitingPayloadMessageCount++;
         break;
       }
     }
@@ -548,6 +552,12 @@ export class NetworkProcessor {
       return;
     }
 
+    // Atomically remove from map and update counter before async iteration to
+    // prevent double-decrement race with onClockSlot during yield points below
+    if (this.awaitingMessagesByBlockRoot.delete(rootHex)) {
+      this.awaitingBlockMessageCount -= waitingGossipsubMessages.size;
+    }
+
     const nowSec = Date.now() / 1000;
     let count = 0;
     // TODO: we can group attestations to process in batches but since we have the SeenAttestationDatas
@@ -567,8 +577,6 @@ export class NetworkProcessor {
         await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
       }
     }
-
-    this.awaitingMessagesByBlockRoot.delete(rootHex);
   };
 
   private onPayloadEnvelopeProcessed = async ({blockRoot: rootHex}: {blockRoot: RootHex}): Promise<void> => {
@@ -577,6 +585,12 @@ export class NetworkProcessor {
       return;
     }
 
+    // Atomically remove from map and update counter before async iteration to
+    // prevent double-decrement race with onClockSlot during yield points below
+    if (this.awaitingMessagesByPayloadBlockRoot.delete(rootHex)) {
+      this.awaitingPayloadMessageCount -= waitingGossipsubMessages.size;
+    }
+
     const nowSec = Date.now() / 1000;
     let count = 0;
     for (const message of waitingGossipsubMessages) {
@@ -593,8 +607,6 @@ export class NetworkProcessor {
         await sleep(AWAITING_GOSSIP_OBJECTS_YIELD_EVERY_MS);
       }
     }
-
-    this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
   };
 
   private onClockSlot = (clockSlot: Slot): void => {
@@ -618,7 +630,9 @@ export class NetworkProcessor {
             );
             // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           }
-          this.awaitingMessagesByBlockRoot.delete(rootHex);
+          if (this.awaitingMessagesByBlockRoot.delete(rootHex)) {
+            this.awaitingBlockMessageCount -= gossipMessages.size;
+          }
         }
       }
       this.unknownBlocksBySlot.delete(slot);
@@ -641,7 +655,9 @@ export class NetworkProcessor {
             );
             // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
           }
-          this.awaitingMessagesByPayloadBlockRoot.delete(rootHex);
+          if (this.awaitingMessagesByPayloadBlockRoot.delete(rootHex)) {
+            this.awaitingPayloadMessageCount -= gossipMessages.size;
+          }
         }
       }
       this.unknownEnvelopesBySlot.delete(slot);
@@ -784,20 +800,4 @@ export class NetworkProcessor {
 
     return null;
   }
-
-  private get unknownBlockGossipsubMessagesCount(): number {
-    let count = 0;
-    for (const messages of this.awaitingMessagesByBlockRoot.values()) {
-      count += messages.size;
-    }
-    return count;
-  }
-
-  private get unknownPayloadGossipsubMessagesCount(): number {
-    let count = 0;
-    for (const messages of this.awaitingMessagesByPayloadBlockRoot.values()) {
-      count += messages.size;
-    }
-    return count;
-  }
 }

package/src/network/reqresp/handlers/beaconBlocksByRange.ts

@@ -1,6 +1,6 @@
 import {PeerId} from "@libp2p/interface";
 import {BeaconConfig} from "@lodestar/config";
-import {GENESIS_SLOT, isForkPostDeneb, isForkPostFulu} from "@lodestar/params";
+import {GENESIS_SLOT, isForkPostDeneb} from "@lodestar/params";
 import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {deneb, phase0} from "@lodestar/types";
@@ -29,13 +29,20 @@ export async function* onBeaconBlocksByRange(
   // starts above it to avoid duplicate yields. See archiveBlocks.ts for the migration logic.
   const archiveMaxSlot = finalizedSlot;
 
-  const forkName = chain.config.getForkName(startSlot);
-  if (isForkPostFulu(forkName) && startSlot < chain.earliestAvailableSlot) {
-    chain.logger.verbose("Peer did not respect earliestAvailableSlot for BeaconBlocksByRange", {
+  // endSlot is exclusive, so highest served slot is endSlot - 1.
+  // Throw only when the entire requested range is below earliestAvailableSlot.
+  if (endSlot - 1 < chain.earliestAvailableSlot) {
+    chain.logger.verbose("Peer requested range before earliestAvailableSlot for BeaconBlocksByRange", {
       peer: prettyPrintPeerId(peerId),
       client: peerClient,
+      startSlot,
+      count,
+      earliestAvailableSlot: chain.earliestAvailableSlot,
     });
-    return;
+    throw new ResponseError(
+      RespStatus.RESOURCE_UNAVAILABLE,
+      `Requested range is before earliestAvailableSlot startSlot=${startSlot} count=${count} earliestAvailableSlot=${chain.earliestAvailableSlot}`
+    );
   }
 
   // Finalized range of blocks

package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts

@@ -34,12 +34,20 @@ export async function* onDataColumnSidecarsByRange(
     return;
   }
 
-  if (startSlot < chain.earliestAvailableSlot) {
-    chain.logger.verbose("Peer did not respect earliestAvailableSlot for DataColumnSidecarsByRange", {
+  // endSlot is exclusive, so highest served slot is endSlot - 1.
+  // Throw only when the entire requested range is below earliestAvailableSlot.
+  if (endSlot - 1 < chain.earliestAvailableSlot) {
+    chain.logger.verbose("Peer requested range before earliestAvailableSlot for DataColumnSidecarsByRange", {
       peer: prettyPrintPeerId(peerId),
       client: peerClient,
+      startSlot,
+      count,
+      earliestAvailableSlot: chain.earliestAvailableSlot,
     });
-    return;
+    throw new ResponseError(
+      RespStatus.RESOURCE_UNAVAILABLE,
+      `Requested range is before earliestAvailableSlot startSlot=${startSlot} count=${count} earliestAvailableSlot=${chain.earliestAvailableSlot}`
+    );
   }
 
   const finalized = db.dataColumnSidecarArchive;

package/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts

@@ -30,7 +30,7 @@ export async function* onDataColumnSidecarsByRoot(
     const {blockRoot, columns: requestedColumns} = dataColumnsByRootIdentifier;
     const availableColumns = validateRequestedDataColumns(chain, requestedColumns);
     if (availableColumns.length === 0) {
-      return;
+      continue;
     }
 
     const blockRootHex = toRootHex(blockRoot);