@lodestar/beacon-node 1.49.0-dev.b27f498c2c → 1.49.0-dev.c256679454

package/src/network/reqresp/handlers/index.ts

@@ -1,5 +1,5 @@
 import type {Type} from "@chainsafe/ssz";
-import {ProtocolHandler, RespStatus, ResponseError} from "@lodestar/reqresp";
+import {ProtocolHandler, RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {ssz} from "@lodestar/types";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
@@ -38,6 +38,29 @@ function deserializeRequestBody<T>(type: Type<T>, data: Uint8Array): T {
   }
 }
 
+async function* trackByRangeResponse(
+  chain: IBeaconChain,
+  method: ReqRespMethod,
+  count: number,
+  source: AsyncIterable<ResponseOutgoing>,
+  columns?: number
+): AsyncIterable<ResponseOutgoing> {
+  chain.metrics?.reqRespByRange.requestedSlots.observe({method}, count);
+  if (columns !== undefined) {
+    chain.metrics?.reqRespByRange.requestedColumns.observe(columns);
+  }
+
+  let servedBytes = 0;
+  try {
+    for await (const chunk of source) {
+      servedBytes += chunk.data.length;
+      yield chunk;
+    }
+  } finally {
+    chain.metrics?.reqRespByRange.servedBytes.observe({method}, servedBytes);
+  }
+}
+
 /**
  * The ReqRespHandler module handles app-level requests / responses from other peers,
  * fetching state from the chain and database as needed.
@@ -50,7 +73,12 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
     [ReqRespMethod.Metadata]: notImplemented(ReqRespMethod.Metadata),
     [ReqRespMethod.BeaconBlocksByRange]: (req, peerId, peerClient) => {
       const body = deserializeRequestBody(ssz.phase0.BeaconBlocksByRangeRequest, req.data);
-      return onBeaconBlocksByRange(body, chain, db, peerId, peerClient);
+      return trackByRangeResponse(
+        chain,
+        ReqRespMethod.BeaconBlocksByRange,
+        body.count,
+        onBeaconBlocksByRange(body, chain, db, peerId, peerClient)
+      );
     },
     [ReqRespMethod.BeaconBlocksByRoot]: (req) => {
       const fork = chain.config.getForkName(chain.clock.currentSlot);
@@ -68,11 +96,22 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
     },
     [ReqRespMethod.BlobSidecarsByRange]: (req) => {
       const body = deserializeRequestBody(ssz.deneb.BlobSidecarsByRangeRequest, req.data);
-      return onBlobSidecarsByRange(body, chain, db);
+      return trackByRangeResponse(
+        chain,
+        ReqRespMethod.BlobSidecarsByRange,
+        body.count,
+        onBlobSidecarsByRange(body, chain, db)
+      );
     },
     [ReqRespMethod.DataColumnSidecarsByRange]: (req, peerId, peerClient) => {
       const body = deserializeRequestBody(ssz.fulu.DataColumnSidecarsByRangeRequest, req.data);
-      return onDataColumnSidecarsByRange(body, chain, db, peerId, peerClient);
+      return trackByRangeResponse(
+        chain,
+        ReqRespMethod.DataColumnSidecarsByRange,
+        body.count,
+        onDataColumnSidecarsByRange(body, chain, db, peerId, peerClient),
+        body.columns.length
+      );
     },
     [ReqRespMethod.DataColumnSidecarsByRoot]: (req, peerId, peerClient) => {
       const body = deserializeRequestBody(DataColumnSidecarsByRootRequestType(chain.config), req.data);
@@ -85,7 +124,12 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
     },
     [ReqRespMethod.ExecutionPayloadEnvelopesByRange]: (req, peerId, peerClient) => {
       const body = deserializeRequestBody(ssz.gloas.ExecutionPayloadEnvelopesByRangeRequest, req.data);
-      return onExecutionPayloadEnvelopesByRange(body, chain, db, peerId, peerClient);
+      return trackByRangeResponse(
+        chain,
+        ReqRespMethod.ExecutionPayloadEnvelopesByRange,
+        body.count,
+        onExecutionPayloadEnvelopesByRange(body, chain, db, peerId, peerClient)
+      );
     },
 
     [ReqRespMethod.LightClientBootstrap]: (req) => {

package/src/node/nodejs.ts

@@ -8,7 +8,6 @@ import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ZERO_HASH_HEX} from "@lodestar/params";
 import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
-import {phase0} from "@lodestar/types";
 import {sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
 import {BeaconRestApiServer, getApi} from "../api/index.js";
@@ -19,7 +18,6 @@ import {initializeExecutionBuilder, initializeExecutionEngine} from "../executio
 import {HttpMetricsServer, Metrics, createMetrics, getHttpMetricsServer} from "../metrics/index.js";
 import {MonitoringService} from "../monitoring/index.js";
 import {Network, getReqRespHandlers} from "../network/index.js";
-import {BackfillSync} from "../sync/backfill/index.js";
 import {BeaconSync, IBeaconSync} from "../sync/index.js";
 import {Clock} from "../util/clock.js";
 import {startDeferredVoluntaryExitPublisher} from "./deferredVoluntaryExitPublisher.js";
@@ -38,7 +36,6 @@ export type BeaconNodeModules = {
   chain: IBeaconChain;
   api: BeaconApiMethods;
   sync: IBeaconSync;
-  backfillSync: BackfillSync | null;
   metricsServer: HttpMetricsServer | null;
   monitoring: MonitoringService | null;
   restApi?: BeaconRestApiServer;
@@ -58,7 +55,6 @@ export type BeaconNodeInitModules = {
   peerStoreDir?: string;
   anchorState: IBeaconStateView;
   isAnchorStateFinalized: boolean;
-  wsCheckpoint?: phase0.Checkpoint;
   metricsRegistries?: Registry[];
 };
 
@@ -70,7 +66,6 @@ export enum BeaconNodeStatus {
 
 enum LoggerModule {
   api = "api",
-  backfill = "backfill",
   chain = "chain",
   execution = "execution",
   metrics = "metrics",
@@ -105,7 +100,6 @@ export class BeaconNode {
   api: BeaconApiMethods;
   restApi?: BeaconRestApiServer;
   sync: IBeaconSync;
-  backfillSync: BackfillSync | null;
 
   status: BeaconNodeStatus;
   private controller?: AbortController;
@@ -123,7 +117,6 @@ export class BeaconNode {
     api,
     restApi,
     sync,
-    backfillSync,
     controller,
   }: BeaconNodeModules) {
     this.opts = opts;
@@ -138,7 +131,6 @@ export class BeaconNode {
     this.restApi = restApi;
     this.network = network;
     this.sync = sync;
-    this.backfillSync = backfillSync;
     this.controller = controller;
 
     this.status = BeaconNodeStatus.started;
@@ -161,7 +153,6 @@ export class BeaconNode {
     peerStoreDir,
     anchorState,
     isAnchorStateFinalized,
-    wsCheckpoint,
     metricsRegistries = [],
   }: BeaconNodeInitModules): Promise<T> {
     if (hasher.name !== "hashtree") {
@@ -295,25 +286,9 @@ export class BeaconNode {
       chain,
       metrics,
       network,
-      wsCheckpoint,
       logger: logger.child({module: LoggerModule.sync}),
     });
 
-    const backfillSync =
-      opts.sync.backfillBatchSize > 0
-        ? await BackfillSync.init(opts.sync, {
-            config,
-            db,
-            chain,
-            metrics,
-            network,
-            wsCheckpoint,
-            anchorState,
-            logger: logger.child({module: LoggerModule.backfill}),
-            signal,
-          })
-        : null;
-
     const api = getApi(opts.api, {
       config,
       logger: logger.child({module: LoggerModule.api}),
@@ -361,7 +336,6 @@ export class BeaconNode {
       api,
       restApi,
       sync,
-      backfillSync,
       controller,
     }) as T;
   }
@@ -373,7 +347,6 @@ export class BeaconNode {
     if (this.status === BeaconNodeStatus.started) {
       this.status = BeaconNodeStatus.closing;
       this.sync.close();
-      this.backfillSync?.close();
       if (this.restApi) await this.restApi.close();
       await this.network.close();
       if (this.metricsServer) await this.metricsServer.close();

package/src/sync/interface.ts

@@ -1,6 +1,6 @@
 import {routes} from "@lodestar/api";
 import {BeaconConfig} from "@lodestar/config";
-import {Slot, phase0} from "@lodestar/types";
+import {Slot} from "@lodestar/types";
 import {Logger} from "@lodestar/utils";
 import {IBeaconChain} from "../chain/index.js";
 import {IBeaconDb} from "../db/index.js";
@@ -51,5 +51,4 @@ export interface SyncModules {
   metrics: Metrics | null;
   logger: Logger;
   chain: IBeaconChain;
-  wsCheckpoint?: phase0.Checkpoint;
 }

package/src/sync/options.ts

@@ -16,12 +16,6 @@ export type SyncOptions = {
   disableRangeSync?: boolean;
   /** USE FOR TESTING ONLY. Disable unknown block sync completely */
   disableBlockInputSync?: boolean;
-  /**
-   * The batch size of slots for backfill sync can attempt to sync/process before yielding
-   * to sync loop. This number can be increased or decreased to make a suitable resource
-   * allocation to backfill sync. The default of 0 would mean backfill sync will be skipped
-   */
-  backfillBatchSize: number;
   /** For testing only, MAX_PENDING_BLOCKS by default */
   maxPendingBlocks?: number;
 
@@ -39,7 +33,5 @@ export type SyncOptions = {
 export const defaultSyncOptions: SyncOptions = {
   isSingleNode: false,
   disableProcessAsChainSegment: false,
-  /** By default skip the backfill sync */
-  backfillBatchSize: 0,
   slotImportTolerance: SLOTS_PER_EPOCH,
 };

package/src/sync/types.ts

@@ -23,6 +23,14 @@ export enum PendingBlockType {
    * Payload analog of UNKNOWN_BLOCK_ROOT: we have a beacon block root but not its execution payload envelope.
    */
   UNKNOWN_PAYLOAD_BLOCK_ROOT = "unknown_payload_block_root",
+  /**
+   * Same as UNKNOWN_PAYLOAD_BLOCK_ROOT, but the trigger also carries the payload's slot
+   */
+  UNKNOWN_PAYLOAD_BLOCK_ROOT_SLOT = "unknown_payload_block_root_slot",
+  /**
+   * Same as UNKNOWN_PAYLOAD_BLOCK_ROOT_SLOT for the current slot
+   */
+  DEFERRED_PAYLOAD_BLOCK_ROOT = "deferred_payload_block_root",
   /**
    * Payload analog of INCOMPLETE_BLOCK_INPUT: we have a partial payload input that did not complete in time.
    */
@@ -49,6 +57,20 @@ export enum FetchResult {
   FailureMaxAttempts = "failure_max_attempts",
 }
 
+/**
+ * Outcome of a slot's deferred optimistic payload searches, recorded once when its poll loop ends.
+ */
+export enum DeferredPayloadResult {
+  /** gossip delivered every watched payload before PAYLOAD_DUE, no search was needed */
+  ImportedBeforeSearch = "imported_before_search",
+  /** at least one was missing at PAYLOAD_DUE, we searched, and all were imported afterwards */
+  ImportedAfterSearch = "imported_after_search",
+  /** something was still not in fork choice when the slot ended */
+  Unresolved = "unresolved",
+  /** the slot had no optimistic search and no payload import at all, eg a skipped slot */
+  ResolvedNoRoot = "resolved_no_root",
+}
+
 export enum PendingBlockInputStatus {
   pending = "pending",
   fetching = "fetching",
@@ -83,8 +105,6 @@ export enum DroppedItemReason {
   invalidEnvelope = "invalid_envelope",
   /** a payload had an invalid signature (INVALID_SIGNATURE) */
   invalidSignature = "invalid_signature",
-  /** a block root was given up on after exhausting download attempts (data could not be fetched) */
-  unavailable = "unavailable",
   /** an unexpected / unhandled error code led to removal */
   unknown = "unknown",
 }
@@ -130,9 +150,8 @@ export type PendingPayloadInput = {
 export type PendingPayloadRootHex = {
   status: PendingPayloadInputStatus.pending | PendingPayloadInputStatus.fetching;
   rootHex: RootHex;
-  // Trusted slot only (fork choice / validated data), may be missing until resolved. NOT the gossip
-  // message slot from ChainEvent.unknownEnvelopeBlockRoot, which is untrusted and not necessarily the
-  // payload/block slot. See BlockInputSync.resolvePayloadSlot.
+  // slot could be via fork choice or ChainEvent.unknownEnvelopeBlockRootSlot
+  // while ChainEvent.unknownEnvelopeBlockRoot does not provide one
   slot?: Slot;
   timeAddedSec: number;
   timeSyncedSec?: number;

package/src/sync/unknownBlock.ts

@@ -5,7 +5,7 @@ import {ForkSeq, SLOTS_PER_EPOCH, isForkPostGloas} from "@lodestar/params";
 import {RequestError, RequestErrorCode} from "@lodestar/reqresp";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
 import {RootHex, Slot, gloas} from "@lodestar/types";
-import {Logger, fromHex, prettyPrintIndices, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
+import {Logger, fromHex, isErrorAborted, prettyPrintIndices, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../chain/blocks/blockInput/types.js";
 import {PayloadError, PayloadErrorCode} from "../chain/blocks/importExecutionPayload.js";
@@ -18,6 +18,7 @@ import {Metrics} from "../metrics/index.js";
 import {INetwork, NetworkEvent, NetworkEventData, prettyPrintPeerIdStr} from "../network/index.js";
 import {PeerSyncMeta} from "../network/peers/peersData.js";
 import {MAX_PEERS_PER_ROOT} from "../network/processor/constants.js";
+import {ClockEvent} from "../util/clock.js";
 import {PeerIdStr} from "../util/peerId.js";
 import {shuffle} from "../util/shuffle.js";
 import {sortBy} from "../util/sortBy.js";
@@ -26,6 +27,7 @@ import {MAX_CONCURRENT_REQUESTS} from "./constants.js";
 import {SyncOptions} from "./options.js";
 import {
   BlockInputSyncCacheItem,
+  DeferredPayloadResult,
   DownloadResult,
   DroppedItemReason,
   FetchResult,
@@ -57,6 +59,11 @@ const MAX_PENDING_BLOCKS = 100;
  */
 const PRUNE_UNRESOLVED_SLOT_EPOCHS = 2;
 
+/**
+ * Poll pending payloads of the current slot this often, between PAYLOAD_DUE and the end of the slot.
+ */
+export const PAYLOAD_POLL_INTERVAL_MS = 1_000;
+
 type AdvancePendingBlockResult =
   | "ready"
   | "queued_block"
@@ -116,6 +123,11 @@ export class BlockInputSync {
   private subscribedToNetworkEvents = false;
   private peerBalancer: UnknownBlockPeerBalancer;
   private rateLimitBackoffTimeout: NodeJS.Timeout | undefined;
+  /**
+   * Search unknown payload starting at PAYLOAD_DUE_BPS.
+   */
+  private readonly deferredPayloadRoots = new Map<RootHex, {slot: Slot; source: BlockInputSource; peer?: PeerIdStr}>();
+  private payloadPollAbortController: AbortController | undefined;
 
   constructor(
     private readonly config: ChainForkConfig,
@@ -152,6 +164,7 @@ export class BlockInputSync {
       this.logger.verbose("BlockInputSync enabled.");
       this.chain.emitter.on(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
       this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
+      this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRootSlot, this.onUnknownEnvelopeBlockRootSlot);
       this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
       this.chain.emitter.on(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
       this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
@@ -160,6 +173,7 @@ export class BlockInputSync {
       this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadImported);
       this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
       this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
+      this.chain.clock.on(ClockEvent.slot, this.startPayloadPollAtSlot);
 
       // Seed the balancer with peers that connected before we subscribed
       for (const peerId of this.network.getConnectedPeers()) {
@@ -171,14 +185,22 @@ export class BlockInputSync {
       }
 
       this.subscribedToNetworkEvents = true;
+      // we subscribe mid-slot, and a root can be deferred at any point in the slot, so start this
+      // slot's loop now instead of waiting for the next ClockEvent.slot
+      this.startPayloadPollAtSlot(this.chain.clock.currentSlot);
     }
   }
 
   unsubscribeFromNetwork(): void {
     this.logger.verbose("BlockInputSync disabled.");
     this.clearRateLimitBackoffTimer();
+    this.chain.clock.off(ClockEvent.slot, this.startPayloadPollAtSlot);
+    this.payloadPollAbortController?.abort();
+    this.payloadPollAbortController = undefined;
+    this.deferredPayloadRoots.clear();
     this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
     this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
+    this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRootSlot, this.onUnknownEnvelopeBlockRootSlot);
     this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
     this.chain.emitter.off(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
     this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
@@ -205,8 +227,11 @@ export class BlockInputSync {
   private onUnknownBlockRoot = (data: ChainEventData[ChainEvent.unknownBlockRoot]): void => {
     try {
       const isNewRoot = this.addByRootHex(data.rootHex, data.peer);
-      if (isNewRoot) {
+      // A retained entry (an earlier download failed, #10018) is not new but still needs a retry pass.
+      if (this.pendingBlocks.get(data.rootHex)?.status === PendingBlockInputStatus.pending) {
         this.triggerUnknownBlockSearch();
+      }
+      if (isNewRoot) {
         this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_BLOCK_ROOT});
         this.metrics?.blockInputSync.source.inc({source: data.source});
       }
@@ -242,6 +267,33 @@ export class BlockInputSync {
     }
   };
 
+  private onUnknownEnvelopeBlockRootSlot = (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRootSlot]): void => {
+    try {
+      if (data.slot === this.chain.clock.currentSlot) {
+        // we receive the optimistic search, the poll loop searches it at PAYLOAD_DUE or on its next
+        // tick. If that loop already returned - every root it watched was imported - this root waits
+        // for the next slot, where startPayloadPollAtSlot moves it to pendingPayloads. That is not the normal
+        // scenario, and in a forking condition it only delays the search of the extra root
+        const deferred = this.deferredPayloadRoots.get(data.rootHex);
+        if (deferred === undefined) {
+          this.deferredPayloadRoots.set(data.rootHex, {slot: data.slot, source: data.source, peer: data.peer});
+        } else if (deferred.peer === undefined) {
+          deferred.peer = data.peer;
+        }
+        return;
+      }
+
+      const isNewRoot = this.addByPayloadRootHex(data.rootHex, data.peer, data.slot);
+      if (isNewRoot) {
+        this.triggerUnknownBlockSearch();
+        this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_PAYLOAD_BLOCK_ROOT_SLOT});
+        this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
+      }
+    } catch (e) {
+      this.logger.debug("Error handling unknownEnvelopeBlockRootSlot event", {}, e as Error);
+    }
+  };
+
   private onIncompletePayloadEnvelope = (data: ChainEventData[ChainEvent.incompletePayloadEnvelope]): void => {
     try {
       this.addByPayloadInput(data.payloadInput, data.peer);
@@ -344,12 +396,21 @@ export class BlockInputSync {
       }
     }
 
-    let deletedBlocks = 0;
+    let belowFinalizedBlocks = 0;
+    let agedOutBlocks = 0;
     for (const [rootHex, block] of this.pendingBlocks) {
       const slot = getBlockInputSyncCacheItemSlot(block);
-      if (typeof slot === "number" && slot < finalizedSlot) {
+      if (typeof slot !== "number") {
+        // we may not be able to download a root, hence there is no slot for it
+        if (nowSec - block.timeAddedSec > maxSecInCache) {
+          this.pendingBlocks.delete(rootHex);
+          agedOutBlocks++;
+        }
+        continue;
+      }
+      if (slot < finalizedSlot) {
         this.pendingBlocks.delete(rootHex);
-        deletedBlocks++;
+        belowFinalizedBlocks++;
       }
     }
 
@@ -362,16 +423,20 @@ export class BlockInputSync {
     if (agedOutPayloads > 0) {
       this.metrics?.blockInputSync.removedPayloads.inc({reason: DroppedItemReason.agedOut}, agedOutPayloads);
     }
-    if (deletedBlocks > 0) {
-      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.belowFinalized}, deletedBlocks);
+    if (belowFinalizedBlocks > 0) {
+      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.belowFinalized}, belowFinalizedBlocks);
+    }
+    if (agedOutBlocks > 0) {
+      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.agedOut}, agedOutBlocks);
     }
 
-    this.logger.debug("BlockInputSync.pruneFinalized dropped pre-finalized pending items", {
+    this.logger.debug("BlockInputSync.pruneFinalized dropped stale pending items", {
       finalizedSlot,
       finalizedRoot: checkpoint.rootHex,
       belowFinalizedPayloads,
       agedOutPayloads,
-      deletedBlocks,
+      belowFinalizedBlocks,
+      agedOutBlocks,
       unresolvedSlotPayloads,
     });
   };
@@ -785,6 +850,122 @@ export class BlockInputSync {
     }
   }
 
+  /**
+   * Start this slot's payload poll: end the previous slot's loop, move roots it never got to
+   * into pendingPayloads, then poll from PAYLOAD_DUE to the end of the slot.
+   */
+  private startPayloadPollAtSlot = (slot: Slot): void => {
+    if (!isForkPostGloas(this.config.getForkName(slot))) {
+      return;
+    }
+    // end the previous slot's loop, wherever it is: its sleep is bound to this same controller
+    this.payloadPollAbortController?.abort();
+    this.payloadPollAbortController = new AbortController();
+    // entries from an earlier slot is not polled
+    for (const [rootHex, {slot: deferredSlot, source, peer}] of this.deferredPayloadRoots) {
+      if (deferredSlot < slot) {
+        if (!this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+          this.addDeferredPayloadRoot(rootHex, deferredSlot, source, peer);
+        }
+        this.deferredPayloadRoots.delete(rootHex);
+      }
+    }
+    this.pollPayloadsAtSlot(slot, this.payloadPollAbortController.signal).catch((e) => {
+      if (!isErrorAborted(e)) {
+        this.logger.debug("Error polling payloads", {slot}, e as Error);
+      }
+    });
+  };
+
+  /**
+   * Poll unknown payloads of the current slot.
+   */
+  private async pollPayloadsAtSlot(slot: Slot, signal: AbortSignal): Promise<void> {
+    // fixed tick count so the loop cannot outlive its slot: for mainnet, it's 6 ticks
+    const tickCount = Math.floor(
+      (this.config.SLOT_DURATION_MS - this.config.getPayloadDueMs()) / PAYLOAD_POLL_INTERVAL_MS
+    );
+    // a root still missing on several ticks is searched on each of them, recorded once
+    const searchedRoots = new Set<RootHex>();
+    let polls = 0;
+
+    try {
+      await sleep(Math.max(0, this.config.getPayloadDueMs() - this.chain.clock.msFromSlot(slot)), signal);
+
+      for (polls = 0; polls < tickCount; polls++) {
+        let importedCount = 0;
+        for (const [rootHex, {slot: deferredSlot, source, peer}] of this.deferredPayloadRoots) {
+          if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+            importedCount++;
+            continue;
+          }
+          // copy to the regular pendingPayloads so that we can search
+          this.addDeferredPayloadRoot(rootHex, deferredSlot, source, peer);
+          searchedRoots.add(rootHex);
+        }
+
+        // an empty deferredPayloadRoots means there is no optimistic search comes before PAYLOAD_DUE
+        // we have to poll until the payload of current slot is imported or slot end
+        if (this.deferredPayloadRoots.size > 0 && importedCount === this.deferredPayloadRoots.size) {
+          if (searchedRoots.size > 0) {
+            // the backup search was needed
+            this.metrics?.blockInputSync.deferredPayloadResult.inc({
+              result: DeferredPayloadResult.ImportedAfterSearch,
+            });
+            this.metrics?.blockInputSync.deferredPayloadPolls.observe(polls);
+            this.logger.verbose("Payload imported after unknown sync search", {
+              slot,
+              roots: Array.from(searchedRoots).join(","),
+              polls,
+            });
+          } else {
+            // this happens most of the time on devnet
+            // gossip delivered everything before PAYLOAD_DUE
+            this.metrics?.blockInputSync.deferredPayloadResult.inc({
+              result: DeferredPayloadResult.ImportedBeforeSearch,
+            });
+          }
+          return;
+        }
+
+        this.triggerUnknownBlockSearch();
+        if (polls < tickCount - 1) {
+          // no sleep after the last tick: the loop would otherwise end right at the slot boundary,
+          // where startPayloadPollAtSlot aborts it
+          await sleep(PAYLOAD_POLL_INTERVAL_MS, signal);
+        }
+      }
+
+      // reaching here means the early return never fired
+      if (this.deferredPayloadRoots.size > 0) {
+        this.metrics?.blockInputSync.deferredPayloadResult.inc({result: DeferredPayloadResult.Unresolved});
+        this.logger.debug("Deferred payloads still missing at end of slot", {
+          slot,
+          roots: Array.from(this.deferredPayloadRoots.keys()).join(","),
+        });
+      } else {
+        // no optimistic search and no payload import ended the loop, eg a skipped slot
+        this.metrics?.blockInputSync.deferredPayloadResult.inc({result: DeferredPayloadResult.ResolvedNoRoot});
+        this.logger.debug("No deferred payloads at slot", {
+          slot,
+        });
+      }
+    } catch (e) {
+      if (!isErrorAborted(e)) {
+        throw e;
+      }
+      this.logger.verbose("Payload poll aborted", {slot, polls, msIntoSlot: this.chain.clock.msFromSlot(slot)});
+    }
+  }
+
+  private addDeferredPayloadRoot(rootHex: RootHex, slot: Slot, source: BlockInputSource, peer?: PeerIdStr): void {
+    const isNewRoot = this.addByPayloadRootHex(rootHex, peer, slot);
+    if (isNewRoot) {
+      this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.DEFERRED_PAYLOAD_BLOCK_ROOT});
+      this.metrics?.blockInputSync.payloadSource.inc({source});
+    }
+  }
+
   private async downloadBlock(block: BlockInputSyncCacheItem): Promise<void> {
     if (block.status !== PendingBlockInputStatus.pending) {
       return;
@@ -808,7 +989,12 @@ export class BlockInputSync {
     if (!res.err) {
       this.metrics?.blockInputSync.downloadedBlocksSuccess.inc();
       const pending = res.result;
-      this.pendingBlocks.set(pending.blockInput.blockRootHex, pending);
+      // pruning may have deleted this entry while we were awaiting the network, do not resurrect it via the set below
+      if (!this.pendingBlocks.has(rootHex)) {
+        this.logger.verbose("Dropping downloaded block, entry pruned during fetch", logCtx);
+        return;
+      }
+      this.pendingBlocks.set(rootHex, pending);
       const blockSlot = pending.blockInput.slot;
       const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
       const delaySec = Date.now() / 1000 - computeTimeAtSlot(this.config, blockSlot, this.chain.genesisTime);
@@ -870,8 +1056,11 @@ export class BlockInputSync {
       }
 
       this.metrics?.blockInputSync.downloadedBlocksError.inc();
-      this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
-      this.removeAndDownScoreAllDescendants(block, DroppedItemReason.unavailable);
+      this.logger.debug("Failed to download unknown block root, retained for retry", logCtx, res.err);
+      const pendingBlock = this.pendingBlocks.get(rootHex);
+      if (pendingBlock && pendingBlock.status === PendingBlockInputStatus.fetching) {
+        pendingBlock.status = PendingBlockInputStatus.pending;
+      }
     }
   }
 
@@ -1138,7 +1327,7 @@ export class BlockInputSync {
     }
 
     this.metrics?.blockInputSync.downloadedPayloadsResult.inc({result: DownloadResult.Failed});
-    this.logger.debug("Ignoring unknown payload root after failed download", logCtx, res.err);
+    this.logger.debug("Failed to download unknown payload root, retained for retry", logCtx, res.err);
     if (!isPendingPayloadEnvelope(payload)) {
       payload.status = PendingPayloadInputStatus.pending;
     }
@@ -1598,7 +1787,11 @@ export class BlockInputSync {
         this.peerBalancer.onRequestCompleted(peerId);
       }
 
-      this.pendingBlocks.set(getBlockInputSyncCacheItemRootHex(cacheItem), cacheItem);
+      const cacheItemRootHex = getBlockInputSyncCacheItemRootHex(cacheItem);
+      // pruning may have dropped it mid-fetch
+      if (this.pendingBlocks.has(cacheItemRootHex)) {
+        this.pendingBlocks.set(cacheItemRootHex, cacheItem);
+      }
 
       if (cacheItem.status === PendingBlockInputStatus.downloaded) {
         // download was successful, no need to go with another peer, return

package/src/util/shuffle.ts

@@ -10,12 +10,3 @@ export function shuffle<T>(arr: T[]): T[] {
   }
   return _arr;
 }
-
-/**
- * Return one random item from array
- */
-export function shuffleOne<T>(arr: T[]): T | undefined {
-  if (arr.length === 0) return undefined;
-
-  return arr[Math.floor(Math.random() * arr.length)];
-}

package/lib/chain/archiveStore/utils/updateBackfillRange.d.ts

@@ -1,20 +0,0 @@
-import { CheckpointWithHex } from "@lodestar/fork-choice";
-import { Logger } from "@lodestar/logger";
-import { IBeaconDb } from "../../../db/interface.js";
-import { IBeaconChain } from "../../interface.js";
-/**
- * Backfill sync relies on verified connected ranges (which are represented as key,value
- * with a verified jump from a key back to value). Since the node could have progressed
- * ahead from, we need to save the forward progress of this node as another backfill
- * range entry, that backfill sync will use to jump back if this node is restarted
- * for any reason.
- * The current backfill has its own backfill entry from anchor slot to last backfilled
- * slot. And this would create the entry from the current finalized slot to the anchor
- * slot.
- */
-export declare function updateBackfillRange({ chain, db, logger }: {
-    chain: IBeaconChain;
-    db: IBeaconDb;
-    logger: Logger;
-}, finalized: CheckpointWithHex): Promise<void>;
-//# sourceMappingURL=updateBackfillRange.d.ts.map