@lodestar/beacon-node 1.49.0-dev.4d35c8b82e → 1.49.0-dev.57ffc39074

package/lib/sync/unknownBlock.d.ts

@@ -6,6 +6,10 @@ import { INetwork } from "../network/index.js";
 import { PeerSyncMeta } from "../network/peers/peersData.js";
 import { PeerIdStr } from "../util/peerId.js";
 import { SyncOptions } from "./options.js";
+/**
+ * Poll pending payloads of the current slot this often, between PAYLOAD_DUE and the end of the slot.
+ */
+export declare const PAYLOAD_POLL_INTERVAL_MS = 1000;
 /**
  * BlockInputSync is a class that handles ReqResp to find blocks and data related to a specific blockRoot.  The
  * blockRoot may have been found via object gossip, or the API.  Gossip objects that can trigger a search are block,
@@ -55,6 +59,11 @@ export declare class BlockInputSync {
     private subscribedToNetworkEvents;
     private peerBalancer;
     private rateLimitBackoffTimeout;
+    /**
+     * Search unknown payload starting at PAYLOAD_DUE_BPS.
+     */
+    private readonly deferredPayloadRoots;
+    private payloadPollAbortController;
     constructor(config: ChainForkConfig, network: INetwork, chain: IBeaconChain, logger: Logger, metrics: Metrics | null, opts?: SyncOptions | undefined);
     subscribeToNetwork(): void;
     unsubscribeFromNetwork(): void;
@@ -69,6 +78,7 @@ export declare class BlockInputSync {
      */
     private onIncompleteBlockInput;
     private onUnknownEnvelopeBlockRoot;
+    private onUnknownEnvelopeBlockRootSlot;
     private onIncompletePayloadEnvelope;
     /**
      * Process an unknownBlockParent event and register the block in `pendingBlocks` Map.
@@ -104,6 +114,12 @@ export declare class BlockInputSync {
     private triggerUnknownBlockSearch;
     private scheduleRateLimitBackoffRetry;
     private clearRateLimitBackoffTimer;
+    private onClockSlot;
+    /**
+     * Poll unknown payloads of the current slot.
+     */
+    private pollPayloadsAtSlot;
+    private addDeferredPayloadRoot;
     private downloadBlock;
     /**
      * Import a block that has already passed the local dependency checks in BlockInputSync.

package/lib/sync/unknownBlock.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"unknownBlock.d.ts","sourceRoot":"","sources":["../../src/sync/unknownBlock.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,MAAM,EAA+D,MAAM,iBAAiB,CAAC;AAMrG,OAAO,EAA6B,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAG3E,OAAO,EAAC,OAAO,EAAC,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAC,QAAQ,EAAuD,MAAM,qBAAqB,CAAC;AACnG,OAAO,EAAC,YAAY,EAAC,MAAM,+BAA+B,CAAC;AAE3D,OAAO,EAAC,SAAS,EAAC,MAAM,mBAAmB,CAAC;AAK5C,OAAO,EAAC,WAAW,EAAC,MAAM,cAAc,CAAC;AAuDzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,cAAc;IAcvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;IAlBxB;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+C;IAE7E,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA4C;IAC5E,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAClC,OAAO,CAAC,yBAAyB,CAAS;IAC1C,OAAO,CAAC,YAAY,CAA2B;IAC/C,OAAO,CAAC,uBAAuB,CAA6B;IAE5D,YACmB,MAAM,EAAE,eAAe,EACvB,OAAO,EAAE,QAAQ,EACjB,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,IAAI,CAAC,EAAE,WAAW,YAAA,EAgBpC;IAED,kBAAkB,IAAI,IAAI,CA+BzB;IAED,sBAAsB,IAAI,IAAI,CAe7B;IAED,KAAK,IAAI,IAAI,CAEZ;IAED,qBAAqB,IAAI,OAAO,CAE/B;IAED;;OAEG;IACH,OAAO,CAAC,kBAAkB,CAWxB;IAEF;;OAEG;IACH,OAAO,CAAC,sBAAsB,CAS5B;IAEF,OAAO,CAAC,0BAA0B,CAWhC;IAEF,OAAO,CAAC,2BAA2B,CASjC;IAEF;;OAEG;IACH,OAAO,CAAC,eAAe,CA+BrB;IAEF,OAAO,CAAC,eAAe,CAIrB;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF;;;OAGG;IACH,OAAO,CAAC,cAAc,CAoEpB;IAEF,OAAO,CAAC,YAAY,CAiClB;IAEF,OAAO,CAAC,eAAe,CAkCrB;IAEF,OAAO,CAAC,mBAAmB,CAuCzB;IAEF;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAoB1B,OAAO,CAAC,iBAAiB,CA4BvB;IAEF,OAAO,CAAC,eAAe,CASrB;IAEF,OAAO,CAAC,kBAAkB,CAIxB;IAEF;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAmCjC,OAAO,CAAC,mBAAmB;IAuC3B,OAAO,CAAC,qBAAqB;IA0B7B;;OAEG;IACH,OAAO,CAAC,yBAAyB,CAwF/B;IAEF,OAAO,CAAC,6BAA6B;IAuBrC,OAAO,CAAC,0BAA0B;YAOpB,aAAa;IA0F3B;;;OAGG;YACW,iBAAiB;IA4H/B;;;;OAIG;YACW,wBAAwB;YAqExB,eAAe;YAyDf,cAAc;IA0F5B;;;;OAIG;YACW,iBAAiB;YAmJjB,6BAA6B;YAmB7B,mBAAmB;IAwCjC;;;;;;;;OAQG;YACW,eAAe;IAqK7B;;;;;OAKG;IACH,OAAO,CAAC,gCAAgC;IA6BxC,OAAO,CAAC,kCAAkC;IAyB1C,OAAO,CAAC,oBAAoB;IA+B5B,OAAO,CAAC,sBAAsB;CAY/B;AAED;;GAEG;AACH,qBAAa,wBAAwB;IACnC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IACjD,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAChD,QAAQ,CAAC,sBAAsB,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAExD,cAIC;IAED,qCAAqC;IACrC,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,GAAG,IAAI,CAM/D;IAED,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAI1C;IAED,4FAA4F;IAC5F,KAAK,IAAI,IAAI,CAIZ;IAED,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAEjE;IAED,uBAAuB,CAAC,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,GAAG,MAAM,GAAG,IAAI,CA4BnG;IAED;;;;;;OAMG;IACH,yBAAyB,CACvB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,aAAa,EAAE,GAAG,CAAC,SAAS,CAAC,EAC7B,cAAc,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,GAC9B,YAAY,GAAG,IAAI,CAiBrB;IAED;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAEjC;IAED;;OAEG;IACH,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAE1C;IAED,sBAAsB,IAAI,MAAM,CAM/B;IAED,OAAO,CAAC,WAAW;IA8DnB,OAAO,CAAC,qBAAqB;CAO9B"}
+{"version":3,"file":"unknownBlock.d.ts","sourceRoot":"","sources":["../../src/sync/unknownBlock.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,MAAM,EAA+E,MAAM,iBAAiB,CAAC;AAMrH,OAAO,EAA6B,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAG3E,OAAO,EAAC,OAAO,EAAC,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAC,QAAQ,EAAuD,MAAM,qBAAqB,CAAC;AACnG,OAAO,EAAC,YAAY,EAAC,MAAM,+BAA+B,CAAC;AAG3D,OAAO,EAAC,SAAS,EAAC,MAAM,mBAAmB,CAAC;AAK5C,OAAO,EAAC,WAAW,EAAC,MAAM,cAAc,CAAC;AAmCzC;;GAEG;AACH,eAAO,MAAM,wBAAwB,OAAQ,CAAC;AAiB9C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,cAAc;IAmBvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;IAvBxB;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+C;IAE7E,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA4C;IAC5E,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAsB;IACrD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAClC,OAAO,CAAC,yBAAyB,CAAS;IAC1C,OAAO,CAAC,YAAY,CAA2B;IAC/C,OAAO,CAAC,uBAAuB,CAA6B;IAC5D;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAgF;IACrH,OAAO,CAAC,0BAA0B,CAA8B;IAEhE,YACmB,MAAM,EAAE,eAAe,EACvB,OAAO,EAAE,QAAQ,EACjB,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,IAAI,CAAC,EAAE,WAAW,YAAA,EAgBpC;IAED,kBAAkB,IAAI,IAAI,CAoCzB;IAED,sBAAsB,IAAI,IAAI,CAoB7B;IAED,KAAK,IAAI,IAAI,CAEZ;IAED,qBAAqB,IAAI,OAAO,CAE/B;IAED;;OAEG;IACH,OAAO,CAAC,kBAAkB,CAWxB;IAEF;;OAEG;IACH,OAAO,CAAC,sBAAsB,CAS5B;IAEF,OAAO,CAAC,0BAA0B,CAWhC;IAEF,OAAO,CAAC,8BAA8B,CAyBpC;IAEF,OAAO,CAAC,2BAA2B,CASjC;IAEF;;OAEG;IACH,OAAO,CAAC,eAAe,CA+BrB;IAEF,OAAO,CAAC,eAAe,CAIrB;IAEF,OAAO,CAAC,iBAAiB,CAKvB;IAEF;;;OAGG;IACH,OAAO,CAAC,cAAc,CAoEpB;IAEF,OAAO,CAAC,YAAY,CAiClB;IAEF,OAAO,CAAC,eAAe,CAkCrB;IAEF,OAAO,CAAC,mBAAmB,CAuCzB;IAEF;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAoB1B,OAAO,CAAC,iBAAiB,CA4BvB;IAEF,OAAO,CAAC,eAAe,CASrB;IAEF,OAAO,CAAC,kBAAkB,CAIxB;IAEF;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAmCjC,OAAO,CAAC,mBAAmB;IAuC3B,OAAO,CAAC,qBAAqB;IA0B7B;;OAEG;IACH,OAAO,CAAC,yBAAyB,CAwF/B;IAEF,OAAO,CAAC,6BAA6B;IAuBrC,OAAO,CAAC,0BAA0B;IAOlC,OAAO,CAAC,WAAW,CAqBjB;IAEF;;OAEG;YACW,kBAAkB;IA8EhC,OAAO,CAAC,sBAAsB;YAQhB,aAAa;IA0F3B;;;OAGG;YACW,iBAAiB;IA4H/B;;;;OAIG;YACW,wBAAwB;YAqExB,eAAe;YAmEf,cAAc;IA0F5B;;;;OAIG;YACW,iBAAiB;YAkLjB,6BAA6B;YAmB7B,mBAAmB;IAwCjC;;;;;;;;OAQG;YACW,eAAe;IAqK7B;;;;;OAKG;IACH,OAAO,CAAC,gCAAgC;IA6BxC,OAAO,CAAC,kCAAkC;IAyB1C,OAAO,CAAC,oBAAoB;IA+B5B,OAAO,CAAC,sBAAsB;CAY/B;AAED;;GAEG;AACH,qBAAa,wBAAwB;IACnC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IACjD,QAAQ,CAAC,cAAc,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAChD,QAAQ,CAAC,sBAAsB,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAExD,cAIC;IAED,qCAAqC;IACrC,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,GAAG,IAAI,CAM/D;IAED,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAI1C;IAED,4FAA4F;IAC5F,KAAK,IAAI,IAAI,CAIZ;IAED,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAEjE;IAED,uBAAuB,CAAC,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,GAAG,MAAM,GAAG,IAAI,CA4BnG;IAED;;;;;;OAMG;IACH,yBAAyB,CACvB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,aAAa,EAAE,GAAG,CAAC,SAAS,CAAC,EAC7B,cAAc,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,GAC9B,YAAY,GAAG,IAAI,CAiBrB;IAED;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAEjC;IAED;;OAEG;IACH,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAE1C;IAED,sBAAsB,IAAI,MAAM,CAM/B;IAED,OAAO,CAAC,WAAW;IA8DnB,OAAO,CAAC,qBAAqB;CAO9B"}

package/lib/sync/unknownBlock.js

@@ -2,7 +2,7 @@ import { routes } from "@lodestar/api";
 import { ForkSeq, SLOTS_PER_EPOCH, isForkPostGloas } from "@lodestar/params";
 import { RequestError, RequestErrorCode } from "@lodestar/reqresp";
 import { computeTimeAtSlot } from "@lodestar/state-transition";
-import { fromHex, prettyPrintIndices, pruneSetToMax, sleep, toRootHex } from "@lodestar/utils";
+import { fromHex, isErrorAborted, prettyPrintIndices, pruneSetToMax, sleep, toRootHex } from "@lodestar/utils";
 import { isBlockInputBlobs, isBlockInputColumns } from "../chain/blocks/blockInput/blockInput.js";
 import { BlockInputSource } from "../chain/blocks/blockInput/types.js";
 import { PayloadError, PayloadErrorCode } from "../chain/blocks/importExecutionPayload.js";
@@ -13,11 +13,12 @@ import { validateGloasBlockDataColumnSidecars } from "../chain/validation/dataCo
 import { validateGossipExecutionPayloadEnvelope } from "../chain/validation/executionPayloadEnvelope.js";
 import { NetworkEvent, prettyPrintPeerIdStr } from "../network/index.js";
 import { MAX_PEERS_PER_ROOT } from "../network/processor/constants.js";
+import { ClockEvent } from "../util/clock.js";
 import { shuffle } from "../util/shuffle.js";
 import { sortBy } from "../util/sortBy.js";
 import { wrapError } from "../util/wrapError.js";
 import { MAX_CONCURRENT_REQUESTS } from "./constants.js";
-import { DroppedItemReason, PendingBlockInputStatus, PendingBlockType, PendingPayloadInputStatus, getBlockInputSyncCacheItemRootHex, getBlockInputSyncCacheItemSlot, getPayloadSyncCacheItemRootHex, getPayloadSyncCacheItemSlot, isPendingBlockInput, isPendingPayloadEnvelope, isPendingPayloadInput, } from "./types.js";
+import { DeferredPayloadResult, DownloadResult, DroppedItemReason, FetchResult, PendingBlockInputStatus, PendingBlockType, PendingPayloadInputStatus, getBlockInputSyncCacheItemRootHex, getBlockInputSyncCacheItemSlot, getPayloadSyncCacheItemRootHex, getPayloadSyncCacheItemSlot, isPendingBlockInput, isPendingPayloadEnvelope, isPendingPayloadInput, } from "./types.js";
 import { DownloadByRootError, downloadByRoot } from "./utils/downloadByRoot.js";
 import { getAllDescendantBlocks, getUnknownAndAncestorBlocks } from "./utils/pendingBlocksTree.js";
 import { getRateLimitedUntilMs } from "./utils/rateLimit.js";
@@ -28,14 +29,10 @@ const MAX_PENDING_BLOCKS = 100;
  * A pending payload without slot can only stay at most 2 epochs.
  */
 const PRUNE_UNRESOLVED_SLOT_EPOCHS = 2;
-var FetchResult;
-(function (FetchResult) {
-    FetchResult["SuccessResolved"] = "success_resolved";
-    FetchResult["SuccessMissingParent"] = "success_missing_parent";
-    FetchResult["SuccessLate"] = "success_late";
-    FetchResult["FailureTriedAllPeers"] = "failure_tried_all_peers";
-    FetchResult["FailureMaxAttempts"] = "failure_max_attempts";
-})(FetchResult || (FetchResult = {}));
+/**
+ * Poll pending payloads of the current slot this often, between PAYLOAD_DUE and the end of the slot.
+ */
+export const PAYLOAD_POLL_INTERVAL_MS = 1_000;
 class UnknownBlockRateLimitedError extends Error {
     constructor(message) {
         super(message);
@@ -92,6 +89,11 @@ export class BlockInputSync {
     subscribedToNetworkEvents = false;
     peerBalancer;
     rateLimitBackoffTimeout;
+    /**
+     * Search unknown payload starting at PAYLOAD_DUE_BPS.
+     */
+    deferredPayloadRoots = new Map();
+    payloadPollAbortController;
     constructor(config, network, chain, logger, metrics, opts) {
         this.config = config;
         this.network = network;
@@ -117,6 +119,7 @@ export class BlockInputSync {
             this.logger.verbose("BlockInputSync enabled.");
             this.chain.emitter.on(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
             this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
+            this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRootSlot, this.onUnknownEnvelopeBlockRootSlot);
             this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
             this.chain.emitter.on(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
             this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
@@ -125,6 +128,7 @@ export class BlockInputSync {
             this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadImported);
             this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
             this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
+            this.chain.clock.on(ClockEvent.slot, this.onClockSlot);
             // Seed the balancer with peers that connected before we subscribed
             for (const peerId of this.network.getConnectedPeers()) {
                 try {
@@ -135,13 +139,21 @@ export class BlockInputSync {
                 }
             }
             this.subscribedToNetworkEvents = true;
+            // we subscribe mid-slot, and a root can be deferred at any point in the slot, so start this
+            // slot's loop now instead of waiting for the next ClockEvent.slot
+            this.onClockSlot(this.chain.clock.currentSlot);
         }
     }
     unsubscribeFromNetwork() {
         this.logger.verbose("BlockInputSync disabled.");
         this.clearRateLimitBackoffTimer();
+        this.chain.clock.off(ClockEvent.slot, this.onClockSlot);
+        this.payloadPollAbortController?.abort();
+        this.payloadPollAbortController = undefined;
+        this.deferredPayloadRoots.clear();
         this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
         this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
+        this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRootSlot, this.onUnknownEnvelopeBlockRootSlot);
         this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
         this.chain.emitter.off(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
         this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
@@ -202,6 +214,33 @@ export class BlockInputSync {
             this.logger.debug("Error handling unknownEnvelopeBlockRoot event", {}, e);
         }
     };
+    onUnknownEnvelopeBlockRootSlot = (data) => {
+        try {
+            if (data.slot === this.chain.clock.currentSlot) {
+                // we receive the optimistic search, the poll loop searches it at PAYLOAD_DUE or on its next
+                // tick. If that loop already returned - every root it watched was imported - this root waits
+                // for the next slot, where onClockSlot moves it to pendingPayloads. That is not the normal
+                // scenario, and in a forking condition it only delays the search of the extra root
+                const deferred = this.deferredPayloadRoots.get(data.rootHex);
+                if (deferred === undefined) {
+                    this.deferredPayloadRoots.set(data.rootHex, { slot: data.slot, source: data.source, peer: data.peer });
+                }
+                else if (deferred.peer === undefined) {
+                    deferred.peer = data.peer;
+                }
+                return;
+            }
+            const isNewRoot = this.addByPayloadRootHex(data.rootHex, data.peer, data.slot);
+            if (isNewRoot) {
+                this.triggerUnknownBlockSearch();
+                this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_PAYLOAD_BLOCK_ROOT_SLOT });
+                this.metrics?.blockInputSync.payloadSource.inc({ source: data.source });
+            }
+        }
+        catch (e) {
+            this.logger.debug("Error handling unknownEnvelopeBlockRootSlot event", {}, e);
+        }
+    };
     onIncompletePayloadEnvelope = (data) => {
         try {
             this.addByPayloadInput(data.payloadInput, data.peer);
@@ -660,6 +699,111 @@ export class BlockInputSync {
             this.rateLimitBackoffTimeout = undefined;
         }
     }
+    onClockSlot = (slot) => {
+        if (!isForkPostGloas(this.config.getForkName(slot))) {
+            return;
+        }
+        // end the previous slot's loop, wherever it is: its sleep is bound to this same controller
+        this.payloadPollAbortController?.abort();
+        this.payloadPollAbortController = new AbortController();
+        // entries from an earlier slot is not polled
+        for (const [rootHex, { slot: deferredSlot, source, peer }] of this.deferredPayloadRoots) {
+            if (deferredSlot < slot) {
+                if (!this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+                    this.addDeferredPayloadRoot(rootHex, deferredSlot, source, peer);
+                }
+                this.deferredPayloadRoots.delete(rootHex);
+            }
+        }
+        this.pollPayloadsAtSlot(slot, this.payloadPollAbortController.signal).catch((e) => {
+            if (!isErrorAborted(e)) {
+                this.logger.debug("Error polling payloads", { slot }, e);
+            }
+        });
+    };
+    /**
+     * Poll unknown payloads of the current slot.
+     */
+    async pollPayloadsAtSlot(slot, signal) {
+        // fixed tick count so the loop cannot outlive its slot: for mainnet, it's 6 ticks
+        const tickCount = Math.floor((this.config.SLOT_DURATION_MS - this.config.getPayloadDueMs()) / PAYLOAD_POLL_INTERVAL_MS);
+        // a root still missing on several ticks is searched on each of them, recorded once
+        const searchedRoots = new Set();
+        let polls = 0;
+        try {
+            await sleep(Math.max(0, this.config.getPayloadDueMs() - this.chain.clock.msFromSlot(slot)), signal);
+            for (polls = 0; polls < tickCount; polls++) {
+                let importedCount = 0;
+                for (const [rootHex, { slot: deferredSlot, source, peer }] of this.deferredPayloadRoots) {
+                    if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+                        importedCount++;
+                        continue;
+                    }
+                    // copy to the regular pendingPayloads so that we can search
+                    this.addDeferredPayloadRoot(rootHex, deferredSlot, source, peer);
+                    searchedRoots.add(rootHex);
+                }
+                // an empty deferredPayloadRoots means there is no optimistic search comes before PAYLOAD_DUE
+                // we have to poll until the payload of current slot is imported or slot end
+                if (this.deferredPayloadRoots.size > 0 && importedCount === this.deferredPayloadRoots.size) {
+                    if (searchedRoots.size > 0) {
+                        // the backup search was needed
+                        this.metrics?.blockInputSync.deferredPayloadResult.inc({
+                            result: DeferredPayloadResult.ImportedAfterSearch,
+                        });
+                        this.metrics?.blockInputSync.deferredPayloadPolls.observe(polls);
+                        this.logger.verbose("Payload imported after unknown sync search", {
+                            slot,
+                            roots: Array.from(searchedRoots).join(","),
+                            polls,
+                        });
+                    }
+                    else {
+                        // this happens most of the time on devnet
+                        // gossip delivered everything before PAYLOAD_DUE
+                        this.metrics?.blockInputSync.deferredPayloadResult.inc({
+                            result: DeferredPayloadResult.ImportedBeforeSearch,
+                        });
+                    }
+                    return;
+                }
+                this.triggerUnknownBlockSearch();
+                if (polls < tickCount - 1) {
+                    // no sleep after the last tick: the loop would otherwise end right at the slot boundary,
+                    // where onClockSlot aborts it
+                    await sleep(PAYLOAD_POLL_INTERVAL_MS, signal);
+                }
+            }
+            // reaching here means the early return never fired
+            if (this.deferredPayloadRoots.size > 0) {
+                this.metrics?.blockInputSync.deferredPayloadResult.inc({ result: DeferredPayloadResult.Unresolved });
+                this.logger.debug("Deferred payloads still missing at end of slot", {
+                    slot,
+                    roots: Array.from(this.deferredPayloadRoots.keys()).join(","),
+                });
+            }
+            else {
+                // no optimistic search and no payload import ended the loop, eg a skipped slot
+                this.metrics?.blockInputSync.deferredPayloadResult.inc({ result: DeferredPayloadResult.ResolvedNoRoot });
+                this.logger.debug("No deferred payloads at slot", {
+                    slot,
+                });
+            }
+        }
+        catch (e) {
+            if (!isErrorAborted(e)) {
+                throw e;
+            }
+            this.logger.verbose("Payload poll aborted", { slot, polls, msIntoSlot: this.chain.clock.msFromSlot(slot) });
+        }
+    }
+    addDeferredPayloadRoot(rootHex, slot, source, peer) {
+        const isNewRoot = this.addByPayloadRootHex(rootHex, peer, slot);
+        if (isNewRoot) {
+            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.DEFERRED_PAYLOAD_BLOCK_ROOT });
+            this.metrics?.blockInputSync.payloadSource.inc({ source });
+        }
+    }
     async downloadBlock(block) {
         if (block.status !== PendingBlockInputStatus.pending) {
             return;
@@ -946,14 +1090,22 @@ export class BlockInputSync {
                 return;
             }
             this.pendingPayloads.set(resultRootHex, pendingPayload);
+            const result = isPendingPayloadEnvelope(pendingPayload)
+                ? DownloadResult.WaitingForBlock
+                : this.chain.forkChoice.hasPayloadHexUnsafe(resultRootHex)
+                    ? DownloadResult.Late
+                    : DownloadResult.Resolved;
+            this.metrics?.blockInputSync.downloadedPayloadsResult.inc({ result });
             if (isPendingPayloadEnvelope(pendingPayload)) {
                 await this.reconcilePayloadEnvelope(pendingPayload);
             }
             else if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
+                this.metrics?.blockInputSync.elapsedTimeTillPayloadReceived.observe(Date.now() / 1000 - computeTimeAtSlot(this.config, pendingPayload.payloadInput.slot, this.chain.genesisTime));
                 await this.processPayload(pendingPayload);
             }
             return;
         }
+        this.metrics?.blockInputSync.downloadedPayloadsResult.inc({ result: DownloadResult.Failed });
         this.logger.debug("Ignoring unknown payload root after failed download", logCtx, res.err);
         if (!isPendingPayloadEnvelope(payload)) {
             payload.status = PendingPayloadInputStatus.pending;
@@ -1050,6 +1202,10 @@ export class BlockInputSync {
             ? cacheItem.payloadInput
             : await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
         let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
+        const fetchStartSec = Date.now() / 1000;
+        if (typeof slot === "number") {
+            this.metrics?.blockInputSync.payloadFetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
+        }
         let i = 0;
         let deferredByRateLimit = false;
         while (i++ < this.getMaxDownloadAttempts()) {
@@ -1065,6 +1221,8 @@ export class BlockInputSync {
                 const reason = pendingColumns.size > 0
                     ? `cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
                     : "no peer available to download pending payload";
+                this.metrics?.blockInputSync.payloadFetchTime.observe({ result: FetchResult.FailureTriedAllPeers }, Date.now() / 1000 - fetchStartSec);
+                this.metrics?.blockInputSync.payloadFetchPeers.set({ result: FetchResult.FailureTriedAllPeers }, i);
                 throw Error(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: ${reason}`);
             }
             const { peerId, client: peerClient } = peerMeta;
@@ -1072,7 +1230,12 @@ export class BlockInputSync {
             try {
                 if (!envelope) {
                     envelope = await this.fetchExecutionPayloadEnvelope(peerId, blockRoot, rootHex);
+                    const slotWasUnknown = typeof slot !== "number";
                     slot = envelope.message.payload.slotNumber;
+                    if (slotWasUnknown) {
+                        // we were not able to observe the time into slot when starting the fetch, do it now
+                        this.metrics?.blockInputSync.payloadFetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
+                    }
                 }
                 payloadInput ??= await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
                 if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
@@ -1080,6 +1243,8 @@ export class BlockInputSync {
                     // envelope and wait for the block body; reconcilePayloadEnvelope validates once the block lands.
                     // payloadInput may be seeded from the block body during download, so a non-null payloadInput does not
                     // imply the block is imported.
+                    this.metrics?.blockInputSync.payloadFetchTime.observe({ result: FetchResult.SuccessWaitingForBlock }, Date.now() / 1000 - fetchStartSec);
+                    this.metrics?.blockInputSync.payloadFetchPeers.set({ result: FetchResult.SuccessWaitingForBlock }, i);
                     return {
                         status: PendingPayloadInputStatus.waitingForBlock,
                         envelope,
@@ -1124,6 +1289,12 @@ export class BlockInputSync {
                     hasAllData: pendingPayload.payloadInput.hasAllData(),
                 });
                 if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
+                    // late = the payload got imported (via gossip) while we were downloading, same as the block path
+                    const result = this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)
+                        ? FetchResult.SuccessLate
+                        : FetchResult.SuccessResolved;
+                    this.metrics?.blockInputSync.payloadFetchTime.observe({ result }, Date.now() / 1000 - fetchStartSec);
+                    this.metrics?.blockInputSync.payloadFetchPeers.set({ result }, i);
                     return pendingPayload;
                 }
                 cacheItem = pendingPayload;
@@ -1155,6 +1326,8 @@ export class BlockInputSync {
                 this.peerBalancer.onRequestCompleted(peerId);
             }
         }
+        this.metrics?.blockInputSync.payloadFetchTime.observe({ result: FetchResult.FailureMaxAttempts }, Date.now() / 1000 - fetchStartSec);
+        this.metrics?.blockInputSync.payloadFetchPeers.set({ result: FetchResult.FailureMaxAttempts }, i - 1);
         if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
             throw new UnknownBlockRateLimitedError(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`);
         }
@@ -1227,7 +1400,7 @@ export class BlockInputSync {
                 }
                 // no more peer with needed columns to try, throw error
                 const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
-                this.metrics?.blockInputSync.fetchTimeSec.observe({ result: FetchResult.FailureTriedAllPeers }, Date.now() / 1000 - fetchStartSec);
+                this.metrics?.blockInputSync.fetchTime.observe({ result: FetchResult.FailureTriedAllPeers }, Date.now() / 1000 - fetchStartSec);
                 this.metrics?.blockInputSync.fetchPeers.set({ result: FetchResult.FailureTriedAllPeers }, i);
                 throw Error(message);
             }
@@ -1309,7 +1482,7 @@ export class BlockInputSync {
                     : this.chain.forkChoice.hasBlockHex(cacheItem.blockInput.parentRootHex)
                         ? FetchResult.SuccessResolved
                         : FetchResult.SuccessMissingParent;
-                this.metrics?.blockInputSync.fetchTimeSec.observe({ result }, Date.now() / 1000 - fetchStartSec);
+                this.metrics?.blockInputSync.fetchTime.observe({ result }, Date.now() / 1000 - fetchStartSec);
                 this.metrics?.blockInputSync.fetchPeers.set({ result }, i);
                 return cacheItem;
             }
@@ -1336,7 +1509,7 @@ export class BlockInputSync {
                 throw new Error(`${message} Missing column indices=${prettyPrintIndices(missing)}.`);
             }
         }
-        this.metrics?.blockInputSync.fetchTimeSec.observe({ result: FetchResult.FailureMaxAttempts }, Date.now() / 1000 - fetchStartSec);
+        this.metrics?.blockInputSync.fetchTime.observe({ result: FetchResult.FailureMaxAttempts }, Date.now() / 1000 - fetchStartSec);
         this.metrics?.blockInputSync.fetchPeers.set({ result: FetchResult.FailureMaxAttempts }, i - 1);
         throw Error(message);
     }