@lodestar/beacon-node 1.49.0-dev.4d35c8b82e → 1.49.0-dev.535f5dd45f

package/src/sync/unknownBlock.ts

@@ -5,7 +5,7 @@ import {ForkSeq, SLOTS_PER_EPOCH, isForkPostGloas} from "@lodestar/params";
 import {RequestError, RequestErrorCode} from "@lodestar/reqresp";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
 import {RootHex, Slot, gloas} from "@lodestar/types";
-import {Logger, fromHex, prettyPrintIndices, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
+import {Logger, fromHex, isErrorAborted, prettyPrintIndices, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../chain/blocks/blockInput/types.js";
 import {PayloadError, PayloadErrorCode} from "../chain/blocks/importExecutionPayload.js";
@@ -18,6 +18,7 @@ import {Metrics} from "../metrics/index.js";
 import {INetwork, NetworkEvent, NetworkEventData, prettyPrintPeerIdStr} from "../network/index.js";
 import {PeerSyncMeta} from "../network/peers/peersData.js";
 import {MAX_PEERS_PER_ROOT} from "../network/processor/constants.js";
+import {ClockEvent} from "../util/clock.js";
 import {PeerIdStr} from "../util/peerId.js";
 import {shuffle} from "../util/shuffle.js";
 import {sortBy} from "../util/sortBy.js";
@@ -26,7 +27,10 @@ import {MAX_CONCURRENT_REQUESTS} from "./constants.js";
 import {SyncOptions} from "./options.js";
 import {
   BlockInputSyncCacheItem,
+  DeferredPayloadResult,
+  DownloadResult,
   DroppedItemReason,
+  FetchResult,
   PayloadSyncCacheItem,
   PendingBlockInput,
   PendingBlockInputStatus,
@@ -55,6 +59,11 @@ const MAX_PENDING_BLOCKS = 100;
  */
 const PRUNE_UNRESOLVED_SLOT_EPOCHS = 2;
 
+/**
+ * Poll pending payloads of the current slot this often, between PAYLOAD_DUE and the end of the slot.
+ */
+export const PAYLOAD_POLL_INTERVAL_MS = 1_000;
+
 type AdvancePendingBlockResult =
   | "ready"
   | "queued_block"
@@ -63,14 +72,6 @@ type AdvancePendingBlockResult =
   | "blocked"
   | "removed";
 
-enum FetchResult {
-  SuccessResolved = "success_resolved",
-  SuccessMissingParent = "success_missing_parent",
-  SuccessLate = "success_late",
-  FailureTriedAllPeers = "failure_tried_all_peers",
-  FailureMaxAttempts = "failure_max_attempts",
-}
-
 class UnknownBlockRateLimitedError extends Error {
   constructor(message: string) {
     super(message);
@@ -122,6 +123,11 @@ export class BlockInputSync {
   private subscribedToNetworkEvents = false;
   private peerBalancer: UnknownBlockPeerBalancer;
   private rateLimitBackoffTimeout: NodeJS.Timeout | undefined;
+  /**
+   * Search unknown payload starting at PAYLOAD_DUE_BPS.
+   */
+  private readonly deferredPayloadRoots = new Map<RootHex, {slot: Slot; source: BlockInputSource; peer?: PeerIdStr}>();
+  private payloadPollAbortController: AbortController | undefined;
 
   constructor(
     private readonly config: ChainForkConfig,
@@ -158,6 +164,7 @@ export class BlockInputSync {
       this.logger.verbose("BlockInputSync enabled.");
       this.chain.emitter.on(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
       this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
+      this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRootSlot, this.onUnknownEnvelopeBlockRootSlot);
       this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
       this.chain.emitter.on(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
       this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
@@ -166,6 +173,7 @@ export class BlockInputSync {
       this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadImported);
       this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
       this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
+      this.chain.clock.on(ClockEvent.slot, this.startPayloadPollAtSlot);
 
       // Seed the balancer with peers that connected before we subscribed
       for (const peerId of this.network.getConnectedPeers()) {
@@ -177,14 +185,22 @@ export class BlockInputSync {
       }
 
       this.subscribedToNetworkEvents = true;
+      // we subscribe mid-slot, and a root can be deferred at any point in the slot, so start this
+      // slot's loop now instead of waiting for the next ClockEvent.slot
+      this.startPayloadPollAtSlot(this.chain.clock.currentSlot);
     }
   }
 
   unsubscribeFromNetwork(): void {
     this.logger.verbose("BlockInputSync disabled.");
     this.clearRateLimitBackoffTimer();
+    this.chain.clock.off(ClockEvent.slot, this.startPayloadPollAtSlot);
+    this.payloadPollAbortController?.abort();
+    this.payloadPollAbortController = undefined;
+    this.deferredPayloadRoots.clear();
     this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
     this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
+    this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRootSlot, this.onUnknownEnvelopeBlockRootSlot);
     this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
     this.chain.emitter.off(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
     this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
@@ -211,8 +227,11 @@ export class BlockInputSync {
   private onUnknownBlockRoot = (data: ChainEventData[ChainEvent.unknownBlockRoot]): void => {
     try {
       const isNewRoot = this.addByRootHex(data.rootHex, data.peer);
-      if (isNewRoot) {
+      // A retained entry (an earlier download failed, #10018) is not new but still needs a retry pass.
+      if (this.pendingBlocks.get(data.rootHex)?.status === PendingBlockInputStatus.pending) {
         this.triggerUnknownBlockSearch();
+      }
+      if (isNewRoot) {
         this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_BLOCK_ROOT});
         this.metrics?.blockInputSync.source.inc({source: data.source});
       }
@@ -248,6 +267,33 @@ export class BlockInputSync {
     }
   };
 
+  private onUnknownEnvelopeBlockRootSlot = (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRootSlot]): void => {
+    try {
+      if (data.slot === this.chain.clock.currentSlot) {
+        // we receive the optimistic search, the poll loop searches it at PAYLOAD_DUE or on its next
+        // tick. If that loop already returned - every root it watched was imported - this root waits
+        // for the next slot, where startPayloadPollAtSlot moves it to pendingPayloads. That is not the normal
+        // scenario, and in a forking condition it only delays the search of the extra root
+        const deferred = this.deferredPayloadRoots.get(data.rootHex);
+        if (deferred === undefined) {
+          this.deferredPayloadRoots.set(data.rootHex, {slot: data.slot, source: data.source, peer: data.peer});
+        } else if (deferred.peer === undefined) {
+          deferred.peer = data.peer;
+        }
+        return;
+      }
+
+      const isNewRoot = this.addByPayloadRootHex(data.rootHex, data.peer, data.slot);
+      if (isNewRoot) {
+        this.triggerUnknownBlockSearch();
+        this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_PAYLOAD_BLOCK_ROOT_SLOT});
+        this.metrics?.blockInputSync.payloadSource.inc({source: data.source});
+      }
+    } catch (e) {
+      this.logger.debug("Error handling unknownEnvelopeBlockRootSlot event", {}, e as Error);
+    }
+  };
+
   private onIncompletePayloadEnvelope = (data: ChainEventData[ChainEvent.incompletePayloadEnvelope]): void => {
     try {
       this.addByPayloadInput(data.payloadInput, data.peer);
@@ -350,12 +396,21 @@ export class BlockInputSync {
       }
     }
 
-    let deletedBlocks = 0;
+    let belowFinalizedBlocks = 0;
+    let agedOutBlocks = 0;
     for (const [rootHex, block] of this.pendingBlocks) {
       const slot = getBlockInputSyncCacheItemSlot(block);
-      if (typeof slot === "number" && slot < finalizedSlot) {
+      if (typeof slot !== "number") {
+        // we may not be able to download a root, hence there is no slot for it
+        if (nowSec - block.timeAddedSec > maxSecInCache) {
+          this.pendingBlocks.delete(rootHex);
+          agedOutBlocks++;
+        }
+        continue;
+      }
+      if (slot < finalizedSlot) {
         this.pendingBlocks.delete(rootHex);
-        deletedBlocks++;
+        belowFinalizedBlocks++;
       }
     }
 
@@ -368,16 +423,20 @@ export class BlockInputSync {
     if (agedOutPayloads > 0) {
       this.metrics?.blockInputSync.removedPayloads.inc({reason: DroppedItemReason.agedOut}, agedOutPayloads);
     }
-    if (deletedBlocks > 0) {
-      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.belowFinalized}, deletedBlocks);
+    if (belowFinalizedBlocks > 0) {
+      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.belowFinalized}, belowFinalizedBlocks);
+    }
+    if (agedOutBlocks > 0) {
+      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.agedOut}, agedOutBlocks);
     }
 
-    this.logger.debug("BlockInputSync.pruneFinalized dropped pre-finalized pending items", {
+    this.logger.debug("BlockInputSync.pruneFinalized dropped stale pending items", {
       finalizedSlot,
       finalizedRoot: checkpoint.rootHex,
       belowFinalizedPayloads,
       agedOutPayloads,
-      deletedBlocks,
+      belowFinalizedBlocks,
+      agedOutBlocks,
       unresolvedSlotPayloads,
     });
   };
@@ -791,6 +850,122 @@ export class BlockInputSync {
     }
   }
 
+  /**
+   * Start this slot's payload poll: end the previous slot's loop, move roots it never got to
+   * into pendingPayloads, then poll from PAYLOAD_DUE to the end of the slot.
+   */
+  private startPayloadPollAtSlot = (slot: Slot): void => {
+    if (!isForkPostGloas(this.config.getForkName(slot))) {
+      return;
+    }
+    // end the previous slot's loop, wherever it is: its sleep is bound to this same controller
+    this.payloadPollAbortController?.abort();
+    this.payloadPollAbortController = new AbortController();
+    // entries from an earlier slot is not polled
+    for (const [rootHex, {slot: deferredSlot, source, peer}] of this.deferredPayloadRoots) {
+      if (deferredSlot < slot) {
+        if (!this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+          this.addDeferredPayloadRoot(rootHex, deferredSlot, source, peer);
+        }
+        this.deferredPayloadRoots.delete(rootHex);
+      }
+    }
+    this.pollPayloadsAtSlot(slot, this.payloadPollAbortController.signal).catch((e) => {
+      if (!isErrorAborted(e)) {
+        this.logger.debug("Error polling payloads", {slot}, e as Error);
+      }
+    });
+  };
+
+  /**
+   * Poll unknown payloads of the current slot.
+   */
+  private async pollPayloadsAtSlot(slot: Slot, signal: AbortSignal): Promise<void> {
+    // fixed tick count so the loop cannot outlive its slot: for mainnet, it's 6 ticks
+    const tickCount = Math.floor(
+      (this.config.SLOT_DURATION_MS - this.config.getPayloadDueMs()) / PAYLOAD_POLL_INTERVAL_MS
+    );
+    // a root still missing on several ticks is searched on each of them, recorded once
+    const searchedRoots = new Set<RootHex>();
+    let polls = 0;
+
+    try {
+      await sleep(Math.max(0, this.config.getPayloadDueMs() - this.chain.clock.msFromSlot(slot)), signal);
+
+      for (polls = 0; polls < tickCount; polls++) {
+        let importedCount = 0;
+        for (const [rootHex, {slot: deferredSlot, source, peer}] of this.deferredPayloadRoots) {
+          if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+            importedCount++;
+            continue;
+          }
+          // copy to the regular pendingPayloads so that we can search
+          this.addDeferredPayloadRoot(rootHex, deferredSlot, source, peer);
+          searchedRoots.add(rootHex);
+        }
+
+        // an empty deferredPayloadRoots means there is no optimistic search comes before PAYLOAD_DUE
+        // we have to poll until the payload of current slot is imported or slot end
+        if (this.deferredPayloadRoots.size > 0 && importedCount === this.deferredPayloadRoots.size) {
+          if (searchedRoots.size > 0) {
+            // the backup search was needed
+            this.metrics?.blockInputSync.deferredPayloadResult.inc({
+              result: DeferredPayloadResult.ImportedAfterSearch,
+            });
+            this.metrics?.blockInputSync.deferredPayloadPolls.observe(polls);
+            this.logger.verbose("Payload imported after unknown sync search", {
+              slot,
+              roots: Array.from(searchedRoots).join(","),
+              polls,
+            });
+          } else {
+            // this happens most of the time on devnet
+            // gossip delivered everything before PAYLOAD_DUE
+            this.metrics?.blockInputSync.deferredPayloadResult.inc({
+              result: DeferredPayloadResult.ImportedBeforeSearch,
+            });
+          }
+          return;
+        }
+
+        this.triggerUnknownBlockSearch();
+        if (polls < tickCount - 1) {
+          // no sleep after the last tick: the loop would otherwise end right at the slot boundary,
+          // where startPayloadPollAtSlot aborts it
+          await sleep(PAYLOAD_POLL_INTERVAL_MS, signal);
+        }
+      }
+
+      // reaching here means the early return never fired
+      if (this.deferredPayloadRoots.size > 0) {
+        this.metrics?.blockInputSync.deferredPayloadResult.inc({result: DeferredPayloadResult.Unresolved});
+        this.logger.debug("Deferred payloads still missing at end of slot", {
+          slot,
+          roots: Array.from(this.deferredPayloadRoots.keys()).join(","),
+        });
+      } else {
+        // no optimistic search and no payload import ended the loop, eg a skipped slot
+        this.metrics?.blockInputSync.deferredPayloadResult.inc({result: DeferredPayloadResult.ResolvedNoRoot});
+        this.logger.debug("No deferred payloads at slot", {
+          slot,
+        });
+      }
+    } catch (e) {
+      if (!isErrorAborted(e)) {
+        throw e;
+      }
+      this.logger.verbose("Payload poll aborted", {slot, polls, msIntoSlot: this.chain.clock.msFromSlot(slot)});
+    }
+  }
+
+  private addDeferredPayloadRoot(rootHex: RootHex, slot: Slot, source: BlockInputSource, peer?: PeerIdStr): void {
+    const isNewRoot = this.addByPayloadRootHex(rootHex, peer, slot);
+    if (isNewRoot) {
+      this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.DEFERRED_PAYLOAD_BLOCK_ROOT});
+      this.metrics?.blockInputSync.payloadSource.inc({source});
+    }
+  }
+
   private async downloadBlock(block: BlockInputSyncCacheItem): Promise<void> {
     if (block.status !== PendingBlockInputStatus.pending) {
       return;
@@ -814,7 +989,12 @@ export class BlockInputSync {
     if (!res.err) {
       this.metrics?.blockInputSync.downloadedBlocksSuccess.inc();
       const pending = res.result;
-      this.pendingBlocks.set(pending.blockInput.blockRootHex, pending);
+      // pruning may have deleted this entry while we were awaiting the network, do not resurrect it via the set below
+      if (!this.pendingBlocks.has(rootHex)) {
+        this.logger.verbose("Dropping downloaded block, entry pruned during fetch", logCtx);
+        return;
+      }
+      this.pendingBlocks.set(rootHex, pending);
       const blockSlot = pending.blockInput.slot;
       const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
       const delaySec = Date.now() / 1000 - computeTimeAtSlot(this.config, blockSlot, this.chain.genesisTime);
@@ -876,8 +1056,11 @@ export class BlockInputSync {
       }
 
       this.metrics?.blockInputSync.downloadedBlocksError.inc();
-      this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
-      this.removeAndDownScoreAllDescendants(block, DroppedItemReason.unavailable);
+      this.logger.debug("Failed to download unknown block root, retained for retry", logCtx, res.err);
+      const pendingBlock = this.pendingBlocks.get(rootHex);
+      if (pendingBlock && pendingBlock.status === PendingBlockInputStatus.fetching) {
+        pendingBlock.status = PendingBlockInputStatus.pending;
+      }
     }
   }
 
@@ -1125,16 +1308,26 @@ export class BlockInputSync {
         return;
       }
       this.pendingPayloads.set(resultRootHex, pendingPayload);
+      const result = isPendingPayloadEnvelope(pendingPayload)
+        ? DownloadResult.WaitingForBlock
+        : this.chain.forkChoice.hasPayloadHexUnsafe(resultRootHex)
+          ? DownloadResult.Late
+          : DownloadResult.Resolved;
+      this.metrics?.blockInputSync.downloadedPayloadsResult.inc({result});
 
       if (isPendingPayloadEnvelope(pendingPayload)) {
         await this.reconcilePayloadEnvelope(pendingPayload);
       } else if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
+        this.metrics?.blockInputSync.elapsedTimeTillPayloadReceived.observe(
+          Date.now() / 1000 - computeTimeAtSlot(this.config, pendingPayload.payloadInput.slot, this.chain.genesisTime)
+        );
         await this.processPayload(pendingPayload);
       }
       return;
     }
 
-    this.logger.debug("Ignoring unknown payload root after failed download", logCtx, res.err);
+    this.metrics?.blockInputSync.downloadedPayloadsResult.inc({result: DownloadResult.Failed});
+    this.logger.debug("Failed to download unknown payload root, retained for retry", logCtx, res.err);
     if (!isPendingPayloadEnvelope(payload)) {
       payload.status = PendingPayloadInputStatus.pending;
     }
@@ -1248,6 +1441,11 @@ export class BlockInputSync {
       : await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
     let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
 
+    const fetchStartSec = Date.now() / 1000;
+    if (typeof slot === "number") {
+      this.metrics?.blockInputSync.payloadFetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
+    }
+
     let i = 0;
     let deferredByRateLimit = false;
     while (i++ < this.getMaxDownloadAttempts()) {
@@ -1271,6 +1469,11 @@ export class BlockInputSync {
           pendingColumns.size > 0
             ? `cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
             : "no peer available to download pending payload";
+        this.metrics?.blockInputSync.payloadFetchTime.observe(
+          {result: FetchResult.FailureTriedAllPeers},
+          Date.now() / 1000 - fetchStartSec
+        );
+        this.metrics?.blockInputSync.payloadFetchPeers.set({result: FetchResult.FailureTriedAllPeers}, i);
         throw Error(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: ${reason}`);
       }
 
@@ -1280,7 +1483,12 @@ export class BlockInputSync {
       try {
         if (!envelope) {
           envelope = await this.fetchExecutionPayloadEnvelope(peerId, blockRoot, rootHex);
+          const slotWasUnknown = typeof slot !== "number";
           slot = envelope.message.payload.slotNumber;
+          if (slotWasUnknown) {
+            // we were not able to observe the time into slot when starting the fetch, do it now
+            this.metrics?.blockInputSync.payloadFetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
+          }
         }
 
         payloadInput ??= await this.chain.seenPayloadEnvelopeInputCache.getOrReload(rootHex);
@@ -1289,6 +1497,11 @@ export class BlockInputSync {
           // envelope and wait for the block body; reconcilePayloadEnvelope validates once the block lands.
           // payloadInput may be seeded from the block body during download, so a non-null payloadInput does not
           // imply the block is imported.
+          this.metrics?.blockInputSync.payloadFetchTime.observe(
+            {result: FetchResult.SuccessWaitingForBlock},
+            Date.now() / 1000 - fetchStartSec
+          );
+          this.metrics?.blockInputSync.payloadFetchPeers.set({result: FetchResult.SuccessWaitingForBlock}, i);
           return {
             status: PendingPayloadInputStatus.waitingForBlock,
             envelope,
@@ -1339,6 +1552,12 @@ export class BlockInputSync {
         });
 
         if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
+          // late = the payload got imported (via gossip) while we were downloading, same as the block path
+          const result = this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)
+            ? FetchResult.SuccessLate
+            : FetchResult.SuccessResolved;
+          this.metrics?.blockInputSync.payloadFetchTime.observe({result}, Date.now() / 1000 - fetchStartSec);
+          this.metrics?.blockInputSync.payloadFetchPeers.set({result}, i);
           return pendingPayload;
         }
 
@@ -1373,12 +1592,17 @@ export class BlockInputSync {
       }
     }
 
+    this.metrics?.blockInputSync.payloadFetchTime.observe(
+      {result: FetchResult.FailureMaxAttempts},
+      Date.now() / 1000 - fetchStartSec
+    );
+    this.metrics?.blockInputSync.payloadFetchPeers.set({result: FetchResult.FailureMaxAttempts}, i - 1);
+
     if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
       throw new UnknownBlockRateLimitedError(
         `Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`
       );
     }
-
     throw Error(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts.`);
   }
 
@@ -1483,7 +1707,7 @@ export class BlockInputSync {
 
         // no more peer with needed columns to try, throw error
         const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
-        this.metrics?.blockInputSync.fetchTimeSec.observe(
+        this.metrics?.blockInputSync.fetchTime.observe(
           {result: FetchResult.FailureTriedAllPeers},
           Date.now() / 1000 - fetchStartSec
         );
@@ -1563,7 +1787,11 @@ export class BlockInputSync {
         this.peerBalancer.onRequestCompleted(peerId);
       }
 
-      this.pendingBlocks.set(getBlockInputSyncCacheItemRootHex(cacheItem), cacheItem);
+      const cacheItemRootHex = getBlockInputSyncCacheItemRootHex(cacheItem);
+      // pruning may have dropped it mid-fetch
+      if (this.pendingBlocks.has(cacheItemRootHex)) {
+        this.pendingBlocks.set(cacheItemRootHex, cacheItem);
+      }
 
       if (cacheItem.status === PendingBlockInputStatus.downloaded) {
         // download was successful, no need to go with another peer, return
@@ -1572,7 +1800,7 @@ export class BlockInputSync {
           : this.chain.forkChoice.hasBlockHex(cacheItem.blockInput.parentRootHex)
             ? FetchResult.SuccessResolved
             : FetchResult.SuccessMissingParent;
-        this.metrics?.blockInputSync.fetchTimeSec.observe({result}, Date.now() / 1000 - fetchStartSec);
+        this.metrics?.blockInputSync.fetchTime.observe({result}, Date.now() / 1000 - fetchStartSec);
         this.metrics?.blockInputSync.fetchPeers.set({result}, i);
         return cacheItem;
       }
@@ -1606,7 +1834,7 @@ export class BlockInputSync {
       }
     }
 
-    this.metrics?.blockInputSync.fetchTimeSec.observe(
+    this.metrics?.blockInputSync.fetchTime.observe(
       {result: FetchResult.FailureMaxAttempts},
       Date.now() / 1000 - fetchStartSec
     );

package/src/util/shuffle.ts

@@ -10,12 +10,3 @@ export function shuffle<T>(arr: T[]): T[] {
   }
   return _arr;
 }
-
-/**
- * Return one random item from array
- */
-export function shuffleOne<T>(arr: T[]): T | undefined {
-  if (arr.length === 0) return undefined;
-
-  return arr[Math.floor(Math.random() * arr.length)];
-}

package/src/util/sszBytes.ts

@@ -1,6 +1,7 @@
 import {BitArray, deserializeUint8ArrayBitListFromBytes} from "@chainsafe/ssz";
 import {ChainForkConfig} from "@lodestar/config";
 import {
+  BYTES_PER_CELL,
   BYTES_PER_FIELD_ELEMENT,
   FIELD_ELEMENTS_PER_BLOB,
   ForkName,
@@ -551,6 +552,23 @@ export function getBeaconBlockRootFromDataColumnSidecarSerialized(data: Uint8Arr
   return "0x" + blockRootBuf.toString("hex");
 }
 
+// beaconBlockRoot is the last fixed field in the Gloas layout
+const DATA_COLUMN_SIDECAR_GLOAS_FIXED_SIZE = BEACON_BLOCK_ROOT_POSITION_IN_GLOAS_DATA_COLUMN_SIDECAR + ROOT_SIZE; // 56 bytes
+const KZG_PROOF_SIZE = 48; // Bytes48
+const DATA_COLUMN_SIDECAR_GLOAS_PER_BLOB_SIZE = BYTES_PER_CELL + KZG_PROOF_SIZE; // 2048 + 48 = 2096 bytes
+
+/**
+ * Gloas sidecar size bound uses the largest blob limit across the entire schedule, including future entries.
+ * https://github.com/ethereum/consensus-specs/pull/5613
+ */
+export function computeMaxGloasDataColumnSidecarSize(config: ChainForkConfig): number {
+  const maxBlobs = config.BLOB_SCHEDULE.reduce(
+    (max, entry) => Math.max(max, entry.MAX_BLOBS_PER_BLOCK),
+    config.MAX_BLOBS_PER_BLOCK_ELECTRA
+  );
+  return DATA_COLUMN_SIDECAR_GLOAS_FIXED_SIZE + maxBlobs * DATA_COLUMN_SIDECAR_GLOAS_PER_BLOB_SIZE;
+}
+
 /**
  * SignedExecutionPayloadEnvelope SSZ Layout:
  * ├─ 4 bytes: message offset (points to byte 100)

package/lib/chain/archiveStore/utils/updateBackfillRange.d.ts

@@ -1,20 +0,0 @@
-import { CheckpointWithHex } from "@lodestar/fork-choice";
-import { Logger } from "@lodestar/logger";
-import { IBeaconDb } from "../../../db/interface.js";
-import { IBeaconChain } from "../../interface.js";
-/**
- * Backfill sync relies on verified connected ranges (which are represented as key,value
- * with a verified jump from a key back to value). Since the node could have progressed
- * ahead from, we need to save the forward progress of this node as another backfill
- * range entry, that backfill sync will use to jump back if this node is restarted
- * for any reason.
- * The current backfill has its own backfill entry from anchor slot to last backfilled
- * slot. And this would create the entry from the current finalized slot to the anchor
- * slot.
- */
-export declare function updateBackfillRange({ chain, db, logger }: {
-    chain: IBeaconChain;
-    db: IBeaconDb;
-    logger: Logger;
-}, finalized: CheckpointWithHex): Promise<void>;
-//# sourceMappingURL=updateBackfillRange.d.ts.map

package/lib/chain/archiveStore/utils/updateBackfillRange.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"updateBackfillRange.d.ts","sourceRoot":"","sources":["../../../../src/chain/archiveStore/utils/updateBackfillRange.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAEhD;;;;;;;;;GASG;AACH,wBAAsB,mBAAmB,CACvC,EAAC,KAAK,EAAE,EAAE,EAAE,MAAM,EAAC,EAAE;IAAC,KAAK,EAAE,YAAY,CAAC;IAAC,EAAE,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAC,EACzE,SAAS,EAAE,iBAAiB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA+Bf"}

package/lib/chain/archiveStore/utils/updateBackfillRange.js

@@ -1,40 +0,0 @@
-/**
- * Backfill sync relies on verified connected ranges (which are represented as key,value
- * with a verified jump from a key back to value). Since the node could have progressed
- * ahead from, we need to save the forward progress of this node as another backfill
- * range entry, that backfill sync will use to jump back if this node is restarted
- * for any reason.
- * The current backfill has its own backfill entry from anchor slot to last backfilled
- * slot. And this would create the entry from the current finalized slot to the anchor
- * slot.
- */
-export async function updateBackfillRange({ chain, db, logger }, finalized) {
-    try {
-        // Mark the sequence in backfill db from finalized block's slot till anchor slot as
-        // filled.
-        const finalizedBlockFC = chain.forkChoice.getBlockHexDefaultStatus(finalized.rootHex);
-        if (finalizedBlockFC && finalizedBlockFC.slot > chain.anchorStateLatestBlockSlot) {
-            await db.backfilledRanges.put(finalizedBlockFC.slot, chain.anchorStateLatestBlockSlot);
-            // Clear previously marked sequence till anchorStateLatestBlockSlot, without
-            // touching backfill sync process sequence which are at
-            // <=anchorStateLatestBlockSlot i.e. clear >anchorStateLatestBlockSlot
-            // and < currentSlot
-            const filteredSeqs = await db.backfilledRanges.entries({
-                gt: chain.anchorStateLatestBlockSlot,
-                lt: finalizedBlockFC.slot,
-            });
-            logger.debug("updated backfilledRanges", {
-                key: finalizedBlockFC.slot,
-                value: chain.anchorStateLatestBlockSlot,
-            });
-            if (filteredSeqs.length > 0) {
-                await db.backfilledRanges.batchDelete(filteredSeqs.map((entry) => entry.key));
-                logger.debug(`Forward Sync - cleaned up backfilledRanges between ${finalizedBlockFC.slot},${chain.anchorStateLatestBlockSlot}`, { seqs: JSON.stringify(filteredSeqs) });
-            }
-        }
-    }
-    catch (e) {
-        logger.error("Error updating backfilledRanges on finalization", { epoch: finalized.epoch }, e);
-    }
-}
-//# sourceMappingURL=updateBackfillRange.js.map

package/lib/chain/archiveStore/utils/updateBackfillRange.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"updateBackfillRange.js","sourceRoot":"","sources":["../../../../src/chain/archiveStore/utils/updateBackfillRange.ts"],"names":[],"mappings":"AAKA;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,EAAC,KAAK,EAAE,EAAE,EAAE,MAAM,EAAuD,EACzE,SAA4B;IAE5B,IAAI,CAAC;QACH,mFAAmF;QACnF,UAAU;QACV,MAAM,gBAAgB,GAAG,KAAK,CAAC,UAAU,CAAC,wBAAwB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACtF,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,IAAI,GAAG,KAAK,CAAC,0BAA0B,EAAE,CAAC;YACjF,MAAM,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAEvF,4EAA4E;YAC5E,uDAAuD;YACvD,sEAAsE;YACtE,oBAAoB;YACpB,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,OAAO,CAAC;gBACrD,EAAE,EAAE,KAAK,CAAC,0BAA0B;gBACpC,EAAE,EAAE,gBAAgB,CAAC,IAAI;aAC1B,CAAC,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE;gBACvC,GAAG,EAAE,gBAAgB,CAAC,IAAI;gBAC1B,KAAK,EAAE,KAAK,CAAC,0BAA0B;aACxC,CAAC,CAAC;YACH,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,EAAE,CAAC,gBAAgB,CAAC,WAAW,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC9E,MAAM,CAAC,KAAK,CACV,sDAAsD,gBAAgB,CAAC,IAAI,IAAI,KAAK,CAAC,0BAA0B,EAAE,EACjH,EAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,EAAC,CACrC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,EAAC,KAAK,EAAE,SAAS,CAAC,KAAK,EAAC,EAAE,CAAU,CAAC,CAAC;IACxG,CAAC;AACH,CAAC"}

package/lib/db/repositories/backfilledRanges.d.ts

@@ -1,18 +0,0 @@
-import { ChainForkConfig } from "@lodestar/config";
-import { DatabaseController, Repository } from "@lodestar/db";
-import { Slot } from "@lodestar/types";
-/**
- * Slot to slot ranges that ensure that block range is fully backfilled
- *
- * If node starts backfilling at slots 1000, and backfills to 800, there will be an entry
- * 1000 -> 800
- *
- * When the node is backfilling if it starts at 1200 and backfills to 1000, it will find this sequence and,
- * jump directly to 800 and delete the key 1000.
- */
-export declare class BackfilledRanges extends Repository<Slot, Slot> {
-    constructor(config: ChainForkConfig, db: DatabaseController<Uint8Array, Uint8Array>);
-    decodeKey(data: Buffer): number;
-    getId(_value: Slot): number;
-}
-//# sourceMappingURL=backfilledRanges.d.ts.map

package/lib/db/repositories/backfilledRanges.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"backfilledRanges.d.ts","sourceRoot":"","sources":["../../../src/db/repositories/backfilledRanges.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAC,kBAAkB,EAAE,UAAU,EAAC,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAC,IAAI,EAAM,MAAM,iBAAiB,CAAC;AAI1C;;;;;;;;GAQG;AACH,qBAAa,gBAAiB,SAAQ,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC;IAC1D,YAAY,MAAM,EAAE,eAAe,EAAE,EAAE,EAAE,kBAAkB,CAAC,UAAU,EAAE,UAAU,CAAC,EAGlF;IAED,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE9B;IAED,KAAK,CAAC,MAAM,EAAE,IAAI,GAAG,MAAM,CAE1B;CACF"}

package/lib/db/repositories/backfilledRanges.js

@@ -1,26 +0,0 @@
-import { Repository } from "@lodestar/db";
-import { ssz } from "@lodestar/types";
-import { bytesToInt } from "@lodestar/utils";
-import { Bucket, getBucketNameByValue } from "../buckets.js";
-/**
- * Slot to slot ranges that ensure that block range is fully backfilled
- *
- * If node starts backfilling at slots 1000, and backfills to 800, there will be an entry
- * 1000 -> 800
- *
- * When the node is backfilling if it starts at 1200 and backfills to 1000, it will find this sequence and,
- * jump directly to 800 and delete the key 1000.
- */
-export class BackfilledRanges extends Repository {
-    constructor(config, db) {
-        const bucket = Bucket.backfilled_ranges;
-        super(config, db, bucket, ssz.Slot, getBucketNameByValue(bucket));
-    }
-    decodeKey(data) {
-        return bytesToInt(super.decodeKey(data), "be");
-    }
-    getId(_value) {
-        throw new Error("Cannot get the db key from slot");
-    }
-}
-//# sourceMappingURL=backfilledRanges.js.map

package/lib/db/repositories/backfilledRanges.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"backfilledRanges.js","sourceRoot":"","sources":["../../../src/db/repositories/backfilledRanges.ts"],"names":[],"mappings":"AACA,OAAO,EAAqB,UAAU,EAAC,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAO,GAAG,EAAC,MAAM,iBAAiB,CAAC;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAC,MAAM,EAAE,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAE3D;;;;;;;;GAQG;AACH,MAAM,OAAO,gBAAiB,SAAQ,UAAsB;IAC1D,YAAY,MAAuB,EAAE,EAA8C;QACjF,MAAM,MAAM,GAAG,MAAM,CAAC,iBAAiB,CAAC;QACxC,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,SAAS,CAAC,IAAY;QACpB,OAAO,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAA0B,EAAE,IAAI,CAAC,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,MAAY;QAChB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;CACF"}