@lodestar/beacon-node 1.49.0-dev.3f33d73d55 → 1.49.0-dev.4631f9fe5a

package/src/sync/unknownBlock.ts

@@ -5,7 +5,7 @@ import {ForkSeq, SLOTS_PER_EPOCH, isForkPostGloas} from "@lodestar/params";
 import {RequestError, RequestErrorCode} from "@lodestar/reqresp";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
 import {RootHex, Slot, gloas} from "@lodestar/types";
-import {Logger, fromHex, prettyPrintIndices, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
+import {Logger, fromHex, isErrorAborted, prettyPrintIndices, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../chain/blocks/blockInput/types.js";
 import {PayloadError, PayloadErrorCode} from "../chain/blocks/importExecutionPayload.js";
@@ -18,6 +18,7 @@ import {Metrics} from "../metrics/index.js";
 import {INetwork, NetworkEvent, NetworkEventData, prettyPrintPeerIdStr} from "../network/index.js";
 import {PeerSyncMeta} from "../network/peers/peersData.js";
 import {MAX_PEERS_PER_ROOT} from "../network/processor/constants.js";
+import {ClockEvent} from "../util/clock.js";
 import {PeerIdStr} from "../util/peerId.js";
 import {shuffle} from "../util/shuffle.js";
 import {sortBy} from "../util/sortBy.js";
@@ -26,6 +27,7 @@ import {MAX_CONCURRENT_REQUESTS} from "./constants.js";
 import {SyncOptions} from "./options.js";
 import {
   BlockInputSyncCacheItem,
+  DeferredPayloadResult,
   DownloadResult,
   DroppedItemReason,
   FetchResult,
@@ -57,6 +59,11 @@ const MAX_PENDING_BLOCKS = 100;
  */
 const PRUNE_UNRESOLVED_SLOT_EPOCHS = 2;
 
+/**
+ * Poll pending payloads of the current slot this often, between PAYLOAD_DUE and the end of the slot.
+ */
+export const PAYLOAD_POLL_INTERVAL_MS = 1_000;
+
 type AdvancePendingBlockResult =
   | "ready"
   | "queued_block"
@@ -116,6 +123,11 @@ export class BlockInputSync {
   private subscribedToNetworkEvents = false;
   private peerBalancer: UnknownBlockPeerBalancer;
   private rateLimitBackoffTimeout: NodeJS.Timeout | undefined;
+  /**
+   * Search unknown payload starting at PAYLOAD_DUE_BPS.
+   */
+  private readonly deferredPayloadRoots = new Map<RootHex, {slot: Slot; source: BlockInputSource; peer?: PeerIdStr}>();
+  private payloadPollAbortController: AbortController | undefined;
 
   constructor(
     private readonly config: ChainForkConfig,
@@ -161,6 +173,7 @@ export class BlockInputSync {
       this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadImported);
       this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
       this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
+      this.chain.clock.on(ClockEvent.slot, this.startPayloadPollAtSlot);
 
       // Seed the balancer with peers that connected before we subscribed
       for (const peerId of this.network.getConnectedPeers()) {
@@ -172,12 +185,19 @@ export class BlockInputSync {
       }
 
       this.subscribedToNetworkEvents = true;
+      // we subscribe mid-slot, and a root can be deferred at any point in the slot, so start this
+      // slot's loop now instead of waiting for the next ClockEvent.slot
+      this.startPayloadPollAtSlot(this.chain.clock.currentSlot);
     }
   }
 
   unsubscribeFromNetwork(): void {
     this.logger.verbose("BlockInputSync disabled.");
     this.clearRateLimitBackoffTimer();
+    this.chain.clock.off(ClockEvent.slot, this.startPayloadPollAtSlot);
+    this.payloadPollAbortController?.abort();
+    this.payloadPollAbortController = undefined;
+    this.deferredPayloadRoots.clear();
     this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
     this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
     this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRootSlot, this.onUnknownEnvelopeBlockRootSlot);
@@ -207,8 +227,11 @@ export class BlockInputSync {
   private onUnknownBlockRoot = (data: ChainEventData[ChainEvent.unknownBlockRoot]): void => {
     try {
       const isNewRoot = this.addByRootHex(data.rootHex, data.peer);
-      if (isNewRoot) {
+      // A retained entry (an earlier download failed, #10018) is not new but still needs a retry pass.
+      if (this.pendingBlocks.get(data.rootHex)?.status === PendingBlockInputStatus.pending) {
         this.triggerUnknownBlockSearch();
+      }
+      if (isNewRoot) {
         this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_BLOCK_ROOT});
         this.metrics?.blockInputSync.source.inc({source: data.source});
       }
@@ -246,6 +269,20 @@ export class BlockInputSync {
 
   private onUnknownEnvelopeBlockRootSlot = (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRootSlot]): void => {
     try {
+      if (data.slot === this.chain.clock.currentSlot) {
+        // we receive the optimistic search, the poll loop searches it at PAYLOAD_DUE or on its next
+        // tick. If that loop already returned - every root it watched was imported - this root waits
+        // for the next slot, where startPayloadPollAtSlot moves it to pendingPayloads. That is not the normal
+        // scenario, and in a forking condition it only delays the search of the extra root
+        const deferred = this.deferredPayloadRoots.get(data.rootHex);
+        if (deferred === undefined) {
+          this.deferredPayloadRoots.set(data.rootHex, {slot: data.slot, source: data.source, peer: data.peer});
+        } else if (deferred.peer === undefined) {
+          deferred.peer = data.peer;
+        }
+        return;
+      }
+
       const isNewRoot = this.addByPayloadRootHex(data.rootHex, data.peer, data.slot);
       if (isNewRoot) {
         this.triggerUnknownBlockSearch();
@@ -359,12 +396,21 @@ export class BlockInputSync {
       }
     }
 
-    let deletedBlocks = 0;
+    let belowFinalizedBlocks = 0;
+    let agedOutBlocks = 0;
     for (const [rootHex, block] of this.pendingBlocks) {
       const slot = getBlockInputSyncCacheItemSlot(block);
-      if (typeof slot === "number" && slot < finalizedSlot) {
+      if (typeof slot !== "number") {
+        // we may not be able to download a root, hence there is no slot for it
+        if (nowSec - block.timeAddedSec > maxSecInCache) {
+          this.pendingBlocks.delete(rootHex);
+          agedOutBlocks++;
+        }
+        continue;
+      }
+      if (slot < finalizedSlot) {
         this.pendingBlocks.delete(rootHex);
-        deletedBlocks++;
+        belowFinalizedBlocks++;
       }
     }
 
@@ -377,16 +423,20 @@ export class BlockInputSync {
     if (agedOutPayloads > 0) {
       this.metrics?.blockInputSync.removedPayloads.inc({reason: DroppedItemReason.agedOut}, agedOutPayloads);
     }
-    if (deletedBlocks > 0) {
-      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.belowFinalized}, deletedBlocks);
+    if (belowFinalizedBlocks > 0) {
+      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.belowFinalized}, belowFinalizedBlocks);
+    }
+    if (agedOutBlocks > 0) {
+      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.agedOut}, agedOutBlocks);
     }
 
-    this.logger.debug("BlockInputSync.pruneFinalized dropped pre-finalized pending items", {
+    this.logger.debug("BlockInputSync.pruneFinalized dropped stale pending items", {
       finalizedSlot,
       finalizedRoot: checkpoint.rootHex,
       belowFinalizedPayloads,
       agedOutPayloads,
-      deletedBlocks,
+      belowFinalizedBlocks,
+      agedOutBlocks,
       unresolvedSlotPayloads,
     });
   };
@@ -800,6 +850,122 @@ export class BlockInputSync {
     }
   }
 
+  /**
+   * Start this slot's payload poll: end the previous slot's loop, move roots it never got to
+   * into pendingPayloads, then poll from PAYLOAD_DUE to the end of the slot.
+   */
+  private startPayloadPollAtSlot = (slot: Slot): void => {
+    if (!isForkPostGloas(this.config.getForkName(slot))) {
+      return;
+    }
+    // end the previous slot's loop, wherever it is: its sleep is bound to this same controller
+    this.payloadPollAbortController?.abort();
+    this.payloadPollAbortController = new AbortController();
+    // entries from an earlier slot is not polled
+    for (const [rootHex, {slot: deferredSlot, source, peer}] of this.deferredPayloadRoots) {
+      if (deferredSlot < slot) {
+        if (!this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+          this.addDeferredPayloadRoot(rootHex, deferredSlot, source, peer);
+        }
+        this.deferredPayloadRoots.delete(rootHex);
+      }
+    }
+    this.pollPayloadsAtSlot(slot, this.payloadPollAbortController.signal).catch((e) => {
+      if (!isErrorAborted(e)) {
+        this.logger.debug("Error polling payloads", {slot}, e as Error);
+      }
+    });
+  };
+
+  /**
+   * Poll unknown payloads of the current slot.
+   */
+  private async pollPayloadsAtSlot(slot: Slot, signal: AbortSignal): Promise<void> {
+    // fixed tick count so the loop cannot outlive its slot: for mainnet, it's 6 ticks
+    const tickCount = Math.floor(
+      (this.config.SLOT_DURATION_MS - this.config.getPayloadDueMs()) / PAYLOAD_POLL_INTERVAL_MS
+    );
+    // a root still missing on several ticks is searched on each of them, recorded once
+    const searchedRoots = new Set<RootHex>();
+    let polls = 0;
+
+    try {
+      await sleep(Math.max(0, this.config.getPayloadDueMs() - this.chain.clock.msFromSlot(slot)), signal);
+
+      for (polls = 0; polls < tickCount; polls++) {
+        let importedCount = 0;
+        for (const [rootHex, {slot: deferredSlot, source, peer}] of this.deferredPayloadRoots) {
+          if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
+            importedCount++;
+            continue;
+          }
+          // copy to the regular pendingPayloads so that we can search
+          this.addDeferredPayloadRoot(rootHex, deferredSlot, source, peer);
+          searchedRoots.add(rootHex);
+        }
+
+        // an empty deferredPayloadRoots means there is no optimistic search comes before PAYLOAD_DUE
+        // we have to poll until the payload of current slot is imported or slot end
+        if (this.deferredPayloadRoots.size > 0 && importedCount === this.deferredPayloadRoots.size) {
+          if (searchedRoots.size > 0) {
+            // the backup search was needed
+            this.metrics?.blockInputSync.deferredPayloadResult.inc({
+              result: DeferredPayloadResult.ImportedAfterSearch,
+            });
+            this.metrics?.blockInputSync.deferredPayloadPolls.observe(polls);
+            this.logger.verbose("Payload imported after unknown sync search", {
+              slot,
+              roots: Array.from(searchedRoots).join(","),
+              polls,
+            });
+          } else {
+            // this happens most of the time on devnet
+            // gossip delivered everything before PAYLOAD_DUE
+            this.metrics?.blockInputSync.deferredPayloadResult.inc({
+              result: DeferredPayloadResult.ImportedBeforeSearch,
+            });
+          }
+          return;
+        }
+
+        this.triggerUnknownBlockSearch();
+        if (polls < tickCount - 1) {
+          // no sleep after the last tick: the loop would otherwise end right at the slot boundary,
+          // where startPayloadPollAtSlot aborts it
+          await sleep(PAYLOAD_POLL_INTERVAL_MS, signal);
+        }
+      }
+
+      // reaching here means the early return never fired
+      if (this.deferredPayloadRoots.size > 0) {
+        this.metrics?.blockInputSync.deferredPayloadResult.inc({result: DeferredPayloadResult.Unresolved});
+        this.logger.debug("Deferred payloads still missing at end of slot", {
+          slot,
+          roots: Array.from(this.deferredPayloadRoots.keys()).join(","),
+        });
+      } else {
+        // no optimistic search and no payload import ended the loop, eg a skipped slot
+        this.metrics?.blockInputSync.deferredPayloadResult.inc({result: DeferredPayloadResult.ResolvedNoRoot});
+        this.logger.debug("No deferred payloads at slot", {
+          slot,
+        });
+      }
+    } catch (e) {
+      if (!isErrorAborted(e)) {
+        throw e;
+      }
+      this.logger.verbose("Payload poll aborted", {slot, polls, msIntoSlot: this.chain.clock.msFromSlot(slot)});
+    }
+  }
+
+  private addDeferredPayloadRoot(rootHex: RootHex, slot: Slot, source: BlockInputSource, peer?: PeerIdStr): void {
+    const isNewRoot = this.addByPayloadRootHex(rootHex, peer, slot);
+    if (isNewRoot) {
+      this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.DEFERRED_PAYLOAD_BLOCK_ROOT});
+      this.metrics?.blockInputSync.payloadSource.inc({source});
+    }
+  }
+
   private async downloadBlock(block: BlockInputSyncCacheItem): Promise<void> {
     if (block.status !== PendingBlockInputStatus.pending) {
       return;
@@ -823,7 +989,12 @@ export class BlockInputSync {
     if (!res.err) {
       this.metrics?.blockInputSync.downloadedBlocksSuccess.inc();
       const pending = res.result;
-      this.pendingBlocks.set(pending.blockInput.blockRootHex, pending);
+      // pruning may have deleted this entry while we were awaiting the network, do not resurrect it via the set below
+      if (!this.pendingBlocks.has(rootHex)) {
+        this.logger.verbose("Dropping downloaded block, entry pruned during fetch", logCtx);
+        return;
+      }
+      this.pendingBlocks.set(rootHex, pending);
       const blockSlot = pending.blockInput.slot;
       const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
       const delaySec = Date.now() / 1000 - computeTimeAtSlot(this.config, blockSlot, this.chain.genesisTime);
@@ -885,8 +1056,11 @@ export class BlockInputSync {
       }
 
       this.metrics?.blockInputSync.downloadedBlocksError.inc();
-      this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
-      this.removeAndDownScoreAllDescendants(block, DroppedItemReason.unavailable);
+      this.logger.debug("Failed to download unknown block root, retained for retry", logCtx, res.err);
+      const pendingBlock = this.pendingBlocks.get(rootHex);
+      if (pendingBlock && pendingBlock.status === PendingBlockInputStatus.fetching) {
+        pendingBlock.status = PendingBlockInputStatus.pending;
+      }
     }
   }
 
@@ -1153,7 +1327,7 @@ export class BlockInputSync {
     }
 
     this.metrics?.blockInputSync.downloadedPayloadsResult.inc({result: DownloadResult.Failed});
-    this.logger.debug("Ignoring unknown payload root after failed download", logCtx, res.err);
+    this.logger.debug("Failed to download unknown payload root, retained for retry", logCtx, res.err);
     if (!isPendingPayloadEnvelope(payload)) {
       payload.status = PendingPayloadInputStatus.pending;
     }
@@ -1613,7 +1787,11 @@ export class BlockInputSync {
         this.peerBalancer.onRequestCompleted(peerId);
       }
 
-      this.pendingBlocks.set(getBlockInputSyncCacheItemRootHex(cacheItem), cacheItem);
+      const cacheItemRootHex = getBlockInputSyncCacheItemRootHex(cacheItem);
+      // pruning may have dropped it mid-fetch
+      if (this.pendingBlocks.has(cacheItemRootHex)) {
+        this.pendingBlocks.set(cacheItemRootHex, cacheItem);
+      }
 
       if (cacheItem.status === PendingBlockInputStatus.downloaded) {
         // download was successful, no need to go with another peer, return

package/src/util/execution.ts

@@ -2,12 +2,18 @@ import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkPostFulu, ForkPreFulu} from "@lodestar/params";
 import {signedBlockToSignedHeader} from "@lodestar/state-transition";
-import {DataColumnSidecar, SignedBeaconBlock, deneb, isGloasDataColumnSidecar} from "@lodestar/types";
-import {fromHex, toHex} from "@lodestar/utils";
+import {DataColumnSidecar, SignedBeaconBlock, Slot, deneb, gloas, isGloasDataColumnSidecar, ssz} from "@lodestar/types";
+import {Logger, fromHex, toHex, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../chain/blocks/blockInput/types.js";
 import {PayloadEnvelopeInput, PayloadEnvelopeInputSource} from "../chain/blocks/payloadEnvelopeInput/index.js";
 import {ChainEvent, ChainEventEmitter} from "../chain/emitter.js";
+import {
+  EnvelopeReconstructionError,
+  EnvelopeReconstructionErrorCode,
+} from "../chain/errors/envelopeReconstructionError.js";
+import {IBeaconDb} from "../db/index.js";
+import {ArchivedEnvelope, decodeArchivedEnvelope} from "../db/repositories/index.js";
 import {IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/index.js";
 import {computePreFuluKzgCommitmentsInclusionProof} from "./blobs.js";
@@ -17,6 +23,7 @@ import {
   getDataColumnSidecarsFromColumnSidecar,
   getGloasDataColumnSidecars,
 } from "./dataColumns.js";
+import {signedHeaderEnvelopeToFull} from "./headerEnvelope.js";
 
 export enum DataColumnEngineResult {
   PreFulu = "pre_fulu",
@@ -251,3 +258,169 @@ export async function getDataColumnSidecarsFromExecution(
   );
   return DataColumnEngineResult.SuccessResolved;
 }
+
+/** engine_getPayloadBodiesByHashV2: ELs MUST support at least 32 hashes per request. */
+const MAX_BODIES_REQUEST = 32;
+
+type SlotEnvelopeBytes = {slot: Slot; envelopeBytes: Uint8Array};
+
+type RangeEntry = ArchivedEnvelope & {slot: Slot};
+
+/** What a body root mismatch means on a given serving path */
+export type ReconstructMismatchPolicy = "throw" | "omit";
+
+export type RebuildMiss =
+  /** EL does not have the block, or has pruned its block access list */
+  | {slot: Slot; reason: "unavailable"}
+  /** An EL body does not hash to its stored root (local inconsistency) */
+  | {slot: Slot; reason: "mismatch"; error: EnvelopeReconstructionError};
+
+/**
+ * Stream finalized envelopes over [startSlot, endSlot) as serialized bytes, rebuilding header
+ * entries from EL bodies 32 per round-trip.
+ *
+ * Ends at the first entry that cannot be served by throwing RANGE_UNSERVABLE with that slot: the
+ * by-range spec inherits BeaconBlocksByRange v2 semantics (consecutive, MAY be short), and a hole
+ * looks like a lying peer to one that already holds the blocks. Entries are attempted regardless of
+ * age; the EL's block access list retention is the floor, not MIN_EPOCHS_FOR_BLOCK_REQUESTS.
+ *
+ * Throws ENGINE_UNAVAILABLE if the EL call fails. Either may surface after envelopes were yielded.
+ */
+export async function* reconstructExecutionPayloadEnvelopesByRange(
+  db: IBeaconDb,
+  executionEngine: IExecutionEngine,
+  logger: Logger,
+  metrics: Metrics | null,
+  startSlot: Slot,
+  endSlot: Slot
+): AsyncIterable<SlotEnvelopeBytes> {
+  const archive = db.executionPayloadEnvelopeArchive;
+  let batch: RangeEntry[] = [];
+
+  for await (const {key, value: bytes} of archive.binaryEntriesStream({gte: startSlot, lt: endSlot})) {
+    batch.push({slot: archive.decodeKey(key), ...decodeArchivedEnvelope(bytes)});
+    if (batch.length === MAX_BODIES_REQUEST) {
+      yield* reconstructBatch(executionEngine, logger, metrics, batch);
+      batch = [];
+    }
+  }
+  if (batch.length > 0) {
+    yield* reconstructBatch(executionEngine, logger, metrics, batch);
+  }
+}
+
+/**
+ * Rebuild a range batch, yielding what precedes the first unservable entry before throwing
+ * RANGE_UNSERVABLE for it
+ */
+async function* reconstructBatch(
+  executionEngine: IExecutionEngine,
+  logger: Logger,
+  metrics: Metrics | null,
+  batch: RangeEntry[]
+): AsyncIterable<SlotEnvelopeBytes> {
+  const headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[] = [];
+  for (const entry of batch) {
+    if (entry.headerEnvelope !== undefined) headerEnvelopes.push(entry.headerEnvelope);
+  }
+  const rebuilt = await reconstructEnvelopesBatch(executionEngine, metrics, headerEnvelopes);
+
+  let rebuiltIdx = 0;
+  for (const entry of batch) {
+    if (entry.envelopeBytes !== undefined) {
+      yield {slot: entry.slot, envelopeBytes: entry.envelopeBytes};
+      continue;
+    }
+    const result = rebuilt[rebuiltIdx++];
+    if (isRebuildMiss(result)) {
+      // Peer-triggered, so debug: a persistent local mismatch would otherwise log on every request
+      if (result.reason === "mismatch") {
+        logger.debug("Archived envelope failed body root check against EL bodies", {slot: entry.slot}, result.error);
+      } else {
+        logger.debug("EL cannot serve bodies for archived envelope, ending range", {slot: entry.slot});
+      }
+      throw new EnvelopeReconstructionError(
+        {code: EnvelopeReconstructionErrorCode.RANGE_UNSERVABLE, slot: entry.slot},
+        `archived envelope range unservable from slot=${entry.slot}`
+      );
+    }
+    yield {slot: entry.slot, envelopeBytes: ssz.gloas.SignedExecutionPayloadEnvelope.serialize(result)};
+  }
+}
+
+/**
+ * Rebuild header envelopes from EL bodies, 32 per round-trip. Aligned with the input, with a
+ * {@link RebuildMiss} where the envelope could not be rebuilt; the caller decides what a miss means
+ * on its path. Throws ENGINE_UNAVAILABLE only.
+ */
+export async function reconstructExecutionPayloadEnvelopes(
+  executionEngine: IExecutionEngine,
+  metrics: Metrics | null,
+  headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[]
+): Promise<(gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[]> {
+  const out: (gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[] = [];
+  for (let i = 0; i < headerEnvelopes.length; i += MAX_BODIES_REQUEST) {
+    out.push(
+      ...(await reconstructEnvelopesBatch(executionEngine, metrics, headerEnvelopes.slice(i, i + MAX_BODIES_REQUEST)))
+    );
+  }
+  return out;
+}
+
+export function isRebuildMiss(result: gloas.SignedExecutionPayloadEnvelope | RebuildMiss): result is RebuildMiss {
+  return "reason" in result;
+}
+
+/**
+ * One EL round-trip. Aligned with the input; never throws per envelope, only ENGINE_UNAVAILABLE.
+ * Every serving path (by-range, by-root, REST) comes through here, so the outcome metrics are
+ * incremented once, in this function.
+ */
+async function reconstructEnvelopesBatch(
+  executionEngine: IExecutionEngine,
+  metrics: Metrics | null,
+  headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[]
+): Promise<(gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[]> {
+  if (headerEnvelopes.length === 0) return [];
+  const hashes = headerEnvelopes.map((headerEnvelope) => toRootHex(headerEnvelope.message.payloadHeader.blockHash));
+
+  let bodies: Awaited<ReturnType<IExecutionEngine["getPayloadBodiesByHashV2"]>>;
+  try {
+    bodies = await executionEngine.getPayloadBodiesByHashV2(hashes);
+  } catch (e) {
+    metrics?.payloadEnvelopeReconstruction.engineErrors.inc();
+    throw new EnvelopeReconstructionError(
+      {code: EnvelopeReconstructionErrorCode.ENGINE_UNAVAILABLE},
+      `engine_getPayloadBodiesByHashV2 failed: ${(e as Error).message}`
+    );
+  }
+
+  return headerEnvelopes.map((headerEnvelope, i) => {
+    const slot = headerEnvelope.message.payloadHeader.slotNumber;
+    const body = bodies[i];
+    // A zero-length block access list cannot be valid, RLP encodes an empty list as 0xc0
+    if (body == null || body.withdrawals == null || body.blockAccessList == null || body.blockAccessList.length === 0) {
+      metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "unavailable"});
+      return {slot, reason: "unavailable"};
+    }
+    try {
+      const envelope = signedHeaderEnvelopeToFull(headerEnvelope, {
+        transactions: body.transactions,
+        withdrawals: body.withdrawals,
+        blockAccessList: body.blockAccessList,
+      });
+      metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "ok"});
+      return envelope;
+    } catch (e) {
+      if (
+        e instanceof EnvelopeReconstructionError &&
+        e.type.code === EnvelopeReconstructionErrorCode.BODY_ROOT_MISMATCH
+      ) {
+        metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "mismatch"});
+        metrics?.payloadEnvelopeReconstruction.mismatchByField.inc({field: e.type.field});
+        return {slot, reason: "mismatch", error: e};
+      }
+      throw e;
+    }
+  });
+}

package/src/util/headerEnvelope.ts

@@ -0,0 +1,76 @@
+import {ssz} from "@lodestar/types";
+import {
+  BlockAccessList,
+  ExecutionPayloadEnvelope,
+  ExecutionPayloadHeaderEnvelope,
+  SignedExecutionPayloadEnvelope,
+  SignedExecutionPayloadHeaderEnvelope,
+  Transactions,
+  Withdrawals,
+} from "@lodestar/types/gloas";
+import {toRootHex} from "@lodestar/utils";
+import {
+  EnvelopeReconstructionError,
+  EnvelopeReconstructionErrorCode,
+} from "../chain/errors/envelopeReconstructionError.js";
+
+/** The parts of an execution payload the EL serves back via engine_getPayloadBodiesByHashV2 */
+export type ExecutionPayloadBodies = {
+  transactions: Transactions;
+  withdrawals: Withdrawals;
+  blockAccessList: BlockAccessList;
+};
+
+export type ExecutionPayloadBodyField = keyof ExecutionPayloadBodies;
+
+const {transactions, withdrawals, blockAccessList} = ssz.gloas.ExecutionPayload.fields;
+
+/** Replace the three body lists by their roots; hashes to the same root as the full envelope */
+export function toHeaderEnvelope(envelope: ExecutionPayloadEnvelope): ExecutionPayloadHeaderEnvelope {
+  const {payload, ...envelopeFields} = envelope;
+  const {transactions: txs, withdrawals: wds, blockAccessList: bal, ...scalars} = payload;
+  return {
+    ...envelopeFields,
+    payloadHeader: {
+      ...scalars,
+      transactionsRoot: transactions.hashTreeRoot(txs),
+      withdrawalsRoot: withdrawals.hashTreeRoot(wds),
+      blockAccessListRoot: blockAccessList.hashTreeRoot(bal),
+    },
+  };
+}
+
+export function toSignedHeaderEnvelope(envelope: SignedExecutionPayloadEnvelope): SignedExecutionPayloadHeaderEnvelope {
+  return {message: toHeaderEnvelope(envelope.message), signature: envelope.signature};
+}
+
+/**
+ * Rebuild the full signed envelope from header envelope + EL bodies, each body verified against its stored
+ * root. Throws {@link EnvelopeReconstructionError} BODY_ROOT_MISMATCH naming the first body that differs.
+ */
+export function signedHeaderEnvelopeToFull(
+  headerEnvelope: SignedExecutionPayloadHeaderEnvelope,
+  bodies: ExecutionPayloadBodies
+): SignedExecutionPayloadEnvelope {
+  const {payloadHeader, ...envelopeFields} = headerEnvelope.message;
+  const {transactionsRoot, withdrawalsRoot, blockAccessListRoot, ...scalars} = payloadHeader;
+  const slot = scalars.slotNumber;
+
+  assertBodyRoot("transactions", slot, transactionsRoot, transactions.hashTreeRoot(bodies.transactions));
+  assertBodyRoot("withdrawals", slot, withdrawalsRoot, withdrawals.hashTreeRoot(bodies.withdrawals));
+  assertBodyRoot("blockAccessList", slot, blockAccessListRoot, blockAccessList.hashTreeRoot(bodies.blockAccessList));
+
+  return {
+    message: {...envelopeFields, payload: {...scalars, ...bodies}},
+    signature: headerEnvelope.signature,
+  };
+}
+
+function assertBodyRoot(field: ExecutionPayloadBodyField, slot: number, stored: Uint8Array, rebuilt: Uint8Array): void {
+  if (!ssz.Root.equals(stored, rebuilt)) {
+    throw new EnvelopeReconstructionError(
+      {code: EnvelopeReconstructionErrorCode.BODY_ROOT_MISMATCH, slot, field},
+      `reconstructed ${field} root mismatch slot=${slot} stored=${toRootHex(stored)} rebuilt=${toRootHex(rebuilt)}`
+    );
+  }
+}

package/src/util/shuffle.ts

@@ -10,12 +10,3 @@ export function shuffle<T>(arr: T[]): T[] {
   }
   return _arr;
 }
-
-/**
- * Return one random item from array
- */
-export function shuffleOne<T>(arr: T[]): T | undefined {
-  if (arr.length === 0) return undefined;
-
-  return arr[Math.floor(Math.random() * arr.length)];
-}

package/lib/chain/archiveStore/utils/updateBackfillRange.d.ts

@@ -1,20 +0,0 @@
-import { CheckpointWithHex } from "@lodestar/fork-choice";
-import { Logger } from "@lodestar/logger";
-import { IBeaconDb } from "../../../db/interface.js";
-import { IBeaconChain } from "../../interface.js";
-/**
- * Backfill sync relies on verified connected ranges (which are represented as key,value
- * with a verified jump from a key back to value). Since the node could have progressed
- * ahead from, we need to save the forward progress of this node as another backfill
- * range entry, that backfill sync will use to jump back if this node is restarted
- * for any reason.
- * The current backfill has its own backfill entry from anchor slot to last backfilled
- * slot. And this would create the entry from the current finalized slot to the anchor
- * slot.
- */
-export declare function updateBackfillRange({ chain, db, logger }: {
-    chain: IBeaconChain;
-    db: IBeaconDb;
-    logger: Logger;
-}, finalized: CheckpointWithHex): Promise<void>;
-//# sourceMappingURL=updateBackfillRange.d.ts.map

package/lib/chain/archiveStore/utils/updateBackfillRange.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"updateBackfillRange.d.ts","sourceRoot":"","sources":["../../../../src/chain/archiveStore/utils/updateBackfillRange.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAEhD;;;;;;;;;GASG;AACH,wBAAsB,mBAAmB,CACvC,EAAC,KAAK,EAAE,EAAE,EAAE,MAAM,EAAC,EAAE;IAAC,KAAK,EAAE,YAAY,CAAC;IAAC,EAAE,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAC,EACzE,SAAS,EAAE,iBAAiB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA+Bf"}