@lodestar/beacon-node 1.49.0-dev.ee3c432d85 → 1.49.0-dev.f1b14bce8f

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRange.ts

@@ -5,8 +5,10 @@ import {GENESIS_SLOT} from "@lodestar/params";
 import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {gloas} from "@lodestar/types";
+import {EnvelopeReconstructionError} from "../../../chain/errors/index.js";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
+import {reconstructExecutionPayloadEnvelopesByRange} from "../../../util/execution.js";
 import {prettyPrintPeerId} from "../../util.js";
 
 export async function* onExecutionPayloadEnvelopesByRange(
@@ -35,58 +37,74 @@ export async function* onExecutionPayloadEnvelopesByRange(
     );
   }
 
-  const finalized = db.executionPayloadEnvelopeArchive;
   // Use the finalized block's actual slot as the checkpoint epoch-boundary slot may be skipped
   const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
   // The finalized block's envelope stays in the hot db until the next finalization run
   const archiveMaxSlot = finalizedSlot - 1;
 
-  // Finalized range of envelopes
-  if (startSlot <= archiveMaxSlot) {
-    for await (const {key, value: envelopeBytes} of finalized.binaryEntriesStream({
-      gte: startSlot,
-      lt: Math.min(endSlot, archiveMaxSlot + 1),
-    })) {
-      const slot = finalized.decodeKey(key);
-      yield {
-        data: envelopeBytes,
-        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(slot)),
-      };
+  let yielded = 0;
+  try {
+    if (startSlot <= archiveMaxSlot) {
+      for await (const {slot, envelopeBytes} of reconstructExecutionPayloadEnvelopesByRange(
+        db,
+        chain.executionEngine,
+        chain.logger,
+        chain.metrics,
+        startSlot,
+        Math.min(endSlot, archiveMaxSlot + 1)
+      )) {
+        yielded++;
+        yield {
+          data: envelopeBytes,
+          boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(slot)),
+        };
+      }
     }
-  }
 
-  // Non-finalized range of envelopes
-  if (endSlot > archiveMaxSlot) {
-    const headBlock = chain.forkChoice.getHead();
-    const headRoot = headBlock.blockRoot;
-    const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
+    // Non-finalized range of envelopes. The finalized block's own envelope may be archived by a
+    // migration that ran since archiveMaxSlot was computed, so this read can reconstruct too.
+    if (endSlot > archiveMaxSlot) {
+      const headBlock = chain.forkChoice.getHead();
+      const headRoot = headBlock.blockRoot;
+      const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
 
-    // Iterate head chain with ascending block numbers
-    for (let i = headChain.length - 1; i >= 0; i--) {
-      const block = headChain[i];
+      // Iterate head chain with ascending block numbers
+      for (let i = headChain.length - 1; i >= 0; i--) {
+        const block = headChain[i];
 
-      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
-        // Skip EMPTY blocks
-        if (block.payloadStatus !== PayloadStatus.FULL) {
-          continue;
-        }
+        if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
+          // Skip EMPTY blocks
+          if (block.payloadStatus !== PayloadStatus.FULL) {
+            continue;
+          }
 
-        const envelopeBytes = await chain.getSerializedExecutionPayloadEnvelope(block.slot, block.blockRoot);
-        if (!envelopeBytes) {
-          throw new ResponseError(
-            RespStatus.SERVER_ERROR,
-            `No envelope for root ${block.blockRoot} slot ${block.slot}, startSlot=${startSlot} endSlot=${endSlot} finalizedSlot=${finalizedSlot}`
-          );
-        }
+          const envelopeBytes = await chain.getSerializedExecutionPayloadEnvelope(block.slot, block.blockRoot);
+          if (!envelopeBytes) {
+            throw new ResponseError(
+              RespStatus.SERVER_ERROR,
+              `No envelope for root ${block.blockRoot} slot ${block.slot}, startSlot=${startSlot} endSlot=${endSlot} finalizedSlot=${finalizedSlot}`
+            );
+          }
 
-        yield {
-          data: envelopeBytes,
-          boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(block.slot)),
-        };
-      } else if (block.slot >= endSlot) {
-        break;
+          yielded++;
+          yield {
+            data: envelopeBytes,
+            boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(block.slot)),
+          };
+        } else if (block.slot >= endSlot) {
+          break;
+        }
       }
     }
+  } catch (e) {
+    if (e instanceof EnvelopeReconstructionError) {
+      // An unservable entry or our own EL failing: after some envelopes, end short (spec-legal, the
+      // peer fetches the rest elsewhere); with nothing served, RESOURCE_UNAVAILABLE rather than
+      // SERVER_ERROR so peers don't downscore us for it
+      if (yielded > 0) return;
+      throw new ResponseError(RespStatus.RESOURCE_UNAVAILABLE, `Failed to serve envelopes: ${e.message}`);
+    }
+    throw e;
   }
 }
 

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRoot.ts

@@ -1,7 +1,9 @@
 import {PeerId} from "@libp2p/interface";
-import {ResponseOutgoing} from "@lodestar/reqresp";
+import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
+import {RootHex, Slot} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
+import {EnvelopeReconstructionError} from "../../../chain/errors/index.js";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
 import {ExecutionPayloadEnvelopesByRootRequest} from "../../../util/types.js";
@@ -17,6 +19,8 @@ export async function* onExecutionPayloadEnvelopesByRoot(
   // The gloas req/resp spec uses MIN_EPOCHS_FOR_BLOCK_REQUESTS to define the minimum range peers MUST serve.
   // Archival nodes may still serve older retained payloads to allow genesis sync.
 
+  // Resolve slots first so archived header envelopes are rebuilt in one EL batch, not one call per root
+  const requests: {blockSlot: Slot; blockRootHex: RootHex}[] = [];
   for (const root of requestBody) {
     const rootHex = toRootHex(root);
     const block = chain.forkChoice.getBlockHexDefaultStatus(rootHex);
@@ -34,17 +38,35 @@ export async function* onExecutionPayloadEnvelopesByRoot(
       );
       continue;
     }
+    requests.push({blockSlot: slot, blockRootHex: rootHex});
+  }
+
+  let envelopesBytes: (Uint8Array | null)[];
+  try {
+    // by-root allows omission, so a mismatched envelope is left out rather than failing the response
+    envelopesBytes = await chain.getSerializedExecutionPayloadEnvelopes(requests, "omit");
+  } catch (e) {
+    if (e instanceof EnvelopeReconstructionError) {
+      throw new ResponseError(
+        RespStatus.RESOURCE_UNAVAILABLE,
+        `Failed to reconstruct archived envelopes: ${e.message}`
+      );
+    }
+    throw e;
+  }
 
-    const envelopeBytes = await chain.getSerializedExecutionPayloadEnvelope(slot, rootHex);
+  for (let i = 0; i < requests.length; i++) {
+    const {blockSlot, blockRootHex} = requests[i];
+    const envelopeBytes = envelopesBytes[i];
     if (envelopeBytes) {
       yield {
         data: envelopeBytes,
-        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(slot)),
+        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(blockSlot)),
       };
     } else {
       chain.logger.debug("Cannot serve ExecutionPayloadEnvelopesByRoot: envelope not found", {
-        slot,
-        root: rootHex,
+        slot: blockSlot,
+        root: blockRootHex,
         peer: prettyPrintPeerId(peerId),
         client: peerClient,
       });

package/src/network/reqresp/handlers/index.ts

@@ -1,5 +1,5 @@
 import type {Type} from "@chainsafe/ssz";
-import {ProtocolHandler, RespStatus, ResponseError} from "@lodestar/reqresp";
+import {ProtocolHandler, RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {ssz} from "@lodestar/types";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
@@ -38,6 +38,29 @@ function deserializeRequestBody<T>(type: Type<T>, data: Uint8Array): T {
   }
 }
 
+async function* trackByRangeResponse(
+  chain: IBeaconChain,
+  method: ReqRespMethod,
+  count: number,
+  source: AsyncIterable<ResponseOutgoing>,
+  columns?: number
+): AsyncIterable<ResponseOutgoing> {
+  chain.metrics?.reqRespByRange.requestedSlots.observe({method}, count);
+  if (columns !== undefined) {
+    chain.metrics?.reqRespByRange.requestedColumns.observe(columns);
+  }
+
+  let servedBytes = 0;
+  try {
+    for await (const chunk of source) {
+      servedBytes += chunk.data.length;
+      yield chunk;
+    }
+  } finally {
+    chain.metrics?.reqRespByRange.servedBytes.observe({method}, servedBytes);
+  }
+}
+
 /**
  * The ReqRespHandler module handles app-level requests / responses from other peers,
  * fetching state from the chain and database as needed.
@@ -50,7 +73,12 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
     [ReqRespMethod.Metadata]: notImplemented(ReqRespMethod.Metadata),
     [ReqRespMethod.BeaconBlocksByRange]: (req, peerId, peerClient) => {
       const body = deserializeRequestBody(ssz.phase0.BeaconBlocksByRangeRequest, req.data);
-      return onBeaconBlocksByRange(body, chain, db, peerId, peerClient);
+      return trackByRangeResponse(
+        chain,
+        ReqRespMethod.BeaconBlocksByRange,
+        body.count,
+        onBeaconBlocksByRange(body, chain, db, peerId, peerClient)
+      );
     },
     [ReqRespMethod.BeaconBlocksByRoot]: (req) => {
       const fork = chain.config.getForkName(chain.clock.currentSlot);
@@ -68,11 +96,22 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
     },
     [ReqRespMethod.BlobSidecarsByRange]: (req) => {
       const body = deserializeRequestBody(ssz.deneb.BlobSidecarsByRangeRequest, req.data);
-      return onBlobSidecarsByRange(body, chain, db);
+      return trackByRangeResponse(
+        chain,
+        ReqRespMethod.BlobSidecarsByRange,
+        body.count,
+        onBlobSidecarsByRange(body, chain, db)
+      );
     },
     [ReqRespMethod.DataColumnSidecarsByRange]: (req, peerId, peerClient) => {
       const body = deserializeRequestBody(ssz.fulu.DataColumnSidecarsByRangeRequest, req.data);
-      return onDataColumnSidecarsByRange(body, chain, db, peerId, peerClient);
+      return trackByRangeResponse(
+        chain,
+        ReqRespMethod.DataColumnSidecarsByRange,
+        body.count,
+        onDataColumnSidecarsByRange(body, chain, db, peerId, peerClient),
+        body.columns.length
+      );
     },
     [ReqRespMethod.DataColumnSidecarsByRoot]: (req, peerId, peerClient) => {
       const body = deserializeRequestBody(DataColumnSidecarsByRootRequestType(chain.config), req.data);
@@ -85,7 +124,12 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
     },
     [ReqRespMethod.ExecutionPayloadEnvelopesByRange]: (req, peerId, peerClient) => {
       const body = deserializeRequestBody(ssz.gloas.ExecutionPayloadEnvelopesByRangeRequest, req.data);
-      return onExecutionPayloadEnvelopesByRange(body, chain, db, peerId, peerClient);
+      return trackByRangeResponse(
+        chain,
+        ReqRespMethod.ExecutionPayloadEnvelopesByRange,
+        body.count,
+        onExecutionPayloadEnvelopesByRange(body, chain, db, peerId, peerClient)
+      );
     },
 
     [ReqRespMethod.LightClientBootstrap]: (req) => {

package/src/sync/types.ts

@@ -105,8 +105,6 @@ export enum DroppedItemReason {
   invalidEnvelope = "invalid_envelope",
   /** a payload had an invalid signature (INVALID_SIGNATURE) */
   invalidSignature = "invalid_signature",
-  /** a block root was given up on after exhausting download attempts (data could not be fetched) */
-  unavailable = "unavailable",
   /** an unexpected / unhandled error code led to removal */
   unknown = "unknown",
 }

package/src/sync/unknownBlock.ts

@@ -227,8 +227,11 @@ export class BlockInputSync {
   private onUnknownBlockRoot = (data: ChainEventData[ChainEvent.unknownBlockRoot]): void => {
     try {
       const isNewRoot = this.addByRootHex(data.rootHex, data.peer);
-      if (isNewRoot) {
+      // A retained entry (an earlier download failed, #10018) is not new but still needs a retry pass.
+      if (this.pendingBlocks.get(data.rootHex)?.status === PendingBlockInputStatus.pending) {
         this.triggerUnknownBlockSearch();
+      }
+      if (isNewRoot) {
         this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_BLOCK_ROOT});
         this.metrics?.blockInputSync.source.inc({source: data.source});
       }
@@ -393,12 +396,21 @@ export class BlockInputSync {
       }
     }
 
-    let deletedBlocks = 0;
+    let belowFinalizedBlocks = 0;
+    let agedOutBlocks = 0;
     for (const [rootHex, block] of this.pendingBlocks) {
       const slot = getBlockInputSyncCacheItemSlot(block);
-      if (typeof slot === "number" && slot < finalizedSlot) {
+      if (typeof slot !== "number") {
+        // we may not be able to download a root, hence there is no slot for it
+        if (nowSec - block.timeAddedSec > maxSecInCache) {
+          this.pendingBlocks.delete(rootHex);
+          agedOutBlocks++;
+        }
+        continue;
+      }
+      if (slot < finalizedSlot) {
         this.pendingBlocks.delete(rootHex);
-        deletedBlocks++;
+        belowFinalizedBlocks++;
       }
     }
 
@@ -411,16 +423,20 @@ export class BlockInputSync {
     if (agedOutPayloads > 0) {
       this.metrics?.blockInputSync.removedPayloads.inc({reason: DroppedItemReason.agedOut}, agedOutPayloads);
     }
-    if (deletedBlocks > 0) {
-      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.belowFinalized}, deletedBlocks);
+    if (belowFinalizedBlocks > 0) {
+      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.belowFinalized}, belowFinalizedBlocks);
+    }
+    if (agedOutBlocks > 0) {
+      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.agedOut}, agedOutBlocks);
     }
 
-    this.logger.debug("BlockInputSync.pruneFinalized dropped pre-finalized pending items", {
+    this.logger.debug("BlockInputSync.pruneFinalized dropped stale pending items", {
       finalizedSlot,
       finalizedRoot: checkpoint.rootHex,
       belowFinalizedPayloads,
       agedOutPayloads,
-      deletedBlocks,
+      belowFinalizedBlocks,
+      agedOutBlocks,
       unresolvedSlotPayloads,
     });
   };
@@ -973,7 +989,12 @@ export class BlockInputSync {
     if (!res.err) {
       this.metrics?.blockInputSync.downloadedBlocksSuccess.inc();
       const pending = res.result;
-      this.pendingBlocks.set(pending.blockInput.blockRootHex, pending);
+      // pruning may have deleted this entry while we were awaiting the network, do not resurrect it via the set below
+      if (!this.pendingBlocks.has(rootHex)) {
+        this.logger.verbose("Dropping downloaded block, entry pruned during fetch", logCtx);
+        return;
+      }
+      this.pendingBlocks.set(rootHex, pending);
       const blockSlot = pending.blockInput.slot;
       const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
       const delaySec = Date.now() / 1000 - computeTimeAtSlot(this.config, blockSlot, this.chain.genesisTime);
@@ -1035,8 +1056,11 @@ export class BlockInputSync {
       }
 
       this.metrics?.blockInputSync.downloadedBlocksError.inc();
-      this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
-      this.removeAndDownScoreAllDescendants(block, DroppedItemReason.unavailable);
+      this.logger.debug("Failed to download unknown block root, retained for retry", logCtx, res.err);
+      const pendingBlock = this.pendingBlocks.get(rootHex);
+      if (pendingBlock && pendingBlock.status === PendingBlockInputStatus.fetching) {
+        pendingBlock.status = PendingBlockInputStatus.pending;
+      }
     }
   }
 
@@ -1303,7 +1327,7 @@ export class BlockInputSync {
     }
 
     this.metrics?.blockInputSync.downloadedPayloadsResult.inc({result: DownloadResult.Failed});
-    this.logger.debug("Ignoring unknown payload root after failed download", logCtx, res.err);
+    this.logger.debug("Failed to download unknown payload root, retained for retry", logCtx, res.err);
     if (!isPendingPayloadEnvelope(payload)) {
       payload.status = PendingPayloadInputStatus.pending;
     }
@@ -1763,7 +1787,11 @@ export class BlockInputSync {
         this.peerBalancer.onRequestCompleted(peerId);
       }
 
-      this.pendingBlocks.set(getBlockInputSyncCacheItemRootHex(cacheItem), cacheItem);
+      const cacheItemRootHex = getBlockInputSyncCacheItemRootHex(cacheItem);
+      // pruning may have dropped it mid-fetch
+      if (this.pendingBlocks.has(cacheItemRootHex)) {
+        this.pendingBlocks.set(cacheItemRootHex, cacheItem);
+      }
 
       if (cacheItem.status === PendingBlockInputStatus.downloaded) {
         // download was successful, no need to go with another peer, return

package/src/util/execution.ts

@@ -2,12 +2,18 @@ import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkPostFulu, ForkPreFulu} from "@lodestar/params";
 import {signedBlockToSignedHeader} from "@lodestar/state-transition";
-import {DataColumnSidecar, SignedBeaconBlock, deneb, isGloasDataColumnSidecar} from "@lodestar/types";
-import {fromHex, toHex} from "@lodestar/utils";
+import {DataColumnSidecar, SignedBeaconBlock, Slot, deneb, gloas, isGloasDataColumnSidecar, ssz} from "@lodestar/types";
+import {Logger, fromHex, toHex, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../chain/blocks/blockInput/types.js";
 import {PayloadEnvelopeInput, PayloadEnvelopeInputSource} from "../chain/blocks/payloadEnvelopeInput/index.js";
 import {ChainEvent, ChainEventEmitter} from "../chain/emitter.js";
+import {
+  EnvelopeReconstructionError,
+  EnvelopeReconstructionErrorCode,
+} from "../chain/errors/envelopeReconstructionError.js";
+import {IBeaconDb} from "../db/index.js";
+import {ArchivedEnvelope, decodeArchivedEnvelope} from "../db/repositories/index.js";
 import {IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/index.js";
 import {computePreFuluKzgCommitmentsInclusionProof} from "./blobs.js";
@@ -17,6 +23,7 @@ import {
   getDataColumnSidecarsFromColumnSidecar,
   getGloasDataColumnSidecars,
 } from "./dataColumns.js";
+import {signedHeaderEnvelopeToFull} from "./headerEnvelope.js";
 
 export enum DataColumnEngineResult {
   PreFulu = "pre_fulu",
@@ -251,3 +258,169 @@ export async function getDataColumnSidecarsFromExecution(
   );
   return DataColumnEngineResult.SuccessResolved;
 }
+
+/** engine_getPayloadBodiesByHashV2: ELs MUST support at least 32 hashes per request. */
+const MAX_BODIES_REQUEST = 32;
+
+type SlotEnvelopeBytes = {slot: Slot; envelopeBytes: Uint8Array};
+
+type RangeEntry = ArchivedEnvelope & {slot: Slot};
+
+/** What a body root mismatch means on a given serving path */
+export type ReconstructMismatchPolicy = "throw" | "omit";
+
+export type RebuildMiss =
+  /** EL does not have the block, or has pruned its block access list */
+  | {slot: Slot; reason: "unavailable"}
+  /** An EL body does not hash to its stored root (local inconsistency) */
+  | {slot: Slot; reason: "mismatch"; error: EnvelopeReconstructionError};
+
+/**
+ * Stream finalized envelopes over [startSlot, endSlot) as serialized bytes, rebuilding header
+ * entries from EL bodies 32 per round-trip.
+ *
+ * Ends at the first entry that cannot be served by throwing RANGE_UNSERVABLE with that slot: the
+ * by-range spec inherits BeaconBlocksByRange v2 semantics (consecutive, MAY be short), and a hole
+ * looks like a lying peer to one that already holds the blocks. Entries are attempted regardless of
+ * age; the EL's block access list retention is the floor, not MIN_EPOCHS_FOR_BLOCK_REQUESTS.
+ *
+ * Throws ENGINE_UNAVAILABLE if the EL call fails. Either may surface after envelopes were yielded.
+ */
+export async function* reconstructExecutionPayloadEnvelopesByRange(
+  db: IBeaconDb,
+  executionEngine: IExecutionEngine,
+  logger: Logger,
+  metrics: Metrics | null,
+  startSlot: Slot,
+  endSlot: Slot
+): AsyncIterable<SlotEnvelopeBytes> {
+  const archive = db.executionPayloadEnvelopeArchive;
+  let batch: RangeEntry[] = [];
+
+  for await (const {key, value: bytes} of archive.binaryEntriesStream({gte: startSlot, lt: endSlot})) {
+    batch.push({slot: archive.decodeKey(key), ...decodeArchivedEnvelope(bytes)});
+    if (batch.length === MAX_BODIES_REQUEST) {
+      yield* reconstructBatch(executionEngine, logger, metrics, batch);
+      batch = [];
+    }
+  }
+  if (batch.length > 0) {
+    yield* reconstructBatch(executionEngine, logger, metrics, batch);
+  }
+}
+
+/**
+ * Rebuild a range batch, yielding what precedes the first unservable entry before throwing
+ * RANGE_UNSERVABLE for it
+ */
+async function* reconstructBatch(
+  executionEngine: IExecutionEngine,
+  logger: Logger,
+  metrics: Metrics | null,
+  batch: RangeEntry[]
+): AsyncIterable<SlotEnvelopeBytes> {
+  const headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[] = [];
+  for (const entry of batch) {
+    if (entry.headerEnvelope !== undefined) headerEnvelopes.push(entry.headerEnvelope);
+  }
+  const rebuilt = await reconstructEnvelopesBatch(executionEngine, metrics, headerEnvelopes);
+
+  let rebuiltIdx = 0;
+  for (const entry of batch) {
+    if (entry.envelopeBytes !== undefined) {
+      yield {slot: entry.slot, envelopeBytes: entry.envelopeBytes};
+      continue;
+    }
+    const result = rebuilt[rebuiltIdx++];
+    if (isRebuildMiss(result)) {
+      // Peer-triggered, so debug: a persistent local mismatch would otherwise log on every request
+      if (result.reason === "mismatch") {
+        logger.debug("Archived envelope failed body root check against EL bodies", {slot: entry.slot}, result.error);
+      } else {
+        logger.debug("EL cannot serve bodies for archived envelope, ending range", {slot: entry.slot});
+      }
+      throw new EnvelopeReconstructionError(
+        {code: EnvelopeReconstructionErrorCode.RANGE_UNSERVABLE, slot: entry.slot},
+        `archived envelope range unservable from slot=${entry.slot}`
+      );
+    }
+    yield {slot: entry.slot, envelopeBytes: ssz.gloas.SignedExecutionPayloadEnvelope.serialize(result)};
+  }
+}
+
+/**
+ * Rebuild header envelopes from EL bodies, 32 per round-trip. Aligned with the input, with a
+ * {@link RebuildMiss} where the envelope could not be rebuilt; the caller decides what a miss means
+ * on its path. Throws ENGINE_UNAVAILABLE only.
+ */
+export async function reconstructExecutionPayloadEnvelopes(
+  executionEngine: IExecutionEngine,
+  metrics: Metrics | null,
+  headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[]
+): Promise<(gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[]> {
+  const out: (gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[] = [];
+  for (let i = 0; i < headerEnvelopes.length; i += MAX_BODIES_REQUEST) {
+    out.push(
+      ...(await reconstructEnvelopesBatch(executionEngine, metrics, headerEnvelopes.slice(i, i + MAX_BODIES_REQUEST)))
+    );
+  }
+  return out;
+}
+
+export function isRebuildMiss(result: gloas.SignedExecutionPayloadEnvelope | RebuildMiss): result is RebuildMiss {
+  return "reason" in result;
+}
+
+/**
+ * One EL round-trip. Aligned with the input; never throws per envelope, only ENGINE_UNAVAILABLE.
+ * Every serving path (by-range, by-root, REST) comes through here, so the outcome metrics are
+ * incremented once, in this function.
+ */
+async function reconstructEnvelopesBatch(
+  executionEngine: IExecutionEngine,
+  metrics: Metrics | null,
+  headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[]
+): Promise<(gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[]> {
+  if (headerEnvelopes.length === 0) return [];
+  const hashes = headerEnvelopes.map((headerEnvelope) => toRootHex(headerEnvelope.message.payloadHeader.blockHash));
+
+  let bodies: Awaited<ReturnType<IExecutionEngine["getPayloadBodiesByHashV2"]>>;
+  try {
+    bodies = await executionEngine.getPayloadBodiesByHashV2(hashes);
+  } catch (e) {
+    metrics?.payloadEnvelopeReconstruction.engineErrors.inc();
+    throw new EnvelopeReconstructionError(
+      {code: EnvelopeReconstructionErrorCode.ENGINE_UNAVAILABLE},
+      `engine_getPayloadBodiesByHashV2 failed: ${(e as Error).message}`
+    );
+  }
+
+  return headerEnvelopes.map((headerEnvelope, i) => {
+    const slot = headerEnvelope.message.payloadHeader.slotNumber;
+    const body = bodies[i];
+    // A zero-length block access list cannot be valid, RLP encodes an empty list as 0xc0
+    if (body == null || body.withdrawals == null || body.blockAccessList == null || body.blockAccessList.length === 0) {
+      metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "unavailable"});
+      return {slot, reason: "unavailable"};
+    }
+    try {
+      const envelope = signedHeaderEnvelopeToFull(headerEnvelope, {
+        transactions: body.transactions,
+        withdrawals: body.withdrawals,
+        blockAccessList: body.blockAccessList,
+      });
+      metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "ok"});
+      return envelope;
+    } catch (e) {
+      if (
+        e instanceof EnvelopeReconstructionError &&
+        e.type.code === EnvelopeReconstructionErrorCode.BODY_ROOT_MISMATCH
+      ) {
+        metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "mismatch"});
+        metrics?.payloadEnvelopeReconstruction.mismatchByField.inc({field: e.type.field});
+        return {slot, reason: "mismatch", error: e};
+      }
+      throw e;
+    }
+  });
+}

package/src/util/headerEnvelope.ts

@@ -0,0 +1,76 @@
+import {ssz} from "@lodestar/types";
+import {
+  BlockAccessList,
+  ExecutionPayloadEnvelope,
+  ExecutionPayloadHeaderEnvelope,
+  SignedExecutionPayloadEnvelope,
+  SignedExecutionPayloadHeaderEnvelope,
+  Transactions,
+  Withdrawals,
+} from "@lodestar/types/gloas";
+import {toRootHex} from "@lodestar/utils";
+import {
+  EnvelopeReconstructionError,
+  EnvelopeReconstructionErrorCode,
+} from "../chain/errors/envelopeReconstructionError.js";
+
+/** The parts of an execution payload the EL serves back via engine_getPayloadBodiesByHashV2 */
+export type ExecutionPayloadBodies = {
+  transactions: Transactions;
+  withdrawals: Withdrawals;
+  blockAccessList: BlockAccessList;
+};
+
+export type ExecutionPayloadBodyField = keyof ExecutionPayloadBodies;
+
+const {transactions, withdrawals, blockAccessList} = ssz.gloas.ExecutionPayload.fields;
+
+/** Replace the three body lists by their roots; hashes to the same root as the full envelope */
+export function toHeaderEnvelope(envelope: ExecutionPayloadEnvelope): ExecutionPayloadHeaderEnvelope {
+  const {payload, ...envelopeFields} = envelope;
+  const {transactions: txs, withdrawals: wds, blockAccessList: bal, ...scalars} = payload;
+  return {
+    ...envelopeFields,
+    payloadHeader: {
+      ...scalars,
+      transactionsRoot: transactions.hashTreeRoot(txs),
+      withdrawalsRoot: withdrawals.hashTreeRoot(wds),
+      blockAccessListRoot: blockAccessList.hashTreeRoot(bal),
+    },
+  };
+}
+
+export function toSignedHeaderEnvelope(envelope: SignedExecutionPayloadEnvelope): SignedExecutionPayloadHeaderEnvelope {
+  return {message: toHeaderEnvelope(envelope.message), signature: envelope.signature};
+}
+
+/**
+ * Rebuild the full signed envelope from header envelope + EL bodies, each body verified against its stored
+ * root. Throws {@link EnvelopeReconstructionError} BODY_ROOT_MISMATCH naming the first body that differs.
+ */
+export function signedHeaderEnvelopeToFull(
+  headerEnvelope: SignedExecutionPayloadHeaderEnvelope,
+  bodies: ExecutionPayloadBodies
+): SignedExecutionPayloadEnvelope {
+  const {payloadHeader, ...envelopeFields} = headerEnvelope.message;
+  const {transactionsRoot, withdrawalsRoot, blockAccessListRoot, ...scalars} = payloadHeader;
+  const slot = scalars.slotNumber;
+
+  assertBodyRoot("transactions", slot, transactionsRoot, transactions.hashTreeRoot(bodies.transactions));
+  assertBodyRoot("withdrawals", slot, withdrawalsRoot, withdrawals.hashTreeRoot(bodies.withdrawals));
+  assertBodyRoot("blockAccessList", slot, blockAccessListRoot, blockAccessList.hashTreeRoot(bodies.blockAccessList));
+
+  return {
+    message: {...envelopeFields, payload: {...scalars, ...bodies}},
+    signature: headerEnvelope.signature,
+  };
+}
+
+function assertBodyRoot(field: ExecutionPayloadBodyField, slot: number, stored: Uint8Array, rebuilt: Uint8Array): void {
+  if (!ssz.Root.equals(stored, rebuilt)) {
+    throw new EnvelopeReconstructionError(
+      {code: EnvelopeReconstructionErrorCode.BODY_ROOT_MISMATCH, slot, field},
+      `reconstructed ${field} root mismatch slot=${slot} stored=${toRootHex(stored)} rebuilt=${toRootHex(rebuilt)}`
+    );
+  }
+}