@lodestar/beacon-node 1.50.0-dev.cc4428df4b → 1.50.0-dev.cddd01a6c6

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRoot.ts

@@ -6,6 +6,7 @@ import {toRootHex} from "@lodestar/utils";
 import {EnvelopeReconstructionError} from "../../../chain/errors/index.js";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
+import {MAX_BODIES_PER_REQUEST} from "../../../util/execution.js";
 import {ExecutionPayloadEnvelopesByRootRequest} from "../../../util/types.js";
 import {prettyPrintPeerId} from "../../util.js";
 
@@ -19,10 +20,15 @@ export async function* onExecutionPayloadEnvelopesByRoot(
   // The gloas req/resp spec uses MIN_EPOCHS_FOR_BLOCK_REQUESTS to define the minimum range peers MUST serve.
   // Archival nodes may still serve older retained payloads to allow genesis sync.
 
-  // Resolve slots first so archived header envelopes are rebuilt in one EL batch, not one call per root
+  // Resolve slots first so archived envelopes can be reconstructed in EL batches.
+  // Duplicate roots are served once, since each archived envelope costs an EL fetch and a rebuild.
   const requests: {blockSlot: Slot; blockRootHex: RootHex}[] = [];
+  const seenRoots = new Set<RootHex>();
   for (const root of requestBody) {
     const rootHex = toRootHex(root);
+    if (seenRoots.has(rootHex)) continue;
+    seenRoots.add(rootHex);
+
     const block = chain.forkChoice.getBlockHexDefaultStatus(rootHex);
     // If the block is not in fork choice, it may be finalized. Attempt to find its slot in block archive
     const slot = block ? block.slot : await db.blockArchive.getSlotByRoot(root);
@@ -41,35 +47,43 @@ export async function* onExecutionPayloadEnvelopesByRoot(
     requests.push({blockSlot: slot, blockRootHex: rootHex});
   }
 
-  let envelopesBytes: (Uint8Array | null)[];
-  try {
-    // by-root allows omission, so a mismatched envelope is left out rather than failing the response
-    envelopesBytes = await chain.getSerializedExecutionPayloadEnvelopes(requests, "omit");
-  } catch (e) {
-    if (e instanceof EnvelopeReconstructionError) {
-      throw new ResponseError(
-        RespStatus.RESOURCE_UNAVAILABLE,
-        `Failed to reconstruct archived envelopes: ${e.message}`
-      );
+  // Yield between EL batches so the first response does not wait for the entire request.
+  let yielded = 0;
+  for (let i = 0; i < requests.length; i += MAX_BODIES_PER_REQUEST) {
+    const batch = requests.slice(i, i + MAX_BODIES_PER_REQUEST);
+    let envelopesBytes: (Uint8Array | null)[];
+    try {
+      // By-root allows omission, so an envelope that cannot be rebuilt is left out rather than failing the response.
+      envelopesBytes = await chain.getSerializedExecutionPayloadEnvelopes(batch, "omit");
+    } catch (e) {
+      if (e instanceof EnvelopeReconstructionError) {
+        // By-root permits partial responses, so an EL failure can end the response here.
+        if (yielded > 0) return;
+        throw new ResponseError(
+          RespStatus.RESOURCE_UNAVAILABLE,
+          `Failed to reconstruct archived envelopes: ${e.message}`
+        );
+      }
+      throw e;
     }
-    throw e;
-  }
 
-  for (let i = 0; i < requests.length; i++) {
-    const {blockSlot, blockRootHex} = requests[i];
-    const envelopeBytes = envelopesBytes[i];
-    if (envelopeBytes) {
-      yield {
-        data: envelopeBytes,
-        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(blockSlot)),
-      };
-    } else {
-      chain.logger.debug("Cannot serve ExecutionPayloadEnvelopesByRoot: envelope not found", {
-        slot: blockSlot,
-        root: blockRootHex,
-        peer: prettyPrintPeerId(peerId),
-        client: peerClient,
-      });
+    for (let j = 0; j < batch.length; j++) {
+      const {blockSlot, blockRootHex} = batch[j];
+      const envelopeBytes = envelopesBytes[j];
+      if (envelopeBytes) {
+        yielded++;
+        yield {
+          data: envelopeBytes,
+          boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(blockSlot)),
+        };
+      } else {
+        chain.logger.debug("Cannot serve ExecutionPayloadEnvelopesByRoot: envelope not found", {
+          slot: blockSlot,
+          root: blockRootHex,
+          peer: prettyPrintPeerId(peerId),
+          client: peerClient,
+        });
+      }
     }
   }
 }

package/src/sync/range/batch.ts

@@ -773,6 +773,7 @@ export class Batch {
     // remove any downloaded blocks and re-attempt
     // TODO(fulu): need to remove the bad blocks from the SeenBlockInputCache
     this.state = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
+    this.resetRequests();
   }
 
   private onProcessingError(attempt: FailedAttempt): void {
@@ -784,6 +785,16 @@ export class Batch {
     // remove any downloaded blocks and re-attempt
     // TODO(fulu): need to remove the bad blocks from the SeenBlockInputCache
     this.state = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
+    this.resetRequests();
+  }
+
+  /**
+   * After a partial download `requests` only cover the data still missing from the downloaded blocks,
+   * e.g. a single column. Once those blocks are dropped, request the whole batch again, otherwise
+   * every peer fails the retry for lack of blocks to validate the data against.
+   */
+  private resetRequests(): void {
+    this.requests = this.getRequests([]);
   }
 
   /** Helper to construct typed BatchError. Stack traces are correct as the error is thrown above */

package/src/sync/range/range.ts

@@ -5,6 +5,7 @@ import {IBeaconStateViewGloas, computeStartSlotAtEpoch, isStatePostGloas} from "
 import {Epoch, Status, fulu} from "@lodestar/types";
 import {Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
+import {PayloadError, PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
 import {AttestationImportOpt, ImportBlockOpts} from "../../chain/blocks/index.js";
 import {assertLinearChainSegment} from "../../chain/blocks/utils/chainSegment.js";
 import {BlockError} from "../../chain/errors/index.js";
@@ -207,22 +208,50 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
       blsVerifyOnMainThread: false,
     };
 
-    if (this.opts?.disableProcessAsChainSegment) {
-      // Should only be used for debugging or testing
-      for (const block of blocks) {
-        await this.chain.processBlock(block, flags);
-        const payloadEnvelope = payloadEnvelopes?.get(block.slot);
-        if (payloadEnvelope?.hasPayloadEnvelope()) {
-          await this.chain.processExecutionPayload(payloadEnvelope);
+    try {
+      if (this.opts?.disableProcessAsChainSegment) {
+        // Should only be used for debugging or testing
+        for (const block of blocks) {
+          await this.chain.processBlock(block, flags);
+          const payloadEnvelope = payloadEnvelopes?.get(block.slot);
+          if (payloadEnvelope?.hasPayloadEnvelope()) {
+            await this.chain.processExecutionPayload(payloadEnvelope);
+          }
+        }
+      } else {
+        const {orphaned, skipped} = await this.chain.processChainSegment(blocks, payloadEnvelopes, flags);
+        // We log orphaned payloads to work with different clients
+        // in the future, consider applying penalties in certain conditions
+        // make sure the payload source was from range sync in that case
+        for (const {slot, payloadEnvelopeInput} of orphaned) {
+          const {peerIdStr, source} = payloadEnvelopeInput.getPayloadEnvelopeSource();
+          let client = "unknown";
+          if (peerIdStr !== undefined) {
+            try {
+              client = this.getConnectedPeerSyncMeta(peerIdStr).client;
+            } catch {
+              // peer disconnected since serving the envelope, keep "unknown"
+            }
+          }
+          this.logger.debug("Orphaned payload envelope in range sync batch", {
+            slot,
+            root: payloadEnvelopeInput.blockRootHex,
+            peer: peerIdStr ?? "unknown",
+            source,
+            client,
+            skipped,
+          });
         }
       }
-    } else {
-      const {orphaned, skipped} = await this.chain.processChainSegment(blocks, payloadEnvelopes, flags);
-      // We log orphaned payloads to work with different clients
-      // in the future, consider applying penalties in certain conditions
-      // make sure the payload source was from range sync in that case
-      for (const {slot, payloadEnvelopeInput} of orphaned) {
-        const {peerIdStr, source} = payloadEnvelopeInput.getPayloadEnvelopeSource();
+    } catch (e) {
+      if (
+        e instanceof PayloadError &&
+        (e.type.code === PayloadErrorCode.INVALID_SIGNATURE ||
+          e.type.code === PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR ||
+          e.type.code === PayloadErrorCode.EXECUTION_ENGINE_INVALID)
+      ) {
+        const {payloadInput} = e;
+        const {peerIdStr, source} = payloadInput.getPayloadEnvelopeSource();
         let client = "unknown";
         if (peerIdStr !== undefined) {
           try {
@@ -231,15 +260,18 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
             // peer disconnected since serving the envelope, keep "unknown"
           }
         }
-        this.logger.debug("Orphaned payload envelope in range sync batch", {
-          slot,
-          root: payloadEnvelopeInput.blockRootHex,
+        this.logger.debug("Removing invalid payload envelope in range sync batch", {
+          slot: payloadInput.slot,
+          root: payloadInput.blockRootHex,
           peer: peerIdStr ?? "unknown",
           source,
           client,
-          skipped,
+          code: e.type.code,
         });
+        // The batch is downloaded again, it would otherwise reuse the cached envelope and fail the same way
+        this.chain.seenPayloadEnvelopeInputCache.removeInvalid(payloadInput);
       }
+      throw e;
     }
   };
 

package/src/sync/unknownBlock.ts

@@ -15,7 +15,7 @@ import {ChainEvent, ChainEventData, IBeaconChain} from "../chain/index.js";
 import {validateGloasBlockDataColumnSidecars} from "../chain/validation/dataColumnSidecar.js";
 import {validateGossipExecutionPayloadEnvelope} from "../chain/validation/executionPayloadEnvelope.js";
 import {Metrics} from "../metrics/index.js";
-import {INetwork, NetworkEvent, NetworkEventData, prettyPrintPeerIdStr} from "../network/index.js";
+import {INetwork, NetworkEvent, NetworkEventData, PeerAction, prettyPrintPeerIdStr} from "../network/index.js";
 import {PeerSyncMeta} from "../network/peers/peersData.js";
 import {MAX_PEERS_PER_ROOT} from "../network/processor/constants.js";
 import {ClockEvent} from "../util/clock.js";
@@ -442,6 +442,11 @@ export class BlockInputSync {
   };
 
   private addByRootHex = (rootHex: RootHex, peerIdStr?: PeerIdStr): boolean => {
+    if (this.knownBadBlocks.has(rootHex)) {
+      this.logger.debug("Ignoring known bad block root", {root: rootHex, peerIdStr: peerIdStr ?? "unknown peer"});
+      return false;
+    }
+
     let pendingBlock = this.pendingBlocks.get(rootHex);
     let added = false;
     if (!pendingBlock) {
@@ -477,6 +482,11 @@ export class BlockInputSync {
   };
 
   private addByBlockInput = (blockInput: IBlockInput, peerIdStr?: string): void => {
+    if (this.knownBadBlocks.has(blockInput.blockRootHex)) {
+      this.logger.debug("Ignoring known bad block", {slot: blockInput.slot, root: blockInput.blockRootHex, peerIdStr});
+      return;
+    }
+
     let pendingBlock = this.pendingBlocks.get(blockInput.blockRootHex);
     // if entry is missing or was added via rootHex and now we have more complete information overwrite
     // the existing information with the more complete cache entry
@@ -513,6 +523,15 @@ export class BlockInputSync {
   };
 
   private addByPayloadRootHex = (rootHex: RootHex, peerIdStr?: PeerIdStr, slot?: Slot): boolean => {
+    if (this.knownBadBlocks.has(rootHex)) {
+      this.logger.debug("Ignoring payload for known bad block root", {
+        slot: slot ?? "unknown",
+        root: rootHex,
+        peerIdStr,
+      });
+      return false;
+    }
+
     let pendingPayload = this.pendingPayloads.get(rootHex);
     let added = false;
     if (!pendingPayload) {
@@ -581,6 +600,15 @@ export class BlockInputSync {
     peerIdStr?: PeerIdStr,
     envelope?: gloas.SignedExecutionPayloadEnvelope
   ): void => {
+    if (this.knownBadBlocks.has(payloadInput.blockRootHex)) {
+      this.logger.debug("Ignoring payload input for known bad block root", {
+        slot: payloadInput.slot,
+        root: payloadInput.blockRootHex,
+        peerIdStr,
+      });
+      return;
+    }
+
     const pendingPayload = this.toPendingPayloadInput(
       payloadInput,
       this.pendingPayloads.get(payloadInput.blockRootHex),
@@ -654,6 +682,7 @@ export class BlockInputSync {
       return {kind: "ready"};
     }
 
+    // The parent's payload is known but does not match `parentBlockHashHex`
     if (this.chain.forkChoice.hasPayloadHexUnsafe(parentRootHex)) {
       return {kind: "invalidParentPayload", parentRootHex, parentBlockHashHex};
     }
@@ -1032,7 +1061,8 @@ export class BlockInputSync {
         }
       } else if (blockSlot <= finalizedSlot) {
         // the common ancestor of the downloading chain and canonical chain should be at least the finalized slot and
-        // we should found it through forkchoice. If not, we should penalize all peers sending us this block chain
+        // we should found it through forkchoice. If not, drop the chain without penalizing peers because pending entries are
+        // block retained on failed download, so finalization can pass the block while it is still pending
         // 0 - 1 - ... - n - finalizedSlot
         //                \
         //                parent 1 - parent 2 - ... - unknownParent block
@@ -1040,7 +1070,7 @@ export class BlockInputSync {
           ...logCtx2,
           finalizedSlot,
         });
-        this.removeAndDownScoreAllDescendants(block, DroppedItemReason.belowFinalized);
+        this.removeAllDescendants(block, DroppedItemReason.belowFinalized);
       } else {
         this.onUnknownBlockRoot({rootHex: pending.blockInput.parentRootHex, source: BlockInputSource.byRoot});
       }
@@ -1172,15 +1202,18 @@ export class BlockInputSync {
             break;
 
           case BlockErrorCode.EXECUTION_ENGINE_INVALID:
-            // the peer served a bad block
+            // Ban but do not downscore: optimistic-sync peers may forward and serve blocks whose payload is later
+            // rejected by the execution engine. Per the spec, an INVALID execution payload SHOULD NOT cause a peer
+            // to be down-scored or disconnected, only consensus-layer invalidity MAY.
+            // https://github.com/ethereum/consensus-specs/blob/master/specs/bellatrix/p2p-interface.md#the-reqresp-domain
             this.logger.debug("Execution engine rejected block from unknown parent sync", errorData, res.err);
-            this.removeAndDownScoreAllDescendants(pendingBlock, DroppedItemReason.elInvalid);
+            this.removeAndBanAllDescendants(pendingBlock, DroppedItemReason.elInvalid);
             break;
 
           default:
             // Block is not correct with respect to our chain. Log error loudly
             this.logger.debug("Error processing block from unknown parent sync", errorData, res.err);
-            this.removeAndDownScoreAllDescendants(pendingBlock, DroppedItemReason.invalidBlock);
+            this.removeAllDescendants(pendingBlock, DroppedItemReason.invalidBlock);
         }
       }
 
@@ -1393,20 +1426,23 @@ export class BlockInputSync {
           pendingPayload.status = PendingPayloadInputStatus.downloaded;
           break;
 
+        // The invalid envelope is evicted from the seen cache in the cases below, otherwise the next
+        // attempt would reuse it instead of fetching another envelope from peers
         case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
           this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+          this.chain.seenPayloadEnvelopeInputCache.removeInvalid(pendingPayload.payloadInput);
           this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.elInvalid);
           break;
 
         case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
           this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+          this.chain.seenPayloadEnvelopeInputCache.removeInvalid(pendingPayload.payloadInput);
           this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.invalidEnvelope);
           break;
 
         case PayloadErrorCode.INVALID_SIGNATURE:
-          // TODO GLOAS: Decide how invalid payload inputs should eventually leave memory without
-          // reintroducing envelope replacement / recreation flows.
           this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
+          this.chain.seenPayloadEnvelopeInputCache.removeInvalid(pendingPayload.payloadInput);
           this.removePendingPayloadAndDescendants(rootHex, DroppedItemReason.invalidSignature);
           break;
 
@@ -1478,11 +1514,11 @@ export class BlockInputSync {
       }
 
       const {peerId, client: peerClient} = peerMeta;
-      cacheItem.peerIdStrings.add(peerId);
 
       try {
         if (!envelope) {
           envelope = await this.fetchExecutionPayloadEnvelope(peerId, blockRoot, rootHex);
+          cacheItem.peerIdStrings.add(peerId);
           const slotWasUnknown = typeof slot !== "number";
           slot = envelope.message.payload.slotNumber;
           if (slotWasUnknown) {
@@ -1524,6 +1560,8 @@ export class BlockInputSync {
           const missing = pendingPayload.payloadInput.getMissingSampledColumnMeta().missing;
           if (missing.length > 0) {
             const columnSidecars = await this.fetchPayloadColumns(peerMeta, pendingPayload.payloadInput, missing);
+            // toPendingPayloadInput copied cacheItem's set, so add to the object that is carried forward
+            pendingPayload.peerIdStrings.add(peerId);
             const seenTimestampSec = Date.now() / 1000;
             for (const columnSidecar of columnSidecars) {
               if (pendingPayload.payloadInput.hasColumn(columnSidecar.index)) {
@@ -1716,8 +1754,6 @@ export class BlockInputSync {
       }
       const {peerId, client: peerClient} = peerMeta;
 
-      cacheItem.peerIdStrings.add(peerId);
-
       try {
         const downloadResult = await downloadByRoot({
           config: this.config,
@@ -1728,6 +1764,7 @@ export class BlockInputSync {
           cacheItem,
         });
         cacheItem = downloadResult.result;
+        cacheItem.peerIdStrings.add(peerId);
         if (slot === undefined) {
           slot = cacheItem.blockInput.slot;
           // we were not able to observe the time into slot when starting the fetch, do it now
@@ -1843,44 +1880,8 @@ export class BlockInputSync {
     throw Error(message);
   }
 
-  /**
-   * Gets all descendant blocks of `block` recursively from `pendingBlocks`.
-   * Assumes that if a parent block does not exist or is not processable, all descendant blocks are bad too.
-   * Downscore all peers that have referenced any of this bad blocks. May report peers multiple times if they have
-   * referenced more than one bad block.
-   */
-  private removeAndDownScoreAllDescendants(block: BlockInputSyncCacheItem, headReason: DroppedItemReason): void {
-    // Get all blocks that are a descendant of this one
-    const badPendingBlocks = this.removeAllDescendants(block, headReason, DroppedItemReason.invalidParent);
-    // just console log and do not penalize on pending/bad blocks for debugging
-    // console.log("removeAndDownscoreAllDescendants", {block});
-
-    for (const block of badPendingBlocks) {
-      //
-      // TODO(fulu): why is this commented out here?
-      //
-      //   this.knownBadBlocks.add(block.blockRootHex);
-      //   for (const peerIdStr of block.peerIdStrings) {
-      //     // TODO: Refactor peerRpcScores to work with peerIdStr only
-      //     this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadBlockByRoot");
-      //   }
-      this.logger.debug("ignored Banning unknown block", {
-        slot: getBlockInputSyncCacheItemSlot(block),
-        root: getBlockInputSyncCacheItemRootHex(block),
-        peerIdStrings: Array.from(block.peerIdStrings)
-          .map((id) => prettyPrintPeerIdStr(id))
-          .join(","),
-      });
-    }
-
-    // Prune knownBadBlocks
-    pruneSetToMax(this.knownBadBlocks, MAX_KNOWN_BAD_BLOCKS);
-  }
-
   // Once a parent payload is invalid, every descendant waiting on that payload lineage becomes unrecoverable too.
   private removePendingPayloadAndDescendants(rootHex: RootHex, headReason: DroppedItemReason): void {
-    // Keep PayloadEnvelopeInput resident in the seen cache. importBlock() owns that object and
-    // later validation/finalization logic decides when it can leave memory.
     if (this.pendingPayloads.delete(rootHex)) {
       this.metrics?.blockInputSync.removedPayloads.inc({reason: headReason}, 1);
     }
@@ -1903,13 +1904,61 @@ export class BlockInputSync {
     }
   }
 
+  /**
+   * Same as `removeAndBanAllDescendants`, and downscores every peer that referenced or served a banned item, once per
+   * peer. Only call for consensus-layer invalidity, never for an INVALID execution payload:
+   * https://github.com/ethereum/consensus-specs/blob/master/specs/bellatrix/p2p-interface.md#the-reqresp-domain
+   */
+  private removeAndDownScoreAllDescendants(block: BlockInputSyncCacheItem, headReason: DroppedItemReason): void {
+    const peersToReport = new Set<PeerIdStr>();
+    for (const bannedBlock of this.removeAndBanAllDescendants(block, headReason)) {
+      for (const peerIdStr of bannedBlock.peerIdStrings) {
+        peersToReport.add(peerIdStr);
+      }
+    }
+
+    for (const peerIdStr of peersToReport) {
+      this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, headReason);
+    }
+  }
+
+  /**
+   * Removes `block` and all its pending descendants and bans their roots in `knownBadBlocks`. Only call when the chain
+   * is provably bad: a banned root is never searched for again. Returns the banned items.
+   */
+  private removeAndBanAllDescendants(
+    block: BlockInputSyncCacheItem,
+    headReason: DroppedItemReason
+  ): BlockInputSyncCacheItem[] {
+    const bannedBlocks: BlockInputSyncCacheItem[] = [];
+    for (const badBlock of this.removeAllDescendants(block, headReason, DroppedItemReason.invalidParent)) {
+      const rootHex = getBlockInputSyncCacheItemRootHex(badBlock);
+      // a gossip import may race a failing sync path, never ban a root fork choice has imported
+      if (this.chain.forkChoice.hasBlockHex(rootHex)) {
+        continue;
+      }
+      this.knownBadBlocks.add(rootHex);
+      bannedBlocks.push(badBlock);
+      this.logger.debug("Banning unknown block", {
+        slot: getBlockInputSyncCacheItemSlot(badBlock),
+        root: rootHex,
+        reason: headReason,
+        peerIdStrings: Array.from(badBlock.peerIdStrings)
+          .map((id) => prettyPrintPeerIdStr(id))
+          .join(","),
+      });
+    }
+
+    pruneSetToMax(this.knownBadBlocks, MAX_KNOWN_BAD_BLOCKS);
+    return bannedBlocks;
+  }
+
   private removeAllDescendants(
     block: BlockInputSyncCacheItem,
     headReason: DroppedItemReason,
     descendantReason: DroppedItemReason = DroppedItemReason.invalidParent
   ): BlockInputSyncCacheItem[] {
     const rootHex = getBlockInputSyncCacheItemRootHex(block);
-    const slot = getBlockInputSyncCacheItemSlot(block);
     // Get all blocks that are a descendant of this one (index 0 is the head item itself)
     const badPendingBlocks = [block, ...getAllDescendantBlocks(rootHex, this.pendingBlocks)];
 
@@ -1926,8 +1975,8 @@ export class BlockInputSync {
       // Keep PayloadEnvelopeInput resident in the seen cache for consistency with the
       // importBlock()-owned lifecycle.
       this.logger.debug("Removing bad/unknown/incomplete BlockInputSyncCacheItem", {
-        slot,
-        blockRoot: rootHex,
+        slot: getBlockInputSyncCacheItemSlot(block),
+        root: rootHex,
       });
     }
 

package/src/util/execution.ts

@@ -259,21 +259,22 @@ export async function getDataColumnSidecarsFromExecution(
   return DataColumnEngineResult.SuccessResolved;
 }
 
-/** engine_getPayloadBodiesByHashV2: ELs MUST support at least 32 hashes per request. */
-const MAX_BODIES_REQUEST = 32;
+/** engine_getPayloadBodiesByHashV2: every EL must accept requests of up to 32 hashes, larger ones may fail with -38004 */
+export const MAX_BODIES_PER_REQUEST = 32;
 
 type SlotEnvelopeBytes = {slot: Slot; envelopeBytes: Uint8Array};
 
 type RangeEntry = ArchivedEnvelope & {slot: Slot};
 
-/** What a body root mismatch means on a given serving path */
-export type ReconstructMismatchPolicy = "throw" | "omit";
+/** What an envelope that cannot be rebuilt (EL-unavailable body or body root mismatch) means on a given serving path */
+export type ReconstructMissPolicy = "throw" | "omit";
 
-export type RebuildMiss =
-  /** EL does not have the block, or has pruned its block access list */
-  | {slot: Slot; reason: "unavailable"}
-  /** An EL body does not hash to its stored root (local inconsistency) */
-  | {slot: Slot; reason: "mismatch"; error: EnvelopeReconstructionError};
+/**
+ * An archived envelope that could not be rebuilt. `"unavailable"`: the EL does not have the block or its
+ * block access list (BODY_UNAVAILABLE). `"mismatch"`: an EL body does not hash to its stored root, a local
+ * inconsistency (BODY_ROOT_MISMATCH). `error` carries the matching code for paths that surface it.
+ */
+export type RebuildMiss = {slot: Slot; reason: "unavailable" | "mismatch"; error: EnvelopeReconstructionError};
 
 /**
  * Stream finalized envelopes over [startSlot, endSlot) as serialized bytes, rebuilding header
@@ -299,7 +300,7 @@ export async function* reconstructExecutionPayloadEnvelopesByRange(
 
   for await (const {key, value: bytes} of archive.binaryEntriesStream({gte: startSlot, lt: endSlot})) {
     batch.push({slot: archive.decodeKey(key), ...decodeArchivedEnvelope(bytes)});
-    if (batch.length === MAX_BODIES_REQUEST) {
+    if (batch.length === MAX_BODIES_PER_REQUEST) {
       yield* reconstructBatch(executionEngine, logger, metrics, batch);
       batch = [];
     }
@@ -359,9 +360,13 @@ export async function reconstructExecutionPayloadEnvelopes(
   headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[]
 ): Promise<(gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[]> {
   const out: (gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[] = [];
-  for (let i = 0; i < headerEnvelopes.length; i += MAX_BODIES_REQUEST) {
+  for (let i = 0; i < headerEnvelopes.length; i += MAX_BODIES_PER_REQUEST) {
     out.push(
-      ...(await reconstructEnvelopesBatch(executionEngine, metrics, headerEnvelopes.slice(i, i + MAX_BODIES_REQUEST)))
+      ...(await reconstructEnvelopesBatch(
+        executionEngine,
+        metrics,
+        headerEnvelopes.slice(i, i + MAX_BODIES_PER_REQUEST)
+      ))
     );
   }
   return out;
@@ -401,7 +406,14 @@ async function reconstructEnvelopesBatch(
     // A zero-length block access list cannot be valid, RLP encodes an empty list as 0xc0
     if (body == null || body.withdrawals == null || body.blockAccessList == null || body.blockAccessList.length === 0) {
       metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "unavailable"});
-      return {slot, reason: "unavailable"};
+      return {
+        slot,
+        reason: "unavailable",
+        error: new EnvelopeReconstructionError(
+          {code: EnvelopeReconstructionErrorCode.BODY_UNAVAILABLE, slot},
+          `execution client cannot serve the payload body or block access list for archived envelope slot=${slot}`
+        ),
+      };
     }
     try {
       const envelope = signedHeaderEnvelopeToFull(headerEnvelope, {
@@ -409,7 +421,7 @@ async function reconstructEnvelopesBatch(
         withdrawals: body.withdrawals,
         blockAccessList: body.blockAccessList,
       });
-      metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "ok"});
+      metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "success"});
       return envelope;
     } catch (e) {
       if (