@lodestar/beacon-node 1.49.0-dev.fa4ae94ccc → 1.49.0-rc.0

package/src/db/repositories/executionPayloadEnvelopeArchive.ts

@@ -1,25 +1,44 @@
 import {ChainForkConfig} from "@lodestar/config";
-import {BUCKET_LENGTH, Db, Repository, encodeKey as encodeDbKey} from "@lodestar/db";
+import {BUCKET_LENGTH, BinaryRepository, Db, DbBatch, encodeKey as encodeDbKey} from "@lodestar/db";
 import {Slot, gloas, ssz} from "@lodestar/types";
 import {bytesToInt} from "@lodestar/utils";
 import {Bucket, getBucketNameByValue} from "../buckets.js";
 
 /**
- * Used to store finalized `SignedExecutionPayloadEnvelope`
- *
- * Indexed by slot for chronological archival
+ * Prefix of a header entry. A full `SignedExecutionPayloadEnvelope` starts with the offset of its
+ * variable-size `message` field (4 + 96 = 0x64), so a leading 0x00 byte is unambiguous and full
+ * entries can be stored as the hot db bytes, untouched.
  */
-export class ExecutionPayloadEnvelopeArchiveRepository extends Repository<Slot, gloas.SignedExecutionPayloadEnvelope> {
+const HEADER_ENVELOPE_PREFIX = 0x00;
+
+/** An archive entry as the serving paths need it: full stays bytes, header is deserialized for reconstruction */
+export type ArchivedEnvelope =
+  | {headerEnvelope: gloas.SignedExecutionPayloadHeaderEnvelope; envelopeBytes?: undefined}
+  | {headerEnvelope?: undefined; envelopeBytes: Uint8Array};
+
+export function decodeArchivedEnvelope(bytes: Uint8Array): ArchivedEnvelope {
+  return bytes[0] === HEADER_ENVELOPE_PREFIX
+    ? {headerEnvelope: ssz.gloas.SignedExecutionPayloadHeaderEnvelope.deserialize(bytes.subarray(1))}
+    : {envelopeBytes: bytes};
+}
+
+export function encodeArchivedHeaderEnvelope(headerEnvelope: gloas.SignedExecutionPayloadHeaderEnvelope): Uint8Array {
+  const value = ssz.gloas.SignedExecutionPayloadHeaderEnvelope.serialize(headerEnvelope);
+  const out = new Uint8Array(1 + value.length);
+  out[0] = HEADER_ENVELOPE_PREFIX;
+  out.set(value, 1);
+  return out;
+}
+
+/**
+ * Finalized envelopes indexed by slot: `SignedExecutionPayloadEnvelope` bytes as-is
+ * (`--chain.dedupePayloads=false`, or the block was still optimistic when archived), or a
+ * `SignedExecutionPayloadHeaderEnvelope` behind a 0x00 prefix. See {@link decodeArchivedEnvelope}.
+ */
+export class ExecutionPayloadEnvelopeArchiveRepository extends BinaryRepository<Slot> {
   constructor(config: ChainForkConfig, db: Db) {
     const bucket = Bucket.gloas_executionPayloadEnvelopeArchive;
-    super(config, db, bucket, ssz.gloas.SignedExecutionPayloadEnvelope, getBucketNameByValue(bucket));
-  }
-
-  /**
-   * Id is the slot from the envelope
-   */
-  getId(value: gloas.SignedExecutionPayloadEnvelope): Slot {
-    return value.message.payload.slotNumber;
+    super(config, db, bucket, getBucketNameByValue(bucket));
   }
 
   encodeKey(id: Slot): Uint8Array {
@@ -29,4 +48,15 @@ export class ExecutionPayloadEnvelopeArchiveRepository extends Repository<Slot,
   decodeKey(data: Uint8Array): number {
     return bytesToInt(data.subarray(BUCKET_LENGTH), "be");
   }
+
+  /** Archive entries and delete their hot counterparts (`hotKey` pre-encoded) in one atomic batch */
+  async batchArchiveAndDeleteHot(
+    entries: {slot: Slot; archivedBytes: Uint8Array; hotKey: Uint8Array}[]
+  ): Promise<void> {
+    const operations: DbBatch<Uint8Array, Uint8Array> = [];
+    for (const {slot, archivedBytes, hotKey} of entries) {
+      operations.push({type: "put", key: this.encodeKey(slot), value: archivedBytes}, {type: "del", key: hotKey});
+    }
+    await this.db.batch(operations, this.dbReqOpts);
+  }
 }

package/src/db/repositories/index.ts

@@ -8,7 +8,12 @@ export {BLSToExecutionChangeRepository} from "./blsToExecutionChange.js";
 export {DataColumnSidecarRepository} from "./dataColumnSidecar.js";
 export {DataColumnSidecarArchiveRepository} from "./dataColumnSidecarArchive.js";
 export {ExecutionPayloadEnvelopeRepository} from "./executionPayloadEnvelope.js";
-export {ExecutionPayloadEnvelopeArchiveRepository} from "./executionPayloadEnvelopeArchive.js";
+export {
+  type ArchivedEnvelope,
+  ExecutionPayloadEnvelopeArchiveRepository,
+  decodeArchivedEnvelope,
+  encodeArchivedHeaderEnvelope,
+} from "./executionPayloadEnvelopeArchive.js";
 export {BestLightClientUpdateRepository} from "./lightclientBestUpdate.js";
 export {CheckpointHeaderRepository} from "./lightclientCheckpointHeader.js";
 export {SyncCommitteeRepository} from "./lightclientSyncCommittee.js";

package/src/execution/engine/disabled.ts

@@ -25,6 +25,10 @@ export class ExecutionEngineDisabled implements IExecutionEngine {
     throw Error("Execution engine disabled");
   }
 
+  getPayloadBodiesByHashV2(): Promise<never> {
+    throw Error("Execution engine disabled");
+  }
+
   getPayloadBodiesByRange(): Promise<never> {
     throw Error("Execution engine disabled");
   }

package/src/execution/engine/http.ts

@@ -32,11 +32,13 @@ import {
   EngineApiRpcParamTypes,
   EngineApiRpcReturnTypes,
   ExecutionPayloadBody,
+  ExecutionPayloadBodyV2,
   assertReqSizeLimit,
   deserializeBlobAndProofs,
   deserializeBlobAndProofsV2,
   deserializeBlobAndProofsV2IntoBytes,
   deserializeExecutionPayloadBody,
+  deserializeExecutionPayloadBodyV2,
   parseExecutionPayload,
   serializeBeaconBlockRoot,
   serializeExecutionPayload,
@@ -476,6 +478,16 @@ export class ExecutionEngineHttp implements IExecutionEngine {
     return response.map(deserializeExecutionPayloadBody);
   }
 
+  async getPayloadBodiesByHashV2(blockHashes: RootHex[]): Promise<(ExecutionPayloadBodyV2 | null)[]> {
+    const method = "engine_getPayloadBodiesByHashV2";
+    assertReqSizeLimit(blockHashes.length, 32);
+    const response = await this.rpc.fetchWithRetries<
+      EngineApiRpcReturnTypes[typeof method],
+      EngineApiRpcParamTypes[typeof method]
+    >({method, params: [blockHashes]}, getPayloadBodiesByHashOpts);
+    return response.map(deserializeExecutionPayloadBodyV2);
+  }
+
   async getPayloadBodiesByRange(
     _fork: ForkName,
     startBlockNumber: number,

package/src/execution/engine/interface.ts

@@ -13,7 +13,7 @@ import {BlobsBundle, ExecutionPayload, ExecutionRequests, Root, RootHex, Wei, ca
 import {BlobAndProof} from "@lodestar/types/deneb";
 import {BlobAndProofV2} from "@lodestar/types/fulu";
 import {PayloadId, PayloadIdCache, WithdrawalV1} from "./payloadIdCache.js";
-import {ExecutionPayloadBody} from "./types.js";
+import {ExecutionPayloadBody, ExecutionPayloadBodyV2} from "./types.js";
 import {DATA} from "./utils.js";
 
 export {PayloadIdCache, type PayloadId, type WithdrawalV1};
@@ -172,6 +172,9 @@ export interface IExecutionEngine {
 
   getPayloadBodiesByHash(fork: ForkName, blockHash: DATA[]): Promise<(ExecutionPayloadBody | null)[]>;
 
+  /** Amsterdam: like V1 but also returns the block access list (null if the EL no longer has it) */
+  getPayloadBodiesByHashV2(blockHash: DATA[]): Promise<(ExecutionPayloadBodyV2 | null)[]>;
+
   getPayloadBodiesByRange(fork: ForkName, start: number, count: number): Promise<(ExecutionPayloadBody | null)[]>;
 
   getBlobs(

package/src/execution/engine/mock.ts

@@ -23,6 +23,7 @@ import {
   EngineApiRpcParamTypes,
   EngineApiRpcReturnTypes,
   ExecutionPayloadBodyRpc,
+  ExecutionPayloadBodyV2Rpc,
   ExecutionPayloadRpc,
   ExecutionRequestsRpc,
   PayloadStatus,
@@ -49,6 +50,8 @@ type ExecutionBlock = {
   blockHash: RootHex;
   timestamp: number;
   blockNumber: number;
+  /** Bodies as received via newPayload, served back by engine_getPayloadBodiesByHash{V1,V2} */
+  body: ExecutionPayloadBodyV2Rpc;
 };
 
 const TX_TYPE_EIP1559 = 2;
@@ -93,6 +96,7 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
       blockHash: ZERO_HASH_HEX,
       timestamp: 0,
       blockNumber: 0,
+      body: {transactions: [], withdrawals: null, blockAccessList: null},
     });
 
     const eth1BlockHash = opts.eth1BlockHash ?? toRootHex(INTEROP_BLOCK_HASH);
@@ -102,6 +106,7 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
       blockHash: eth1BlockHash,
       timestamp: 0,
       blockNumber: 1,
+      body: {transactions: [], withdrawals: null, blockAccessList: null},
     });
 
     const {config} = opts;
@@ -144,6 +149,7 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
       engine_getPayloadV5: this.getPayloadV5.bind(this),
       engine_getPayloadV6: this.getPayloadV5.bind(this),
       engine_getPayloadBodiesByHashV1: this.getPayloadBodiesByHash.bind(this),
+      engine_getPayloadBodiesByHashV2: this.getPayloadBodiesByHashV2.bind(this),
       engine_getPayloadBodiesByRangeV1: this.getPayloadBodiesByRange.bind(this),
       engine_getClientVersionV1: this.getClientVersionV1.bind(this),
       engine_getBlobsV1: this.getBlobs.bind(this),
@@ -152,9 +158,18 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
   }
 
   private getPayloadBodiesByHash(
-    _blockHex: EngineApiRpcParamTypes["engine_getPayloadBodiesByHashV1"][0]
+    blockHashes: EngineApiRpcParamTypes["engine_getPayloadBodiesByHashV1"][0]
   ): EngineApiRpcReturnTypes["engine_getPayloadBodiesByHashV1"] {
-    return [] as ExecutionPayloadBodyRpc[];
+    return this.getPayloadBodiesByHashV2(blockHashes).map((body) =>
+      body ? {transactions: body.transactions, withdrawals: body.withdrawals} : null
+    );
+  }
+
+  private getPayloadBodiesByHashV2(
+    blockHashes: EngineApiRpcParamTypes["engine_getPayloadBodiesByHashV2"][0]
+  ): EngineApiRpcReturnTypes["engine_getPayloadBodiesByHashV2"] {
+    // null for unknown blocks, as a real EL does; the genesis/eth1 seed blocks carry no body
+    return blockHashes.map((hash) => this.validBlocks.get(hash)?.body ?? null);
   }
 
   private getPayloadBodiesByRange(
@@ -227,6 +242,11 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
       blockHash,
       timestamp: quantityToNum(executionPayloadRpc.timestamp),
       blockNumber: quantityToNum(executionPayloadRpc.blockNumber),
+      body: {
+        transactions: executionPayloadRpc.transactions,
+        withdrawals: executionPayloadRpc.withdrawals ?? null,
+        blockAccessList: executionPayloadRpc.blockAccessList ?? null,
+      },
     });
 
     // IF the payload has been fully validated while processing the call

package/src/execution/engine/types.ts

@@ -89,6 +89,7 @@ export type EngineApiRpcParamTypes = {
    * 1. Array of DATA - Array of block_hash field values of the ExecutionPayload structure
    *  */
   engine_getPayloadBodiesByHashV1: DATA[][];
+  engine_getPayloadBodiesByHashV2: DATA[][];
 
   /**
    *  1. start: QUANTITY, 64 bits - Starting block number
@@ -148,6 +149,7 @@ export type EngineApiRpcReturnTypes = {
   engine_getPayloadV6: ExecutionPayloadResponse;
 
   engine_getPayloadBodiesByHashV1: (ExecutionPayloadBodyRpc | null)[];
+  engine_getPayloadBodiesByHashV2: (ExecutionPayloadBodyV2Rpc | null)[];
 
   engine_getPayloadBodiesByRangeV1: (ExecutionPayloadBodyRpc | null)[];
 
@@ -177,6 +179,15 @@ export type ExecutionPayloadBody = {
   withdrawals: capella.Withdrawals | null;
 };
 
+/** engine_getPayloadBodiesByHashV2 (Amsterdam): adds the RLP-encoded block access list, or null if the EL has pruned it */
+export type ExecutionPayloadBodyV2Rpc = ExecutionPayloadBodyRpc & {
+  blockAccessList?: DATA | null;
+};
+
+export type ExecutionPayloadBodyV2 = ExecutionPayloadBody & {
+  blockAccessList: Uint8Array | null;
+};
+
 export type ExecutionPayloadRpc = {
   parentHash: DATA; // 32 bytes
   feeRecipient: DATA; // 20 bytes
@@ -678,6 +689,20 @@ export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc |
     : null;
 }
 
+export function deserializeExecutionPayloadBodyV2(
+  data: ExecutionPayloadBodyV2Rpc | null
+): ExecutionPayloadBodyV2 | null {
+  const body = deserializeExecutionPayloadBody(data);
+  return body
+    ? {...body, blockAccessList: data?.blockAccessList ? dataToBytes(data.blockAccessList, null) : null}
+    : null;
+}
+
+export function serializeExecutionPayloadBodyV2(data: ExecutionPayloadBodyV2 | null): ExecutionPayloadBodyV2Rpc | null {
+  const body = serializeExecutionPayloadBody(data);
+  return body ? {...body, blockAccessList: data?.blockAccessList ? bytesToData(data.blockAccessList) : null} : null;
+}
+
 export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null): ExecutionPayloadBodyRpc | null {
   return data
     ? {

package/src/metrics/metrics/beacon.ts

@@ -215,6 +215,22 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
         help: "Number of getBlobsV1 hits where a versioned hash returns blob and the blob is needed so call is useful",
       }),
     },
+    payloadEnvelopeReconstruction: {
+      envelopes: register.counter<{result: "ok" | "unavailable" | "mismatch"}>({
+        name: "beacon_payload_envelope_reconstruction_envelopes_total",
+        help: "Archived header envelopes rebuilt from EL bodies, by outcome",
+        labelNames: ["result"],
+      }),
+      mismatchByField: register.counter<{field: "transactions" | "withdrawals" | "blockAccessList"}>({
+        name: "beacon_payload_envelope_reconstruction_mismatch_total",
+        help: "Body root mismatches, by which body differed from the stored root",
+        labelNames: ["field"],
+      }),
+      engineErrors: register.counter({
+        name: "beacon_payload_envelope_reconstruction_engine_errors_total",
+        help: "engine_getPayloadBodiesByHashV2 round-trips that failed while rebuilding envelopes",
+      }),
+    },
 
     blockInputFetchStats: {
       // of already available blocks which didn't have to go through blobs pull

package/src/metrics/metrics/lodestar.ts

@@ -180,6 +180,30 @@ export function createLodestarMetrics(
       buckets: [0.001, 0.003, 0.01, 0.03, 0.1],
     }),
 
+    reqRespByRange: {
+      requestedSlots: register.histogram<{method: string}>({
+        name: "lodestar_reqresp_by_range_requested_slots",
+        help: "Histogram of slot count requested per served by_range request",
+        labelNames: ["method"],
+        buckets: [1, 16, 32, 64, 128],
+      }),
+      // data_column_sidecars_by_range is the only by_range request carrying columns
+      requestedColumns: register.histogram({
+        name: "lodestar_reqresp_by_range_requested_columns",
+        help: "Histogram of column count requested per served data_column_sidecars_by_range request",
+        // 128 is NUMBER_OF_COLUMNS.
+        buckets: [1, 4, 8, 32, 128],
+      }),
+      servedBytes: register.histogram<{method: string}>({
+        name: "lodestar_reqresp_by_range_served_bytes",
+        help: "Histogram of total serialized bytes served per by_range request",
+        labelNames: ["method"],
+        // min bucket is roughly a typical mainnet block is 100MB,
+        // max bucket is roughly 100 * MAX_PAYLOAD_SIZE
+        buckets: [100e3, 1e6, 10e6, 100e6, 1000e6],
+      }),
+    },
+
     regenQueue: {
       length: register.gauge({
         name: "lodestar_regen_queue_length",
@@ -1011,6 +1035,13 @@ export function createLodestarMetrics(
         help: "Count of errors, by error type, while processing execution payload envelopes",
         labelNames: ["error"],
       }),
+      executionPayload: {
+        recvToValidation: register.histogram({
+          name: "lodestar_gossip_execution_payload_envelope_received_to_execution_payload_verification_seconds",
+          help: "Time elapsed between execution payload envelope received and execution payload verification",
+          buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 3, 4, 6, 8],
+        }),
+      },
     },
     gossipExecutionPayloadBid: {
       elapsedTimeTillReceived: register.histogram<{source: OpSource}>({

package/src/metrics/metrics.ts

@@ -12,7 +12,12 @@ export type Metrics = BeaconMetrics &
   BeaconStateTransitionMetrics &
   LodestarMetrics & {register: RegistryMetricCreator; close: () => void};
 
-export function createMetrics(opts: MetricsOptions, genesisTime: number, externalRegistries: Registry[] = []): Metrics {
+export function createMetrics(
+  opts: MetricsOptions,
+  genesisTime: number,
+  externalRegistries: Registry[] = [],
+  {collectNodeMetrics = true}: {collectNodeMetrics?: boolean} = {}
+): Metrics {
   const register = new RegistryMetricCreator();
   const beacon = createBeaconMetrics(register);
   const forkChoice = getForkChoiceMetrics(register);
@@ -24,7 +29,7 @@ export function createMetrics(opts: MetricsOptions, genesisTime: number, externa
   };
   process.on("unhandledRejection", onUnhandledRejection);
 
-  const nodeJsMetricsClose = collectNodeJSMetrics(register);
+  const nodeJsMetricsClose = collectNodeMetrics ? collectNodeJSMetrics(register) : (): void => {};
   const close = (): void => {
     process.removeListener("unhandledRejection", onUnhandledRejection);
     nodeJsMetricsClose();

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRange.ts

@@ -5,8 +5,10 @@ import {GENESIS_SLOT} from "@lodestar/params";
 import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
 import {gloas} from "@lodestar/types";
+import {EnvelopeReconstructionError} from "../../../chain/errors/index.js";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
+import {reconstructExecutionPayloadEnvelopesByRange} from "../../../util/execution.js";
 import {prettyPrintPeerId} from "../../util.js";
 
 export async function* onExecutionPayloadEnvelopesByRange(
@@ -35,58 +37,74 @@ export async function* onExecutionPayloadEnvelopesByRange(
     );
   }
 
-  const finalized = db.executionPayloadEnvelopeArchive;
   // Use the finalized block's actual slot as the checkpoint epoch-boundary slot may be skipped
   const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
   // The finalized block's envelope stays in the hot db until the next finalization run
   const archiveMaxSlot = finalizedSlot - 1;
 
-  // Finalized range of envelopes
-  if (startSlot <= archiveMaxSlot) {
-    for await (const {key, value: envelopeBytes} of finalized.binaryEntriesStream({
-      gte: startSlot,
-      lt: Math.min(endSlot, archiveMaxSlot + 1),
-    })) {
-      const slot = finalized.decodeKey(key);
-      yield {
-        data: envelopeBytes,
-        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(slot)),
-      };
+  let yielded = 0;
+  try {
+    if (startSlot <= archiveMaxSlot) {
+      for await (const {slot, envelopeBytes} of reconstructExecutionPayloadEnvelopesByRange(
+        db,
+        chain.executionEngine,
+        chain.logger,
+        chain.metrics,
+        startSlot,
+        Math.min(endSlot, archiveMaxSlot + 1)
+      )) {
+        yielded++;
+        yield {
+          data: envelopeBytes,
+          boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(slot)),
+        };
+      }
     }
-  }
 
-  // Non-finalized range of envelopes
-  if (endSlot > archiveMaxSlot) {
-    const headBlock = chain.forkChoice.getHead();
-    const headRoot = headBlock.blockRoot;
-    const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
+    // Non-finalized range of envelopes. The finalized block's own envelope may be archived by a
+    // migration that ran since archiveMaxSlot was computed, so this read can reconstruct too.
+    if (endSlot > archiveMaxSlot) {
+      const headBlock = chain.forkChoice.getHead();
+      const headRoot = headBlock.blockRoot;
+      const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
 
-    // Iterate head chain with ascending block numbers
-    for (let i = headChain.length - 1; i >= 0; i--) {
-      const block = headChain[i];
+      // Iterate head chain with ascending block numbers
+      for (let i = headChain.length - 1; i >= 0; i--) {
+        const block = headChain[i];
 
-      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
-        // Skip EMPTY blocks
-        if (block.payloadStatus !== PayloadStatus.FULL) {
-          continue;
-        }
+        if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
+          // Skip EMPTY blocks
+          if (block.payloadStatus !== PayloadStatus.FULL) {
+            continue;
+          }
 
-        const envelopeBytes = await chain.getSerializedExecutionPayloadEnvelope(block.slot, block.blockRoot);
-        if (!envelopeBytes) {
-          throw new ResponseError(
-            RespStatus.SERVER_ERROR,
-            `No envelope for root ${block.blockRoot} slot ${block.slot}, startSlot=${startSlot} endSlot=${endSlot} finalizedSlot=${finalizedSlot}`
-          );
-        }
+          const envelopeBytes = await chain.getSerializedExecutionPayloadEnvelope(block.slot, block.blockRoot);
+          if (!envelopeBytes) {
+            throw new ResponseError(
+              RespStatus.SERVER_ERROR,
+              `No envelope for root ${block.blockRoot} slot ${block.slot}, startSlot=${startSlot} endSlot=${endSlot} finalizedSlot=${finalizedSlot}`
+            );
+          }
 
-        yield {
-          data: envelopeBytes,
-          boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(block.slot)),
-        };
-      } else if (block.slot >= endSlot) {
-        break;
+          yielded++;
+          yield {
+            data: envelopeBytes,
+            boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(block.slot)),
+          };
+        } else if (block.slot >= endSlot) {
+          break;
+        }
       }
     }
+  } catch (e) {
+    if (e instanceof EnvelopeReconstructionError) {
+      // An unservable entry or our own EL failing: after some envelopes, end short (spec-legal, the
+      // peer fetches the rest elsewhere); with nothing served, RESOURCE_UNAVAILABLE rather than
+      // SERVER_ERROR so peers don't downscore us for it
+      if (yielded > 0) return;
+      throw new ResponseError(RespStatus.RESOURCE_UNAVAILABLE, `Failed to serve envelopes: ${e.message}`);
+    }
+    throw e;
   }
 }
 

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRoot.ts

@@ -1,7 +1,9 @@
 import {PeerId} from "@libp2p/interface";
-import {ResponseOutgoing} from "@lodestar/reqresp";
+import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
+import {RootHex, Slot} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
+import {EnvelopeReconstructionError} from "../../../chain/errors/index.js";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
 import {ExecutionPayloadEnvelopesByRootRequest} from "../../../util/types.js";
@@ -17,6 +19,8 @@ export async function* onExecutionPayloadEnvelopesByRoot(
   // The gloas req/resp spec uses MIN_EPOCHS_FOR_BLOCK_REQUESTS to define the minimum range peers MUST serve.
   // Archival nodes may still serve older retained payloads to allow genesis sync.
 
+  // Resolve slots first so archived header envelopes are rebuilt in one EL batch, not one call per root
+  const requests: {blockSlot: Slot; blockRootHex: RootHex}[] = [];
   for (const root of requestBody) {
     const rootHex = toRootHex(root);
     const block = chain.forkChoice.getBlockHexDefaultStatus(rootHex);
@@ -34,17 +38,35 @@ export async function* onExecutionPayloadEnvelopesByRoot(
       );
       continue;
     }
+    requests.push({blockSlot: slot, blockRootHex: rootHex});
+  }
+
+  let envelopesBytes: (Uint8Array | null)[];
+  try {
+    // by-root allows omission, so a mismatched envelope is left out rather than failing the response
+    envelopesBytes = await chain.getSerializedExecutionPayloadEnvelopes(requests, "omit");
+  } catch (e) {
+    if (e instanceof EnvelopeReconstructionError) {
+      throw new ResponseError(
+        RespStatus.RESOURCE_UNAVAILABLE,
+        `Failed to reconstruct archived envelopes: ${e.message}`
+      );
+    }
+    throw e;
+  }
 
-    const envelopeBytes = await chain.getSerializedExecutionPayloadEnvelope(slot, rootHex);
+  for (let i = 0; i < requests.length; i++) {
+    const {blockSlot, blockRootHex} = requests[i];
+    const envelopeBytes = envelopesBytes[i];
     if (envelopeBytes) {
       yield {
         data: envelopeBytes,
-        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(slot)),
+        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(blockSlot)),
       };
     } else {
       chain.logger.debug("Cannot serve ExecutionPayloadEnvelopesByRoot: envelope not found", {
-        slot,
-        root: rootHex,
+        slot: blockSlot,
+        root: blockRootHex,
         peer: prettyPrintPeerId(peerId),
         client: peerClient,
       });

package/src/network/reqresp/handlers/index.ts

@@ -1,5 +1,5 @@
 import type {Type} from "@chainsafe/ssz";
-import {ProtocolHandler, RespStatus, ResponseError} from "@lodestar/reqresp";
+import {ProtocolHandler, RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {ssz} from "@lodestar/types";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
@@ -38,6 +38,29 @@ function deserializeRequestBody<T>(type: Type<T>, data: Uint8Array): T {
   }
 }
 
+async function* trackByRangeResponse(
+  chain: IBeaconChain,
+  method: ReqRespMethod,
+  count: number,
+  source: AsyncIterable<ResponseOutgoing>,
+  columns?: number
+): AsyncIterable<ResponseOutgoing> {
+  chain.metrics?.reqRespByRange.requestedSlots.observe({method}, count);
+  if (columns !== undefined) {
+    chain.metrics?.reqRespByRange.requestedColumns.observe(columns);
+  }
+
+  let servedBytes = 0;
+  try {
+    for await (const chunk of source) {
+      servedBytes += chunk.data.length;
+      yield chunk;
+    }
+  } finally {
+    chain.metrics?.reqRespByRange.servedBytes.observe({method}, servedBytes);
+  }
+}
+
 /**
  * The ReqRespHandler module handles app-level requests / responses from other peers,
  * fetching state from the chain and database as needed.
@@ -50,7 +73,12 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
     [ReqRespMethod.Metadata]: notImplemented(ReqRespMethod.Metadata),
     [ReqRespMethod.BeaconBlocksByRange]: (req, peerId, peerClient) => {
       const body = deserializeRequestBody(ssz.phase0.BeaconBlocksByRangeRequest, req.data);
-      return onBeaconBlocksByRange(body, chain, db, peerId, peerClient);
+      return trackByRangeResponse(
+        chain,
+        ReqRespMethod.BeaconBlocksByRange,
+        body.count,
+        onBeaconBlocksByRange(body, chain, db, peerId, peerClient)
+      );
     },
     [ReqRespMethod.BeaconBlocksByRoot]: (req) => {
       const fork = chain.config.getForkName(chain.clock.currentSlot);
@@ -68,11 +96,22 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
     },
     [ReqRespMethod.BlobSidecarsByRange]: (req) => {
       const body = deserializeRequestBody(ssz.deneb.BlobSidecarsByRangeRequest, req.data);
-      return onBlobSidecarsByRange(body, chain, db);
+      return trackByRangeResponse(
+        chain,
+        ReqRespMethod.BlobSidecarsByRange,
+        body.count,
+        onBlobSidecarsByRange(body, chain, db)
+      );
     },
     [ReqRespMethod.DataColumnSidecarsByRange]: (req, peerId, peerClient) => {
       const body = deserializeRequestBody(ssz.fulu.DataColumnSidecarsByRangeRequest, req.data);
-      return onDataColumnSidecarsByRange(body, chain, db, peerId, peerClient);
+      return trackByRangeResponse(
+        chain,
+        ReqRespMethod.DataColumnSidecarsByRange,
+        body.count,
+        onDataColumnSidecarsByRange(body, chain, db, peerId, peerClient),
+        body.columns.length
+      );
     },
     [ReqRespMethod.DataColumnSidecarsByRoot]: (req, peerId, peerClient) => {
       const body = deserializeRequestBody(DataColumnSidecarsByRootRequestType(chain.config), req.data);
@@ -85,7 +124,12 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh
     },
     [ReqRespMethod.ExecutionPayloadEnvelopesByRange]: (req, peerId, peerClient) => {
       const body = deserializeRequestBody(ssz.gloas.ExecutionPayloadEnvelopesByRangeRequest, req.data);
-      return onExecutionPayloadEnvelopesByRange(body, chain, db, peerId, peerClient);
+      return trackByRangeResponse(
+        chain,
+        ReqRespMethod.ExecutionPayloadEnvelopesByRange,
+        body.count,
+        onExecutionPayloadEnvelopesByRange(body, chain, db, peerId, peerClient)
+      );
     },
 
     [ReqRespMethod.LightClientBootstrap]: (req) => {

package/src/sync/types.ts

@@ -105,8 +105,6 @@ export enum DroppedItemReason {
   invalidEnvelope = "invalid_envelope",
   /** a payload had an invalid signature (INVALID_SIGNATURE) */
   invalidSignature = "invalid_signature",
-  /** a block root was given up on after exhausting download attempts (data could not be fetched) */
-  unavailable = "unavailable",
   /** an unexpected / unhandled error code led to removal */
   unknown = "unknown",
 }