@lodestar/beacon-node 1.50.0-dev.6fb6c0bad4 → 1.50.0-dev.9c3f7bebe9

package/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.50.0-dev.6fb6c0bad4",
+  "version": "1.50.0-dev.9c3f7bebe9",
   "type": "module",
   "exports": {
     ".": {
@@ -127,17 +127,17 @@
     "@libp2p/prometheus-metrics": "^5.0.28",
     "@libp2p/tcp": "^11.0.27",
     "@libp2p/utils": "^7.4.0",
-    "@lodestar/api": "^1.50.0-dev.6fb6c0bad4",
-    "@lodestar/config": "^1.50.0-dev.6fb6c0bad4",
-    "@lodestar/db": "^1.50.0-dev.6fb6c0bad4",
-    "@lodestar/fork-choice": "^1.50.0-dev.6fb6c0bad4",
-    "@lodestar/logger": "^1.50.0-dev.6fb6c0bad4",
-    "@lodestar/params": "^1.50.0-dev.6fb6c0bad4",
-    "@lodestar/reqresp": "^1.50.0-dev.6fb6c0bad4",
-    "@lodestar/state-transition": "^1.50.0-dev.6fb6c0bad4",
-    "@lodestar/types": "^1.50.0-dev.6fb6c0bad4",
-    "@lodestar/utils": "^1.50.0-dev.6fb6c0bad4",
-    "@lodestar/validator": "^1.50.0-dev.6fb6c0bad4",
+    "@lodestar/api": "^1.50.0-dev.9c3f7bebe9",
+    "@lodestar/config": "^1.50.0-dev.9c3f7bebe9",
+    "@lodestar/db": "^1.50.0-dev.9c3f7bebe9",
+    "@lodestar/fork-choice": "^1.50.0-dev.9c3f7bebe9",
+    "@lodestar/logger": "^1.50.0-dev.9c3f7bebe9",
+    "@lodestar/params": "^1.50.0-dev.9c3f7bebe9",
+    "@lodestar/reqresp": "^1.50.0-dev.9c3f7bebe9",
+    "@lodestar/state-transition": "^1.50.0-dev.9c3f7bebe9",
+    "@lodestar/types": "^1.50.0-dev.9c3f7bebe9",
+    "@lodestar/utils": "^1.50.0-dev.9c3f7bebe9",
+    "@lodestar/validator": "^1.50.0-dev.9c3f7bebe9",
     "@multiformats/multiaddr": "^13.0.1",
     "datastore-core": "^12.0.1",
     "datastore-level": "^13.0.1",
@@ -158,8 +158,8 @@
   "devDependencies": {
     "@libp2p/interface-internal": "^3.1.12",
     "@libp2p/logger": "^6.2.13",
-    "@lodestar/spec-test-util": "^1.50.0-dev.6fb6c0bad4",
-    "@lodestar/test-utils": "^1.50.0-dev.6fb6c0bad4",
+    "@lodestar/spec-test-util": "^1.50.0-dev.9c3f7bebe9",
+    "@lodestar/test-utils": "^1.50.0-dev.9c3f7bebe9",
     "@types/js-yaml": "^4.0.5",
     "@types/qs": "^6.9.7",
     "@types/tmp": "^0.2.3",
@@ -174,5 +174,5 @@
     "beacon",
     "blockchain"
   ],
-  "gitHead": "b0cf4ad48ca035db258dd93537a960a310f89a86"
+  "gitHead": "b6622aa0e5694b1b9bae9ff8c73324eb54745012"
 }

package/src/chain/archiveStore/archiveStore.ts

@@ -1,6 +1,7 @@
 import {CheckpointWithHex} from "@lodestar/fork-choice";
 import {LoggerNode} from "@lodestar/logger/node";
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import {Slot} from "@lodestar/types";
 import {Checkpoint} from "@lodestar/types/phase0";
 import {callFnWhenAwait} from "@lodestar/utils";
 import {IBeaconDb} from "../../db/index.js";
@@ -47,6 +48,8 @@ export class ArchiveStore {
   private jobQueue: JobItemQueue<[CheckpointWithHex], void>;
 
   private archiveDataEpochs?: number;
+  // Finalized state writes are monotonic; reset on restart to include the newly persisted anchor.
+  private statePruneFromSlot: Slot = 0;
   private readonly statesArchiverStrategy: StateArchiveStrategy;
   private readonly chain: IBeaconChain;
   private readonly db: IBeaconDb;
@@ -103,7 +106,7 @@ export class ArchiveStore {
     if (this.opts.pruneHistory) {
       // prune ALL stale data before starting
       this.logger.info("Pruning historical data");
-      await callFnWhenAwait(
+      const {stateCutoffSlot} = await callFnWhenAwait(
         pruneHistory(
           this.chain.config,
           this.db,
@@ -116,6 +119,7 @@ export class ArchiveStore {
         30_000,
         this.signal
       );
+      this.statePruneFromSlot = stateCutoffSlot;
     }
 
     // Initialize earliestAvailableSlot from the earliest block actually retained (after any pruning
@@ -255,14 +259,16 @@ export class ArchiveStore {
 
       if (this.opts.pruneHistory) {
         timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer();
-        const blockCutoffSlot = await pruneHistory(
+        const {blockCutoffSlot, stateCutoffSlot} = await pruneHistory(
           this.chain.config,
           this.db,
           this.logger,
           this.metrics,
           finalizedEpoch,
-          this.chain.clock.currentEpoch
+          this.chain.clock.currentEpoch,
+          this.statePruneFromSlot
         );
+        this.statePruneFromSlot = stateCutoffSlot;
         this.chain.earliestAvailableSlot = Math.max(this.chain.earliestAvailableSlot, blockCutoffSlot);
         timer?.({source: ArchiveStoreTask.PruneHistory});
       }

package/src/chain/archiveStore/utils/pruneHistory.ts

@@ -11,8 +11,9 @@ export async function pruneHistory(
   logger: Logger,
   metrics: Metrics | null | undefined,
   finalizedEpoch: Epoch,
-  currentEpoch: Epoch
-): Promise<Slot> {
+  currentEpoch: Epoch,
+  statePruneFromSlot: Slot = 0
+): Promise<{blockCutoffSlot: Slot; stateCutoffSlot: Slot}> {
   const blockCutoffEpoch = Math.min(
     // set by config, with underflow protection
     Math.max(currentEpoch - config.MIN_EPOCHS_FOR_BLOCK_REQUESTS, 0),
@@ -28,6 +29,7 @@ export async function pruneHistory(
     currentEpoch,
     finalizedEpoch,
     blockCutoffEpoch,
+    statePruneFromSlot,
     stateCutoffSlot,
   });
 
@@ -35,7 +37,7 @@ export async function pruneHistory(
   const [blocks, envelopes, states] = await Promise.all([
     db.blockArchive.keys({gte: 0, lt: blockCutoffSlot}),
     db.executionPayloadEnvelopeArchive.keys({gte: 0, lt: blockCutoffSlot}),
-    db.stateArchive.keys({gte: 0, lt: stateCutoffSlot}),
+    stateCutoffSlot > statePruneFromSlot ? db.stateArchive.keys({gte: statePruneFromSlot, lt: stateCutoffSlot}) : [],
   ]);
   step0?.();
 
@@ -64,5 +66,5 @@ export async function pruneHistory(
 
   metrics?.pruneHistory.pruneCount.inc();
 
-  return blockCutoffSlot;
+  return {blockCutoffSlot, stateCutoffSlot};
 }

package/src/chain/initState.ts

@@ -1,14 +1,8 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkPostGloas, ForkSeq, ZERO_HASH} from "@lodestar/params";
-import {
-  BeaconStateAllForks,
-  BeaconStateGloas,
-  IBeaconStateView,
-  computeEpochAtSlot,
-  computeStartSlotAtEpoch,
-} from "@lodestar/state-transition";
+import {BeaconStateAllForks, BeaconStateGloas, IBeaconStateView, computeEpochAtSlot} from "@lodestar/state-transition";
 import {SignedBeaconBlock, ssz} from "@lodestar/types";
-import {Logger, byteArrayEquals, toHex, toRootHex} from "@lodestar/utils";
+import {LodestarError, Logger, byteArrayEquals, toRootHex} from "@lodestar/utils";
 import {GENESIS_SLOT} from "../constants/index.js";
 import {IBeaconDb} from "../db/index.js";
 import {Metrics} from "../metrics/index.js";
@@ -33,7 +27,8 @@ export async function persistAnchorState(
     const latestBlockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(latestBlockHeader);
 
     if (!byteArrayEquals(blockRoot, latestBlockRoot)) {
-      throw Error(
+      throw new LodestarError(
+        {code: "GENESIS_BLOCK_ROOT_MISMATCH"},
         `Genesis block root ${toRootHex(blockRoot)} does not match genesis state latest block root ${toRootHex(latestBlockRoot)}`
       );
     }
@@ -86,52 +81,6 @@ export async function initStateFromDb(
   return state;
 }
 
-/**
- * Initialize and persist an anchor state (either weak subjectivity or genesis)
- */
-export async function checkAndPersistAnchorState(
-  config: ChainForkConfig,
-  db: IBeaconDb,
-  logger: Logger,
-  anchorState: BeaconStateAllForks,
-  anchorStateBytes: Uint8Array,
-  {
-    isWithinWeakSubjectivityPeriod,
-    isCheckpointState,
-  }: {isWithinWeakSubjectivityPeriod: boolean; isCheckpointState: boolean}
-): Promise<void> {
-  const expectedFork = config.getForkInfo(computeStartSlotAtEpoch(anchorState.fork.epoch));
-  const expectedForkVersion = toHex(expectedFork.version);
-  const stateFork = toHex(anchorState.fork.currentVersion);
-  if (stateFork !== expectedForkVersion) {
-    throw Error(
-      `State current fork version ${stateFork} not equal to current config ${expectedForkVersion}. Maybe caused by importing a state from a different network`
-    );
-  }
-
-  const stateInfo = isCheckpointState ? "checkpoint" : "db";
-  if (isWithinWeakSubjectivityPeriod) {
-    logger.info(`Initializing beacon from a valid ${stateInfo} state`, {
-      slot: anchorState.slot,
-      epoch: computeEpochAtSlot(anchorState.slot),
-      stateRoot: toRootHex(anchorState.hashTreeRoot()),
-      isWithinWeakSubjectivityPeriod,
-    });
-  } else {
-    logger.warn(`Initializing from a stale ${stateInfo} state vulnerable to long range attacks`, {
-      slot: anchorState.slot,
-      epoch: computeEpochAtSlot(anchorState.slot),
-      stateRoot: toRootHex(anchorState.hashTreeRoot()),
-      isWithinWeakSubjectivityPeriod,
-    });
-    logger.warn("Checkpoint sync recommended, please use --help to see checkpoint sync options");
-  }
-
-  if (isCheckpointState || anchorState.slot === GENESIS_SLOT) {
-    await persistAnchorState(config, db, anchorState, anchorStateBytes);
-  }
-}
-
 export function initBeaconMetrics(metrics: Metrics, state: IBeaconStateView): void {
   metrics.headSlot.set(state.slot);
   metrics.previousJustifiedEpoch.set(state.previousJustifiedCheckpoint.epoch);

package/src/db/beacon.ts

@@ -122,6 +122,8 @@ export class BeaconDb implements IBeaconDb {
   async pruneHotDb(): Promise<void> {
     // Prune all hot blobs
     await this.blobSidecars.batchDelete(await this.blobSidecars.keys());
+    // New columns are persisted in flat files, so this bucket only contains legacy hot data.
+    await this.deleteBucketData(Bucket.allForks_dataColumnSidecars);
     // Prune all hot blocks
     // TODO: Enable once it's deemed safe
     // await this.block.batchDelete(await this.block.keys());

package/src/execution/engine/http.ts

@@ -516,12 +516,13 @@ export class ExecutionEngineHttp implements IExecutionEngine {
   ): Promise<(BlobAndProof | null)[]>;
   async getBlobs(
     fork: ForkName,
-    versionedHashes: VersionedHashes
+    versionedHashes: VersionedHashes,
+    buffers?: Uint8Array[]
   ): Promise<BlobAndProofV2[] | (BlobAndProof | null)[] | null> {
     assertReqSizeLimit(versionedHashes.length, MAX_VERSIONED_HASHES);
     const versionedHashesHex = versionedHashes.map(bytesToData);
     if (isForkPostFulu(fork)) {
-      return await this.getBlobsV2(versionedHashesHex);
+      return await this.getBlobsV2(versionedHashesHex, buffers);
     }
     return await this.getBlobsV1(versionedHashesHex);
   }
@@ -551,8 +552,9 @@ export class ExecutionEngineHttp implements IExecutionEngine {
 
   private async getBlobsV2(versionedHashesHex: string[], buffers?: Uint8Array[]) {
     if (buffers) {
-      if (buffers.length !== versionedHashesHex.length) {
-        throw Error(`Invalid buffers length=${buffers.length} versionedHashes=${versionedHashesHex.length}`);
+      // Callers preallocate one buffer per max blobs of the epoch, only the first entries are used
+      if (buffers.length < versionedHashesHex.length) {
+        throw Error(`Not enough buffers length=${buffers.length} versionedHashes=${versionedHashesHex.length}`);
       }
 
       for (const [i, buffer] of buffers.entries()) {

package/src/index.ts

@@ -2,7 +2,7 @@
 
 export type {RestApiServerMetrics, RestApiServerModules, RestApiServerOpts} from "./api/rest/base.js";
 export {RestApiServer} from "./api/rest/base.js";
-export {checkAndPersistAnchorState} from "./chain/index.js";
+export {persistAnchorState} from "./chain/index.js";
 export {DbCPStateDatastore} from "./chain/stateCache/datastore/db.js";
 export {FileCPStateDatastore} from "./chain/stateCache/datastore/file.js";
 export {BeaconDb, type IBeaconDb} from "./db/index.js";

package/src/metrics/metrics/beacon.ts

@@ -216,7 +216,7 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
       }),
     },
     payloadEnvelopeReconstruction: {
-      envelopes: register.counter<{result: "ok" | "unavailable" | "mismatch"}>({
+      envelopes: register.counter<{result: "success" | "unavailable" | "mismatch"}>({
         name: "beacon_payload_envelope_reconstruction_envelopes_total",
         help: "Archived header envelopes rebuilt from EL bodies, by outcome",
         labelNames: ["result"],

package/src/network/metadata.ts

@@ -42,13 +42,13 @@ export class MetadataController {
   private onSetValue: (key: string, value: Uint8Array) => void;
   private networkConfig: NetworkConfig;
   private logger: Logger;
-  private _metadata: fulu.Metadata;
+  private metadata: fulu.Metadata;
 
   constructor(opts: MetadataOpts, modules: MetadataModules) {
     this.networkConfig = modules.networkConfig;
     this.logger = modules.logger;
     this.onSetValue = modules.onSetValue;
-    this._metadata = opts.metadata ?? {
+    this.metadata = opts.metadata ?? {
       ...ssz.fulu.Metadata.defaultValue(),
       custodyGroupCount: modules.networkConfig.custodyConfig.targetCustodyGroupCount,
     };
@@ -58,61 +58,61 @@ export class MetadataController {
     // updateEth2Field() MUST be called with clock epoch
     this.updateEth2Field(currentEpoch);
 
-    this.onSetValue(ENRKey.attnets, ssz.phase0.AttestationSubnets.serialize(this._metadata.attnets));
+    this.onSetValue(ENRKey.attnets, ssz.phase0.AttestationSubnets.serialize(this.metadata.attnets));
 
     const config = this.networkConfig.config;
 
     if (config.getForkSeq(computeStartSlotAtEpoch(currentEpoch)) >= ForkSeq.altair) {
       // Only persist syncnets if altair fork is already activated. If currentFork is altair but head is phase0
       // adding syncnets to the ENR is not a problem, we will just have a useless field for a few hours.
-      this.onSetValue(ENRKey.syncnets, ssz.altair.SyncSubnets.serialize(this._metadata.syncnets));
+      this.onSetValue(ENRKey.syncnets, ssz.altair.SyncSubnets.serialize(this.metadata.syncnets));
     }
 
     // Set CGC regardless of fork. It may be useful to clients before Fulu, and will be ignored otherwise.
-    this.onSetValue(ENRKey.cgc, serializeCgc(this._metadata.custodyGroupCount));
+    this.onSetValue(ENRKey.cgc, serializeCgc(this.metadata.custodyGroupCount));
   }
 
   get seqNumber(): bigint {
-    return this._metadata.seqNumber;
+    return this.metadata.seqNumber;
   }
 
   get syncnets(): BitArray {
-    return this._metadata.syncnets;
+    return this.metadata.syncnets;
   }
 
   set syncnets(syncnets: BitArray) {
     this.onSetValue(ENRKey.syncnets, ssz.altair.SyncSubnets.serialize(syncnets));
-    this._metadata.seqNumber++;
-    this._metadata.syncnets = syncnets;
+    this.metadata.seqNumber++;
+    this.metadata.syncnets = syncnets;
   }
 
   get attnets(): BitArray {
-    return this._metadata.attnets;
+    return this.metadata.attnets;
   }
 
   set attnets(attnets: BitArray) {
     this.onSetValue(ENRKey.attnets, ssz.phase0.AttestationSubnets.serialize(attnets));
-    this._metadata.seqNumber++;
-    this._metadata.attnets = attnets;
+    this.metadata.seqNumber++;
+    this.metadata.attnets = attnets;
   }
 
   get custodyGroupCount(): number {
-    return this._metadata.custodyGroupCount;
+    return this.metadata.custodyGroupCount;
   }
 
   set custodyGroupCount(custodyGroupCount: number) {
-    if (custodyGroupCount === this._metadata.custodyGroupCount) {
+    if (custodyGroupCount === this.metadata.custodyGroupCount) {
       return;
     }
     this.onSetValue(ENRKey.cgc, serializeCgc(custodyGroupCount));
     this.logger.debug("Updated cgc field in ENR", {custodyGroupCount});
-    this._metadata.seqNumber++;
-    this._metadata.custodyGroupCount = custodyGroupCount;
+    this.metadata.seqNumber++;
+    this.metadata.custodyGroupCount = custodyGroupCount;
   }
 
   /** Consumers that need the phase0.Metadata type can just ignore the .syncnets property */
   get json(): fulu.Metadata {
-    return this._metadata;
+    return this.metadata;
   }
 
   /**

package/src/network/peers/datastore.ts

@@ -16,23 +16,23 @@ type AwaitGenerator<T, TReturn = any, TNext = any> = Generator<T, TReturn, TNext
  * This has a memory issue because all peer data stays in memory and loaded at startup time
  * This is written for libp2p >=0.35, we maintain the same mechanism but with bounded data structure
  * This datastore includes a memory datastore and fallback to db datastore
- * Use an in-memory datastore with last accessed time and _maxMemoryItems, on start it's empty (lazy load)
+ * Use an in-memory datastore with last accessed time and maxMemoryItems, on start it's empty (lazy load)
  * - get: Search in-memory datastore first, if not found search from db.
  *     - If found from db, add back to the in-memory datastore
  *     - Update lastAccessedMs
- * - put: move oldest items from memory to db if there are more than _maxMemoryItems items in memory
- *     -  update memory datastore, only update db datastore if there are at least _threshold dirty items
+ * - put: move oldest items from memory to db if there are more than maxMemoryItems items in memory
+ *     -  update memory datastore, only update db datastore if there are at least threshold dirty items
  *     -  Update lastAccessedMs
  */
 export class Eth2PeerDataStore extends BaseDatastore {
-  private _dbDatastore: LevelDatastore;
-  private _memoryDatastore: Map<string, MemoryItem>;
+  private dbDatastore: LevelDatastore;
+  private memoryDatastore: Map<string, MemoryItem>;
   /** Same to PersistentPeerStore of the old libp2p implementation */
-  private _dirtyItems = new Set<string>();
+  private dirtyItems = new Set<string>();
   /** If there are more dirty items than threshold, commit data to db */
-  private _threshold: number;
+  private threshold: number;
   /** If there are more memory items than this, prune oldest ones from memory and move to db */
-  private _maxMemoryItems: number;
+  private maxMemoryItems: number;
 
   constructor(
     dbDatastore: LevelDatastore | string,
@@ -47,18 +47,18 @@ export class Eth2PeerDataStore extends BaseDatastore {
       throw Error(`Threshold ${threshold} should be at most maxMemoryItems ${maxMemoryItems}`);
     }
 
-    this._dbDatastore = typeof dbDatastore === "string" ? new LevelDatastore(dbDatastore) : dbDatastore;
-    this._memoryDatastore = new Map();
-    this._threshold = threshold;
-    this._maxMemoryItems = maxMemoryItems;
+    this.dbDatastore = typeof dbDatastore === "string" ? new LevelDatastore(dbDatastore) : dbDatastore;
+    this.memoryDatastore = new Map();
+    this.threshold = threshold;
+    this.maxMemoryItems = maxMemoryItems;
   }
 
   async open(): Promise<void> {
-    return this._dbDatastore.open();
+    return this.dbDatastore.open();
   }
 
   async close(): Promise<void> {
-    return this._dbDatastore.close();
+    return this.dbDatastore.close();
   }
 
   async put(key: Key, val: Uint8Array, _options?: AbortOptions): Promise<Key> {
@@ -67,23 +67,23 @@ export class Eth2PeerDataStore extends BaseDatastore {
 
   /**
    * Same interface to put with "fromDb" option, if this item is updated back from db
-   * Move oldest items from memory data store to db if it's over this._maxMemoryItems
+   * Move oldest items from memory data store to db if it's over this.maxMemoryItems
    */
   async _put(key: Key, val: Uint8Array, fromDb = false): Promise<Key> {
-    while (this._memoryDatastore.size >= this._maxMemoryItems) {
+    while (this.memoryDatastore.size >= this.maxMemoryItems) {
       // it's likely this is called only 1 time
       await this.pruneMemoryDatastore();
     }
 
     const keyStr = key.toString();
-    const memoryItem = this._memoryDatastore.get(keyStr);
+    const memoryItem = this.memoryDatastore.get(keyStr);
     if (memoryItem) {
       // update existing
       memoryItem.lastAccessedMs = Date.now();
       memoryItem.data = val;
     } else {
       // new
-      this._memoryDatastore.set(keyStr, {data: val, lastAccessedMs: Date.now()});
+      this.memoryDatastore.set(keyStr, {data: val, lastAccessedMs: Date.now()});
     }
 
     if (!fromDb) await this._addDirtyItem(keyStr);
@@ -98,15 +98,15 @@ export class Eth2PeerDataStore extends BaseDatastore {
    */
   async get(key: Key, options?: AbortOptions): Promise<Uint8Array> {
     const keyStr = key.toString();
-    const memoryItem = this._memoryDatastore.get(keyStr);
+    const memoryItem = this.memoryDatastore.get(keyStr);
     if (memoryItem) {
       memoryItem.lastAccessedMs = Date.now();
       return memoryItem.data;
     }
 
     // this throws error if not found
-    const dbValue = await this._dbDatastore.get(key, options);
-    // don't call this._memoryDatastore.set directly
+    const dbValue = await this.dbDatastore.get(key, options);
+    // don't call this.memoryDatastore.set directly
     // we want to get through prune() logic with fromDb as true
     await this._put(key, dbValue, true);
     return dbValue;
@@ -125,30 +125,30 @@ export class Eth2PeerDataStore extends BaseDatastore {
   }
 
   async delete(key: Key, options?: AbortOptions): Promise<void> {
-    this._memoryDatastore.delete(key.toString());
-    await this._dbDatastore.delete(key, options);
+    this.memoryDatastore.delete(key.toString());
+    await this.dbDatastore.delete(key, options);
   }
 
   async *_all(q: Query, options?: AbortOptions): AwaitGenerator<Pair> {
-    for (const [key, value] of this._memoryDatastore.entries()) {
+    for (const [key, value] of this.memoryDatastore.entries()) {
       yield {
         key: new Key(key),
         value: value.data,
       };
     }
-    yield* this._dbDatastore.query(q, options);
+    yield* this.dbDatastore.query(q, options);
   }
 
   async *_allKeys(q: KeyQuery, options?: AbortOptions): AwaitGenerator<Key> {
-    for (const key of this._memoryDatastore.keys()) {
+    for (const key of this.memoryDatastore.keys()) {
       yield new Key(key);
     }
-    yield* this._dbDatastore.queryKeys(q, options);
+    yield* this.dbDatastore.queryKeys(q, options);
   }
 
   private async _addDirtyItem(keyStr: string): Promise<void> {
-    this._dirtyItems.add(keyStr);
-    if (this._dirtyItems.size >= this._threshold) {
+    this.dirtyItems.add(keyStr);
+    if (this.dirtyItems.size >= this.threshold) {
       try {
         await this._commitData();
       } catch (_e) {}
@@ -156,15 +156,15 @@ export class Eth2PeerDataStore extends BaseDatastore {
   }
 
   private async _commitData(): Promise<void> {
-    const batch = this._dbDatastore.batch();
-    for (const keyStr of this._dirtyItems) {
-      const memoryItem = this._memoryDatastore.get(keyStr);
+    const batch = this.dbDatastore.batch();
+    for (const keyStr of this.dirtyItems) {
+      const memoryItem = this.memoryDatastore.get(keyStr);
       if (memoryItem) {
         batch.put(new Key(keyStr), memoryItem.data);
       }
     }
     await batch.commit();
-    this._dirtyItems.clear();
+    this.dirtyItems.clear();
   }
 
   /**
@@ -175,7 +175,7 @@ export class Eth2PeerDataStore extends BaseDatastore {
     let oldestKey: string | undefined = undefined;
     let oldestValue: Uint8Array | undefined = undefined;
 
-    for (const [key, value] of this._memoryDatastore) {
+    for (const [key, value] of this.memoryDatastore) {
       if (value.lastAccessedMs < oldestAccessedMs) {
         oldestAccessedMs = value.lastAccessedMs;
         oldestKey = key;
@@ -184,8 +184,8 @@ export class Eth2PeerDataStore extends BaseDatastore {
     }
 
     if (oldestKey && oldestValue) {
-      await this._dbDatastore.put(new Key(oldestKey), oldestValue);
-      this._memoryDatastore.delete(oldestKey);
+      await this.dbDatastore.put(new Key(oldestKey), oldestValue);
+      this.memoryDatastore.delete(oldestKey);
     }
   }
 }

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRoot.ts

@@ -6,6 +6,7 @@ import {toRootHex} from "@lodestar/utils";
 import {EnvelopeReconstructionError} from "../../../chain/errors/index.js";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
+import {MAX_BODIES_PER_REQUEST} from "../../../util/execution.js";
 import {ExecutionPayloadEnvelopesByRootRequest} from "../../../util/types.js";
 import {prettyPrintPeerId} from "../../util.js";
 
@@ -19,10 +20,15 @@ export async function* onExecutionPayloadEnvelopesByRoot(
   // The gloas req/resp spec uses MIN_EPOCHS_FOR_BLOCK_REQUESTS to define the minimum range peers MUST serve.
   // Archival nodes may still serve older retained payloads to allow genesis sync.
 
-  // Resolve slots first so archived header envelopes are rebuilt in one EL batch, not one call per root
+  // Resolve slots first so archived envelopes can be reconstructed in EL batches.
+  // Duplicate roots are served once, since each archived envelope costs an EL fetch and a rebuild.
   const requests: {blockSlot: Slot; blockRootHex: RootHex}[] = [];
+  const seenRoots = new Set<RootHex>();
   for (const root of requestBody) {
     const rootHex = toRootHex(root);
+    if (seenRoots.has(rootHex)) continue;
+    seenRoots.add(rootHex);
+
     const block = chain.forkChoice.getBlockHexDefaultStatus(rootHex);
     // If the block is not in fork choice, it may be finalized. Attempt to find its slot in block archive
     const slot = block ? block.slot : await db.blockArchive.getSlotByRoot(root);
@@ -41,35 +47,43 @@ export async function* onExecutionPayloadEnvelopesByRoot(
     requests.push({blockSlot: slot, blockRootHex: rootHex});
   }
 
-  let envelopesBytes: (Uint8Array | null)[];
-  try {
-    // by-root allows omission, so a mismatched envelope is left out rather than failing the response
-    envelopesBytes = await chain.getSerializedExecutionPayloadEnvelopes(requests, "omit");
-  } catch (e) {
-    if (e instanceof EnvelopeReconstructionError) {
-      throw new ResponseError(
-        RespStatus.RESOURCE_UNAVAILABLE,
-        `Failed to reconstruct archived envelopes: ${e.message}`
-      );
+  // Yield between EL batches so the first response does not wait for the entire request.
+  let yielded = 0;
+  for (let i = 0; i < requests.length; i += MAX_BODIES_PER_REQUEST) {
+    const batch = requests.slice(i, i + MAX_BODIES_PER_REQUEST);
+    let envelopesBytes: (Uint8Array | null)[];
+    try {
+      // By-root allows omission, so a mismatched envelope is left out rather than failing the response.
+      envelopesBytes = await chain.getSerializedExecutionPayloadEnvelopes(batch, "omit");
+    } catch (e) {
+      if (e instanceof EnvelopeReconstructionError) {
+        // By-root permits partial responses, so an EL failure can end the response here.
+        if (yielded > 0) return;
+        throw new ResponseError(
+          RespStatus.RESOURCE_UNAVAILABLE,
+          `Failed to reconstruct archived envelopes: ${e.message}`
+        );
+      }
+      throw e;
     }
-    throw e;
-  }
 
-  for (let i = 0; i < requests.length; i++) {
-    const {blockSlot, blockRootHex} = requests[i];
-    const envelopeBytes = envelopesBytes[i];
-    if (envelopeBytes) {
-      yield {
-        data: envelopeBytes,
-        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(blockSlot)),
-      };
-    } else {
-      chain.logger.debug("Cannot serve ExecutionPayloadEnvelopesByRoot: envelope not found", {
-        slot: blockSlot,
-        root: blockRootHex,
-        peer: prettyPrintPeerId(peerId),
-        client: peerClient,
-      });
+    for (let j = 0; j < batch.length; j++) {
+      const {blockSlot, blockRootHex} = batch[j];
+      const envelopeBytes = envelopesBytes[j];
+      if (envelopeBytes) {
+        yielded++;
+        yield {
+          data: envelopeBytes,
+          boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(blockSlot)),
+        };
+      } else {
+        chain.logger.debug("Cannot serve ExecutionPayloadEnvelopesByRoot: envelope not found", {
+          slot: blockSlot,
+          root: blockRootHex,
+          peer: prettyPrintPeerId(peerId),
+          client: peerClient,
+        });
+      }
     }
   }
 }