@lodestar/beacon-node 1.50.0-dev.f2d839a25c → 1.50.0-dev.fc618d94e9

package/src/network/peers/datastore.ts

@@ -16,23 +16,23 @@ type AwaitGenerator<T, TReturn = any, TNext = any> = Generator<T, TReturn, TNext
  * This has a memory issue because all peer data stays in memory and loaded at startup time
  * This is written for libp2p >=0.35, we maintain the same mechanism but with bounded data structure
  * This datastore includes a memory datastore and fallback to db datastore
- * Use an in-memory datastore with last accessed time and _maxMemoryItems, on start it's empty (lazy load)
+ * Use an in-memory datastore with last accessed time and maxMemoryItems, on start it's empty (lazy load)
  * - get: Search in-memory datastore first, if not found search from db.
  *     - If found from db, add back to the in-memory datastore
  *     - Update lastAccessedMs
- * - put: move oldest items from memory to db if there are more than _maxMemoryItems items in memory
- *     -  update memory datastore, only update db datastore if there are at least _threshold dirty items
+ * - put: move oldest items from memory to db if there are more than maxMemoryItems items in memory
+ *     -  update memory datastore, only update db datastore if there are at least threshold dirty items
  *     -  Update lastAccessedMs
  */
 export class Eth2PeerDataStore extends BaseDatastore {
-  private _dbDatastore: LevelDatastore;
-  private _memoryDatastore: Map<string, MemoryItem>;
+  private dbDatastore: LevelDatastore;
+  private memoryDatastore: Map<string, MemoryItem>;
   /** Same to PersistentPeerStore of the old libp2p implementation */
-  private _dirtyItems = new Set<string>();
+  private dirtyItems = new Set<string>();
   /** If there are more dirty items than threshold, commit data to db */
-  private _threshold: number;
+  private threshold: number;
   /** If there are more memory items than this, prune oldest ones from memory and move to db */
-  private _maxMemoryItems: number;
+  private maxMemoryItems: number;
 
   constructor(
     dbDatastore: LevelDatastore | string,
@@ -47,18 +47,18 @@ export class Eth2PeerDataStore extends BaseDatastore {
       throw Error(`Threshold ${threshold} should be at most maxMemoryItems ${maxMemoryItems}`);
     }
 
-    this._dbDatastore = typeof dbDatastore === "string" ? new LevelDatastore(dbDatastore) : dbDatastore;
-    this._memoryDatastore = new Map();
-    this._threshold = threshold;
-    this._maxMemoryItems = maxMemoryItems;
+    this.dbDatastore = typeof dbDatastore === "string" ? new LevelDatastore(dbDatastore) : dbDatastore;
+    this.memoryDatastore = new Map();
+    this.threshold = threshold;
+    this.maxMemoryItems = maxMemoryItems;
   }
 
   async open(): Promise<void> {
-    return this._dbDatastore.open();
+    return this.dbDatastore.open();
   }
 
   async close(): Promise<void> {
-    return this._dbDatastore.close();
+    return this.dbDatastore.close();
   }
 
   async put(key: Key, val: Uint8Array, _options?: AbortOptions): Promise<Key> {
@@ -67,23 +67,23 @@ export class Eth2PeerDataStore extends BaseDatastore {
 
   /**
    * Same interface to put with "fromDb" option, if this item is updated back from db
-   * Move oldest items from memory data store to db if it's over this._maxMemoryItems
+   * Move oldest items from memory data store to db if it's over this.maxMemoryItems
    */
   async _put(key: Key, val: Uint8Array, fromDb = false): Promise<Key> {
-    while (this._memoryDatastore.size >= this._maxMemoryItems) {
+    while (this.memoryDatastore.size >= this.maxMemoryItems) {
       // it's likely this is called only 1 time
       await this.pruneMemoryDatastore();
     }
 
     const keyStr = key.toString();
-    const memoryItem = this._memoryDatastore.get(keyStr);
+    const memoryItem = this.memoryDatastore.get(keyStr);
     if (memoryItem) {
       // update existing
       memoryItem.lastAccessedMs = Date.now();
       memoryItem.data = val;
     } else {
       // new
-      this._memoryDatastore.set(keyStr, {data: val, lastAccessedMs: Date.now()});
+      this.memoryDatastore.set(keyStr, {data: val, lastAccessedMs: Date.now()});
     }
 
     if (!fromDb) await this._addDirtyItem(keyStr);
@@ -98,15 +98,15 @@ export class Eth2PeerDataStore extends BaseDatastore {
    */
   async get(key: Key, options?: AbortOptions): Promise<Uint8Array> {
     const keyStr = key.toString();
-    const memoryItem = this._memoryDatastore.get(keyStr);
+    const memoryItem = this.memoryDatastore.get(keyStr);
     if (memoryItem) {
       memoryItem.lastAccessedMs = Date.now();
       return memoryItem.data;
     }
 
     // this throws error if not found
-    const dbValue = await this._dbDatastore.get(key, options);
-    // don't call this._memoryDatastore.set directly
+    const dbValue = await this.dbDatastore.get(key, options);
+    // don't call this.memoryDatastore.set directly
     // we want to get through prune() logic with fromDb as true
     await this._put(key, dbValue, true);
     return dbValue;
@@ -125,30 +125,30 @@ export class Eth2PeerDataStore extends BaseDatastore {
   }
 
   async delete(key: Key, options?: AbortOptions): Promise<void> {
-    this._memoryDatastore.delete(key.toString());
-    await this._dbDatastore.delete(key, options);
+    this.memoryDatastore.delete(key.toString());
+    await this.dbDatastore.delete(key, options);
   }
 
   async *_all(q: Query, options?: AbortOptions): AwaitGenerator<Pair> {
-    for (const [key, value] of this._memoryDatastore.entries()) {
+    for (const [key, value] of this.memoryDatastore.entries()) {
       yield {
         key: new Key(key),
         value: value.data,
       };
     }
-    yield* this._dbDatastore.query(q, options);
+    yield* this.dbDatastore.query(q, options);
   }
 
   async *_allKeys(q: KeyQuery, options?: AbortOptions): AwaitGenerator<Key> {
-    for (const key of this._memoryDatastore.keys()) {
+    for (const key of this.memoryDatastore.keys()) {
       yield new Key(key);
     }
-    yield* this._dbDatastore.queryKeys(q, options);
+    yield* this.dbDatastore.queryKeys(q, options);
   }
 
   private async _addDirtyItem(keyStr: string): Promise<void> {
-    this._dirtyItems.add(keyStr);
-    if (this._dirtyItems.size >= this._threshold) {
+    this.dirtyItems.add(keyStr);
+    if (this.dirtyItems.size >= this.threshold) {
       try {
         await this._commitData();
       } catch (_e) {}
@@ -156,15 +156,15 @@ export class Eth2PeerDataStore extends BaseDatastore {
   }
 
   private async _commitData(): Promise<void> {
-    const batch = this._dbDatastore.batch();
-    for (const keyStr of this._dirtyItems) {
-      const memoryItem = this._memoryDatastore.get(keyStr);
+    const batch = this.dbDatastore.batch();
+    for (const keyStr of this.dirtyItems) {
+      const memoryItem = this.memoryDatastore.get(keyStr);
       if (memoryItem) {
         batch.put(new Key(keyStr), memoryItem.data);
       }
     }
     await batch.commit();
-    this._dirtyItems.clear();
+    this.dirtyItems.clear();
   }
 
   /**
@@ -175,7 +175,7 @@ export class Eth2PeerDataStore extends BaseDatastore {
     let oldestKey: string | undefined = undefined;
     let oldestValue: Uint8Array | undefined = undefined;
 
-    for (const [key, value] of this._memoryDatastore) {
+    for (const [key, value] of this.memoryDatastore) {
       if (value.lastAccessedMs < oldestAccessedMs) {
         oldestAccessedMs = value.lastAccessedMs;
         oldestKey = key;
@@ -184,8 +184,8 @@ export class Eth2PeerDataStore extends BaseDatastore {
     }
 
     if (oldestKey && oldestValue) {
-      await this._dbDatastore.put(new Key(oldestKey), oldestValue);
-      this._memoryDatastore.delete(oldestKey);
+      await this.dbDatastore.put(new Key(oldestKey), oldestValue);
+      this.memoryDatastore.delete(oldestKey);
     }
   }
 }

package/src/network/processor/gossipHandlers.ts

@@ -1065,12 +1065,12 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       const {serializedData} = gossipData;
       const {fork} = topic.boundary;
       const attesterSlashing = sszDeserialize(topic, serializedData);
-      await validateGossipAttesterSlashing(chain, attesterSlashing);
+      const verifiedDomains = await validateGossipAttesterSlashing(chain, attesterSlashing);
 
       // Handler - deferred to next event loop so the validation result propagates first
       callInNextEventLoop(() => {
         try {
-          chain.opPool.insertAttesterSlashing(fork, attesterSlashing);
+          chain.opPool.insertAttesterSlashing(fork, attesterSlashing, verifiedDomains);
           chain.forkChoice.onAttesterSlashing(attesterSlashing);
           chain.emitter.emit(routes.events.EventType.attesterSlashing, attesterSlashing);
         } catch (e) {
@@ -1089,12 +1089,12 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     }: GossipHandlerParamGeneric<GossipType.proposer_slashing>) => {
       const {serializedData} = gossipData;
       const proposerSlashing = sszDeserialize(topic, serializedData);
-      await validateGossipProposerSlashing(chain, proposerSlashing);
+      const verifiedDomain = await validateGossipProposerSlashing(chain, proposerSlashing);
 
       // Handler - deferred to next event loop so the validation result propagates first
       callInNextEventLoop(() => {
         try {
-          chain.opPool.insertProposerSlashing(proposerSlashing);
+          chain.opPool.insertProposerSlashing(proposerSlashing, verifiedDomain);
           chain.emitter.emit(routes.events.EventType.proposerSlashing, proposerSlashing);
         } catch (e) {
           logger.debug(
@@ -1341,6 +1341,9 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
                 case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
                 case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
                   core.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadGossipPayload");
+                  // The builder may have signed another envelope that is valid, keeping this one would make
+                  // by-root and range sync reuse it and never import the payload
+                  chain.seenPayloadEnvelopeInputCache.removeInvalid(payloadInput);
                   // Misbehaving peer, but could highlight an issue in another client
                   logLevel = LogLevel.warn;
                   break;

package/src/network/processor/gossipValidatorFn.ts

@@ -20,6 +20,7 @@ import {
   VoluntaryExitErrorCode,
 } from "../../chain/errors/index.js";
 import {Metrics} from "../../metrics/index.js";
+import {PeerIdStr} from "../../util/peerId.js";
 import {INetworkCore} from "../core/index.js";
 import {
   BatchGossipHandlerFn,
@@ -40,6 +41,8 @@ export type ValidatorFnModules = {
   core: INetworkCore;
 };
 
+export type OnFatalPeerFn = (peer: PeerIdStr) => void;
+
 type RejectPeerActionRule = {default: PeerAction; byCode?: Record<string, PeerAction>};
 
 /**
@@ -112,7 +115,7 @@ const gossipRejectPeerAction: Record<GossipType, RejectPeerActionRule> = {
   [GossipType.light_client_optimistic_update]: {default: PeerAction.HighToleranceError},
 };
 
-function rejectPeerAction(type: GossipType, code: string): PeerAction {
+export function rejectPeerAction(type: GossipType, code: string): PeerAction {
   const rule = gossipRejectPeerAction[type];
   return rule.byCode?.[code] ?? rule.default;
 }
@@ -123,7 +126,8 @@ function rejectPeerAction(type: GossipType, code: string): PeerAction {
  */
 export function getGossipValidatorBatchFn(
   gossipHandlers: GossipHandlers,
-  modules: ValidatorFnModules
+  modules: ValidatorFnModules,
+  onFatalPeer: OnFatalPeerFn
 ): GossipValidatorBatchFn {
   const {logger, metrics, core} = modules;
 
@@ -173,6 +177,7 @@ export function getGossipValidatorBatchFn(
             metrics?.networkProcessor.gossipAttestationRejectByReason.inc({reason: e.type.code});
             const peerAction = rejectPeerAction(type, e.type.code);
             core.reportPeer(propagationSource, peerAction, e.type.code);
+            if (peerAction === PeerAction.Fatal) onFatalPeer(propagationSource);
             logger.debug(
               `Gossip validation ${type} rejected`,
               {peer: propagationSource, clientAgent, clientVersion, peerAction},
@@ -208,7 +213,11 @@ export function getGossipValidatorBatchFn(
  *
  * @see getGossipHandlers for reasoning on why GossipHandlerFn are used for gossip validation.
  */
-export function getGossipValidatorFn(gossipHandlers: GossipHandlers, modules: ValidatorFnModules): GossipValidatorFn {
+export function getGossipValidatorFn(
+  gossipHandlers: GossipHandlers,
+  modules: ValidatorFnModules,
+  onFatalPeer: OnFatalPeerFn
+): GossipValidatorFn {
   const {logger, metrics, core} = modules;
 
   return async function gossipValidatorFn({
@@ -260,6 +269,7 @@ export function getGossipValidatorFn(gossipHandlers: GossipHandlers, modules: Va
           metrics?.networkProcessor.gossipValidationReject.inc({topic: type});
           const peerAction = rejectPeerAction(type, e.type.code);
           core.reportPeer(propagationSource, peerAction, e.type.code);
+          if (peerAction === PeerAction.Fatal) onFatalPeer(propagationSource);
           logger.debug(
             `Gossip validation ${type} rejected`,
             {peer: propagationSource, clientAgent, clientVersion, peerAction},

package/src/network/processor/index.ts

@@ -22,6 +22,7 @@ import {
   getParentRootFromSignedBeaconBlockSerialized,
   getPayloadPresentFromPayloadAttestationMessageSerialized,
 } from "../../util/sszBytes.js";
+import {INetworkCore} from "../core/index.js";
 import {NetworkEvent, NetworkEventBus} from "../events.js";
 import {
   GossipHandlers,
@@ -34,7 +35,12 @@ import {MAX_PEERS_PER_ROOT} from "./constants.js";
 import {createExtractBlockSlotRootFns} from "./extractSlotRootFns.js";
 import {GossipHandlerOpts, ValidatorFnsModules, getGossipHandlers} from "./gossipHandlers.js";
 import {createGossipQueues} from "./gossipQueues/index.js";
-import {ValidatorFnModules, getGossipValidatorBatchFn, getGossipValidatorFn} from "./gossipValidatorFn.js";
+import {
+  ValidatorFnModules,
+  getGossipValidatorBatchFn,
+  getGossipValidatorFn,
+  rejectPeerAction,
+} from "./gossipValidatorFn.js";
 import {PendingGossipsubMessage} from "./types.js";
 
 export * from "./types.js";
@@ -102,6 +108,12 @@ type SearchedRootEntry = {
  */
 const DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE = 32;
 
+/**
+ * The network worker bans a Fatal peer only on its next PeerManager heartbeat (30s). Until then the main thread
+ * would keep validating that peer's messages, so drop them here. 3 slots (36s) outlasts one heartbeat.
+ */
+export const FATAL_PEER_EXPIRY_SLOTS = 3;
+
 type WorkOpts = {
   bypassQueue?: boolean;
 };
@@ -232,6 +244,7 @@ type SearchUnknownRootTarget =
  */
 export class NetworkProcessor {
   private readonly chain: IBeaconChain;
+  private readonly core: INetworkCore;
   private readonly events: NetworkEventBus;
   private readonly logger: Logger;
   private readonly metrics: Metrics | null;
@@ -256,23 +269,31 @@ export class NetworkProcessor {
     () => new MapDef<RootHex, SearchedRootEntry>(() => ({}))
   );
   private bufferedRootsBySlot = new MapDef<Slot, Set<RootHex>>(() => new Set());
+  /** peer -> clock slot when it was Fatal'ed */
+  private readonly fatalPeers = new Map<PeerIdStr, Slot>();
 
   constructor(
     modules: NetworkProcessorModules,
     private readonly opts: NetworkProcessorOpts
   ) {
-    const {chain, events, logger, metrics} = modules;
+    const {chain, core, events, logger, metrics} = modules;
     this.chain = chain;
+    this.core = core;
     this.events = events;
     this.metrics = metrics;
     this.logger = logger;
     this.events = events;
     this.gossipQueues = createGossipQueues();
     this.gossipTopicConcurrency = mapValues(this.gossipQueues, () => 0);
-    this.gossipValidatorFn = getGossipValidatorFn(modules.gossipHandlers ?? getGossipHandlers(modules, opts), modules);
+    this.gossipValidatorFn = getGossipValidatorFn(
+      modules.gossipHandlers ?? getGossipHandlers(modules, opts),
+      modules,
+      this.onFatalPeer
+    );
     this.gossipValidatorBatchFn = getGossipValidatorBatchFn(
       modules.gossipHandlers ?? getGossipHandlers(modules, opts),
-      modules
+      modules,
+      this.onFatalPeer
     );
 
     events.on(NetworkEvent.pendingGossipsubMessage, this.onPendingGossipsubMessage);
@@ -406,19 +427,47 @@ export class NetworkProcessor {
     }
   }
 
+  private onFatalPeer = (peer: PeerIdStr): void => {
+    this.fatalPeers.set(peer, this.chain.clock.currentSlot);
+  };
+
   private onPendingGossipsubMessage = (message: PendingGossipsubMessage): void => {
     const topicType = message.topic.type;
+    if (this.fatalPeers.has(message.propagationSource)) {
+      this.metrics?.networkProcessor.gossipValidationError.inc({topic: topicType, error: GossipErrorCode.FATAL_PEER});
+      // No need to report the dropped job to gossip. It will be eventually pruned from the mcache
+      return;
+    }
+
     const extractBlockSlotRootFn = this.extractBlockSlotRootFns[topicType];
+    if (extractBlockSlotRootFn === undefined) {
+      // some messages don't have slot and root
+      this.pushPendingGossipsubMessageToQueue(message);
+      return;
+    }
 
     // 1st extract round: make sure slot is in range and if block root is not available
     // proactively search for it + queue the message
-    const slotRoot = extractBlockSlotRootFn
-      ? extractBlockSlotRootFn(message.msg.data, message.topic.boundary.fork)
-      : null;
+    const slotRoot = extractBlockSlotRootFn(message.msg.data, message.topic.boundary.fork);
     if (slotRoot === null) {
-      // some messages don't have slot and root
-      // if the msg.data is invalid, message will be rejected when deserializing data in later phase (gossipValidatorFn)
-      this.pushPendingGossipsubMessageToQueue(message);
+      // DOS protection: null means malformed bytes or slot >= 2^32, neither is forwarded by honest peers.
+      const {propagationSource, clientAgent, clientVersion} = message;
+      const code = GossipErrorCode.INVALID_SLOT;
+      const peerAction = rejectPeerAction(topicType, code);
+      this.metrics?.networkProcessor.gossipValidationReject.inc({topic: topicType});
+      this.core.reportPeer(propagationSource, peerAction, code);
+      this.logger.debug(`Gossip validation ${topicType} rejected due to null slot`, {
+        peer: propagationSource,
+        clientAgent,
+        clientVersion,
+        peerAction,
+        code,
+      });
+      this.events.emit(NetworkEvent.gossipMessageValidationResult, {
+        msgId: message.msgId,
+        propagationSource,
+        acceptance: TopicValidatorResult.Reject,
+      });
       return;
     }
 
@@ -872,6 +921,10 @@ export class NetworkProcessor {
     for (const slot of this.bufferedRootsBySlot.keys()) {
       if (slot <= minSlot) this.bufferedRootsBySlot.delete(slot);
     }
+
+    for (const [peer, slot] of this.fatalPeers) {
+      if (clockSlot - slot > FATAL_PEER_EXPIRY_SLOTS) this.fatalPeers.delete(peer);
+    }
   };
 
   private executeWork(): void {
@@ -927,6 +980,32 @@ export class NetworkProcessor {
   private async processPendingGossipsubMessage(
     messageOrArray: PendingGossipsubMessage | PendingGossipsubMessage[]
   ): Promise<void> {
+    // drop messages from peers Fatal'ed after these were queued or started awaiting a block/envelope
+    // No need to report the dropped jobs to gossip. They will be eventually pruned from the mcache
+    if (this.fatalPeers.size > 0) {
+      if (Array.isArray(messageOrArray)) {
+        const allowed: PendingGossipsubMessage[] = [];
+        for (const msg of messageOrArray) {
+          if (this.fatalPeers.has(msg.propagationSource)) {
+            this.metrics?.networkProcessor.gossipValidationError.inc({
+              topic: msg.topic.type,
+              error: GossipErrorCode.FATAL_PEER,
+            });
+          } else {
+            allowed.push(msg);
+          }
+        }
+        if (allowed.length === 0) return;
+        messageOrArray = allowed;
+      } else if (this.fatalPeers.has(messageOrArray.propagationSource)) {
+        this.metrics?.networkProcessor.gossipValidationError.inc({
+          topic: messageOrArray.topic.type,
+          error: GossipErrorCode.FATAL_PEER,
+        });
+        return;
+      }
+    }
+
     const nowSec = Date.now() / 1000;
     if (Array.isArray(messageOrArray)) {
       for (const msg of messageOrArray) {

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRoot.ts

@@ -6,6 +6,7 @@ import {toRootHex} from "@lodestar/utils";
 import {EnvelopeReconstructionError} from "../../../chain/errors/index.js";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
+import {MAX_BODIES_PER_REQUEST} from "../../../util/execution.js";
 import {ExecutionPayloadEnvelopesByRootRequest} from "../../../util/types.js";
 import {prettyPrintPeerId} from "../../util.js";
 
@@ -19,10 +20,15 @@ export async function* onExecutionPayloadEnvelopesByRoot(
   // The gloas req/resp spec uses MIN_EPOCHS_FOR_BLOCK_REQUESTS to define the minimum range peers MUST serve.
   // Archival nodes may still serve older retained payloads to allow genesis sync.
 
-  // Resolve slots first so archived header envelopes are rebuilt in one EL batch, not one call per root
+  // Resolve slots first so archived envelopes can be reconstructed in EL batches.
+  // Duplicate roots are served once, since each archived envelope costs an EL fetch and a rebuild.
   const requests: {blockSlot: Slot; blockRootHex: RootHex}[] = [];
+  const seenRoots = new Set<RootHex>();
   for (const root of requestBody) {
     const rootHex = toRootHex(root);
+    if (seenRoots.has(rootHex)) continue;
+    seenRoots.add(rootHex);
+
     const block = chain.forkChoice.getBlockHexDefaultStatus(rootHex);
     // If the block is not in fork choice, it may be finalized. Attempt to find its slot in block archive
     const slot = block ? block.slot : await db.blockArchive.getSlotByRoot(root);
@@ -41,35 +47,43 @@ export async function* onExecutionPayloadEnvelopesByRoot(
     requests.push({blockSlot: slot, blockRootHex: rootHex});
   }
 
-  let envelopesBytes: (Uint8Array | null)[];
-  try {
-    // by-root allows omission, so a mismatched envelope is left out rather than failing the response
-    envelopesBytes = await chain.getSerializedExecutionPayloadEnvelopes(requests, "omit");
-  } catch (e) {
-    if (e instanceof EnvelopeReconstructionError) {
-      throw new ResponseError(
-        RespStatus.RESOURCE_UNAVAILABLE,
-        `Failed to reconstruct archived envelopes: ${e.message}`
-      );
+  // Yield between EL batches so the first response does not wait for the entire request.
+  let yielded = 0;
+  for (let i = 0; i < requests.length; i += MAX_BODIES_PER_REQUEST) {
+    const batch = requests.slice(i, i + MAX_BODIES_PER_REQUEST);
+    let envelopesBytes: (Uint8Array | null)[];
+    try {
+      // By-root allows omission, so an envelope that cannot be rebuilt is left out rather than failing the response.
+      envelopesBytes = await chain.getSerializedExecutionPayloadEnvelopes(batch, "omit");
+    } catch (e) {
+      if (e instanceof EnvelopeReconstructionError) {
+        // By-root permits partial responses, so an EL failure can end the response here.
+        if (yielded > 0) return;
+        throw new ResponseError(
+          RespStatus.RESOURCE_UNAVAILABLE,
+          `Failed to reconstruct archived envelopes: ${e.message}`
+        );
+      }
+      throw e;
     }
-    throw e;
-  }
 
-  for (let i = 0; i < requests.length; i++) {
-    const {blockSlot, blockRootHex} = requests[i];
-    const envelopeBytes = envelopesBytes[i];
-    if (envelopeBytes) {
-      yield {
-        data: envelopeBytes,
-        boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(blockSlot)),
-      };
-    } else {
-      chain.logger.debug("Cannot serve ExecutionPayloadEnvelopesByRoot: envelope not found", {
-        slot: blockSlot,
-        root: blockRootHex,
-        peer: prettyPrintPeerId(peerId),
-        client: peerClient,
-      });
+    for (let j = 0; j < batch.length; j++) {
+      const {blockSlot, blockRootHex} = batch[j];
+      const envelopeBytes = envelopesBytes[j];
+      if (envelopeBytes) {
+        yielded++;
+        yield {
+          data: envelopeBytes,
+          boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(blockSlot)),
+        };
+      } else {
+        chain.logger.debug("Cannot serve ExecutionPayloadEnvelopesByRoot: envelope not found", {
+          slot: blockSlot,
+          root: blockRootHex,
+          peer: prettyPrintPeerId(peerId),
+          client: peerClient,
+        });
+      }
     }
   }
 }

package/src/node/nodejs.ts

@@ -1,6 +1,12 @@
 import {setMaxListeners} from "node:events";
 import {PrivateKey} from "@libp2p/interface";
 import {Registry} from "prom-client";
+import {
+  init as initNativeMetrics,
+  registerLocalValidator as registerNativeLocalValidator,
+  scrapeMetrics as scrapeNativeMetrics,
+  unregisterLocalValidator as unregisterNativeLocalValidator,
+} from "@chainsafe/lodestar-z/metrics";
 import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
 import {hasher} from "@chainsafe/persistent-merkle-tree";
 import {BeaconApiMethods} from "@lodestar/api/beacon/server";
@@ -171,7 +177,13 @@ export class BeaconNode {
       // monitoring relies on metrics data
       opts.monitoring.endpoint
     ) {
-      metrics = createMetrics(opts.metrics, anchorState.genesisTime, metricsRegistries);
+      if (opts.chain.nativeStateTransition) {
+        initNativeMetrics();
+      }
+
+      metrics = createMetrics(opts.metrics, anchorState.genesisTime, metricsRegistries, {
+        includeStateTransitionMetrics: !opts.chain.nativeStateTransition,
+      });
       initBeaconMetrics(metrics, anchorState);
       // Since the db is instantiated before this, metrics must be injected manually afterwards
       db.setMetrics(metrics.db, metrics.flatFileStore);
@@ -185,7 +197,13 @@ export class BeaconNode {
             config,
             anchorState.genesisTime,
             logger.child({module: LoggerModule.vmon}),
-            opts.validatorMonitor
+            opts.validatorMonitor,
+            opts.chain.nativeStateTransition
+              ? {
+                  registerLocalValidator: registerNativeLocalValidator,
+                  unregisterLocalValidator: unregisterNativeLocalValidator,
+                }
+              : null
           )
         : null;
 
@@ -303,7 +321,17 @@ export class BeaconNode {
     const metricsServer = opts.metrics.enabled
       ? await getHttpMetricsServer(opts.metrics, {
           register: (metrics as Metrics).register,
-          getOtherMetrics: async () => Promise.all([network.scrapeMetrics(), chain.archiveStore.scrapeMetrics()]),
+          getOtherMetrics: async () => {
+            const otherMetrics = await Promise.all([network.scrapeMetrics(), chain.archiveStore.scrapeMetrics()]);
+            if (opts.chain.nativeStateTransition) {
+              try {
+                otherMetrics.push(scrapeNativeMetrics());
+              } catch (e) {
+                logger.warn("Failed to scrape native state-transition metrics", {}, e as Error);
+              }
+            }
+            return otherMetrics;
+          },
           logger: logger.child({module: LoggerModule.metrics}),
         })
       : null;

package/src/sync/range/batch.ts

@@ -773,6 +773,7 @@ export class Batch {
     // remove any downloaded blocks and re-attempt
     // TODO(fulu): need to remove the bad blocks from the SeenBlockInputCache
     this.state = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
+    this.resetRequests();
   }
 
   private onProcessingError(attempt: FailedAttempt): void {
@@ -784,6 +785,16 @@ export class Batch {
     // remove any downloaded blocks and re-attempt
     // TODO(fulu): need to remove the bad blocks from the SeenBlockInputCache
     this.state = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
+    this.resetRequests();
+  }
+
+  /**
+   * After a partial download `requests` only cover the data still missing from the downloaded blocks,
+   * e.g. a single column. Once those blocks are dropped, request the whole batch again, otherwise
+   * every peer fails the retry for lack of blocks to validate the data against.
+   */
+  private resetRequests(): void {
+    this.requests = this.getRequests([]);
   }
 
   /** Helper to construct typed BatchError. Stack traces are correct as the error is thrown above */

package/src/sync/range/chain.ts

@@ -225,6 +225,9 @@ export class SyncChain {
       Math.floor((localFinalizedEpoch - this.lastEpochWithProcessBlocks) / EPOCHS_PER_BATCH) * EPOCHS_PER_BATCH;
     this.advanceChain(lastEpochWithProcessBlocksAligned);
 
+    // stopSyncing() cleared the retry timer, but peers can still be in rate limit backoff
+    this.scheduleRateLimitBackoffRetry();
+
     // Potentially download new batches and process pending
     this.triggerBatchDownloader();
     this.triggerBatchProcessor();
@@ -417,8 +420,10 @@ export class SyncChain {
     this.rateLimitBackoffTimeout = setTimeout(
       () => {
         this.rateLimitBackoffTimeout = undefined;
-        this.triggerBatchDownloader();
+        // Must run before the downloader, it drops expired peers without triggering a download. If it ran
+        // after, a backoff expiring in between would leave the peer idle with no retry scheduled.
         this.scheduleRateLimitBackoffRetry();
+        this.triggerBatchDownloader();
       },
       Math.max(0, retryAt - now)
     );