@lodestar/beacon-node 1.46.0-dev.d9065a2b16 → 1.46.0-dev.de5f89eae1

package/src/network/core/networkCore.ts

@@ -70,6 +70,15 @@ export type BaseNetworkInit = {
   initialCustodyGroupCount: number;
 };
 
+/** Counts of what is still keeping this thread's event loop alive, e.g. `TCPWrap=3,Timeout=1` */
+function formatActiveResources(): string {
+  const counts = new Map<string, number>();
+  for (const resource of process.getActiveResourcesInfo()) {
+    counts.set(resource, (counts.get(resource) ?? 0) + 1);
+  }
+  return Array.from(counts, ([name, count]) => `${name}=${count}`).join(",");
+}
+
 /**
  * This class is meant to work both:
  * - In a libp2p worker
@@ -289,7 +298,10 @@ export class NetworkCore implements INetworkCore {
     this.attnetsService.close();
     this.syncnetsService.close();
     await this.libp2p.stop();
-    this.logger.debug("network lib2p closed");
+    // Diagnostic for the shutdown hang, this thread can spin in `Environment::CleanupHandles()` on
+    // a handle that never closes and `Worker.terminate()` then never resolves. Diffing this list
+    // between a clean and a stuck shutdown should narrow down which handle it is
+    this.logger.debug("network lib2p closed", {activeResources: formatActiveResources()});
 
     this.closed = true;
   }

package/src/network/core/networkCoreWorkerHandler.ts

@@ -10,6 +10,7 @@ import {BeaconConfig, chainConfigToJson} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ResponseIncoming, ResponseOutgoing} from "@lodestar/reqresp";
 import {Status} from "@lodestar/types";
+import {withTimeout} from "@lodestar/utils";
 import {Metrics} from "../../metrics/index.js";
 import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js";
 import {PeerIdStr, peerIdFromString} from "../../util/peerId.js";
@@ -58,6 +59,10 @@ type WorkerNetworkCoreModules = WorkerNetworkCoreInitModules & {
 
 const NETWORK_WORKER_EXIT_TIMEOUT_MS = 1000;
 const NETWORK_WORKER_EXIT_RETRY_COUNT = 3;
+/** `getApi().close()` is an unbounded RPC into the worker and runs before `BeaconNode.close()`
+ * archives chain state to disk, so it is bounded here. Set well above the ~2-4s a close takes on
+ * mainnet, tripping it leaves libp2p half torn down */
+const NETWORK_CORE_CLOSE_TIMEOUT_MS = 5000;
 
 /**
  * NetworkCore implementation using a Worker thread
@@ -165,15 +170,25 @@ export class WorkerNetworkCore implements INetworkCore {
 
   async close(): Promise<void> {
     this.modules.logger.debug("closing network core running in network worker");
-    await this.getApi().close();
+    try {
+      await withTimeout(async () => this.getApi().close(), NETWORK_CORE_CLOSE_TIMEOUT_MS);
+    } catch (e) {
+      this.modules.logger.debug("Error closing network core", {}, e as Error);
+    }
     this.modules.logger.debug("terminating network worker");
-    await terminateWorkerThread({
+    const terminated = await terminateWorkerThread({
       worker: this.getApi(),
       retryCount: NETWORK_WORKER_EXIT_RETRY_COUNT,
       retryMs: NETWORK_WORKER_EXIT_TIMEOUT_MS,
       logger: this.modules.logger,
     });
-    this.modules.logger.debug("terminated network worker");
+    if (terminated) {
+      this.modules.logger.debug("terminated network worker");
+    } else {
+      // Nothing more to do, `process.exit()` joins the thread regardless so the process still waits
+      // on it. What matters is returning, so the caller can archive state and close the db
+      this.modules.logger.debug("Network worker did not terminate in time, continuing shutdown without it");
+    }
   }
 
   async test(): Promise<void> {

package/src/network/gossip/topic.ts

@@ -8,11 +8,13 @@ import {
   MAX_DATA_COLUMN_SIDECAR_SIZE,
   MAX_SIGNED_AGGREGATE_AND_PROOF_SIZE,
   MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE,
+  MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE_HEZE,
   SYNC_COMMITTEE_SUBNET_COUNT,
   isForkPostAltair,
   isForkPostElectra,
   isForkPostFulu,
   isForkPostGloas,
+  isForkPostHeze,
 } from "@lodestar/params";
 import {Attestation, SingleAttestation, ssz, sszTypesFor} from "@lodestar/types";
 import {GossipAction, GossipActionError, GossipErrorCode} from "../../chain/errors/gossipValidation.js";
@@ -130,7 +132,7 @@ export function getGossipSSZType(topic: GossipTopic) {
     case GossipType.payload_attestation_message:
       return ssz.gloas.PayloadAttestationMessage;
     case GossipType.execution_payload_bid:
-      return ssz.gloas.SignedExecutionPayloadBid;
+      return isForkPostGloas(fork) ? sszTypesFor(fork).SignedExecutionPayloadBid : ssz.gloas.SignedExecutionPayloadBid;
     case GossipType.proposer_preferences:
       return ssz.gloas.SignedProposerPreferences;
   }
@@ -154,7 +156,7 @@ export function getGossipSSZMaxSize(topic: GossipTopic, maxPayloadSize: number,
     case GossipType.execution_payload:
       return maxPayloadSize;
     case GossipType.execution_payload_bid:
-      return MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE;
+      return isForkPostHeze(fork) ? MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE_HEZE : MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE;
     default:
       return (sszType ?? getGossipSSZType(topic)).maxSize;
   }

package/src/network/network.ts

@@ -147,6 +147,7 @@ export class Network implements INetwork {
     this.chain.emitter.on(ChainEvent.updateTargetCustodyGroupCount, this.onTargetGroupCountUpdated);
     this.chain.emitter.on(ChainEvent.publishDataColumns, this.onPublishDataColumns);
     this.chain.emitter.on(ChainEvent.publishBlobSidecars, this.onPublishBlobSidecars);
+    this.chain.emitter.on(ChainEvent.publishProposerSlashing, this.onPublishProposerSlashing);
     this.chain.emitter.on(ChainEvent.updateStatus, this.onUpdateStatus);
   }
 
@@ -244,6 +245,7 @@ export class Network implements INetwork {
     this.chain.emitter.off(ChainEvent.updateTargetCustodyGroupCount, this.onTargetGroupCountUpdated);
     this.chain.emitter.off(ChainEvent.publishDataColumns, this.onPublishDataColumns);
     this.chain.emitter.off(ChainEvent.publishBlobSidecars, this.onPublishBlobSidecars);
+    this.chain.emitter.off(ChainEvent.publishProposerSlashing, this.onPublishProposerSlashing);
     this.chain.emitter.off(ChainEvent.updateStatus, this.onUpdateStatus);
     await this.core.close();
 
@@ -455,7 +457,8 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.proposer_slashing>(
       {type: GossipType.proposer_slashing, boundary},
-      proposerSlashing
+      proposerSlashing,
+      {ignoreDuplicatePublishError: true}
     );
   }
 
@@ -877,6 +880,18 @@ export class Network implements INetwork {
     return promiseAllMaybeAsync(sidecars.map((sidecar) => () => this.publishBlobSidecar(sidecar)));
   };
 
+  private onPublishProposerSlashing = async (proposerSlashing: phase0.ProposerSlashing): Promise<void> => {
+    try {
+      await this.publishProposerSlashing(proposerSlashing);
+    } catch (e) {
+      this.logger.debug(
+        "Error publishing proposer slashing",
+        {proposerIndex: proposerSlashing.signedHeader1.message.proposerIndex},
+        e as Error
+      );
+    }
+  };
+
   private onUpdateStatus = async (): Promise<void> => {
     await this.core.updateStatus(this.chain.getStatus());
   };

package/src/network/processor/gossipHandlers.ts

@@ -257,6 +257,13 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         chain.seenPayloadEnvelopeInputCache.prune(blockRootHex);
       }
       throw e;
+    } finally {
+      // The block received from the network may have established an equivocation, either by conflicting
+      // with a previously observed block root (REPEAT_PROPOSAL) or with a root observed during validation
+      const proposerIndex = signedBlock.message.proposerIndex;
+      if (chain.seenBlockProposers.isEquivocating(slot, proposerIndex)) {
+        chain.processProposerEquivocation(slot, proposerIndex);
+      }
     }
   }
 

package/src/network/processor/index.ts

@@ -461,7 +461,8 @@ export class NetworkProcessor {
           break;
         }
         case GossipType.execution_payload_bid: {
-          // instead of searching for the message root, this searches for the parent root
+          // Search for the parent independently of current head compatibility. A bid may arrive
+          // before its parent block, so compatibility is checked later during gossip validation.
           const parentBlockRoot = getParentBlockRootFromSignedExecutionPayloadBidSerialized(message.msg.data);
           const parentBlockHash = getParentBlockHashFromSignedExecutionPayloadBidSerialized(message.msg.data);
           if (

package/src/network/reqresp/handlers/lightClientFinalityUpdate.ts

@@ -12,7 +12,7 @@ export async function* onLightClientFinalityUpdate(chain: IBeaconChain): AsyncIt
     throw new ResponseError(RespStatus.RESOURCE_UNAVAILABLE, "No latest finality update available");
   }
 
-  const boundary = chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(update.signatureSlot));
+  const boundary = chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(update.attestedHeader.beacon.slot));
   const type = responseSszTypeByMethod[ReqRespMethod.LightClientFinalityUpdate](boundary.fork, 0);
   yield {
     data: type.serialize(update),

package/src/network/reqresp/handlers/lightClientOptimisticUpdate.ts

@@ -12,7 +12,7 @@ export async function* onLightClientOptimisticUpdate(chain: IBeaconChain): Async
     throw new ResponseError(RespStatus.RESOURCE_UNAVAILABLE, "No latest optimistic update available");
   }
 
-  const boundary = chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(update.signatureSlot));
+  const boundary = chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(update.attestedHeader.beacon.slot));
   const type = responseSszTypeByMethod[ReqRespMethod.LightClientOptimisticUpdate](boundary.fork, 0);
   yield {
     data: type.serialize(update),

package/src/network/reqresp/handlers/lightClientUpdatesByRange.ts

@@ -19,11 +19,15 @@ export async function* onLightClientUpdatesByRange(
   assertLightClientServer(chain.lightClientServer);
 
   const count = Math.min(MAX_REQUEST_LIGHT_CLIENT_UPDATES, requestBody.count);
+  if (!Number.isSafeInteger(requestBody.startPeriod)) {
+    throw new ResponseError(RespStatus.INVALID_REQUEST, "startPeriod out of range");
+  }
   let started = false;
-  for (let period = requestBody.startPeriod; period < requestBody.startPeriod + count; period++) {
+  for (let i = 0; i < count; i++) {
+    const period = requestBody.startPeriod + i;
     try {
       const update = await chain.lightClientServer.getUpdate(period);
-      const boundary = chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(update.signatureSlot));
+      const boundary = chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(update.attestedHeader.beacon.slot));
       const type = responseSszTypeByMethod[ReqRespMethod.LightClientUpdatesByRange](boundary.fork, 0);
 
       yield {

package/src/network/reqresp/protocols.ts

@@ -1,6 +1,6 @@
 import {BeaconConfig} from "@lodestar/config";
-import {ForkName} from "@lodestar/params";
-import {ContextBytesFactory, ContextBytesType, Encoding} from "@lodestar/reqresp";
+import {ForkName, MAX_DATA_COLUMN_SIDECAR_SIZE, isForkPostGloas} from "@lodestar/params";
+import {ContextBytesFactory, ContextBytesType, Encoding, TypeSizes} from "@lodestar/reqresp";
 import {rateLimitQuotas} from "./rateLimit.js";
 import {ProtocolNoHandler, ReqRespMethod, Version, requestSszTypeByMethod, responseSszTypeByMethod} from "./types.js";
 
@@ -143,15 +143,43 @@ type ProtocolSummary = {
 };
 
 function toProtocol(protocol: ProtocolSummary) {
-  return (fork: ForkName, config: BeaconConfig): ProtocolNoHandler => ({
-    method: protocol.method,
-    version: protocol.version,
-    encoding: Encoding.SSZ_SNAPPY,
-    contextBytes: toContextBytes(protocol.contextBytesType, config),
-    inboundRateLimits: rateLimitQuotas(fork, config)[protocol.method],
-    requestSizes: requestSszTypeByMethod(fork, config)[protocol.method],
-    responseSizes: (fork) => responseSszTypeByMethod[protocol.method](fork, protocol.version),
-  });
+  return (fork: ForkName, config: BeaconConfig): ProtocolNoHandler => {
+    const requestType = requestSszTypeByMethod(fork, config)[protocol.method];
+    return {
+      method: protocol.method,
+      version: protocol.version,
+      encoding: Encoding.SSZ_SNAPPY,
+      contextBytes: toContextBytes(protocol.contextBytesType, config),
+      inboundRateLimits: rateLimitQuotas(fork, config)[protocol.method],
+      requestSizes: requestType === null ? null : clampTypeSizes(requestType, protocol.method, fork, config),
+      responseSizes: (fork) =>
+        clampTypeSizes(responseSszTypeByMethod[protocol.method](fork, protocol.version), protocol.method, fork, config),
+    };
+  };
+}
+
+/**
+ * Bound the sizes accepted from the ssz-snappy length-prefix. Gloas progressive containers have broad
+ * theoretical SSZ max sizes so the preset p2p bounds must be used instead.
+ *
+ * The length-prefix must be within the size bounds derived from the payload SSZ type or `MAX_PAYLOAD_SIZE`,
+ * whichever is smaller, see
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.12/specs/phase0/p2p-interface.md#encoding-strategies.
+ * Type-specific SSZ bounds supersede the bounds derived from the SSZ type, see
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.12/specs/gloas/p2p-interface.md#type-specific-ssz-bounds.
+ */
+function clampTypeSizes(type: TypeSizes, method: ReqRespMethod, fork: ForkName, config: BeaconConfig): TypeSizes {
+  let typeSpecificBound = config.MAX_PAYLOAD_SIZE;
+  if (isForkPostGloas(fork)) {
+    switch (method) {
+      case ReqRespMethod.DataColumnSidecarsByRange:
+      case ReqRespMethod.DataColumnSidecarsByRoot:
+        typeSpecificBound = MAX_DATA_COLUMN_SIDECAR_SIZE;
+        break;
+    }
+  }
+
+  return {minSize: type.minSize, maxSize: Math.min(type.maxSize, typeSpecificBound)};
 }
 
 function toContextBytes(type: ContextBytesType, config: BeaconConfig): ContextBytesFactory {

package/src/network/reqresp/rateLimit.ts

@@ -87,7 +87,7 @@ export const rateLimitQuotas: (fork: ForkName, config: BeaconConfig) => Record<R
     ),
   },
   [ReqRespMethod.ExecutionPayloadEnvelopesByRange]: {
-    byPeer: {quota: config.MAX_REQUEST_BLOCKS_DENEB, quotaTimeMs: 10_000},
+    byPeer: {quota: config.MAX_REQUEST_PAYLOADS, quotaTimeMs: 10_000},
     getRequestCount: getRequestCountFn(
       fork,
       config,

package/src/network/reqresp/score.ts

@@ -33,9 +33,28 @@ export function onOutgoingReqRespError(e: RequestError, method: ReqRespMethod):
 
     case RequestErrorCode.DIAL_TIMEOUT:
     case RequestErrorCode.DIAL_ERROR:
-      return e.message.includes(multiStreamSelectErrorCodes.protocolSelectionFailed) && method === ReqRespMethod.Ping
-        ? PeerAction.Fatal
-        : PeerAction.LowToleranceError;
+      // `RequestError` renders `e.message` as just the error code (see `renderErrorMessage`), so the
+      // multistream "protocol selection failed" text is only on the wrapped inner error, which is
+      // available on `DIAL_ERROR` via `e.type.error` (`DIAL_TIMEOUT` carries no inner error).
+      if (
+        e.type.code === RequestErrorCode.DIAL_ERROR &&
+        e.type.error.message.includes(multiStreamSelectErrorCodes.protocolSelectionFailed)
+      ) {
+        // Peer does not support the protocol, a real incompatibility rather than a transient
+        // failure, so keep the stronger penalty (Fatal for Ping, as before).
+        return method === ReqRespMethod.Ping ? PeerAction.Fatal : PeerAction.LowToleranceError;
+      }
+      switch (method) {
+        // Ping and Status are liveness probes; their dial timeouts are dominated by transient
+        // network congestion rather than peer misbehavior, so penalize leniently to avoid
+        // self-inflicted peer starvation (https://github.com/ChainSafe/lodestar/issues/9562),
+        // while still applying some penalty so genuinely dead peers eventually free the slot.
+        case ReqRespMethod.Ping:
+        case ReqRespMethod.Status:
+          return PeerAction.HighToleranceError;
+        default:
+          return PeerAction.LowToleranceError;
+      }
     // TODO: Detect SSZDecodeError and return PeerAction.Fatal
 
     case RequestErrorCode.RESP_TIMEOUT:

package/src/sync/constants.ts

@@ -15,12 +15,13 @@ export const MAX_BATCH_DOWNLOAD_ATTEMPTS = 5;
 export const RATE_LIMITED_PEER_BACKOFF_MS = 5_000;
 
 /**
- * Consider batch faulty after downloading and processing this number of times
- * as in https://github.com/ChainSafe/lodestar/issues/8147 we cannot proceed the sync chain if there is unknown parent
- * from prior batch. For example a peer may send us a non-canonical chain segment or not returning all blocks
- * in that case we should throw error and `RangeSync` should remove that error chain and add a new one.
+ * Consider a batch faulty once its failed processing attempts EXCEED this number, i.e. the batch is
+ * processed up to `MAX_BATCH_PROCESSING_ATTEMPTS + 1` times before teardown (the check is `>`, same
+ * convention as `MAX_BATCH_DOWNLOAD_ATTEMPTS`).
+ * Some processing failures can be caused by a different batch, so range sync classifies the fault
+ * before deciding which batch to redownload.
  **/
-export const MAX_BATCH_PROCESSING_ATTEMPTS = 0;
+export const MAX_BATCH_PROCESSING_ATTEMPTS = 3;
 
 /**
  * Number of slots to offset batches.

package/src/sync/range/batch.ts

@@ -5,6 +5,7 @@ import {LodestarError, byteArrayEquals, prettyPrintIndices, toRootHex} from "@lo
 import {isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
 import {isDaOutOfRange} from "../../chain/blocks/blockInput/utils.js";
+import {PayloadError, PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
 import {PayloadEnvelopeInput} from "../../chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js";
 import {BlockError, BlockErrorCode} from "../../chain/errors/index.js";
 import {ZERO_HASH} from "../../constants/constants.js";
@@ -64,6 +65,7 @@ export type DownloadSuccessState = {
   status: BatchStatus.AwaitingProcessing;
   blocks: IBlockInput[];
   payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null;
+  attempt: Attempt;
 };
 
 export type BatchState =
@@ -589,7 +591,11 @@ export class Batch {
     }
 
     if (allComplete) {
-      this.state = {status: BatchStatus.AwaitingProcessing, blocks, payloadEnvelopes: newPayloadEnvelopes};
+      const attempt: Attempt = {
+        peers: this.getSuccessfulPeers(),
+        hash: hashBlocks(blocks, this.config),
+      };
+      this.state = {status: BatchStatus.AwaitingProcessing, blocks, payloadEnvelopes: newPayloadEnvelopes, attempt};
     } else {
       this.state = {status: BatchStatus.AwaitingDownload, blocks, payloadEnvelopes: newPayloadEnvelopes};
       this.requests = this.getRequests(blocks);
@@ -646,15 +652,11 @@ export class Batch {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.AwaitingProcessing));
     }
 
-    const blocks = this.state.blocks;
-    const payloadEnvelopes = this.state.payloadEnvelopes;
-    const hash = hashBlocks(blocks, this.config); // tracks blocks to report peer on processing error
-    // Reset successfulDownloads in case another download attempt needs to be made. When Attempt is successful or not
-    // the peers that the data came from will be handled by the Attempt that goes for processing.
-    const peers = this.getSuccessfulPeers();
+    const {blocks, payloadEnvelopes, attempt} = this.state;
+    // No need to track successfulDownloads anymore, the batch goes to Processing status.
     this.successfulDownloads.clear();
-    this.state = {status: BatchStatus.Processing, blocks, payloadEnvelopes, attempt: {peers, hash}};
-    return {blocks, payloadEnvelopes, peers};
+    this.state = {status: BatchStatus.Processing, blocks, payloadEnvelopes, attempt};
+    return {blocks, payloadEnvelopes, peers: attempt.peers};
   }
 
   /**
@@ -673,6 +675,26 @@ export class Batch {
     };
   }
 
+  /**
+   * Processing -> AwaitingProcessing
+   *
+   * The batch's own blocks are valid but processing failed because a previous batch did not
+   * deliver the parent. Keep the downloaded blocks and process them again once the previous
+   * batch is repaired.
+   */
+  retainForReprocessing(): void {
+    if (this.state.status !== BatchStatus.Processing) {
+      throw new BatchError(this.wrongStatusErrorType(BatchStatus.Processing));
+    }
+
+    this.state = {
+      status: BatchStatus.AwaitingProcessing,
+      blocks: this.state.blocks,
+      payloadEnvelopes: this.state.payloadEnvelopes,
+      attempt: this.state.attempt,
+    };
+  }
+
   /**
    * Processing -> AwaitingDownload
    */
@@ -681,7 +703,7 @@ export class Batch {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.Processing));
     }
 
-    if (err instanceof BlockError && err.type.code === BlockErrorCode.EXECUTION_ENGINE_ERROR) {
+    if (isExecutionEngineError(err)) {
       this.onExecutionEngineError(this.state.attempt);
     } else {
       this.onProcessingError(this.state.attempt);
@@ -696,7 +718,7 @@ export class Batch {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.AwaitingValidation));
     }
 
-    if (err instanceof BlockError && err.type.code === BlockErrorCode.EXECUTION_ENGINE_ERROR) {
+    if (isExecutionEngineError(err)) {
       this.onExecutionEngineError(this.state.attempt);
     } else {
       this.onProcessingError(this.state.attempt);
@@ -750,7 +772,7 @@ export enum BatchErrorCode {
   INVALID_COUNT = "BATCH_ERROR_INVALID_COUNT",
   MAX_DOWNLOAD_ATTEMPTS = "BATCH_ERROR_MAX_DOWNLOAD_ATTEMPTS",
   MAX_PROCESSING_ATTEMPTS = "BATCH_ERROR_MAX_PROCESSING_ATTEMPTS",
-  MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS = "MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS",
+  MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS = "BATCH_ERROR_MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS",
 }
 
 type BatchErrorType =
@@ -766,3 +788,19 @@ type BatchErrorMetadata = {
 };
 
 export class BatchError extends LodestarError<BatchErrorType & BatchErrorMetadata> {}
+
+function isExecutionEngineError(err: Error): boolean {
+  if (!(err instanceof BlockError)) {
+    return false;
+  }
+
+  if (err.type.code === BlockErrorCode.EXECUTION_ENGINE_ERROR) {
+    return true;
+  }
+
+  return (
+    err.type.code === BlockErrorCode.BEACON_CHAIN_ERROR &&
+    err.type.error instanceof PayloadError &&
+    err.type.error.type.code === PayloadErrorCode.EXECUTION_ENGINE_ERROR
+  );
+}

package/src/sync/range/chain.ts

@@ -23,7 +23,9 @@ import {Batch, BatchError, BatchErrorCode, BatchMetadata, BatchStatus} from "./b
 import {
   ChainPeersBalancer,
   PeerSyncInfo,
+  ProcessingFaultKind,
   batchStartEpochIsAfterSlot,
+  classifyProcessingFault,
   computeHighestTarget,
   getBatchSlotRange,
   getNextBatchToProcess,
@@ -708,23 +710,41 @@ export class SyncChain {
       this.triggerBatchProcessor();
     } else {
       this.logger.verbose("Batch process error", logCtx, res.err);
-      batch.processingError(res.err); // Throws after MAX_BATCH_PROCESSING_ATTEMPTS
-
-      // At least one block was successfully verified and imported, so we can be sure all
-      // previous batches are valid and we only need to download the current failed batch.
-      // TODO: Disabled for now
-      // if (res.err instanceof ChainSegmentError && res.err.importedBlocks > 0) {
-      //   this.advanceChain(batch.startEpoch);
-      // }
-
-      // The current batch could not be processed, so either this or previous batches are invalid.
-      // All previous batches (AwaitingValidation) are potentially faulty and marked for retry.
-      // Progress will be drop back to `this.startEpoch`
-      for (const pendingBatch of this.batches.values()) {
-        if (pendingBatch.startEpoch < batch.startEpoch) {
-          this.logger.verbose("Batch validation error", {id: this.logId, ...pendingBatch.getMetadata()});
-          pendingBatch.validationError(res.err); // Throws after MAX_BATCH_PROCESSING_ATTEMPTS
+
+      const invalidatePreviousBatches = (): number => {
+        let invalidatedCount = 0;
+        for (const pendingBatch of this.batches.values()) {
+          if (
+            pendingBatch.startEpoch < batch.startEpoch &&
+            pendingBatch.state.status === BatchStatus.AwaitingValidation
+          ) {
+            this.logger.verbose("Batch validation error", {id: this.logId, ...pendingBatch.getMetadata()});
+            pendingBatch.validationError(res.err);
+            invalidatedCount++;
+          }
         }
+        return invalidatedCount;
+      };
+
+      const prevBatch = this.batches.get(batch.startEpoch - EPOCHS_PER_BATCH);
+      switch (classifyProcessingFault(res.err, batch, prevBatch)) {
+        case ProcessingFaultKind.CurrentBatch:
+          batch.processingError(res.err);
+          break;
+
+        case ProcessingFaultKind.PreviousBatch:
+          // If no previous batch was invalidated, it means the error is on this batch
+          if (invalidatePreviousBatches() === 0) {
+            batch.processingError(res.err);
+          } else {
+            batch.retainForReprocessing();
+          }
+          break;
+
+        case ProcessingFaultKind.Ambiguous:
+          batch.processingError(res.err);
+          invalidatePreviousBatches();
+          break;
       }
     }
 

package/src/sync/range/utils/batches.ts

@@ -1,6 +1,7 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Epoch, Slot} from "@lodestar/types";
+import {BlockError, BlockErrorCode} from "../../../chain/errors/index.js";
 import {BATCH_SLOT_OFFSET, EPOCHS_PER_BATCH} from "../../constants.js";
 import {Batch, BatchStatus} from "../batch.js";
 
@@ -117,3 +118,46 @@ export function isSyncChainDone(batches: Batch[], lastEpochWithProcessBlocks: Ep
 
   return batchStartEpochIsAfterSlot(lastEpochWithProcessBlocks, targetSlot);
 }
+
+/**
+ * Where a batch processing error should be attributed.
+ */
+export enum ProcessingFaultKind {
+  /** This batch's own blocks are bad or incomplete, so redownload this batch. */
+  CurrentBatch = "CurrentBatch",
+  /** This batch is valid, but a previous batch did not deliver the parent. */
+  PreviousBatch = "PreviousBatch",
+  /** The fault could be in this batch or a previous batch, so hedge both ways. */
+  Ambiguous = "Ambiguous",
+}
+
+/**
+ * Classify a processChainSegment error for a batch so range sync only redownloads
+ * the batch or batches that could actually be at fault.
+ * Note that range sync is supposed to only handle linear chain, for forky condition we'll handle it via backward sync
+ * in BlockInputSync or the upcoming/TODO beacon_blocks_by_head req/resp.
+ */
+export function classifyProcessingFault(err: Error, batch: Batch, prevBatch: Batch | undefined): ProcessingFaultKind {
+  if (!(err instanceof BlockError)) {
+    return ProcessingFaultKind.CurrentBatch;
+  }
+
+  const {code} = err.type;
+  if (code !== BlockErrorCode.PARENT_BLOCK_UNKNOWN && code !== BlockErrorCode.PARENT_PAYLOAD_UNKNOWN) {
+    return ProcessingFaultKind.CurrentBatch;
+  }
+
+  const firstErrorSlot = err.signedBlock.message.slot;
+  if (firstErrorSlot === batch.startSlot) {
+    // note that range sync is for linear chain, so this is clearly an issue of the previous batch
+    return prevBatch === undefined ? ProcessingFaultKind.CurrentBatch : ProcessingFaultKind.PreviousBatch;
+  }
+
+  const prevTailSlot = batch.startSlot - 1;
+  const prevHasFullTail =
+    code === BlockErrorCode.PARENT_PAYLOAD_UNKNOWN
+      ? (prevBatch?.getPayloadEnvelopes()?.get(prevTailSlot)?.hasPayloadEnvelope() ?? false)
+      : (prevBatch?.getBlocks().some((b) => b.slot === prevTailSlot) ?? false);
+
+  return prevHasFullTail ? ProcessingFaultKind.CurrentBatch : ProcessingFaultKind.Ambiguous;
+}

package/src/util/clock.ts

@@ -94,10 +94,7 @@ export class Clock extends EventEmitter implements IClock {
   }
   /**
    * If it's too close to next slot given MAXIMUM_GOSSIP_CLOCK_DISPARITY, return currentSlot + 1.
-   * Otherwise return currentSlot
-   *
-   * Spec: phase0/p2p-interface.md - gossip validation uses `current_time + MAXIMUM_GOSSIP_CLOCK_DISPARITY < message_time`
-   * to reject future messages (strict `<`), so the boundary (exactly equal) is accepted, hence `<=` here.
+   * Spec: phase0/p2p-interface.md rejects future messages with strict `<`, hence `<=` here.
    */
   get currentSlotWithGossipDisparity(): Slot {
     const currentSlot = this.currentSlot;
@@ -123,9 +120,7 @@ export class Clock extends EventEmitter implements IClock {
 
   /**
    * Check if a slot is current slot given MAXIMUM_GOSSIP_CLOCK_DISPARITY.
-   *
-   * Uses `<=` for disparity checks because the spec rejects with strict `<`
-   * (phase0/p2p-interface.md), meaning the boundary (exactly equal) is accepted.
+   * Spec: phase0/p2p-interface.md rejects future messages with strict `<`, hence `<=`.
    */
   isCurrentSlotGivenGossipDisparity(slot: Slot): boolean {
     const currentSlot = this.currentSlot;