@lodestar/beacon-node 1.46.0-dev.9b97bb94f2 → 1.46.0-dev.b22e48117e

package/src/network/reqresp/protocols.ts

@@ -1,6 +1,6 @@
 import {BeaconConfig} from "@lodestar/config";
-import {ForkName} from "@lodestar/params";
-import {ContextBytesFactory, ContextBytesType, Encoding} from "@lodestar/reqresp";
+import {ForkName, MAX_DATA_COLUMN_SIDECAR_SIZE, isForkPostGloas} from "@lodestar/params";
+import {ContextBytesFactory, ContextBytesType, Encoding, TypeSizes} from "@lodestar/reqresp";
 import {rateLimitQuotas} from "./rateLimit.js";
 import {ProtocolNoHandler, ReqRespMethod, Version, requestSszTypeByMethod, responseSszTypeByMethod} from "./types.js";
 
@@ -143,15 +143,43 @@ type ProtocolSummary = {
 };
 
 function toProtocol(protocol: ProtocolSummary) {
-  return (fork: ForkName, config: BeaconConfig): ProtocolNoHandler => ({
-    method: protocol.method,
-    version: protocol.version,
-    encoding: Encoding.SSZ_SNAPPY,
-    contextBytes: toContextBytes(protocol.contextBytesType, config),
-    inboundRateLimits: rateLimitQuotas(fork, config)[protocol.method],
-    requestSizes: requestSszTypeByMethod(fork, config)[protocol.method],
-    responseSizes: (fork) => responseSszTypeByMethod[protocol.method](fork, protocol.version),
-  });
+  return (fork: ForkName, config: BeaconConfig): ProtocolNoHandler => {
+    const requestType = requestSszTypeByMethod(fork, config)[protocol.method];
+    return {
+      method: protocol.method,
+      version: protocol.version,
+      encoding: Encoding.SSZ_SNAPPY,
+      contextBytes: toContextBytes(protocol.contextBytesType, config),
+      inboundRateLimits: rateLimitQuotas(fork, config)[protocol.method],
+      requestSizes: requestType === null ? null : clampTypeSizes(requestType, protocol.method, fork, config),
+      responseSizes: (fork) =>
+        clampTypeSizes(responseSszTypeByMethod[protocol.method](fork, protocol.version), protocol.method, fork, config),
+    };
+  };
+}
+
+/**
+ * Bound the sizes accepted from the ssz-snappy length-prefix. Gloas progressive containers have broad
+ * theoretical SSZ max sizes so the preset p2p bounds must be used instead.
+ *
+ * The length-prefix must be within the size bounds derived from the payload SSZ type or `MAX_PAYLOAD_SIZE`,
+ * whichever is smaller, see
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.12/specs/phase0/p2p-interface.md#encoding-strategies.
+ * Type-specific SSZ bounds supersede the bounds derived from the SSZ type, see
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.12/specs/gloas/p2p-interface.md#type-specific-ssz-bounds.
+ */
+function clampTypeSizes(type: TypeSizes, method: ReqRespMethod, fork: ForkName, config: BeaconConfig): TypeSizes {
+  let typeSpecificBound = config.MAX_PAYLOAD_SIZE;
+  if (isForkPostGloas(fork)) {
+    switch (method) {
+      case ReqRespMethod.DataColumnSidecarsByRange:
+      case ReqRespMethod.DataColumnSidecarsByRoot:
+        typeSpecificBound = MAX_DATA_COLUMN_SIDECAR_SIZE;
+        break;
+    }
+  }
+
+  return {minSize: type.minSize, maxSize: Math.min(type.maxSize, typeSpecificBound)};
 }
 
 function toContextBytes(type: ContextBytesType, config: BeaconConfig): ContextBytesFactory {

package/src/network/reqresp/rateLimit.ts

@@ -87,7 +87,7 @@ export const rateLimitQuotas: (fork: ForkName, config: BeaconConfig) => Record<R
     ),
   },
   [ReqRespMethod.ExecutionPayloadEnvelopesByRange]: {
-    byPeer: {quota: config.MAX_REQUEST_BLOCKS_DENEB, quotaTimeMs: 10_000},
+    byPeer: {quota: config.MAX_REQUEST_PAYLOADS, quotaTimeMs: 10_000},
     getRequestCount: getRequestCountFn(
       fork,
       config,

package/src/sync/constants.ts

@@ -15,12 +15,13 @@ export const MAX_BATCH_DOWNLOAD_ATTEMPTS = 5;
 export const RATE_LIMITED_PEER_BACKOFF_MS = 5_000;
 
 /**
- * Consider batch faulty after downloading and processing this number of times
- * as in https://github.com/ChainSafe/lodestar/issues/8147 we cannot proceed the sync chain if there is unknown parent
- * from prior batch. For example a peer may send us a non-canonical chain segment or not returning all blocks
- * in that case we should throw error and `RangeSync` should remove that error chain and add a new one.
+ * Consider a batch faulty once its failed processing attempts EXCEED this number, i.e. the batch is
+ * processed up to `MAX_BATCH_PROCESSING_ATTEMPTS + 1` times before teardown (the check is `>`, same
+ * convention as `MAX_BATCH_DOWNLOAD_ATTEMPTS`).
+ * Some processing failures can be caused by a different batch, so range sync classifies the fault
+ * before deciding which batch to redownload.
  **/
-export const MAX_BATCH_PROCESSING_ATTEMPTS = 0;
+export const MAX_BATCH_PROCESSING_ATTEMPTS = 3;
 
 /**
  * Number of slots to offset batches.

package/src/sync/range/batch.ts

@@ -64,6 +64,7 @@ export type DownloadSuccessState = {
   status: BatchStatus.AwaitingProcessing;
   blocks: IBlockInput[];
   payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null;
+  attempt: Attempt;
 };
 
 export type BatchState =
@@ -589,7 +590,11 @@ export class Batch {
     }
 
     if (allComplete) {
-      this.state = {status: BatchStatus.AwaitingProcessing, blocks, payloadEnvelopes: newPayloadEnvelopes};
+      const attempt: Attempt = {
+        peers: this.getSuccessfulPeers(),
+        hash: hashBlocks(blocks, this.config),
+      };
+      this.state = {status: BatchStatus.AwaitingProcessing, blocks, payloadEnvelopes: newPayloadEnvelopes, attempt};
     } else {
       this.state = {status: BatchStatus.AwaitingDownload, blocks, payloadEnvelopes: newPayloadEnvelopes};
       this.requests = this.getRequests(blocks);
@@ -646,15 +651,11 @@ export class Batch {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.AwaitingProcessing));
     }
 
-    const blocks = this.state.blocks;
-    const payloadEnvelopes = this.state.payloadEnvelopes;
-    const hash = hashBlocks(blocks, this.config); // tracks blocks to report peer on processing error
-    // Reset successfulDownloads in case another download attempt needs to be made. When Attempt is successful or not
-    // the peers that the data came from will be handled by the Attempt that goes for processing.
-    const peers = this.getSuccessfulPeers();
+    const {blocks, payloadEnvelopes, attempt} = this.state;
+    // No need to track successfulDownloads anymore, the batch goes to Processing status.
     this.successfulDownloads.clear();
-    this.state = {status: BatchStatus.Processing, blocks, payloadEnvelopes, attempt: {peers, hash}};
-    return {blocks, payloadEnvelopes, peers};
+    this.state = {status: BatchStatus.Processing, blocks, payloadEnvelopes, attempt};
+    return {blocks, payloadEnvelopes, peers: attempt.peers};
   }
 
   /**
@@ -673,6 +674,26 @@ export class Batch {
     };
   }
 
+  /**
+   * Processing -> AwaitingProcessing
+   *
+   * The batch's own blocks are valid but processing failed because a previous batch did not
+   * deliver the parent. Keep the downloaded blocks and process them again once the previous
+   * batch is repaired.
+   */
+  retainForReprocessing(): void {
+    if (this.state.status !== BatchStatus.Processing) {
+      throw new BatchError(this.wrongStatusErrorType(BatchStatus.Processing));
+    }
+
+    this.state = {
+      status: BatchStatus.AwaitingProcessing,
+      blocks: this.state.blocks,
+      payloadEnvelopes: this.state.payloadEnvelopes,
+      attempt: this.state.attempt,
+    };
+  }
+
   /**
    * Processing -> AwaitingDownload
    */
@@ -750,7 +771,7 @@ export enum BatchErrorCode {
   INVALID_COUNT = "BATCH_ERROR_INVALID_COUNT",
   MAX_DOWNLOAD_ATTEMPTS = "BATCH_ERROR_MAX_DOWNLOAD_ATTEMPTS",
   MAX_PROCESSING_ATTEMPTS = "BATCH_ERROR_MAX_PROCESSING_ATTEMPTS",
-  MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS = "MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS",
+  MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS = "BATCH_ERROR_MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS",
 }
 
 type BatchErrorType =

package/src/sync/range/chain.ts

@@ -23,7 +23,9 @@ import {Batch, BatchError, BatchErrorCode, BatchMetadata, BatchStatus} from "./b
 import {
   ChainPeersBalancer,
   PeerSyncInfo,
+  ProcessingFaultKind,
   batchStartEpochIsAfterSlot,
+  classifyProcessingFault,
   computeHighestTarget,
   getBatchSlotRange,
   getNextBatchToProcess,
@@ -708,23 +710,41 @@ export class SyncChain {
       this.triggerBatchProcessor();
     } else {
       this.logger.verbose("Batch process error", logCtx, res.err);
-      batch.processingError(res.err); // Throws after MAX_BATCH_PROCESSING_ATTEMPTS
-
-      // At least one block was successfully verified and imported, so we can be sure all
-      // previous batches are valid and we only need to download the current failed batch.
-      // TODO: Disabled for now
-      // if (res.err instanceof ChainSegmentError && res.err.importedBlocks > 0) {
-      //   this.advanceChain(batch.startEpoch);
-      // }
-
-      // The current batch could not be processed, so either this or previous batches are invalid.
-      // All previous batches (AwaitingValidation) are potentially faulty and marked for retry.
-      // Progress will be drop back to `this.startEpoch`
-      for (const pendingBatch of this.batches.values()) {
-        if (pendingBatch.startEpoch < batch.startEpoch) {
-          this.logger.verbose("Batch validation error", {id: this.logId, ...pendingBatch.getMetadata()});
-          pendingBatch.validationError(res.err); // Throws after MAX_BATCH_PROCESSING_ATTEMPTS
+
+      const invalidatePreviousBatches = (): number => {
+        let invalidatedCount = 0;
+        for (const pendingBatch of this.batches.values()) {
+          if (
+            pendingBatch.startEpoch < batch.startEpoch &&
+            pendingBatch.state.status === BatchStatus.AwaitingValidation
+          ) {
+            this.logger.verbose("Batch validation error", {id: this.logId, ...pendingBatch.getMetadata()});
+            pendingBatch.validationError(res.err);
+            invalidatedCount++;
+          }
         }
+        return invalidatedCount;
+      };
+
+      const prevBatch = this.batches.get(batch.startEpoch - EPOCHS_PER_BATCH);
+      switch (classifyProcessingFault(res.err, batch, prevBatch)) {
+        case ProcessingFaultKind.CurrentBatch:
+          batch.processingError(res.err);
+          break;
+
+        case ProcessingFaultKind.PreviousBatch:
+          // If no previous batch was invalidated, it means the error is on this batch
+          if (invalidatePreviousBatches() === 0) {
+            batch.processingError(res.err);
+          } else {
+            batch.retainForReprocessing();
+          }
+          break;
+
+        case ProcessingFaultKind.Ambiguous:
+          batch.processingError(res.err);
+          invalidatePreviousBatches();
+          break;
       }
     }
 

package/src/sync/range/utils/batches.ts

@@ -1,6 +1,7 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Epoch, Slot} from "@lodestar/types";
+import {BlockError, BlockErrorCode} from "../../../chain/errors/index.js";
 import {BATCH_SLOT_OFFSET, EPOCHS_PER_BATCH} from "../../constants.js";
 import {Batch, BatchStatus} from "../batch.js";
 
@@ -117,3 +118,46 @@ export function isSyncChainDone(batches: Batch[], lastEpochWithProcessBlocks: Ep
 
   return batchStartEpochIsAfterSlot(lastEpochWithProcessBlocks, targetSlot);
 }
+
+/**
+ * Where a batch processing error should be attributed.
+ */
+export enum ProcessingFaultKind {
+  /** This batch's own blocks are bad or incomplete, so redownload this batch. */
+  CurrentBatch = "CurrentBatch",
+  /** This batch is valid, but a previous batch did not deliver the parent. */
+  PreviousBatch = "PreviousBatch",
+  /** The fault could be in this batch or a previous batch, so hedge both ways. */
+  Ambiguous = "Ambiguous",
+}
+
+/**
+ * Classify a processChainSegment error for a batch so range sync only redownloads
+ * the batch or batches that could actually be at fault.
+ * Note that range sync is supposed to only handle linear chain, for forky condition we'll handle it via backward sync
+ * in BlockInputSync or the upcoming/TODO beacon_blocks_by_head req/resp.
+ */
+export function classifyProcessingFault(err: Error, batch: Batch, prevBatch: Batch | undefined): ProcessingFaultKind {
+  if (!(err instanceof BlockError)) {
+    return ProcessingFaultKind.CurrentBatch;
+  }
+
+  const {code} = err.type;
+  if (code !== BlockErrorCode.PARENT_BLOCK_UNKNOWN && code !== BlockErrorCode.PARENT_PAYLOAD_UNKNOWN) {
+    return ProcessingFaultKind.CurrentBatch;
+  }
+
+  const firstErrorSlot = err.signedBlock.message.slot;
+  if (firstErrorSlot === batch.startSlot) {
+    // note that range sync is for linear chain, so this is clearly an issue of the previous batch
+    return prevBatch === undefined ? ProcessingFaultKind.CurrentBatch : ProcessingFaultKind.PreviousBatch;
+  }
+
+  const prevTailSlot = batch.startSlot - 1;
+  const prevHasFullTail =
+    code === BlockErrorCode.PARENT_PAYLOAD_UNKNOWN
+      ? (prevBatch?.getPayloadEnvelopes()?.get(prevTailSlot)?.hasPayloadEnvelope() ?? false)
+      : (prevBatch?.getBlocks().some((b) => b.slot === prevTailSlot) ?? false);
+
+  return prevHasFullTail ? ProcessingFaultKind.CurrentBatch : ProcessingFaultKind.Ambiguous;
+}

package/src/util/sszBytes.ts

@@ -706,7 +706,7 @@ export function getBlockRootFromPayloadAttestationMessageSerialized(data: Uint8A
  *   blockHash: Bytes32            (32 bytes)
  *   prevRandao: Bytes32           (32 bytes)
  *   feeRecipient: ExecutionAddress(20 bytes)
- *   gasLimit: UintNum64           (8 bytes)
+ *   gasLimit: UintBn64            (8 bytes)
  *   builderIndex: BuilderIndex    (8 bytes)
  *   slot: Slot                    (8 bytes)  ← absolute offset 264
  */