@lodestar/beacon-node 1.47.0-dev.16cb8ec36c → 1.47.0-dev.1c8babbe6e

package/src/sync/range/batch.ts

@@ -39,6 +39,7 @@ export enum BatchStatus {
   AwaitingValidation = "AwaitingValidation",
 }
 
+/** Identity of one download attempt*/
 export type Attempt = {
   /** The peer that made the attempt */
   peers: PeerIdStr[];
@@ -46,6 +47,16 @@ export type Attempt = {
   hash: RootHex;
 };
 
+/**
+ * A failed attempt that has been classified for blame.
+ */
+export type FailedAttempt = Attempt & {
+  /**
+   * True when the failure is evidence the peers served bad data, so they may be downscored..
+   */
+  peerAttributable: boolean;
+};
+
 type TrackedRequest = {
   /** only happen for the 1st batch in checkpoint sync */
   parentPayload: boolean;
@@ -108,10 +119,12 @@ export type BatchMetadata = {
   // Retry counters
   downloadAttempts: number;
   processingAttempts: number;
+  executionErrorAttempts: number;
 
   // Cumulative peer attribution for failed attempts (only present when non-empty)
   failedDownloadPeers?: string;
   failedProcessingPeers?: string;
+  executionErrorPeers?: string;
 };
 
 function formatRangeReq(req: {startSlot: Slot; count: number}): string {
@@ -152,10 +165,10 @@ export class Batch {
   state: BatchState = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
   /** Peers that provided good data, with column coverage for by_range requests */
   private readonly successfulDownloads = new Map<PeerIdStr, TrackedRequest>();
-  /** The `Attempts` that have been made and failed to send us this batch. */
-  readonly failedProcessingAttempts: Attempt[] = [];
-  /** The `Attempts` that have been made and failed because of execution malfunction. */
-  readonly executionErrorAttempts: Attempt[] = [];
+  /** The attempts that have been made and failed to send us this batch. */
+  readonly failedProcessingAttempts: FailedAttempt[] = [];
+  /** The attempts that have been made and failed because of execution malfunction. */
+  readonly executionErrorAttempts: FailedAttempt[] = [];
   /** The number of download retries this batch has undergone due to a failed request. */
   private readonly failedDownloadAttempts: PeerIdStr[] = [];
   private readonly config: ChainForkConfig;
@@ -419,10 +432,18 @@ export class Batch {
   }
 
   /**
-   * Gives a list of peers from which this batch has had a failed download or processing attempt.
+   * Gives a list of peers from which this batch has had a failed download attempt, or a
+   * peer-attributable failed processing/execution attempt.
    */
   getFailedPeers(): PeerIdStr[] {
-    return [...this.failedDownloadAttempts, ...this.failedProcessingAttempts.flatMap((a) => a.peers)];
+    return [...this.failedDownloadAttempts, ...this.getPeerAttributableAttempts().flatMap((a) => a.peers)];
+  }
+
+  /**
+   * Attempts whose failure is peer-attributable, across both the processing and execution error
+   */
+  getPeerAttributableAttempts(): FailedAttempt[] {
+    return [...this.failedProcessingAttempts, ...this.executionErrorAttempts].filter((a) => a.peerAttributable);
   }
 
   /**
@@ -457,6 +478,7 @@ export class Batch {
   getMetadata(): BatchMetadata {
     const {blocksRequest, blobsRequest, columnsRequest, envelopesRequest} = this.requests;
     const failedProcessingPeerList = this.failedProcessingAttempts.flatMap((a) => a.peers);
+    const executionErrorPeerList = this.executionErrorAttempts.flatMap((a) => a.peers);
     return {
       startEpoch: this.startEpoch,
       startSlot: this.startSlot,
@@ -468,12 +490,16 @@ export class Batch {
       ...(envelopesRequest && {envelopesReq: formatRangeReq(envelopesRequest)}),
       downloadAttempts: this.failedDownloadAttempts.length,
       processingAttempts: this.failedProcessingAttempts.length,
+      executionErrorAttempts: this.executionErrorAttempts.length,
       ...(this.failedDownloadAttempts.length > 0 && {
         failedDownloadPeers: this.failedDownloadAttempts.join(","),
       }),
       ...(failedProcessingPeerList.length > 0 && {
         failedProcessingPeers: failedProcessingPeerList.join(","),
       }),
+      ...(executionErrorPeerList.length > 0 && {
+        executionErrorPeers: executionErrorPeerList.join(","),
+      }),
     };
   }
 
@@ -703,11 +729,7 @@ export class Batch {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.Processing));
     }
 
-    if (isExecutionEngineError(err)) {
-      this.onExecutionEngineError(this.state.attempt);
-    } else {
-      this.onProcessingError(this.state.attempt);
-    }
+    this.routeProcessingFailure(err, this.state.attempt);
   }
 
   /**
@@ -718,10 +740,17 @@ export class Batch {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.AwaitingValidation));
     }
 
-    if (isExecutionEngineError(err)) {
-      this.onExecutionEngineError(this.state.attempt);
+    this.routeProcessingFailure(err, this.state.attempt);
+  }
+
+  private routeProcessingFailure(err: Error, attempt: Attempt): void {
+    const code = err instanceof BlockError || err instanceof PayloadError ? err.type.code : null;
+    const failedAttempt: FailedAttempt = {...attempt, peerAttributable: isPeerAttributableFailure(code)};
+
+    if (isExecutionEngineFailure(code)) {
+      this.onExecutionEngineError(failedAttempt);
     } else {
-      this.onProcessingError(this.state.attempt);
+      this.onProcessingError(failedAttempt);
     }
   }
 
@@ -735,7 +764,7 @@ export class Batch {
     return this.state.attempt;
   }
 
-  private onExecutionEngineError(attempt: Attempt): void {
+  private onExecutionEngineError(attempt: FailedAttempt): void {
     this.executionErrorAttempts.push(attempt);
     if (this.executionErrorAttempts.length > MAX_BATCH_PROCESSING_ATTEMPTS) {
       throw new BatchError(this.errorType({code: BatchErrorCode.MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS}));
@@ -746,7 +775,7 @@ export class Batch {
     this.state = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
   }
 
-  private onProcessingError(attempt: Attempt): void {
+  private onProcessingError(attempt: FailedAttempt): void {
     this.failedProcessingAttempts.push(attempt);
     if (this.failedProcessingAttempts.length > MAX_BATCH_PROCESSING_ATTEMPTS) {
       throw new BatchError(this.errorType({code: BatchErrorCode.MAX_PROCESSING_ATTEMPTS}));
@@ -789,18 +818,44 @@ type BatchErrorMetadata = {
 
 export class BatchError extends LodestarError<BatchErrorType & BatchErrorMetadata> {}
 
-function isExecutionEngineError(err: Error): boolean {
-  if (!(err instanceof BlockError)) {
-    return false;
-  }
-
-  if (err.type.code === BlockErrorCode.EXECUTION_ENGINE_ERROR) {
-    return true;
-  }
-
+/**
+ * Whether a processing failure is an execution-engine failure. Pre-gloas, all EL errors stay in
+ * BlockErrorCode, post-gloas, they stay in PayloadErrorCode.
+ */
+function isExecutionEngineFailure(code: BlockErrorCode | PayloadErrorCode | null): boolean {
   return (
-    err.type.code === BlockErrorCode.BEACON_CHAIN_ERROR &&
-    err.type.error instanceof PayloadError &&
-    err.type.error.type.code === PayloadErrorCode.EXECUTION_ENGINE_ERROR
+    code === BlockErrorCode.EXECUTION_ENGINE_INVALID ||
+    code === BlockErrorCode.EXECUTION_ENGINE_ERROR ||
+    code === PayloadErrorCode.EXECUTION_ENGINE_INVALID ||
+    code === PayloadErrorCode.EXECUTION_ENGINE_ERROR
   );
 }
+
+/**
+ * Based on an error code, return if peer is attributable for FailedAttempt.
+ */
+function isPeerAttributableFailure(code: BlockErrorCode | PayloadErrorCode | null): boolean {
+  switch (code) {
+    // EL returned a definitive INVALID verdict on the payload
+    case BlockErrorCode.EXECUTION_ENGINE_INVALID:
+    case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
+    // a block or envelope signature is invalid
+    case BlockErrorCode.INVALID_SIGNATURE:
+    case PayloadErrorCode.INVALID_SIGNATURE:
+    // the block's state transition produced an invalid state root, or failed per_slot/per_block processing
+    case BlockErrorCode.INVALID_STATE_ROOT:
+    case BlockErrorCode.PER_BLOCK_PROCESSING_ERROR:
+    // the segment this peer served is internally inconsistent (in-segment link breaks, see PR #9684)
+    case BlockErrorCode.NON_LINEAR_SLOTS:
+    case BlockErrorCode.NON_LINEAR_PARENT_ROOTS:
+    case BlockErrorCode.NON_LINEAR_PAYLOAD_ROOTS:
+    // the envelope references a mismatched block root, or fails field verification
+    case BlockErrorCode.ENVELOPE_BLOCK_ROOT_MISMATCH:
+    case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
+    // the peer served a block we have explicitly blacklisted
+    case BlockErrorCode.BLACKLISTED_BLOCK:
+      return true;
+    default:
+      return false;
+  }
+}

package/src/sync/range/chain.ts

@@ -358,9 +358,10 @@ export class SyncChain {
       // If a batch exceeds it's retry limit, maybe downscore peers.
       // shouldDownscoreOnBatchError() functions enforces that all BatchErrorCode values are covered
       if (e instanceof BatchError) {
-        const shouldReportPeer = shouldReportPeerOnBatchError(e.type.code);
+        const shouldReportPeer = shouldReportPeerOnBatchError(e.type.code, this.batches.get(e.type.startEpoch));
         if (shouldReportPeer) {
-          for (const peer of this.peerset.keys()) {
+          // Only the peers actually at fault are reported, never the whole peerset.
+          for (const peer of shouldReportPeer.peers) {
             this.reportPeer(peer, shouldReportPeer.action, shouldReportPeer.reason);
           }
         }
@@ -768,8 +769,11 @@ export class SyncChain {
 
         // The last batch attempt is right, all others are wrong. Penalize other peers
         const attemptOk = batch.validationSuccess();
-        for (const attempt of batch.failedProcessingAttempts) {
+        for (const attempt of batch.getPeerAttributableAttempts()) {
           if (attempt.hash !== attemptOk.hash) {
+            // attempt.peers contains block/column/envelope peers.
+            // And if there is one bad envelope we will downscore block and column peers to
+            // TODO: resolve this
             for (const badAttemptPeer of attempt.peers) {
               if (attemptOk.peers.find((goodPeer) => goodPeer === badAttemptPeer)) {
                 // The same peer corrected its previous attempt
@@ -830,21 +834,32 @@ export class SyncChain {
  * If peer should not be downscored, returns null.
  */
 export function shouldReportPeerOnBatchError(
-  code: BatchErrorCode
-): {action: PeerAction.LowToleranceError; reason: string} | null {
+  code: BatchErrorCode,
+  batch: Batch | undefined
+): {action: PeerAction.LowToleranceError; reason: string; peers: PeerIdStr[]} | null {
   switch (code) {
-    // A batch could not be processed after max retry limit. It's likely that all peers
-    // in this chain are sending invalid batches repeatedly so are either malicious or faulty.
-    // We drop the chain and report all peers.
-    // There are some edge cases with forks that could cause this situation, but it's unlikely.
+    //  A batch could not be processed after max retry limit. Report
+    // only the peers that served a peer-attributable failed attempt.
     case BatchErrorCode.MAX_PROCESSING_ATTEMPTS:
-      return {action: PeerAction.LowToleranceError, reason: "SyncChainMaxProcessingAttempts"};
+    case BatchErrorCode.MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS: {
+      // penalize both the processing-error and execution-error peers
+      const attributablePeers = [
+        ...new Set(batch?.getPeerAttributableAttempts().flatMap((attempt) => attempt.peers) ?? []),
+      ];
+      if (attributablePeers.length === 0) {
+        return null;
+      }
+      const reason =
+        code === BatchErrorCode.MAX_PROCESSING_ATTEMPTS
+          ? "SyncChainMaxProcessingAttempts"
+          : "SyncChainMaxExecutionEngineErrorAttempts";
+      return {action: PeerAction.LowToleranceError, reason, peers: attributablePeers};
+    }
 
     // TODO: Should peers be reported for MAX_DOWNLOAD_ATTEMPTS?
     case BatchErrorCode.MAX_DOWNLOAD_ATTEMPTS:
     case BatchErrorCode.INVALID_COUNT:
     case BatchErrorCode.WRONG_STATUS:
-    case BatchErrorCode.MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS:
       return null;
   }
 }

package/lib/chain/bls/maybeBatch.d.ts

@@ -1,12 +0,0 @@
-import { PublicKey } from "@chainsafe/blst";
-export type SignatureSetDeserialized = {
-    publicKey: PublicKey;
-    message: Uint8Array;
-    signature: Uint8Array;
-};
-/**
- * Verify signatures sets with batch verification or regular core verify depending on the set count.
- * Abstracted in a separate file to be consumed by the threaded pool and the main thread implementation.
- */
-export declare function verifySignatureSetsMaybeBatch(sets: SignatureSetDeserialized[]): boolean;
-//# sourceMappingURL=maybeBatch.d.ts.map

package/lib/chain/bls/maybeBatch.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"maybeBatch.d.ts","sourceRoot":"","sources":["../../../src/chain/bls/maybeBatch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,SAAS,EAAuD,MAAM,iBAAiB,CAAC;AAIhG,MAAM,MAAM,wBAAwB,GAAG;IACrC,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,EAAE,UAAU,CAAC;IACpB,SAAS,EAAE,UAAU,CAAC;CACvB,CAAC;AAEF;;;GAGG;AACH,wBAAgB,6BAA6B,CAAC,IAAI,EAAE,wBAAwB,EAAE,GAAG,OAAO,CA8BvF"}

package/lib/chain/bls/maybeBatch.js

@@ -1,35 +0,0 @@
-import { Signature, verify, verifyMultipleAggregateSignatures } from "@chainsafe/blst";
-const MIN_SET_COUNT_TO_BATCH = 2;
-/**
- * Verify signatures sets with batch verification or regular core verify depending on the set count.
- * Abstracted in a separate file to be consumed by the threaded pool and the main thread implementation.
- */
-export function verifySignatureSetsMaybeBatch(sets) {
-    try {
-        if (sets.length >= MIN_SET_COUNT_TO_BATCH) {
-            return verifyMultipleAggregateSignatures(sets.map((s) => ({
-                pk: s.publicKey,
-                msg: s.message,
-                // true = validate signature
-                sig: Signature.fromBytes(s.signature, true),
-            })));
-        }
-        // .every on an empty array returns true
-        if (sets.length === 0) {
-            throw Error("Empty signature set");
-        }
-        // If too few signature sets verify them without batching
-        return sets.every((set) => {
-            // true = validate signature
-            const sig = Signature.fromBytes(set.signature, true);
-            return verify(set.message, set.publicKey, sig);
-        });
-    }
-    catch (_) {
-        // A signature could be malformed, in that case fromBytes throws error
-        // blst-ts `verifyMultipleSignatures` is also a fallible operation if mul_n_aggregate fails
-        // see https://github.com/ChainSafe/blst-ts/blob/b1ba6333f664b08e5c50b2b0d18c4f079203962b/src/lib.ts#L291
-        return false;
-    }
-}
-//# sourceMappingURL=maybeBatch.js.map

package/lib/chain/bls/maybeBatch.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"maybeBatch.js","sourceRoot":"","sources":["../../../src/chain/bls/maybeBatch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,SAAS,EAAE,MAAM,EAAE,iCAAiC,EAAC,MAAM,iBAAiB,CAAC;AAEhG,MAAM,sBAAsB,GAAG,CAAC,CAAC;AAQjC;;;GAGG;AACH,MAAM,UAAU,6BAA6B,CAAC,IAAgC;IAC5E,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,MAAM,IAAI,sBAAsB,EAAE,CAAC;YAC1C,OAAO,iCAAiC,CACtC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACf,EAAE,EAAE,CAAC,CAAC,SAAS;gBACf,GAAG,EAAE,CAAC,CAAC,OAAO;gBACd,4BAA4B;gBAC5B,GAAG,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC;aAC5C,CAAC,CAAC,CACJ,CAAC;QACJ,CAAC;QAED,wCAAwC;QACxC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACrC,CAAC;QAED,yDAAyD;QACzD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACxB,4BAA4B;YAC5B,MAAM,GAAG,GAAG,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACrD,OAAO,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,sEAAsE;QACtE,2FAA2F;QAC3F,yGAAyG;QACzG,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/src/chain/bls/maybeBatch.ts

@@ -1,45 +0,0 @@
-import {PublicKey, Signature, verify, verifyMultipleAggregateSignatures} from "@chainsafe/blst";
-
-const MIN_SET_COUNT_TO_BATCH = 2;
-
-export type SignatureSetDeserialized = {
-  publicKey: PublicKey;
-  message: Uint8Array;
-  signature: Uint8Array;
-};
-
-/**
- * Verify signatures sets with batch verification or regular core verify depending on the set count.
- * Abstracted in a separate file to be consumed by the threaded pool and the main thread implementation.
- */
-export function verifySignatureSetsMaybeBatch(sets: SignatureSetDeserialized[]): boolean {
-  try {
-    if (sets.length >= MIN_SET_COUNT_TO_BATCH) {
-      return verifyMultipleAggregateSignatures(
-        sets.map((s) => ({
-          pk: s.publicKey,
-          msg: s.message,
-          // true = validate signature
-          sig: Signature.fromBytes(s.signature, true),
-        }))
-      );
-    }
-
-    // .every on an empty array returns true
-    if (sets.length === 0) {
-      throw Error("Empty signature set");
-    }
-
-    // If too few signature sets verify them without batching
-    return sets.every((set) => {
-      // true = validate signature
-      const sig = Signature.fromBytes(set.signature, true);
-      return verify(set.message, set.publicKey, sig);
-    });
-  } catch (_) {
-    // A signature could be malformed, in that case fromBytes throws error
-    // blst-ts `verifyMultipleSignatures` is also a fallible operation if mul_n_aggregate fails
-    // see https://github.com/ChainSafe/blst-ts/blob/b1ba6333f664b08e5c50b2b0d18c4f079203962b/src/lib.ts#L291
-    return false;
-  }
-}