@lodestar/beacon-node 1.45.0 → 1.46.0-dev.18a1510f34

package/src/sync/range/chain.ts

@@ -23,7 +23,9 @@ import {Batch, BatchError, BatchErrorCode, BatchMetadata, BatchStatus} from "./b
 import {
   ChainPeersBalancer,
   PeerSyncInfo,
+  ProcessingFaultKind,
   batchStartEpochIsAfterSlot,
+  classifyProcessingFault,
   computeHighestTarget,
   getBatchSlotRange,
   getNextBatchToProcess,
@@ -708,23 +710,41 @@ export class SyncChain {
       this.triggerBatchProcessor();
     } else {
       this.logger.verbose("Batch process error", logCtx, res.err);
-      batch.processingError(res.err); // Throws after MAX_BATCH_PROCESSING_ATTEMPTS
-
-      // At least one block was successfully verified and imported, so we can be sure all
-      // previous batches are valid and we only need to download the current failed batch.
-      // TODO: Disabled for now
-      // if (res.err instanceof ChainSegmentError && res.err.importedBlocks > 0) {
-      //   this.advanceChain(batch.startEpoch);
-      // }
-
-      // The current batch could not be processed, so either this or previous batches are invalid.
-      // All previous batches (AwaitingValidation) are potentially faulty and marked for retry.
-      // Progress will be drop back to `this.startEpoch`
-      for (const pendingBatch of this.batches.values()) {
-        if (pendingBatch.startEpoch < batch.startEpoch) {
-          this.logger.verbose("Batch validation error", {id: this.logId, ...pendingBatch.getMetadata()});
-          pendingBatch.validationError(res.err); // Throws after MAX_BATCH_PROCESSING_ATTEMPTS
+
+      const invalidatePreviousBatches = (): number => {
+        let invalidatedCount = 0;
+        for (const pendingBatch of this.batches.values()) {
+          if (
+            pendingBatch.startEpoch < batch.startEpoch &&
+            pendingBatch.state.status === BatchStatus.AwaitingValidation
+          ) {
+            this.logger.verbose("Batch validation error", {id: this.logId, ...pendingBatch.getMetadata()});
+            pendingBatch.validationError(res.err);
+            invalidatedCount++;
+          }
         }
+        return invalidatedCount;
+      };
+
+      const prevBatch = this.batches.get(batch.startEpoch - EPOCHS_PER_BATCH);
+      switch (classifyProcessingFault(res.err, batch, prevBatch)) {
+        case ProcessingFaultKind.CurrentBatch:
+          batch.processingError(res.err);
+          break;
+
+        case ProcessingFaultKind.PreviousBatch:
+          // If no previous batch was invalidated, it means the error is on this batch
+          if (invalidatePreviousBatches() === 0) {
+            batch.processingError(res.err);
+          } else {
+            batch.retainForReprocessing();
+          }
+          break;
+
+        case ProcessingFaultKind.Ambiguous:
+          batch.processingError(res.err);
+          invalidatePreviousBatches();
+          break;
       }
     }
 

package/src/sync/range/utils/batches.ts

@@ -1,6 +1,7 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Epoch, Slot} from "@lodestar/types";
+import {BlockError, BlockErrorCode} from "../../../chain/errors/index.js";
 import {BATCH_SLOT_OFFSET, EPOCHS_PER_BATCH} from "../../constants.js";
 import {Batch, BatchStatus} from "../batch.js";
 
@@ -117,3 +118,46 @@ export function isSyncChainDone(batches: Batch[], lastEpochWithProcessBlocks: Ep
 
   return batchStartEpochIsAfterSlot(lastEpochWithProcessBlocks, targetSlot);
 }
+
+/**
+ * Where a batch processing error should be attributed.
+ */
+export enum ProcessingFaultKind {
+  /** This batch's own blocks are bad or incomplete, so redownload this batch. */
+  CurrentBatch = "CurrentBatch",
+  /** This batch is valid, but a previous batch did not deliver the parent. */
+  PreviousBatch = "PreviousBatch",
+  /** The fault could be in this batch or a previous batch, so hedge both ways. */
+  Ambiguous = "Ambiguous",
+}
+
+/**
+ * Classify a processChainSegment error for a batch so range sync only redownloads
+ * the batch or batches that could actually be at fault.
+ * Note that range sync is supposed to only handle linear chain, for forky condition we'll handle it via backward sync
+ * in BlockInputSync or the upcoming/TODO beacon_blocks_by_head req/resp.
+ */
+export function classifyProcessingFault(err: Error, batch: Batch, prevBatch: Batch | undefined): ProcessingFaultKind {
+  if (!(err instanceof BlockError)) {
+    return ProcessingFaultKind.CurrentBatch;
+  }
+
+  const {code} = err.type;
+  if (code !== BlockErrorCode.PARENT_BLOCK_UNKNOWN && code !== BlockErrorCode.PARENT_PAYLOAD_UNKNOWN) {
+    return ProcessingFaultKind.CurrentBatch;
+  }
+
+  const firstErrorSlot = err.signedBlock.message.slot;
+  if (firstErrorSlot === batch.startSlot) {
+    // note that range sync is for linear chain, so this is clearly an issue of the previous batch
+    return prevBatch === undefined ? ProcessingFaultKind.CurrentBatch : ProcessingFaultKind.PreviousBatch;
+  }
+
+  const prevTailSlot = batch.startSlot - 1;
+  const prevHasFullTail =
+    code === BlockErrorCode.PARENT_PAYLOAD_UNKNOWN
+      ? (prevBatch?.getPayloadEnvelopes()?.get(prevTailSlot)?.hasPayloadEnvelope() ?? false)
+      : (prevBatch?.getBlocks().some((b) => b.slot === prevTailSlot) ?? false);
+
+  return prevHasFullTail ? ProcessingFaultKind.CurrentBatch : ProcessingFaultKind.Ambiguous;
+}

package/src/sync/sync.ts

@@ -75,6 +75,11 @@ export class BeaconSync implements IBeaconSync {
     if (metrics) {
       metrics.syncStatus.addCollect(() => this.scrapeMetrics(metrics));
     }
+
+    // A from-scratch or large-gap start boots not synced; updateSyncState resumes once synced
+    if (this.state !== SyncState.Synced) {
+      this.chain.forkChoice.pauseFastConfirmation();
+    }
   }
 
   close(): void {
@@ -228,6 +233,8 @@ export class BeaconSync implements IBeaconSync {
 
     // We have become synced, subscribe to all the gossip core topics
     if (state === SyncState.Synced && this.chain.clock.currentEpoch >= MIN_EPOCH_TO_START_GOSSIP) {
+      this.chain.forkChoice.resumeFastConfirmation();
+
       if (!this.network.isSubscribedToGossipCoreTopics()) {
         this.network
           .subscribeGossipCoreTopics()
@@ -251,6 +258,9 @@ export class BeaconSync implements IBeaconSync {
     else if (state !== SyncState.Synced) {
       const syncDiff = this.chain.clock.currentSlot - this.chain.forkChoice.getHead().slot;
       if (syncDiff > this.slotImportTolerance * 2) {
+        // Same debounce as gossip: transient blips keep the rule running, only a real gap pauses it
+        this.chain.forkChoice.pauseFastConfirmation();
+
         if (this.network.isSubscribedToGossipCoreTopics()) {
           this.logger.warn(`Node sync has fallen behind by ${syncDiff} slots`);
           this.network

package/src/sync/unknownBlock.ts

@@ -819,7 +819,7 @@ export class BlockInputSync {
           // case BlockErrorCode.ALREADY_KNOWN:
           // case BlockErrorCode.GENESIS_BLOCK:
 
-          case BlockErrorCode.PARENT_UNKNOWN:
+          case BlockErrorCode.PARENT_BLOCK_UNKNOWN:
           case BlockErrorCode.PRESTATE_MISSING:
             // Should not happen, mark as downloaded to try again latter
             this.logger.debug("Attempted to process block but its parent was still unknown", errorData, res.err);
@@ -841,10 +841,16 @@ export class BlockInputSync {
 
           case BlockErrorCode.EXECUTION_ENGINE_ERROR:
             // Removing the block(s) without penalizing the peers, hoping for EL to
-            // recover on a latter download + verify attempt
+            // recover on a latter download + verify attempt.
             this.removeAllDescendants(pendingBlock);
             break;
 
+          case BlockErrorCode.EXECUTION_ENGINE_INVALID:
+            // the peer served a bad block
+            this.logger.debug("Execution engine rejected block from unknown parent sync", errorData, res.err);
+            this.removeAndDownScoreAllDescendants(pendingBlock);
+            break;
+
           default:
             // Block is not correct with respect to our chain. Log error loudly
             this.logger.debug("Error processing block from unknown parent sync", errorData, res.err);

package/src/util/execution.ts

@@ -199,7 +199,7 @@ export async function getDataColumnSidecarsFromExecution(
   const previouslyMissingColumns = input.getMissingSampledColumnMeta().missing;
   const sampledColumns = previouslyMissingColumns.map((columnIndex) => dataColumnSidecars[columnIndex]);
 
-  // for columns that we already seen, it will be ignored through `ignoreDuplicatePublishError` gossip option
+  // for columns we have already seen, publishDataColumnSidecar() catches PublishError.Duplicate and marks alreadyPublished=true
   emitter.emit(ChainEvent.publishDataColumns, sampledColumns);
   // TODO: Can we record dataColumns.sentPeersPerSubnet metric here somehow
 

package/src/util/profile.ts

@@ -8,21 +8,10 @@ export enum ProfileThread {
   DISC5 = "discv5",
 }
 
-/**
- * The time to take a Bun profile.
- * If we increase this time it'll potentiall cause the app to crash.
- * If we decrease this time, profile recorded will be fragmented and hard to analyze.
- */
-const BUN_PROFILE_MS = 3 * 1000;
-
-export async function profileThread(thread: ProfileThread, durationMs: number, dirpath: string): Promise<string> {
-  return globalThis.Bun ? profileBun(thread, durationMs) : profileNodeJS(thread, durationMs, dirpath);
-}
-
 /**
  * Take `durationMs` profile of the current thread and return the persisted file path.
  */
-async function profileNodeJS(thread: ProfileThread, durationMs: number, dirpath: string): Promise<string> {
+export async function profileThread(thread: ProfileThread, durationMs: number, dirpath: string): Promise<string> {
   const inspector = await import("node:inspector");
 
   // due to some typing issues, not able to use promisify here
@@ -54,25 +43,6 @@ async function profileNodeJS(thread: ProfileThread, durationMs: number, dirpath:
   return filePath;
 }
 
-/**
- * Unlike NodeJS, Bun console.profile() api flush data to the inspector,
- * so this api returns ms taken of this profile instead of file path.
- */
-async function profileBun(thread: ProfileThread, durationMs: number): Promise<string> {
-  const start = Date.now();
-  let now = Date.now();
-  while (now - start < durationMs) {
-    // biome-ignore lint/suspicious/noConsole: need to use console api to profile in Bun
-    console.profile(String(now));
-    await sleep(BUN_PROFILE_MS);
-    // biome-ignore lint/suspicious/noConsole: need to use console api to profile in Bun
-    console.profileEnd(String(now));
-    now = Date.now();
-  }
-
-  return `Successfully take Bun ${thread} thread profile in ${now - start}ms. Check your inspector to see the profile.`;
-}
-
 /**
  * Write heap snapshot of the current thread to the specified file.
  */

package/src/util/sszBytes.ts

@@ -706,7 +706,7 @@ export function getBlockRootFromPayloadAttestationMessageSerialized(data: Uint8A
  *   blockHash: Bytes32            (32 bytes)
  *   prevRandao: Bytes32           (32 bytes)
  *   feeRecipient: ExecutionAddress(20 bytes)
- *   gasLimit: UintNum64           (8 bytes)
+ *   gasLimit: UintBn64            (8 bytes)
  *   builderIndex: BuilderIndex    (8 bytes)
  *   slot: Slot                    (8 bytes)  ← absolute offset 264
  */

package/lib/bun-wrappers/prometheus-gc-stats.d.ts

@@ -1,2 +0,0 @@
-export declare function gcStats(): () => void;
-//# sourceMappingURL=prometheus-gc-stats.d.ts.map

package/lib/bun-wrappers/prometheus-gc-stats.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"prometheus-gc-stats.d.ts","sourceRoot":"","sources":["../../src/bun-wrappers/prometheus-gc-stats.ts"],"names":[],"mappings":"AAIA,wBAAgB,OAAO,eAEtB"}

package/lib/bun-wrappers/prometheus-gc-stats.js

@@ -1,8 +0,0 @@
-// Bun does not support `GCProfiler` from `node:v8`a
-// Which is exposed from `@chainsafe/prometheus-gc-stats` and used in `packages/beacon-node/src/metrics/nodeJsMetrics.ts`
-// So we expose a dummy function for Bun runtime which do nothing for now
-// TODO: May look get some useful data from `bun:jsc`
-export function gcStats() {
-    return () => { };
-}
-//# sourceMappingURL=prometheus-gc-stats.js.map

package/lib/bun-wrappers/prometheus-gc-stats.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"prometheus-gc-stats.js","sourceRoot":"","sources":["../../src/bun-wrappers/prometheus-gc-stats.ts"],"names":[],"mappings":"AAAA,oDAAoD;AACpD,yHAAyH;AACzH,yEAAyE;AACzE,qDAAqD;AACrD,MAAM,UAAU,OAAO,GAAG;IACxB,OAAO,GAAG,EAAE,CAAC,EAAC,CAAC,CAAC;AAAA,CACjB"}

package/lib/network/peers/datastore_bun.d.ts

@@ -1,3 +0,0 @@
-import { FsDatastore } from "datastore-fs";
-export declare const LevelDatastore: typeof FsDatastore;
-//# sourceMappingURL=datastore_bun.d.ts.map

package/lib/network/peers/datastore_bun.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"datastore_bun.d.ts","sourceRoot":"","sources":["../../../src/network/peers/datastore_bun.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,WAAW,EAAC,MAAM,cAAc,CAAC;AAEzC,eAAO,MAAM,cAAc,oBAAc,CAAC"}

package/lib/network/peers/datastore_bun.js

@@ -1,3 +0,0 @@
-import { FsDatastore } from "datastore-fs";
-export const LevelDatastore = FsDatastore;
-//# sourceMappingURL=datastore_bun.js.map

package/lib/network/peers/datastore_bun.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"datastore_bun.js","sourceRoot":"","sources":["../../../src/network/peers/datastore_bun.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,WAAW,EAAC,MAAM,cAAc,CAAC;AAEzC,MAAM,CAAC,MAAM,cAAc,GAAG,WAAW,CAAC"}

package/src/bun-wrappers/prometheus-gc-stats.ts

@@ -1,7 +0,0 @@
-// Bun does not support `GCProfiler` from `node:v8`a
-// Which is exposed from `@chainsafe/prometheus-gc-stats` and used in `packages/beacon-node/src/metrics/nodeJsMetrics.ts`
-// So we expose a dummy function for Bun runtime which do nothing for now
-// TODO: May look get some useful data from `bun:jsc`
-export function gcStats() {
-  return () => {};
-}

package/src/network/peers/datastore_bun.ts

@@ -1,3 +0,0 @@
-import {FsDatastore} from "datastore-fs";
-
-export const LevelDatastore = FsDatastore;