@lodestar/beacon-node 1.50.0-dev.5866a397b6 → 1.50.0-dev.58d40975f9

package/src/node/nodejs.ts

@@ -1,6 +1,12 @@
 import {setMaxListeners} from "node:events";
 import {PrivateKey} from "@libp2p/interface";
 import {Registry} from "prom-client";
+import {
+  init as initNativeMetrics,
+  registerLocalValidator as registerNativeLocalValidator,
+  scrapeMetrics as scrapeNativeMetrics,
+  unregisterLocalValidator as unregisterNativeLocalValidator,
+} from "@chainsafe/lodestar-z/metrics";
 import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
 import {hasher} from "@chainsafe/persistent-merkle-tree";
 import {BeaconApiMethods} from "@lodestar/api/beacon/server";
@@ -8,6 +14,7 @@ import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ZERO_HASH_HEX} from "@lodestar/params";
 import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
+import {Slot} from "@lodestar/types";
 import {sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
 import {BeaconRestApiServer, getApi} from "../api/index.js";
@@ -55,6 +62,7 @@ export type BeaconNodeInitModules = {
   peerStoreDir?: string;
   anchorState: IBeaconStateView;
   isAnchorStateFinalized: boolean;
+  earliestAvailableSlot: Slot;
   metricsRegistries?: Registry[];
 };
 
@@ -153,6 +161,7 @@ export class BeaconNode {
     peerStoreDir,
     anchorState,
     isAnchorStateFinalized,
+    earliestAvailableSlot,
     metricsRegistries = [],
   }: BeaconNodeInitModules): Promise<T> {
     if (hasher.name !== "hashtree") {
@@ -171,7 +180,13 @@ export class BeaconNode {
       // monitoring relies on metrics data
       opts.monitoring.endpoint
     ) {
-      metrics = createMetrics(opts.metrics, anchorState.genesisTime, metricsRegistries);
+      if (opts.chain.nativeStateTransition) {
+        initNativeMetrics();
+      }
+
+      metrics = createMetrics(opts.metrics, anchorState.genesisTime, metricsRegistries, {
+        includeStateTransitionMetrics: !opts.chain.nativeStateTransition,
+      });
       initBeaconMetrics(metrics, anchorState);
       // Since the db is instantiated before this, metrics must be injected manually afterwards
       db.setMetrics(metrics.db, metrics.flatFileStore);
@@ -185,7 +200,13 @@ export class BeaconNode {
             config,
             anchorState.genesisTime,
             logger.child({module: LoggerModule.vmon}),
-            opts.validatorMonitor
+            opts.validatorMonitor,
+            opts.chain.nativeStateTransition
+              ? {
+                  registerLocalValidator: registerNativeLocalValidator,
+                  unregisterLocalValidator: unregisterNativeLocalValidator,
+                }
+              : null
           )
         : null;
 
@@ -248,6 +269,7 @@ export class BeaconNode {
       validatorMonitor,
       anchorState,
       isAnchorStateFinalized,
+      earliestAvailableSlot,
       executionEngine: initializeExecutionEngine(executionEngineOpts, {
         metrics,
         signal,
@@ -303,7 +325,17 @@ export class BeaconNode {
     const metricsServer = opts.metrics.enabled
       ? await getHttpMetricsServer(opts.metrics, {
           register: (metrics as Metrics).register,
-          getOtherMetrics: async () => Promise.all([network.scrapeMetrics(), chain.archiveStore.scrapeMetrics()]),
+          getOtherMetrics: async () => {
+            const otherMetrics = await Promise.all([network.scrapeMetrics(), chain.archiveStore.scrapeMetrics()]);
+            if (opts.chain.nativeStateTransition) {
+              try {
+                otherMetrics.push(scrapeNativeMetrics());
+              } catch (e) {
+                logger.warn("Failed to scrape native state-transition metrics", {}, e as Error);
+              }
+            }
+            return otherMetrics;
+          },
           logger: logger.child({module: LoggerModule.metrics}),
         })
       : null;

package/src/sync/range/batch.ts

@@ -14,7 +14,7 @@ import {IClock} from "../../util/clock.js";
 import {CustodyConfig} from "../../util/dataColumns.js";
 import {PeerIdStr} from "../../util/peerId.js";
 import {MAX_BATCH_DOWNLOAD_ATTEMPTS, MAX_BATCH_PROCESSING_ATTEMPTS} from "../constants.js";
-import {DownloadByRangeRequests, ParentPayloadCommitments} from "../utils/downloadByRange.js";
+import {DownloadByRangeRequests, ParentPayload} from "../utils/downloadByRange.js";
 import {getBatchSlotRange, hashBlocks} from "./utils/index.js";
 
 /**
@@ -175,7 +175,7 @@ export class Batch {
   private readonly clock: IClock;
   private readonly custodyConfig: CustodyConfig;
   private readonly isFirstBatchInChain: boolean;
-  private readonly latestBid: gloas.ExecutionPayloadBid | undefined;
+  private readonly parentPayload: ParentPayload | undefined;
 
   constructor(
     startEpoch: Epoch,
@@ -183,7 +183,7 @@ export class Batch {
     clock: IClock,
     custodyConfig: CustodyConfig,
     isFirstBatchInChain: boolean,
-    latestBid: gloas.ExecutionPayloadBid | undefined,
+    parentPayload: ParentPayload | undefined,
     targetSlot: Slot
   ) {
     this.config = config;
@@ -196,7 +196,7 @@ export class Batch {
     this.startSlot = startSlot;
     this.count = Math.min(count, targetSlot - startSlot + 1);
     this.isFirstBatchInChain = isFirstBatchInChain;
-    this.latestBid = latestBid;
+    this.parentPayload = parentPayload;
     this.requests = this.getRequests([]);
   }
 
@@ -208,24 +208,19 @@ export class Batch {
     }
 
     // we only know if we should download parent envelope if firstBlock is downloaded
-    if (firstBlock === undefined) return false;
-    if (this.latestBid === undefined) return false;
-    const firstBlockBidParentHash = (firstBlock.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message
-      .parentBlockHash;
-    return byteArrayEquals(firstBlockBidParentHash, this.latestBid.blockHash);
+    if (firstBlock === undefined || this.parentPayload === undefined) return false;
+    const firstBlockBid = (firstBlock.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message;
+    return (
+      byteArrayEquals(firstBlock.message.parentRoot, this.parentPayload.blockRoot) &&
+      byteArrayEquals(firstBlockBid.parentBlockHash, this.parentPayload.bid.blockHash)
+    );
   }
 
-  getParentPayloadCommitments(parentBlockRoot: Uint8Array): ParentPayloadCommitments {
-    if (this.latestBid === undefined) {
-      throw new Error(
-        `Coding error: getParentPayloadCommitments called without latestBid for parentBlockRoot=${toRootHex(parentBlockRoot)}`
-      );
+  getParentPayload(): ParentPayload {
+    if (this.parentPayload === undefined) {
+      throw new Error("Coding error: getParentPayload called without parentPayload");
     }
-    return {
-      blockRoot: parentBlockRoot,
-      blockRootHex: toRootHex(parentBlockRoot),
-      kzgCommitments: this.latestBid.blobKzgCommitments,
-    };
+    return this.parentPayload;
   }
 
   /**
@@ -773,6 +768,7 @@ export class Batch {
     // remove any downloaded blocks and re-attempt
     // TODO(fulu): need to remove the bad blocks from the SeenBlockInputCache
     this.state = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
+    this.resetRequests();
   }
 
   private onProcessingError(attempt: FailedAttempt): void {
@@ -784,6 +780,16 @@ export class Batch {
     // remove any downloaded blocks and re-attempt
     // TODO(fulu): need to remove the bad blocks from the SeenBlockInputCache
     this.state = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
+    this.resetRequests();
+  }
+
+  /**
+   * After a partial download `requests` only cover the data still missing from the downloaded blocks,
+   * e.g. a single column. Once those blocks are dropped, request the whole batch again, otherwise
+   * every peer fails the retry for lack of blocks to validate the data against.
+   */
+  private resetRequests(): void {
+    this.requests = this.getRequests([]);
   }
 
   /** Helper to construct typed BatchError. Stack traces are correct as the error is thrown above */

package/src/sync/range/chain.ts

@@ -1,5 +1,5 @@
 import {ChainForkConfig} from "@lodestar/config";
-import {Epoch, Root, Slot, gloas} from "@lodestar/types";
+import {Epoch, Root, Slot} from "@lodestar/types";
 import {ErrorAborted, LodestarError, Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js";
 import {BlockInputErrorCode} from "../../chain/blocks/blockInput/errors.js";
@@ -16,7 +16,7 @@ import {ItTrigger} from "../../util/itTrigger.js";
 import {PeerIdStr} from "../../util/peerId.js";
 import {WarnResult, wrapError} from "../../util/wrapError.js";
 import {BATCH_BUFFER_SIZE, EPOCHS_PER_BATCH, MAX_LOOK_AHEAD_EPOCHS} from "../constants.js";
-import {DownloadByRangeError, DownloadByRangeErrorCode} from "../utils/downloadByRange.js";
+import {DownloadByRangeError, DownloadByRangeErrorCode, ParentPayload} from "../utils/downloadByRange.js";
 import {getRateLimitedUntilMs} from "../utils/rateLimit.js";
 import {RangeSyncType} from "../utils/remoteSyncType.js";
 import {Batch, BatchError, BatchErrorCode, BatchMetadata, BatchStatus} from "./batch.js";
@@ -160,7 +160,7 @@ export class SyncChain {
   private readonly clock: IClock;
   private readonly metrics: Metrics | null;
   private readonly custodyConfig: CustodyConfig;
-  private readonly latestBid: gloas.ExecutionPayloadBid | undefined;
+  private readonly parentPayload: ParentPayload | undefined;
 
   constructor(
     initialBatchEpoch: Epoch,
@@ -168,7 +168,7 @@ export class SyncChain {
     syncType: RangeSyncType,
     fns: SyncChainFns,
     modules: SyncChainModules,
-    latestBid: gloas.ExecutionPayloadBid | undefined
+    parentPayload: ParentPayload | undefined
   ) {
     const {config, clock, custodyConfig, logger, metrics} = modules;
     this.firstBatchEpoch = initialBatchEpoch;
@@ -184,7 +184,7 @@ export class SyncChain {
     this.clock = clock;
     this.metrics = metrics;
     this.custodyConfig = custodyConfig;
-    this.latestBid = latestBid;
+    this.parentPayload = parentPayload;
     this.logger = logger;
     this.logId = `${syncType}-${nextChainId++}`;
 
@@ -225,6 +225,9 @@ export class SyncChain {
       Math.floor((localFinalizedEpoch - this.lastEpochWithProcessBlocks) / EPOCHS_PER_BATCH) * EPOCHS_PER_BATCH;
     this.advanceChain(lastEpochWithProcessBlocksAligned);
 
+    // stopSyncing() cleared the retry timer, but peers can still be in rate limit backoff
+    this.scheduleRateLimitBackoffRetry();
+
     // Potentially download new batches and process pending
     this.triggerBatchDownloader();
     this.triggerBatchProcessor();
@@ -417,8 +420,10 @@ export class SyncChain {
     this.rateLimitBackoffTimeout = setTimeout(
       () => {
         this.rateLimitBackoffTimeout = undefined;
-        this.triggerBatchDownloader();
+        // Must run before the downloader, it drops expired peers without triggering a download. If it ran
+        // after, a backoff expiring in between would leave the peer idle with no retry scheduled.
         this.scheduleRateLimitBackoffRetry();
+        this.triggerBatchDownloader();
       },
       Math.max(0, retryAt - now)
     );
@@ -531,8 +536,8 @@ export class SyncChain {
       this.clock,
       this.custodyConfig,
       this.isFirstBatch,
-      // `latestBid` is only meaningful for the first batch's parent-payload check
-      this.isFirstBatch ? this.latestBid : undefined,
+      // `parentPayload` is only meaningful for the first batch
+      this.isFirstBatch ? this.parentPayload : undefined,
       this.target.slot
     );
     this.isFirstBatch = false;
@@ -581,6 +586,8 @@ export class SyncChain {
           case DownloadByRangeErrorCode.OUT_OF_RANGE_BLOCKS:
           case DownloadByRangeErrorCode.PARENT_ROOT_MISMATCH:
           case DownloadByRangeErrorCode.INVALID_ENVELOPE_BEACON_BLOCK_ROOT:
+          case DownloadByRangeErrorCode.INVALID_ENVELOPE_BLOCK_HASH:
+          case DownloadByRangeErrorCode.INVALID_ENVELOPE_SLOT:
           case DownloadByRangeErrorCode.INVALID_CHAIN_SEGMENT:
           case BlobSidecarErrorCode.INCLUSION_PROOF_INVALID:
           case BlobSidecarErrorCode.INVALID_KZG_PROOF_BATCH:

package/src/sync/range/range.ts

@@ -2,9 +2,10 @@ import {EventEmitter} from "node:events";
 import {StrictEventEmitter} from "strict-event-emitter-types";
 import {BeaconConfig} from "@lodestar/config";
 import {IBeaconStateViewGloas, computeStartSlotAtEpoch, isStatePostGloas} from "@lodestar/state-transition";
-import {Epoch, Status, fulu} from "@lodestar/types";
-import {Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
+import {Epoch, Status, fulu, ssz} from "@lodestar/types";
+import {Logger, byteArrayEquals, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
+import {PayloadError, PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
 import {AttestationImportOpt, ImportBlockOpts} from "../../chain/blocks/index.js";
 import {assertLinearChainSegment} from "../../chain/blocks/utils/chainSegment.js";
 import {BlockError} from "../../chain/errors/index.js";
@@ -15,6 +16,7 @@ import {PeerIdStr} from "../../util/peerId.js";
 import {
   DownloadByRangeError,
   DownloadByRangeErrorCode,
+  ParentPayload,
   cacheByRangeResponses,
   downloadByRange,
 } from "../utils/downloadByRange.js";
@@ -207,22 +209,50 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
       blsVerifyOnMainThread: false,
     };
 
-    if (this.opts?.disableProcessAsChainSegment) {
-      // Should only be used for debugging or testing
-      for (const block of blocks) {
-        await this.chain.processBlock(block, flags);
-        const payloadEnvelope = payloadEnvelopes?.get(block.slot);
-        if (payloadEnvelope?.hasPayloadEnvelope()) {
-          await this.chain.processExecutionPayload(payloadEnvelope);
+    try {
+      if (this.opts?.disableProcessAsChainSegment) {
+        // Should only be used for debugging or testing
+        for (const block of blocks) {
+          await this.chain.processBlock(block, flags);
+          const payloadEnvelope = payloadEnvelopes?.get(block.slot);
+          if (payloadEnvelope?.hasPayloadEnvelope()) {
+            await this.chain.processExecutionPayload(payloadEnvelope);
+          }
+        }
+      } else {
+        const {orphaned, skipped} = await this.chain.processChainSegment(blocks, payloadEnvelopes, flags);
+        // We log orphaned payloads to work with different clients
+        // in the future, consider applying penalties in certain conditions
+        // make sure the payload source was from range sync in that case
+        for (const {slot, payloadEnvelopeInput} of orphaned) {
+          const {peerIdStr, source} = payloadEnvelopeInput.getPayloadEnvelopeSource();
+          let client = "unknown";
+          if (peerIdStr !== undefined) {
+            try {
+              client = this.getConnectedPeerSyncMeta(peerIdStr).client;
+            } catch {
+              // peer disconnected since serving the envelope, keep "unknown"
+            }
+          }
+          this.logger.debug("Orphaned payload envelope in range sync batch", {
+            slot,
+            root: payloadEnvelopeInput.blockRootHex,
+            peer: peerIdStr ?? "unknown",
+            source,
+            client,
+            skipped,
+          });
         }
       }
-    } else {
-      const {orphaned, skipped} = await this.chain.processChainSegment(blocks, payloadEnvelopes, flags);
-      // We log orphaned payloads to work with different clients
-      // in the future, consider applying penalties in certain conditions
-      // make sure the payload source was from range sync in that case
-      for (const {slot, payloadEnvelopeInput} of orphaned) {
-        const {peerIdStr, source} = payloadEnvelopeInput.getPayloadEnvelopeSource();
+    } catch (e) {
+      if (
+        e instanceof PayloadError &&
+        (e.type.code === PayloadErrorCode.INVALID_SIGNATURE ||
+          e.type.code === PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR ||
+          e.type.code === PayloadErrorCode.EXECUTION_ENGINE_INVALID)
+      ) {
+        const {payloadInput} = e;
+        const {peerIdStr, source} = payloadInput.getPayloadEnvelopeSource();
         let client = "unknown";
         if (peerIdStr !== undefined) {
           try {
@@ -231,30 +261,32 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
             // peer disconnected since serving the envelope, keep "unknown"
           }
         }
-        this.logger.debug("Orphaned payload envelope in range sync batch", {
-          slot,
-          root: payloadEnvelopeInput.blockRootHex,
+        this.logger.debug("Removing invalid payload envelope in range sync batch", {
+          slot: payloadInput.slot,
+          root: payloadInput.blockRootHex,
           peer: peerIdStr ?? "unknown",
           source,
           client,
-          skipped,
+          code: e.type.code,
         });
+        // The batch is downloaded again, it would otherwise reuse the cached envelope and fail the same way
+        this.chain.seenPayloadEnvelopeInputCache.removeInvalid(payloadInput);
       }
+      throw e;
     }
   };
 
   private downloadByRange: SyncChainFns["downloadByRange"] = async (peer, batch) => {
     const batchBlocks = batch.getBlocks();
     const requests = batch.getRequestsForPeer(peer);
-    const parentRoot = requests.parentPayloadRequest?.envelopeBlockRoot ?? requests.parentPayloadRequest?.blockRoot;
-    const parentPayloadCommitments = parentRoot ? batch.getParentPayloadCommitments(parentRoot) : undefined;
+    const parentPayload = requests.parentPayloadRequest ? batch.getParentPayload() : undefined;
     const {result, warnings} = await downloadByRange({
       config: this.config,
       network: this.network,
       logger: this.logger,
       peerIdStr: peer.peerId,
       batchBlocks,
-      parentPayloadCommitments,
+      parentPayload,
       peerDasMetrics: this.chain.metrics?.peerDas,
       ...requests,
     });
@@ -267,6 +299,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
       batchBlocks,
       downloadedPayloadEnvelopes,
       existingPayloadEnvelopes: batch.getPayloadEnvelopes(),
+      parentPayload,
       custodyConfig: this.chain.custodyConfig,
       seenTimestampSec: Date.now() / 1000,
     });
@@ -344,13 +377,26 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
   private addPeerOrCreateChain(startEpoch: Epoch, target: ChainTarget, peer: PeerIdStr, syncType: RangeSyncType): void {
     let syncChain = this.chains.get(syncType);
     if (!syncChain) {
-      // The first batch of a new sync chain may need to detect whether the parent block was an
-      // gloas "empty" block (no envelope produced). It does so by comparing the first
-      // downloaded block's `bid.parentBlockHash` against the head state's `latestExecutionPayloadBid.blockHash`.
+      // The first block of the first batch may build on the head's FULL variant before its payload is imported,
+      // the batch then downloads the head's payload as dangling parent and validates it against the bid
       const headState = this.chain.getHeadState();
-      const latestBid = isStatePostGloas(headState)
-        ? (headState as IBeaconStateViewGloas).latestExecutionPayloadBid
-        : undefined;
+      let parentPayload: ParentPayload | undefined;
+      if (isStatePostGloas(headState) && headState.latestBlockHeader.slot > 0) {
+        const header = ssz.phase0.BeaconBlockHeader.clone(headState.latestBlockHeader);
+        if (byteArrayEquals(header.stateRoot, ssz.Root.defaultValue())) {
+          header.stateRoot = headState.hashTreeRoot();
+        }
+        const headRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(header);
+        const headSlot = headState.latestBlockHeader.slot;
+        parentPayload = {
+          blockRoot: headRoot,
+          blockRootHex: toRootHex(headRoot),
+          slot: headSlot,
+          proposerIndex: headState.latestBlockHeader.proposerIndex,
+          forkName: this.config.getForkName(headSlot),
+          bid: (headState as IBeaconStateViewGloas).latestExecutionPayloadBid,
+        };
+      }
 
       syncChain = new SyncChain(
         startEpoch,
@@ -371,7 +417,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
           custodyConfig: this.chain.custodyConfig,
           metrics: this.metrics,
         },
-        latestBid
+        parentPayload
       );
       this.chains.set(syncType, syncChain);