@lodestar/beacon-node 1.50.0-dev.70314c9caa → 1.50.0-dev.773f874b05

package/src/metrics/metrics.ts

@@ -9,27 +9,42 @@ import {RegistryMetricCreator} from "./utils/registryMetricCreator.js";
 
 export type Metrics = BeaconMetrics &
   ForkChoiceMetrics &
-  BeaconStateTransitionMetrics &
-  LodestarMetrics & {register: RegistryMetricCreator; close: () => void};
+  LodestarMetrics & {
+    /** Null when we use native state transition metrics instead */
+    stateTransition: BeaconStateTransitionMetrics | null;
+    register: RegistryMetricCreator;
+    close: () => void;
+  };
+
+export type CreateMetricsOptions = {
+  /*
+   * Toggle inclusion of state transition metrics.
+   *
+   * False when native state transition is used, so we
+   * can grab metrics from the native implementation instead.
+   * */
+  includeStateTransitionMetrics?: boolean;
+  collectNodeMetrics?: boolean;
+};
 
 export function createMetrics(
   opts: MetricsOptions,
   genesisTime: number,
   externalRegistries: Registry[] = [],
-  {collectNodeMetrics = true}: {collectNodeMetrics?: boolean} = {}
+  createOpts: CreateMetricsOptions = {}
 ): Metrics {
   const register = new RegistryMetricCreator();
   const beacon = createBeaconMetrics(register);
   const forkChoice = getForkChoiceMetrics(register);
   const lodestar = createLodestarMetrics(register, opts.metadata, genesisTime);
-  const stateTransition = getMetrics(register);
+  const stateTransition = createOpts.includeStateTransitionMetrics === false ? null : getMetrics(register);
 
   const onUnhandledRejection = (_error: unknown): void => {
     lodestar.unhandledPromiseRejections.inc();
   };
   process.on("unhandledRejection", onUnhandledRejection);
 
-  const nodeJsMetricsClose = collectNodeMetrics ? collectNodeJSMetrics(register) : (): void => {};
+  const nodeJsMetricsClose = createOpts.collectNodeMetrics === false ? (): void => {} : collectNodeJSMetrics(register);
   const close = (): void => {
     process.removeListener("unhandledRejection", onUnhandledRejection);
     nodeJsMetricsClose();
@@ -46,7 +61,7 @@ export function createMetrics(
     ...beacon,
     ...forkChoice,
     ...lodestar,
-    ...stateTransition,
+    stateTransition,
     register,
     close,
   };

package/src/network/reqresp/handlers/executionPayloadEnvelopesByRoot.ts

@@ -53,7 +53,7 @@ export async function* onExecutionPayloadEnvelopesByRoot(
     const batch = requests.slice(i, i + MAX_BODIES_PER_REQUEST);
     let envelopesBytes: (Uint8Array | null)[];
     try {
-      // By-root allows omission, so a mismatched envelope is left out rather than failing the response.
+      // By-root allows omission, so an envelope that cannot be rebuilt is left out rather than failing the response.
       envelopesBytes = await chain.getSerializedExecutionPayloadEnvelopes(batch, "omit");
     } catch (e) {
       if (e instanceof EnvelopeReconstructionError) {

package/src/node/nodejs.ts

@@ -1,6 +1,12 @@
 import {setMaxListeners} from "node:events";
 import {PrivateKey} from "@libp2p/interface";
 import {Registry} from "prom-client";
+import {
+  init as initNativeMetrics,
+  registerLocalValidator as registerNativeLocalValidator,
+  scrapeMetrics as scrapeNativeMetrics,
+  unregisterLocalValidator as unregisterNativeLocalValidator,
+} from "@chainsafe/lodestar-z/metrics";
 import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
 import {hasher} from "@chainsafe/persistent-merkle-tree";
 import {BeaconApiMethods} from "@lodestar/api/beacon/server";
@@ -171,7 +177,13 @@ export class BeaconNode {
       // monitoring relies on metrics data
       opts.monitoring.endpoint
     ) {
-      metrics = createMetrics(opts.metrics, anchorState.genesisTime, metricsRegistries);
+      if (opts.chain.nativeStateTransition) {
+        initNativeMetrics();
+      }
+
+      metrics = createMetrics(opts.metrics, anchorState.genesisTime, metricsRegistries, {
+        includeStateTransitionMetrics: !opts.chain.nativeStateTransition,
+      });
       initBeaconMetrics(metrics, anchorState);
       // Since the db is instantiated before this, metrics must be injected manually afterwards
       db.setMetrics(metrics.db, metrics.flatFileStore);
@@ -185,7 +197,13 @@ export class BeaconNode {
             config,
             anchorState.genesisTime,
             logger.child({module: LoggerModule.vmon}),
-            opts.validatorMonitor
+            opts.validatorMonitor,
+            opts.chain.nativeStateTransition
+              ? {
+                  registerLocalValidator: registerNativeLocalValidator,
+                  unregisterLocalValidator: unregisterNativeLocalValidator,
+                }
+              : null
           )
         : null;
 
@@ -303,7 +321,17 @@ export class BeaconNode {
     const metricsServer = opts.metrics.enabled
       ? await getHttpMetricsServer(opts.metrics, {
           register: (metrics as Metrics).register,
-          getOtherMetrics: async () => Promise.all([network.scrapeMetrics(), chain.archiveStore.scrapeMetrics()]),
+          getOtherMetrics: async () => {
+            const otherMetrics = await Promise.all([network.scrapeMetrics(), chain.archiveStore.scrapeMetrics()]);
+            if (opts.chain.nativeStateTransition) {
+              try {
+                otherMetrics.push(scrapeNativeMetrics());
+              } catch (e) {
+                logger.warn("Failed to scrape native state-transition metrics", {}, e as Error);
+              }
+            }
+            return otherMetrics;
+          },
           logger: logger.child({module: LoggerModule.metrics}),
         })
       : null;

package/src/sync/range/batch.ts

@@ -773,6 +773,7 @@ export class Batch {
     // remove any downloaded blocks and re-attempt
     // TODO(fulu): need to remove the bad blocks from the SeenBlockInputCache
     this.state = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
+    this.resetRequests();
   }
 
   private onProcessingError(attempt: FailedAttempt): void {
@@ -784,6 +785,16 @@ export class Batch {
     // remove any downloaded blocks and re-attempt
     // TODO(fulu): need to remove the bad blocks from the SeenBlockInputCache
     this.state = {status: BatchStatus.AwaitingDownload, blocks: [], payloadEnvelopes: null};
+    this.resetRequests();
+  }
+
+  /**
+   * After a partial download `requests` only cover the data still missing from the downloaded blocks,
+   * e.g. a single column. Once those blocks are dropped, request the whole batch again, otherwise
+   * every peer fails the retry for lack of blocks to validate the data against.
+   */
+  private resetRequests(): void {
+    this.requests = this.getRequests([]);
   }
 
   /** Helper to construct typed BatchError. Stack traces are correct as the error is thrown above */

package/src/sync/range/chain.ts

@@ -225,6 +225,9 @@ export class SyncChain {
       Math.floor((localFinalizedEpoch - this.lastEpochWithProcessBlocks) / EPOCHS_PER_BATCH) * EPOCHS_PER_BATCH;
     this.advanceChain(lastEpochWithProcessBlocksAligned);
 
+    // stopSyncing() cleared the retry timer, but peers can still be in rate limit backoff
+    this.scheduleRateLimitBackoffRetry();
+
     // Potentially download new batches and process pending
     this.triggerBatchDownloader();
     this.triggerBatchProcessor();
@@ -417,8 +420,10 @@ export class SyncChain {
     this.rateLimitBackoffTimeout = setTimeout(
       () => {
         this.rateLimitBackoffTimeout = undefined;
-        this.triggerBatchDownloader();
+        // Must run before the downloader, it drops expired peers without triggering a download. If it ran
+        // after, a backoff expiring in between would leave the peer idle with no retry scheduled.
         this.scheduleRateLimitBackoffRetry();
+        this.triggerBatchDownloader();
       },
       Math.max(0, retryAt - now)
     );

package/src/util/execution.ts

@@ -266,14 +266,15 @@ type SlotEnvelopeBytes = {slot: Slot; envelopeBytes: Uint8Array};
 
 type RangeEntry = ArchivedEnvelope & {slot: Slot};
 
-/** What a body root mismatch means on a given serving path */
-export type ReconstructMismatchPolicy = "throw" | "omit";
+/** What an envelope that cannot be rebuilt (EL-unavailable body or body root mismatch) means on a given serving path */
+export type ReconstructMissPolicy = "throw" | "omit";
 
-export type RebuildMiss =
-  /** EL does not have the block, or has pruned its block access list */
-  | {slot: Slot; reason: "unavailable"}
-  /** An EL body does not hash to its stored root (local inconsistency) */
-  | {slot: Slot; reason: "mismatch"; error: EnvelopeReconstructionError};
+/**
+ * An archived envelope that could not be rebuilt. `"unavailable"`: the EL does not have the block or its
+ * block access list (BODY_UNAVAILABLE). `"mismatch"`: an EL body does not hash to its stored root, a local
+ * inconsistency (BODY_ROOT_MISMATCH). `error` carries the matching code for paths that surface it.
+ */
+export type RebuildMiss = {slot: Slot; reason: "unavailable" | "mismatch"; error: EnvelopeReconstructionError};
 
 /**
  * Stream finalized envelopes over [startSlot, endSlot) as serialized bytes, rebuilding header
@@ -405,7 +406,14 @@ async function reconstructEnvelopesBatch(
     // A zero-length block access list cannot be valid, RLP encodes an empty list as 0xc0
     if (body == null || body.withdrawals == null || body.blockAccessList == null || body.blockAccessList.length === 0) {
       metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "unavailable"});
-      return {slot, reason: "unavailable"};
+      return {
+        slot,
+        reason: "unavailable",
+        error: new EnvelopeReconstructionError(
+          {code: EnvelopeReconstructionErrorCode.BODY_UNAVAILABLE, slot},
+          `execution client cannot serve the payload body or block access list for archived envelope slot=${slot}`
+        ),
+      };
     }
     try {
       const envelope = signedHeaderEnvelopeToFull(headerEnvelope, {