@lodestar/beacon-node 1.51.0-dev.312812503e → 1.51.0-dev.36022cc5cb

package/src/node/nodejs.ts

@@ -1,23 +1,29 @@
 import {setMaxListeners} from "node:events";
 import {PrivateKey} from "@libp2p/interface";
 import {Registry} from "prom-client";
+import {hasher} from "@chainsafe/persistent-merkle-tree";
+import {BeaconApiMethods} from "@lodestar/api/beacon/server";
+import {BeaconConfig} from "@lodestar/config";
+import type {LoggerNode} from "@lodestar/logger/node";
 import {
   init as initNativeMetrics,
   registerLocalValidator as registerNativeLocalValidator,
   scrapeMetrics as scrapeNativeMetrics,
   unregisterLocalValidator as unregisterNativeLocalValidator,
-} from "@chainsafe/lodestar-z/metrics";
-import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
-import {hasher} from "@chainsafe/persistent-merkle-tree";
-import {BeaconApiMethods} from "@lodestar/api/beacon/server";
-import {BeaconConfig} from "@lodestar/config";
-import type {LoggerNode} from "@lodestar/logger/node";
+} from "@lodestar/native/metrics";
+import {type PubkeyCache} from "@lodestar/native/pubkeys";
 import {ZERO_HASH_HEX} from "@lodestar/params";
-import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
+import {
+  IBeaconStateView,
+  computeStartSlotAtEpoch,
+  isStatePostBellatrix,
+  isStatePostGloas,
+} from "@lodestar/state-transition";
 import {Slot} from "@lodestar/types";
-import {sleep, toRootHex} from "@lodestar/utils";
+import {defer, sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
 import {BeaconRestApiServer, getApi} from "../api/index.js";
+import {getDataColumnSidecarsMinEpoch} from "../chain/archiveStore/utils/archiveBlocks.js";
 import {BeaconChain, IBeaconChain, initBeaconMetrics} from "../chain/index.js";
 import {ValidatorMonitor, createValidatorMonitor} from "../chain/validatorMonitor.js";
 import {IBeaconDb} from "../db/index.js";
@@ -111,6 +117,7 @@ export class BeaconNode {
 
   status: BeaconNodeStatus;
   private controller?: AbortController;
+  private closePromise: Promise<void> | undefined;
 
   constructor({
     opts,
@@ -212,6 +219,17 @@ export class BeaconNode {
 
     const clock = new Clock({config, genesisTime: anchorState.genesisTime, signal});
 
+    // Before any lookup into the legacy column buckets, which can be slow until compacted
+    const columnsMinEpoch =
+      opts.chain.archiveDataEpochs === Infinity
+        ? null
+        : getDataColumnSidecarsMinEpoch(config, clock.currentEpoch, opts.chain.archiveDataEpochs);
+    await db.compactLegacyDataColumns(
+      columnsMinEpoch !== null && columnsMinEpoch >= config.FULU_FORK_EPOCH
+        ? computeStartSlotAtEpoch(columnsMinEpoch)
+        : 0
+    );
+
     await db.init();
 
     // Prune hot db repos
@@ -280,6 +298,7 @@ export class BeaconNode {
         : undefined,
       builderApiClientOpts: {
         timeout: opts.executionBuilder.timeout,
+        bidDeadline: opts.executionBuilder.bidDeadline,
         // Sent with all builder api requests, unless the node runs in private mode
         userAgent: opts.executionBuilder.userAgent,
       },
@@ -375,23 +394,40 @@ export class BeaconNode {
   /**
    * Stop beacon node and its sub-components.
    */
-  async close(): Promise<void> {
-    if (this.status === BeaconNodeStatus.started) {
-      this.status = BeaconNodeStatus.closing;
-      this.sync.close();
-      if (this.restApi) await this.restApi.close();
-      await this.network.close();
-      if (this.metricsServer) await this.metricsServer.close();
-      if (this.monitoring) await this.monitoring.close();
-      await this.chain.persistToDisk();
-      await this.chain.close();
+  close(): Promise<void> {
+    if (this.closePromise !== undefined) return this.closePromise;
+    const completion = defer<void>();
+    this.closePromise = completion.promise;
+    this.status = BeaconNodeStatus.closing;
+    void this.closeInternal().then(completion.resolve, completion.reject);
+    return this.closePromise;
+  }
+
+  private async closeInternal(): Promise<void> {
+    const errors: unknown[] = [];
+    for (const cleanup of [
+      () => this.sync.close(),
+      () => this.restApi?.close(),
+      () => this.network.close(),
+      () => this.metricsServer?.close(),
+      () => this.monitoring?.close(),
+      () => this.chain.persistToDisk(),
+      () => this.chain.close(),
       // Abort signal last: close() calls above clear intervals/timeouts so no new
       // operations get scheduled. If we aborted first, a still-pending interval could
       // fire and schedule a new operation after abort, leaving it stuck and delaying shutdown.
-      if (this.controller) this.controller.abort();
-      await sleep(DELAY_BEFORE_CLOSING_DB_MS);
-      await this.db.close();
-      this.status = BeaconNodeStatus.closed;
+      () => this.controller?.abort(),
+      () => sleep(DELAY_BEFORE_CLOSING_DB_MS),
+      () => this.db.close(),
+    ]) {
+      try {
+        await cleanup();
+      } catch (error) {
+        errors.push(error);
+      }
     }
+    this.status = BeaconNodeStatus.closed;
+    if (errors.length === 1) throw errors[0];
+    if (errors.length > 1) throw new AggregateError(errors, "Beacon node cleanup failed");
   }
 }

package/src/node/options.ts

@@ -1,7 +1,9 @@
+import {BLOCK_PRODUCTION_RACE_CUTOFF_GLOAS_BPS} from "../api/impl/validator/index.js";
 import {ApiOptions, defaultApiOptions} from "../api/options.js";
 import {ArchiveMode, DEFAULT_ARCHIVE_MODE, IChainOptions, defaultChainOptions} from "../chain/options.js";
 import {ValidatorMonitorOpts, defaultValidatorMonitorOpts} from "../chain/validatorMonitor.js";
 import {DatabaseOptions, defaultDbOptions} from "../db/options.js";
+import {BUILDER_BID_LAG_BUFFER_MS} from "../execution/builder/apiClient.js";
 import {
   EngineApiMode,
   ExecutionBuilderOpts,
@@ -19,16 +21,17 @@ import {SyncOptions, defaultSyncOptions} from "../sync/options.js";
 
 // Re-export so the CLI doesn't need to depend on lodestar-api
 export {allNamespaces} from "../api/rest/index.js";
-
-// Re-export to use as default values in CLI args
+export type {EngineApiMode};
+// Re-export to use as default values and for validation in CLI args
 export {
-  defaultExecutionEngineHttpOpts,
-  defaultExecutionBuilderHttpOpts,
-  engineApiModes,
   ArchiveMode,
+  BLOCK_PRODUCTION_RACE_CUTOFF_GLOAS_BPS,
+  BUILDER_BID_LAG_BUFFER_MS,
   DEFAULT_ARCHIVE_MODE,
+  defaultExecutionBuilderHttpOpts,
+  defaultExecutionEngineHttpOpts,
+  engineApiModes,
 };
-export type {EngineApiMode};
 
 export interface IBeaconNodeOptions {
   api: ApiOptions;

package/src/sync/interface.ts

@@ -7,6 +7,7 @@ import {IBeaconDb} from "../db/index.js";
 import {Metrics} from "../metrics/index.js";
 import {INetwork} from "../network/index.js";
 import {SyncChainDebugState} from "./range/chain.js";
+
 export type {SyncChainDebugState};
 
 export type SyncingStatus = routes.node.SyncingStatus;

package/src/sync/sync.ts

@@ -1,13 +1,17 @@
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
-import {Slot} from "@lodestar/types";
+import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {Slot, Status} from "@lodestar/types";
 import {Logger, toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../chain/index.js";
-import {GENESIS_SLOT} from "../constants/constants.js";
+import {GENESIS_EPOCH, GENESIS_SLOT} from "../constants/constants.js";
+import {GoodByeReasonCode} from "../constants/index.js";
+import {IBeaconDb} from "../db/index.js";
 import {ExecutionEngineState} from "../execution/index.js";
 import {Metrics} from "../metrics/index.js";
-import {INetwork, NetworkEvent, NetworkEventData} from "../network/index.js";
+import {INetwork, NetworkEvent, NetworkEventData, PeerAction} from "../network/index.js";
 import {ClockEvent} from "../util/clock.js";
 import {isOptimisticBlock} from "../util/forkChoice.js";
+import {PeerIdStr} from "../util/peerId.js";
 import {MIN_EPOCH_TO_START_GOSSIP} from "./constants.js";
 import {IBeaconSync, SyncChainDebugState, SyncModules, SyncState, SyncingStatus, syncStateMetric} from "./interface.js";
 import {SyncOptions} from "./options.js";
@@ -19,6 +23,7 @@ export class BeaconSync implements IBeaconSync {
   private readonly logger: Logger;
   private readonly network: INetwork;
   private readonly chain: IBeaconChain;
+  private readonly db: IBeaconDb;
   private readonly metrics: Metrics | null;
   private readonly opts: SyncOptions;
 
@@ -30,10 +35,11 @@ export class BeaconSync implements IBeaconSync {
   private readonly slotImportTolerance: Slot;
 
   constructor(opts: SyncOptions, modules: SyncModules) {
-    const {config, chain, metrics, network, logger} = modules;
+    const {config, chain, db, metrics, network, logger} = modules;
     this.opts = opts;
     this.network = network;
     this.chain = chain;
+    this.db = db;
     this.metrics = metrics;
     this.logger = logger;
     this.rangeSync = new RangeSync(modules, opts);
@@ -192,6 +198,11 @@ export class BeaconSync implements IBeaconSync {
    */
   private addPeer = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
     const localStatus = this.chain.getStatus();
+    // Genesis and equal finalized epochs are already checked by assertPeerRelevance in the network worker
+    if (data.status.finalizedEpoch > GENESIS_EPOCH && data.status.finalizedEpoch < localStatus.finalizedEpoch) {
+      void this.verifyPastFinalizedCheckpoint(data.peer, data.status);
+    }
+
     const syncType = getPeerSyncType(localStatus, data.status, this.chain.forkChoice, this.slotImportTolerance);
     this.logger.verbose("Peer sync type classified", {
       peer: data.peer,
@@ -216,6 +227,56 @@ export class BeaconSync implements IBeaconSync {
     this.updateSyncState();
   };
 
+  /**
+   * A behind peer's finalized checkpoint must be the canonical block in effect at the start slot of its epoch
+   */
+  async verifyPastFinalizedCheckpoint(peer: PeerIdStr, remote: Status): Promise<void> {
+    const startSlot = computeStartSlotAtEpoch(remote.finalizedEpoch);
+    const root = toRootHex(remote.finalizedRoot);
+    try {
+      let reason: string | null = null;
+      // After a finality jump, finalized blocks stay in fork choice until the archiver migrates them.
+      // Fork choice is pruned only after archiving, so a miss here means the db already has this range.
+      const canonicalBlock = this.chain.forkChoice.getCanonicalBlockClosestLteSlot(startSlot);
+      if (canonicalBlock !== null) {
+        if (canonicalBlock.blockRoot === root) return;
+        reason = "finalized_root_not_canonical";
+      } else {
+        const slot = await this.db.blockArchive.getSlotByRoot(remote.finalizedRoot);
+        if (slot !== null && slot > startSlot) {
+          reason = "finalized_root_after_epoch_start";
+        } else if (slot !== startSlot) {
+          const childSlot = await this.db.blockArchive.getSlotByParentRoot(remote.finalizedRoot);
+          if (childSlot !== null) {
+            if (childSlot <= startSlot) reason = "finalized_root_not_in_effect_at_epoch_start";
+          } else if (slot === null && startSlot > this.chain.earliestAvailableSlot) {
+            // childSlot == null too, neither the root nor a child is canonical
+            // or they are canonical and both pruned but peer's finalized epoch starts after earliestAvailableSlot
+            reason = "unknown_finalized_root";
+          }
+        }
+      }
+
+      if (reason !== null) {
+        this.logger.debug("Peer finalized checkpoint not in our chain", {
+          peer,
+          finalizedEpoch: remote.finalizedEpoch,
+          root,
+          reason,
+        });
+        this.metrics?.syncPeerFinalizedCheckpointMismatch.inc({reason});
+        this.network.reportPeer(peer, PeerAction.Fatal, "PeerFinalizedCheckpointMismatch");
+        await this.network.goodbyeAndDisconnect(peer, GoodByeReasonCode.IRRELEVANT_NETWORK);
+      }
+    } catch (e) {
+      this.logger.debug(
+        "Error verifying peer finalized checkpoint",
+        {peer, finalizedEpoch: remote.finalizedEpoch, root},
+        e as Error
+      );
+    }
+  }
+
   /**
    * Must be called by libp2p when a peer is removed from the peer manager
    */

package/src/util/execution.ts

@@ -42,7 +42,7 @@ export enum DataColumnEngineResult {
 
 export async function getBlobSidecarsFromExecution(
   config: ChainForkConfig,
-  executionEngine: IExecutionEngine,
+  executionEngine: Pick<IExecutionEngine, "getBlobs">,
   metrics: Metrics | null,
   emitter: ChainEventEmitter,
   blockInput: IBlockInput
@@ -140,7 +140,7 @@ export async function getBlobSidecarsFromExecution(
  */
 export async function getDataColumnSidecarsFromExecution(
   config: ChainForkConfig,
-  executionEngine: IExecutionEngine,
+  executionEngine: Pick<IExecutionEngine, "getBlobs">,
   emitter: ChainEventEmitter,
   input: IBlockInput | PayloadEnvelopeInput,
   metrics: Metrics | null,
@@ -289,7 +289,7 @@ export type RebuildMiss = {slot: Slot; reason: "unavailable" | "mismatch"; error
  */
 export async function* reconstructExecutionPayloadEnvelopesByRange(
   db: IBeaconDb,
-  executionEngine: IExecutionEngine,
+  executionEngine: Pick<IExecutionEngine, "getPayloadBodiesByHashV2">,
   logger: Logger,
   metrics: Metrics | null,
   startSlot: Slot,
@@ -315,7 +315,7 @@ export async function* reconstructExecutionPayloadEnvelopesByRange(
  * RANGE_UNSERVABLE for it
  */
 async function* reconstructBatch(
-  executionEngine: IExecutionEngine,
+  executionEngine: Pick<IExecutionEngine, "getPayloadBodiesByHashV2">,
   logger: Logger,
   metrics: Metrics | null,
   batch: RangeEntry[]
@@ -355,7 +355,7 @@ async function* reconstructBatch(
  * on its path. Throws ENGINE_UNAVAILABLE only.
  */
 export async function reconstructExecutionPayloadEnvelopes(
-  executionEngine: IExecutionEngine,
+  executionEngine: Pick<IExecutionEngine, "getPayloadBodiesByHashV2">,
   metrics: Metrics | null,
   headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[]
 ): Promise<(gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[]> {
@@ -382,7 +382,7 @@ export function isRebuildMiss(result: gloas.SignedExecutionPayloadEnvelope | Reb
  * incremented once, in this function.
  */
 async function reconstructEnvelopesBatch(
-  executionEngine: IExecutionEngine,
+  executionEngine: Pick<IExecutionEngine, "getPayloadBodiesByHashV2">,
   metrics: Metrics | null,
   headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[]
 ): Promise<(gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[]> {

package/src/util/file.ts

@@ -1,25 +1,24 @@
-import fs from "node:fs";
-import {promisify} from "node:util";
+import {readFile as fsReadFile, mkdir, readdir, stat, unlink, writeFile} from "node:fs/promises";
 
 /** Ensure a directory exists */
 export async function ensureDir(path: string): Promise<void> {
   try {
-    await promisify(fs.stat)(path);
+    await stat(path);
   } catch (_) {
     // not exists
-    await promisify(fs.mkdir)(path, {recursive: true});
+    await mkdir(path, {recursive: true});
   }
 }
 
 /** Write data to a file if it does not exist */
 export async function writeIfNotExist(filepath: string, bytes: Uint8Array): Promise<boolean> {
   try {
-    await promisify(fs.stat)(filepath);
+    await stat(filepath);
     return false;
     // file exists, do nothing
   } catch (_) {
     // not exists
-    await promisify(fs.writeFile)(filepath, bytes);
+    await writeFile(filepath, bytes);
     return true;
   }
 }
@@ -27,7 +26,7 @@ export async function writeIfNotExist(filepath: string, bytes: Uint8Array): Prom
 /** Remove a file if it exists */
 export async function removeFile(path: string): Promise<boolean> {
   try {
-    await promisify(fs.unlink)(path);
+    await unlink(path);
     return true;
   } catch (_) {
     // may not exists
@@ -37,7 +36,7 @@ export async function removeFile(path: string): Promise<boolean> {
 
 export async function readFile(path: string): Promise<Uint8Array | null> {
   try {
-    return await fs.promises.readFile(path);
+    return await fsReadFile(path);
   } catch (_) {
     return null;
   }
@@ -45,7 +44,7 @@ export async function readFile(path: string): Promise<Uint8Array | null> {
 
 export async function readFileNames(folderPath: string): Promise<string[]> {
   try {
-    return await fs.promises.readdir(folderPath);
+    return await readdir(folderPath);
   } catch (_) {
     return [];
   }

package/src/util/queue/itemQueue.ts

@@ -69,7 +69,7 @@ export class JobItemQueue<Args extends any[], R> {
       if (this.jobs.length === 1 && this.opts.noYieldIfOneItem) {
         void this.runJob();
       } else if (this.runningJobs < this.opts.maxConcurrency) {
-        callInNextEventLoop(this.runJob);
+        callInNextEventLoop(() => void this.runJob());
       }
     });
   }

package/src/util/sszBytes.ts

@@ -652,6 +652,46 @@ export function getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(data: U
   return "0x" + blockRootBuf.toString("hex");
 }
 
+// transactions offset within ExecutionPayload fixed portion, see SLOT_NUMBER_OFFSET_IN_EXECUTION_PAYLOAD:
+// everything up to and including blockHash(32) = 504, followed by withdrawals_offset(4)
+const PAYLOAD_START_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE =
+  ENVELOPE_START_IN_SIGNED + EXECUTION_PAYLOAD_ENVELOPE_FIXED_SIZE; // 180
+const TRANSACTIONS_OFFSET_POSITION_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE =
+  PAYLOAD_START_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + 504; // 684
+const WITHDRAWALS_OFFSET_POSITION_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE =
+  TRANSACTIONS_OFFSET_POSITION_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + 4; // 688
+
+/**
+ * Transaction count of the payload, read from the SSZ offsets without deserializing the envelope.
+ * Returns null on malformed offsets, deserialization reports the exact error.
+ */
+export function getTransactionCountFromExecutionPayloadEnvelopeSerialized(data: Uint8Array): number | null {
+  if (data.length < WITHDRAWALS_OFFSET_POSITION_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + 4) {
+    return null;
+  }
+
+  const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+  const transactionsOffset = view.getUint32(TRANSACTIONS_OFFSET_POSITION_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE, true);
+  const withdrawalsOffset = view.getUint32(WITHDRAWALS_OFFSET_POSITION_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE, true);
+  if (transactionsOffset > withdrawalsOffset) {
+    return null;
+  }
+  if (transactionsOffset === withdrawalsOffset) {
+    return 0;
+  }
+
+  const transactionsStart = PAYLOAD_START_IN_SIGNED_EXECUTION_PAYLOAD_ENVELOPE + transactionsOffset;
+  if (transactionsStart + 4 > data.length) {
+    return null;
+  }
+  // first offset of a non-empty list is the size of its offset table, 4 bytes per element
+  const firstTransactionOffset = view.getUint32(transactionsStart, true);
+  if (firstTransactionOffset % 4 !== 0 || firstTransactionOffset > withdrawalsOffset - transactionsOffset) {
+    return null;
+  }
+  return firstTransactionOffset / 4;
+}
+
 /**
  * BeaconState of all forks (up until Electra, check with new forks)
  * class BeaconState(Container):
@@ -854,7 +894,7 @@ export function getBlobKzgCommitmentsCountFromSignedBeaconBlockSerialized(
 
     const start = bidMsgStart + commitmentsRange.start;
     const end = bidMsgStart + commitmentsRange.end;
-    return Math.round(((end > blockBytes.byteLength ? blockBytes.byteLength : end) - start) / commitmentSize);
+    return Math.round((Math.min(end, blockBytes.byteLength) - start) / commitmentSize);
   }
 
   const {SignedBeaconBlock, BeaconBlock, BeaconBlockBody, KZGCommitment} = ssz[forkName as ForkPostDeneb];
@@ -880,5 +920,5 @@ export function getBlobKzgCommitmentsCountFromSignedBeaconBlockSerialized(
   const end = messageRange.end + bodyRange.end + kzgCommitmentsRange.end;
   const start = messageRange.start + bodyRange.start + kzgCommitmentsRange.start;
 
-  return Math.round(((end > blockBytes.byteLength ? blockBytes.byteLength : end) - start) / commitmentSize);
+  return Math.round((Math.min(end, blockBytes.byteLength) - start) / commitmentSize);
 }