@lodestar/beacon-node 1.35.0-dev.b42a298a7c → 1.35.0-dev.ba92bd8a88

package/src/sync/range/range.ts

@@ -0,0 +1,354 @@
+import {EventEmitter} from "node:events";
+import {StrictEventEmitter} from "strict-event-emitter-types";
+import {BeaconConfig} from "@lodestar/config";
+import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {Epoch, Status, fulu} from "@lodestar/types";
+import {Logger, toRootHex} from "@lodestar/utils";
+import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
+import {AttestationImportOpt, ImportBlockOpts} from "../../chain/blocks/index.js";
+import {IBeaconChain} from "../../chain/index.js";
+import {Metrics} from "../../metrics/index.js";
+import {INetwork} from "../../network/index.js";
+import {PeerIdStr} from "../../util/peerId.js";
+import {cacheByRangeResponses, downloadByRange} from "../utils/downloadByRange.js";
+import {RangeSyncType, getRangeSyncTarget, rangeSyncTypes} from "../utils/remoteSyncType.js";
+import {ChainTarget, SyncChain, SyncChainDebugState, SyncChainFns} from "./chain.js";
+import {updateChains} from "./utils/index.js";
+
+export enum RangeSyncEvent {
+  completedChain = "RangeSync-completedChain",
+}
+
+type RangeSyncEvents = {
+  [RangeSyncEvent.completedChain]: () => void;
+};
+
+type RangeSyncEmitter = StrictEventEmitter<EventEmitter, RangeSyncEvents>;
+
+export enum RangeSyncStatus {
+  /** A finalized chain is being synced */
+  Finalized,
+  /** There are no finalized chains and we are syncing one more head chains */
+  Head,
+  /** There are no head or finalized chains and no long range sync is in progress */
+  Idle,
+}
+
+type RangeSyncState =
+  | {status: RangeSyncStatus.Finalized; target: ChainTarget}
+  | {status: RangeSyncStatus.Head; targets: ChainTarget[]}
+  | {status: RangeSyncStatus.Idle};
+
+export type RangeSyncModules = {
+  chain: IBeaconChain;
+  network: INetwork;
+  metrics: Metrics | null;
+  config: BeaconConfig;
+  logger: Logger;
+};
+
+export type RangeSyncOpts = {
+  disableProcessAsChainSegment?: boolean;
+};
+
+/**
+ * RangeSync groups peers by their `status` into static target `SyncChain` instances
+ * Peers on each chain will be queried for batches until reaching their target.
+ *
+ * Not all SyncChain-s will sync at once, and are grouped by sync type:
+ * - Finalized Chain Sync
+ * - Head Chain Sync
+ *
+ * ### Finalized Chain Sync
+ *
+ * At least one peer's status finalized checkpoint is greater than ours. Then we'll form
+ * a chain starting from our finalized epoch and sync up to their finalized checkpoint.
+ * - Only one finalized chain can sync at a time
+ * - The finalized chain with the largest peer pool takes priority
+ * - As peers' status progresses we will switch to a SyncChain with a better target
+ *
+ * ### Head Chain Sync
+ *
+ * If no Finalized Chain Sync is active, and the peer's STATUS head is beyond
+ * `SLOT_IMPORT_TOLERANCE`, then we'll form a chain starting from our finalized epoch and sync
+ * up to their head.
+ * - More than one head chain can sync in parallel
+ * - If there are many head chains the ones with more peers take priority
+ */
+export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
+  private readonly chain: IBeaconChain;
+  private readonly network: INetwork;
+  private readonly metrics: Metrics | null;
+  private readonly config: BeaconConfig;
+  private readonly logger: Logger;
+  /** There is a single chain per type, 1 finalized sync, 1 head sync */
+  private readonly chains = new Map<RangeSyncType, SyncChain>();
+
+  private opts?: RangeSyncOpts;
+
+  constructor(modules: RangeSyncModules, opts?: RangeSyncOpts) {
+    super();
+    const {chain, network, metrics, config, logger} = modules;
+    this.chain = chain;
+    this.network = network;
+    this.metrics = metrics;
+    this.config = config;
+    this.logger = logger;
+    this.opts = opts;
+
+    if (metrics) {
+      metrics.syncStatus.addCollect(() => this.scrapeMetrics(metrics));
+    }
+  }
+
+  /** Throw / return all AsyncGenerators inside every SyncChain instance */
+  close(): void {
+    for (const chain of this.chains.values()) {
+      chain.remove();
+    }
+  }
+
+  /**
+   * A peer with a relevant STATUS message has been found, which also is advanced from us.
+   * Add this peer to an existing chain or create a new one. The update the chains status.
+   */
+  addPeer(peerId: PeerIdStr, localStatus: Status, peerStatus: Status): void {
+    // Compute if we should do a Finalized or Head sync with this peer
+    const {syncType, startEpoch, target} = getRangeSyncTarget(localStatus, peerStatus, this.chain.forkChoice);
+    this.logger.debug("Sync peer joined", {
+      peer: peerId,
+      syncType,
+      startEpoch,
+      targetSlot: target.slot,
+      targetRoot: toRootHex(target.root),
+      earliestAvailableSlot: (peerStatus as fulu.Status).earliestAvailableSlot ?? Infinity,
+    });
+
+    // If the peer existed in any other chain, remove it.
+    // re-status'd peers can exist in multiple finalized chains, only one sync at a time
+    if (syncType === RangeSyncType.Head) {
+      this.removePeer(peerId);
+    }
+
+    this.addPeerOrCreateChain(startEpoch, target, peerId, syncType);
+    this.update(localStatus.finalizedEpoch);
+  }
+
+  /**
+   * Remove this peer from all head and finalized chains. A chain may become peer-empty and be dropped
+   */
+  removePeer(peerId: PeerIdStr): void {
+    for (const syncChain of this.chains.values()) {
+      syncChain.removePeer(peerId);
+    }
+  }
+
+  /**
+   * Compute the current RangeSync state, not cached
+   */
+  get state(): RangeSyncState {
+    const syncingHeadTargets: ChainTarget[] = [];
+    for (const chain of this.chains.values()) {
+      if (chain.isSyncing) {
+        if (chain.syncType === RangeSyncType.Finalized) {
+          return {status: RangeSyncStatus.Finalized, target: chain.target};
+        }
+        syncingHeadTargets.push(chain.target);
+      }
+    }
+
+    if (syncingHeadTargets.length > 0) {
+      return {status: RangeSyncStatus.Head, targets: syncingHeadTargets};
+    }
+    return {status: RangeSyncStatus.Idle};
+  }
+
+  /** Full debug state for lodestar API */
+  getSyncChainsDebugState(): SyncChainDebugState[] {
+    return Array.from(this.chains.values())
+      .map((syncChain) => syncChain.getDebugState())
+      .reverse(); // Newest additions first
+  }
+
+  /** Convenience method for `SyncChain` */
+  private processChainSegment: SyncChainFns["processChainSegment"] = async (blocks, syncType) => {
+    // Not trusted, verify signatures
+    const flags: ImportBlockOpts = {
+      // Only skip importing attestations for finalized sync. For head sync attestation are valuable.
+      // Importing attestations also triggers a head update, see https://github.com/ChainSafe/lodestar/issues/3804
+      // TODO: Review if this is okay, can we prevent some attacks by importing attestations?
+      importAttestations: syncType === RangeSyncType.Finalized ? AttestationImportOpt.Skip : undefined,
+      // Ignores ALREADY_KNOWN or GENESIS_BLOCK errors, and continues with the next block in chain segment
+      ignoreIfKnown: true,
+      // Ignore WOULD_REVERT_FINALIZED_SLOT error, continue with the next block in chain segment
+      ignoreIfFinalized: true,
+      // We won't attest to this block so it's okay to ignore a SYNCING message from execution layer
+      fromRangeSync: true,
+      // when this runs, syncing is the most important thing and gossip is not likely to run
+      // so we can utilize worker threads to verify signatures
+      blsVerifyOnMainThread: false,
+      // we want to be safe to only persist blocks after verifying it to avoid any attacks that may cause our DB
+      // to grow too much
+      eagerPersistBlock: false,
+    };
+
+    if (this.opts?.disableProcessAsChainSegment) {
+      // Should only be used for debugging or testing
+      for (const block of blocks) await this.chain.processBlock(block, flags);
+    } else {
+      await this.chain.processChainSegment(blocks, flags);
+    }
+  };
+
+  private downloadByRange: SyncChainFns["downloadByRange"] = async (peer, batch) => {
+    const batchBlocks = batch.getBlocks();
+    const {result, warnings} = await downloadByRange({
+      config: this.config,
+      network: this.network,
+      logger: this.logger,
+      peerIdStr: peer.peerId,
+      batchBlocks,
+      ...batch.getRequestsForPeer(peer),
+    });
+    const cached = cacheByRangeResponses({
+      cache: this.chain.seenBlockInputCache,
+      peerIdStr: peer.peerId,
+      responses: result,
+      batchBlocks,
+    });
+    return {result: cached, warnings};
+  };
+
+  private pruneBlockInputs: SyncChainFns["pruneBlockInputs"] = (blocks: IBlockInput[]) => {
+    for (const block of blocks) {
+      this.chain.seenBlockInputCache.prune(block.blockRootHex);
+    }
+  };
+
+  /** Convenience method for `SyncChain` */
+  private reportPeer: SyncChainFns["reportPeer"] = (peer, action, actionName) => {
+    this.network.reportPeer(peer, action, actionName);
+  };
+
+  private getConnectedPeerSyncMeta: SyncChainFns["getConnectedPeerSyncMeta"] = (peerId) => {
+    return this.network.getConnectedPeerSyncMeta(peerId);
+  };
+
+  /** Convenience method for `SyncChain` */
+  private onSyncChainEnd: SyncChainFns["onEnd"] = (err, target) => {
+    this.update(this.chain.forkChoice.getFinalizedCheckpoint().epoch);
+    this.emit(RangeSyncEvent.completedChain);
+
+    if (err === null && target !== null) {
+      this.metrics?.syncRange.syncChainHighestTargetSlotCompleted.set(target.slot);
+    }
+  };
+
+  private addPeerOrCreateChain(startEpoch: Epoch, target: ChainTarget, peer: PeerIdStr, syncType: RangeSyncType): void {
+    let syncChain = this.chains.get(syncType);
+    if (!syncChain) {
+      syncChain = new SyncChain(
+        startEpoch,
+        target,
+        syncType,
+        {
+          processChainSegment: this.processChainSegment,
+          downloadByRange: this.downloadByRange,
+          reportPeer: this.reportPeer,
+          getConnectedPeerSyncMeta: this.getConnectedPeerSyncMeta,
+          pruneBlockInputs: this.pruneBlockInputs,
+          onEnd: this.onSyncChainEnd,
+        },
+        {
+          config: this.config,
+          clock: this.chain.clock,
+          logger: this.logger,
+          custodyConfig: this.chain.custodyConfig,
+          metrics: this.metrics,
+        }
+      );
+      this.chains.set(syncType, syncChain);
+
+      this.metrics?.syncRange.syncChainsEvents.inc({syncType: syncChain.syncType, event: "add"});
+      this.logger.debug("SyncChain added", {
+        syncType,
+        firstEpoch: syncChain.firstBatchEpoch,
+        targetSlot: syncChain.target.slot,
+        targetRoot: toRootHex(syncChain.target.root),
+        peer,
+      });
+    }
+
+    syncChain.addPeer(peer, target);
+  }
+
+  private update(localFinalizedEpoch: Epoch): void {
+    const localFinalizedSlot = computeStartSlotAtEpoch(localFinalizedEpoch);
+
+    // Remove chains that are out-dated, peer-empty, completed or failed
+    for (const [id, syncChain] of this.chains.entries()) {
+      // Checks if a Finalized or Head chain should be removed
+      if (
+        // Sync chain has completed syncing or encountered an error
+        syncChain.isRemovable ||
+        // Sync chain has no more peers to download from
+        syncChain.peers === 0 ||
+        // Outdated: our chain has progressed beyond this sync chain
+        syncChain.target.slot < localFinalizedSlot ||
+        this.chain.forkChoice.hasBlock(syncChain.target.root)
+      ) {
+        syncChain.remove();
+        this.chains.delete(id);
+
+        this.metrics?.syncRange.syncChainsEvents.inc({syncType: syncChain.syncType, event: "remove"});
+        this.logger.debug("SyncChain removed", {
+          id: syncChain.logId,
+          localFinalizedSlot,
+          lastValidatedSlot: syncChain.lastValidatedSlot,
+          firstEpoch: syncChain.firstBatchEpoch,
+          targetSlot: syncChain.target.slot,
+          targetRoot: toRootHex(syncChain.target.root),
+          validatedEpochs: syncChain.validatedEpochs,
+        });
+
+        // Re-status peers from successful chain. Potentially trigger a Head sync
+        this.network
+          .reStatusPeers(syncChain.getPeers())
+          .catch((e) => this.logger.error("Error resyncing peers", {}, e));
+      }
+    }
+
+    const {toStop, toStart} = updateChains(Array.from(this.chains.values()));
+
+    for (const syncChain of toStop) {
+      syncChain.stopSyncing();
+      if (syncChain.isSyncing) {
+        this.metrics?.syncRange.syncChainsEvents.inc({syncType: syncChain.syncType, event: "stop"});
+      }
+    }
+
+    for (const syncChain of toStart) {
+      syncChain.startSyncing(localFinalizedEpoch);
+      if (!syncChain.isSyncing) {
+        this.metrics?.syncRange.syncChainsEvents.inc({syncType: syncChain.syncType, event: "start"});
+      }
+    }
+  }
+
+  private scrapeMetrics(metrics: Metrics): void {
+    metrics.syncRange.syncChainsPeers.reset();
+    const syncChainsByType: Record<RangeSyncType, number> = {
+      [RangeSyncType.Finalized]: 0,
+      [RangeSyncType.Head]: 0,
+    };
+
+    for (const chain of this.chains.values()) {
+      metrics.syncRange.syncChainsPeers.observe({syncType: chain.syncType}, chain.peers);
+      syncChainsByType[chain.syncType]++;
+    }
+
+    for (const syncType of rangeSyncTypes) {
+      metrics.syncRange.syncChains.set({syncType}, syncChainsByType[syncType]);
+    }
+  }
+}

package/src/sync/range/utils/batches.ts

@@ -0,0 +1,119 @@
+import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {Epoch, Slot} from "@lodestar/types";
+import {BATCH_SLOT_OFFSET, EPOCHS_PER_BATCH} from "../../constants.js";
+import {Batch, BatchStatus} from "../batch.js";
+
+/**
+ * Validates that the status and ordering of batches is valid
+ * ```
+ * [AwaitingValidation]* [Processing]? [AwaitingDownload,Downloading,AwaitingProcessing]*
+ * ```
+ */
+export function validateBatchesStatus(batches: Batch[]): void {
+  let processing = 0;
+  let preProcessing = 0;
+  for (const batch of batches) {
+    const status = batch.state.status;
+    switch (status) {
+      case BatchStatus.AwaitingValidation:
+        if (processing > 0) throw Error("AwaitingValidation state found after Processing");
+        if (preProcessing > 0) throw Error("AwaitingValidation state found after PreProcessing");
+        break;
+
+      case BatchStatus.Processing:
+        if (preProcessing > 0) throw Error("Processing state found after PreProcessing");
+        if (processing > 0) throw Error("More than one Processing state found");
+        processing++;
+        break;
+
+      case BatchStatus.AwaitingDownload:
+      case BatchStatus.Downloading:
+      case BatchStatus.AwaitingProcessing:
+        preProcessing++;
+        break;
+
+      default:
+        throw Error(`Unknown status: ${status}`);
+    }
+  }
+}
+
+/**
+ * Return the next batch to process if any.
+ * @see validateBatchesStatus for batches state description
+ */
+export function getNextBatchToProcess(batches: Batch[]): Batch | null {
+  for (const batch of batches) {
+    switch (batch.state.status) {
+      // If an AwaitingProcessing batch exists it can only be preceeded by AwaitingValidation
+      case BatchStatus.AwaitingValidation:
+        break;
+
+      case BatchStatus.AwaitingProcessing:
+        return batch;
+
+      // There MUST be no AwaitingProcessing state after AwaitingDownload, Downloading, Processing
+      case BatchStatus.AwaitingDownload:
+      case BatchStatus.Downloading:
+      case BatchStatus.Processing:
+        return null;
+    }
+  }
+  // Exhausted batches
+  return null;
+}
+
+/**
+ * Compute the startEpoch of the next batch to be downloaded
+ */
+export function toBeDownloadedStartEpoch(batches: Batch[], startEpoch: Epoch): Epoch {
+  // Note: batches are inserted in ascending `startEpoch` order
+  const lastBatch = batches.at(-1);
+  return lastBatch ? lastBatch.startEpoch + EPOCHS_PER_BATCH : startEpoch;
+}
+
+export function toArr<K, V>(map: Map<K, V>): V[] {
+  return Array.from(map.values());
+}
+
+export function getBatchSlotRange(startEpoch: Epoch): {startSlot: number; count: number} {
+  return {
+    startSlot: computeStartSlotAtEpoch(startEpoch) + BATCH_SLOT_OFFSET,
+    count: EPOCHS_PER_BATCH * SLOTS_PER_EPOCH,
+  };
+}
+
+/**
+ * Given a batch's startEpoch, return true if batch does not include slot and is strictly after
+ * ```
+ *        Batch1   Batch2   Batch3
+ *  ----|--------|-----X--|--------|---
+ * ```
+ *  - Batch1 = not includes and before = false
+ *  - Batch2 = includes                = false
+ *  - Batch3 = not includes and after  = true
+ */
+export function batchStartEpochIsAfterSlot(startEpoch: Epoch, targetSlot: Slot): boolean {
+  // The range of slots (inclusive) downloaded by a batch
+  const {startSlot} = getBatchSlotRange(startEpoch);
+
+  return startSlot > targetSlot;
+}
+
+/**
+ * Returns true if SyncChain has processed all possible blocks with slot <= target.slot
+ */
+export function isSyncChainDone(batches: Batch[], lastEpochWithProcessBlocks: Epoch, targetSlot: Slot): boolean {
+  // In case of full epochs of skipped slots, lastEpochWithProcessBlocks won't be updated.
+  // In that case it is assumed that the batches are valid only to be able to mark this SyncChain as done
+  const lastAwaitingValidation = batches
+    .reverse()
+    .find((batch) => batch.state.status === BatchStatus.AwaitingValidation);
+
+  if (lastAwaitingValidation) {
+    return batchStartEpochIsAfterSlot(lastAwaitingValidation.startEpoch + EPOCHS_PER_BATCH, targetSlot);
+  }
+
+  return batchStartEpochIsAfterSlot(lastEpochWithProcessBlocks, targetSlot);
+}

package/src/sync/range/utils/chainTarget.ts

@@ -0,0 +1,62 @@
+import {Root, Slot} from "@lodestar/types";
+import {toRootHex} from "@lodestar/utils";
+
+/**
+ * Sync this up to this target. Uses slot instead of epoch to re-use logic for finalized sync
+ * and head sync. The root is used to uniquely identify this chain on different forks
+ */
+export type ChainTarget = {
+  slot: Slot;
+  root: Root;
+};
+
+/**
+ * Previously we use computeMostCommonTarget to compute the target for a chain.
+ * Starting from fulu, we use computeHighestTarget to compute the target for a chain.
+ */
+export function computeHighestTarget(targets: ChainTarget[]): ChainTarget {
+  if (targets.length === 0) {
+    throw Error("Must provide at least one target");
+  }
+
+  let highestSlot = -1;
+  let highestTargets: ChainTarget[] = [];
+  for (const target of targets) {
+    if (target.slot > highestSlot) {
+      highestSlot = target.slot;
+      highestTargets = [target];
+    } else if (target.slot === highestSlot) {
+      highestTargets.push(target);
+    }
+    // ignore if target.slot < highestSlot
+  }
+
+  if (highestTargets.length === 1) {
+    return highestTargets[0];
+  }
+
+  return computeMostCommonTarget(highestTargets);
+}
+
+function computeMostCommonTarget(targets: ChainTarget[]): ChainTarget {
+  if (targets.length === 0) {
+    throw Error("Must provide at least one target");
+  }
+
+  const countById = new Map<string, number>();
+
+  let mostCommonTarget = targets[0];
+  let mostCommonCount = 0;
+
+  for (const target of targets) {
+    const targetId = `${target.slot}-${toRootHex(target.root)}`;
+    const count = 1 + (countById.get(targetId) ?? 0);
+    countById.set(targetId, count);
+    if (count > mostCommonCount) {
+      mostCommonCount = count;
+      mostCommonTarget = target;
+    }
+  }
+
+  return mostCommonTarget;
+}

package/src/sync/range/utils/hashBlocks.ts

@@ -0,0 +1,27 @@
+import {ChainForkConfig} from "@lodestar/config";
+import {RootHex, SignedBeaconBlock} from "@lodestar/types";
+import {toRootHex} from "@lodestar/utils";
+import {IBlockInput} from "../../../chain/blocks/blockInput/types.js";
+
+/**
+ * String to uniquely identify block segments. Used for peer scoring and to compare if batches are equivalent.
+ */
+export function hashBlocks(blocks: IBlockInput[], config: ChainForkConfig): RootHex {
+  switch (blocks.length) {
+    case 0:
+      return "0x";
+    case 1: {
+      const block0 = blocks[0].getBlock();
+      return toRootHex(config.getForkTypes(block0.message.slot).SignedBeaconBlock.hashTreeRoot(block0));
+    }
+    default: {
+      const block0 = blocks[0].getBlock();
+      const blockN = blocks.at(-1)?.getBlock() as SignedBeaconBlock;
+      return (
+        // TODO(fulu): should we be doing checks for presence to make sure these do not blow up?
+        toRootHex(config.getForkTypes(block0.message.slot).SignedBeaconBlock.hashTreeRoot(block0)) +
+        toRootHex(config.getForkTypes(blockN.message.slot).SignedBeaconBlock.hashTreeRoot(blockN))
+      );
+    }
+  }
+}

package/src/sync/range/utils/index.ts

@@ -0,0 +1,5 @@
+export * from "./batches.js";
+export * from "./chainTarget.js";
+export * from "./hashBlocks.js";
+export * from "./peerBalancer.js";
+export * from "./updateChains.js";