@lodestar/beacon-node 1.35.0-dev.f80d2d52da → 1.35.0-dev.fcf8d024ea

package/src/sync/range/utils/peerBalancer.ts

@@ -0,0 +1,184 @@
+import {isForkPostFulu} from "@lodestar/params";
+import {PeerSyncMeta} from "../../../network/peers/peersData.js";
+import {CustodyConfig} from "../../../util/dataColumns.js";
+import {PeerIdStr} from "../../../util/peerId.js";
+import {shuffle} from "../../../util/shuffle.js";
+import {sortBy} from "../../../util/sortBy.js";
+import {MAX_CONCURRENT_REQUESTS} from "../../constants.js";
+import {RangeSyncType} from "../../utils/remoteSyncType.js";
+import {Batch, BatchStatus} from "../batch.js";
+import {ChainTarget} from "./chainTarget.js";
+
+export type PeerSyncInfo = PeerSyncMeta & {
+  target: ChainTarget;
+};
+
+type PeerInfoColumn = {syncInfo: PeerSyncInfo; columns: number; hasEarliestAvailableSlots: boolean};
+
+/**
+ * Balance and organize peers to perform requests with a SyncChain
+ * Shuffles peers only once on instantiation
+ */
+export class ChainPeersBalancer {
+  private peers: PeerSyncInfo[];
+  private activeRequestsByPeer = new Map<PeerIdStr, number>();
+  private readonly custodyConfig: CustodyConfig;
+  private readonly syncType: RangeSyncType;
+  private readonly maxConcurrentRequests: number;
+
+  /**
+   * No need to specify `maxConcurrentRequests` for production code
+   * It is used for testing purposes to limit the number of concurrent requests
+   */
+  constructor(
+    peers: PeerSyncInfo[],
+    batches: Batch[],
+    custodyConfig: CustodyConfig,
+    syncType: RangeSyncType,
+    maxConcurrentRequests = MAX_CONCURRENT_REQUESTS
+  ) {
+    this.peers = shuffle(peers);
+    this.custodyConfig = custodyConfig;
+    this.syncType = syncType;
+    this.maxConcurrentRequests = maxConcurrentRequests;
+
+    // Compute activeRequestsByPeer from all batches internal states
+    for (const batch of batches) {
+      if (batch.state.status === BatchStatus.Downloading) {
+        this.activeRequestsByPeer.set(batch.state.peer, (this.activeRequestsByPeer.get(batch.state.peer) ?? 0) + 1);
+      }
+    }
+  }
+
+  /**
+   * Return the most suitable peer to retry
+   * Sort peers by (1) no failed request (2) less active requests, then pick first
+   */
+  bestPeerToRetryBatch(batch: Batch): PeerSyncMeta | undefined {
+    if (batch.state.status !== BatchStatus.AwaitingDownload) {
+      return;
+    }
+    const {columnsRequest} = batch.requests;
+    // TODO(fulu): This is fulu specific and hinders our peer selection PreFulu
+    const pendingDataColumns = columnsRequest?.columns ?? this.custodyConfig.sampledColumns;
+    const eligiblePeers = this.filterPeers(batch, pendingDataColumns, false);
+
+    const failedPeers = new Set(batch.getFailedPeers());
+    const sortedBestPeers = sortBy(
+      eligiblePeers,
+      ({syncInfo}) => (failedPeers.has(syncInfo.peerId) ? 1 : 0), // prefer peers without failed requests
+      ({syncInfo}) => this.activeRequestsByPeer.get(syncInfo.peerId) ?? 0, // prefer peers with least active req
+      ({columns}) => -1 * columns // prefer peers with the most columns
+    );
+
+    if (sortedBestPeers.length > 0) {
+      const bestPeer = sortedBestPeers[0];
+      // we will use this peer for batch in SyncChain right after this call
+      this.activeRequestsByPeer.set(
+        bestPeer.syncInfo.peerId,
+        (this.activeRequestsByPeer.get(bestPeer.syncInfo.peerId) ?? 0) + 1
+      );
+      return bestPeer.syncInfo;
+    }
+
+    return undefined;
+  }
+
+  /**
+   * Return peers with 0 or no active requests that has a higher target slot than this batch and has columns we need.
+   */
+  idlePeerForBatch(batch: Batch): PeerSyncInfo | undefined {
+    if (batch.state.status !== BatchStatus.AwaitingDownload) {
+      return;
+    }
+    const eligiblePeers = this.filterPeers(batch, this.custodyConfig.sampledColumns, true);
+
+    // pick idle peer that has (for pre-fulu they are the same)
+    // - earliestAvailableSlot defined
+    // - the most columns we need
+    const sortedBestPeers = sortBy(
+      eligiblePeers,
+      ({columns}) => -1 * columns // prefer peers with most columns we need
+    );
+    const bestPeer = sortedBestPeers[0];
+    if (bestPeer != null) {
+      // we will use this peer for batch in SyncChain right after this call
+      this.activeRequestsByPeer.set(bestPeer.syncInfo.peerId, 1);
+      return bestPeer.syncInfo;
+    }
+
+    return undefined;
+  }
+
+  private filterPeers(batch: Batch, requestColumns: number[], noActiveRequest: boolean): PeerInfoColumn[] {
+    const eligiblePeers: PeerInfoColumn[] = [];
+
+    if (batch.state.status !== BatchStatus.AwaitingDownload) {
+      return eligiblePeers;
+    }
+
+    for (const peer of this.peers) {
+      const {earliestAvailableSlot, target, peerId} = peer;
+
+      const activeRequest = this.activeRequestsByPeer.get(peerId) ?? 0;
+      if (noActiveRequest && activeRequest > 0) {
+        // consumer wants to find peer with no active request, but this peer has active request
+        continue;
+      }
+
+      if (activeRequest >= this.maxConcurrentRequests) {
+        // consumer wants to find peer with no more than MAX_CONCURRENT_REQUESTS active requests
+        continue;
+      }
+
+      if (target.slot < batch.startSlot) {
+        continue;
+      }
+
+      if (isForkPostFulu(batch.forkName) && this.syncType === RangeSyncType.Head) {
+        // for head sync, target slot is head slot and each peer may have a different head slot
+        // we don't want to retry a batch with a peer that's not as up-to-date as the previous peer
+        // see https://github.com/ChainSafe/lodestar/issues/8193
+        const blocks = batch.state?.blocks;
+        const lastBlock = blocks?.at(-1);
+        const lastBlockSlot = lastBlock?.slot;
+        if (lastBlockSlot && lastBlockSlot > target.slot) {
+          continue;
+        }
+      }
+
+      if (!isForkPostFulu(batch.forkName)) {
+        // pre-fulu logic, we don't care columns and earliestAvailableSlot
+        eligiblePeers.push({syncInfo: peer, columns: 0, hasEarliestAvailableSlots: false});
+        continue;
+      }
+
+      // we don't accept peers without earliestAvailableSlot because it may return 0 blocks and we get stuck
+      // see https://github.com/ChainSafe/lodestar/issues/8147
+      if (earliestAvailableSlot == null) {
+        continue;
+      }
+
+      if (earliestAvailableSlot > batch.startSlot) {
+        continue;
+      }
+
+      const columns = peer.custodyColumns.reduce((acc, elem) => {
+        if (requestColumns.includes(elem)) {
+          acc.push(elem);
+        }
+        return acc;
+      }, [] as number[]);
+
+      if (columns.length > 0) {
+        eligiblePeers.push({
+          syncInfo: peer,
+          columns: columns.length,
+          hasEarliestAvailableSlots: earliestAvailableSlot != null,
+        });
+      }
+    }
+
+    return eligiblePeers;
+  }
+}

package/src/sync/range/utils/updateChains.ts

@@ -0,0 +1,66 @@
+import {sortBy} from "../../../util/sortBy.js";
+import {MIN_FINALIZED_CHAIN_VALIDATED_EPOCHS, PARALLEL_HEAD_CHAINS} from "../../constants.js";
+import {RangeSyncType} from "../../utils/remoteSyncType.js";
+import {SyncChain} from "../chain.js";
+
+/**
+ * Priotize existing chains based on their target and peer count
+ * Returns an array of chains toStart and toStop to comply with the priotization
+ */
+export function updateChains(chains: SyncChain[]): {toStart: SyncChain[]; toStop: SyncChain[]} {
+  const finalizedChains: SyncChain[] = [];
+  const headChains: SyncChain[] = [];
+  for (const chain of chains) {
+    if (chain.syncType === RangeSyncType.Finalized) {
+      finalizedChains.push(chain);
+    } else {
+      headChains.push(chain);
+    }
+  }
+
+  const toStart: SyncChain[] = [];
+  const toStop: SyncChain[] = [];
+
+  if (finalizedChains.length > 0) {
+    // Pick first only
+    const [newSyncChain] = prioritizeSyncChains(finalizedChains);
+
+    // TODO: Should it stop all HEAD chains if going from a head sync to a finalized sync?
+
+    const currentSyncChain = finalizedChains.find((syncChain) => syncChain.isSyncing);
+
+    // Only switch from currentSyncChain to newSyncChain if necessary
+    // Avoid unnecesary switchings and try to advance it
+    if (
+      !currentSyncChain ||
+      (newSyncChain !== currentSyncChain &&
+        newSyncChain.peers > currentSyncChain.peers &&
+        currentSyncChain.validatedEpochs > MIN_FINALIZED_CHAIN_VALIDATED_EPOCHS)
+    ) {
+      toStart.push(newSyncChain);
+      if (currentSyncChain) toStop.push(currentSyncChain);
+    }
+  } else {
+    for (const syncChain of prioritizeSyncChains(headChains)) {
+      if (toStart.length < PARALLEL_HEAD_CHAINS) {
+        toStart.push(syncChain);
+      } else {
+        toStop.push(syncChain);
+      }
+    }
+  }
+
+  return {toStart, toStop};
+}
+
+/**
+ * Order `syncChains` by most peers and already syncing first
+ * If two chains have the same number of peers, prefer the already syncing to not drop progress
+ */
+function prioritizeSyncChains(syncChains: SyncChain[]): SyncChain[] {
+  return sortBy(
+    syncChains,
+    (syncChain) => -syncChain.peers, // Sort from high peer count to low: negative to reverse
+    (syncChain) => (syncChain.isSyncing ? 0 : 1) // Sort by isSyncing first = 0
+  );
+}

package/src/sync/sync.ts

@@ -0,0 +1,290 @@
+import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import {Slot} from "@lodestar/types";
+import {Logger} from "@lodestar/utils";
+import {IBeaconChain} from "../chain/index.js";
+import {GENESIS_SLOT} from "../constants/constants.js";
+import {ExecutionEngineState} from "../execution/index.js";
+import {Metrics} from "../metrics/index.js";
+import {INetwork, NetworkEvent, NetworkEventData} from "../network/index.js";
+import {ClockEvent} from "../util/clock.js";
+import {isOptimisticBlock} from "../util/forkChoice.js";
+import {MIN_EPOCH_TO_START_GOSSIP} from "./constants.js";
+import {IBeaconSync, SyncChainDebugState, SyncModules, SyncState, SyncingStatus, syncStateMetric} from "./interface.js";
+import {SyncOptions} from "./options.js";
+import {RangeSync, RangeSyncEvent, RangeSyncStatus} from "./range/range.js";
+import {BlockInputSync} from "./unknownBlock.js";
+import {PeerSyncType, getPeerSyncType, peerSyncTypes} from "./utils/remoteSyncType.js";
+
+export class BeaconSync implements IBeaconSync {
+  private readonly logger: Logger;
+  private readonly network: INetwork;
+  private readonly chain: IBeaconChain;
+  private readonly metrics: Metrics | null;
+  private readonly opts: SyncOptions;
+
+  private readonly rangeSync: RangeSync;
+  private readonly unknownBlockSync: BlockInputSync;
+
+  /** For metrics only */
+  private readonly peerSyncType = new Map<string, PeerSyncType>();
+  private readonly slotImportTolerance: Slot;
+
+  constructor(opts: SyncOptions, modules: SyncModules) {
+    const {config, chain, metrics, network, logger} = modules;
+    this.opts = opts;
+    this.network = network;
+    this.chain = chain;
+    this.metrics = metrics;
+    this.logger = logger;
+    this.rangeSync = new RangeSync(modules, opts);
+    this.unknownBlockSync = new BlockInputSync(config, network, chain, logger, metrics, opts);
+    this.slotImportTolerance = opts.slotImportTolerance ?? SLOTS_PER_EPOCH;
+
+    // Subscribe to RangeSync completing a SyncChain and recompute sync state
+    if (!opts.disableRangeSync) {
+      // prod code
+      this.logger.debug("RangeSync enabled.");
+      this.rangeSync.on(RangeSyncEvent.completedChain, this.updateSyncState);
+      this.network.events.on(NetworkEvent.peerConnected, this.addPeer);
+      this.network.events.on(NetworkEvent.peerDisconnected, this.removePeer);
+      this.chain.clock.on(ClockEvent.epoch, this.onClockEpoch);
+    } else {
+      // test code, this is needed for Unknown block sync sim test
+      this.unknownBlockSync.subscribeToNetwork();
+      this.logger.debug("RangeSync disabled.");
+
+      // In case node is started with `rangeSync` disabled and `unknownBlockSync` is enabled.
+      // If the epoch boundary happens right away the `onClockEpoch` will check for the `syncDiff` and if
+      // it's more than 2 epoch will disable the disabling the `unknownBlockSync` as well.
+      // This will result into node hanging on the head slot and not syncing any blocks.
+      // This was the scenario in the test case `Unknown block sync` in `packages/cli/test/sim/multi_fork.test.ts`
+      // So we are adding a particular delay to ensure that the `unknownBlockSync` is enabled.
+      const syncStartSlot = this.chain.clock.currentSlot;
+      // Having one epoch time for the node to connect to peers and start a syncing process
+      const epochCheckForSyncSlot = syncStartSlot + SLOTS_PER_EPOCH;
+      const initiateEpochCheckForSync = (): void => {
+        if (this.chain.clock.currentSlot > epochCheckForSyncSlot) {
+          this.logger.info("Initiating epoch check for sync progress");
+          this.chain.clock.off(ClockEvent.slot, initiateEpochCheckForSync);
+          this.chain.clock.on(ClockEvent.epoch, this.onClockEpoch);
+        }
+      };
+      this.chain.clock.on(ClockEvent.slot, initiateEpochCheckForSync);
+    }
+
+    if (metrics) {
+      metrics.syncStatus.addCollect(() => this.scrapeMetrics(metrics));
+    }
+  }
+
+  close(): void {
+    this.network.events.off(NetworkEvent.peerConnected, this.addPeer);
+    this.network.events.off(NetworkEvent.peerDisconnected, this.removePeer);
+    this.chain.clock.off(ClockEvent.epoch, this.onClockEpoch);
+    this.rangeSync.close();
+    this.unknownBlockSync.close();
+  }
+
+  getSyncStatus(): SyncingStatus {
+    const currentSlot = this.chain.clock.currentSlot;
+    const elOffline =
+      this.chain.executionEngine.state === ExecutionEngineState.OFFLINE ||
+      this.chain.executionEngine.state === ExecutionEngineState.AUTH_FAILED;
+
+    // If we are pre/at genesis, signal ready
+    if (currentSlot <= GENESIS_SLOT) {
+      return {
+        headSlot: 0,
+        syncDistance: 0,
+        isSyncing: false,
+        isOptimistic: false,
+        elOffline,
+      };
+    }
+
+    const head = this.chain.forkChoice.getHead();
+
+    switch (this.state) {
+      case SyncState.SyncingFinalized:
+      case SyncState.SyncingHead:
+      case SyncState.Stalled:
+        return {
+          headSlot: head.slot,
+          syncDistance: currentSlot - head.slot,
+          isSyncing: true,
+          isOptimistic: isOptimisticBlock(head),
+          elOffline,
+        };
+      case SyncState.Synced:
+        return {
+          headSlot: head.slot,
+          syncDistance: 0,
+          isSyncing: false,
+          isOptimistic: isOptimisticBlock(head),
+          elOffline,
+        };
+      default:
+        throw new Error("Node is stopped, cannot get sync status");
+    }
+  }
+
+  isSyncing(): boolean {
+    const state = this.state; // Don't run the getter twice
+    return state === SyncState.SyncingFinalized || state === SyncState.SyncingHead;
+  }
+
+  isSynced(): boolean {
+    return this.state === SyncState.Synced;
+  }
+
+  get state(): SyncState {
+    const currentSlot = this.chain.clock.currentSlot;
+    const headSlot = this.chain.forkChoice.getHead().slot;
+
+    if (
+      // Consider node synced IF
+      // Before genesis OR
+      (currentSlot < 0 ||
+        // head is behind clock but close enough with some tolerance
+        (headSlot <= currentSlot && headSlot >= currentSlot - this.slotImportTolerance)) &&
+      // Ensure there at least one connected peer to not claim synced if has no peers
+      // Allow to bypass this conditions for local networks with a single node
+      (this.opts.isSingleNode || this.network.getConnectedPeerCount() > 0)
+      // TODO: Consider enabling this condition (used in Lighthouse)
+      // && headSlot > 0
+    ) {
+      return SyncState.Synced;
+    }
+
+    const rangeSyncState = this.rangeSync.state;
+    switch (rangeSyncState.status) {
+      case RangeSyncStatus.Finalized:
+        return SyncState.SyncingFinalized;
+      case RangeSyncStatus.Head:
+        return SyncState.SyncingHead;
+      case RangeSyncStatus.Idle:
+        return SyncState.Stalled;
+      default:
+        throw new Error("Unreachable code");
+    }
+  }
+
+  /** Full debug state for lodestar API */
+  getSyncChainsDebugState(): SyncChainDebugState[] {
+    return this.rangeSync.getSyncChainsDebugState();
+  }
+
+  /**
+   * A peer has connected which has blocks that are unknown to us.
+   *
+   * This function handles the logic associated with the connection of a new peer. If the peer
+   * is sufficiently ahead of our current head, a range-sync (batch) sync is started and
+   * batches of blocks are queued to download from the peer. Batched blocks begin at our latest
+   * finalized head.
+   *
+   * If the peer is within the `SLOT_IMPORT_TOLERANCE`, then it's head is sufficiently close to
+   * ours that we consider it fully sync'd with respect to our current chain.
+   */
+  private addPeer = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
+    const localStatus = this.chain.getStatus();
+    const syncType = getPeerSyncType(localStatus, data.status, this.chain.forkChoice, this.slotImportTolerance);
+
+    // For metrics only
+    this.peerSyncType.set(data.peer, syncType);
+
+    if (syncType === PeerSyncType.Advanced) {
+      this.rangeSync.addPeer(data.peer, localStatus, data.status);
+    }
+
+    this.updateSyncState();
+  };
+
+  /**
+   * Must be called by libp2p when a peer is removed from the peer manager
+   */
+  private removePeer = (data: NetworkEventData[NetworkEvent.peerDisconnected]): void => {
+    this.rangeSync.removePeer(data.peer);
+
+    this.peerSyncType.delete(data.peer.toString());
+  };
+
+  /**
+   * Run this function when the sync state can potentially change.
+   */
+  private updateSyncState = (): void => {
+    const state = this.state; // Don't run the getter twice
+
+    // We have become synced, subscribe to all the gossip core topics
+    if (state === SyncState.Synced && this.chain.clock.currentEpoch >= MIN_EPOCH_TO_START_GOSSIP) {
+      if (!this.network.isSubscribedToGossipCoreTopics()) {
+        this.network
+          .subscribeGossipCoreTopics()
+          .then(() => {
+            this.metrics?.syncSwitchGossipSubscriptions.inc({action: "subscribed"});
+            this.logger.info("Subscribed gossip core topics");
+          })
+          .catch((e) => {
+            this.logger.error("Error subscribing to gossip core topics", {}, e);
+          });
+      }
+
+      // also start searching for unknown blocks
+      if (!this.unknownBlockSync.isSubscribedToNetwork()) {
+        this.unknownBlockSync.subscribeToNetwork();
+        this.metrics?.blockInputSync.switchNetworkSubscriptions.inc({action: "subscribed"});
+      }
+    }
+
+    // If we stopped being synced and fallen significantly behind, stop gossip
+    else if (state !== SyncState.Synced) {
+      const syncDiff = this.chain.clock.currentSlot - this.chain.forkChoice.getHead().slot;
+      if (syncDiff > this.slotImportTolerance * 2) {
+        if (this.network.isSubscribedToGossipCoreTopics()) {
+          this.logger.warn(`Node sync has fallen behind by ${syncDiff} slots`);
+          this.network
+            .unsubscribeGossipCoreTopics()
+            .then(() => {
+              this.metrics?.syncSwitchGossipSubscriptions.inc({action: "unsubscribed"});
+              this.logger.info("Un-subscribed gossip core topics");
+            })
+            .catch((e) => {
+              this.logger.error("Error unsubscribing to gossip core topics", {}, e);
+            });
+        }
+
+        // also stop searching for unknown blocks
+        if (this.unknownBlockSync.isSubscribedToNetwork()) {
+          this.unknownBlockSync.unsubscribeFromNetwork();
+          this.metrics?.blockInputSync.switchNetworkSubscriptions.inc({action: "unsubscribed"});
+        }
+      }
+    }
+  };
+
+  private onClockEpoch = (): void => {
+    // If a node witness the genesis event consider starting gossip
+    // Also, ensure that updateSyncState is run at least once per epoch.
+    // If the chain gets stuck or very overloaded it could helps to resolve the situation
+    // by realizing it's way behind and turning gossip off.
+    this.updateSyncState();
+  };
+
+  private scrapeMetrics(metrics: Metrics): void {
+    // Compute current sync state
+    metrics.syncStatus.set(syncStateMetric[this.state]);
+
+    // Count peers by syncType
+    const peerCountByType: Record<PeerSyncType, number> = {
+      [PeerSyncType.Advanced]: 0,
+      [PeerSyncType.FullySynced]: 0,
+      [PeerSyncType.Behind]: 0,
+    };
+    for (const syncType of this.peerSyncType.values()) {
+      peerCountByType[syncType]++;
+    }
+
+    for (const syncType of peerSyncTypes) {
+      metrics.syncPeersBySyncType.set({syncType}, peerCountByType[syncType]);
+    }
+  }
+}

package/src/sync/types.ts

@@ -0,0 +1,57 @@
+import {RootHex, Slot} from "@lodestar/types";
+import {IBlockInput} from "../chain/blocks/blockInput/index.js";
+
+export enum PendingBlockType {
+  /**
+   * We got a block root (from a gossip attestation, for exxample) but we don't have the block in forkchoice.
+   */
+  UNKNOWN_BLOCK_ROOT = "UnknownBlockRoot",
+  /**
+   * During gossip time, we may get a block but the parent root is unknown (not in forkchoice).
+   */
+  UNKNOWN_PARENT = "unknown_parent",
+  /**
+   * During gossip we wait for a set amount of time to receive the complete block input but if it does not
+   * arrive in time we turn to req/resp to pull the remainder so that it can be processed
+   */
+  INCOMPLETE_BLOCK_INPUT = "IncompleteBlockInput",
+
+  UNKNOWN_DATA = "unknown_data",
+}
+
+export enum PendingBlockInputStatus {
+  pending = "pending",
+  fetching = "fetching",
+  downloaded = "downloaded",
+  processing = "processing",
+}
+
+export type PendingBlockInput = {
+  status: PendingBlockInputStatus;
+  blockInput: IBlockInput;
+  timeAddedSec: number;
+  timeSyncedSec?: number;
+  peerIdStrings: Set<string>;
+};
+
+export type PendingRootHex = {
+  status: PendingBlockInputStatus.pending | PendingBlockInputStatus.fetching;
+  rootHex: RootHex;
+  timeAddedSec: number;
+  timeSyncedSec?: number;
+  peerIdStrings: Set<string>;
+};
+
+export type BlockInputSyncCacheItem = PendingBlockInput | PendingRootHex;
+
+export function isPendingBlockInput(pending: BlockInputSyncCacheItem): pending is PendingBlockInput {
+  return "blockInput" in pending;
+}
+
+export function getBlockInputSyncCacheItemRootHex(block: BlockInputSyncCacheItem): RootHex {
+  return isPendingBlockInput(block) ? block.blockInput.blockRootHex : block.rootHex;
+}
+
+export function getBlockInputSyncCacheItemSlot(block: BlockInputSyncCacheItem): Slot | string {
+  return isPendingBlockInput(block) ? block.blockInput.slot : "unknown";
+}