@lodestar/beacon-node 1.35.0-dev.8689cc3545 → 1.35.0-dev.8b45b1e978

package/src/sync/range/utils/peerBalancer.ts

@@ -1,184 +0,0 @@
-import {isForkPostFulu} from "@lodestar/params";
-import {PeerSyncMeta} from "../../../network/peers/peersData.js";
-import {CustodyConfig} from "../../../util/dataColumns.js";
-import {PeerIdStr} from "../../../util/peerId.js";
-import {shuffle} from "../../../util/shuffle.js";
-import {sortBy} from "../../../util/sortBy.js";
-import {MAX_CONCURRENT_REQUESTS} from "../../constants.js";
-import {RangeSyncType} from "../../utils/remoteSyncType.js";
-import {Batch, BatchStatus} from "../batch.js";
-import {ChainTarget} from "./chainTarget.js";
-
-export type PeerSyncInfo = PeerSyncMeta & {
-  target: ChainTarget;
-};
-
-type PeerInfoColumn = {syncInfo: PeerSyncInfo; columns: number; hasEarliestAvailableSlots: boolean};
-
-/**
- * Balance and organize peers to perform requests with a SyncChain
- * Shuffles peers only once on instantiation
- */
-export class ChainPeersBalancer {
-  private peers: PeerSyncInfo[];
-  private activeRequestsByPeer = new Map<PeerIdStr, number>();
-  private readonly custodyConfig: CustodyConfig;
-  private readonly syncType: RangeSyncType;
-  private readonly maxConcurrentRequests: number;
-
-  /**
-   * No need to specify `maxConcurrentRequests` for production code
-   * It is used for testing purposes to limit the number of concurrent requests
-   */
-  constructor(
-    peers: PeerSyncInfo[],
-    batches: Batch[],
-    custodyConfig: CustodyConfig,
-    syncType: RangeSyncType,
-    maxConcurrentRequests = MAX_CONCURRENT_REQUESTS
-  ) {
-    this.peers = shuffle(peers);
-    this.custodyConfig = custodyConfig;
-    this.syncType = syncType;
-    this.maxConcurrentRequests = maxConcurrentRequests;
-
-    // Compute activeRequestsByPeer from all batches internal states
-    for (const batch of batches) {
-      if (batch.state.status === BatchStatus.Downloading) {
-        this.activeRequestsByPeer.set(batch.state.peer, (this.activeRequestsByPeer.get(batch.state.peer) ?? 0) + 1);
-      }
-    }
-  }
-
-  /**
-   * Return the most suitable peer to retry
-   * Sort peers by (1) no failed request (2) less active requests, then pick first
-   */
-  bestPeerToRetryBatch(batch: Batch): PeerSyncMeta | undefined {
-    if (batch.state.status !== BatchStatus.AwaitingDownload) {
-      return;
-    }
-    const {columnsRequest} = batch.requests;
-    // TODO(fulu): This is fulu specific and hinders our peer selection PreFulu
-    const pendingDataColumns = columnsRequest?.columns ?? this.custodyConfig.sampledColumns;
-    const eligiblePeers = this.filterPeers(batch, pendingDataColumns, false);
-
-    const failedPeers = new Set(batch.getFailedPeers());
-    const sortedBestPeers = sortBy(
-      eligiblePeers,
-      ({syncInfo}) => (failedPeers.has(syncInfo.peerId) ? 1 : 0), // prefer peers without failed requests
-      ({syncInfo}) => this.activeRequestsByPeer.get(syncInfo.peerId) ?? 0, // prefer peers with least active req
-      ({columns}) => -1 * columns // prefer peers with the most columns
-    );
-
-    if (sortedBestPeers.length > 0) {
-      const bestPeer = sortedBestPeers[0];
-      // we will use this peer for batch in SyncChain right after this call
-      this.activeRequestsByPeer.set(
-        bestPeer.syncInfo.peerId,
-        (this.activeRequestsByPeer.get(bestPeer.syncInfo.peerId) ?? 0) + 1
-      );
-      return bestPeer.syncInfo;
-    }
-
-    return undefined;
-  }
-
-  /**
-   * Return peers with 0 or no active requests that has a higher target slot than this batch and has columns we need.
-   */
-  idlePeerForBatch(batch: Batch): PeerSyncInfo | undefined {
-    if (batch.state.status !== BatchStatus.AwaitingDownload) {
-      return;
-    }
-    const eligiblePeers = this.filterPeers(batch, this.custodyConfig.sampledColumns, true);
-
-    // pick idle peer that has (for pre-fulu they are the same)
-    // - earliestAvailableSlot defined
-    // - the most columns we need
-    const sortedBestPeers = sortBy(
-      eligiblePeers,
-      ({columns}) => -1 * columns // prefer peers with most columns we need
-    );
-    const bestPeer = sortedBestPeers[0];
-    if (bestPeer != null) {
-      // we will use this peer for batch in SyncChain right after this call
-      this.activeRequestsByPeer.set(bestPeer.syncInfo.peerId, 1);
-      return bestPeer.syncInfo;
-    }
-
-    return undefined;
-  }
-
-  private filterPeers(batch: Batch, requestColumns: number[], noActiveRequest: boolean): PeerInfoColumn[] {
-    const eligiblePeers: PeerInfoColumn[] = [];
-
-    if (batch.state.status !== BatchStatus.AwaitingDownload) {
-      return eligiblePeers;
-    }
-
-    for (const peer of this.peers) {
-      const {earliestAvailableSlot, target, peerId} = peer;
-
-      const activeRequest = this.activeRequestsByPeer.get(peerId) ?? 0;
-      if (noActiveRequest && activeRequest > 0) {
-        // consumer wants to find peer with no active request, but this peer has active request
-        continue;
-      }
-
-      if (activeRequest >= this.maxConcurrentRequests) {
-        // consumer wants to find peer with no more than MAX_CONCURRENT_REQUESTS active requests
-        continue;
-      }
-
-      if (target.slot < batch.startSlot) {
-        continue;
-      }
-
-      if (isForkPostFulu(batch.forkName) && this.syncType === RangeSyncType.Head) {
-        // for head sync, target slot is head slot and each peer may have a different head slot
-        // we don't want to retry a batch with a peer that's not as up-to-date as the previous peer
-        // see https://github.com/ChainSafe/lodestar/issues/8193
-        const blocks = batch.state?.blocks;
-        const lastBlock = blocks?.at(-1);
-        const lastBlockSlot = lastBlock?.slot;
-        if (lastBlockSlot && lastBlockSlot > target.slot) {
-          continue;
-        }
-      }
-
-      if (!isForkPostFulu(batch.forkName)) {
-        // pre-fulu logic, we don't care columns and earliestAvailableSlot
-        eligiblePeers.push({syncInfo: peer, columns: 0, hasEarliestAvailableSlots: false});
-        continue;
-      }
-
-      // we don't accept peers without earliestAvailableSlot because it may return 0 blocks and we get stuck
-      // see https://github.com/ChainSafe/lodestar/issues/8147
-      if (earliestAvailableSlot == null) {
-        continue;
-      }
-
-      if (earliestAvailableSlot > batch.startSlot) {
-        continue;
-      }
-
-      const columns = peer.custodyColumns.reduce((acc, elem) => {
-        if (requestColumns.includes(elem)) {
-          acc.push(elem);
-        }
-        return acc;
-      }, [] as number[]);
-
-      if (columns.length > 0) {
-        eligiblePeers.push({
-          syncInfo: peer,
-          columns: columns.length,
-          hasEarliestAvailableSlots: earliestAvailableSlot != null,
-        });
-      }
-    }
-
-    return eligiblePeers;
-  }
-}

package/src/sync/range/utils/updateChains.ts

@@ -1,66 +0,0 @@
-import {sortBy} from "../../../util/sortBy.js";
-import {MIN_FINALIZED_CHAIN_VALIDATED_EPOCHS, PARALLEL_HEAD_CHAINS} from "../../constants.js";
-import {RangeSyncType} from "../../utils/remoteSyncType.js";
-import {SyncChain} from "../chain.js";
-
-/**
- * Priotize existing chains based on their target and peer count
- * Returns an array of chains toStart and toStop to comply with the priotization
- */
-export function updateChains(chains: SyncChain[]): {toStart: SyncChain[]; toStop: SyncChain[]} {
-  const finalizedChains: SyncChain[] = [];
-  const headChains: SyncChain[] = [];
-  for (const chain of chains) {
-    if (chain.syncType === RangeSyncType.Finalized) {
-      finalizedChains.push(chain);
-    } else {
-      headChains.push(chain);
-    }
-  }
-
-  const toStart: SyncChain[] = [];
-  const toStop: SyncChain[] = [];
-
-  if (finalizedChains.length > 0) {
-    // Pick first only
-    const [newSyncChain] = prioritizeSyncChains(finalizedChains);
-
-    // TODO: Should it stop all HEAD chains if going from a head sync to a finalized sync?
-
-    const currentSyncChain = finalizedChains.find((syncChain) => syncChain.isSyncing);
-
-    // Only switch from currentSyncChain to newSyncChain if necessary
-    // Avoid unnecesary switchings and try to advance it
-    if (
-      !currentSyncChain ||
-      (newSyncChain !== currentSyncChain &&
-        newSyncChain.peers > currentSyncChain.peers &&
-        currentSyncChain.validatedEpochs > MIN_FINALIZED_CHAIN_VALIDATED_EPOCHS)
-    ) {
-      toStart.push(newSyncChain);
-      if (currentSyncChain) toStop.push(currentSyncChain);
-    }
-  } else {
-    for (const syncChain of prioritizeSyncChains(headChains)) {
-      if (toStart.length < PARALLEL_HEAD_CHAINS) {
-        toStart.push(syncChain);
-      } else {
-        toStop.push(syncChain);
-      }
-    }
-  }
-
-  return {toStart, toStop};
-}
-
-/**
- * Order `syncChains` by most peers and already syncing first
- * If two chains have the same number of peers, prefer the already syncing to not drop progress
- */
-function prioritizeSyncChains(syncChains: SyncChain[]): SyncChain[] {
-  return sortBy(
-    syncChains,
-    (syncChain) => -syncChain.peers, // Sort from high peer count to low: negative to reverse
-    (syncChain) => (syncChain.isSyncing ? 0 : 1) // Sort by isSyncing first = 0
-  );
-}

package/src/sync/sync.ts

@@ -1,290 +0,0 @@
-import {SLOTS_PER_EPOCH} from "@lodestar/params";
-import {Slot} from "@lodestar/types";
-import {Logger} from "@lodestar/utils";
-import {IBeaconChain} from "../chain/index.js";
-import {GENESIS_SLOT} from "../constants/constants.js";
-import {ExecutionEngineState} from "../execution/index.js";
-import {Metrics} from "../metrics/index.js";
-import {INetwork, NetworkEvent, NetworkEventData} from "../network/index.js";
-import {ClockEvent} from "../util/clock.js";
-import {isOptimisticBlock} from "../util/forkChoice.js";
-import {MIN_EPOCH_TO_START_GOSSIP} from "./constants.js";
-import {IBeaconSync, SyncChainDebugState, SyncModules, SyncState, SyncingStatus, syncStateMetric} from "./interface.js";
-import {SyncOptions} from "./options.js";
-import {RangeSync, RangeSyncEvent, RangeSyncStatus} from "./range/range.js";
-import {BlockInputSync} from "./unknownBlock.js";
-import {PeerSyncType, getPeerSyncType, peerSyncTypes} from "./utils/remoteSyncType.js";
-
-export class BeaconSync implements IBeaconSync {
-  private readonly logger: Logger;
-  private readonly network: INetwork;
-  private readonly chain: IBeaconChain;
-  private readonly metrics: Metrics | null;
-  private readonly opts: SyncOptions;
-
-  private readonly rangeSync: RangeSync;
-  private readonly unknownBlockSync: BlockInputSync;
-
-  /** For metrics only */
-  private readonly peerSyncType = new Map<string, PeerSyncType>();
-  private readonly slotImportTolerance: Slot;
-
-  constructor(opts: SyncOptions, modules: SyncModules) {
-    const {config, chain, metrics, network, logger} = modules;
-    this.opts = opts;
-    this.network = network;
-    this.chain = chain;
-    this.metrics = metrics;
-    this.logger = logger;
-    this.rangeSync = new RangeSync(modules, opts);
-    this.unknownBlockSync = new BlockInputSync(config, network, chain, logger, metrics, opts);
-    this.slotImportTolerance = opts.slotImportTolerance ?? SLOTS_PER_EPOCH;
-
-    // Subscribe to RangeSync completing a SyncChain and recompute sync state
-    if (!opts.disableRangeSync) {
-      // prod code
-      this.logger.debug("RangeSync enabled.");
-      this.rangeSync.on(RangeSyncEvent.completedChain, this.updateSyncState);
-      this.network.events.on(NetworkEvent.peerConnected, this.addPeer);
-      this.network.events.on(NetworkEvent.peerDisconnected, this.removePeer);
-      this.chain.clock.on(ClockEvent.epoch, this.onClockEpoch);
-    } else {
-      // test code, this is needed for Unknown block sync sim test
-      this.unknownBlockSync.subscribeToNetwork();
-      this.logger.debug("RangeSync disabled.");
-
-      // In case node is started with `rangeSync` disabled and `unknownBlockSync` is enabled.
-      // If the epoch boundary happens right away the `onClockEpoch` will check for the `syncDiff` and if
-      // it's more than 2 epoch will disable the disabling the `unknownBlockSync` as well.
-      // This will result into node hanging on the head slot and not syncing any blocks.
-      // This was the scenario in the test case `Unknown block sync` in `packages/cli/test/sim/multi_fork.test.ts`
-      // So we are adding a particular delay to ensure that the `unknownBlockSync` is enabled.
-      const syncStartSlot = this.chain.clock.currentSlot;
-      // Having one epoch time for the node to connect to peers and start a syncing process
-      const epochCheckForSyncSlot = syncStartSlot + SLOTS_PER_EPOCH;
-      const initiateEpochCheckForSync = (): void => {
-        if (this.chain.clock.currentSlot > epochCheckForSyncSlot) {
-          this.logger.info("Initiating epoch check for sync progress");
-          this.chain.clock.off(ClockEvent.slot, initiateEpochCheckForSync);
-          this.chain.clock.on(ClockEvent.epoch, this.onClockEpoch);
-        }
-      };
-      this.chain.clock.on(ClockEvent.slot, initiateEpochCheckForSync);
-    }
-
-    if (metrics) {
-      metrics.syncStatus.addCollect(() => this.scrapeMetrics(metrics));
-    }
-  }
-
-  close(): void {
-    this.network.events.off(NetworkEvent.peerConnected, this.addPeer);
-    this.network.events.off(NetworkEvent.peerDisconnected, this.removePeer);
-    this.chain.clock.off(ClockEvent.epoch, this.onClockEpoch);
-    this.rangeSync.close();
-    this.unknownBlockSync.close();
-  }
-
-  getSyncStatus(): SyncingStatus {
-    const currentSlot = this.chain.clock.currentSlot;
-    const elOffline =
-      this.chain.executionEngine.state === ExecutionEngineState.OFFLINE ||
-      this.chain.executionEngine.state === ExecutionEngineState.AUTH_FAILED;
-
-    // If we are pre/at genesis, signal ready
-    if (currentSlot <= GENESIS_SLOT) {
-      return {
-        headSlot: 0,
-        syncDistance: 0,
-        isSyncing: false,
-        isOptimistic: false,
-        elOffline,
-      };
-    }
-
-    const head = this.chain.forkChoice.getHead();
-
-    switch (this.state) {
-      case SyncState.SyncingFinalized:
-      case SyncState.SyncingHead:
-      case SyncState.Stalled:
-        return {
-          headSlot: head.slot,
-          syncDistance: currentSlot - head.slot,
-          isSyncing: true,
-          isOptimistic: isOptimisticBlock(head),
-          elOffline,
-        };
-      case SyncState.Synced:
-        return {
-          headSlot: head.slot,
-          syncDistance: 0,
-          isSyncing: false,
-          isOptimistic: isOptimisticBlock(head),
-          elOffline,
-        };
-      default:
-        throw new Error("Node is stopped, cannot get sync status");
-    }
-  }
-
-  isSyncing(): boolean {
-    const state = this.state; // Don't run the getter twice
-    return state === SyncState.SyncingFinalized || state === SyncState.SyncingHead;
-  }
-
-  isSynced(): boolean {
-    return this.state === SyncState.Synced;
-  }
-
-  get state(): SyncState {
-    const currentSlot = this.chain.clock.currentSlot;
-    const headSlot = this.chain.forkChoice.getHead().slot;
-
-    if (
-      // Consider node synced IF
-      // Before genesis OR
-      (currentSlot < 0 ||
-        // head is behind clock but close enough with some tolerance
-        (headSlot <= currentSlot && headSlot >= currentSlot - this.slotImportTolerance)) &&
-      // Ensure there at least one connected peer to not claim synced if has no peers
-      // Allow to bypass this conditions for local networks with a single node
-      (this.opts.isSingleNode || this.network.getConnectedPeerCount() > 0)
-      // TODO: Consider enabling this condition (used in Lighthouse)
-      // && headSlot > 0
-    ) {
-      return SyncState.Synced;
-    }
-
-    const rangeSyncState = this.rangeSync.state;
-    switch (rangeSyncState.status) {
-      case RangeSyncStatus.Finalized:
-        return SyncState.SyncingFinalized;
-      case RangeSyncStatus.Head:
-        return SyncState.SyncingHead;
-      case RangeSyncStatus.Idle:
-        return SyncState.Stalled;
-      default:
-        throw new Error("Unreachable code");
-    }
-  }
-
-  /** Full debug state for lodestar API */
-  getSyncChainsDebugState(): SyncChainDebugState[] {
-    return this.rangeSync.getSyncChainsDebugState();
-  }
-
-  /**
-   * A peer has connected which has blocks that are unknown to us.
-   *
-   * This function handles the logic associated with the connection of a new peer. If the peer
-   * is sufficiently ahead of our current head, a range-sync (batch) sync is started and
-   * batches of blocks are queued to download from the peer. Batched blocks begin at our latest
-   * finalized head.
-   *
-   * If the peer is within the `SLOT_IMPORT_TOLERANCE`, then it's head is sufficiently close to
-   * ours that we consider it fully sync'd with respect to our current chain.
-   */
-  private addPeer = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
-    const localStatus = this.chain.getStatus();
-    const syncType = getPeerSyncType(localStatus, data.status, this.chain.forkChoice, this.slotImportTolerance);
-
-    // For metrics only
-    this.peerSyncType.set(data.peer, syncType);
-
-    if (syncType === PeerSyncType.Advanced) {
-      this.rangeSync.addPeer(data.peer, localStatus, data.status);
-    }
-
-    this.updateSyncState();
-  };
-
-  /**
-   * Must be called by libp2p when a peer is removed from the peer manager
-   */
-  private removePeer = (data: NetworkEventData[NetworkEvent.peerDisconnected]): void => {
-    this.rangeSync.removePeer(data.peer);
-
-    this.peerSyncType.delete(data.peer.toString());
-  };
-
-  /**
-   * Run this function when the sync state can potentially change.
-   */
-  private updateSyncState = (): void => {
-    const state = this.state; // Don't run the getter twice
-
-    // We have become synced, subscribe to all the gossip core topics
-    if (state === SyncState.Synced && this.chain.clock.currentEpoch >= MIN_EPOCH_TO_START_GOSSIP) {
-      if (!this.network.isSubscribedToGossipCoreTopics()) {
-        this.network
-          .subscribeGossipCoreTopics()
-          .then(() => {
-            this.metrics?.syncSwitchGossipSubscriptions.inc({action: "subscribed"});
-            this.logger.info("Subscribed gossip core topics");
-          })
-          .catch((e) => {
-            this.logger.error("Error subscribing to gossip core topics", {}, e);
-          });
-      }
-
-      // also start searching for unknown blocks
-      if (!this.unknownBlockSync.isSubscribedToNetwork()) {
-        this.unknownBlockSync.subscribeToNetwork();
-        this.metrics?.blockInputSync.switchNetworkSubscriptions.inc({action: "subscribed"});
-      }
-    }
-
-    // If we stopped being synced and fallen significantly behind, stop gossip
-    else if (state !== SyncState.Synced) {
-      const syncDiff = this.chain.clock.currentSlot - this.chain.forkChoice.getHead().slot;
-      if (syncDiff > this.slotImportTolerance * 2) {
-        if (this.network.isSubscribedToGossipCoreTopics()) {
-          this.logger.warn(`Node sync has fallen behind by ${syncDiff} slots`);
-          this.network
-            .unsubscribeGossipCoreTopics()
-            .then(() => {
-              this.metrics?.syncSwitchGossipSubscriptions.inc({action: "unsubscribed"});
-              this.logger.info("Un-subscribed gossip core topics");
-            })
-            .catch((e) => {
-              this.logger.error("Error unsubscribing to gossip core topics", {}, e);
-            });
-        }
-
-        // also stop searching for unknown blocks
-        if (this.unknownBlockSync.isSubscribedToNetwork()) {
-          this.unknownBlockSync.unsubscribeFromNetwork();
-          this.metrics?.blockInputSync.switchNetworkSubscriptions.inc({action: "unsubscribed"});
-        }
-      }
-    }
-  };
-
-  private onClockEpoch = (): void => {
-    // If a node witness the genesis event consider starting gossip
-    // Also, ensure that updateSyncState is run at least once per epoch.
-    // If the chain gets stuck or very overloaded it could helps to resolve the situation
-    // by realizing it's way behind and turning gossip off.
-    this.updateSyncState();
-  };
-
-  private scrapeMetrics(metrics: Metrics): void {
-    // Compute current sync state
-    metrics.syncStatus.set(syncStateMetric[this.state]);
-
-    // Count peers by syncType
-    const peerCountByType: Record<PeerSyncType, number> = {
-      [PeerSyncType.Advanced]: 0,
-      [PeerSyncType.FullySynced]: 0,
-      [PeerSyncType.Behind]: 0,
-    };
-    for (const syncType of this.peerSyncType.values()) {
-      peerCountByType[syncType]++;
-    }
-
-    for (const syncType of peerSyncTypes) {
-      metrics.syncPeersBySyncType.set({syncType}, peerCountByType[syncType]);
-    }
-  }
-}

package/src/sync/types.ts

@@ -1,59 +0,0 @@
-import {RootHex, Slot} from "@lodestar/types";
-import {IBlockInput} from "../chain/blocks/blockInput/index.js";
-
-export enum PendingBlockType {
-  /**
-   * We got a block root (from a gossip attestation, for exxample) but we don't have the block in forkchoice.
-   */
-  UNKNOWN_BLOCK_ROOT = "UnknownBlockRoot",
-  /**
-   * During gossip time, we may get a block but the parent root is unknown (not in forkchoice).
-   */
-  UNKNOWN_PARENT = "unknown_parent",
-  /**
-   * During gossip we wait for a set amount of time to receive the complete block input but if it does not
-   * arrive in time we turn to req/resp to pull the remainder so that it can be processed
-   */
-  INCOMPLETE_BLOCK_INPUT = "IncompleteBlockInput",
-
-  UNKNOWN_DATA = "unknown_data",
-}
-
-export enum PendingBlockInputStatus {
-  pending = "pending",
-  fetching = "fetching",
-  downloaded = "downloaded",
-  processing = "processing",
-}
-
-export type PendingBlockInput = {
-  status: PendingBlockInputStatus;
-  blockInput: IBlockInput;
-  timeAddedSec: number;
-  timeSyncedSec?: number;
-  peerIdStrings: Set<string>;
-};
-
-export type PendingRootHex = {
-  status: PendingBlockInputStatus.pending | PendingBlockInputStatus.fetching;
-  rootHex: RootHex;
-  // optional because we may not know the slot of parent_unknown event
-  slot?: Slot;
-  timeAddedSec: number;
-  timeSyncedSec?: number;
-  peerIdStrings: Set<string>;
-};
-
-export type BlockInputSyncCacheItem = PendingBlockInput | PendingRootHex;
-
-export function isPendingBlockInput(pending: BlockInputSyncCacheItem): pending is PendingBlockInput {
-  return "blockInput" in pending;
-}
-
-export function getBlockInputSyncCacheItemRootHex(block: BlockInputSyncCacheItem): RootHex {
-  return isPendingBlockInput(block) ? block.blockInput.blockRootHex : block.rootHex;
-}
-
-export function getBlockInputSyncCacheItemSlot(block: BlockInputSyncCacheItem): Slot | string {
-  return isPendingBlockInput(block) ? block.blockInput.slot : "unknown";
-}