@lodestar/beacon-node 1.35.0-dev.39be8f0882 → 1.35.0-dev.3d8b04e1a0

package/src/chain/regen/regen.ts

@@ -1,424 +0,0 @@
-import {ChainForkConfig} from "@lodestar/config";
-import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
-import {SLOTS_PER_EPOCH} from "@lodestar/params";
-import {
-  CachedBeaconStateAllForks,
-  DataAvailabilityStatus,
-  ExecutionPayloadStatus,
-  StateHashTreeRootSource,
-  computeEpochAtSlot,
-  computeStartSlotAtEpoch,
-  processSlots,
-  stateTransition,
-} from "@lodestar/state-transition";
-import {BeaconBlock, RootHex, SignedBeaconBlock, Slot, phase0} from "@lodestar/types";
-import {Logger, fromHex, toRootHex} from "@lodestar/utils";
-import {IBeaconDb} from "../../db/index.js";
-import {Metrics} from "../../metrics/index.js";
-import {nextEventLoop} from "../../util/eventLoop.js";
-import {getCheckpointFromState} from "../blocks/utils/checkpoint.js";
-import {ChainEvent, ChainEventEmitter} from "../emitter.js";
-import {BlockStateCache, CheckpointStateCache} from "../stateCache/types.js";
-import {ValidatorMonitor} from "../validatorMonitor.js";
-import {RegenError, RegenErrorCode} from "./errors.js";
-import {IStateRegeneratorInternal, RegenCaller, StateRegenerationOpts} from "./interface.js";
-
-export type RegenModules = {
-  db: IBeaconDb;
-  forkChoice: IForkChoice;
-  blockStateCache: BlockStateCache;
-  checkpointStateCache: CheckpointStateCache;
-  config: ChainForkConfig;
-  emitter: ChainEventEmitter;
-  logger: Logger;
-  metrics: Metrics | null;
-  validatorMonitor: ValidatorMonitor | null;
-};
-
-/**
- * Regenerates states that have already been processed by the fork choice
- * Since Feb 2024, we support reloading checkpoint state from disk via allowDiskReload flag. Due to its performance impact
- * this flag is only set to true in this case:
- *    - getPreState: this is for block processing, it's important to reload state in unfinality time
- *    - updateHeadState: rarely happen, but it's important to make sure we always can regen head state
- */
-export class StateRegenerator implements IStateRegeneratorInternal {
-  constructor(private readonly modules: RegenModules) {}
-
-  /**
-   * Get the state to run with `block`. May be:
-   * - If parent is in same epoch -> Exact state at `block.parentRoot`
-   * - If parent is in prev epoch -> State after `block.parentRoot` dialed forward through epoch transition
-   * - reload state if needed in this flow
-   */
-  async getPreState(
-    block: BeaconBlock,
-    opts: StateRegenerationOpts,
-    regenCaller: RegenCaller
-  ): Promise<CachedBeaconStateAllForks> {
-    const parentBlock = this.modules.forkChoice.getBlock(block.parentRoot);
-    if (!parentBlock) {
-      throw new RegenError({
-        code: RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE,
-        blockRoot: block.parentRoot,
-      });
-    }
-
-    const parentEpoch = computeEpochAtSlot(parentBlock.slot);
-    const blockEpoch = computeEpochAtSlot(block.slot);
-    const allowDiskReload = true;
-
-    // This may save us at least one epoch transition.
-    // If the requested state crosses an epoch boundary
-    // then we may use the checkpoint state before the block
-    // We may have the checkpoint state with parent root inside the checkpoint state cache
-    // through gossip validation.
-    if (parentEpoch < blockEpoch) {
-      return this.getCheckpointState({root: block.parentRoot, epoch: blockEpoch}, opts, regenCaller, allowDiskReload);
-    }
-
-    // Otherwise, get the state normally.
-    return this.getState(parentBlock.stateRoot, regenCaller, opts, allowDiskReload);
-  }
-
-  /**
-   * Get state after block `cp.root` dialed forward to first slot of `cp.epoch`
-   */
-  async getCheckpointState(
-    cp: phase0.Checkpoint,
-    opts: StateRegenerationOpts,
-    regenCaller: RegenCaller,
-    allowDiskReload = false
-  ): Promise<CachedBeaconStateAllForks> {
-    const checkpointStartSlot = computeStartSlotAtEpoch(cp.epoch);
-    return this.getBlockSlotState(toRootHex(cp.root), checkpointStartSlot, opts, regenCaller, allowDiskReload);
-  }
-
-  /**
-   * Get state after block `blockRoot` dialed forward to `slot`
-   *   - allowDiskReload should be used with care, as it will cause the state to be reloaded from disk
-   */
-  async getBlockSlotState(
-    blockRoot: RootHex,
-    slot: Slot,
-    opts: StateRegenerationOpts,
-    regenCaller: RegenCaller,
-    allowDiskReload = false
-  ): Promise<CachedBeaconStateAllForks> {
-    const block = this.modules.forkChoice.getBlockHex(blockRoot);
-    if (!block) {
-      throw new RegenError({
-        code: RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE,
-        blockRoot,
-      });
-    }
-
-    if (slot < block.slot) {
-      throw new RegenError({
-        code: RegenErrorCode.SLOT_BEFORE_BLOCK_SLOT,
-        slot,
-        blockSlot: block.slot,
-      });
-    }
-
-    const {checkpointStateCache} = this.modules;
-    const epoch = computeEpochAtSlot(slot);
-    const latestCheckpointStateCtx = allowDiskReload
-      ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch, opts)
-      : checkpointStateCache.getLatest(blockRoot, epoch, opts);
-
-    // If a checkpoint state exists with the given checkpoint root, it either is in requested epoch
-    // or needs to have empty slots processed until the requested epoch
-    if (latestCheckpointStateCtx) {
-      return processSlotsByCheckpoint(this.modules, latestCheckpointStateCtx, slot, regenCaller, opts);
-    }
-
-    // Otherwise, use the fork choice to get the stateRoot from block at the checkpoint root
-    // regenerate that state,
-    // then process empty slots until the requested epoch
-    const blockStateCtx = await this.getState(block.stateRoot, regenCaller, opts, allowDiskReload);
-    return processSlotsByCheckpoint(this.modules, blockStateCtx, slot, regenCaller, opts);
-  }
-
-  /**
-   * Get state by exact root. If not in cache directly, requires finding the block that references the state from the
-   * forkchoice and replaying blocks to get to it.
-   *   - allowDiskReload should be used with care, as it will cause the state to be reloaded from disk
-   */
-  async getState(
-    stateRoot: RootHex,
-    caller: RegenCaller,
-    opts?: StateRegenerationOpts,
-    // internal option, don't want to expose to external caller
-    allowDiskReload = false
-  ): Promise<CachedBeaconStateAllForks> {
-    // Trivial case, state at stateRoot is already cached
-    const cachedStateCtx = this.modules.blockStateCache.get(stateRoot, opts);
-    if (cachedStateCtx) {
-      return cachedStateCtx;
-    }
-
-    // in block gossip validation (getPreState() call), dontTransferCache is specified as true because we only want to transfer cache in verifyBlocksStateTransitionOnly()
-    // but here we want to process blocks as fast as possible so force to transfer cache in this case
-    if (opts && allowDiskReload) {
-      // if there is no `opts` specified, it already means "false"
-      opts.dontTransferCache = false;
-    }
-
-    // Otherwise we have to use the fork choice to traverse backwards, block by block,
-    // searching the state caches
-    // then replay blocks forward to the desired stateRoot
-    const block = this.findFirstStateBlock(stateRoot);
-
-    // blocks to replay, ordered highest to lowest
-    // gets reversed when replayed
-    const blocksToReplay = [block];
-    let state: CachedBeaconStateAllForks | null = null;
-    const {checkpointStateCache} = this.modules;
-
-    const getSeedStateTimer = this.modules.metrics?.regenGetState.getSeedState.startTimer({caller});
-    // iterateAncestorBlocks only returns ancestor blocks, not the block itself
-    for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot)) {
-      state = this.modules.blockStateCache.get(b.stateRoot, opts);
-      if (state) {
-        break;
-      }
-      const lastBlockToReplay = blocksToReplay.at(-1);
-      if (!lastBlockToReplay) continue;
-      const epoch = computeEpochAtSlot(lastBlockToReplay.slot - 1);
-      state = allowDiskReload
-        ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch, opts)
-        : checkpointStateCache.getLatest(b.blockRoot, epoch, opts);
-      if (state) {
-        break;
-      }
-      blocksToReplay.push(b);
-    }
-    getSeedStateTimer?.();
-
-    if (state === null) {
-      throw new RegenError({
-        code: RegenErrorCode.NO_SEED_STATE,
-      });
-    }
-
-    const blockCount = blocksToReplay.length;
-    const MAX_EPOCH_TO_PROCESS = 5;
-    if (blockCount > MAX_EPOCH_TO_PROCESS * SLOTS_PER_EPOCH) {
-      throw new RegenError({
-        code: RegenErrorCode.TOO_MANY_BLOCK_PROCESSED,
-        stateRoot,
-      });
-    }
-
-    this.modules.metrics?.regenGetState.blockCount.observe({caller}, blockCount);
-
-    const replaySlots = new Array<Slot>(blockCount);
-    const blockPromises = new Array<Promise<SignedBeaconBlock | null>>(blockCount);
-
-    const protoBlocksAsc = blocksToReplay.reverse();
-    for (const [i, protoBlock] of protoBlocksAsc.entries()) {
-      replaySlots[i] = protoBlock.slot;
-      blockPromises[i] = this.modules.db.block.get(fromHex(protoBlock.blockRoot));
-    }
-
-    const logCtx = {stateRoot, caller, replaySlots: replaySlots.join(",")};
-    this.modules.logger.debug("Replaying blocks to get state", logCtx);
-
-    const loadBlocksTimer = this.modules.metrics?.regenGetState.loadBlocks.startTimer({caller});
-    const blockOrNulls = await Promise.all(blockPromises);
-    loadBlocksTimer?.();
-
-    const blocksByRoot = new Map<RootHex, SignedBeaconBlock>();
-    for (const [i, blockOrNull] of blockOrNulls.entries()) {
-      // checking early here helps prevent unneccessary state transition below
-      if (blockOrNull === null) {
-        throw new RegenError({
-          code: RegenErrorCode.BLOCK_NOT_IN_DB,
-          blockRoot: protoBlocksAsc[i].blockRoot,
-        });
-      }
-      blocksByRoot.set(protoBlocksAsc[i].blockRoot, blockOrNull);
-    }
-
-    const stateTransitionTimer = this.modules.metrics?.regenGetState.stateTransition.startTimer({caller});
-    for (const b of protoBlocksAsc) {
-      const block = blocksByRoot.get(b.blockRoot);
-      // just to make compiler happy, we checked in the above for loop already
-      if (block === undefined) {
-        throw new RegenError({
-          code: RegenErrorCode.BLOCK_NOT_IN_DB,
-          blockRoot: b.blockRoot,
-        });
-      }
-
-      try {
-        // Only advances state trusting block's signture and hashes.
-        // We are only running the state transition to get a specific state's data.
-        state = stateTransition(
-          state,
-          block,
-          {
-            // Replay previously imported blocks, assume valid and available
-            executionPayloadStatus: ExecutionPayloadStatus.valid,
-            dataAvailabilityStatus: DataAvailabilityStatus.Available,
-            verifyStateRoot: false,
-            verifyProposer: false,
-            verifySignatures: false,
-          },
-          this.modules
-        );
-
-        const hashTreeRootTimer = this.modules.metrics?.stateHashTreeRootTime.startTimer({
-          source: StateHashTreeRootSource.regenState,
-        });
-        const stateRoot = toRootHex(state.hashTreeRoot());
-        hashTreeRootTimer?.();
-
-        if (b.stateRoot !== stateRoot) {
-          throw new RegenError({
-            slot: b.slot,
-            code: RegenErrorCode.INVALID_STATE_ROOT,
-            actual: stateRoot,
-            expected: b.stateRoot,
-          });
-        }
-
-        if (allowDiskReload) {
-          // also with allowDiskReload flag, we "reload" it to the state cache too
-          this.modules.blockStateCache.add(state);
-        }
-      } catch (e) {
-        throw new RegenError({
-          code: RegenErrorCode.STATE_TRANSITION_ERROR,
-          error: e as Error,
-        });
-      }
-    }
-    stateTransitionTimer?.();
-
-    this.modules.logger.debug("Replayed blocks to get state", {...logCtx, stateSlot: state.slot});
-
-    return state;
-  }
-
-  private findFirstStateBlock(stateRoot: RootHex): ProtoBlock {
-    for (const block of this.modules.forkChoice.forwarditerateAncestorBlocks()) {
-      if (block.stateRoot === stateRoot) {
-        return block;
-      }
-    }
-
-    throw new RegenError({
-      code: RegenErrorCode.STATE_NOT_IN_FORKCHOICE,
-      stateRoot,
-    });
-  }
-}
-
-/**
- * Starting at `state.slot`,
- * process slots forward towards `slot`,
- * emitting "checkpoint" events after every epoch processed.
- */
-async function processSlotsByCheckpoint(
-  modules: {
-    checkpointStateCache: CheckpointStateCache;
-    metrics: Metrics | null;
-    validatorMonitor: ValidatorMonitor | null;
-    emitter: ChainEventEmitter;
-    logger: Logger;
-  },
-  preState: CachedBeaconStateAllForks,
-  slot: Slot,
-  regenCaller: RegenCaller,
-  opts: StateRegenerationOpts
-): Promise<CachedBeaconStateAllForks> {
-  let postState = await processSlotsToNearestCheckpoint(modules, preState, slot, regenCaller, opts);
-  if (postState.slot < slot) {
-    postState = processSlots(postState, slot, opts, modules);
-  }
-  return postState;
-}
-
-/**
- * Starting at `state.slot`,
- * process slots forward towards `slot`,
- * emitting "checkpoint" events after every epoch processed.
- *
- * Stops processing after no more full epochs can be processed.
- */
-export async function processSlotsToNearestCheckpoint(
-  modules: {
-    checkpointStateCache: CheckpointStateCache;
-    metrics: Metrics | null;
-    validatorMonitor: ValidatorMonitor | null;
-    emitter: ChainEventEmitter | null;
-    logger: Logger | null;
-  },
-  preState: CachedBeaconStateAllForks,
-  slot: Slot,
-  regenCaller: RegenCaller,
-  opts: StateRegenerationOpts
-): Promise<CachedBeaconStateAllForks> {
-  const preSlot = preState.slot;
-  const postSlot = slot;
-  const preEpoch = computeEpochAtSlot(preSlot);
-  let postState = preState;
-  const {checkpointStateCache, emitter, metrics, logger} = modules;
-  let count = 0;
-
-  for (
-    let nextEpochSlot = computeStartSlotAtEpoch(preEpoch + 1);
-    nextEpochSlot <= postSlot;
-    nextEpochSlot += SLOTS_PER_EPOCH
-  ) {
-    logger?.verbose("Processing slots over epochs", {
-      slot: postState.slot,
-      nextEpochSlot,
-      postSlot,
-      caller: regenCaller,
-    });
-    // processSlots calls .clone() before mutating
-    postState = processSlots(postState, nextEpochSlot, opts, modules);
-    metrics?.epochTransitionByCaller.inc({caller: regenCaller});
-
-    // this is usually added when we prepare for next slot or validate gossip block
-    // then when we process the 1st block of epoch, we don't have to do state transition again
-    // This adds Previous Root Checkpoint State to the checkpoint state cache
-    // This may becomes the "official" checkpoint state if the 1st block of epoch is skipped
-    const checkpointState = postState;
-    const cp = getCheckpointFromState(checkpointState);
-    checkpointStateCache.add(cp, checkpointState);
-    // consumers should not mutate or get the transfered cache
-    emitter?.emit(ChainEvent.checkpoint, cp, checkpointState.clone(true));
-
-    if (count >= 1) {
-      // in normal condition, we only process 1 epoch so never reach this
-      // in that case, we want to prune state at the last 1/3 slot of slot 0 of the next epoch after importing the 1st block of epoch
-      // in non-finality time, we may process a lot of epochs so need to prune the cache to keep the node healthy
-      // this happened to holesky on Feb 2025, see https://github.com/ChainSafe/lodestar/issues/7495#issuecomment-2680800898
-      // cannot use getBlockRootAtSlot() because nextEpochSlot = postState
-      const latestBlockHex = toRootHex(cp.root);
-      try {
-        const persistCount = await checkpointStateCache.processState(latestBlockHex, checkpointState);
-        logger?.verbose("pruning checkpointStateCache during processSlotsToNearestCheckpoint", {
-          root: latestBlockHex,
-          epoch: cp.epoch,
-          persistCount,
-        });
-      } catch (e) {
-        logger?.debug(
-          "CheckpointStateCache failed to process checkpoint state",
-          {root: latestBlockHex, epoch: cp.epoch},
-          e as Error
-        );
-      }
-    }
-    count++;
-
-    // this avoids keeping our node busy processing blocks
-    await nextEventLoop();
-  }
-  return postState;
-}

package/src/chain/reprocess.ts

@@ -1,161 +0,0 @@
-import {RootHex, Slot} from "@lodestar/types";
-import {MapDef} from "@lodestar/utils";
-import {Metrics} from "../metrics/index.js";
-
-/**
- * To prevent our node from having to reprocess while struggling to sync,
- * we only want to reprocess attestations if block reaches our node before this time.
- */
-export const REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC = 2;
-
-/**
- * Reprocess status for metrics
- */
-export enum ReprocessStatus {
-  /**
-   * There are too many attestations that have unknown block root.
-   */
-  reached_limit = "reached_limit",
-  /**
-   * The awaiting attestation is pruned per clock slot.
-   */
-  expired = "expired",
-}
-
-type AwaitingAttestationPromise = {
-  resolve: (foundBlock: boolean) => void;
-  promise: Promise<boolean>;
-  // there are multiple subnet/aggregated attestations waiting for same promise
-  awaitingAttestationsCount: number;
-  // we only resolve to true or false to make the code simpler so no need to keep reject here
-  addedTimeMs: number;
-};
-
-// How many attestations (aggregate + unaggregate) we keep before new ones get dropped.
-const MAXIMUM_QUEUED_ATTESTATIONS = 16_384;
-
-type SlotRoot = {slot: Slot; root: RootHex};
-
-/**
- * Some attestations may reach our node before the voted block, so we manage a cache to reprocess them
- * when the block come.
- * (n)                                               (n + 1)
- *  |----------------|----------------|----------|------|
- *                   |                |          |
- *                  att           agg att        |
- *                                              block
- * Since the gossip handler has to return validation result to js-libp2p-gossipsub, this class should not
- * reprocess attestations, it should control when the attestations are ready to reprocess instead.
- */
-export class ReprocessController {
-  private readonly awaitingPromisesByRootBySlot: MapDef<Slot, Map<RootHex, AwaitingAttestationPromise>>;
-  private awaitingPromisesCount = 0;
-
-  constructor(private readonly metrics: Metrics | null) {
-    this.awaitingPromisesByRootBySlot = new MapDef(() => new Map<RootHex, AwaitingAttestationPromise>());
-  }
-
-  /**
-   * Returns Promise that resolves either on block found or once 1 slot passes.
-   * Used to handle unknown block root for both unaggregated and aggregated attestations.
-   * @returns true if blockFound
-   */
-  waitForBlockOfAttestation(slot: Slot, root: RootHex): Promise<boolean> {
-    this.metrics?.reprocessApiAttestations.total.inc();
-
-    if (this.awaitingPromisesCount >= MAXIMUM_QUEUED_ATTESTATIONS) {
-      this.metrics?.reprocessApiAttestations.reject.inc({reason: ReprocessStatus.reached_limit});
-      return Promise.resolve(false);
-    }
-
-    this.awaitingPromisesCount++;
-    const awaitingPromisesByRoot = this.awaitingPromisesByRootBySlot.getOrDefault(slot);
-    const promiseCached = awaitingPromisesByRoot.get(root);
-    if (promiseCached) {
-      promiseCached.awaitingAttestationsCount++;
-      return promiseCached.promise;
-    }
-
-    // Capture both the promise and its callbacks.
-    // It is not spec'ed but in tests in Firefox and NodeJS the promise constructor is run immediately
-    let resolve: AwaitingAttestationPromise["resolve"] | null = null;
-    const promise = new Promise<boolean>((resolveCB) => {
-      resolve = resolveCB;
-    });
-
-    if (resolve === null) {
-      throw Error("Promise Constructor was not executed immediately");
-    }
-
-    awaitingPromisesByRoot.set(root, {
-      promise,
-      awaitingAttestationsCount: 1,
-      resolve,
-      addedTimeMs: Date.now(),
-    });
-
-    return promise;
-  }
-
-  /**
-   * It's important to make sure our node is synced before we reprocess,
-   * it means the processed slot is same to clock slot
-   * Note that we want to use clock advanced by REPROCESS_MIN_TIME_TO_NEXT_SLOT instead of
-   * clockSlot because we want to make sure our node is healthy while reprocessing attestations.
-   * If a block reach our node 1s before the next slot, for example, then probably node
-   * is struggling and we don't want to reprocess anything at that time.
-   */
-  onBlockImported({slot: blockSlot, root}: SlotRoot, advancedSlot: Slot): void {
-    // we are probably resyncing, don't want to reprocess attestations here
-    if (blockSlot < advancedSlot) return;
-
-    // resolve all related promises
-    const awaitingPromisesBySlot = this.awaitingPromisesByRootBySlot.getOrDefault(blockSlot);
-    const awaitingPromise = awaitingPromisesBySlot.get(root);
-    if (awaitingPromise) {
-      const {resolve, addedTimeMs, awaitingAttestationsCount} = awaitingPromise;
-      resolve(true);
-      this.awaitingPromisesCount -= awaitingAttestationsCount;
-      this.metrics?.reprocessApiAttestations.resolve.inc(awaitingAttestationsCount);
-      this.metrics?.reprocessApiAttestations.waitSecBeforeResolve.set((Date.now() - addedTimeMs) / 1000);
-    }
-
-    // prune
-    awaitingPromisesBySlot.delete(root);
-  }
-
-  /**
-   * It's important to make sure our node is synced before reprocessing attestations,
-   * it means clockSlot is the same to last processed block's slot, and we don't reprocess
-   * attestations of old slots.
-   * So we reject and prune all old awaiting promises per clock slot.
-   * @param clockSlot
-   */
-  onSlot(clockSlot: Slot): void {
-    const now = Date.now();
-
-    for (const [key, awaitingPromisesByRoot] of this.awaitingPromisesByRootBySlot.entries()) {
-      if (key < clockSlot) {
-        // reject all related promises
-        for (const awaitingPromise of awaitingPromisesByRoot.values()) {
-          const {resolve, addedTimeMs} = awaitingPromise;
-          resolve(false);
-          this.metrics?.reprocessApiAttestations.waitSecBeforeReject.set(
-            {reason: ReprocessStatus.expired},
-            (now - addedTimeMs) / 1000
-          );
-          this.metrics?.reprocessApiAttestations.reject.inc({reason: ReprocessStatus.expired});
-        }
-
-        // prune
-        this.awaitingPromisesByRootBySlot.delete(key);
-      } else {
-        break;
-      }
-    }
-
-    // in theory there are maybe some awaiting promises waiting for a slot > clockSlot
-    // in reality this never happens so reseting awaitingPromisesCount to 0 to make it simple
-    this.awaitingPromisesCount = 0;
-  }
-}