@lodestar/beacon-node 1.35.0-dev.47470aab65 → 1.35.0-dev.47c570ab76

package/src/chain/shufflingCache.ts

@@ -1,238 +0,0 @@
-import {
-  BeaconStateAllForks,
-  EpochShuffling,
-  IShufflingCache,
-  ShufflingBuildProps,
-  computeEpochShuffling,
-  computeEpochShufflingAsync,
-} from "@lodestar/state-transition";
-import {Epoch, RootHex} from "@lodestar/types";
-import {LodestarError, Logger, MapDef, pruneSetToMax} from "@lodestar/utils";
-import {Metrics} from "../metrics/metrics.js";
-
-/**
- * Same value to CheckpointBalancesCache, with the assumption that we don't have to use it for old epochs. In the worse case:
- * - when loading state bytes from disk, we need to compute shuffling for all epochs (~1s as of Sep 2023)
- * - don't have shuffling to verify attestations, need to do 1 epoch transition to add shuffling to this cache. This never happens
- * with default chain option of maxSkipSlots = 32
- **/
-const MAX_EPOCHS = 4;
-
-/**
- * With default chain option of maxSkipSlots = 32, there should be no shuffling promise. If that happens a lot, it could blow up Lodestar,
- * with MAX_EPOCHS = 4, only allow 2 promise at a time. Note that regen already bounds number of concurrent requests at 1 already.
- */
-const MAX_PROMISES = 2;
-
-enum CacheItemType {
-  shuffling,
-  promise,
-}
-
-type ShufflingCacheItem = {
-  type: CacheItemType.shuffling;
-  shuffling: EpochShuffling;
-};
-
-type PromiseCacheItem = {
-  type: CacheItemType.promise;
-  timeInsertedMs: number;
-  promise: Promise<EpochShuffling>;
-  resolveFn: (shuffling: EpochShuffling) => void;
-};
-
-type CacheItem = ShufflingCacheItem | PromiseCacheItem;
-
-export type ShufflingCacheOpts = {
-  maxShufflingCacheEpochs?: number;
-};
-
-/**
- * A shuffling cache to help:
- * - get committee quickly for attestation verification
- * - if a shuffling is not available (which does not happen with default chain option of maxSkipSlots = 32), track a promise to make sure we don't compute the same shuffling twice
- * - skip computing shuffling when loading state bytes from disk
- */
-export class ShufflingCache implements IShufflingCache {
-  /** LRU cache implemented as a map, pruned every time we add an item */
-  private readonly itemsByDecisionRootByEpoch: MapDef<Epoch, Map<RootHex, CacheItem>> = new MapDef(
-    () => new Map<RootHex, CacheItem>()
-  );
-
-  private readonly maxEpochs: number;
-
-  constructor(
-    readonly metrics: Metrics | null = null,
-    readonly logger: Logger | null = null,
-    opts: ShufflingCacheOpts = {},
-    precalculatedShufflings?: {shuffling: EpochShuffling | null; decisionRoot: RootHex}[]
-  ) {
-    if (metrics) {
-      metrics.shufflingCache.size.addCollect(() =>
-        metrics.shufflingCache.size.set(
-          Array.from(this.itemsByDecisionRootByEpoch.values()).reduce((total, innerMap) => total + innerMap.size, 0)
-        )
-      );
-    }
-
-    this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS;
-
-    precalculatedShufflings?.map(({shuffling, decisionRoot}) => {
-      if (shuffling !== null) {
-        this.set(shuffling, decisionRoot);
-      }
-    });
-  }
-
-  /**
-   * Insert a promise to make sure we don't regen state for the same shuffling.
-   * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up.
-   */
-  insertPromise(epoch: Epoch, decisionRoot: RootHex): void {
-    const promiseCount = Array.from(this.itemsByDecisionRootByEpoch.values())
-      .flatMap((innerMap) => Array.from(innerMap.values()))
-      .filter((item) => isPromiseCacheItem(item)).length;
-    if (promiseCount >= MAX_PROMISES) {
-      throw new Error(
-        `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${epoch}, decisionRootHex: ${decisionRoot}`
-      );
-    }
-    let resolveFn: ((shuffling: EpochShuffling) => void) | null = null;
-    const promise = new Promise<EpochShuffling>((resolve) => {
-      resolveFn = resolve;
-    });
-    if (resolveFn === null) {
-      throw new Error("Promise Constructor was not executed immediately");
-    }
-
-    const cacheItem: PromiseCacheItem = {
-      type: CacheItemType.promise,
-      timeInsertedMs: Date.now(),
-      promise,
-      resolveFn,
-    };
-    this.itemsByDecisionRootByEpoch.getOrDefault(epoch).set(decisionRoot, cacheItem);
-    this.metrics?.shufflingCache.insertPromiseCount.inc();
-  }
-
-  /**
-   * Most of the time, this should return a shuffling immediately.
-   * If there's a promise, it means we are computing the same shuffling, so we wait for the promise to resolve.
-   * Return null if we don't have a shuffling for this epoch and dependentRootHex.
-   */
-  async get(epoch: Epoch, decisionRoot: RootHex): Promise<EpochShuffling | null> {
-    const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
-    if (cacheItem === undefined) {
-      this.metrics?.shufflingCache.miss.inc();
-      return null;
-    }
-
-    if (isShufflingCacheItem(cacheItem)) {
-      this.metrics?.shufflingCache.hit.inc();
-      return cacheItem.shuffling;
-    }
-    this.metrics?.shufflingCache.shufflingPromiseNotResolved.inc();
-    return cacheItem.promise;
-  }
-
-  /**
-   * Gets a cached shuffling via the epoch and decision root.  If the shuffling is not
-   * available it will build it synchronously and return the shuffling.
-   *
-   * NOTE: If a shuffling is already queued and not calculated it will build and resolve
-   * the promise but the already queued build will happen at some later time
-   */
-  getSync<T extends ShufflingBuildProps | undefined>(
-    epoch: Epoch,
-    decisionRoot: RootHex,
-    buildProps?: T
-  ): T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null {
-    const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
-    if (!cacheItem) {
-      this.metrics?.shufflingCache.miss.inc();
-    } else if (isShufflingCacheItem(cacheItem)) {
-      this.metrics?.shufflingCache.hit.inc();
-      return cacheItem.shuffling;
-    } else if (buildProps) {
-      // TODO: (@matthewkeil) This should possible log a warning??
-      this.metrics?.shufflingCache.shufflingPromiseNotResolvedAndThrownAway.inc();
-    } else {
-      this.metrics?.shufflingCache.shufflingPromiseNotResolved.inc();
-    }
-
-    let shuffling: EpochShuffling | null = null;
-    if (buildProps) {
-      const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer({source: "getSync"});
-      shuffling = computeEpochShuffling(buildProps.state, buildProps.activeIndices, epoch);
-      timer?.();
-      this.set(shuffling, decisionRoot);
-    }
-    return shuffling as T extends ShufflingBuildProps ? EpochShuffling : EpochShuffling | null;
-  }
-
-  /**
-   * Queue asynchronous build for an EpochShuffling, triggered from state-transition
-   */
-  build(epoch: number, decisionRoot: string, state: BeaconStateAllForks, activeIndices: Uint32Array): void {
-    this.insertPromise(epoch, decisionRoot);
-    /**
-     * TODO: (@matthewkeil) This will get replaced by a proper build queue and a worker to do calculations
-     * on a NICE thread
-     */
-    const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer({source: "build"});
-    computeEpochShufflingAsync(state, activeIndices, epoch)
-      .then((shuffling) => {
-        this.set(shuffling, decisionRoot);
-      })
-      .catch((err) =>
-        this.logger?.error(`error building shuffling for epoch ${epoch} at decisionRoot ${decisionRoot}`, {}, err)
-      )
-      .finally(() => {
-        timer?.();
-      });
-  }
-
-  /**
-   * Add an EpochShuffling to the ShufflingCache. If a promise for the shuffling is present it will
-   * resolve the promise with the built shuffling
-   */
-  private set(shuffling: EpochShuffling, decisionRoot: string): void {
-    const shufflingAtEpoch = this.itemsByDecisionRootByEpoch.getOrDefault(shuffling.epoch);
-    // if a pending shuffling promise exists, resolve it
-    const cacheItem = shufflingAtEpoch.get(decisionRoot);
-    if (cacheItem) {
-      if (isPromiseCacheItem(cacheItem)) {
-        cacheItem.resolveFn(shuffling);
-        this.metrics?.shufflingCache.shufflingPromiseResolutionTime.observe(
-          (Date.now() - cacheItem.timeInsertedMs) / 1000
-        );
-      } else {
-        this.metrics?.shufflingCache.shufflingBuiltMultipleTimes.inc();
-      }
-    }
-    // set the shuffling
-    shufflingAtEpoch.set(decisionRoot, {type: CacheItemType.shuffling, shuffling});
-    // prune the cache
-    pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs);
-  }
-}
-
-function isShufflingCacheItem(item: CacheItem): item is ShufflingCacheItem {
-  return item.type === CacheItemType.shuffling;
-}
-
-function isPromiseCacheItem(item: CacheItem): item is PromiseCacheItem {
-  return item.type === CacheItemType.promise;
-}
-
-export enum ShufflingCacheErrorCode {
-  NO_SHUFFLING_FOUND = "SHUFFLING_CACHE_ERROR_NO_SHUFFLING_FOUND",
-}
-
-type ShufflingCacheErrorType = {
-  code: ShufflingCacheErrorCode.NO_SHUFFLING_FOUND;
-  epoch: Epoch;
-  decisionRoot: RootHex;
-};
-
-export class ShufflingCacheError extends LodestarError<ShufflingCacheErrorType> {}

package/src/chain/stateCache/blockStateCacheImpl.ts

@@ -1,150 +0,0 @@
-import {routes} from "@lodestar/api";
-import {CachedBeaconStateAllForks} from "@lodestar/state-transition";
-import {Epoch, RootHex} from "@lodestar/types";
-import {toRootHex} from "@lodestar/utils";
-import {Metrics} from "../../metrics/index.js";
-import {StateRegenerationOpts} from "../regen/interface.js";
-import {MapTracker} from "./mapMetrics.js";
-import {BlockStateCache} from "./types.js";
-
-const MAX_STATES = 3 * 32;
-
-/**
- * Old implementation of StateCache (used to call `StateContextCache`)
- * - Prune per checkpoint so number of states ranges from 96 to 128
- * - Keep a separate head state to make sure it is always available
- */
-export class BlockStateCacheImpl implements BlockStateCache {
-  /**
-   * Max number of states allowed in the cache
-   */
-  readonly maxStates: number;
-
-  private readonly cache: MapTracker<string, CachedBeaconStateAllForks>;
-  /** Epoch -> Set<blockRoot> */
-  private readonly epochIndex = new Map<Epoch, Set<string>>();
-  private readonly metrics: Metrics["stateCache"] | null | undefined;
-  /**
-   * Strong reference to prevent head state from being pruned.
-   * null if head state is being regen and not available at the moment.
-   */
-  private head: {state: CachedBeaconStateAllForks; stateRoot: RootHex} | null = null;
-
-  constructor({maxStates = MAX_STATES, metrics}: {maxStates?: number; metrics?: Metrics | null}) {
-    this.maxStates = maxStates;
-    this.cache = new MapTracker(metrics?.stateCache);
-    if (metrics) {
-      this.metrics = metrics.stateCache;
-      metrics.stateCache.size.addCollect(() => metrics.stateCache.size.set(this.cache.size));
-    }
-  }
-
-  get(rootHex: RootHex, opts?: StateRegenerationOpts): CachedBeaconStateAllForks | null {
-    this.metrics?.lookups.inc();
-    const item = this.head?.stateRoot === rootHex ? this.head.state : this.cache.get(rootHex);
-    if (!item) {
-      return null;
-    }
-
-    this.metrics?.hits.inc();
-    this.metrics?.stateClonedCount.observe(item.clonedCount);
-
-    return item.clone(opts?.dontTransferCache);
-  }
-
-  add(item: CachedBeaconStateAllForks): void {
-    const key = toRootHex(item.hashTreeRoot());
-    if (this.cache.get(key)) {
-      return;
-    }
-    this.metrics?.adds.inc();
-    this.cache.set(key, item);
-    const epoch = item.epochCtx.epoch;
-    const blockRoots = this.epochIndex.get(epoch);
-    if (blockRoots) {
-      blockRoots.add(key);
-    } else {
-      this.epochIndex.set(epoch, new Set([key]));
-    }
-  }
-
-  setHeadState(item: CachedBeaconStateAllForks | null): void {
-    if (item) {
-      const key = toRootHex(item.hashTreeRoot());
-      this.head = {state: item, stateRoot: key};
-    } else {
-      this.head = null;
-    }
-  }
-
-  /**
-   * Get a seed state for state reload.
-   * This is to conform to the api only as this cache is not used in n-historical state.
-   * See ./fifoBlockStateCache.ts for implementation
-   */
-  getSeedState(): CachedBeaconStateAllForks {
-    throw Error("Not implemented for BlockStateCacheImpl");
-  }
-
-  clear(): void {
-    this.cache.clear();
-    this.epochIndex.clear();
-  }
-
-  get size(): number {
-    return this.cache.size;
-  }
-
-  /**
-   * TODO make this more robust.
-   * Without more thought, this currently breaks our assumptions about recent state availablity
-   */
-  prune(headStateRootHex: RootHex): void {
-    const keys = Array.from(this.cache.keys());
-    if (keys.length > this.maxStates) {
-      // object keys are stored in insertion order, delete keys starting from the front
-      for (const key of keys.slice(0, keys.length - this.maxStates)) {
-        if (key !== headStateRootHex) {
-          const item = this.cache.get(key);
-          if (item) {
-            this.epochIndex.get(item.epochCtx.epoch)?.delete(key);
-            this.cache.delete(key);
-          }
-        }
-      }
-    }
-  }
-
-  /**
-   * Prune per finalized epoch.
-   */
-  deleteAllBeforeEpoch(finalizedEpoch: Epoch): void {
-    for (const epoch of this.epochIndex.keys()) {
-      if (epoch < finalizedEpoch) {
-        this.deleteAllEpochItems(epoch);
-      }
-    }
-  }
-
-  /** ONLY FOR DEBUGGING PURPOSES. For lodestar debug API */
-  dumpSummary(): routes.lodestar.StateCacheItem[] {
-    return Array.from(this.cache.entries()).map(([key, state]) => ({
-      slot: state.slot,
-      root: toRootHex(state.hashTreeRoot()),
-      reads: this.cache.readCount.get(key) ?? 0,
-      lastRead: this.cache.lastRead.get(key) ?? 0,
-      checkpointState: false,
-    }));
-  }
-
-  getStates(): IterableIterator<CachedBeaconStateAllForks> {
-    return this.cache.values();
-  }
-
-  private deleteAllEpochItems(epoch: Epoch): void {
-    for (const rootHex of this.epochIndex.get(epoch) || []) {
-      this.cache.delete(rootHex);
-    }
-    this.epochIndex.delete(epoch);
-  }
-}

package/src/chain/stateCache/datastore/db.ts

@@ -1,36 +0,0 @@
-import {phase0, ssz} from "@lodestar/types";
-import {IBeaconDb} from "../../../db/interface.js";
-import {CPStateDatastore, DatastoreKey} from "./types.js";
-
-/**
- * Implementation of CPStateDatastore using db.
- */
-export class DbCPStateDatastore implements CPStateDatastore {
-  constructor(private readonly db: IBeaconDb) {}
-
-  async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise<DatastoreKey> {
-    const serializedCheckpoint = checkpointToDatastoreKey(cpKey);
-    await this.db.checkpointState.putBinary(serializedCheckpoint, stateBytes);
-    return serializedCheckpoint;
-  }
-
-  async remove(serializedCheckpoint: DatastoreKey): Promise<void> {
-    await this.db.checkpointState.delete(serializedCheckpoint);
-  }
-
-  async read(serializedCheckpoint: DatastoreKey): Promise<Uint8Array | null> {
-    return this.db.checkpointState.getBinary(serializedCheckpoint);
-  }
-
-  async readKeys(): Promise<DatastoreKey[]> {
-    return this.db.checkpointState.keys();
-  }
-}
-
-export function datastoreKeyToCheckpoint(key: DatastoreKey): phase0.Checkpoint {
-  return ssz.phase0.Checkpoint.deserialize(key);
-}
-
-export function checkpointToDatastoreKey(cp: phase0.Checkpoint): DatastoreKey {
-  return ssz.phase0.Checkpoint.serialize(cp);
-}

package/src/chain/stateCache/datastore/file.ts

@@ -1,53 +0,0 @@
-import path from "node:path";
-import {phase0, ssz} from "@lodestar/types";
-import {fromHex, toHex} from "@lodestar/utils";
-import {ensureDir, readFile, readFileNames, removeFile, writeIfNotExist} from "../../../util/file.js";
-import {CPStateDatastore, DatastoreKey} from "./types.js";
-
-const CHECKPOINT_STATES_FOLDER = "checkpoint_states";
-const CHECKPOINT_FILE_NAME_LENGTH = 82;
-
-/**
- * Implementation of CPStateDatastore using file system, this is beneficial for debugging.
- */
-export class FileCPStateDatastore implements CPStateDatastore {
-  private readonly folderPath: string;
-
-  constructor(dataDir: string) {
-    // service deployment: `/beacon/checkpoint_states`
-    // docker deployment: `/data/checkpoint_states`
-    this.folderPath = path.join(dataDir, CHECKPOINT_STATES_FOLDER);
-  }
-
-  async init(): Promise<void> {
-    try {
-      await ensureDir(this.folderPath);
-    } catch (_) {
-      // do nothing
-    }
-  }
-
-  async write(cpKey: phase0.Checkpoint, stateBytes: Uint8Array): Promise<DatastoreKey> {
-    const serializedCheckpoint = ssz.phase0.Checkpoint.serialize(cpKey);
-    const filePath = path.join(this.folderPath, toHex(serializedCheckpoint));
-    await writeIfNotExist(filePath, stateBytes);
-    return serializedCheckpoint;
-  }
-
-  async remove(serializedCheckpoint: DatastoreKey): Promise<void> {
-    const filePath = path.join(this.folderPath, toHex(serializedCheckpoint));
-    await removeFile(filePath);
-  }
-
-  async read(serializedCheckpoint: DatastoreKey): Promise<Uint8Array | null> {
-    const filePath = path.join(this.folderPath, toHex(serializedCheckpoint));
-    return readFile(filePath);
-  }
-
-  async readKeys(): Promise<DatastoreKey[]> {
-    const fileNames = await readFileNames(this.folderPath);
-    return fileNames
-      .filter((fileName) => fileName.startsWith("0x") && fileName.length === CHECKPOINT_FILE_NAME_LENGTH)
-      .map((fileName) => fromHex(fileName));
-  }
-}

package/src/chain/stateCache/datastore/index.ts

@@ -1,2 +0,0 @@
-export * from "./db.js";
-export * from "./types.js";

package/src/chain/stateCache/datastore/types.ts

@@ -1,13 +0,0 @@
-import {phase0} from "@lodestar/types";
-
-// With db implementation, persistedKey is serialized data of a checkpoint
-export type DatastoreKey = Uint8Array;
-
-// Make this generic to support testing
-export interface CPStateDatastore {
-  write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array) => Promise<DatastoreKey>;
-  remove: (key: DatastoreKey) => Promise<void>;
-  read: (key: DatastoreKey) => Promise<Uint8Array | null>;
-  readKeys: () => Promise<DatastoreKey[]>;
-  init?: () => Promise<void>;
-}