@lodestar/beacon-node 1.35.0-dev.fcf8d024ea → 1.35.0-dev.feed916580

package/src/chain/produceBlock/validateBlobsAndKzgCommitments.ts

@@ -1,54 +0,0 @@
-import {CELLS_PER_EXT_BLOB} from "@lodestar/params";
-import {deneb, fulu} from "@lodestar/types";
-import {kzg} from "../../util/kzg.js";
-
-/**
- * Optionally sanity-check that the KZG commitments match the versioned hashes in the transactions
- * https://github.com/ethereum/consensus-specs/blob/11a037fd9227e29ee809c9397b09f8cc3383a8c0/specs/eip4844/validator.md#blob-kzg-commitments
- */
-export async function validateBlobsAndKzgCommitments(
-  commitments: deneb.KZGCommitment[],
-  proofs: deneb.KZGProof[],
-  blobs: deneb.Blobs
-): Promise<void> {
-  if (blobs.length !== commitments.length) {
-    throw Error(`Blobs bundle blobs len ${blobs.length} != commitments len ${commitments.length}`);
-  }
-
-  if (proofs.length !== blobs.length) {
-    throw new Error(`Invalid proofs length for BlobsBundleV1 format: expected ${blobs.length}, got ${proofs.length}`);
-  }
-
-  if (!(await kzg.asyncVerifyBlobKzgProofBatch(blobs, commitments, proofs))) {
-    throw new Error("Error in verifyBlobKzgProofBatch");
-  }
-}
-
-/**
- * Optionally sanity-check that the KZG commitments match the versioned hashes in the transactions
- */
-export async function validateCellsAndKzgCommitments(
-  commitments: deneb.KZGCommitment[],
-  proofs: fulu.KZGProof[],
-  cells: fulu.Cell[][]
-): Promise<void> {
-  if (cells.length !== commitments.length) {
-    throw Error(`Blobs bundle cells len ${cells.length} != commitments len ${commitments.length}`);
-  }
-
-  const expectedProofsLength = cells.length * CELLS_PER_EXT_BLOB;
-  if (proofs.length !== expectedProofsLength) {
-    throw Error(
-      `Invalid proofs length for BlobsBundleV2 format: expected ${expectedProofsLength}, got ${proofs.length}`
-    );
-  }
-
-  const commitmentBytes = commitments.flatMap((commitment) => Array(CELLS_PER_EXT_BLOB).fill(commitment));
-  const cellIndices = Array.from({length: cells.length}).flatMap(() =>
-    Array.from({length: CELLS_PER_EXT_BLOB}, (_, i) => i)
-  );
-
-  if (!(await kzg.asyncVerifyCellKzgProofBatch(commitmentBytes, cellIndices, cells.flat(), proofs.flat()))) {
-    throw new Error("Error in verifyCellKzgProofBatch");
-  }
-}

package/src/chain/regen/errors.ts

@@ -1,30 +0,0 @@
-import {Root, RootHex, Slot} from "@lodestar/types";
-
-export enum RegenErrorCode {
-  BLOCK_NOT_IN_FORKCHOICE = "REGEN_ERROR_BLOCK_NOT_IN_FORKCHOICE",
-  STATE_NOT_IN_FORKCHOICE = "REGEN_ERROR_STATE_NOT_IN_FORKCHOICE",
-  SLOT_BEFORE_BLOCK_SLOT = "REGEN_ERROR_SLOT_BEFORE_BLOCK_SLOT",
-  NO_SEED_STATE = "REGEN_ERROR_NO_SEED_STATE",
-  TOO_MANY_BLOCK_PROCESSED = "REGEN_ERROR_TOO_MANY_BLOCK_PROCESSED",
-  BLOCK_NOT_IN_DB = "REGEN_ERROR_BLOCK_NOT_IN_DB",
-  STATE_TRANSITION_ERROR = "REGEN_ERROR_STATE_TRANSITION_ERROR",
-  INVALID_STATE_ROOT = "REGEN_ERROR_INVALID_STATE_ROOT",
-}
-
-export type RegenErrorType =
-  | {code: RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE; blockRoot: RootHex | Root}
-  | {code: RegenErrorCode.STATE_NOT_IN_FORKCHOICE; stateRoot: RootHex | Root}
-  | {code: RegenErrorCode.SLOT_BEFORE_BLOCK_SLOT; slot: Slot; blockSlot: Slot}
-  | {code: RegenErrorCode.NO_SEED_STATE}
-  | {code: RegenErrorCode.TOO_MANY_BLOCK_PROCESSED; stateRoot: RootHex | Root}
-  | {code: RegenErrorCode.BLOCK_NOT_IN_DB; blockRoot: RootHex | Root}
-  | {code: RegenErrorCode.STATE_TRANSITION_ERROR; error: Error}
-  | {code: RegenErrorCode.INVALID_STATE_ROOT; slot: Slot; expected: RootHex; actual: RootHex};
-
-export class RegenError extends Error {
-  type: RegenErrorType;
-  constructor(type: RegenErrorType) {
-    super(type.code);
-    this.type = type;
-  }
-}

package/src/chain/regen/index.ts

@@ -1,4 +0,0 @@
-export * from "./errors.js";
-export * from "./interface.js";
-export * from "./queued.js";
-export * from "./regen.js";

package/src/chain/regen/interface.ts

@@ -1,93 +0,0 @@
-import {routes} from "@lodestar/api";
-import {ProtoBlock} from "@lodestar/fork-choice";
-import {CachedBeaconStateAllForks} from "@lodestar/state-transition";
-import {BeaconBlock, Epoch, RootHex, Slot, phase0} from "@lodestar/types";
-import {CheckpointHex} from "../stateCache/index.js";
-
-export enum RegenCaller {
-  getDuties = "getDuties",
-  processBlock = "processBlock",
-  produceBlock = "produceBlock",
-  validateGossipBlock = "validateGossipBlock",
-  validateGossipBlob = "validateGossipBlob",
-  precomputeEpoch = "precomputeEpoch",
-  predictProposerHead = "predictProposerHead",
-  produceAttestationData = "produceAttestationData",
-  processBlocksInEpoch = "processBlocksInEpoch",
-  validateGossipAggregateAndProof = "validateGossipAggregateAndProof",
-  validateGossipAttestation = "validateGossipAttestation",
-  validateGossipVoluntaryExit = "validateGossipVoluntaryExit",
-  onForkChoiceFinalized = "onForkChoiceFinalized",
-  restApi = "restApi",
-}
-
-export enum RegenFnName {
-  getBlockSlotState = "getBlockSlotState",
-  getState = "getState",
-  getPreState = "getPreState",
-  getCheckpointState = "getCheckpointState",
-}
-
-export type StateRegenerationOpts = {
-  dontTransferCache: boolean;
-  /**
-   * Do not queue shuffling calculation async. Forces sync JIT calculation in afterProcessEpoch if not passed as `true`
-   */
-  asyncShufflingCalculation?: boolean;
-};
-
-export interface IStateRegenerator extends IStateRegeneratorInternal {
-  dropCache(): void;
-  dumpCacheSummary(): routes.lodestar.StateCacheItem[];
-  getStateSync(stateRoot: RootHex): CachedBeaconStateAllForks | null;
-  getPreStateSync(block: BeaconBlock): CachedBeaconStateAllForks | null;
-  getCheckpointStateOrBytes(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | Uint8Array | null>;
-  getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null;
-  getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null;
-  pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void;
-  pruneOnFinalized(finalizedEpoch: Epoch): void;
-  processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void;
-  addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void;
-  updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void;
-  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null;
-}
-
-/**
- * Regenerates states that have already been processed by the fork choice
- */
-export interface IStateRegeneratorInternal {
-  /**
-   * Return a valid pre-state for a beacon block
-   * This will always return a state in the latest viable epoch
-   */
-  getPreState(
-    block: BeaconBlock,
-    opts: StateRegenerationOpts,
-    rCaller: RegenCaller
-  ): Promise<CachedBeaconStateAllForks>;
-
-  /**
-   * Return a valid checkpoint state
-   * This will always return a state with `state.slot % SLOTS_PER_EPOCH === 0`
-   */
-  getCheckpointState(
-    cp: phase0.Checkpoint,
-    opts: StateRegenerationOpts,
-    rCaller: RegenCaller
-  ): Promise<CachedBeaconStateAllForks>;
-
-  /**
-   * Return the state of `blockRoot` processed to slot `slot`
-   */
-  getBlockSlotState(
-    blockRoot: RootHex,
-    slot: Slot,
-    opts: StateRegenerationOpts,
-    rCaller: RegenCaller
-  ): Promise<CachedBeaconStateAllForks>;
-
-  /**
-   * Return the exact state with `stateRoot`
-   */
-  getState(stateRoot: RootHex, rCaller: RegenCaller, opts?: StateRegenerationOpts): Promise<CachedBeaconStateAllForks>;
-}

package/src/chain/regen/queued.ts

@@ -1,317 +0,0 @@
-import {routes} from "@lodestar/api";
-import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
-import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition";
-import {BeaconBlock, Epoch, RootHex, Slot, phase0} from "@lodestar/types";
-import {Logger, toRootHex} from "@lodestar/utils";
-import {Metrics} from "../../metrics/index.js";
-import {JobItemQueue} from "../../util/queue/index.js";
-import {CheckpointHex, toCheckpointHex} from "../stateCache/index.js";
-import {BlockStateCache, CheckpointStateCache} from "../stateCache/types.js";
-import {RegenError, RegenErrorCode} from "./errors.js";
-import {
-  IStateRegenerator,
-  IStateRegeneratorInternal,
-  RegenCaller,
-  RegenFnName,
-  StateRegenerationOpts,
-} from "./interface.js";
-import {RegenModules, StateRegenerator} from "./regen.js";
-
-const REGEN_QUEUE_MAX_LEN = 256;
-// TODO: Should this constant be lower than above? 256 feels high
-const REGEN_CAN_ACCEPT_WORK_THRESHOLD = 16;
-
-type QueuedStateRegeneratorModules = RegenModules & {
-  signal: AbortSignal;
-};
-
-type RegenRequestKey = keyof IStateRegeneratorInternal;
-type RegenRequestByKey = {[K in RegenRequestKey]: {key: K; args: Parameters<IStateRegeneratorInternal[K]>}};
-export type RegenRequest = RegenRequestByKey[RegenRequestKey];
-
-/**
- * Regenerates states that have already been processed by the fork choice
- *
- * All requests are queued so that only a single state at a time may be regenerated at a time
- */
-export class QueuedStateRegenerator implements IStateRegenerator {
-  readonly jobQueue: JobItemQueue<[RegenRequest], CachedBeaconStateAllForks>;
-  private readonly regen: StateRegenerator;
-
-  private readonly forkChoice: IForkChoice;
-  private readonly blockStateCache: BlockStateCache;
-  private readonly checkpointStateCache: CheckpointStateCache;
-  private readonly metrics: Metrics | null;
-  private readonly logger: Logger;
-
-  constructor(modules: QueuedStateRegeneratorModules) {
-    this.regen = new StateRegenerator(modules);
-    this.jobQueue = new JobItemQueue<[RegenRequest], CachedBeaconStateAllForks>(
-      this.jobQueueProcessor,
-      {maxLength: REGEN_QUEUE_MAX_LEN, signal: modules.signal},
-      modules.metrics ? modules.metrics.regenQueue : undefined
-    );
-    this.forkChoice = modules.forkChoice;
-    this.blockStateCache = modules.blockStateCache;
-    this.checkpointStateCache = modules.checkpointStateCache;
-    this.metrics = modules.metrics;
-    this.logger = modules.logger;
-  }
-
-  async init(): Promise<void> {
-    if (this.checkpointStateCache.init) {
-      return this.checkpointStateCache.init();
-    }
-  }
-
-  canAcceptWork(): boolean {
-    return this.jobQueue.jobLen < REGEN_CAN_ACCEPT_WORK_THRESHOLD;
-  }
-
-  dropCache(): void {
-    this.blockStateCache.clear();
-    this.checkpointStateCache.clear();
-  }
-
-  dumpCacheSummary(): routes.lodestar.StateCacheItem[] {
-    return [...this.blockStateCache.dumpSummary(), ...this.checkpointStateCache.dumpSummary()];
-  }
-
-  /**
-   * Get a state from block state cache.
-   * This is not for block processing so don't transfer cache
-   */
-  getStateSync(stateRoot: RootHex): CachedBeaconStateAllForks | null {
-    return this.blockStateCache.get(stateRoot, {dontTransferCache: true});
-  }
-
-  /**
-   * Get state for block processing.
-   * By default, do not transfer cache except for the block at clock slot
-   * which is usually the gossip block.
-   */
-  getPreStateSync(
-    block: BeaconBlock,
-    opts: StateRegenerationOpts = {dontTransferCache: true}
-  ): CachedBeaconStateAllForks | null {
-    const parentRoot = toRootHex(block.parentRoot);
-    const parentBlock = this.forkChoice.getBlockHex(parentRoot);
-    if (!parentBlock) {
-      throw new RegenError({
-        code: RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE,
-        blockRoot: block.parentRoot,
-      });
-    }
-
-    const parentEpoch = computeEpochAtSlot(parentBlock.slot);
-    const blockEpoch = computeEpochAtSlot(block.slot);
-
-    // Check the checkpoint cache (if the pre-state is a checkpoint state)
-    if (parentEpoch < blockEpoch) {
-      const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch, opts);
-      if (checkpointState && computeEpochAtSlot(checkpointState.slot) === blockEpoch) {
-        return checkpointState;
-      }
-    }
-
-    // Check the state cache, only if the state doesn't need to go through an epoch transition.
-    // Otherwise the state transition may not be cached and wasted. Queue for regen since the
-    // work required will still be significant.
-    if (parentEpoch === blockEpoch) {
-      const state = this.blockStateCache.get(parentBlock.stateRoot, opts);
-      if (state) {
-        return state;
-      }
-    }
-
-    return null;
-  }
-
-  async getCheckpointStateOrBytes(cp: CheckpointHex): Promise<CachedBeaconStateAllForks | Uint8Array | null> {
-    return this.checkpointStateCache.getStateOrBytes(cp);
-  }
-
-  /**
-   * Get checkpoint state from cache, this function is not for block processing so don't transfer cache
-   */
-  getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null {
-    return this.checkpointStateCache.get(cp, {dontTransferCache: true});
-  }
-
-  /**
-   * Get state closest to head, this function is not for block processing so don't transfer cache
-   */
-  getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null {
-    const opts = {dontTransferCache: true};
-    return (
-      this.checkpointStateCache.getLatest(head.blockRoot, Infinity, opts) ||
-      this.blockStateCache.get(head.stateRoot, opts)
-    );
-  }
-
-  pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void {
-    this.checkpointStateCache.prune(finalizedEpoch, justifiedEpoch);
-    this.blockStateCache.prune(headStateRoot);
-  }
-
-  pruneOnFinalized(finalizedEpoch: number): void {
-    this.checkpointStateCache.pruneFinalized(finalizedEpoch);
-    this.blockStateCache.deleteAllBeforeEpoch(finalizedEpoch);
-  }
-
-  processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void {
-    this.blockStateCache.add(postState);
-    this.checkpointStateCache.processState(blockRootHex, postState).catch((e) => {
-      this.logger.debug("Error processing block state", {blockRootHex, slot: postState.slot}, e);
-    });
-  }
-
-  addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void {
-    this.checkpointStateCache.add(cp, item);
-  }
-
-  updateHeadState(newHead: ProtoBlock, maybeHeadState: CachedBeaconStateAllForks): void {
-    const {stateRoot: newHeadStateRoot, blockRoot: newHeadBlockRoot, slot: newHeadSlot} = newHead;
-    const maybeHeadStateRoot = toRootHex(maybeHeadState.hashTreeRoot());
-    const logCtx = {
-      newHeadSlot,
-      newHeadBlockRoot,
-      newHeadStateRoot,
-      maybeHeadSlot: maybeHeadState.slot,
-      maybeHeadStateRoot,
-    };
-    const headState =
-      newHeadStateRoot === maybeHeadStateRoot
-        ? maybeHeadState
-        : // maybeHeadState was already in block state cache so we don't transfer the cache
-          this.blockStateCache.get(newHeadStateRoot, {dontTransferCache: true});
-
-    if (headState) {
-      this.blockStateCache.setHeadState(headState);
-    } else {
-      // Trigger regen on head change if necessary
-      this.logger.warn("Head state not available, triggering regen", logCtx);
-      // for the old BlockStateCacheImpl only
-      //     - head has changed, so the existing cached head state is no longer useful. Set strong reference to null to free
-      //       up memory for regen step below. During regen, node won't be functional but eventually head will be available
-      // for the new FIFOBlockStateCache, this has no affect
-      this.blockStateCache.setHeadState(null);
-
-      // for the new FIFOBlockStateCache, it's important to reload state to regen head state here if needed
-      const allowDiskReload = true;
-      // transfer cache here because we want to regen state asap
-      const cloneOpts = {dontTransferCache: false};
-      this.regen.getState(newHeadStateRoot, RegenCaller.processBlock, cloneOpts, allowDiskReload).then(
-        (headStateRegen) => this.blockStateCache.setHeadState(headStateRegen),
-        (e) => this.logger.error("Error on head state regen", logCtx, e)
-      );
-    }
-  }
-
-  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null {
-    return this.checkpointStateCache.updatePreComputedCheckpoint(rootHex, epoch);
-  }
-
-  /**
-   * Get the state to run with `block`.
-   * - State after `block.parentRoot` dialed forward to block.slot
-   */
-  async getPreState(
-    block: BeaconBlock,
-    opts: StateRegenerationOpts,
-    rCaller: RegenCaller
-  ): Promise<CachedBeaconStateAllForks> {
-    this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getPreState});
-
-    // First attempt to fetch the state from caches before queueing
-    const cachedState = this.getPreStateSync(block, opts);
-
-    if (cachedState !== null) {
-      return cachedState;
-    }
-
-    // The state is not immediately available in the caches, enqueue the job
-    this.metrics?.regenFnQueuedTotal.inc({caller: rCaller, entrypoint: RegenFnName.getPreState});
-    return this.jobQueue.push({key: "getPreState", args: [block, opts, rCaller]});
-  }
-
-  async getCheckpointState(
-    cp: phase0.Checkpoint,
-    opts: StateRegenerationOpts,
-    rCaller: RegenCaller
-  ): Promise<CachedBeaconStateAllForks> {
-    this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getCheckpointState});
-
-    // First attempt to fetch the state from cache before queueing
-    const checkpointState = this.checkpointStateCache.get(toCheckpointHex(cp), opts);
-    if (checkpointState) {
-      return checkpointState;
-    }
-
-    // The state is not immediately available in the caches, enqueue the job
-    this.metrics?.regenFnQueuedTotal.inc({caller: rCaller, entrypoint: RegenFnName.getCheckpointState});
-    return this.jobQueue.push({key: "getCheckpointState", args: [cp, opts, rCaller]});
-  }
-
-  /**
-   * Get state of provided `blockRoot` and dial forward to `slot`
-   * Use this api with care because we don't want the queue to be busy
-   * For the context, gossip block validation uses this api so we want it to be as fast as possible
-   * @returns
-   */
-  async getBlockSlotState(
-    blockRoot: RootHex,
-    slot: Slot,
-    opts: StateRegenerationOpts,
-    rCaller: RegenCaller
-  ): Promise<CachedBeaconStateAllForks> {
-    this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getBlockSlotState});
-
-    // The state is not immediately available in the caches, enqueue the job
-    return this.jobQueue.push({key: "getBlockSlotState", args: [blockRoot, slot, opts, rCaller]});
-  }
-
-  async getState(
-    stateRoot: RootHex,
-    rCaller: RegenCaller,
-    opts: StateRegenerationOpts = {dontTransferCache: true}
-  ): Promise<CachedBeaconStateAllForks> {
-    this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getState});
-
-    // First attempt to fetch the state from cache before queueing
-    const state = this.blockStateCache.get(stateRoot, opts);
-    if (state) {
-      return state;
-    }
-
-    // The state is not immediately available in the cache, enqueue the job
-    this.metrics?.regenFnQueuedTotal.inc({caller: rCaller, entrypoint: RegenFnName.getState});
-    return this.jobQueue.push({key: "getState", args: [stateRoot, rCaller, opts]});
-  }
-
-  private jobQueueProcessor = async (regenRequest: RegenRequest): Promise<CachedBeaconStateAllForks> => {
-    const metricsLabels = {
-      caller: regenRequest.args.at(-1) as RegenCaller,
-      entrypoint: regenRequest.key as RegenFnName,
-    };
-    let timer: (() => number) | undefined;
-    try {
-      timer = this.metrics?.regenFnCallDuration.startTimer(metricsLabels);
-      switch (regenRequest.key) {
-        case "getPreState":
-          return await this.regen.getPreState(...regenRequest.args);
-        case "getCheckpointState":
-          return await this.regen.getCheckpointState(...regenRequest.args);
-        case "getBlockSlotState":
-          return await this.regen.getBlockSlotState(...regenRequest.args);
-        case "getState":
-          return await this.regen.getState(...regenRequest.args);
-      }
-    } catch (e) {
-      this.metrics?.regenFnTotalErrors.inc(metricsLabels);
-      throw e;
-    } finally {
-      if (timer) timer();
-    }
-  };
-}