@lodestar/beacon-node 1.35.0-dev.98d359db41 → 1.35.0-dev.b42a298a7c

package/src/chain/blocks/verifyBlocksStateTransitionOnly.ts

@@ -1,114 +0,0 @@
-import {
-  CachedBeaconStateAllForks,
-  DataAvailabilityStatus,
-  ExecutionPayloadStatus,
-  StateHashTreeRootSource,
-  stateTransition,
-} from "@lodestar/state-transition";
-import {ErrorAborted, Logger} from "@lodestar/utils";
-import {Metrics} from "../../metrics/index.js";
-import {byteArrayEquals} from "../../util/bytes.js";
-import {nextEventLoop} from "../../util/eventLoop.js";
-import {BlockError, BlockErrorCode} from "../errors/index.js";
-import {BlockProcessOpts} from "../options.js";
-import {ValidatorMonitor} from "../validatorMonitor.js";
-import {IBlockInput} from "./blockInput/index.js";
-import {ImportBlockOpts} from "./types.js";
-
-/**
- * Verifies 1 or more blocks are fully valid running the full state transition; from a linear sequence of blocks.
- *
- * - Advance state to block's slot - per_slot_processing()
- * - For each block:
- *   - STFN - per_block_processing()
- *   - Check state root matches
- */
-export async function verifyBlocksStateTransitionOnly(
-  preState0: CachedBeaconStateAllForks,
-  blocks: IBlockInput[],
-  dataAvailabilityStatuses: DataAvailabilityStatus[],
-  logger: Logger,
-  metrics: Metrics | null,
-  validatorMonitor: ValidatorMonitor | null,
-  signal: AbortSignal,
-  opts: BlockProcessOpts & ImportBlockOpts
-): Promise<{postStates: CachedBeaconStateAllForks[]; proposerBalanceDeltas: number[]; verifyStateTime: number}> {
-  const postStates: CachedBeaconStateAllForks[] = [];
-  const proposerBalanceDeltas: number[] = [];
-  const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
-
-  for (let i = 0; i < blocks.length; i++) {
-    const {validProposerSignature, validSignatures} = opts;
-    const block = blocks[i].getBlock();
-    const preState = i === 0 ? preState0 : postStates[i - 1];
-    const dataAvailabilityStatus = dataAvailabilityStatuses[i];
-
-    // STFN - per_slot_processing() + per_block_processing()
-    // NOTE: `regen.getPreState()` should have dialed forward the state already caching checkpoint states
-    const useBlsBatchVerify = !opts?.disableBlsBatchVerify;
-    const postState = stateTransition(
-      preState,
-      block,
-      {
-        // NOTE: Assume valid for now while sending payload to execution engine in parallel
-        // Latter verifyBlocksInEpoch() will make sure that payload is indeed valid
-        executionPayloadStatus: ExecutionPayloadStatus.valid,
-        dataAvailabilityStatus,
-        // false because it's verified below with better error typing
-        verifyStateRoot: false,
-        // if block is trusted don't verify proposer or op signature
-        verifyProposer: !useBlsBatchVerify && !validSignatures && !validProposerSignature,
-        verifySignatures: !useBlsBatchVerify && !validSignatures,
-      },
-      {metrics, validatorMonitor}
-    );
-
-    const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({
-      source: StateHashTreeRootSource.blockTransition,
-    });
-    const stateRoot = postState.hashTreeRoot();
-    hashTreeRootTimer?.();
-
-    // Check state root matches
-    if (!byteArrayEquals(block.message.stateRoot, stateRoot)) {
-      throw new BlockError(block, {
-        code: BlockErrorCode.INVALID_STATE_ROOT,
-        root: postState.hashTreeRoot(),
-        expectedRoot: block.message.stateRoot,
-        preState,
-        postState,
-      });
-    }
-
-    postStates[i] = postState;
-
-    // For metric block profitability
-    const proposerIndex = block.message.proposerIndex;
-    proposerBalanceDeltas[i] = postState.balances.get(proposerIndex) - preState.balances.get(proposerIndex);
-
-    // If blocks are invalid in execution the main promise could resolve before this loop ends.
-    // In that case stop processing blocks and return early.
-    if (signal.aborted) {
-      throw new ErrorAborted("verifyBlockStateTransitionOnly");
-    }
-
-    // this avoids keeping our node busy processing blocks
-    if (i < blocks.length - 1) {
-      await nextEventLoop();
-    }
-  }
-
-  const verifyStateTime = Date.now();
-  if (blocks.length === 1 && opts.seenTimestampSec !== undefined) {
-    const slot = blocks[0].getBlock().message.slot;
-    const recvToValidation = verifyStateTime / 1000 - opts.seenTimestampSec;
-    const validationTime = recvToValidation - recvToValLatency;
-
-    metrics?.gossipBlock.stateTransition.recvToValidation.observe(recvToValidation);
-    metrics?.gossipBlock.stateTransition.validationTime.observe(validationTime);
-
-    logger.debug("Verified block state transition", {slot, recvToValLatency, recvToValidation, validationTime});
-  }
-
-  return {postStates, proposerBalanceDeltas, verifyStateTime};
-}

package/src/chain/blocks/writeBlockInputToDb.ts

@@ -1,128 +0,0 @@
-import {fulu} from "@lodestar/types";
-import {prettyPrintIndices, toRootHex} from "@lodestar/utils";
-import {BeaconChain} from "../chain.js";
-import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blockInput/index.js";
-import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js";
-
-/**
- * Persists block input data to DB. This operation must be eventually completed if a block is imported to the fork-choice.
- * Else the node will be in an inconsistent state that can lead to being stuck.
- *
- * This operation may be performed before, during or after importing to the fork-choice. As long as errors
- * are handled properly for eventual consistency.
- */
-export async function writeBlockInputToDb(this: BeaconChain, blocksInputs: IBlockInput[]): Promise<void> {
-  const fnPromises: Promise<void>[] = [];
-  // track slots for logging
-  const slots: number[] = [];
-
-  for (const blockInput of blocksInputs) {
-    const block = blockInput.getBlock();
-    const slot = block.message.slot;
-    slots.push(slot);
-    const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
-    const blockRootHex = toRootHex(blockRoot);
-    const blockBytes = this.serializedCache.get(block);
-    if (blockBytes) {
-      // skip serializing data if we already have it
-      this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc();
-      fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes));
-    } else {
-      this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc();
-      fnPromises.push(this.db.block.add(block));
-    }
-
-    this.logger.debug("Persist block to hot DB", {
-      slot: block.message.slot,
-      root: blockRootHex,
-      inputType: blockInput.type,
-    });
-
-    if (!blockInput.hasAllData()) {
-      await blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT);
-    }
-
-    // NOTE: Old data is pruned on archive
-    if (isBlockInputColumns(blockInput)) {
-      const {custodyColumns} = this.custodyConfig;
-      const blobsLen = (block.message as fulu.BeaconBlock).body.blobKzgCommitments.length;
-      let dataColumnsLen: number;
-      if (blobsLen === 0) {
-        dataColumnsLen = 0;
-      } else {
-        dataColumnsLen = custodyColumns.length;
-      }
-
-      const dataColumnSidecars = blockInput.getCustodyColumns();
-      if (dataColumnSidecars.length !== dataColumnsLen) {
-        this.logger.debug(
-          `Invalid dataColumnSidecars=${dataColumnSidecars.length} for custody expected custodyColumnsLen=${dataColumnsLen}`
-        );
-      }
-
-      fnPromises.push(this.db.dataColumnSidecar.putMany(blockRoot, dataColumnSidecars));
-      this.logger.debug("Persisted dataColumnSidecars to hot DB", {
-        slot: block.message.slot,
-        root: blockRootHex,
-        dataColumnSidecars: dataColumnSidecars.length,
-        numBlobs: blobsLen,
-        custodyColumns: custodyColumns.length,
-      });
-    } else if (isBlockInputBlobs(blockInput)) {
-      const blobSidecars = blockInput.getBlobs();
-      fnPromises.push(this.db.blobSidecars.add({blockRoot, slot: block.message.slot, blobSidecars}));
-      this.logger.debug("Persisted blobSidecars to hot DB", {
-        blobsLen: blobSidecars.length,
-        slot: block.message.slot,
-        root: blockRootHex,
-      });
-    }
-
-    await Promise.all(fnPromises);
-    this.logger.debug("Persisted blocksInput to db", {
-      blocksInput: blocksInputs.length,
-      slots: prettyPrintIndices(slots),
-    });
-  }
-}
-
-/**
- * Prunes eagerly persisted block inputs only if not known to the fork-choice
- */
-export async function removeEagerlyPersistedBlockInputs(this: BeaconChain, blockInputs: IBlockInput[]): Promise<void> {
-  const blockToRemove = [];
-  const blobsToRemove = [];
-  const dataColumnsToRemove = [];
-
-  for (const blockInput of blockInputs) {
-    const block = blockInput.getBlock();
-    const slot = block.message.slot;
-    const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message);
-    const blockRootHex = toRootHex(blockRoot);
-    if (!this.forkChoice.hasBlockHex(blockRootHex)) {
-      blockToRemove.push(block);
-
-      if (isBlockInputColumns(blockInput)) {
-        const {custodyColumns} = this.custodyConfig;
-        const dataColumnsLen = custodyColumns.length;
-        const dataColumnSidecars = blockInput.getCustodyColumns();
-        if (dataColumnSidecars.length !== dataColumnsLen) {
-          throw Error(
-            `Invalid dataColumnSidecars=${dataColumnSidecars.length} for custody expected custodyColumnsLen=${dataColumnsLen}`
-          );
-        }
-        dataColumnsToRemove.push(blockRoot);
-      } else if (isBlockInputBlobs(blockInput)) {
-        const blobSidecars = blockInput.getBlobs();
-        blobsToRemove.push({blockRoot, slot, blobSidecars});
-      }
-    }
-  }
-
-  await Promise.all([
-    // TODO: Batch DB operations not with Promise.all but with level db ops
-    this.db.block.batchRemove(blockToRemove),
-    this.db.blobSidecars.batchRemove(blobsToRemove),
-    this.db.dataColumnSidecar.deleteMany(dataColumnsToRemove),
-  ]);
-}

package/src/chain/bls/index.ts

@@ -1,4 +0,0 @@
-export type {IBlsVerifier} from "./interface.js";
-export type {BlsMultiThreadWorkerPoolModules, JobQueueItemType} from "./multithread/index.js";
-export {BlsMultiThreadWorkerPool} from "./multithread/index.js";
-export {BlsSingleThreadVerifier} from "./singleThread.js";

package/src/chain/bls/interface.ts

@@ -1,68 +0,0 @@
-import {PublicKey} from "@chainsafe/blst";
-import {ISignatureSet} from "@lodestar/state-transition";
-
-export type VerifySignatureOpts = {
-  /**
-   * A batchable set MAY be verified with more sets to reduce the verification costs.
-   * Multiple sets may be merged and verified as one set. If the result is correct, success is returned
-   * for all them. If at least one set is invalid, all sets are reverified individually. For normal network
-   * conditions this strategy can yield 50% improvement in CPU time spent verifying gossip objects.
-   * Only non-time critical objects should be marked as batchable, since the pool may hold them for 100ms.
-   */
-  batchable?: boolean;
-
-  /**
-   * Use main thread to verify signatures, use this with care.
-   * Ignore the batchable option if this is true.
-   */
-  verifyOnMainThread?: boolean;
-  /**
-   * Some signature sets are more important than others, and should be verified first.
-   */
-  priority?: boolean;
-};
-
-export interface IBlsVerifier {
-  /**
-   * Verify 1 or more signature sets. Sets may be verified on batch or not depending on their count
-   *
-   * Signatures all come from the wire (untrusted) are all bytes compressed, must be:
-   * - Parsed from bytes
-   * - Uncompressed
-   * - subgroup_check
-   * - consume in Pairing.aggregate as affine, or mul_n_aggregate as affine
-   * Just send the raw signture recevied as bytes to the thread and verify there
-   *
-   * Pubkeys all come from cache (trusted) have already been checked for subgroup and infinity
-   * - Some pubkeys will have to be aggregated, some don't
-   * - Pubkeys must be available in jacobian coordinates to make aggregation x3 faster
-   * - Then, consume in Pairing.aggregate as affine, or mul_n_aggregate as affine
-   *
-   * All signatures are not trusted and must be group checked (p2.subgroup_check)
-   *
-   * Public keys have already been checked for subgroup and infinity
-   * Signatures have already been checked for subgroup
-   * Signature checks above could be done here for convienence as well
-   */
-  verifySignatureSets(sets: ISignatureSet[], opts?: VerifySignatureOpts): Promise<boolean>;
-
-  /**
-   * Similar to verifySignatureSets but:
-   *   - all signatures have the same message
-   *   - return an array of boolean, each element indicates whether the corresponding signature set is valid
-   *   - only support `batchable` option
-   */
-  verifySignatureSetsSameMessage(
-    sets: {publicKey: PublicKey; signature: Uint8Array}[],
-    messsage: Uint8Array,
-    opts?: Omit<VerifySignatureOpts, "verifyOnMainThread">
-  ): Promise<boolean[]>;
-
-  /** For multithread pool awaits terminating all workers */
-  close(): Promise<void>;
-
-  /**
-   * Returns true if BLS worker pool is ready to accept more work jobs.
-   */
-  canAcceptWork(): boolean;
-}

package/src/chain/bls/maybeBatch.ts

@@ -1,45 +0,0 @@
-import {PublicKey, Signature, verify, verifyMultipleAggregateSignatures} from "@chainsafe/blst";
-
-const MIN_SET_COUNT_TO_BATCH = 2;
-
-export type SignatureSetDeserialized = {
-  publicKey: PublicKey;
-  message: Uint8Array;
-  signature: Uint8Array;
-};
-
-/**
- * Verify signatures sets with batch verification or regular core verify depending on the set count.
- * Abstracted in a separate file to be consumed by the threaded pool and the main thread implementation.
- */
-export function verifySignatureSetsMaybeBatch(sets: SignatureSetDeserialized[]): boolean {
-  try {
-    if (sets.length >= MIN_SET_COUNT_TO_BATCH) {
-      return verifyMultipleAggregateSignatures(
-        sets.map((s) => ({
-          pk: s.publicKey,
-          msg: s.message,
-          // true = validate signature
-          sig: Signature.fromBytes(s.signature, true),
-        }))
-      );
-    }
-
-    // .every on an empty array returns true
-    if (sets.length === 0) {
-      throw Error("Empty signature set");
-    }
-
-    // If too few signature sets verify them without batching
-    return sets.every((set) => {
-      // true = validate signature
-      const sig = Signature.fromBytes(set.signature, true);
-      return verify(set.message, set.publicKey, sig);
-    });
-  } catch (_) {
-    // A signature could be malformed, in that case fromBytes throws error
-    // blst-ts `verifyMultipleSignatures` is also a fallible operation if mul_n_aggregate fails
-    // see https://github.com/ChainSafe/blst-ts/blob/b1ba6333f664b08e5c50b2b0d18c4f079203962b/src/lib.ts#L291
-    return false;
-  }
-}