@lodestar/beacon-node 1.35.0-dev.21ba2777a9 → 1.35.0-dev.36a31f3a2f

package/src/chain/opPools/syncContributionAndProofPool.ts

@@ -1,240 +0,0 @@
-import {Signature, aggregateSignatures} from "@chainsafe/blst";
-import {BitArray} from "@chainsafe/ssz";
-import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params";
-import {G2_POINT_AT_INFINITY} from "@lodestar/state-transition";
-import {Root, Slot, SubnetID, altair, ssz} from "@lodestar/types";
-import {Logger, MapDef, toRootHex} from "@lodestar/utils";
-import {MAXIMUM_GOSSIP_CLOCK_DISPARITY} from "../../constants/constants.js";
-import {Metrics} from "../../metrics/metrics.js";
-import {IClock} from "../../util/clock.js";
-import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js";
-import {pruneBySlot, signatureFromBytesNoCheck} from "./utils.js";
-
-/**
- * SyncCommittee aggregates are only useful for the next block they have signed.
- */
-const SLOTS_RETAINED = 8;
-
-/**
- * The maximum number of distinct `SyncAggregateFast` that will be stored in each slot.
- *
- * This is a DoS protection measure.
- */
-const MAX_ITEMS_PER_SLOT = 512;
-
-// SyncContributionAndProofPool constructor ensures SYNC_COMMITTEE_SUBNET_SIZE is multiple of 8
-const SYNC_COMMITTEE_SUBNET_BYTES = SYNC_COMMITTEE_SUBNET_SIZE / 8;
-
-/**
- * A one-one mapping to SyncContribution with fast data structure to help speed up the aggregation.
- */
-export type SyncContributionFast = {
-  syncSubcommitteeBits: BitArray;
-  numParticipants: number;
-  syncSubcommitteeSignature: Uint8Array;
-};
-
-/** Hex string of `contribution.beaconBlockRoot` */
-type BlockRootHex = string;
-
-/**
- * Cache SyncCommitteeContribution and seen ContributionAndProof.
- * This is used for SignedContributionAndProof validation and block factory.
- * This stays in-memory and should be pruned per slot.
- */
-export class SyncContributionAndProofPool {
-  private readonly bestContributionBySubnetRootBySlot = new MapDef<
-    Slot,
-    MapDef<BlockRootHex, Map<SubnetID, SyncContributionFast>>
-  >(() => new MapDef<BlockRootHex, Map<SubnetID, SyncContributionFast>>(() => new Map<number, SyncContributionFast>()));
-
-  private lowestPermissibleSlot = 0;
-
-  constructor(
-    private readonly clock: IClock,
-    private readonly metrics: Metrics | null = null,
-    private logger: Logger | null = null
-  ) {
-    // Param guarantee for optimizations below that merge syncSubcommitteeBits as bytes
-    if (SYNC_COMMITTEE_SUBNET_SIZE % 8 !== 0) {
-      throw Error("SYNC_COMMITTEE_SUBNET_SIZE must be multiple of 8");
-    }
-    metrics?.opPool.syncContributionAndProofPool.size.addCollect(() => this.onScrapeMetrics(metrics));
-  }
-
-  /** Returns current count of unique SyncContributionFast by block root and subnet */
-  get size(): number {
-    let count = 0;
-    for (const bestContributionByRootBySubnet of this.bestContributionBySubnetRootBySlot.values()) {
-      for (const bestContributionByRoot of bestContributionByRootBySubnet.values()) {
-        count += bestContributionByRoot.size;
-      }
-    }
-    return count;
-  }
-
-  /**
-   * Only call this once we pass all validation.
-   */
-  add(
-    contributionAndProof: altair.ContributionAndProof,
-    syncCommitteeParticipants: number,
-    priority?: boolean
-  ): InsertOutcome {
-    const {contribution} = contributionAndProof;
-    const {slot, beaconBlockRoot} = contribution;
-    const rootHex = toRootHex(beaconBlockRoot);
-
-    // Reject if too old.
-    if (slot < this.lowestPermissibleSlot) {
-      return InsertOutcome.Old;
-    }
-
-    // Reject ContributionAndProofs of previous slots
-    // for api ContributionAndProofs, we allow them to be added to the pool
-    if (!priority && slot < this.clock.slotWithPastTolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY)) {
-      return InsertOutcome.Late;
-    }
-
-    // Limit object per slot
-    const bestContributionBySubnetByRoot = this.bestContributionBySubnetRootBySlot.getOrDefault(slot);
-    if (bestContributionBySubnetByRoot.size >= MAX_ITEMS_PER_SLOT) {
-      throw new OpPoolError({code: OpPoolErrorCode.REACHED_MAX_PER_SLOT});
-    }
-
-    const bestContributionBySubnet = bestContributionBySubnetByRoot.getOrDefault(rootHex);
-    const subnet = contribution.subcommitteeIndex;
-    const bestContribution = bestContributionBySubnet.get(subnet);
-    if (bestContribution) {
-      return replaceIfBetter(bestContribution, contribution, syncCommitteeParticipants);
-    }
-    bestContributionBySubnet.set(subnet, contributionToFast(contribution, syncCommitteeParticipants));
-    return InsertOutcome.NewData;
-  }
-
-  /**
-   * This is for producing blocks, the same to process_sync_committee_contributions in the spec.
-   */
-  getAggregate(slot: Slot, prevBlockRoot: Root): altair.SyncAggregate {
-    const opPoolMetrics = this.metrics?.opPool.syncContributionAndProofPool;
-    const bestContributionBySubnetByRoot = this.bestContributionBySubnetRootBySlot.getOrDefault(slot);
-    opPoolMetrics?.getAggregateRoots.set(bestContributionBySubnetByRoot.size);
-    const prevBlockRootHex = toRootHex(prevBlockRoot);
-    const bestContributionBySubnet = bestContributionBySubnetByRoot.get(prevBlockRootHex) ?? new Map();
-    opPoolMetrics?.getAggregateSubnets.set(bestContributionBySubnet.size);
-
-    if (bestContributionBySubnet.size === 0) {
-      opPoolMetrics?.getAggregateReturnsEmpty.inc();
-      // this may happen, see https://github.com/ChainSafe/lodestar/issues/7299
-      const availableRoots = Array.from(bestContributionBySubnetByRoot.keys()).join(",");
-      this.logger?.warn("SyncContributionAndProofPool.getAggregate: no contributions for root", {
-        slot,
-        root: prevBlockRootHex,
-        availableRoots,
-      });
-      // Must return signature as G2_POINT_AT_INFINITY when participating bits are empty
-      // https://github.com/ethereum/consensus-specs/blob/30f2a076377264677e27324a8c3c78c590ae5e20/specs/altair/bls.md#eth2_fast_aggregate_verify
-      return {
-        syncCommitteeBits: ssz.altair.SyncCommitteeBits.defaultValue(),
-        syncCommitteeSignature: G2_POINT_AT_INFINITY,
-      };
-    }
-
-    return aggregate(bestContributionBySubnet, this.metrics);
-  }
-
-  /**
-   * Prune per head slot.
-   * SyncCommittee aggregates are only useful for the next block they have signed.
-   * We don't want to prune by clock slot in case there's a long period of skipped slots.
-   */
-  prune(headSlot: Slot): void {
-    pruneBySlot(this.bestContributionBySubnetRootBySlot, headSlot, SLOTS_RETAINED);
-    this.lowestPermissibleSlot = Math.max(headSlot - SLOTS_RETAINED, 0);
-  }
-
-  private onScrapeMetrics(metrics: Metrics): void {
-    const poolMetrics = metrics.opPool.syncContributionAndProofPool;
-    poolMetrics.size.set(this.size);
-    const previousSlot = this.clock.currentSlot - 1;
-    const contributionBySubnetByBlockRoot = this.bestContributionBySubnetRootBySlot.getOrDefault(previousSlot);
-    poolMetrics.blockRootsPerSlot.set(contributionBySubnetByBlockRoot.size);
-    let index = 0;
-    for (const contributionsBySubnet of contributionBySubnetByBlockRoot.values()) {
-      let participationCount = 0;
-      for (const contribution of contributionsBySubnet.values()) {
-        participationCount += contribution.numParticipants;
-      }
-      poolMetrics.subnetsByBlockRoot.set({index}, contributionsBySubnet.size);
-      poolMetrics.participantsByBlockRoot.set({index}, participationCount);
-      index++;
-    }
-  }
-}
-
-/**
- * Mutate bestContribution if new contribution has more participants
- */
-export function replaceIfBetter(
-  bestContribution: SyncContributionFast,
-  newContribution: altair.SyncCommitteeContribution,
-  newNumParticipants: number
-): InsertOutcome {
-  const {numParticipants} = bestContribution;
-
-  if (newNumParticipants <= numParticipants) {
-    return InsertOutcome.NotBetterThan;
-  }
-
-  bestContribution.syncSubcommitteeBits = newContribution.aggregationBits;
-  bestContribution.numParticipants = newNumParticipants;
-  bestContribution.syncSubcommitteeSignature = newContribution.signature;
-  return InsertOutcome.NewData;
-}
-
-/**
- * Format `contribution` into an efficient data structure to aggregate later.
- */
-export function contributionToFast(
-  contribution: altair.SyncCommitteeContribution,
-  numParticipants: number
-): SyncContributionFast {
-  return {
-    // No need to clone, aggregationBits are not mutated, only replaced
-    syncSubcommitteeBits: contribution.aggregationBits,
-    numParticipants,
-    // No need to deserialize, signatures are not aggregated until when calling .getAggregate()
-    syncSubcommitteeSignature: contribution.signature,
-  };
-}
-
-/**
- * Aggregate best contributions of each subnet into SyncAggregate
- * @returns SyncAggregate to be included in block body.
- */
-export function aggregate(
-  bestContributionBySubnet: Map<number, SyncContributionFast>,
-  metrics: Metrics | null = null
-): altair.SyncAggregate {
-  // check for empty/undefined bestContributionBySubnet earlier
-  const syncCommitteeBits = BitArray.fromBitLen(SYNC_COMMITTEE_SIZE);
-
-  const signatures: Signature[] = [];
-  let participationCount = 0;
-  for (const [subnet, bestContribution] of bestContributionBySubnet.entries()) {
-    participationCount += bestContribution.numParticipants;
-    const byteOffset = subnet * SYNC_COMMITTEE_SUBNET_BYTES;
-
-    for (let i = 0; i < SYNC_COMMITTEE_SUBNET_BYTES; i++) {
-      syncCommitteeBits.uint8Array[byteOffset + i] = bestContribution.syncSubcommitteeBits.uint8Array[i];
-    }
-
-    signatures.push(signatureFromBytesNoCheck(bestContribution.syncSubcommitteeSignature));
-  }
-
-  metrics?.opPool.syncContributionAndProofPool.getAggregateParticipants.set(participationCount);
-  return {
-    syncCommitteeBits,
-    syncCommitteeSignature: aggregateSignatures(signatures).toBytes(),
-  };
-}

package/src/chain/opPools/types.ts

@@ -1,35 +0,0 @@
-import {Slot} from "@lodestar/types";
-import {LodestarError} from "@lodestar/utils";
-
-/**
- * Result of adding data to an operation pool of an aggregatable object.
- */
-export enum InsertOutcome {
-  /** The data had not been seen before and was added to the pool. */
-  NewData = "NewData",
-  /** A validator signature for a participant of this data is already known. No changes were made. */
-  AlreadyKnown = "AlreadyKnown",
-  /** Not existing in the pool but it's too old to add. No changes were made. */
-  Old = "Old",
-  /** The pool has reached its limit. No changes were made. */
-  ReachLimit = "ReachLimit",
-  /** Messages don't bring any value, for example attestations come to the pool at > 2/3 of slot. No changes were made */
-  Late = "Late",
-  /** The data is know, and the new participants have been added to the aggregated signature */
-  Aggregated = "Aggregated",
-  /** The data is not better than the existing data*/
-  NotBetterThan = "NotBetterThan",
-}
-
-export enum OpPoolErrorCode {
-  /** The given object slot was too low to be stored. No changes were made. */
-  SLOT_TOO_LOW = "OP_POOL_ERROR_SLOT_TOO_LOW",
-  /** Reached max number of unique objects per slot. This is a DoS protection function. */
-  REACHED_MAX_PER_SLOT = "OP_POOL_ERROR_REACHED_MAX_PER_SLOT",
-}
-
-export type OpPoolErrorType =
-  | {code: OpPoolErrorCode.SLOT_TOO_LOW; slot: Slot; lowestPermissibleSlot: Slot}
-  | {code: OpPoolErrorCode.REACHED_MAX_PER_SLOT};
-
-export class OpPoolError extends LodestarError<OpPoolErrorType> {}

package/src/chain/opPools/utils.ts

@@ -1,65 +0,0 @@
-import {Signature} from "@chainsafe/blst";
-import {BLS_WITHDRAWAL_PREFIX} from "@lodestar/params";
-import {CachedBeaconStateAllForks} from "@lodestar/state-transition";
-import {Slot, capella} from "@lodestar/types";
-import {AggregateFast, AggregateFastElectra} from "./attestationPool.js";
-
-/**
- * Prune a Map indexed by slot to keep the most recent slots, up to `slotsRetained`
- */
-export function pruneBySlot(map: Map<Slot, unknown>, slot: Slot, slotsRetained: Slot): Slot {
-  const lowestPermissibleSlot = Math.max(slot - slotsRetained, 0);
-
-  // No need to prune if the lowest permissible slot has not changed and the queue length is less than the maximum
-  if (map.size <= slotsRetained) {
-    return lowestPermissibleSlot;
-  }
-
-  // Remove the oldest slots to keep a max of `slotsRetained` slots
-  const slots = Array.from(map.keys());
-  const slotsToDelete = slots.sort((a, b) => b - a).slice(slotsRetained);
-  for (const slot of slotsToDelete) {
-    map.delete(slot);
-  }
-
-  return lowestPermissibleSlot;
-}
-
-/**
- * De-serialize bytes into Signature.
- * No need to verify Signature is valid, already run sig-verify = false
- */
-export function signatureFromBytesNoCheck(signature: Uint8Array): Signature {
-  return Signature.fromBytes(signature);
-}
-
-/**
- * Ensures that a SignedBLSToExecutionChange object is _still_ valid for block inclusion. An object valid for the pool,
- * can become invalid for certain forks.
- */
-export function isValidBlsToExecutionChangeForBlockInclusion(
-  state: CachedBeaconStateAllForks,
-  signedBLSToExecutionChange: capella.SignedBLSToExecutionChange
-): boolean {
-  // For each condition from https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#new-process_bls_to_execution_change
-  //
-  // 1. assert address_change.validator_index < len(state.validators):
-  //    If valid before will always be valid in the future, no need to check
-  //
-  // 2. assert validator.withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX:
-  //    Must be checked again, since it can already be changed by now.
-  const validator = state.validators.getReadonly(signedBLSToExecutionChange.message.validatorIndex);
-  const {withdrawalCredentials} = validator;
-  if (withdrawalCredentials[0] !== BLS_WITHDRAWAL_PREFIX) {
-    return false;
-  }
-
-  // 3. assert validator.withdrawal_credentials[1:] == hash(address_change.from_bls_pubkey)[1:]:
-  //    If valid before will always be valid in the future, no need to check
-
-  return true;
-}
-
-export function isElectraAggregate(aggregate: AggregateFast): aggregate is AggregateFastElectra {
-  return (aggregate as AggregateFastElectra).committeeBits !== undefined;
-}

package/src/chain/options.ts

@@ -1,138 +0,0 @@
-import {SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY} from "@lodestar/params";
-import {defaultOptions as defaultValidatorOptions} from "@lodestar/validator";
-import {DEFAULT_ARCHIVE_MODE} from "./archiveStore/constants.js";
-import {ArchiveMode, ArchiveStoreOpts} from "./archiveStore/interface.js";
-import {ForkChoiceOpts} from "./forkChoice/index.js";
-import {LightClientServerOpts} from "./lightClient/index.js";
-import {ShufflingCacheOpts} from "./shufflingCache.js";
-import {DEFAULT_MAX_BLOCK_STATES, FIFOBlockStateCacheOpts} from "./stateCache/fifoBlockStateCache.js";
-import {
-  DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY,
-  PersistentCheckpointStateCacheOpts,
-} from "./stateCache/persistentCheckpointsCache.js";
-import {ValidatorMonitorOpts} from "./validatorMonitor.js";
-
-export {ArchiveMode, DEFAULT_ARCHIVE_MODE};
-
-export type IChainOptions = BlockProcessOpts &
-  PoolOpts &
-  SeenCacheOpts &
-  ForkChoiceOpts &
-  ArchiveStoreOpts &
-  FIFOBlockStateCacheOpts &
-  PersistentCheckpointStateCacheOpts &
-  ShufflingCacheOpts &
-  ValidatorMonitorOpts &
-  LightClientServerOpts & {
-    blsVerifyAllMainThread?: boolean;
-    blsVerifyAllMultiThread?: boolean;
-    blacklistedBlocks?: string[];
-    persistProducedBlocks?: boolean;
-    persistInvalidSszObjects?: boolean;
-    persistInvalidSszObjectsDir?: string;
-    persistOrphanedBlocks?: boolean;
-    persistOrphanedBlocksDir?: string;
-    skipCreateStateCacheIfAvailable?: boolean;
-    suggestedFeeRecipient: string;
-    maxSkipSlots?: number;
-    /** Ensure blobs returned by the execution engine are valid */
-    sanityCheckExecutionEngineBlobs?: boolean;
-    /** Max number of produced blobs by local validators to cache */
-    maxCachedBlobSidecars?: number;
-    /** Max number of produced block roots (blinded or full) cached for broadcast validations */
-    maxCachedProducedRoots?: number;
-    /** Subscribe to and custody all data column sidecar subnets */
-    supernode?: boolean;
-    initialCustodyGroupCount?: number;
-    broadcastValidationStrictness?: string;
-    minSameMessageSignatureSetsToBatch: number;
-    archiveDateEpochs?: number;
-    nHistoricalStates?: boolean;
-    nHistoricalStatesFileDataStore?: boolean;
-  };
-
-export type BlockProcessOpts = {
-  /**
-   * Do not use BLS batch verify to validate all block signatures at once.
-   * Will double processing times. Use only for debugging purposes.
-   */
-  disableBlsBatchVerify?: boolean;
-  /**
-   * Override SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY
-   */
-  safeSlotsToImportOptimistically?: number;
-  /**
-   * Assert progressive balances the same to EpochTransitionCache
-   */
-  assertCorrectProgressiveBalances?: boolean;
-  /** Used for fork_choice spec tests */
-  disableOnBlockError?: boolean;
-  /** Used for fork_choice spec tests */
-  disablePrepareNextSlot?: boolean;
-  /**
-   * Used to connect beacon in follow mode to an EL,
-   * will still issue fcU for block proposal
-   */
-  disableImportExecutionFcU?: boolean;
-  emitPayloadAttributes?: boolean;
-
-  /**
-   * Used to specify to specify to run verifications only and not
-   * to save the block or log transitions for e.g. doing
-   * broadcastValidation while publishing the block
-   */
-  verifyOnly?: boolean;
-  /** Used to specify to skip execution payload validation */
-  skipVerifyExecutionPayload?: boolean;
-  /** Used to specify to skip block signatures validation */
-  skipVerifyBlockSignatures?: boolean;
-};
-
-export type PoolOpts = {
-  /**
-   * Only preaggregate attestation/sync committee message since clockSlot - preaggregateSlotDistance
-   */
-  preaggregateSlotDistance?: number;
-};
-
-export type SeenCacheOpts = {
-  /**
-   * Slot distance from current slot to cache AttestationData
-   */
-  attDataCacheSlotDistance?: number;
-};
-
-export const defaultChainOptions: IChainOptions = {
-  blsVerifyAllMainThread: false,
-  blsVerifyAllMultiThread: false,
-  blacklistedBlocks: [],
-  disableBlsBatchVerify: false,
-  proposerBoost: true,
-  proposerBoostReorg: true,
-  computeUnrealized: true,
-  safeSlotsToImportOptimistically: SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY,
-  suggestedFeeRecipient: defaultValidatorOptions.suggestedFeeRecipient,
-  serveHistoricalState: false,
-  assertCorrectProgressiveBalances: false,
-  archiveStateEpochFrequency: 1024,
-  archiveMode: DEFAULT_ARCHIVE_MODE,
-  pruneHistory: false,
-  emitPayloadAttributes: false,
-  supernode: false,
-  // for gossip block validation, it's unlikely we see a reorg with 32 slots
-  // for attestation validation, having this value ensures we don't have to regen states most of the time
-  maxSkipSlots: 32,
-  broadcastValidationStrictness: "warn",
-  // should be less than or equal to MIN_SIGNATURE_SETS_TO_BATCH_VERIFY
-  // batching too much may block the I/O thread so if useWorker=false, suggest this value to be 32
-  // since this batch attestation work is designed to work with useWorker=true, make this the lowest value
-  minSameMessageSignatureSetsToBatch: 2,
-  nHistoricalStates: true,
-  // as of Feb 2025, this option turned out to be very useful:
-  //   - it allows to share a persisted checkpoint state to other nodes
-  //   - users can prune the persisted checkpoint state files manually to save disc space
-  //   - it helps debug easier when network is unfinalized
-  nHistoricalStatesFileDataStore: true,
-  maxBlockStates: DEFAULT_MAX_BLOCK_STATES,
-  maxCPStateEpochsInMemory: DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY,
-};