@lodestar/state-transition 1.41.0-dev.702f7932c2 → 1.41.0-dev.8578102bd2

package/src/stateView/index.ts

@@ -0,0 +1,2 @@
+export * from "./beaconStateView.js";
+export * from "./interface.js";

package/src/stateView/interface.ts

@@ -0,0 +1,196 @@
+import {CompactMultiProof} from "@chainsafe/persistent-merkle-tree";
+import {ByteViews} from "@chainsafe/ssz";
+import {
+  BeaconBlock,
+  BlindedBeaconBlock,
+  BuilderIndex,
+  Bytes32,
+  Epoch,
+  ExecutionPayloadBid,
+  ExecutionPayloadHeader,
+  Root,
+  RootHex,
+  SignedBeaconBlock,
+  SignedBlindedBeaconBlock,
+  Slot,
+  ValidatorIndex,
+  altair,
+  capella,
+  electra,
+  fulu,
+  gloas,
+  phase0,
+  rewards,
+} from "@lodestar/types";
+import {Checkpoint, Fork} from "@lodestar/types/phase0";
+import {VoluntaryExitValidity} from "../block/processVoluntaryExit.js";
+import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js";
+import {EpochTransitionCacheOpts} from "../cache/epochTransitionCache.js";
+import {RewardCache} from "../cache/rewardCache.js";
+import {SyncCommitteeCache} from "../cache/syncCommitteeCache.js";
+import {SyncCommitteeWitness} from "../lightClient/types.js";
+import {StateTransitionModules, StateTransitionOpts} from "../stateTransition.js";
+import {EpochShuffling} from "../util/epochShuffling.js";
+
+/**
+ * A read-only view of the BeaconState.
+ */
+export interface IBeaconStateView {
+  // State access
+
+  // phase0
+  slot: Slot;
+  fork: Fork;
+  epoch: Epoch;
+  genesisTime: number;
+  genesisValidatorsRoot: Root;
+  eth1Data: phase0.Eth1Data;
+  latestBlockHeader: phase0.BeaconBlockHeader;
+  previousJustifiedCheckpoint: Checkpoint;
+  currentJustifiedCheckpoint: Checkpoint;
+  finalizedCheckpoint: Checkpoint;
+  getBlockRootAtSlot(slot: Slot): Root;
+  getBlockRootAtEpoch(epoch: Epoch): Root;
+  getStateRootAtSlot(slot: Slot): Root;
+  getRandaoMix(epoch: Epoch): Bytes32;
+
+  // altair
+  previousEpochParticipation: number[];
+  currentEpochParticipation: number[];
+
+  // bellatrix
+  latestExecutionPayloadHeader: ExecutionPayloadHeader;
+
+  // capella
+  historicalSummaries: capella.HistoricalSummaries;
+
+  // electra
+  pendingDeposits: electra.PendingDeposits;
+  pendingDepositsCount: number;
+  pendingPartialWithdrawals: electra.PendingPartialWithdrawals;
+  pendingPartialWithdrawalsCount: number;
+  pendingConsolidations: electra.PendingConsolidations;
+  pendingConsolidationsCount: number;
+
+  // fulu
+  proposerLookahead: fulu.ProposerLookahead;
+
+  // gloas
+  executionPayloadAvailability: boolean[];
+  latestExecutionPayloadBid: ExecutionPayloadBid;
+  getBuilder(index: BuilderIndex): gloas.Builder;
+  canBuilderCoverBid(builderIndex: BuilderIndex, bidAmount: number): boolean;
+  validatorPTCCommitteeIndex(validatorIndex: ValidatorIndex, slot: Slot): number;
+
+  // Shuffling and committees
+  getShufflingAtEpoch(epoch: Epoch): EpochShuffling;
+  // Decision roots
+  previousDecisionRoot: RootHex;
+  currentDecisionRoot: RootHex;
+  nextDecisionRoot: RootHex;
+  getShufflingDecisionRoot(epoch: Epoch): RootHex;
+  getPreviousShuffling(): EpochShuffling;
+  getCurrentShuffling(): EpochShuffling;
+  getNextShuffling(): EpochShuffling;
+
+  // utils: proposers, anchor checkpoint
+  previousProposers: ValidatorIndex[] | null;
+  currentProposers: ValidatorIndex[];
+  nextProposers: ValidatorIndex[];
+  getBeaconProposer(slot: Slot): ValidatorIndex;
+  computeAnchorCheckpoint(): {checkpoint: phase0.Checkpoint; blockHeader: phase0.BeaconBlockHeader};
+
+  // Sync committees
+  currentSyncCommittee: altair.SyncCommittee;
+  nextSyncCommittee: altair.SyncCommittee;
+  currentSyncCommitteeIndexed: SyncCommitteeCache;
+  syncProposerReward: number;
+  getIndexedSyncCommitteeAtEpoch(epoch: Epoch): SyncCommitteeCache;
+
+  // Validators and balances
+  effectiveBalanceIncrements: EffectiveBalanceIncrements;
+  getEffectiveBalanceIncrementsZeroInactive(): EffectiveBalanceIncrements;
+  getBalance(index: number): number;
+  // readonly
+  getValidator(index: ValidatorIndex): phase0.Validator;
+  getValidatorsByStatus(statuses: Set<string>, currentEpoch: Epoch): phase0.Validator[];
+  validatorCount: number;
+  // this get number of active validators in the current shuffling
+  activeValidatorCount: number;
+  // this is needed for apis only
+  getAllValidators(): phase0.Validator[];
+  getAllBalances(): number[];
+
+  // Merge
+  isExecutionStateType: boolean;
+  isMergeTransitionComplete: boolean;
+  // TODO this should go away (or rather only need block)
+  isExecutionEnabled(block: BeaconBlock | BlindedBeaconBlock): boolean;
+
+  // Block production
+  getExpectedWithdrawals(): {
+    expectedWithdrawals: capella.Withdrawal[];
+    processedBuilderWithdrawalsCount: number;
+    processedPartialWithdrawalsCount: number;
+    processedValidatorSweepCount: number;
+  };
+
+  // API
+  proposerRewards: RewardCache;
+  computeBlockRewards(block: BeaconBlock, proposerRewards?: RewardCache): Promise<rewards.BlockRewards>;
+  computeAttestationsRewards(validatorIds?: (ValidatorIndex | string)[]): Promise<rewards.AttestationsRewards>;
+  computeSyncCommitteeRewards(
+    block: BeaconBlock,
+    validatorIds: (ValidatorIndex | string)[]
+  ): Promise<rewards.SyncCommitteeRewards>;
+  getLatestWeakSubjectivityCheckpointEpoch(): Epoch;
+
+  // Validation
+  getVoluntaryExitValidity(
+    signedVoluntaryExit: phase0.SignedVoluntaryExit,
+    verifySignature: boolean
+  ): VoluntaryExitValidity;
+  isValidVoluntaryExit(signedVoluntaryExit: phase0.SignedVoluntaryExit, verifySignature: boolean): boolean;
+
+  // Proofs
+  getFinalizedRootProof(): Uint8Array[];
+  getSyncCommitteesWitness(): SyncCommitteeWitness;
+  getSingleProof(gindex: bigint): Uint8Array[];
+  createMultiProof(descriptor: Uint8Array): CompactMultiProof;
+
+  // Fork choice
+  computeUnrealizedCheckpoints(): {
+    justifiedCheckpoint: phase0.Checkpoint;
+    finalizedCheckpoint: phase0.Checkpoint;
+  };
+
+  // this is for backward compatible
+  clonedCount: number;
+  clonedCountWithTransferCache: number;
+  createdWithTransferCache: boolean;
+  // TODO is there a better name that is less implementation specific but still conveys the meaning?
+  isStateValidatorsNodesPopulated(): boolean;
+
+  // Serialization
+  loadOtherState(stateBytes: Uint8Array, seedValidatorsBytes?: Uint8Array): IBeaconStateView;
+  serialize(): Uint8Array;
+  serializedSize(): number;
+  serializeToBytes(output: ByteViews, offset: number): number;
+  serializeValidators(): Uint8Array;
+  serializedValidatorsSize(): number;
+  serializeValidatorsToBytes(output: ByteViews, offset: number): number;
+
+  hashTreeRoot(): Uint8Array;
+
+  // State transition
+  stateTransition(
+    signedBlock: SignedBeaconBlock | SignedBlindedBeaconBlock,
+    options: StateTransitionOpts,
+    modules: StateTransitionModules
+  ): IBeaconStateView;
+  processSlots(
+    slot: Slot,
+    epochTransitionCacheOpts?: EpochTransitionCacheOpts & {dontTransferCache?: boolean},
+    modules?: StateTransitionModules
+  ): IBeaconStateView;
+}

package/src/testUtils/cache.ts

@@ -0,0 +1,8 @@
+import path from "node:path";
+import {fileURLToPath} from "node:url";
+
+// Global variable __dirname no longer available in ES6 modules.
+// Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+export const testCachePath = path.join(__dirname, "../../test/test-cache");

package/src/testUtils/index.ts

@@ -0,0 +1,5 @@
+export * from "./cache.js";
+export * from "./interop.js";
+export * from "./params.js";
+export * from "./testFileCache.js";
+export * from "./util.js";

package/src/testUtils/infura.ts

@@ -0,0 +1,10 @@
+import {NetworkName} from "@lodestar/config/networks";
+
+export function getInfuraBeaconUrl(network: NetworkName): string {
+  const INFURA_ETH2_CREDENTIALS = process.env.INFURA_ETH2_CREDENTIALS;
+  if (!INFURA_ETH2_CREDENTIALS) {
+    throw Error("Must set ENV INFURA_ETH2_CREDENTIALS");
+  }
+
+  return `https://${INFURA_ETH2_CREDENTIALS}@eth2-beacon-${network}.infura.io`;
+}

package/src/testUtils/interop.ts

@@ -0,0 +1,29 @@
+import fs from "node:fs";
+import path from "node:path";
+import {fromHexString, toHexString} from "@chainsafe/ssz";
+import {interopSecretKey} from "../index.js";
+import {testCachePath} from "./cache.js";
+
+const interopPubkeysCachedPath = path.join(testCachePath, "interop-pubkeys.json");
+
+export function interopPubkeysCached(validatorCount: number): Uint8Array[] {
+  fs.mkdirSync(path.dirname(interopPubkeysCachedPath), {recursive: true});
+
+  const cachedKeysHex = fs.existsSync(interopPubkeysCachedPath)
+    ? (JSON.parse(fs.readFileSync(interopPubkeysCachedPath, "utf8")) as string[])
+    : ([] as string[]);
+
+  const keys = cachedKeysHex.slice(0, validatorCount).map((hex) => fromHexString(hex));
+
+  if (cachedKeysHex.length < validatorCount) {
+    for (let i = cachedKeysHex.length; i < validatorCount; i++) {
+      const sk = interopSecretKey(i);
+      const pk = sk.toPublicKey().toBytes();
+      keys.push(pk);
+    }
+    const keysHex = keys.map((pk) => toHexString(pk));
+    fs.writeFileSync(interopPubkeysCachedPath, JSON.stringify(keysHex, null, 2));
+  }
+
+  return keys;
+}

package/src/testUtils/params.ts

@@ -0,0 +1,23 @@
+// DON'T MODIFY OR FORMAT THIS FILE
+// USE FOR GA CACHE
+
+export const phase0State = {
+  network: "mainnet" as const,
+  epoch: 58758, // Pre-altair fork
+};
+
+export const altairState = {
+  network: "mainnet" as const,
+  epoch: 81889, // Post altair fork
+};
+
+export const capellaState = {
+  network: "mainnet" as const,
+  epoch: 217614, // Post capella fork
+};
+
+export const rangeSyncTest = {
+  network: "mainnet" as const,
+  startSlot: 3766816, // Post altair, first slot in epoch 117713
+  endSlot: 3766847, // 3766816 + 31, all blocks in same epoch
+};

package/src/testUtils/state.ts

@@ -0,0 +1,110 @@
+import {ChainForkConfig, createBeaconConfig} from "@lodestar/config";
+import {config, config as minimalConfig} from "@lodestar/config/default";
+import {
+  EPOCHS_PER_HISTORICAL_VECTOR,
+  EPOCHS_PER_SLASHINGS_VECTOR,
+  GENESIS_EPOCH,
+  GENESIS_SLOT,
+  SLOTS_PER_HISTORICAL_ROOT,
+} from "@lodestar/params";
+import {phase0, ssz} from "@lodestar/types";
+import {EpochCacheOpts} from "../cache/epochCache.js";
+import {BeaconStateCache} from "../cache/stateCache.js";
+import {ZERO_HASH} from "../constants/index.js";
+import {
+  BeaconStateAllForks,
+  BeaconStatePhase0,
+  CachedBeaconStateAllForks,
+  createCachedBeaconState,
+  createPubkeyCache,
+} from "../index.js";
+import {newZeroedArray} from "../util/index.js";
+
+/**
+ * Copy of BeaconState, but all fields are marked optional to allow for swapping out variables as needed.
+ */
+type TestBeaconState = Partial<phase0.BeaconState>;
+
+/**
+ * Generate beaconState, by default it will use the initial state defined when the `ChainStart` log is emitted.
+ * NOTE: All fields can be overridden through `opts`.
+ * @param {TestBeaconState} opts
+ * @returns {BeaconState}
+ */
+export function generateState(opts?: TestBeaconState): BeaconStatePhase0 {
+  return ssz.phase0.BeaconState.toViewDU({
+    genesisTime: Math.floor(Date.now() / 1000),
+    genesisValidatorsRoot: ZERO_HASH,
+    slot: GENESIS_SLOT,
+    fork: {
+      previousVersion: config.GENESIS_FORK_VERSION,
+      currentVersion: config.GENESIS_FORK_VERSION,
+      epoch: GENESIS_EPOCH,
+    },
+    latestBlockHeader: {
+      slot: 0,
+      proposerIndex: 0,
+      parentRoot: Buffer.alloc(32),
+      stateRoot: Buffer.alloc(32),
+      bodyRoot: ssz.phase0.BeaconBlockBody.hashTreeRoot(ssz.phase0.BeaconBlockBody.defaultValue()),
+    },
+    blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH),
+    stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, () => ZERO_HASH),
+    historicalRoots: [],
+    eth1Data: {
+      depositRoot: Buffer.alloc(32),
+      blockHash: Buffer.alloc(32),
+      depositCount: 0,
+    },
+    eth1DataVotes: [],
+    eth1DepositIndex: 0,
+    validators: [],
+    balances: [],
+    randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, () => ZERO_HASH),
+    slashings: newZeroedArray(EPOCHS_PER_SLASHINGS_VECTOR),
+    previousEpochAttestations: [],
+    currentEpochAttestations: [],
+    justificationBits: ssz.phase0.JustificationBits.defaultValue(),
+    previousJustifiedCheckpoint: {
+      epoch: GENESIS_EPOCH,
+      root: ZERO_HASH,
+    },
+    currentJustifiedCheckpoint: {
+      epoch: GENESIS_EPOCH,
+      root: ZERO_HASH,
+    },
+    finalizedCheckpoint: {
+      epoch: GENESIS_EPOCH,
+      root: ZERO_HASH,
+    },
+    ...opts,
+  });
+}
+
+export function generateCachedState(
+  config: ChainForkConfig = minimalConfig,
+  opts: TestBeaconState = {}
+): CachedBeaconStateAllForks {
+  const state = generateState(opts);
+  return createCachedBeaconState(state, {
+    config: createBeaconConfig(config, state.genesisValidatorsRoot),
+    // This is a test state, there's no need to have a global shared cache of keys
+    pubkeyCache: createPubkeyCache(),
+  });
+}
+
+export function createCachedBeaconStateTest<T extends BeaconStateAllForks>(
+  state: T,
+  configCustom: ChainForkConfig = config,
+  opts?: EpochCacheOpts
+): T & BeaconStateCache {
+  return createCachedBeaconState<T>(
+    state,
+    {
+      config: createBeaconConfig(configCustom, state.genesisValidatorsRoot),
+      // This is a test state, there's no need to have a global shared cache of keys
+      pubkeyCache: createPubkeyCache(),
+    },
+    opts
+  );
+}

package/src/testUtils/testFileCache.ts

@@ -0,0 +1,127 @@
+import fs from "node:fs";
+import path from "node:path";
+import {getClient} from "@lodestar/api";
+import {ChainForkConfig, createChainForkConfig} from "@lodestar/config";
+import {NetworkName, networksChainConfig} from "@lodestar/config/networks";
+import {SignedBeaconBlock} from "@lodestar/types";
+import {fetch} from "@lodestar/utils";
+import {CachedBeaconStateAllForks} from "../index.js";
+import {testCachePath} from "./cache.js";
+import {getInfuraBeaconUrl} from "./infura.js";
+import {createCachedBeaconStateTest} from "./state.js";
+
+/**
+ * Full link example:
+ * ```
+ * https://github.com/dapplion/ethereum-consensus-test-data/releases/download/v0.1.0/block_mainnet_3766821.ssz
+ * ``` */
+const TEST_FILES_BASE_URL = "https://github.com/dapplion/ethereum-consensus-test-data/releases/download/v0.1.0";
+
+/**
+ * Create a network config from known network params
+ */
+export function getNetworkConfig(network: NetworkName): ChainForkConfig {
+  const configNetwork = networksChainConfig[network];
+  return createChainForkConfig(configNetwork);
+}
+
+/**
+ * Download a state from Infura. Caches states in local fs by network and slot to only download once.
+ */
+export async function getNetworkCachedState(
+  network: NetworkName,
+  slot: number,
+  timeout?: number
+): Promise<CachedBeaconStateAllForks> {
+  const config = getNetworkConfig(network);
+  const fileId = `state_${network}_${slot}.ssz`;
+
+  const filepath = path.join(testCachePath, fileId);
+
+  if (fs.existsSync(filepath)) {
+    const stateSsz = fs.readFileSync(filepath);
+    return createCachedBeaconStateTest(config.getForkTypes(slot).BeaconState.deserializeToViewDU(stateSsz), config);
+  }
+
+  const stateSsz = await tryEach([
+    () => downloadTestFile(fileId),
+    () => {
+      const client = getClient(
+        {baseUrl: getInfuraBeaconUrl(network), globalInit: {timeoutMs: timeout ?? 300_000}},
+        {config}
+      );
+      return client.debug.getStateV2({stateId: slot}).then((r) => {
+        return r.ssz();
+      });
+    },
+  ]);
+
+  fs.writeFileSync(filepath, stateSsz);
+  return createCachedBeaconStateTest(config.getForkTypes(slot).BeaconState.deserializeToViewDU(stateSsz), config);
+}
+
+/**
+ * Download a state from Infura. Caches states in local fs by network and slot to only download once.
+ */
+export async function getNetworkCachedBlock(
+  network: NetworkName,
+  slot: number,
+  timeout?: number
+): Promise<SignedBeaconBlock> {
+  const config = getNetworkConfig(network);
+  const fileId = `block_${network}_${slot}.ssz`;
+
+  const filepath = path.join(testCachePath, fileId);
+
+  if (fs.existsSync(filepath)) {
+    const blockSsz = fs.readFileSync(filepath);
+    return config.getForkTypes(slot).SignedBeaconBlock.deserialize(blockSsz);
+  }
+
+  const blockSsz = await tryEach([
+    () => downloadTestFile(fileId),
+    async () => {
+      const client = getClient(
+        {baseUrl: getInfuraBeaconUrl(network), globalInit: {timeoutMs: timeout ?? 300_000}},
+        {config}
+      );
+
+      return (await client.beacon.getBlockV2({blockId: slot})).ssz();
+    },
+  ]);
+
+  fs.writeFileSync(filepath, blockSsz);
+  return config.getForkTypes(slot).SignedBeaconBlock.deserialize(blockSsz);
+}
+
+async function downloadTestFile(fileId: string): Promise<Uint8Array> {
+  const fileUrl = `${TEST_FILES_BASE_URL}/${fileId}`;
+  // biome-ignore lint/suspicious/noConsole: We explicity need to log to console
+  console.log(`Downloading file ${fileUrl}`);
+
+  try {
+    const res = await fetch(fileUrl);
+    if (!res.ok) {
+      throw new Error(`Error downloading ${fileUrl}: ${res.status} ${res.statusText}`);
+    }
+    return new Uint8Array(await res.arrayBuffer());
+  } catch (e) {
+    const error = e as Error;
+    error.message = `Error downloading ${fileUrl}: ${error.message}`;
+    throw error;
+  }
+}
+
+async function tryEach<T>(promises: (() => Promise<T>)[]): Promise<T> {
+  const errors: Error[] = [];
+
+  for (let i = 0; i < promises.length; i++) {
+    try {
+      return await promises[i]();
+    } catch (e) {
+      errors.push(e as Error);
+    }
+  }
+
+  throw Error(errors.map((e, i) => `Error[${i}] ${e.message}`).join("\n"));
+}