@lodestar/state-transition 1.42.0-dev.4411584fd8 → 1.42.0-dev.4e640bd2f7

package/src/stateView/interface.ts

@@ -1,7 +1,8 @@
 import {CompactMultiProof} from "@chainsafe/persistent-merkle-tree";
-import {ByteViews} from "@chainsafe/ssz";
+import {BitArray, ByteViews} from "@chainsafe/ssz";
 import {
   BeaconBlock,
+  BeaconState,
   BlindedBeaconBlock,
   BuilderIndex,
   Bytes32,
@@ -23,6 +24,7 @@ import {
   rewards,
 } from "@lodestar/types";
 import {Checkpoint, Fork} from "@lodestar/types/phase0";
+import {ProcessExecutionPayloadEnvelopeOpts} from "../block/processExecutionPayloadEnvelope.js";
 import {VoluntaryExitValidity} from "../block/processVoluntaryExit.js";
 import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js";
 import {EpochTransitionCacheOpts} from "../cache/epochTransitionCache.js";
@@ -92,11 +94,11 @@ export interface IBeaconStateView {
   proposerLookahead: fulu.ProposerLookahead;
 
   // gloas
-  executionPayloadAvailability: boolean[];
+  executionPayloadAvailability: BitArray;
   latestExecutionPayloadBid: ExecutionPayloadBid;
   getBuilder(index: BuilderIndex): gloas.Builder;
   canBuilderCoverBid(builderIndex: BuilderIndex, bidAmount: number): boolean;
-  validatorPTCCommitteeIndex(validatorIndex: ValidatorIndex, slot: Slot): number;
+  getIndexInPayloadTimelinessCommittee(validatorIndex: ValidatorIndex, slot: Slot): number;
 
   // Shuffling and committees
   getShufflingAtEpoch(epoch: Epoch): EpochShuffling;
@@ -109,12 +111,11 @@ export interface IBeaconStateView {
   getCurrentShuffling(): EpochShuffling;
   getNextShuffling(): EpochShuffling;
 
-  // utils: proposers, anchor checkpoint
+  // Proposer shuffling
   previousProposers: ValidatorIndex[] | null;
   currentProposers: ValidatorIndex[];
   nextProposers: ValidatorIndex[];
   getBeaconProposer(slot: Slot): ValidatorIndex;
-  computeAnchorCheckpoint(): {checkpoint: phase0.Checkpoint; blockHeader: phase0.BeaconBlockHeader};
 
   // Sync committees
   currentSyncCommittee: altair.SyncCommittee;
@@ -122,6 +123,8 @@ export interface IBeaconStateView {
   currentSyncCommitteeIndexed: SyncCommitteeCache;
   syncProposerReward: number;
   getIndexedSyncCommitteeAtEpoch(epoch: Epoch): SyncCommitteeCache;
+  /** Get indexed sync committee with slot+1 offset for duty lookups */
+  getIndexedSyncCommittee(slot: Slot): SyncCommitteeCache;
 
   // Validators and balances
   effectiveBalanceIncrements: EffectiveBalanceIncrements;
@@ -148,6 +151,7 @@ export interface IBeaconStateView {
     expectedWithdrawals: capella.Withdrawal[];
     processedBuilderWithdrawalsCount: number;
     processedPartialWithdrawalsCount: number;
+    processedBuildersSweepCount: number;
     processedValidatorSweepCount: number;
   };
 
@@ -179,6 +183,7 @@ export interface IBeaconStateView {
     justifiedCheckpoint: phase0.Checkpoint;
     finalizedCheckpoint: phase0.Checkpoint;
   };
+  computeAnchorCheckpoint(): {checkpoint: phase0.Checkpoint; blockHeader: phase0.BeaconBlockHeader};
 
   // this is for backward compatible
   clonedCount: number;
@@ -189,6 +194,7 @@ export interface IBeaconStateView {
 
   // Serialization
   loadOtherState(stateBytes: Uint8Array, seedValidatorsBytes?: Uint8Array): IBeaconStateView;
+  toValue(): BeaconState;
   serialize(): Uint8Array;
   serializedSize(): number;
   serializeToBytes(output: ByteViews, offset: number): number;
@@ -209,5 +215,8 @@ export interface IBeaconStateView {
     epochTransitionCacheOpts?: EpochTransitionCacheOpts & {dontTransferCache?: boolean},
     modules?: StateTransitionModules
   ): IBeaconStateView;
-  processExecutionPayloadEnvelope(signedEnvelope: gloas.SignedExecutionPayloadEnvelope, verify: boolean): void;
+  processExecutionPayloadEnvelope(
+    signedEnvelope: gloas.SignedExecutionPayloadEnvelope,
+    opts?: ProcessExecutionPayloadEnvelopeOpts
+  ): IBeaconStateView;
 }

package/src/stateView/stateViewFactory.ts

@@ -0,0 +1,78 @@
+import {BeaconConfig} from "@lodestar/config";
+import {PubkeyCache, createPubkeyCache} from "../cache/pubkeyCache.js";
+import {createCachedBeaconState} from "../cache/stateCache.js";
+import {BeaconStateAllForks} from "../cache/types.js";
+import {getStateTypeFromBytes} from "../util/sszBytes.js";
+import {BeaconStateView} from "./beaconStateView.js";
+import {IBeaconStateView} from "./interface.js";
+
+// ---- createBeaconStateView (startup path) ----
+
+type NodeJSOpts = {
+  useNative: false;
+  anchorState: BeaconStateAllForks;
+  config: BeaconConfig;
+  pubkeyCache: PubkeyCache;
+};
+
+type NativeOpts = {
+  useNative: true;
+  stateBytes: Uint8Array;
+};
+
+/**
+ * Create a BeaconStateView from a pre-deserialized state. Used at node startup.
+ *
+ * The caller is responsible for creating and populating `pubkeyCache` (it is also
+ * passed separately to BeaconNode.init, so it must live outside this factory).
+ *
+ * Set `useNative: true` to use the native (Zig) implementation once available.
+ */
+export function createBeaconStateView(opts: NodeJSOpts | NativeOpts): IBeaconStateView {
+  if (opts.useNative) {
+    throw new Error("Native (Zig) BeaconStateView not yet implemented");
+  }
+  const {anchorState, config, pubkeyCache} = opts;
+  const cachedState = createCachedBeaconState(anchorState, {config, pubkeyCache}, {skipSyncPubkeys: true});
+  return new BeaconStateView(cachedState);
+}
+
+// ---- createBeaconStateViewForHistoricalRegen (regen path) ----
+
+// Reused across all historical state regen calls in the worker thread
+const pubkeyCacheRegen = createPubkeyCache();
+
+function syncPubkeyCache(state: BeaconStateAllForks, pubkeyCache: PubkeyCache): void {
+  const newCount = state.validators.length;
+  for (let i = pubkeyCache.size; i < newCount; i++) {
+    const pubkey = state.validators.getReadonly(i).pubkey;
+    pubkeyCache.set(i, pubkey);
+  }
+}
+
+type RegenNodeJSOpts = {
+  useNative: false;
+  config: BeaconConfig;
+  stateBytes: Uint8Array;
+};
+
+type RegenNativeOpts = {
+  useNative: true;
+  stateBytes: Uint8Array;
+};
+
+/**
+ * Create a BeaconStateView from raw SSZ bytes. Used in the historical state regen worker thread.
+ *
+ * Set `useNative: true` to use the native (Zig) implementation once available.
+ */
+export function createBeaconStateViewForHistoricalRegen(opts: RegenNodeJSOpts | RegenNativeOpts): IBeaconStateView {
+  if (opts.useNative) {
+    throw new Error("Native (Zig) BeaconStateView not yet implemented");
+  }
+  const {config, stateBytes} = opts;
+  const state = getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes);
+  syncPubkeyCache(state, pubkeyCacheRegen);
+  const cachedState = createCachedBeaconState(state, {config, pubkeyCache: pubkeyCacheRegen}, {skipSyncPubkeys: true});
+  return new BeaconStateView(cachedState);
+}

package/src/testUtils/util.ts

@@ -25,9 +25,11 @@ import {
 } from "../index.js";
 import {
   BeaconStateAltair,
+  BeaconStateElectra,
   BeaconStatePhase0,
   CachedBeaconStateAllForks,
   CachedBeaconStateAltair,
+  CachedBeaconStateElectra,
   CachedBeaconStatePhase0,
 } from "../types.js";
 import {getNextSyncCommittee} from "../util/syncCommittee.js";
@@ -41,6 +43,9 @@ let phase0SignedBlock: phase0.SignedBeaconBlock | null = null;
 let altairState: BeaconStateAltair | null = null;
 let altairCachedState23637: CachedBeaconStateAltair | null = null;
 let altairCachedState23638: CachedBeaconStateAltair | null = null;
+let electraState: BeaconStateElectra | null = null;
+let electraCachedState23637: CachedBeaconStateElectra | null = null;
+let electraCachedState23638: CachedBeaconStateElectra | null = null;
 
 /**
  * Number of validators in prater is 210000 as of May 2021
@@ -84,10 +89,10 @@ export function getSecretKeyFromIndexCached(validatorIndex: number): SecretKey {
   return sk;
 }
 
-function getPubkeyCaches({pubkeysMod}: ReturnType<typeof getPubkeys>) {
+function getPubkeyCaches({pubkeysMod}: ReturnType<typeof getPubkeys>, vc = numValidators) {
   // Manually sync pubkeys to prevent doing BLS opts 110_000 times
   const pubkeyCache = createPubkeyCache();
-  for (let i = 0; i < numValidators; i++) {
+  for (let i = 0; i < vc; i++) {
     const pubkey = pubkeysMod[i % keypairsMod];
     pubkeyCache.set(i, pubkey);
   }
@@ -206,29 +211,80 @@ export function generatePerfTestCachedStateAltair(opts?: {
   goBackOneSlot: boolean;
   vc?: number;
 }): CachedBeaconStateAltair {
-  const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(opts?.vc);
-  const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj});
+  const vc = opts?.vc ?? numValidators;
+  const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(vc);
+  const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}, vc);
 
   const altairConfig = createChainForkConfig({ALTAIR_FORK_EPOCH: 0});
 
   const origState = generatePerformanceStateAltair(pubkeys);
 
-  if (!altairCachedState23637) {
+  // For non-default vc, generate fresh without caching to avoid accumulating large states in memory
+  const isDefaultVc = vc === numValidators;
+  let cachedState23637 = isDefaultVc ? altairCachedState23637 : null;
+  if (!cachedState23637) {
     const state = origState.clone();
     state.slot -= 1;
-    altairCachedState23637 = createCachedBeaconState(state, {
+    cachedState23637 = createCachedBeaconState(state, {
       config: createBeaconConfig(altairConfig, state.genesisValidatorsRoot),
       pubkeyCache,
     });
+    if (isDefaultVc) altairCachedState23637 = cachedState23637;
+  }
+
+  let cachedState23638 = isDefaultVc ? altairCachedState23638 : null;
+  if (!cachedState23638) {
+    cachedState23638 = processSlots(cachedState23637, cachedState23637.slot + 1) as CachedBeaconStateAltair;
+    cachedState23638.slot += 1;
+    if (isDefaultVc) altairCachedState23638 = cachedState23638;
+  }
+
+  const resultingState = opts?.goBackOneSlot ? cachedState23637 : cachedState23638;
+
+  return resultingState.clone();
+}
+
+/**
+ * Warning: This function has side effects on the cached state
+ * The order in which the caches are populated is important and can cause stable tests to fail.
+ */
+export function generatePerfTestCachedStateElectra(opts?: {
+  goBackOneSlot: boolean;
+  vc?: number;
+}): CachedBeaconStateElectra {
+  const vc = opts?.vc ?? numValidators;
+  const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(vc);
+  const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}, vc);
+
+  const electraConfig = createChainForkConfig({
+    ALTAIR_FORK_EPOCH: 0,
+    BELLATRIX_FORK_EPOCH: 0,
+    CAPELLA_FORK_EPOCH: 0,
+    DENEB_FORK_EPOCH: 0,
+    ELECTRA_FORK_EPOCH: 0,
+  });
+
+  const origState = generatePerformanceStateElectra(pubkeys);
+
+  // For non-default vc, generate fresh without caching to avoid accumulating large states in memory
+  const isDefaultVc = vc === numValidators;
+  let cachedState23637 = isDefaultVc ? electraCachedState23637 : null;
+  if (!cachedState23637) {
+    const state = origState.clone();
+    state.slot -= 1;
+    cachedState23637 = createCachedBeaconState(state, {
+      config: createBeaconConfig(electraConfig, state.genesisValidatorsRoot),
+      pubkeyCache,
+    });
+    if (isDefaultVc) electraCachedState23637 = cachedState23637;
   }
-  if (!altairCachedState23638) {
-    altairCachedState23638 = processSlots(
-      altairCachedState23637,
-      altairCachedState23637.slot + 1
-    ) as CachedBeaconStateAltair;
-    altairCachedState23638.slot += 1;
+  let cachedState23638 = isDefaultVc ? electraCachedState23638 : null;
+  if (!cachedState23638) {
+    cachedState23638 = processSlots(cachedState23637, cachedState23637.slot + 1) as CachedBeaconStateElectra;
+    cachedState23638.slot += 1;
+    if (isDefaultVc) electraCachedState23638 = cachedState23638;
   }
-  const resultingState = opts?.goBackOneSlot ? altairCachedState23637 : altairCachedState23638;
+  const resultingState = opts?.goBackOneSlot ? cachedState23637 : cachedState23638;
 
   return resultingState.clone();
 }
@@ -237,8 +293,12 @@ export function generatePerfTestCachedStateAltair(opts?: {
  * This is generated from Medalla state 756416
  */
 export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): BeaconStateAltair {
-  if (!altairState) {
-    const pubkeys = pubkeysArg || getPubkeys().pubkeys;
+  const pubkeys = pubkeysArg || getPubkeys().pubkeys;
+  const vc = pubkeys.length;
+  const isDefaultVc = vc === numValidators;
+  // Only use cached state for default vc to avoid accumulating large states in memory
+  let cached = isDefaultVc ? altairState : null;
+  if (!cached) {
     const statePhase0 = buildPerformanceStatePhase0(pubkeys);
     const state = statePhase0 as BeaconState as BeaconState<ForkName.altair>;
 
@@ -251,27 +311,81 @@ export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): Beaco
     state.nextSyncCommittee = state.currentSyncCommittee;
 
     // Now the state is fully populated to convert to ViewDU
-    altairState = ssz.altair.BeaconState.toViewDU(state);
+    cached = ssz.altair.BeaconState.toViewDU(state);
 
     // Now set correct syncCommittees
     const epoch = computeEpochAtSlot(state.slot);
-    const activeValidatorIndices = getActiveValidatorIndices(altairState, epoch);
+    const activeValidatorIndices = getActiveValidatorIndices(cached, epoch);
 
-    const effectiveBalanceIncrements = getEffectiveBalanceIncrements(altairState);
+    const effectiveBalanceIncrements = getEffectiveBalanceIncrements(cached);
     const {syncCommittee} = getNextSyncCommittee(
       ForkSeq.altair,
-      altairState,
+      cached,
       activeValidatorIndices,
       effectiveBalanceIncrements
     );
     state.currentSyncCommittee = syncCommittee;
     state.nextSyncCommittee = syncCommittee;
 
-    altairState = ssz.altair.BeaconState.toViewDU(state);
+    cached = ssz.altair.BeaconState.toViewDU(state);
     // cache roots
-    altairState.hashTreeRoot();
+    cached.hashTreeRoot();
+    if (isDefaultVc) altairState = cached;
+  }
+  return cached.clone();
+}
+
+/**
+ * This is generated from the same performance state as Altair, upgraded to Electra fields.
+ */
+export function generatePerformanceStateElectra(pubkeysArg?: Uint8Array[]): BeaconStateElectra {
+  const pubkeys = pubkeysArg || getPubkeys().pubkeys;
+  const vc = pubkeys.length;
+  const isDefaultVc = vc === numValidators;
+  // Only use cached state for default vc to avoid accumulating large states in memory
+  let cached = isDefaultVc ? electraState : null;
+  if (!cached) {
+    const electraConfig = createChainForkConfig({
+      ALTAIR_FORK_EPOCH: 0,
+      BELLATRIX_FORK_EPOCH: 0,
+      CAPELLA_FORK_EPOCH: 0,
+      DENEB_FORK_EPOCH: 0,
+      ELECTRA_FORK_EPOCH: 0,
+    });
+    const state = ssz.electra.BeaconState.defaultValue();
+
+    Object.assign(state, buildPerformanceStatePhase0(pubkeys));
+
+    state.fork.previousVersion = electraConfig.DENEB_FORK_VERSION;
+    state.fork.currentVersion = electraConfig.ELECTRA_FORK_VERSION;
+    state.fork.epoch = electraConfig.ELECTRA_FORK_EPOCH;
+    state.previousEpochParticipation = newFilledArray(pubkeys.length, 0b111);
+    state.currentEpochParticipation = state.previousEpochParticipation;
+    state.inactivityScores = Array.from({length: pubkeys.length}, (_, i) => i % 2);
+    state.currentSyncCommittee = ssz.altair.SyncCommittee.defaultValue();
+    state.nextSyncCommittee = state.currentSyncCommittee;
+    state.latestExecutionPayloadHeader = ssz.electra.ExecutionPayloadHeader.defaultValue();
+    state.depositRequestsStartIndex = 2023n;
+
+    cached = ssz.electra.BeaconState.toViewDU(state);
+
+    const epoch = computeEpochAtSlot(state.slot);
+    const activeValidatorIndices = getActiveValidatorIndices(cached, epoch);
+    const effectiveBalanceIncrements = getEffectiveBalanceIncrements(cached);
+    const {syncCommittee} = getNextSyncCommittee(
+      ForkSeq.electra,
+      cached,
+      activeValidatorIndices,
+      effectiveBalanceIncrements
+    );
+    state.currentSyncCommittee = syncCommittee;
+    state.nextSyncCommittee = syncCommittee;
+
+    cached = ssz.electra.BeaconState.toViewDU(state);
+    cached.hashTreeRoot();
+    if (isDefaultVc) electraState = cached;
   }
-  return altairState.clone();
+  return cached.clone();
 }
 
 /**
@@ -427,3 +541,25 @@ export function generateTestCachedBeaconStateOnlyValidators({
     {skipSyncPubkeys: true}
   );
 }
+
+/**
+ * Release all cached perf states to free memory.
+ * Only call this before memory-intensive benchmarks that need to allocate
+ * very large states (e.g. loadState with 1.5M validators) where the
+ * accumulated singletons would cause OOM.
+ *
+ * WARNING: Do NOT call this in epoch/block benchmark afterAll blocks —
+ * destroying singletons mid-suite causes GC storms that corrupt measurements.
+ */
+export function clearPerfStateCache(): void {
+  phase0State = null;
+  phase0CachedState23637 = null;
+  phase0CachedState23638 = null;
+  phase0SignedBlock = null;
+  altairState = null;
+  altairCachedState23637 = null;
+  altairCachedState23638 = null;
+  electraState = null;
+  electraCachedState23637 = null;
+  electraCachedState23638 = null;
+}

package/src/util/rootCache.ts

@@ -1,6 +1,5 @@
 import {Epoch, Root, Slot, phase0} from "@lodestar/types";
-import {CachedBeaconStateAllForks} from "../types.js";
-import {getBlockRoot, getBlockRootAtSlot} from "./blockRoot.js";
+import {IBeaconStateView} from "../stateView/interface.js";
 
 /**
  * Cache to prevent accessing the state tree to fetch block roots repeteadly.
@@ -12,7 +11,7 @@ export class RootCache {
   private readonly blockRootEpochCache = new Map<Epoch, Root>();
   private readonly blockRootSlotCache = new Map<Slot, Root>();
 
-  constructor(private readonly state: CachedBeaconStateAllForks) {
+  constructor(private readonly state: IBeaconStateView) {
     this.currentJustifiedCheckpoint = state.currentJustifiedCheckpoint;
     this.previousJustifiedCheckpoint = state.previousJustifiedCheckpoint;
   }
@@ -20,7 +19,7 @@ export class RootCache {
   getBlockRoot(epoch: Epoch): Root {
     let root = this.blockRootEpochCache.get(epoch);
     if (!root) {
-      root = getBlockRoot(this.state, epoch);
+      root = this.state.getBlockRootAtEpoch(epoch);
       this.blockRootEpochCache.set(epoch, root);
     }
     return root;
@@ -29,7 +28,7 @@ export class RootCache {
   getBlockRootAtSlot(slot: Slot): Root {
     let root = this.blockRootSlotCache.get(slot);
     if (!root) {
-      root = getBlockRootAtSlot(this.state, slot);
+      root = this.state.getBlockRootAtSlot(slot);
       this.blockRootSlotCache.set(slot, root);
     }
     return root;

package/src/util/shuffling.ts

@@ -11,6 +11,7 @@ import {
 } from "@lodestar/types";
 import {LodestarError} from "@lodestar/utils";
 import {CachedBeaconStateAllForks} from "../cache/stateCache.js";
+import {IBeaconStateView} from "../stateView/interface.js";
 import {getBlockRootAtSlot} from "./blockRoot.js";
 import {computeStartSlotAtEpoch} from "./epoch.js";
 import {EpochShuffling} from "./epochShuffling.js";
@@ -22,21 +23,21 @@ import {EpochShuffling} from "./epochShuffling.js";
  * Returns `null` on the one-off scenario where the genesis block decides its own shuffling.
  * It should be set to the latest block applied to this `state` or the genesis block root.
  */
-export function proposerShufflingDecisionRoot(fork: ForkName, state: CachedBeaconStateAllForks): Root | null {
+export function proposerShufflingDecisionRoot(fork: ForkName, state: IBeaconStateView): Root | null {
   const decisionSlot = proposerShufflingDecisionSlot(fork, state);
   if (state.slot === decisionSlot) {
     return null;
   }
-  return getBlockRootAtSlot(state, decisionSlot);
+  return state.getBlockRootAtSlot(decisionSlot);
 }
 
 /**
  * Returns the slot at which the proposer shuffling was decided. The block root at this slot
  * can be used to key the proposer shuffling for the current epoch.
  */
-function proposerShufflingDecisionSlot(fork: ForkName, state: CachedBeaconStateAllForks): Slot {
+function proposerShufflingDecisionSlot(fork: ForkName, state: IBeaconStateView): Slot {
   // After fulu, the decision slot is in previous epoch due to deterministic proposer lookahead
-  const epoch = isForkPostFulu(fork) ? state.epochCtx.epoch - 1 : state.epochCtx.epoch;
+  const epoch = isForkPostFulu(fork) ? state.epoch - 1 : state.epoch;
   const startSlot = computeStartSlotAtEpoch(epoch);
   return Math.max(startSlot - 1, 0);
 }