@lodestar/state-transition 1.42.0-dev.1d50253953 → 1.42.0-dev.1e596a7422

package/src/stateView/stateViewFactory.ts

@@ -0,0 +1,78 @@
+import {BeaconConfig} from "@lodestar/config";
+import {PubkeyCache, createPubkeyCache} from "../cache/pubkeyCache.js";
+import {createCachedBeaconState} from "../cache/stateCache.js";
+import {BeaconStateAllForks} from "../cache/types.js";
+import {getStateTypeFromBytes} from "../util/sszBytes.js";
+import {BeaconStateView} from "./beaconStateView.js";
+import {IBeaconStateView} from "./interface.js";
+
+// ---- createBeaconStateView (startup path) ----
+
+type NodeJSOpts = {
+  useNative: false;
+  anchorState: BeaconStateAllForks;
+  config: BeaconConfig;
+  pubkeyCache: PubkeyCache;
+};
+
+type NativeOpts = {
+  useNative: true;
+  stateBytes: Uint8Array;
+};
+
+/**
+ * Create a BeaconStateView from a pre-deserialized state. Used at node startup.
+ *
+ * The caller is responsible for creating and populating `pubkeyCache` (it is also
+ * passed separately to BeaconNode.init, so it must live outside this factory).
+ *
+ * Set `useNative: true` to use the native (Zig) implementation once available.
+ */
+export function createBeaconStateView(opts: NodeJSOpts | NativeOpts): IBeaconStateView {
+  if (opts.useNative) {
+    throw new Error("Native (Zig) BeaconStateView not yet implemented");
+  }
+  const {anchorState, config, pubkeyCache} = opts;
+  const cachedState = createCachedBeaconState(anchorState, {config, pubkeyCache}, {skipSyncPubkeys: true});
+  return new BeaconStateView(cachedState);
+}
+
+// ---- createBeaconStateViewForHistoricalRegen (regen path) ----
+
+// Reused across all historical state regen calls in the worker thread
+const pubkeyCacheRegen = createPubkeyCache();
+
+function syncPubkeyCache(state: BeaconStateAllForks, pubkeyCache: PubkeyCache): void {
+  const newCount = state.validators.length;
+  for (let i = pubkeyCache.size; i < newCount; i++) {
+    const pubkey = state.validators.getReadonly(i).pubkey;
+    pubkeyCache.set(i, pubkey);
+  }
+}
+
+type RegenNodeJSOpts = {
+  useNative: false;
+  config: BeaconConfig;
+  stateBytes: Uint8Array;
+};
+
+type RegenNativeOpts = {
+  useNative: true;
+  stateBytes: Uint8Array;
+};
+
+/**
+ * Create a BeaconStateView from raw SSZ bytes. Used in the historical state regen worker thread.
+ *
+ * Set `useNative: true` to use the native (Zig) implementation once available.
+ */
+export function createBeaconStateViewForHistoricalRegen(opts: RegenNodeJSOpts | RegenNativeOpts): IBeaconStateView {
+  if (opts.useNative) {
+    throw new Error("Native (Zig) BeaconStateView not yet implemented");
+  }
+  const {config, stateBytes} = opts;
+  const state = getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes);
+  syncPubkeyCache(state, pubkeyCacheRegen);
+  const cachedState = createCachedBeaconState(state, {config, pubkeyCache: pubkeyCacheRegen}, {skipSyncPubkeys: true});
+  return new BeaconStateView(cachedState);
+}

package/src/testUtils/cache.ts

@@ -5,4 +5,4 @@ import {fileURLToPath} from "node:url";
 // Solutions: https://stackoverflow.com/questions/46745014/alternative-for-dirname-in-node-js-when-using-es6-modules
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
-export const testCachePath = path.join(__dirname, "../../test/test-cache");
+export const testCachePath = path.join(__dirname, "../../test-cache");

package/src/testUtils/util.ts

@@ -25,9 +25,11 @@ import {
 } from "../index.js";
 import {
   BeaconStateAltair,
+  BeaconStateElectra,
   BeaconStatePhase0,
   CachedBeaconStateAllForks,
   CachedBeaconStateAltair,
+  CachedBeaconStateElectra,
   CachedBeaconStatePhase0,
 } from "../types.js";
 import {getNextSyncCommittee} from "../util/syncCommittee.js";
@@ -41,6 +43,9 @@ let phase0SignedBlock: phase0.SignedBeaconBlock | null = null;
 let altairState: BeaconStateAltair | null = null;
 let altairCachedState23637: CachedBeaconStateAltair | null = null;
 let altairCachedState23638: CachedBeaconStateAltair | null = null;
+let electraState: BeaconStateElectra | null = null;
+let electraCachedState23637: CachedBeaconStateElectra | null = null;
+let electraCachedState23638: CachedBeaconStateElectra | null = null;
 
 /**
  * Number of validators in prater is 210000 as of May 2021
@@ -84,10 +89,10 @@ export function getSecretKeyFromIndexCached(validatorIndex: number): SecretKey {
   return sk;
 }
 
-function getPubkeyCaches({pubkeysMod}: ReturnType<typeof getPubkeys>) {
+function getPubkeyCaches({pubkeysMod}: ReturnType<typeof getPubkeys>, vc = numValidators) {
   // Manually sync pubkeys to prevent doing BLS opts 110_000 times
   const pubkeyCache = createPubkeyCache();
-  for (let i = 0; i < numValidators; i++) {
+  for (let i = 0; i < vc; i++) {
     const pubkey = pubkeysMod[i % keypairsMod];
     pubkeyCache.set(i, pubkey);
   }
@@ -206,29 +211,80 @@ export function generatePerfTestCachedStateAltair(opts?: {
   goBackOneSlot: boolean;
   vc?: number;
 }): CachedBeaconStateAltair {
-  const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(opts?.vc);
-  const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj});
+  const vc = opts?.vc ?? numValidators;
+  const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(vc);
+  const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}, vc);
 
   const altairConfig = createChainForkConfig({ALTAIR_FORK_EPOCH: 0});
 
   const origState = generatePerformanceStateAltair(pubkeys);
 
-  if (!altairCachedState23637) {
+  // For non-default vc, generate fresh without caching to avoid accumulating large states in memory
+  const isDefaultVc = vc === numValidators;
+  let cachedState23637 = isDefaultVc ? altairCachedState23637 : null;
+  if (!cachedState23637) {
     const state = origState.clone();
     state.slot -= 1;
-    altairCachedState23637 = createCachedBeaconState(state, {
+    cachedState23637 = createCachedBeaconState(state, {
       config: createBeaconConfig(altairConfig, state.genesisValidatorsRoot),
       pubkeyCache,
     });
+    if (isDefaultVc) altairCachedState23637 = cachedState23637;
+  }
+
+  let cachedState23638 = isDefaultVc ? altairCachedState23638 : null;
+  if (!cachedState23638) {
+    cachedState23638 = processSlots(cachedState23637, cachedState23637.slot + 1) as CachedBeaconStateAltair;
+    cachedState23638.slot += 1;
+    if (isDefaultVc) altairCachedState23638 = cachedState23638;
+  }
+
+  const resultingState = opts?.goBackOneSlot ? cachedState23637 : cachedState23638;
+
+  return resultingState.clone();
+}
+
+/**
+ * Warning: This function has side effects on the cached state
+ * The order in which the caches are populated is important and can cause stable tests to fail.
+ */
+export function generatePerfTestCachedStateElectra(opts?: {
+  goBackOneSlot: boolean;
+  vc?: number;
+}): CachedBeaconStateElectra {
+  const vc = opts?.vc ?? numValidators;
+  const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(vc);
+  const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj}, vc);
+
+  const electraConfig = createChainForkConfig({
+    ALTAIR_FORK_EPOCH: 0,
+    BELLATRIX_FORK_EPOCH: 0,
+    CAPELLA_FORK_EPOCH: 0,
+    DENEB_FORK_EPOCH: 0,
+    ELECTRA_FORK_EPOCH: 0,
+  });
+
+  const origState = generatePerformanceStateElectra(pubkeys);
+
+  // For non-default vc, generate fresh without caching to avoid accumulating large states in memory
+  const isDefaultVc = vc === numValidators;
+  let cachedState23637 = isDefaultVc ? electraCachedState23637 : null;
+  if (!cachedState23637) {
+    const state = origState.clone();
+    state.slot -= 1;
+    cachedState23637 = createCachedBeaconState(state, {
+      config: createBeaconConfig(electraConfig, state.genesisValidatorsRoot),
+      pubkeyCache,
+    });
+    if (isDefaultVc) electraCachedState23637 = cachedState23637;
   }
-  if (!altairCachedState23638) {
-    altairCachedState23638 = processSlots(
-      altairCachedState23637,
-      altairCachedState23637.slot + 1
-    ) as CachedBeaconStateAltair;
-    altairCachedState23638.slot += 1;
+  let cachedState23638 = isDefaultVc ? electraCachedState23638 : null;
+  if (!cachedState23638) {
+    cachedState23638 = processSlots(cachedState23637, cachedState23637.slot + 1) as CachedBeaconStateElectra;
+    cachedState23638.slot += 1;
+    if (isDefaultVc) electraCachedState23638 = cachedState23638;
   }
-  const resultingState = opts?.goBackOneSlot ? altairCachedState23637 : altairCachedState23638;
+  const resultingState = opts?.goBackOneSlot ? cachedState23637 : cachedState23638;
 
   return resultingState.clone();
 }
@@ -237,8 +293,12 @@ export function generatePerfTestCachedStateAltair(opts?: {
  * This is generated from Medalla state 756416
  */
 export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): BeaconStateAltair {
-  if (!altairState) {
-    const pubkeys = pubkeysArg || getPubkeys().pubkeys;
+  const pubkeys = pubkeysArg || getPubkeys().pubkeys;
+  const vc = pubkeys.length;
+  const isDefaultVc = vc === numValidators;
+  // Only use cached state for default vc to avoid accumulating large states in memory
+  let cached = isDefaultVc ? altairState : null;
+  if (!cached) {
     const statePhase0 = buildPerformanceStatePhase0(pubkeys);
     const state = statePhase0 as BeaconState as BeaconState<ForkName.altair>;
 
@@ -251,27 +311,81 @@ export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): Beaco
     state.nextSyncCommittee = state.currentSyncCommittee;
 
     // Now the state is fully populated to convert to ViewDU
-    altairState = ssz.altair.BeaconState.toViewDU(state);
+    cached = ssz.altair.BeaconState.toViewDU(state);
 
     // Now set correct syncCommittees
     const epoch = computeEpochAtSlot(state.slot);
-    const activeValidatorIndices = getActiveValidatorIndices(altairState, epoch);
+    const activeValidatorIndices = getActiveValidatorIndices(cached, epoch);
 
-    const effectiveBalanceIncrements = getEffectiveBalanceIncrements(altairState);
+    const effectiveBalanceIncrements = getEffectiveBalanceIncrements(cached);
     const {syncCommittee} = getNextSyncCommittee(
       ForkSeq.altair,
-      altairState,
+      cached,
       activeValidatorIndices,
       effectiveBalanceIncrements
     );
     state.currentSyncCommittee = syncCommittee;
     state.nextSyncCommittee = syncCommittee;
 
-    altairState = ssz.altair.BeaconState.toViewDU(state);
+    cached = ssz.altair.BeaconState.toViewDU(state);
     // cache roots
-    altairState.hashTreeRoot();
+    cached.hashTreeRoot();
+    if (isDefaultVc) altairState = cached;
+  }
+  return cached.clone();
+}
+
+/**
+ * This is generated from the same performance state as Altair, upgraded to Electra fields.
+ */
+export function generatePerformanceStateElectra(pubkeysArg?: Uint8Array[]): BeaconStateElectra {
+  const pubkeys = pubkeysArg || getPubkeys().pubkeys;
+  const vc = pubkeys.length;
+  const isDefaultVc = vc === numValidators;
+  // Only use cached state for default vc to avoid accumulating large states in memory
+  let cached = isDefaultVc ? electraState : null;
+  if (!cached) {
+    const electraConfig = createChainForkConfig({
+      ALTAIR_FORK_EPOCH: 0,
+      BELLATRIX_FORK_EPOCH: 0,
+      CAPELLA_FORK_EPOCH: 0,
+      DENEB_FORK_EPOCH: 0,
+      ELECTRA_FORK_EPOCH: 0,
+    });
+    const state = ssz.electra.BeaconState.defaultValue();
+
+    Object.assign(state, buildPerformanceStatePhase0(pubkeys));
+
+    state.fork.previousVersion = electraConfig.DENEB_FORK_VERSION;
+    state.fork.currentVersion = electraConfig.ELECTRA_FORK_VERSION;
+    state.fork.epoch = electraConfig.ELECTRA_FORK_EPOCH;
+    state.previousEpochParticipation = newFilledArray(pubkeys.length, 0b111);
+    state.currentEpochParticipation = state.previousEpochParticipation;
+    state.inactivityScores = Array.from({length: pubkeys.length}, (_, i) => i % 2);
+    state.currentSyncCommittee = ssz.altair.SyncCommittee.defaultValue();
+    state.nextSyncCommittee = state.currentSyncCommittee;
+    state.latestExecutionPayloadHeader = ssz.electra.ExecutionPayloadHeader.defaultValue();
+    state.depositRequestsStartIndex = 2023n;
+
+    cached = ssz.electra.BeaconState.toViewDU(state);
+
+    const epoch = computeEpochAtSlot(state.slot);
+    const activeValidatorIndices = getActiveValidatorIndices(cached, epoch);
+    const effectiveBalanceIncrements = getEffectiveBalanceIncrements(cached);
+    const {syncCommittee} = getNextSyncCommittee(
+      ForkSeq.electra,
+      cached,
+      activeValidatorIndices,
+      effectiveBalanceIncrements
+    );
+    state.currentSyncCommittee = syncCommittee;
+    state.nextSyncCommittee = syncCommittee;
+
+    cached = ssz.electra.BeaconState.toViewDU(state);
+    cached.hashTreeRoot();
+    if (isDefaultVc) electraState = cached;
   }
-  return altairState.clone();
+  return cached.clone();
 }
 
 /**

package/src/util/gloas.ts

@@ -11,6 +11,7 @@ import {
 import {BuilderIndex, Epoch, ValidatorIndex, gloas} from "@lodestar/types";
 import {AttestationData} from "@lodestar/types/phase0";
 import {byteArrayEquals} from "@lodestar/utils";
+import {IBeaconStateViewGloas} from "../stateView/interface.js";
 import {CachedBeaconStateGloas} from "../types.js";
 import {getBlockRootAtSlot} from "./blockRoot.js";
 import {computeEpochAtSlot} from "./epoch.js";
@@ -167,6 +168,6 @@ export function isAttestationSameSlotRootCache(rootCache: RootCache, data: Attes
   return isMatchingBlockRoot && isCurrentBlockRoot;
 }
 
-export function isParentBlockFull(state: CachedBeaconStateGloas): boolean {
+export function isParentBlockFull(state: CachedBeaconStateGloas | IBeaconStateViewGloas): boolean {
   return byteArrayEquals(state.latestExecutionPayloadBid.blockHash, state.latestBlockHash);
 }

package/src/util/rootCache.ts

@@ -1,6 +1,5 @@
 import {Epoch, Root, Slot, phase0} from "@lodestar/types";
-import {CachedBeaconStateAllForks} from "../types.js";
-import {getBlockRoot, getBlockRootAtSlot} from "./blockRoot.js";
+import {IBeaconStateView} from "../stateView/interface.js";
 
 /**
  * Cache to prevent accessing the state tree to fetch block roots repeteadly.
@@ -12,7 +11,7 @@ export class RootCache {
   private readonly blockRootEpochCache = new Map<Epoch, Root>();
   private readonly blockRootSlotCache = new Map<Slot, Root>();
 
-  constructor(private readonly state: CachedBeaconStateAllForks) {
+  constructor(private readonly state: IBeaconStateView) {
     this.currentJustifiedCheckpoint = state.currentJustifiedCheckpoint;
     this.previousJustifiedCheckpoint = state.previousJustifiedCheckpoint;
   }
@@ -20,7 +19,7 @@ export class RootCache {
   getBlockRoot(epoch: Epoch): Root {
     let root = this.blockRootEpochCache.get(epoch);
     if (!root) {
-      root = getBlockRoot(this.state, epoch);
+      root = this.state.getBlockRootAtEpoch(epoch);
       this.blockRootEpochCache.set(epoch, root);
     }
     return root;
@@ -29,7 +28,7 @@ export class RootCache {
   getBlockRootAtSlot(slot: Slot): Root {
     let root = this.blockRootSlotCache.get(slot);
     if (!root) {
-      root = getBlockRootAtSlot(this.state, slot);
+      root = this.state.getBlockRootAtSlot(slot);
       this.blockRootSlotCache.set(slot, root);
     }
     return root;

package/src/util/shuffling.ts

@@ -11,6 +11,7 @@ import {
 } from "@lodestar/types";
 import {LodestarError} from "@lodestar/utils";
 import {CachedBeaconStateAllForks} from "../cache/stateCache.js";
+import {IBeaconStateView} from "../stateView/interface.js";
 import {getBlockRootAtSlot} from "./blockRoot.js";
 import {computeStartSlotAtEpoch} from "./epoch.js";
 import {EpochShuffling} from "./epochShuffling.js";
@@ -22,21 +23,21 @@ import {EpochShuffling} from "./epochShuffling.js";
  * Returns `null` on the one-off scenario where the genesis block decides its own shuffling.
  * It should be set to the latest block applied to this `state` or the genesis block root.
  */
-export function proposerShufflingDecisionRoot(fork: ForkName, state: CachedBeaconStateAllForks): Root | null {
+export function proposerShufflingDecisionRoot(fork: ForkName, state: IBeaconStateView): Root | null {
   const decisionSlot = proposerShufflingDecisionSlot(fork, state);
   if (state.slot === decisionSlot) {
     return null;
   }
-  return getBlockRootAtSlot(state, decisionSlot);
+  return state.getBlockRootAtSlot(decisionSlot);
 }
 
 /**
  * Returns the slot at which the proposer shuffling was decided. The block root at this slot
  * can be used to key the proposer shuffling for the current epoch.
  */
-function proposerShufflingDecisionSlot(fork: ForkName, state: CachedBeaconStateAllForks): Slot {
+function proposerShufflingDecisionSlot(fork: ForkName, state: IBeaconStateView): Slot {
   // After fulu, the decision slot is in previous epoch due to deterministic proposer lookahead
-  const epoch = isForkPostFulu(fork) ? state.epochCtx.epoch - 1 : state.epochCtx.epoch;
+  const epoch = isForkPostFulu(fork) ? state.epoch - 1 : state.epoch;
   const startSlot = computeStartSlotAtEpoch(epoch);
   return Math.max(startSlot - 1, 0);
 }