@lodestar/state-transition 1.42.0-dev.5f9f015475 → 1.42.0-dev.6b6e90ca67

package/src/stateView/beaconStateView.ts

@@ -1,9 +1,10 @@
 import {CompactMultiProof, ProofType, Tree, createProof} from "@chainsafe/persistent-merkle-tree";
-import {ByteViews} from "@chainsafe/ssz";
+import {BitArray, ByteViews} from "@chainsafe/ssz";
 import {BeaconConfig} from "@lodestar/config";
 import {ForkSeq, SLOTS_PER_HISTORICAL_ROOT, isForkPostGloas} from "@lodestar/params";
 import {
   BeaconBlock,
+  BeaconState,
   BlindedBeaconBlock,
   BuilderIndex,
   Bytes32,
@@ -33,7 +34,6 @@ import {VoluntaryExitValidity, getVoluntaryExitValidity} from "../block/processV
 import {getExpectedWithdrawals} from "../block/processWithdrawals.js";
 import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js";
 import {EpochTransitionCacheOpts} from "../cache/epochTransitionCache.js";
-import {PubkeyCache, createPubkeyCache} from "../cache/pubkeyCache.js";
 import {RewardCache} from "../cache/rewardCache.js";
 import {
   CachedBeaconStateAllForks,
@@ -47,7 +47,6 @@ import {
   isStateValidatorsNodesPopulated,
 } from "../cache/stateCache.js";
 import {SyncCommitteeCache} from "../cache/syncCommitteeCache.js";
-import {BeaconStateAllForks} from "../cache/types.js";
 import {computeUnrealizedCheckpoints} from "../epoch/computeUnrealizedCheckpoints.js";
 import {getFinalizedRootProof, getSyncCommitteesWitness} from "../lightClient/proofs.js";
 import {SyncCommitteeWitness} from "../lightClient/types.js";
@@ -64,7 +63,6 @@ import {isExecutionEnabled, isExecutionStateType, isMergeTransitionComplete} fro
 import {canBuilderCoverBid} from "../util/gloas.js";
 import {loadState} from "../util/loadState/loadState.js";
 import {getRandaoMix} from "../util/seed.js";
-import {getStateTypeFromBytes} from "../util/sszBytes.js";
 import {getLatestWeakSubjectivityCheckpointEpoch} from "../util/weakSubjectivity.js";
 import {IBeaconStateView} from "./interface.js";
 
@@ -92,7 +90,7 @@ export class BeaconStateView implements IBeaconStateView {
   // fulu
   private _proposerLookahead: fulu.ProposerLookahead | null = null;
   // gloas
-  private _executionPayloadAvailability: boolean[] | null = null;
+  private _executionPayloadAvailability: BitArray | null = null;
   private _latestExecutionPayloadBid: ExecutionPayloadBid | null = null;
 
   constructor(readonly cachedState: CachedBeaconStateAllForks) {
@@ -357,15 +355,15 @@ export class BeaconStateView implements IBeaconStateView {
 
   // gloas
 
-  get executionPayloadAvailability(): boolean[] {
+  get executionPayloadAvailability(): BitArray {
     if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.gloas) {
       throw new Error("executionPayloadAvailability is not available before GLOAS");
     }
 
     if (this._executionPayloadAvailability === null) {
-      this._executionPayloadAvailability = (this.cachedState as CachedBeaconStateGloas).executionPayloadAvailability
-        .toValue()
-        .toBoolArray();
+      this._executionPayloadAvailability = (
+        this.cachedState as CachedBeaconStateGloas
+      ).executionPayloadAvailability.toValue();
     }
 
     return this._executionPayloadAvailability;
@@ -404,7 +402,7 @@ export class BeaconStateView implements IBeaconStateView {
    * Return the index of the validator in the PTC committee for the given slot.
    * return -1 if validator is not in the PTC committee for the given slot.
    */
-  validatorPTCCommitteeIndex(validatorIndex: ValidatorIndex, slot: Slot): number {
+  getIndexInPayloadTimelinessCommittee(validatorIndex: ValidatorIndex, slot: Slot): number {
     if (this.config.getForkSeq(this.cachedState.slot) < ForkSeq.gloas) {
       throw new Error("PTC committees are not supported before GLOAS");
     }
@@ -503,6 +501,10 @@ export class BeaconStateView implements IBeaconStateView {
     return this.cachedState.epochCtx.syncProposerReward;
   }
 
+  getIndexedSyncCommittee(slot: Slot): SyncCommitteeCache {
+    return this.cachedState.epochCtx.getIndexedSyncCommittee(slot);
+  }
+
   getIndexedSyncCommitteeAtEpoch(epoch: Epoch): SyncCommitteeCache {
     return this.cachedState.epochCtx.getIndexedSyncCommitteeAtEpoch(epoch);
   }
@@ -713,6 +715,10 @@ export class BeaconStateView implements IBeaconStateView {
     return new BeaconStateView(cachedState);
   }
 
+  toValue(): BeaconState {
+    return this.cachedState.toValue();
+  }
+
   serialize(): Uint8Array {
     return this.cachedState.serialize();
   }
@@ -778,41 +784,3 @@ export class BeaconStateView implements IBeaconStateView {
     return new BeaconStateView(postPayloadState);
   }
 }
-
-/**
- * Create BeaconStateView for historical state regen, no need to sync pubkey cache there.
- */
-export function createBeaconStateViewForHistoricalRegen(
-  config: BeaconConfig,
-  stateBytes: Uint8Array
-): IBeaconStateView {
-  const state = getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes);
-
-  const pubkeyCache = createPubkeyCache();
-  syncPubkeyCache(state, pubkeyCache);
-  const cachedState = createCachedBeaconState(
-    state,
-    {
-      config,
-      pubkeyCache,
-    },
-    {
-      skipSyncPubkeys: true,
-    }
-  );
-
-  return new BeaconStateView(cachedState);
-}
-
-/**
- * Populate a PubkeyIndexMap with any new entries based on a BeaconState
- */
-function syncPubkeyCache(state: BeaconStateAllForks, pubkeyCache: PubkeyCache): void {
-  // Get the validators sub tree once for all the loop
-
-  const newCount = state.validators.length;
-  for (let i = pubkeyCache.size; i < newCount; i++) {
-    const pubkey = state.validators.getReadonly(i).pubkey;
-    pubkeyCache.set(i, pubkey);
-  }
-}

package/src/stateView/index.ts

@@ -1,2 +1,3 @@
 export * from "./beaconStateView.js";
 export * from "./interface.js";
+export * from "./stateViewFactory.js";

package/src/stateView/interface.ts

@@ -1,7 +1,8 @@
 import {CompactMultiProof} from "@chainsafe/persistent-merkle-tree";
-import {ByteViews} from "@chainsafe/ssz";
+import {BitArray, ByteViews} from "@chainsafe/ssz";
 import {
   BeaconBlock,
+  BeaconState,
   BlindedBeaconBlock,
   BuilderIndex,
   Bytes32,
@@ -93,11 +94,11 @@ export interface IBeaconStateView {
   proposerLookahead: fulu.ProposerLookahead;
 
   // gloas
-  executionPayloadAvailability: boolean[];
+  executionPayloadAvailability: BitArray;
   latestExecutionPayloadBid: ExecutionPayloadBid;
   getBuilder(index: BuilderIndex): gloas.Builder;
   canBuilderCoverBid(builderIndex: BuilderIndex, bidAmount: number): boolean;
-  validatorPTCCommitteeIndex(validatorIndex: ValidatorIndex, slot: Slot): number;
+  getIndexInPayloadTimelinessCommittee(validatorIndex: ValidatorIndex, slot: Slot): number;
 
   // Shuffling and committees
   getShufflingAtEpoch(epoch: Epoch): EpochShuffling;
@@ -110,12 +111,11 @@ export interface IBeaconStateView {
   getCurrentShuffling(): EpochShuffling;
   getNextShuffling(): EpochShuffling;
 
-  // utils: proposers, anchor checkpoint
+  // Proposer shuffling
   previousProposers: ValidatorIndex[] | null;
   currentProposers: ValidatorIndex[];
   nextProposers: ValidatorIndex[];
   getBeaconProposer(slot: Slot): ValidatorIndex;
-  computeAnchorCheckpoint(): {checkpoint: phase0.Checkpoint; blockHeader: phase0.BeaconBlockHeader};
 
   // Sync committees
   currentSyncCommittee: altair.SyncCommittee;
@@ -123,6 +123,8 @@ export interface IBeaconStateView {
   currentSyncCommitteeIndexed: SyncCommitteeCache;
   syncProposerReward: number;
   getIndexedSyncCommitteeAtEpoch(epoch: Epoch): SyncCommitteeCache;
+  /** Get indexed sync committee with slot+1 offset for duty lookups */
+  getIndexedSyncCommittee(slot: Slot): SyncCommitteeCache;
 
   // Validators and balances
   effectiveBalanceIncrements: EffectiveBalanceIncrements;
@@ -149,6 +151,7 @@ export interface IBeaconStateView {
     expectedWithdrawals: capella.Withdrawal[];
     processedBuilderWithdrawalsCount: number;
     processedPartialWithdrawalsCount: number;
+    processedBuildersSweepCount: number;
     processedValidatorSweepCount: number;
   };
 
@@ -180,6 +183,7 @@ export interface IBeaconStateView {
     justifiedCheckpoint: phase0.Checkpoint;
     finalizedCheckpoint: phase0.Checkpoint;
   };
+  computeAnchorCheckpoint(): {checkpoint: phase0.Checkpoint; blockHeader: phase0.BeaconBlockHeader};
 
   // this is for backward compatible
   clonedCount: number;
@@ -190,6 +194,7 @@ export interface IBeaconStateView {
 
   // Serialization
   loadOtherState(stateBytes: Uint8Array, seedValidatorsBytes?: Uint8Array): IBeaconStateView;
+  toValue(): BeaconState;
   serialize(): Uint8Array;
   serializedSize(): number;
   serializeToBytes(output: ByteViews, offset: number): number;

package/src/stateView/stateViewFactory.ts

@@ -0,0 +1,78 @@
+import {BeaconConfig} from "@lodestar/config";
+import {PubkeyCache, createPubkeyCache} from "../cache/pubkeyCache.js";
+import {createCachedBeaconState} from "../cache/stateCache.js";
+import {BeaconStateAllForks} from "../cache/types.js";
+import {getStateTypeFromBytes} from "../util/sszBytes.js";
+import {BeaconStateView} from "./beaconStateView.js";
+import {IBeaconStateView} from "./interface.js";
+
+// ---- createBeaconStateView (startup path) ----
+
+type NodeJSOpts = {
+  useNative: false;
+  anchorState: BeaconStateAllForks;
+  config: BeaconConfig;
+  pubkeyCache: PubkeyCache;
+};
+
+type NativeOpts = {
+  useNative: true;
+  stateBytes: Uint8Array;
+};
+
+/**
+ * Create a BeaconStateView from a pre-deserialized state. Used at node startup.
+ *
+ * The caller is responsible for creating and populating `pubkeyCache` (it is also
+ * passed separately to BeaconNode.init, so it must live outside this factory).
+ *
+ * Set `useNative: true` to use the native (Zig) implementation once available.
+ */
+export function createBeaconStateView(opts: NodeJSOpts | NativeOpts): IBeaconStateView {
+  if (opts.useNative) {
+    throw new Error("Native (Zig) BeaconStateView not yet implemented");
+  }
+  const {anchorState, config, pubkeyCache} = opts;
+  const cachedState = createCachedBeaconState(anchorState, {config, pubkeyCache}, {skipSyncPubkeys: true});
+  return new BeaconStateView(cachedState);
+}
+
+// ---- createBeaconStateViewForHistoricalRegen (regen path) ----
+
+// Reused across all historical state regen calls in the worker thread
+const pubkeyCacheRegen = createPubkeyCache();
+
+function syncPubkeyCache(state: BeaconStateAllForks, pubkeyCache: PubkeyCache): void {
+  const newCount = state.validators.length;
+  for (let i = pubkeyCache.size; i < newCount; i++) {
+    const pubkey = state.validators.getReadonly(i).pubkey;
+    pubkeyCache.set(i, pubkey);
+  }
+}
+
+type RegenNodeJSOpts = {
+  useNative: false;
+  config: BeaconConfig;
+  stateBytes: Uint8Array;
+};
+
+type RegenNativeOpts = {
+  useNative: true;
+  stateBytes: Uint8Array;
+};
+
+/**
+ * Create a BeaconStateView from raw SSZ bytes. Used in the historical state regen worker thread.
+ *
+ * Set `useNative: true` to use the native (Zig) implementation once available.
+ */
+export function createBeaconStateViewForHistoricalRegen(opts: RegenNodeJSOpts | RegenNativeOpts): IBeaconStateView {
+  if (opts.useNative) {
+    throw new Error("Native (Zig) BeaconStateView not yet implemented");
+  }
+  const {config, stateBytes} = opts;
+  const state = getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes);
+  syncPubkeyCache(state, pubkeyCacheRegen);
+  const cachedState = createCachedBeaconState(state, {config, pubkeyCache: pubkeyCacheRegen}, {skipSyncPubkeys: true});
+  return new BeaconStateView(cachedState);
+}

package/src/testUtils/util.ts

@@ -25,9 +25,11 @@ import {
 } from "../index.js";
 import {
   BeaconStateAltair,
+  BeaconStateElectra,
   BeaconStatePhase0,
   CachedBeaconStateAllForks,
   CachedBeaconStateAltair,
+  CachedBeaconStateElectra,
   CachedBeaconStatePhase0,
 } from "../types.js";
 import {getNextSyncCommittee} from "../util/syncCommittee.js";
@@ -41,6 +43,9 @@ let phase0SignedBlock: phase0.SignedBeaconBlock | null = null;
 let altairState: BeaconStateAltair | null = null;
 let altairCachedState23637: CachedBeaconStateAltair | null = null;
 let altairCachedState23638: CachedBeaconStateAltair | null = null;
+let electraState: BeaconStateElectra | null = null;
+let electraCachedState23637: CachedBeaconStateElectra | null = null;
+let electraCachedState23638: CachedBeaconStateElectra | null = null;
 
 /**
  * Number of validators in prater is 210000 as of May 2021
@@ -233,6 +238,47 @@ export function generatePerfTestCachedStateAltair(opts?: {
   return resultingState.clone();
 }
 
+/**
+ * Warning: This function has side effects on the cached state
+ * The order in which the caches are populated is important and can cause stable tests to fail.
+ */
+export function generatePerfTestCachedStateElectra(opts?: {
+  goBackOneSlot: boolean;
+  vc?: number;
+}): CachedBeaconStateElectra {
+  const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(opts?.vc);
+  const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj});
+
+  const electraConfig = createChainForkConfig({
+    ALTAIR_FORK_EPOCH: 0,
+    BELLATRIX_FORK_EPOCH: 0,
+    CAPELLA_FORK_EPOCH: 0,
+    DENEB_FORK_EPOCH: 0,
+    ELECTRA_FORK_EPOCH: 0,
+  });
+
+  const origState = generatePerformanceStateElectra(pubkeys);
+
+  if (!electraCachedState23637) {
+    const state = origState.clone();
+    state.slot -= 1;
+    electraCachedState23637 = createCachedBeaconState(state, {
+      config: createBeaconConfig(electraConfig, state.genesisValidatorsRoot),
+      pubkeyCache,
+    });
+  }
+  if (!electraCachedState23638) {
+    electraCachedState23638 = processSlots(
+      electraCachedState23637,
+      electraCachedState23637.slot + 1
+    ) as CachedBeaconStateElectra;
+    electraCachedState23638.slot += 1;
+  }
+  const resultingState = opts?.goBackOneSlot ? electraCachedState23637 : electraCachedState23638;
+
+  return resultingState.clone();
+}
+
 /**
  * This is generated from Medalla state 756416
  */
@@ -274,6 +320,54 @@ export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): Beaco
   return altairState.clone();
 }
 
+/**
+ * This is generated from the same performance state as Altair, upgraded to Electra fields.
+ */
+export function generatePerformanceStateElectra(pubkeysArg?: Uint8Array[]): BeaconStateElectra {
+  if (!electraState) {
+    const pubkeys = pubkeysArg || getPubkeys().pubkeys;
+    const electraConfig = createChainForkConfig({
+      ALTAIR_FORK_EPOCH: 0,
+      BELLATRIX_FORK_EPOCH: 0,
+      CAPELLA_FORK_EPOCH: 0,
+      DENEB_FORK_EPOCH: 0,
+      ELECTRA_FORK_EPOCH: 0,
+    });
+    const state = ssz.electra.BeaconState.defaultValue();
+
+    Object.assign(state, buildPerformanceStatePhase0(pubkeys));
+
+    state.fork.previousVersion = electraConfig.DENEB_FORK_VERSION;
+    state.fork.currentVersion = electraConfig.ELECTRA_FORK_VERSION;
+    state.fork.epoch = electraConfig.ELECTRA_FORK_EPOCH;
+    state.previousEpochParticipation = newFilledArray(pubkeys.length, 0b111);
+    state.currentEpochParticipation = state.previousEpochParticipation;
+    state.inactivityScores = Array.from({length: pubkeys.length}, (_, i) => i % 2);
+    state.currentSyncCommittee = ssz.altair.SyncCommittee.defaultValue();
+    state.nextSyncCommittee = state.currentSyncCommittee;
+    state.latestExecutionPayloadHeader = ssz.electra.ExecutionPayloadHeader.defaultValue();
+    state.depositRequestsStartIndex = 2023n;
+
+    electraState = ssz.electra.BeaconState.toViewDU(state);
+
+    const epoch = computeEpochAtSlot(state.slot);
+    const activeValidatorIndices = getActiveValidatorIndices(electraState, epoch);
+    const effectiveBalanceIncrements = getEffectiveBalanceIncrements(electraState);
+    const {syncCommittee} = getNextSyncCommittee(
+      ForkSeq.electra,
+      electraState,
+      activeValidatorIndices,
+      effectiveBalanceIncrements
+    );
+    state.currentSyncCommittee = syncCommittee;
+    state.nextSyncCommittee = syncCommittee;
+
+    electraState = ssz.electra.BeaconState.toViewDU(state);
+    electraState.hashTreeRoot();
+  }
+  return electraState.clone();
+}
+
 /**
  * This is generated from Medalla block 756417
  */

package/src/util/rootCache.ts

@@ -1,6 +1,5 @@
 import {Epoch, Root, Slot, phase0} from "@lodestar/types";
-import {CachedBeaconStateAllForks} from "../types.js";
-import {getBlockRoot, getBlockRootAtSlot} from "./blockRoot.js";
+import {IBeaconStateView} from "../stateView/interface.js";
 
 /**
  * Cache to prevent accessing the state tree to fetch block roots repeteadly.
@@ -12,7 +11,7 @@ export class RootCache {
   private readonly blockRootEpochCache = new Map<Epoch, Root>();
   private readonly blockRootSlotCache = new Map<Slot, Root>();
 
-  constructor(private readonly state: CachedBeaconStateAllForks) {
+  constructor(private readonly state: IBeaconStateView) {
     this.currentJustifiedCheckpoint = state.currentJustifiedCheckpoint;
     this.previousJustifiedCheckpoint = state.previousJustifiedCheckpoint;
   }
@@ -20,7 +19,7 @@ export class RootCache {
   getBlockRoot(epoch: Epoch): Root {
     let root = this.blockRootEpochCache.get(epoch);
     if (!root) {
-      root = getBlockRoot(this.state, epoch);
+      root = this.state.getBlockRootAtEpoch(epoch);
       this.blockRootEpochCache.set(epoch, root);
     }
     return root;
@@ -29,7 +28,7 @@ export class RootCache {
   getBlockRootAtSlot(slot: Slot): Root {
     let root = this.blockRootSlotCache.get(slot);
     if (!root) {
-      root = getBlockRootAtSlot(this.state, slot);
+      root = this.state.getBlockRootAtSlot(slot);
       this.blockRootSlotCache.set(slot, root);
     }
     return root;

package/src/util/shuffling.ts

@@ -11,6 +11,7 @@ import {
 } from "@lodestar/types";
 import {LodestarError} from "@lodestar/utils";
 import {CachedBeaconStateAllForks} from "../cache/stateCache.js";
+import {IBeaconStateView} from "../stateView/interface.js";
 import {getBlockRootAtSlot} from "./blockRoot.js";
 import {computeStartSlotAtEpoch} from "./epoch.js";
 import {EpochShuffling} from "./epochShuffling.js";
@@ -22,21 +23,21 @@ import {EpochShuffling} from "./epochShuffling.js";
  * Returns `null` on the one-off scenario where the genesis block decides its own shuffling.
  * It should be set to the latest block applied to this `state` or the genesis block root.
  */
-export function proposerShufflingDecisionRoot(fork: ForkName, state: CachedBeaconStateAllForks): Root | null {
+export function proposerShufflingDecisionRoot(fork: ForkName, state: IBeaconStateView): Root | null {
   const decisionSlot = proposerShufflingDecisionSlot(fork, state);
   if (state.slot === decisionSlot) {
     return null;
   }
-  return getBlockRootAtSlot(state, decisionSlot);
+  return state.getBlockRootAtSlot(decisionSlot);
 }
 
 /**
  * Returns the slot at which the proposer shuffling was decided. The block root at this slot
  * can be used to key the proposer shuffling for the current epoch.
  */
-function proposerShufflingDecisionSlot(fork: ForkName, state: CachedBeaconStateAllForks): Slot {
+function proposerShufflingDecisionSlot(fork: ForkName, state: IBeaconStateView): Slot {
   // After fulu, the decision slot is in previous epoch due to deterministic proposer lookahead
-  const epoch = isForkPostFulu(fork) ? state.epochCtx.epoch - 1 : state.epochCtx.epoch;
+  const epoch = isForkPostFulu(fork) ? state.epoch - 1 : state.epoch;
   const startSlot = computeStartSlotAtEpoch(epoch);
   return Math.max(startSlot - 1, 0);
 }