@lodestar/state-transition 1.41.0-dev.afd446235e → 1.41.0-dev.b90dff673d

package/src/testUtils/util.ts

@@ -0,0 +1,429 @@
+import {PublicKey, SecretKey} from "@chainsafe/blst";
+import {BitArray, fromHexString} from "@chainsafe/ssz";
+import {createBeaconConfig, createChainForkConfig} from "@lodestar/config";
+import {config} from "@lodestar/config/default";
+import {
+  EPOCHS_PER_ETH1_VOTING_PERIOD,
+  EPOCHS_PER_HISTORICAL_VECTOR,
+  ForkName,
+  ForkSeq,
+  MAX_ATTESTATIONS,
+  MAX_EFFECTIVE_BALANCE,
+  SLOTS_PER_EPOCH,
+  SLOTS_PER_HISTORICAL_ROOT,
+} from "@lodestar/params";
+import {BeaconState, Slot, phase0, ssz} from "@lodestar/types";
+import {getEffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js";
+import {
+  computeCommitteeCount,
+  computeEpochAtSlot,
+  createCachedBeaconState,
+  createPubkeyCache,
+  interopSecretKey,
+  newFilledArray,
+  processSlots,
+} from "../index.js";
+import {
+  BeaconStateAltair,
+  BeaconStatePhase0,
+  CachedBeaconStateAllForks,
+  CachedBeaconStateAltair,
+  CachedBeaconStatePhase0,
+} from "../types.js";
+import {getNextSyncCommittee} from "../util/syncCommittee.js";
+import {getActiveValidatorIndices} from "../util/validator.js";
+import {interopPubkeysCached} from "./interop.js";
+
+let phase0State: BeaconStatePhase0 | null = null;
+let phase0CachedState23637: CachedBeaconStatePhase0 | null = null;
+let phase0CachedState23638: CachedBeaconStatePhase0 | null = null;
+let phase0SignedBlock: phase0.SignedBeaconBlock | null = null;
+let altairState: BeaconStateAltair | null = null;
+let altairCachedState23637: CachedBeaconStateAltair | null = null;
+let altairCachedState23638: CachedBeaconStateAltair | null = null;
+
+/**
+ * Number of validators in prater is 210000 as of May 2021
+ */
+export const numValidators = 250000;
+export const keypairsMod = 100;
+
+/**
+ * As of Jul 07 2021, the performance state has
+ * out.prevEpochUnslashedStake.targetStake 7750000000000000n
+ * out.currEpochUnslashedTargetStake 7750000000000000n
+ * This prefix represent the total stake in Peta Wei
+ */
+export const perfStateId = `${numValidators} vs - 7PWei`;
+
+/** Cache interop secret keys */
+const secretKeyByModIndex = new Map<number, SecretKey>();
+const epoch = 23638;
+export const perfStateEpoch = epoch;
+
+export function getPubkeys(vc = numValidators) {
+  const pubkeysMod = interopPubkeysCached(keypairsMod);
+  const pubkeysModObj = pubkeysMod.map((pk) => PublicKey.fromBytes(pk));
+  const pubkeys = Array.from({length: vc}, (_, i) => pubkeysMod[i % keypairsMod]);
+  return {pubkeysMod, pubkeysModObj, pubkeys};
+}
+
+/** Get secret key of a validatorIndex, if the pubkeys are generated with `getPubkeys()` */
+export function getSecretKeyFromIndex(validatorIndex: number): SecretKey {
+  return interopSecretKey(validatorIndex % keypairsMod);
+}
+
+/** Get secret key of a validatorIndex, if the pubkeys are generated with `getPubkeys()` */
+export function getSecretKeyFromIndexCached(validatorIndex: number): SecretKey {
+  const keyIndex = validatorIndex % keypairsMod;
+  let sk = secretKeyByModIndex.get(keyIndex);
+  if (!sk) {
+    sk = interopSecretKey(keyIndex);
+    secretKeyByModIndex.set(keyIndex, sk);
+  }
+  return sk;
+}
+
+function getPubkeyCaches({pubkeysMod}: ReturnType<typeof getPubkeys>) {
+  // Manually sync pubkeys to prevent doing BLS opts 110_000 times
+  const pubkeyCache = createPubkeyCache();
+  for (let i = 0; i < numValidators; i++) {
+    const pubkey = pubkeysMod[i % keypairsMod];
+    pubkeyCache.set(i, pubkey);
+  }
+
+  return {pubkeyCache};
+}
+
+export function generatePerfTestCachedStatePhase0(opts?: {goBackOneSlot: boolean}): CachedBeaconStatePhase0 {
+  // Generate only some publicKeys
+  const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys();
+  const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj});
+
+  if (!phase0State) {
+    const state = buildPerformanceStatePhase0();
+
+    // no justificationBits
+    phase0State = ssz.phase0.BeaconState.toViewDU(state);
+
+    // cache roots
+    phase0State.hashTreeRoot();
+  }
+
+  if (!phase0CachedState23637) {
+    const state = phase0State.clone();
+    state.slot -= 1;
+    phase0CachedState23637 = createCachedBeaconState(state, {
+      config: createBeaconConfig(config, state.genesisValidatorsRoot),
+      pubkeyCache,
+    });
+
+    const currentEpoch = computeEpochAtSlot(state.slot - 1);
+    const previousEpoch = currentEpoch - 1;
+
+    // previous epoch attestations
+    const numPrevAttestations = SLOTS_PER_EPOCH * MAX_ATTESTATIONS;
+    const activeValidatorCount = pubkeys.length;
+    const committeesPerSlot = computeCommitteeCount(activeValidatorCount);
+    for (let i = 0; i < numPrevAttestations; i++) {
+      const slotInEpoch = i % SLOTS_PER_EPOCH;
+      const slot = previousEpoch * SLOTS_PER_EPOCH + slotInEpoch;
+      const index = i % committeesPerSlot;
+      const shuffling = phase0CachedState23637.epochCtx.getShufflingAtEpoch(previousEpoch);
+      const committee = shuffling.committees[slotInEpoch][index];
+      phase0CachedState23637.previousEpochAttestations.push(
+        ssz.phase0.PendingAttestation.toViewDU({
+          aggregationBits: BitArray.fromBoolArray(Array.from({length: committee.length}, () => true)),
+          data: {
+            beaconBlockRoot: phase0CachedState23637.blockRoots.get(slotInEpoch % SLOTS_PER_HISTORICAL_ROOT),
+            index,
+            slot,
+            source: state.previousJustifiedCheckpoint,
+            target: state.currentJustifiedCheckpoint,
+          },
+          inclusionDelay: 1,
+          proposerIndex: i,
+        })
+      );
+    }
+
+    // current epoch attestations
+    const numCurAttestations = (SLOTS_PER_EPOCH - 1) * MAX_ATTESTATIONS;
+    for (let i = 0; i < numCurAttestations; i++) {
+      const slotInEpoch = i % SLOTS_PER_EPOCH;
+      const slot = currentEpoch * SLOTS_PER_EPOCH + slotInEpoch;
+      const index = i % committeesPerSlot;
+      const shuffling = phase0CachedState23637.epochCtx.getShufflingAtEpoch(previousEpoch);
+      const committee = shuffling.committees[slotInEpoch][index];
+
+      phase0CachedState23637.currentEpochAttestations.push(
+        ssz.phase0.PendingAttestation.toViewDU({
+          aggregationBits: BitArray.fromBoolArray(Array.from({length: committee.length}, () => true)),
+          data: {
+            beaconBlockRoot: phase0CachedState23637.blockRoots.get(slotInEpoch % SLOTS_PER_HISTORICAL_ROOT),
+            index,
+            slot,
+            source: state.currentJustifiedCheckpoint,
+            target: {
+              epoch: currentEpoch,
+              root: phase0CachedState23637.blockRoots.get((currentEpoch * SLOTS_PER_EPOCH) % SLOTS_PER_HISTORICAL_ROOT),
+            },
+          },
+          inclusionDelay: 1,
+          proposerIndex: i,
+        })
+      );
+    }
+  }
+  if (!phase0CachedState23638) {
+    phase0CachedState23638 = processSlots(
+      phase0CachedState23637,
+      phase0CachedState23637.slot + 1
+    ) as CachedBeaconStatePhase0;
+    phase0CachedState23638.slot += 1;
+  }
+  const resultingState = opts?.goBackOneSlot ? phase0CachedState23637 : phase0CachedState23638;
+
+  return resultingState.clone();
+}
+
+export function cachedStateAltairPopulateCaches(state: CachedBeaconStateAltair): void {
+  // Populate caches
+  state.blockRoots.getAllReadonly();
+  state.eth1DataVotes.getAllReadonly();
+  state.validators.getAllReadonly();
+  state.balances.getAll();
+  state.previousEpochParticipation.getAll();
+  state.currentEpochParticipation.getAll();
+  state.inactivityScores.getAll();
+}
+
+/**
+ * Warning: This function has side effects on the cached state
+ * The order in which the caches are populated is important and can cause stable tests to fail.
+ */
+export function generatePerfTestCachedStateAltair(opts?: {
+  goBackOneSlot: boolean;
+  vc?: number;
+}): CachedBeaconStateAltair {
+  const {pubkeys, pubkeysMod, pubkeysModObj} = getPubkeys(opts?.vc);
+  const {pubkeyCache} = getPubkeyCaches({pubkeys, pubkeysMod, pubkeysModObj});
+
+  const altairConfig = createChainForkConfig({ALTAIR_FORK_EPOCH: 0});
+
+  const origState = generatePerformanceStateAltair(pubkeys);
+
+  if (!altairCachedState23637) {
+    const state = origState.clone();
+    state.slot -= 1;
+    altairCachedState23637 = createCachedBeaconState(state, {
+      config: createBeaconConfig(altairConfig, state.genesisValidatorsRoot),
+      pubkeyCache,
+    });
+  }
+  if (!altairCachedState23638) {
+    altairCachedState23638 = processSlots(
+      altairCachedState23637,
+      altairCachedState23637.slot + 1
+    ) as CachedBeaconStateAltair;
+    altairCachedState23638.slot += 1;
+  }
+  const resultingState = opts?.goBackOneSlot ? altairCachedState23637 : altairCachedState23638;
+
+  return resultingState.clone();
+}
+
+/**
+ * This is generated from Medalla state 756416
+ */
+export function generatePerformanceStateAltair(pubkeysArg?: Uint8Array[]): BeaconStateAltair {
+  if (!altairState) {
+    const pubkeys = pubkeysArg || getPubkeys().pubkeys;
+    const statePhase0 = buildPerformanceStatePhase0(pubkeys);
+    const state = statePhase0 as BeaconState as BeaconState<ForkName.altair>;
+
+    state.previousEpochParticipation = newFilledArray(pubkeys.length, 0b111);
+    state.currentEpochParticipation = state.previousEpochParticipation;
+    state.inactivityScores = Array.from({length: pubkeys.length}, (_, i) => i % 2);
+
+    // Placeholder syncCommittees
+    state.currentSyncCommittee = ssz.altair.SyncCommittee.defaultValue();
+    state.nextSyncCommittee = state.currentSyncCommittee;
+
+    // Now the state is fully populated to convert to ViewDU
+    altairState = ssz.altair.BeaconState.toViewDU(state);
+
+    // Now set correct syncCommittees
+    const epoch = computeEpochAtSlot(state.slot);
+    const activeValidatorIndices = getActiveValidatorIndices(altairState, epoch);
+
+    const effectiveBalanceIncrements = getEffectiveBalanceIncrements(altairState);
+    const {syncCommittee} = getNextSyncCommittee(
+      ForkSeq.altair,
+      altairState,
+      activeValidatorIndices,
+      effectiveBalanceIncrements
+    );
+    state.currentSyncCommittee = syncCommittee;
+    state.nextSyncCommittee = syncCommittee;
+
+    altairState = ssz.altair.BeaconState.toViewDU(state);
+    // cache roots
+    altairState.hashTreeRoot();
+  }
+  return altairState.clone();
+}
+
+/**
+ * This is generated from Medalla block 756417
+ */
+export function generatePerformanceBlockPhase0(): phase0.SignedBeaconBlock {
+  if (!phase0SignedBlock) {
+    const block = ssz.phase0.SignedBeaconBlock.defaultValue();
+    const parentState = generatePerfTestCachedStatePhase0();
+    block.message.slot = parentState.slot;
+    block.message.proposerIndex = parentState.epochCtx.getBeaconProposer(parentState.slot);
+    block.message.parentRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(parentState.latestBlockHeader);
+    block.message.stateRoot = fromHexString("0x6c86ca3c4c6688cf189421b8a68bf2dbc91521609965e6f4e207d44347061fee");
+    block.message.body.randaoReveal = fromHexString(
+      "0x8a5d2673c48f22f6ed19462efec35645db490df29eed2f56321dbe4a89b2463b0c902095a7ab74a2dc5b7f67edb1a19507ea3d4361d5af9cb0a524945c91638dfd6568841486813a2c45142659d6d9403f5081febb123a7931edbc248b9d0025"
+    );
+    // eth1Data, graffiti, attestations
+    phase0SignedBlock = block;
+  }
+
+  return phase0SignedBlock;
+}
+
+function buildPerformanceStatePhase0(pubkeysArg?: Uint8Array[]): phase0.BeaconState {
+  const slot = epoch * SLOTS_PER_EPOCH;
+  const pubkeys = pubkeysArg || getPubkeys().pubkeys;
+  const currentEpoch = computeEpochAtSlot(slot - 1);
+
+  return {
+    // Misc
+    genesisTime: 1596546008,
+    genesisValidatorsRoot: fromHexString("0x04700007fabc8282644aed6d1c7c9e21d38a03a0c4ba193f3afe428824b3a673"),
+    slot: epoch * SLOTS_PER_EPOCH,
+    fork: {
+      currentVersion: fromHexString("0x00000001"),
+      previousVersion: fromHexString("0x00000001"),
+      epoch: 0,
+    },
+    // History
+    latestBlockHeader: {
+      slot: slot - 1,
+      proposerIndex: 80882,
+      parentRoot: fromHexString("0x5b83c3078e474b86af60043eda82a34c3c2e5ebf83146b14d9d909aea4163ef2"),
+      stateRoot: fromHexString("0x2761ae355e8a53c11e0e37d5e417f8984db0c53fa83f1bc65f89c6af35a196a7"),
+      bodyRoot: fromHexString("0x249a1962eef90e122fa2447040bfac102798b1dba9c73e5593bc5aa32eb92bfd"),
+    },
+    blockRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, (_, i) => Buffer.alloc(32, i)),
+    stateRoots: Array.from({length: SLOTS_PER_HISTORICAL_ROOT}, (_, i) => Buffer.alloc(32, i)),
+    historicalRoots: [],
+    // Eth1
+    eth1Data: {
+      depositCount: pubkeys.length,
+      depositRoot: fromHexString("0xcb1f89a924cfd31224823db5a41b1643f10faa7aedf231f1e28887f6ee98c047"),
+      blockHash: fromHexString("0x701fb2869ce16d0f1d14f6705725adb0dec6799da29006dfc6fff83960298f21"),
+    },
+    // minus one so that inserting 1 from block works
+    eth1DataVotes: newFilledArray(EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH - 1, {
+      depositCount: 1,
+      depositRoot: Buffer.alloc(32, 1),
+      blockHash: Buffer.alloc(32, 1),
+    }),
+    eth1DepositIndex: pubkeys.length,
+    // Registry
+    validators: pubkeys.map((_, i) => ({
+      pubkey: pubkeys[i],
+      withdrawalCredentials: Buffer.alloc(32, i),
+      effectiveBalance: 31000000000,
+      slashed: false,
+      activationEligibilityEpoch: 0,
+      activationEpoch: 0,
+      exitEpoch: Infinity,
+      withdrawableEpoch: Infinity,
+    })),
+    balances: Array.from({length: pubkeys.length}, () => 31217089836),
+    randaoMixes: Array.from({length: EPOCHS_PER_HISTORICAL_VECTOR}, (_, i) => Buffer.alloc(32, i)),
+    // Slashings
+    slashings: ssz.phase0.Slashings.defaultValue(),
+    previousEpochAttestations: [],
+    currentEpochAttestations: [],
+    // Finality
+    justificationBits: BitArray.fromBitLen(4),
+    previousJustifiedCheckpoint: {
+      epoch: currentEpoch - 2,
+      root: fromHexString("0x3fe60bf06a57b0956cd1f8181d26649cf8bf79e48bf82f55562e04b33d4785d4"),
+    },
+    currentJustifiedCheckpoint: {
+      epoch: currentEpoch - 1,
+      root: fromHexString("0x3ba0913d2fb5e4cbcfb0d39eb15803157c1e769d63b8619285d8fdabbd8181c7"),
+    },
+    finalizedCheckpoint: {
+      epoch: currentEpoch - 3,
+      root: fromHexString("0x122b8ff579d0c8f8a8b66326bdfec3f685007d2842f01615a0768870961ccc17"),
+    },
+  };
+}
+
+export function generateTestCachedBeaconStateOnlyValidators({
+  vc,
+  slot,
+}: {
+  vc: number;
+  slot: Slot;
+}): CachedBeaconStateAllForks {
+  // Generate only some publicKeys
+  const {pubkeys, pubkeysMod} = getPubkeys(vc);
+
+  // Manually sync pubkeys to prevent doing BLS opts 110_000 times
+  const pubkeyCache = createPubkeyCache();
+  for (let i = 0; i < vc; i++) {
+    const pubkey = pubkeysMod[i % keypairsMod];
+    pubkeyCache.set(i, pubkey);
+  }
+
+  const state = ssz.phase0.BeaconState.defaultViewDU();
+  state.slot = slot;
+
+  const activeValidator = ssz.phase0.Validator.toViewDU({
+    pubkey: Buffer.alloc(48, 0),
+    withdrawalCredentials: Buffer.alloc(32, 0),
+    effectiveBalance: MAX_EFFECTIVE_BALANCE,
+    slashed: false,
+    activationEligibilityEpoch: 0,
+    activationEpoch: 0,
+    exitEpoch: Infinity,
+    withdrawableEpoch: Infinity,
+  });
+
+  for (let i = 0; i < vc; i++) {
+    const validator = activeValidator.clone();
+    validator.pubkey = pubkeys[i];
+    state.validators.push(validator);
+  }
+
+  state.balances = ssz.phase0.Balances.toViewDU(newFilledArray(pubkeys.length, MAX_EFFECTIVE_BALANCE));
+  state.randaoMixes = ssz.phase0.RandaoMixes.toViewDU(
+    newFilledArray(EPOCHS_PER_HISTORICAL_VECTOR, Buffer.alloc(32, 0xdd))
+  );
+
+  // Commit ViewDU changes
+  state.commit();
+
+  // Sanity check for .commit() above
+  if (state.validators.length !== vc) {
+    throw Error(`Wrong number of validators in the state: ${state.validators.length} !== ${vc}`);
+  }
+
+  return createCachedBeaconState(
+    state,
+    {
+      config: createBeaconConfig(config, state.genesisValidatorsRoot),
+      pubkeyCache,
+    },
+    {skipSyncPubkeys: true}
+  );
+}

package/src/util/seed.ts

@@ -417,8 +417,6 @@ export function naiveComputePayloadTimelinessCommitteeIndices(
  * https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
  *
  * See the 'generalized domain' algorithm on page 3.
- * This is the naive implementation just to make sure lodestar follows the spec, this is not for production.
- * The optimized version is in `getComputeShuffledIndexFn`.
  */
 export function computeShuffledIndex(index: number, indexCount: number, seed: Bytes32): number {
   let permuted = index;
@@ -439,75 +437,6 @@ export function computeShuffledIndex(index: number, indexCount: number, seed: By
   return permuted;
 }
 
-type ComputeShuffledIndexFn = (index: number) => number;
-
-/**
- * An optimized version of `computeShuffledIndex`, this is for production.
- */
-export function getComputeShuffledIndexFn(indexCount: number, seed: Bytes32): ComputeShuffledIndexFn {
-  // there are possibly SHUFFLE_ROUND_COUNT (90 for mainnet) values for this cache
-  // this cache will always hit after the 1st call
-  const pivotByIndex: Map<number, number> = new Map();
-  // given 2M active validators, there are 2 M / 256 = 8k possible positionDiv
-  // it means there are at most 8k different sources for each round
-  const sourceByPositionDivByIndex: Map<number, Map<number, Uint8Array>> = new Map();
-  // 32 bytes seed + 1 byte i
-  const pivotBuffer = Buffer.alloc(32 + 1);
-  pivotBuffer.set(seed, 0);
-  // 32 bytes seed + 1 byte i + 4 bytes positionDiv
-  const sourceBuffer = Buffer.alloc(32 + 1 + 4);
-  sourceBuffer.set(seed, 0);
-
-  return (index): number => {
-    assert.lt(index, indexCount, "indexCount must be less than index");
-    assert.lte(indexCount, 2 ** 40, "indexCount too big");
-    let permuted = index;
-    // const _seed = seed;
-    for (let i = 0; i < SHUFFLE_ROUND_COUNT; i++) {
-      // optimized version of the below naive code
-      // const pivot = Number(
-      //   bytesToBigInt(digest(Buffer.concat([_seed, intToBytes(i, 1)])).slice(0, 8)) % BigInt(indexCount)
-      // );
-
-      let pivot = pivotByIndex.get(i);
-      if (pivot == null) {
-        // naive version always creates a new buffer, we can reuse the buffer
-        // pivot = Number(
-        //   bytesToBigInt(digest(Buffer.concat([_seed, intToBytes(i, 1)])).slice(0, 8)) % BigInt(indexCount)
-        // );
-        pivotBuffer[32] = i % 256;
-        pivot = Number(bytesToBigInt(digest(pivotBuffer).subarray(0, 8)) % BigInt(indexCount));
-        pivotByIndex.set(i, pivot);
-      }
-
-      const flip = (pivot + indexCount - permuted) % indexCount;
-      const position = Math.max(permuted, flip);
-
-      // optimized version of the below naive code
-      // const source = digest(Buffer.concat([_seed, intToBytes(i, 1), intToBytes(Math.floor(position / 256), 4)]));
-      let sourceByPositionDiv = sourceByPositionDivByIndex.get(i);
-      if (sourceByPositionDiv == null) {
-        sourceByPositionDiv = new Map<number, Uint8Array>();
-        sourceByPositionDivByIndex.set(i, sourceByPositionDiv);
-      }
-      const positionDiv256 = Math.floor(position / 256);
-      let source = sourceByPositionDiv.get(positionDiv256);
-      if (source == null) {
-        // naive version always creates a new buffer, we can reuse the buffer
-        // don't want to go through intToBytes() to avoid BigInt
-        sourceBuffer[32] = i % 256;
-        sourceBuffer.writeUint32LE(positionDiv256, 33);
-        source = digest(sourceBuffer);
-        sourceByPositionDiv.set(positionDiv256, source);
-      }
-      const byte = source[Math.floor((position % 256) / 8)];
-      const bit = (byte >> (position % 8)) % 2;
-      permuted = bit ? flip : permuted;
-    }
-    return permuted;
-  };
-}
-
 /**
  * Return the randao mix at a recent [[epoch]].
  */