@lodestar/state-transition 1.35.0-dev.98d359db41 → 1.35.0-dev.a70bac5bd3

package/src/util/loadState/loadState.ts

@@ -1,225 +0,0 @@
-import {ChainForkConfig} from "@lodestar/config";
-import {ForkSeq} from "@lodestar/params";
-import {deserializeContainerIgnoreFields, ssz} from "@lodestar/types";
-import {BeaconStateAllForks, BeaconStateAltair} from "../../types.js";
-import {VALIDATOR_BYTES_SIZE, getForkFromStateBytes, getStateTypeFromBytes} from "../sszBytes.js";
-import {findModifiedInactivityScores} from "./findModifiedInactivityScores.js";
-import {findModifiedValidators} from "./findModifiedValidators.js";
-import {loadValidator} from "./loadValidator.js";
-
-type MigrateStateOutput = {state: BeaconStateAllForks; modifiedValidators: number[]};
-
-/**
- * Load state from bytes given a seed state so that we share the same base tree. This gives some benefits:
- * - Have single base tree across the application
- * - Faster to load state
- * - Less memory usage
- * - Utilize the cached HashObjects in seed state due to a lot of validators are not changed, also the inactivity scores.
- * @returns the new state and modified validators
- */
-export function loadState(
-  config: ChainForkConfig,
-  seedState: BeaconStateAllForks,
-  stateBytes: Uint8Array,
-  seedValidatorsBytes?: Uint8Array
-): MigrateStateOutput {
-  // casting only to make typescript happy
-  const stateType = getStateTypeFromBytes(config, stateBytes) as typeof ssz.capella.BeaconState;
-  const dataView = new DataView(stateBytes.buffer, stateBytes.byteOffset, stateBytes.byteLength);
-  const fieldRanges = stateType.getFieldRanges(dataView, 0, stateBytes.length);
-  const allFields = Object.keys(stateType.fields);
-  const validatorsFieldIndex = allFields.indexOf("validators");
-  // start with default view has the same performance to start with seed state
-  // and it is not fork dependent
-  const migratedState = deserializeContainerIgnoreFields(
-    stateType,
-    stateBytes,
-    ["validators", "inactivityScores"],
-    fieldRanges
-  ) as BeaconStateAllForks;
-
-  // validators are rarely changed
-  const validatorsRange = fieldRanges[validatorsFieldIndex];
-  const modifiedValidators = loadValidators(
-    migratedState,
-    seedState,
-    stateBytes.subarray(validatorsRange.start, validatorsRange.end),
-    seedValidatorsBytes
-  );
-
-  // inactivityScores are rarely changed
-  // this saves ~500ms of hashTreeRoot() time of state
-  const fork = getForkFromStateBytes(config, stateBytes);
-  const seedFork = config.getForkSeq(seedState.slot);
-
-  if (fork >= ForkSeq.altair && seedFork >= ForkSeq.altair) {
-    const inactivityScoresIndex = allFields.indexOf("inactivityScores");
-    const inactivityScoresRange = fieldRanges[inactivityScoresIndex];
-    loadInactivityScores(
-      migratedState as BeaconStateAltair,
-      seedState as BeaconStateAltair,
-      stateBytes.subarray(inactivityScoresRange.start, inactivityScoresRange.end)
-    );
-  }
-  migratedState.commit();
-
-  return {state: migratedState, modifiedValidators};
-}
-
-/**
- * Load state and validators Uint8Array from state bytes.
- */
-export function loadStateAndValidators(
-  chainForkConfig: ChainForkConfig,
-  stateBytes: Uint8Array
-): {state: BeaconStateAllForks; validatorsBytes: Uint8Array} {
-  // stateType could be any types, casting just to make typescript happy
-  const stateType = getStateTypeFromBytes(chainForkConfig, stateBytes) as typeof ssz.phase0.BeaconState;
-  const state = stateType.deserializeToViewDU(stateBytes);
-  const dataView = new DataView(stateBytes.buffer, stateBytes.byteOffset, stateBytes.byteLength);
-  const fieldRanges = stateType.getFieldRanges(dataView, 0, stateBytes.length);
-  const allFields = Object.keys(stateType.fields);
-  const validatorFieldIndex = allFields.indexOf("validators");
-  const validatorRange = fieldRanges[validatorFieldIndex];
-  const validatorsBytes = stateBytes.subarray(validatorRange.start, validatorRange.end);
-  return {state, validatorsBytes};
-}
-
-/**
- * This value is rarely changed as monitored 3 month state diffs on mainnet as of Sep 2023.
- * Reusing this data helps save hashTreeRoot time of state ~500ms
- *
- * Given the below tree:
- *
- * seedState.inactivityScores ====>  ROOT
- *                                 /     \
- *                            Hash01       Hash23
- *                           /    \       /    \
- *                       Sco0    Sco1   Sco2   Sco3
- *
- * if score 3 is modified, the new tree looks like this:
- *
- * migratedState.inactivityScores ====> ROOTa
- *                                     /      \
- *                                Hash01      Hash23a
- *                               /    \       /    \
- *                           Sco0    Sco1  Sco2   Sco3a
- */
-function loadInactivityScores(
-  migratedState: BeaconStateAltair,
-  seedState: BeaconStateAltair,
-  inactivityScoresBytes: Uint8Array
-): void {
-  // migratedState starts with the same inactivityScores to seed state
-  migratedState.inactivityScores = seedState.inactivityScores.clone();
-  const oldValidator = migratedState.inactivityScores.length;
-  // UintNum64 = 8 bytes
-  const newValidator = inactivityScoresBytes.length / 8;
-  const minValidator = Math.min(oldValidator, newValidator);
-  const oldInactivityScores = migratedState.inactivityScores.serialize();
-  const isMoreValidator = newValidator >= oldValidator;
-  const modifiedValidators: number[] = [];
-  findModifiedInactivityScores(
-    isMoreValidator ? oldInactivityScores : oldInactivityScores.subarray(0, minValidator * 8),
-    isMoreValidator ? inactivityScoresBytes.subarray(0, minValidator * 8) : inactivityScoresBytes,
-    modifiedValidators
-  );
-
-  for (const validatorIndex of modifiedValidators) {
-    migratedState.inactivityScores.set(
-      validatorIndex,
-      ssz.UintNum64.deserialize(inactivityScoresBytes.subarray(validatorIndex * 8, (validatorIndex + 1) * 8))
-    );
-  }
-
-  if (isMoreValidator) {
-    // add new inactivityScores
-    for (let validatorIndex = oldValidator; validatorIndex < newValidator; validatorIndex++) {
-      migratedState.inactivityScores.push(
-        ssz.UintNum64.deserialize(inactivityScoresBytes.subarray(validatorIndex * 8, (validatorIndex + 1) * 8))
-      );
-    }
-  } else {
-    if (newValidator - 1 < 0) {
-      migratedState.inactivityScores = ssz.altair.InactivityScores.defaultViewDU();
-    } else {
-      migratedState.inactivityScores = migratedState.inactivityScores.sliceTo(newValidator - 1);
-    }
-  }
-}
-
-/**
- * As of Sep 2023, common validators of 2 mainnet states are rarely changed. However, the benchmark shows that
- * 10k modified validators is not an issue. (see packages/state-transition/test/perf/util/loadState/findModifiedValidators.test.ts)
- *
- * This method loads validators from bytes given a seed state so that they share the same base tree. This gives some benefits:
- *  - Have single base tree across the application
- *  - Faster to load state
- *  - Less memory usage
- *  - Utilize the cached HashObjects in seed state due to a lot of validators are not changed
- *
- * Given the below tree:
- *
- * seedState.validators ====>  ROOT
- *                            /     \
- *                       Hash01       Hash23
- *                      /    \       /    \
- *                  Val0    Val1   Val2   Val3
- *
- * if validator 3 is modified, the new tree looks like this:
- *
- * migratedState.validators ====>  ROOTa
- *                               /      \
- *                          Hash01      Hash23a
- *                         /    \       /    \
- *                     Val0    Val1   Val2   Val3a
- *
- * @param migratedState state to be migrated, the validators are loaded to this state
- * @returns modified validator indices
- */
-function loadValidators(
-  migratedState: BeaconStateAllForks,
-  seedState: BeaconStateAllForks,
-  newValidatorsBytes: Uint8Array,
-  seedStateValidatorsBytes?: Uint8Array
-): number[] {
-  const seedValidatorCount = seedState.validators.length;
-  const newValidatorCount = Math.floor(newValidatorsBytes.length / VALIDATOR_BYTES_SIZE);
-  const isMoreValidator = newValidatorCount >= seedValidatorCount;
-  const minValidatorCount = Math.min(seedValidatorCount, newValidatorCount);
-  // migrated state starts with the same validators to seed state
-  migratedState.validators = seedState.validators.clone();
-  // 80% of validators serialization time comes from memory allocation
-  // seedStateValidatorsBytes is an optimization at beacon-node side to avoid memory allocation here
-  const seedValidatorsBytes = seedStateValidatorsBytes ?? seedState.validators.serialize();
-  const modifiedValidators: number[] = [];
-  findModifiedValidators(
-    isMoreValidator ? seedValidatorsBytes : seedValidatorsBytes.subarray(0, minValidatorCount * VALIDATOR_BYTES_SIZE),
-    isMoreValidator ? newValidatorsBytes.subarray(0, minValidatorCount * VALIDATOR_BYTES_SIZE) : newValidatorsBytes,
-    modifiedValidators
-  );
-
-  for (const i of modifiedValidators) {
-    const seedValidator = seedState.validators.get(i);
-    const newValidatorBytes = newValidatorsBytes.subarray(i * VALIDATOR_BYTES_SIZE, (i + 1) * VALIDATOR_BYTES_SIZE);
-    migratedState.validators.set(i, loadValidator(seedValidator, newValidatorBytes));
-  }
-
-  if (newValidatorCount >= seedValidatorCount) {
-    // add new validators
-    for (let validatorIndex = seedValidatorCount; validatorIndex < newValidatorCount; validatorIndex++) {
-      migratedState.validators.push(
-        ssz.phase0.Validator.deserializeToViewDU(
-          newValidatorsBytes.subarray(
-            validatorIndex * VALIDATOR_BYTES_SIZE,
-            (validatorIndex + 1) * VALIDATOR_BYTES_SIZE
-          )
-        )
-      );
-      modifiedValidators.push(validatorIndex);
-    }
-  } else {
-    migratedState.validators = migratedState.validators.sliceTo(newValidatorCount - 1);
-  }
-  return modifiedValidators;
-}

package/src/util/loadState/loadValidator.ts

@@ -1,77 +0,0 @@
-import {CompositeViewDU} from "@chainsafe/ssz";
-import {ChainForkConfig} from "@lodestar/config";
-import {ValidatorIndex, deserializeContainerIgnoreFields, ssz} from "@lodestar/types";
-import {getStateTypeFromBytes} from "../sszBytes.js";
-
-/**
- * Load validator from bytes given a seed validator.
- * - Reuse pubkey and withdrawal credentials if possible to save memory
- * - If it's a new validator, deserialize it
- */
-export function loadValidator(
-  seedValidator: CompositeViewDU<typeof ssz.phase0.Validator>,
-  newValidatorBytes: Uint8Array
-): CompositeViewDU<typeof ssz.phase0.Validator> {
-  const ignoredFields = getSameFields(seedValidator, newValidatorBytes);
-  if (ignoredFields.length > 0) {
-    const newValidatorValue = deserializeContainerIgnoreFields(ssz.phase0.Validator, newValidatorBytes, ignoredFields);
-    for (const field of ignoredFields) {
-      newValidatorValue[field] = seedValidator[field];
-    }
-    return ssz.phase0.Validator.toViewDU(newValidatorValue);
-  }
-  return ssz.phase0.Validator.deserializeToViewDU(newValidatorBytes);
-}
-
-/**
- * Return pubkey or withdrawalCredentials or both if they are the same.
- */
-function getSameFields(
-  validator: CompositeViewDU<typeof ssz.phase0.Validator>,
-  validatorBytes: Uint8Array
-): ("pubkey" | "withdrawalCredentials")[] {
-  const ignoredFields: ("pubkey" | "withdrawalCredentials")[] = [];
-  const pubkey = validatorBytes.subarray(0, 48);
-  if (Buffer.compare(pubkey, validator.pubkey) === 0) {
-    ignoredFields.push("pubkey");
-  }
-
-  const withdrawalCredentials = validatorBytes.subarray(48, 80);
-  if (Buffer.compare(withdrawalCredentials, validator.withdrawalCredentials) === 0) {
-    ignoredFields.push("withdrawalCredentials");
-  }
-
-  return ignoredFields;
-}
-
-/**
- * Extract and deserialize validator effective balances from state bytes
- */
-export function getEffectiveBalancesFromStateBytes(
-  config: ChainForkConfig,
-  stateBytes: Uint8Array,
-  validatorIndices: ValidatorIndex[]
-): number[] {
-  // stateType could be any types, casting just to make typescript happy
-  const stateType = getStateTypeFromBytes(config, stateBytes) as typeof ssz.phase0.BeaconState;
-  const stateView = new DataView(stateBytes.buffer, stateBytes.byteOffset, stateBytes.byteLength);
-  const stateFieldRanges = stateType.getFieldRanges(stateView, 0, stateBytes.length);
-  const stateFields = Object.keys(stateType.fields);
-  const validatorsFieldIndex = stateFields.indexOf("validators");
-  const validatorsRange = stateFieldRanges[validatorsFieldIndex];
-  const validatorsBytes = stateBytes.subarray(validatorsRange.start, validatorsRange.end);
-  const validatorSize = ssz.phase0.Validator.fixedSize as number;
-
-  const effectiveBalances: number[] = [];
-
-  for (const index of validatorIndices) {
-    const validatorBytes = validatorsBytes.subarray(index * validatorSize, (index + 1) * validatorSize);
-    if (validatorBytes.byteLength === 0) {
-      throw Error(`Validator index ${index} out of range`);
-    }
-    const validator = ssz.phase0.Validator.deserialize(validatorBytes);
-    effectiveBalances.push(validator.effectiveBalance);
-  }
-
-  return effectiveBalances;
-}

package/src/util/rootCache.ts

@@ -1,37 +0,0 @@
-import {Epoch, Root, Slot, phase0} from "@lodestar/types";
-import {CachedBeaconStateAllForks} from "../types.js";
-import {getBlockRoot, getBlockRootAtSlot} from "./blockRoot.js";
-
-/**
- * Cache to prevent accessing the state tree to fetch block roots repeteadly.
- * In normal network conditions the same root is read multiple times, specially the target.
- */
-export class RootCache {
-  readonly currentJustifiedCheckpoint: phase0.Checkpoint;
-  readonly previousJustifiedCheckpoint: phase0.Checkpoint;
-  private readonly blockRootEpochCache = new Map<Epoch, Root>();
-  private readonly blockRootSlotCache = new Map<Slot, Root>();
-
-  constructor(private readonly state: CachedBeaconStateAllForks) {
-    this.currentJustifiedCheckpoint = state.currentJustifiedCheckpoint;
-    this.previousJustifiedCheckpoint = state.previousJustifiedCheckpoint;
-  }
-
-  getBlockRoot(epoch: Epoch): Root {
-    let root = this.blockRootEpochCache.get(epoch);
-    if (!root) {
-      root = getBlockRoot(this.state, epoch);
-      this.blockRootEpochCache.set(epoch, root);
-    }
-    return root;
-  }
-
-  getBlockRootAtSlot(slot: Slot): Root {
-    let root = this.blockRootSlotCache.get(slot);
-    if (!root) {
-      root = getBlockRootAtSlot(this.state, slot);
-      this.blockRootSlotCache.set(slot, root);
-    }
-    return root;
-  }
-}