@lodestar/state-transition 1.45.0-rc.0 → 1.46.0-dev.13423cff6d

package/src/slot/upgradeStateToGloas.ts

@@ -1,3 +1,15 @@
+import {getNodesAtDepth} from "@chainsafe/persistent-merkle-tree";
+import {
+  BasicType,
+  CompositeType,
+  CompositeView,
+  CompositeViewDU,
+  ListBasicTreeViewDU,
+  ListCompositeTreeViewDU,
+  ProgressiveListBasicType,
+  ProgressiveListCompositeType,
+  ValueOf,
+} from "@chainsafe/ssz";
 import {PAYLOAD_BUILDER_VERSION, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
 import {ssz} from "@lodestar/types";
 import {toPubkeyHex} from "@lodestar/utils";
@@ -7,6 +19,7 @@ import {CachedBeaconStateFulu, CachedBeaconStateGloas} from "../types.js";
 import {addBuilderToRegistry, initializePtcWindow, isBuilderWithdrawalCredential} from "../util/gloas.js";
 import {isValidatorKnown} from "../util/index.js";
 import {PendingDepositsLookup} from "../util/pendingDepositsLookup.js";
+import {progressiveListRootNode} from "../util/ssz.js";
 
 /**
  * Upgrade a state from Fulu to Gloas.
@@ -34,21 +47,30 @@ export function upgradeStateToGloas(stateFulu: CachedBeaconStateFulu): CachedBea
   stateGloasView.eth1Data = stateGloasCloned.eth1Data;
   stateGloasView.eth1DataVotes = stateGloasCloned.eth1DataVotes;
   stateGloasView.eth1DepositIndex = stateGloasCloned.eth1DepositIndex;
-  stateGloasView.validators = stateGloasCloned.validators;
-  stateGloasView.balances = stateGloasCloned.balances;
+  stateGloasView.validators = migrateCompositeListToGloas(stateGloasCloned.validators, ssz.gloas.Validators);
+  stateGloasView.balances = migrateBasicListToGloas(stateGloasCloned.balances, ssz.gloas.Balances);
   stateGloasView.randaoMixes = stateGloasCloned.randaoMixes;
   stateGloasView.slashings = stateGloasCloned.slashings;
-  stateGloasView.previousEpochParticipation = stateGloasCloned.previousEpochParticipation;
-  stateGloasView.currentEpochParticipation = stateGloasCloned.currentEpochParticipation;
+  stateGloasView.previousEpochParticipation = migrateBasicListToGloas(
+    stateGloasCloned.previousEpochParticipation,
+    ssz.gloas.EpochParticipation
+  );
+  stateGloasView.currentEpochParticipation = migrateBasicListToGloas(
+    stateGloasCloned.currentEpochParticipation,
+    ssz.gloas.EpochParticipation
+  );
   stateGloasView.justificationBits = stateGloasCloned.justificationBits;
   stateGloasView.previousJustifiedCheckpoint = stateGloasCloned.previousJustifiedCheckpoint;
   stateGloasView.currentJustifiedCheckpoint = stateGloasCloned.currentJustifiedCheckpoint;
   stateGloasView.finalizedCheckpoint = stateGloasCloned.finalizedCheckpoint;
-  stateGloasView.inactivityScores = stateGloasCloned.inactivityScores;
+  stateGloasView.inactivityScores = migrateBasicListToGloas(
+    stateGloasCloned.inactivityScores,
+    ssz.gloas.InactivityScores
+  );
   stateGloasView.currentSyncCommittee = stateGloasCloned.currentSyncCommittee;
   stateGloasView.nextSyncCommittee = stateGloasCloned.nextSyncCommittee;
   stateGloasView.latestExecutionPayloadBid.blockHash = stateFulu.latestExecutionPayloadHeader.blockHash;
-  stateGloasView.latestExecutionPayloadBid.gasLimit = stateFulu.latestExecutionPayloadHeader.gasLimit;
+  stateGloasView.latestExecutionPayloadBid.gasLimit = BigInt(stateFulu.latestExecutionPayloadHeader.gasLimit);
   stateGloasView.latestExecutionPayloadBid.executionRequestsRoot = ssz.gloas.ExecutionRequests.hashTreeRoot(
     ssz.gloas.ExecutionRequests.defaultValue()
   );
@@ -61,9 +83,18 @@ export function upgradeStateToGloas(stateFulu: CachedBeaconStateFulu): CachedBea
   stateGloasView.earliestExitEpoch = stateGloasCloned.earliestExitEpoch;
   stateGloasView.consolidationBalanceToConsume = stateGloasCloned.consolidationBalanceToConsume;
   stateGloasView.earliestConsolidationEpoch = stateGloasCloned.earliestConsolidationEpoch;
-  stateGloasView.pendingDeposits = stateGloasCloned.pendingDeposits;
-  stateGloasView.pendingPartialWithdrawals = stateGloasCloned.pendingPartialWithdrawals;
-  stateGloasView.pendingConsolidations = stateGloasCloned.pendingConsolidations;
+  stateGloasView.pendingDeposits = migrateCompositeListToGloas(
+    stateGloasCloned.pendingDeposits,
+    ssz.gloas.PendingDeposits
+  );
+  stateGloasView.pendingPartialWithdrawals = migrateCompositeListToGloas(
+    stateGloasCloned.pendingPartialWithdrawals,
+    ssz.gloas.PendingPartialWithdrawals
+  );
+  stateGloasView.pendingConsolidations = migrateCompositeListToGloas(
+    stateGloasCloned.pendingConsolidations,
+    ssz.gloas.PendingConsolidations
+  );
   stateGloasView.proposerLookahead = stateGloasCloned.proposerLookahead;
   stateGloasView.ptcWindow = ssz.gloas.PtcWindow.toViewDU(initializePtcWindow(stateFulu));
 
@@ -85,6 +116,50 @@ export function upgradeStateToGloas(stateFulu: CachedBeaconStateFulu): CachedBea
   return stateGloas;
 }
 
+/**
+ * Migrate a composite list from fulu to its gloas progressive-list equivalent by reusing the fulu
+ * list's element nodes.
+ *
+ * Works whenever the element type is identical across the fork (e.g. validators use ValidatorNodeStruct,
+ * the pending* queues use the same electra element types). Each element's cached subtree root is then
+ * valid under gloas, so only the progressive list superstructure is rebuilt and a subsequent
+ * hashTreeRoot() skips re-hashing every element — the dominant cost for large lists like validators.
+ * Much cheaper than `gloasType.toViewDU(fuluList.getAllReadonlyValues())`, which decodes every element
+ * to a value and forces a full re-hash.
+ *
+ * The chunk nodes of a composite list ARE the element root nodes, so they are extracted directly
+ * with getNodesAtDepth instead of allocating a temporary ViewDU wrapper per element (getAllReadonly).
+ * Requires the fulu view to be committed (done at the top of upgradeStateToGloas).
+ */
+function migrateCompositeListToGloas<
+  ElementType extends CompositeType<ValueOf<ElementType>, CompositeView<ElementType>, CompositeViewDU<ElementType>>,
+>(fuluList: ListCompositeTreeViewDU<ElementType>, gloasType: ProgressiveListCompositeType<ElementType>) {
+  const {length, type} = fuluList;
+  const elementNodes = getNodesAtDepth(fuluList.node.left, type.chunkDepth, 0, length);
+  return gloasType.getViewDU(progressiveListRootNode(elementNodes, length));
+}
+
+/**
+ * Migrate a basic list from fulu to its gloas progressive-list equivalent by reusing the fulu
+ * list's packed chunk leaf nodes.
+ *
+ * Packed leaf chunks are bit-identical between List[T, N] and ProgressiveList[T] (same 32-byte
+ * LE packing, zero-padded final chunk); only the superstructure above the leaves differs. Reusing
+ * the leaves avoids materializing the value array (getAll), re-serializing it, and allocating
+ * fresh LeafNodes — the gloas tree shares the leaf nodes with the fulu tree.
+ * Requires the fulu view to be committed (done at the top of upgradeStateToGloas).
+ */
+function migrateBasicListToGloas<ElementType extends BasicType<unknown>>(
+  fuluList: ListBasicTreeViewDU<ElementType>,
+  gloasType: ProgressiveListBasicType<ElementType>
+) {
+  const {length, type} = fuluList;
+  const chunkCount = Math.ceil(length / type.itemsPerChunk);
+  // List root = BranchNode(chunksNode, lengthNode) → chunks tree is the left child
+  const chunkLeafNodes = getNodesAtDepth(fuluList.node.left, type.chunkDepth, 0, chunkCount);
+  return gloasType.getViewDU(progressiveListRootNode(chunkLeafNodes, length));
+}
+
 /**
  * Applies any pending deposits for builders to onboard builders during the fork transition
  * Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.8/specs/gloas/fork.md#new-onboard_builders_from_pending_deposits
@@ -94,7 +169,7 @@ function onboardBuildersFromPendingDeposits(state: CachedBeaconStateGloas): void
   // at the fork, so every builder pubkey here is one added in an earlier iteration.
   const builderPubkeys = new Set<string>();
 
-  const pendingDeposits = ssz.electra.PendingDeposits.defaultViewDU();
+  const pendingDeposits = ssz.gloas.PendingDeposits.defaultViewDU();
   const pendingDepositsLookup = PendingDepositsLookup.buildEmpty();
 
   for (let i = 0; i < state.pendingDeposits.length; i++) {

package/src/util/clock.ts

@@ -0,0 +1,183 @@
+import {ChainForkConfig} from "@lodestar/config";
+import {GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params";
+import {Epoch, Slot, TimeSeconds} from "@lodestar/types";
+import {ErrorAborted, Logger, isErrorAborted, sleep} from "@lodestar/utils";
+import {computeEpochAtSlot} from "./epoch.js";
+import {computeTimeAtSlot, getCurrentSlot} from "./slot.js";
+
+type RunEveryFn = (slot: Slot, signal: AbortSignal) => Promise<void>;
+
+export type ClockOptions = {
+  skipSlots?: boolean;
+};
+
+export interface IClock {
+  readonly genesisTime: number;
+  readonly secondsPerSlot: number;
+
+  readonly currentEpoch: number;
+
+  start(signal: AbortSignal): void;
+  runEverySlot(fn: (slot: Slot, signal: AbortSignal) => Promise<void>): void;
+  runEveryEpoch(fn: (epoch: Epoch, signal: AbortSignal) => Promise<void>): void;
+  msToSlot(slot: Slot): number;
+  msFromSlot(slot: Slot): number;
+  secFromSlot(slot: Slot): number;
+  getCurrentSlot(): Slot;
+  getCurrentEpoch(): Epoch;
+}
+
+export enum TimeItem {
+  Slot,
+  Epoch,
+}
+
+export class Clock implements IClock {
+  readonly genesisTime: number;
+  readonly secondsPerSlot: number;
+  private readonly config: ChainForkConfig;
+  private readonly logger: Logger;
+  private readonly fns: {timeItem: TimeItem; fn: RunEveryFn}[] = [];
+
+  constructor(
+    config: ChainForkConfig,
+    logger: Logger,
+    private readonly opts: {genesisTime: number} & ClockOptions
+  ) {
+    this.genesisTime = opts.genesisTime;
+    this.secondsPerSlot = config.SLOT_DURATION_MS / 1000;
+    this.config = config;
+    this.logger = logger;
+  }
+
+  get currentEpoch(): Epoch {
+    return computeEpochAtSlot(getCurrentSlot(this.config, this.genesisTime));
+  }
+
+  start(signal: AbortSignal): void {
+    for (const {timeItem, fn} of this.fns) {
+      this.runAtMostEvery(timeItem, signal, fn).catch((e: Error) => {
+        if (!isErrorAborted(e)) {
+          this.logger.error("runAtMostEvery", {}, e);
+        }
+      });
+    }
+  }
+
+  getCurrentSlot(): Slot {
+    return getCurrentSlot(this.config, this.genesisTime);
+  }
+
+  getCurrentEpoch(): Epoch {
+    return computeEpochAtSlot(getCurrentSlot(this.config, this.genesisTime));
+  }
+
+  runEverySlot(fn: RunEveryFn): void {
+    this.fns.push({timeItem: TimeItem.Slot, fn});
+  }
+
+  runEveryEpoch(fn: RunEveryFn): void {
+    this.fns.push({timeItem: TimeItem.Epoch, fn});
+  }
+
+  /** Milliseconds from now to a specific slot */
+  msToSlot(slot: Slot): number {
+    return computeTimeAtSlot(this.config, slot, this.genesisTime) * 1000 - Date.now();
+  }
+
+  /** Milliseconds elapsed from a specific slot to now */
+  msFromSlot(slot: Slot): number {
+    return Date.now() - computeTimeAtSlot(this.config, slot, this.genesisTime) * 1000;
+  }
+
+  /** Seconds elapsed from a specific slot to now */
+  secFromSlot(slot: Slot): number {
+    return Date.now() / 1000 - computeTimeAtSlot(this.config, slot, this.genesisTime);
+  }
+
+  /**
+   * If a task happens to take more than one slot to run, we might skip a slot. This is unfortunate,
+   * however the alternative is to *always* process every slot, which has the chance of creating a
+   * theoretically unlimited backlog of tasks. It was a conscious decision to choose to drop tasks
+   * on an overloaded/latent system rather than overload it even more.
+   */
+  private async runAtMostEvery(timeItem: TimeItem, signal: AbortSignal, fn: RunEveryFn): Promise<void> {
+    // Run immediately first
+    let slot = getCurrentSlot(this.config, this.genesisTime);
+    let slotOrEpoch = timeItem === TimeItem.Slot ? slot : computeEpochAtSlot(slot);
+    while (!signal.aborted) {
+      // Must catch fn() to ensure `sleep()` is awaited both for resolve and reject
+      const task = fn(slotOrEpoch, signal).catch((e: Error) => {
+        if (!isErrorAborted(e)) this.logger.error("Error on runEvery fn", {}, e);
+      });
+
+      if (timeItem !== TimeItem.Slot || this.opts.skipSlots !== false) {
+        // await response to only continue with next task if current task finished within slot
+        await task;
+      }
+
+      try {
+        await sleep(this.timeUntilNext(timeItem), signal);
+        // calling getCurrentSlot here may not be correct when we're close to the next slot
+        // it's safe to call getCurrentSlotAround after we sleep
+        const nextSlot = getCurrentSlotAround(this.config, this.genesisTime);
+
+        if (timeItem === TimeItem.Slot) {
+          if (nextSlot > slot + 1) {
+            // It's not very likely that we skip more than one slot as HTTP timeout is set
+            // to SLOT_DURATION_MS so we will fail task before skipping another slot.
+            this.logger.warn("Skipped slot due to task taking more than one slot to run", {
+              skippedSlot: slot + 1,
+            });
+          }
+          slotOrEpoch = nextSlot;
+        } else {
+          slotOrEpoch = computeEpochAtSlot(nextSlot);
+        }
+        slot = nextSlot;
+      } catch (e) {
+        if (e instanceof ErrorAborted) {
+          return;
+        }
+        throw e;
+      }
+    }
+  }
+
+  private timeUntilNext(timeItem: TimeItem): number {
+    const milliSecondsPerSlot = this.config.SLOT_DURATION_MS;
+    const msFromGenesis = Date.now() - this.genesisTime * 1000;
+
+    if (timeItem === TimeItem.Slot) {
+      if (msFromGenesis >= 0) {
+        return milliSecondsPerSlot - (msFromGenesis % milliSecondsPerSlot);
+      }
+      return Math.abs(msFromGenesis) % milliSecondsPerSlot;
+    }
+    const milliSecondsPerEpoch = SLOTS_PER_EPOCH * milliSecondsPerSlot;
+    if (msFromGenesis >= 0) {
+      return milliSecondsPerEpoch - (msFromGenesis % milliSecondsPerEpoch);
+    }
+    return Math.abs(msFromGenesis) % milliSecondsPerEpoch;
+  }
+}
+
+/**
+ * Same to the spec but we use Math.round instead of Math.floor.
+ */
+export function getCurrentSlotAround(config: ChainForkConfig, genesisTime: TimeSeconds): Slot {
+  const diffInSeconds = Date.now() / 1000 - genesisTime;
+  const slotsSinceGenesis = Math.round((diffInSeconds * 1000) / config.SLOT_DURATION_MS);
+  return GENESIS_SLOT + slotsSinceGenesis;
+}
+
+// function useEventStream() {
+//   this.stream = this.events.getEventStream([BeaconEventType.BLOCK, BeaconEventType.HEAD, BeaconEventType.CHAIN_REORG]);
+//   pipeToEmitter(this.stream, this).catch((e: Error) => {
+//     this.logger.error("Error on stream pipe", {}, e);
+//   });
+
+//   // On stop
+//   this.stream.stop();
+//   this.stream = null;
+// }

package/src/util/execution.ts

@@ -9,6 +9,7 @@ import {
   bellatrix,
   capella,
   deneb,
+  gloas,
   isBlindedBeaconBlockBody,
   isExecutionPayload,
   ssz,
@@ -127,7 +128,10 @@ export function isCapellaPayloadHeader(
 }
 
 export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: ExecutionPayload): ExecutionPayloadHeader {
-  const transactionsRoot = ssz.bellatrix.Transactions.hashTreeRoot(payload.transactions);
+  const transactionsRoot =
+    fork >= ForkSeq.gloas
+      ? ssz.gloas.Transactions.hashTreeRoot((payload as gloas.ExecutionPayload).transactions)
+      : ssz.bellatrix.Transactions.hashTreeRoot(payload.transactions);
 
   const bellatrixPayloadFields: ExecutionPayloadHeader = {
     parentHash: payload.parentHash,
@@ -147,9 +151,10 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio
   };
 
   if (fork >= ForkSeq.capella) {
-    (bellatrixPayloadFields as capella.ExecutionPayloadHeader).withdrawalsRoot = ssz.capella.Withdrawals.hashTreeRoot(
-      (payload as capella.ExecutionPayload).withdrawals
-    );
+    (bellatrixPayloadFields as capella.ExecutionPayloadHeader).withdrawalsRoot =
+      fork >= ForkSeq.gloas
+        ? ssz.gloas.Withdrawals.hashTreeRoot((payload as gloas.ExecutionPayload).withdrawals)
+        : ssz.capella.Withdrawals.hashTreeRoot((payload as capella.ExecutionPayload).withdrawals);
   }
 
   if (fork >= ForkSeq.deneb) {

package/src/util/gloas.ts

@@ -18,7 +18,6 @@ import {AttestationData} from "@lodestar/types/phase0";
 import {byteArrayEquals} from "@lodestar/utils";
 import {ZERO_HASH} from "../constants/index.js";
 import {CachedBeaconStateFulu, CachedBeaconStateGloas} from "../types.js";
-import {getBlockRootAtSlot} from "./blockRoot.js";
 import {computeDomain} from "./domain.js";
 import {computeEpochAtSlot} from "./epoch.js";
 import {computeEpochShuffling} from "./epochShuffling.js";
@@ -32,11 +31,13 @@ export function isBuilderWithdrawalCredential(withdrawalCredentials: Uint8Array)
 }
 
 export function getBuilderPaymentQuorumThreshold(state: CachedBeaconStateGloas): number {
-  const quorum =
-    Math.floor((state.epochCtx.totalActiveBalanceIncrements * EFFECTIVE_BALANCE_INCREMENT) / SLOTS_PER_EPOCH) *
-    BUILDER_PAYMENT_THRESHOLD_NUMERATOR;
+  // total active balance exceeds Number.MAX_SAFE_INTEGER at mainnet scale, keep the intermediate math in bigint
+  const perSlotBalance =
+    (BigInt(state.epochCtx.totalActiveBalanceIncrements) * BigInt(EFFECTIVE_BALANCE_INCREMENT)) /
+    BigInt(SLOTS_PER_EPOCH);
+  const quorum = perSlotBalance * BigInt(BUILDER_PAYMENT_THRESHOLD_NUMERATOR);
 
-  return Math.floor(quorum / BUILDER_PAYMENT_THRESHOLD_DENOMINATOR);
+  return Number(quorum / BigInt(BUILDER_PAYMENT_THRESHOLD_DENOMINATOR));
 }
 
 function hasBuilderIndexFlag(index: number): boolean {
@@ -188,15 +189,9 @@ export function findBuilderIndexByPubkey(state: CachedBeaconStateGloas, pubkey:
   return null;
 }
 
-export function isAttestationSameSlot(state: CachedBeaconStateGloas, data: AttestationData): boolean {
-  if (data.slot === 0) return true;
-
-  const isMatchingBlockRoot = byteArrayEquals(data.beaconBlockRoot, getBlockRootAtSlot(state, data.slot));
-  const isCurrentBlockRoot = !byteArrayEquals(data.beaconBlockRoot, getBlockRootAtSlot(state, data.slot - 1));
-
-  return isMatchingBlockRoot && isCurrentBlockRoot;
-}
-
+/**
+ * Use cached block roots to avoid repeated state root lookups while matching the spec's is_attestation_same_slot behavior.
+ */
 export function isAttestationSameSlotRootCache(rootCache: RootCache, data: AttestationData): boolean {
   if (data.slot === 0) return true;
 

package/src/util/index.ts

@@ -6,6 +6,7 @@ export * from "./balance.js";
 export * from "./blindedBlock.js";
 export * from "./blockRoot.js";
 export * from "./capella.js";
+export * from "./clock.js";
 export * from "./computeAnchorCheckpoint.js";
 export * from "./deposit.js";
 export * from "./domain.js";
@@ -25,6 +26,7 @@ export * from "./shuffling.js";
 export * from "./signatureSets.js";
 export * from "./signingRoot.js";
 export * from "./slot.js";
+export * from "./ssz.js";
 export * from "./syncCommittee.js";
 export * from "./validator.js";
 export * from "./weakSubjectivity.js";

package/src/util/ssz.ts

@@ -0,0 +1,42 @@
+import {BranchNode, LeafNode, Node, zeroNode} from "@chainsafe/persistent-merkle-tree";
+import {progressiveSubtreeFillToContents} from "@chainsafe/ssz";
+
+// TODO: move these utils to @chainsafe/ssz (progressive.ts, next to progressiveSubtreeFillToContents)
+
+/** Root node (chunks + length mix-in) of a zero-filled ProgressiveListBasicType of `length` items */
+export function zeroProgressiveListBasicRootNode(itemsPerChunk: number, length: number): Node {
+  const chunkCount = Math.ceil(length / itemsPerChunk);
+  // mirrors ssz's progressiveSubtreeCount (subtree capacities 1, 4, 16, ...)
+  let numSubtrees = 0;
+  for (let remaining = chunkCount, subtreeLength = 1; remaining > 0; subtreeLength *= 4) {
+    remaining -= Math.min(remaining, subtreeLength);
+    numSubtrees++;
+  }
+  return new BranchNode(zeroProgressiveNode(numSubtrees), LeafNode.fromUint32(length));
+}
+
+/**
+ * Root node of a progressive list from its chunk/element nodes + length mix-in.
+ * `nodes` are packed 32-byte chunk leaves for basic lists, or element root nodes for composite lists.
+ */
+export function progressiveListRootNode(nodes: Node[], length: number): Node {
+  return new BranchNode(progressiveSubtreeFillToContents(nodes), LeafNode.fromUint32(length));
+}
+
+/**
+ * Return the chunks node of a zero-filled progressive merkle list spanning `numSubtrees`
+ * balanced subtrees (chunk capacities 1, 4, 16, ... = 4^i; depths 0, 2, 4, ... = 2i).
+ *
+ * Not memoized: the zero data is already fully shared via the cached zeroNode(2i) subtrees;
+ * only the O(numSubtrees) spine BranchNodes (~10 at 2M validators) are allocated per call.
+ */
+function zeroProgressiveNode(numSubtrees: number): Node {
+  // chain(k) = B(zeroNode(0), B(zeroNode(2), ... B(zeroNode(2(k-1)), zeroNode(0))))
+  let node: Node = zeroNode(0); // terminator
+  for (let i = numSubtrees - 1; i >= 0; i--) {
+    // BranchNode(left: balanced subtree i, right: rest of chain) — chain grows to the right,
+    // same as ssz progressiveSubtreeFillToContents: `root = new BranchNode(subtreeRoots[i], root)`
+    node = new BranchNode(zeroNode(2 * i), node);
+  }
+  return node;
+}