@lodestar/fork-choice 1.44.0-dev.8793c24169 → 1.44.0-dev.9666fc891a

package/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.44.0-dev.8793c24169",
+  "version": "1.44.0-dev.9666fc891a",
   "type": "module",
   "exports": {
     ".": {
@@ -40,11 +40,11 @@
   },
   "dependencies": {
     "@chainsafe/ssz": "^1.4.0",
-    "@lodestar/config": "^1.44.0-dev.8793c24169",
-    "@lodestar/params": "^1.44.0-dev.8793c24169",
-    "@lodestar/state-transition": "^1.44.0-dev.8793c24169",
-    "@lodestar/types": "^1.44.0-dev.8793c24169",
-    "@lodestar/utils": "^1.44.0-dev.8793c24169"
+    "@lodestar/config": "^1.44.0-dev.9666fc891a",
+    "@lodestar/params": "^1.44.0-dev.9666fc891a",
+    "@lodestar/state-transition": "^1.44.0-dev.9666fc891a",
+    "@lodestar/types": "^1.44.0-dev.9666fc891a",
+    "@lodestar/utils": "^1.44.0-dev.9666fc891a"
   },
   "keywords": [
     "ethereum",
@@ -52,5 +52,5 @@
     "beacon",
     "blockchain"
   ],
-  "gitHead": "dc197cf24e7e9a45912f4c018bbb448b1cc3b45a"
+  "gitHead": "584195c4ed5e89d6ef520fe55a444e4c6746a5b5"
 }

package/src/forkChoice/fastConfirmation/data.ts

@@ -0,0 +1,43 @@
+import {computeEpochAtSlot} from "@lodestar/state-transition";
+import {
+  FastConfirmationCache,
+  FastConfirmationContext,
+  FastConfirmationSnapshot,
+  IFastConfirmationStore,
+} from "./types.ts";
+import {getBlock, getUnrealizedJustification} from "./utils.ts";
+
+export function createFastConfirmationCache(): FastConfirmationCache {
+  return {
+    blockByRoot: new Map(),
+    ancestorRoots: new Map(),
+    committeeBySlot: new Map(),
+    isDescendantByRootPair: new Map(),
+    voteWeightBySource: new Map(),
+    checkpointStateByKey: new Map(),
+  };
+}
+
+export function buildFastConfirmationSnapshot(
+  ctx: FastConfirmationContext,
+  store: IFastConfirmationStore,
+  cache: FastConfirmationCache
+): FastConfirmationSnapshot {
+  const currentSlot = ctx.getCurrentSlot();
+  const currentEpoch = computeEpochAtSlot(currentSlot);
+  const headRoot = ctx.getHead().blockRoot;
+  const confirmedRoot = store.confirmedRoot;
+  const confirmedBlock = getBlock(ctx, cache, confirmedRoot);
+
+  return {
+    currentSlot,
+    currentEpoch,
+    headRoot,
+    confirmedRoot,
+    confirmedEpoch: confirmedBlock ? computeEpochAtSlot(confirmedBlock.slot) : null,
+    confirmedSlot: confirmedBlock?.slot ?? null,
+    observedJustified: store.currentEpochObservedJustifiedCheckpoint,
+    headUnrealized: getUnrealizedJustification(ctx, cache, headRoot),
+    finalizedRoot: ctx.getFinalizedCheckpoint().rootHex,
+  };
+}

package/src/forkChoice/fastConfirmation/fastConfirmationRule.ts

@@ -0,0 +1,159 @@
+import {computeEpochAtSlot, isStartSlotOfEpoch} from "@lodestar/state-transition";
+import {RootHex} from "@lodestar/types";
+import {Logger, withObservedDuration} from "@lodestar/utils";
+import {buildFastConfirmationSnapshot, createFastConfirmationCache} from "./data.ts";
+import {FastConfirmationMetrics, FastConfirmationSteps} from "./metrics.ts";
+import {runFastConfirmationRules} from "./rules.ts";
+import {
+  FastConfirmationContext,
+  FastConfirmationResult,
+  IFastConfirmationRule,
+  IFastConfirmationStore,
+} from "./types.ts";
+
+export * from "./metrics.ts";
+export * from "./types.ts";
+
+export class FastConfirmationRule implements IFastConfirmationRule {
+  constructor(
+    private readonly store: IFastConfirmationStore,
+    readonly metrics: FastConfirmationMetrics | null,
+    readonly logger?: Logger
+  ) {}
+
+  getConfirmedRoot(): RootHex {
+    return this.store.confirmedRoot;
+  }
+
+  onSlotStartAfterPastAttestationsApplied(ctx: FastConfirmationContext): FastConfirmationResult {
+    const currentSlot = ctx.getCurrentSlot();
+    const previousConfirmedRoot = this.store.confirmedRoot;
+
+    this.logger?.debug("Running fast confirmation rule", {
+      slot: currentSlot,
+      epoch: computeEpochAtSlot(currentSlot),
+    });
+    this.updateFastConfirmationVariables(ctx);
+
+    const cache = withObservedDuration(
+      this.metrics?.fastConfirmation.stepsDuration.startTimer({
+        step: FastConfirmationSteps.buildCache,
+      }),
+      createFastConfirmationCache
+    );
+
+    const snapshot = withObservedDuration(
+      this.metrics?.fastConfirmation.stepsDuration.startTimer({
+        step: FastConfirmationSteps.buildSnapshot,
+      }),
+      () => buildFastConfirmationSnapshot(ctx, this.store, cache)
+    );
+
+    this.logger?.verbose("Built fast confirmation snapshot", {
+      confirmedSlot: snapshot.confirmedSlot,
+      confirmedEpoch: snapshot.confirmedEpoch,
+      confirmedRoot: snapshot.confirmedRoot,
+      headRoot: snapshot.headRoot,
+      finalizedRoot: snapshot.finalizedRoot,
+      headUnrealizedRoot: snapshot.headUnrealized?.rootHex,
+      headUnrealizedEpoch: snapshot.headUnrealized?.epoch,
+      observedJustifiedRoot: snapshot.observedJustified.rootHex,
+      observedJustifiedEpoch: snapshot.observedJustified.epoch,
+    });
+
+    const {confirmedRoot, didReset, reason} = withObservedDuration(
+      this.metrics?.fastConfirmation.stepsDuration.startTimer({step: FastConfirmationSteps.runRules}),
+      () => runFastConfirmationRules(snapshot, ctx, this.store, cache, this.logger)
+    );
+
+    const changed = confirmedRoot !== previousConfirmedRoot;
+    const confirmedBlock = cache.blockByRoot.get(confirmedRoot) ?? null;
+    const confirmedSlot = confirmedBlock?.slot ?? null;
+    const confirmedEpoch = confirmedBlock ? computeEpochAtSlot(confirmedBlock.slot) : null;
+    const logContext = {
+      previousConfirmedRoot,
+      confirmedRoot,
+      changed,
+      didReset,
+      reason,
+      confirmedSlot,
+      confirmedEpoch,
+    };
+
+    if (changed) {
+      if (didReset) {
+        this.logger?.warn("Reset fast confirmation", logContext);
+      } else {
+        this.logger?.debug("Updated fast confirmation", logContext);
+      }
+    } else {
+      this.logger?.debug("Unchanged fast confirmation", logContext);
+    }
+
+    this.store.confirmedRoot = confirmedRoot;
+    this.updateFastConfirmationMetrics(ctx, {confirmedRoot, didReset});
+
+    return {confirmedRoot, didReset};
+  }
+
+  private updateFastConfirmationVariables(ctx: FastConfirmationContext): void {
+    const previousSlotHead = this.store.currentSlotHead;
+    const currentSlotHead = ctx.getHead().blockRoot;
+    const currentSlot = ctx.getCurrentSlot();
+    const isStartSlotOfCurrentEpoch = isStartSlotOfEpoch(currentSlot);
+    const isLastSlotOfCurrentEpoch = isStartSlotOfEpoch(currentSlot + 1);
+
+    this.store.previousSlotHead = previousSlotHead;
+    this.store.currentSlotHead = currentSlotHead;
+
+    this.logger?.verbose("Updating fast confirmation variables", {
+      previousSlotHead,
+      currentSlotHead,
+      currentSlot,
+      isStartSlotOfCurrentEpoch,
+      isLastSlotOfCurrentEpoch,
+    });
+
+    // Spec step 1: freeze the greatest unrealized justified checkpoint at the
+    // last slot of the epoch so the next epoch consumes a stable snapshot.
+    if (isLastSlotOfCurrentEpoch) {
+      const unrealized = ctx.getUnrealizedJustified();
+      this.store.previousEpochGreatestUnrealizedCheckpoint = unrealized.checkpoint;
+      this.store.previousEpochGreatestUnrealizedBalances = unrealized.balances;
+
+      this.logger?.verbose("Updated fast confirmation greatest unrealized snapshot", {
+        previousEpochGreatestUnrealizedCheckpointRoot: this.store.previousEpochGreatestUnrealizedCheckpoint.rootHex,
+        previousEpochGreatestUnrealizedCheckpointEpoch: this.store.previousEpochGreatestUnrealizedCheckpoint.epoch,
+      });
+    }
+
+    // Spec step 2: rotate observed justified checkpoints at the first slot of
+    // the new epoch using the snapshot taken at the end of the previous epoch.
+    if (isStartSlotOfCurrentEpoch) {
+      this.store.previousEpochObservedJustifiedCheckpoint = this.store.currentEpochObservedJustifiedCheckpoint;
+      this.store.previousEpochObservedJustifiedBalances = this.store.currentEpochObservedJustifiedBalances;
+      this.store.currentEpochObservedJustifiedCheckpoint = this.store.previousEpochGreatestUnrealizedCheckpoint;
+      this.store.currentEpochObservedJustifiedBalances = this.store.previousEpochGreatestUnrealizedBalances;
+
+      this.logger?.verbose("Updated fast confirmation observed justified checkpoints", {
+        previousEpochObservedJustifiedCheckpointRoot: this.store.previousEpochObservedJustifiedCheckpoint.rootHex,
+        previousEpochObservedJustifiedCheckpointEpoch: this.store.previousEpochObservedJustifiedCheckpoint.epoch,
+        currentEpochObservedJustifiedCheckpointRoot: this.store.currentEpochObservedJustifiedCheckpoint.rootHex,
+        currentEpochObservedJustifiedCheckpointEpoch: this.store.currentEpochObservedJustifiedCheckpoint.epoch,
+      });
+    }
+  }
+
+  private updateFastConfirmationMetrics(ctx: FastConfirmationContext, result: FastConfirmationResult): void {
+    if (!this.metrics) return;
+    const confirmedBlock = ctx.getBlock(result.confirmedRoot);
+    if (confirmedBlock) {
+      this.metrics.fastConfirmation.confirmedSlot.set(confirmedBlock.slot);
+      this.metrics.fastConfirmation.confirmedEpoch.set(computeEpochAtSlot(confirmedBlock.slot));
+    }
+    if (result.didReset) {
+      this.metrics.fastConfirmation.resets.inc();
+    }
+    this.metrics.fastConfirmation.votesTracked.set(ctx.getTrackedVotesCount());
+  }
+}

package/src/forkChoice/fastConfirmation/index.ts

@@ -0,0 +1,3 @@
+export * from "./fastConfirmationRule.ts";
+export * from "./metrics.ts";
+export * from "./types.ts";

package/src/forkChoice/fastConfirmation/metrics.ts

@@ -0,0 +1,44 @@
+import {MetricsRegisterExtra} from "@lodestar/utils";
+
+export type FastConfirmationMetrics = ReturnType<typeof getFastConfirmationMetrics>;
+
+export enum FastConfirmationSteps {
+  updateHead = "updateHead",
+  buildCache = "buildCache",
+  buildSnapshot = "buildSnapshot",
+  runRules = "runRules",
+}
+
+export function getFastConfirmationMetrics(register: MetricsRegisterExtra) {
+  return {
+    fastConfirmation: {
+      totalDuration: register.histogram({
+        name: "lodestar_fast_confirmation_duration_seconds",
+        help: "Time to run Fast Confirmation Rule algorithm",
+        buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2],
+      }),
+      stepsDuration: register.histogram<{step: FastConfirmationSteps}>({
+        name: "lodestar_fast_confirmation_steps_duration_seconds",
+        help: "Time to run Fast Confirmation Steps",
+        buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2],
+        labelNames: ["step"],
+      }),
+      confirmedEpoch: register.gauge({
+        name: "lodestar_fast_confirmation_confirmed_epoch",
+        help: "Current confirmed epoch from fast confirmation",
+      }),
+      confirmedSlot: register.gauge({
+        name: "lodestar_fast_confirmation_confirmed_slot",
+        help: "Current confirmed slot from fast confirmation",
+      }),
+      votesTracked: register.gauge({
+        name: "lodestar_fast_confirmation_votes_tracked",
+        help: "Number of checkpoint votes tracked by fast confirmation",
+      }),
+      resets: register.gauge({
+        name: "lodestar_fast_confirmation_resets_total",
+        help: "Count of fast confirmation resets due to reorgs",
+      }),
+    },
+  };
+}

package/src/forkChoice/fastConfirmation/rules.ts

@@ -0,0 +1,124 @@
+import {computeEpochAtSlot, isStartSlotOfEpoch} from "@lodestar/state-transition";
+import {Logger} from "@lodestar/utils";
+import {equalCheckpointWithHex} from "../store.ts";
+import {
+  FastConfirmationCache,
+  FastConfirmationContext,
+  FastConfirmationDecision,
+  FastConfirmationDecisionReason,
+  FastConfirmationRule,
+  FastConfirmationSnapshot,
+  IFastConfirmationStore,
+} from "./types.ts";
+import {findLatestConfirmedDescendant, getBlock, isAncestor, isConfirmedChainSafe} from "./utils.ts";
+
+export const resetIfConfirmedUnavailable: FastConfirmationRule = (snapshot, ctx, _store, cache, decision) => {
+  const confirmedBlock = getBlock(ctx, cache, decision.confirmedRoot);
+  if (!confirmedBlock) {
+    return {
+      confirmedRoot: snapshot.finalizedRoot,
+      didReset: true,
+      reason: FastConfirmationDecisionReason.ConfirmedNotFound,
+    };
+  }
+  return decision;
+};
+
+export const resetIfBehindOrNotAncestorOrUnsafe: FastConfirmationRule = (
+  snapshot,
+  ctx,
+  store,
+  cache,
+  decision,
+  logger
+) => {
+  const confirmedBlock = getBlock(ctx, cache, decision.confirmedRoot);
+  if (!confirmedBlock) return decision;
+  const confirmedEpoch = computeEpochAtSlot(confirmedBlock.slot);
+
+  const confirmedEpochBehindHead = confirmedEpoch + 1 < snapshot.currentEpoch;
+  const notAncestorOfHead = !isAncestor(ctx, cache, snapshot.headRoot, decision.confirmedRoot);
+  const allChildrenNotConfirmed =
+    isStartSlotOfEpoch(snapshot.currentSlot) &&
+    !isConfirmedChainSafe(ctx, store, cache, decision.confirmedRoot, logger);
+
+  if (confirmedEpochBehindHead || notAncestorOfHead || allChildrenNotConfirmed) {
+    const didReset = decision.didReset || decision.confirmedRoot !== snapshot.finalizedRoot;
+    const reason = confirmedEpochBehindHead
+      ? FastConfirmationDecisionReason.ResetBehind
+      : notAncestorOfHead
+        ? FastConfirmationDecisionReason.ResetNotAncestor
+        : FastConfirmationDecisionReason.ResetChainUnsafe;
+    return {confirmedRoot: snapshot.finalizedRoot, didReset, reason};
+  }
+  return decision;
+};
+
+export const advanceIfObservedJustified: FastConfirmationRule = (snapshot, ctx, store, cache, decision) => {
+  if (!isStartSlotOfEpoch(snapshot.currentSlot)) return decision;
+  if (store.currentEpochObservedJustifiedCheckpoint.epoch + 1 !== snapshot.currentEpoch) return decision;
+  if (!snapshot.headUnrealized) return decision;
+  if (!equalCheckpointWithHex(store.currentEpochObservedJustifiedCheckpoint, snapshot.headUnrealized)) return decision;
+  const observedBlock = getBlock(ctx, cache, store.currentEpochObservedJustifiedCheckpoint.rootHex);
+  if (!observedBlock || computeEpochAtSlot(observedBlock.slot) + 1 < snapshot.currentEpoch) return decision;
+
+  const confirmedSlot = getBlock(ctx, cache, decision.confirmedRoot)?.slot ?? null;
+  const observedSlot = observedBlock.slot;
+  if (confirmedSlot !== null && observedSlot !== null && confirmedSlot < observedSlot) {
+    return {
+      ...decision,
+      confirmedRoot: store.currentEpochObservedJustifiedCheckpoint.rootHex,
+      reason: FastConfirmationDecisionReason.ObservedJustified,
+    };
+  }
+  return decision;
+};
+
+export const advanceToLatestConfirmedDescendant: FastConfirmationRule = (
+  snapshot,
+  ctx,
+  store,
+  cache,
+  decision,
+  logger
+) => {
+  const confirmedBlock = getBlock(ctx, cache, decision.confirmedRoot);
+  const confirmedEpoch = confirmedBlock ? computeEpochAtSlot(confirmedBlock.slot) : null;
+  if (confirmedEpoch !== null && confirmedEpoch + 1 >= snapshot.currentEpoch) {
+    const newConfirmed = findLatestConfirmedDescendant(snapshot, ctx, store, cache, decision.confirmedRoot, logger);
+    return {
+      ...decision,
+      confirmedRoot: newConfirmed,
+      reason: FastConfirmationDecisionReason.ConfirmedDescendant,
+    };
+  }
+  return decision;
+};
+
+export const FAST_CONFIRMATION_RULES: FastConfirmationRule[] = [
+  resetIfConfirmedUnavailable,
+  resetIfBehindOrNotAncestorOrUnsafe,
+  advanceIfObservedJustified,
+  advanceToLatestConfirmedDescendant,
+];
+
+// Spec mapping: this rule runner implements the `get_latest_confirmed` decision flow
+// over Lodestar's snapshot/store/cache abstractions.
+export function runFastConfirmationRules(
+  snapshot: FastConfirmationSnapshot,
+  ctx: FastConfirmationContext,
+  store: IFastConfirmationStore,
+  cache: FastConfirmationCache,
+  logger?: Logger
+): FastConfirmationDecision {
+  let decision: FastConfirmationDecision = {
+    confirmedRoot: snapshot.confirmedRoot,
+    didReset: false,
+    reason: FastConfirmationDecisionReason.Unchanged,
+  };
+
+  for (const rule of FAST_CONFIRMATION_RULES) {
+    decision = rule(snapshot, ctx, store, cache, decision, logger);
+  }
+  return decision;
+}

package/src/forkChoice/fastConfirmation/types.ts

@@ -0,0 +1,111 @@
+import {EffectiveBalanceIncrements, IBeaconStateView} from "@lodestar/state-transition";
+import {Epoch, RootHex, Slot, ValidatorIndex} from "@lodestar/types";
+import {Logger} from "@lodestar/utils";
+import {ProtoBlock} from "../../protoArray/interface.ts";
+import {CheckpointWithHex} from "../store.ts";
+
+export type FastConfirmationBalanceSource = {
+  state: IBeaconStateView | null;
+  balances: EffectiveBalanceIncrements;
+};
+
+export type ForkChoiceStateGetter = (
+  opts: {stateRoot: RootHex; checkpoint?: never} | {stateRoot?: never; checkpoint: CheckpointWithHex}
+) => IBeaconStateView | null;
+
+type IFastConfirmationSpecStore = {
+  confirmedRoot: RootHex;
+  previousEpochObservedJustifiedCheckpoint: CheckpointWithHex;
+  currentEpochObservedJustifiedCheckpoint: CheckpointWithHex;
+  previousEpochGreatestUnrealizedCheckpoint: CheckpointWithHex;
+  previousSlotHead: RootHex;
+  currentSlotHead: RootHex;
+};
+
+type IFastConfirmationAuxStore = {
+  previousEpochObservedJustifiedBalances: EffectiveBalanceIncrements;
+  currentEpochObservedJustifiedBalances: EffectiveBalanceIncrements;
+  previousEpochGreatestUnrealizedBalances: EffectiveBalanceIncrements;
+  stateGetter: ForkChoiceStateGetter;
+};
+
+export type IFastConfirmationStore = IFastConfirmationSpecStore & IFastConfirmationAuxStore;
+
+export type FastConfirmationResult = {
+  confirmedRoot: RootHex;
+  didReset?: boolean;
+};
+
+export type FastConfirmationSnapshot = {
+  currentSlot: Slot;
+  currentEpoch: Epoch;
+  headRoot: RootHex;
+  confirmedRoot: RootHex;
+  confirmedEpoch: Epoch | null;
+  confirmedSlot: Slot | null;
+  observedJustified: CheckpointWithHex;
+  headUnrealized: CheckpointWithHex | null;
+  finalizedRoot: RootHex;
+};
+
+export enum FastConfirmationDecisionReason {
+  Unchanged = "unchanged",
+  ConfirmedNotFound = "confirmed_not_found",
+  ResetBehind = "reset_behind",
+  ResetNotAncestor = "reset_not_ancestor",
+  ResetChainUnsafe = "reset_chain_unsafe",
+  ObservedJustified = "observed_justified",
+  ConfirmedDescendant = "confirmed_descendant",
+}
+
+export type FastConfirmationDecision = {
+  confirmedRoot: RootHex;
+  didReset: boolean;
+  reason: FastConfirmationDecisionReason;
+};
+
+export type FastConfirmationRule = (
+  snapshot: FastConfirmationSnapshot,
+  ctx: FastConfirmationContext,
+  store: IFastConfirmationStore,
+  cache: FastConfirmationCache,
+  decision: FastConfirmationDecision,
+  logger?: Logger
+) => FastConfirmationDecision;
+
+export type BalanceSourceKey = "current" | "previous";
+
+// This cache is created once per slot
+export type FastConfirmationCache = {
+  blockByRoot: Map<RootHex, ProtoBlock | null>;
+  ancestorRoots: Map<string, RootHex[] | null>;
+  committeeBySlot: Map<Slot, Set<ValidatorIndex>>;
+  isDescendantByRootPair: Map<string, boolean>;
+  /** voteRoot -> totalWeight, keyed by sourceKey */
+  voteWeightBySource: Map<BalanceSourceKey, Map<RootHex, number>>;
+  headState?: IBeaconStateView;
+  pulledUpHeadState?: IBeaconStateView;
+  checkpointStateByKey: Map<string, IBeaconStateView | null>;
+};
+
+export type FastConfirmationContext = {
+  config: {
+    CONFIRMATION_BYZANTINE_THRESHOLD: number;
+    PROPOSER_SCORE_BOOST: number;
+  };
+  getCurrentSlot(): Slot;
+  getHead(): ProtoBlock;
+  getBlock(root: RootHex): ProtoBlock | null;
+  getAncestor(root: RootHex, slot: Slot): RootHex;
+  isDescendant(ancestor: RootHex, descendant: RootHex): boolean;
+  getLatestMessage(validatorIndex: ValidatorIndex): {root: RootHex; epoch: Epoch} | null;
+  getUnrealizedJustified(): {checkpoint: CheckpointWithHex; balances: EffectiveBalanceIncrements};
+  getFinalizedCheckpoint(): CheckpointWithHex;
+  getEquivocatingIndices(): Set<ValidatorIndex>;
+  getTrackedVotesCount(): number;
+};
+
+export interface IFastConfirmationRule {
+  getConfirmedRoot(): RootHex;
+  onSlotStartAfterPastAttestationsApplied(ctx: FastConfirmationContext): FastConfirmationResult;
+}