@lodestar/beacon-node 1.36.0-dev.8ac1ed5b17 → 1.36.0-dev.9dbc67b579

package/src/chain/forkChoice/index.ts

@@ -5,17 +5,22 @@ import {
   ForkChoiceStore,
   JustifiedBalancesGetter,
   ProtoArray,
+  ProtoBlock,
   ForkChoiceOpts as RawForkChoiceOpts,
 } from "@lodestar/fork-choice";
+import {ZERO_HASH_HEX} from "@lodestar/params";
 import {
   CachedBeaconStateAllForks,
   DataAvailabilityStatus,
   computeAnchorCheckpoint,
+  computeEpochAtSlot,
+  computeStartSlotAtEpoch,
+  getBlockRootAtSlot,
   getEffectiveBalanceIncrementsZeroInactive,
   isExecutionStateType,
   isMergeTransitionComplete,
 } from "@lodestar/state-transition";
-import {Slot} from "@lodestar/types";
+import {Slot, ssz} from "@lodestar/types";
 import {Logger, toRootHex} from "@lodestar/utils";
 import {GENESIS_SLOT} from "../../constants/index.js";
 import {Metrics} from "../../metrics/index.js";
@@ -35,6 +40,43 @@ export enum ForkchoiceCaller {
  * Fork Choice extended with a ChainEventEmitter
  */
 export function initializeForkChoice(
+  config: ChainForkConfig,
+  emitter: ChainEventEmitter,
+  currentSlot: Slot,
+  state: CachedBeaconStateAllForks,
+  isFinalizedState: boolean,
+  opts: ForkChoiceOpts,
+  justifiedBalancesGetter: JustifiedBalancesGetter,
+  metrics: Metrics | null,
+  logger?: Logger
+): ForkChoice {
+  return isFinalizedState
+    ? initializeForkChoiceFromFinalizedState(
+        config,
+        emitter,
+        currentSlot,
+        state,
+        opts,
+        justifiedBalancesGetter,
+        metrics,
+        logger
+      )
+    : initializeForkChoiceFromUnfinalizedState(
+        config,
+        emitter,
+        currentSlot,
+        state,
+        opts,
+        justifiedBalancesGetter,
+        metrics,
+        logger
+      );
+}
+
+/**
+ * Initialize forkchoice from a finalized state.
+ */
+export function initializeForkChoiceFromFinalizedState(
   config: ChainForkConfig,
   emitter: ChainEventEmitter,
   currentSlot: Slot,
@@ -82,7 +124,7 @@ export function initializeForkChoice(
         parentRoot: toRootHex(blockHeader.parentRoot),
         stateRoot: toRootHex(blockHeader.stateRoot),
         blockRoot: toRootHex(checkpoint.root),
-        timeliness: true, // Optimisitcally assume is timely
+        timeliness: true, // Optimistically assume is timely
 
         justifiedEpoch: justifiedCheckpoint.epoch,
         justifiedRoot: toRootHex(justifiedCheckpoint.root),
@@ -111,3 +153,136 @@ export function initializeForkChoice(
     logger
   );
 }
+
+/**
+ * Initialize forkchoice from an unfinalized state.
+ */
+export function initializeForkChoiceFromUnfinalizedState(
+  config: ChainForkConfig,
+  emitter: ChainEventEmitter,
+  currentSlot: Slot,
+  unfinalizedState: CachedBeaconStateAllForks,
+  opts: ForkChoiceOpts,
+  justifiedBalancesGetter: JustifiedBalancesGetter,
+  metrics: Metrics | null,
+  logger?: Logger
+): ForkChoice {
+  const {blockHeader} = computeAnchorCheckpoint(config, unfinalizedState);
+  const finalizedCheckpoint = unfinalizedState.finalizedCheckpoint.toValue();
+  const justifiedCheckpoint = unfinalizedState.currentJustifiedCheckpoint.toValue();
+  const headRoot = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader));
+
+  const logCtx = {
+    currentSlot: currentSlot,
+    stateSlot: unfinalizedState.slot,
+    headSlot: blockHeader.slot,
+    headRoot: headRoot,
+    finalizedEpoch: finalizedCheckpoint.epoch,
+    finalizedRoot: toRootHex(finalizedCheckpoint.root),
+    justifiedEpoch: justifiedCheckpoint.epoch,
+    justifiedRoot: toRootHex(justifiedCheckpoint.root),
+  };
+  logger?.warn("Initializing fork choice from unfinalized state", logCtx);
+
+  // this is not the justified state, but there is no other ways to get justified balances
+  const justifiedBalances = getEffectiveBalanceIncrementsZeroInactive(unfinalizedState);
+  const store = new ForkChoiceStore(
+    currentSlot,
+    justifiedCheckpoint,
+    finalizedCheckpoint,
+    justifiedBalances,
+    justifiedBalancesGetter,
+    {
+      onJustified: (cp) => emitter.emit(ChainEvent.forkChoiceJustified, cp),
+      onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp),
+    }
+  );
+
+  // this is the same to the finalized state
+  const headBlock: ProtoBlock = {
+    slot: blockHeader.slot,
+    parentRoot: toRootHex(blockHeader.parentRoot),
+    stateRoot: toRootHex(blockHeader.stateRoot),
+    blockRoot: headRoot,
+    targetRoot: headRoot,
+    timeliness: true, // Optimistically assume is timely
+
+    justifiedEpoch: justifiedCheckpoint.epoch,
+    justifiedRoot: toRootHex(justifiedCheckpoint.root),
+    finalizedEpoch: finalizedCheckpoint.epoch,
+    finalizedRoot: toRootHex(finalizedCheckpoint.root),
+    unrealizedJustifiedEpoch: justifiedCheckpoint.epoch,
+    unrealizedJustifiedRoot: toRootHex(justifiedCheckpoint.root),
+    unrealizedFinalizedEpoch: finalizedCheckpoint.epoch,
+    unrealizedFinalizedRoot: toRootHex(finalizedCheckpoint.root),
+
+    ...(isExecutionStateType(unfinalizedState) && isMergeTransitionComplete(unfinalizedState)
+      ? {
+          executionPayloadBlockHash: toRootHex(unfinalizedState.latestExecutionPayloadHeader.blockHash),
+          executionPayloadNumber: unfinalizedState.latestExecutionPayloadHeader.blockNumber,
+          executionStatus: blockHeader.slot === GENESIS_SLOT ? ExecutionStatus.Valid : ExecutionStatus.Syncing,
+        }
+      : {executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}),
+
+    dataAvailabilityStatus: DataAvailabilityStatus.PreData,
+  };
+
+  const parentSlot = blockHeader.slot - 1;
+  const parentEpoch = computeEpochAtSlot(parentSlot);
+  // parent of head block
+  const parentBlock: ProtoBlock = {
+    ...headBlock,
+    slot: parentSlot,
+    // link this to the dummy justified block
+    parentRoot: toRootHex(justifiedCheckpoint.root),
+    // dummy data, we're not able to regen state before headBlock
+    stateRoot: ZERO_HASH_HEX,
+    blockRoot: headBlock.parentRoot,
+    targetRoot: toRootHex(getBlockRootAtSlot(unfinalizedState, computeStartSlotAtEpoch(parentEpoch))),
+  };
+
+  const justifiedBlock: ProtoBlock = {
+    ...headBlock,
+    slot: computeStartSlotAtEpoch(justifiedCheckpoint.epoch),
+    // link this to the finalized root so that getAncestors can find the finalized block
+    parentRoot: toRootHex(finalizedCheckpoint.root),
+    // dummy data, we're not able to regen state before headBlock
+    stateRoot: ZERO_HASH_HEX,
+    blockRoot: toRootHex(justifiedCheckpoint.root),
+    // same to blockRoot
+    targetRoot: toRootHex(justifiedCheckpoint.root),
+  };
+
+  const finalizedBlock: ProtoBlock = {
+    ...headBlock,
+    slot: computeStartSlotAtEpoch(finalizedCheckpoint.epoch),
+    // we don't care parent of finalized block
+    parentRoot: ZERO_HASH_HEX,
+    // dummy data, we're not able to regen state before headBlock
+    stateRoot: ZERO_HASH_HEX,
+    blockRoot: toRootHex(finalizedCheckpoint.root),
+    // same to blockRoot
+    targetRoot: toRootHex(finalizedCheckpoint.root),
+  };
+
+  const protoArray = ProtoArray.initialize(finalizedBlock, currentSlot);
+  protoArray.onBlock(justifiedBlock, currentSlot);
+  protoArray.onBlock(parentBlock, currentSlot);
+  protoArray.onBlock(headBlock, currentSlot);
+
+  logger?.verbose("Initialized protoArray successfully", {...logCtx, length: protoArray.length()});
+
+  // forkchoiceConstructor is only used for some test cases
+  // production code use ForkChoice constructor directly
+  const forkchoiceConstructor = opts.forkchoiceConstructor ?? ForkChoice;
+
+  return new forkchoiceConstructor(
+    config,
+    store,
+    protoArray,
+    unfinalizedState.validators.length,
+    metrics,
+    opts,
+    logger
+  );
+}

package/src/chain/interface.ts

@@ -178,6 +178,8 @@ export interface IBeaconChain {
     stateRoot: RootHex,
     opts?: StateGetOpts
   ): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null>;
+  /** Return serialized bytes of a persisted checkpoint state */
+  getPersistedCheckpointState(checkpoint?: phase0.Checkpoint): Promise<Uint8Array | null>;
   /** Returns a cached state by checkpoint */
   getStateByCheckpoint(
     checkpoint: CheckpointWithHex

package/src/chain/stateCache/datastore/db.ts

@@ -1,5 +1,11 @@
-import {phase0, ssz} from "@lodestar/types";
+import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import {Epoch, phase0, ssz} from "@lodestar/types";
+import {MapDef} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/interface.js";
+import {
+  getLastProcessedSlotFromBeaconStateSerialized,
+  getSlotFromBeaconStateSerialized,
+} from "../../../util/sszBytes.js";
 import {CPStateDatastore, DatastoreKey} from "./types.js";
 
 /**
@@ -22,6 +28,13 @@ export class DbCPStateDatastore implements CPStateDatastore {
     return this.db.checkpointState.getBinary(serializedCheckpoint);
   }
 
+  async readLatestSafe(): Promise<Uint8Array | null> {
+    const allKeys = await this.readKeys();
+    if (allKeys.length === 0) return null;
+
+    return getLatestSafeDatastoreKey(allKeys, this.read.bind(this));
+  }
+
   async readKeys(): Promise<DatastoreKey[]> {
     return this.db.checkpointState.keys();
   }
@@ -34,3 +47,78 @@ export function datastoreKeyToCheckpoint(key: DatastoreKey): phase0.Checkpoint {
 export function checkpointToDatastoreKey(cp: phase0.Checkpoint): DatastoreKey {
   return ssz.phase0.Checkpoint.serialize(cp);
 }
+
+/**
+ * Get the latest safe checkpoint state the node can use to boot from
+ *   - it should be the checkpoint state that's unique in its epoch
+ *   - its last processed block slot should be at epoch boundary or last slot of previous epoch
+ *   - state slot should be at epoch boundary
+ *   - state slot should be equal to epoch * SLOTS_PER_EPOCH
+ *
+ * return the serialized data of Current Root Checkpoint State (CRCS) or Previous Root Checkpoint State (PRCS)
+ *
+ */
+export async function getLatestSafeDatastoreKey(
+  allKeys: DatastoreKey[],
+  readFn: (key: DatastoreKey) => Promise<Uint8Array | null>
+): Promise<Uint8Array | null> {
+  const checkpointsByEpoch = new MapDef<Epoch, DatastoreKey[]>(() => []);
+  for (const key of allKeys) {
+    const cp = datastoreKeyToCheckpoint(key);
+    checkpointsByEpoch.getOrDefault(cp.epoch).push(key);
+  }
+
+  const dataStoreKeyByEpoch: Map<Epoch, DatastoreKey> = new Map();
+  for (const [epoch, keys] of checkpointsByEpoch.entries()) {
+    // only consider epochs with a single checkpoint to avoid ambiguity from forks
+    if (keys.length === 1) {
+      dataStoreKeyByEpoch.set(epoch, keys[0]);
+    }
+  }
+
+  const epochsDesc = Array.from(dataStoreKeyByEpoch.keys()).sort((a, b) => b - a);
+  for (const epoch of epochsDesc) {
+    const datastoreKey = dataStoreKeyByEpoch.get(epoch);
+    if (datastoreKey == null) {
+      // should not happen
+      continue;
+    }
+
+    const stateBytes = await readFn(datastoreKey);
+    if (stateBytes == null) {
+      // should not happen
+      continue;
+    }
+
+    const lastProcessedSlot = getLastProcessedSlotFromBeaconStateSerialized(stateBytes);
+    if (lastProcessedSlot == null) {
+      // cannot extract last processed slot from serialized state, skip
+      continue;
+    }
+
+    const stateSlot = getSlotFromBeaconStateSerialized(stateBytes);
+    if (stateSlot == null) {
+      // cannot extract slot from serialized state, skip
+      continue;
+    }
+
+    if (lastProcessedSlot !== stateSlot && lastProcessedSlot !== stateSlot - 1) {
+      // not CRCS or PRCS, skip
+      continue;
+    }
+
+    if (stateSlot % SLOTS_PER_EPOCH !== 0) {
+      // not at epoch boundary, skip
+      continue;
+    }
+
+    if (stateSlot !== SLOTS_PER_EPOCH * epoch) {
+      // should not happen after above checks, but just to be safe
+      continue;
+    }
+
+    return stateBytes;
+  }
+
+  return null;
+}

package/src/chain/stateCache/datastore/file.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 import {phase0, ssz} from "@lodestar/types";
 import {fromHex, toHex} from "@lodestar/utils";
 import {ensureDir, readFile, readFileNames, removeFile, writeIfNotExist} from "../../../util/file.js";
+import {getLatestSafeDatastoreKey} from "./db.js";
 import {CPStateDatastore, DatastoreKey} from "./types.js";
 
 const CHECKPOINT_STATES_FOLDER = "checkpoint_states";
@@ -44,6 +45,13 @@ export class FileCPStateDatastore implements CPStateDatastore {
     return readFile(filePath);
   }
 
+  async readLatestSafe(): Promise<Uint8Array | null> {
+    const allKeys = await this.readKeys();
+    if (allKeys.length === 0) return null;
+
+    return getLatestSafeDatastoreKey(allKeys, this.read.bind(this));
+  }
+
   async readKeys(): Promise<DatastoreKey[]> {
     const fileNames = await readFileNames(this.folderPath);
     return fileNames

package/src/chain/stateCache/datastore/types.ts

@@ -8,6 +8,7 @@ export interface CPStateDatastore {
   write: (cpKey: phase0.Checkpoint, stateBytes: Uint8Array) => Promise<DatastoreKey>;
   remove: (key: DatastoreKey) => Promise<void>;
   read: (key: DatastoreKey) => Promise<Uint8Array | null>;
+  readLatestSafe: () => Promise<Uint8Array | null>;
   readKeys: () => Promise<DatastoreKey[]>;
   init?: () => Promise<void>;
 }

package/src/index.ts

@@ -3,6 +3,8 @@
 export type {RestApiServerMetrics, RestApiServerModules, RestApiServerOpts} from "./api/rest/base.js";
 export {RestApiServer} from "./api/rest/base.js";
 export {checkAndPersistAnchorState, initStateFromDb, initStateFromEth1} from "./chain/index.js";
+export {DbCPStateDatastore} from "./chain/stateCache/datastore/db.js";
+export {FileCPStateDatastore} from "./chain/stateCache/datastore/file.js";
 export {BeaconDb, type IBeaconDb} from "./db/index.js";
 export {Eth1Provider, type IEth1Provider} from "./eth1/index.js";
 // Export metrics utilities to de-duplicate validator metrics

package/src/network/reqresp/handlers/beaconBlocksByRange.ts

@@ -19,7 +19,9 @@ export async function* onBeaconBlocksByRange(
 
   const finalized = db.blockArchive;
   const unfinalized = db.block;
-  const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
+  // in the case of initializing from a non-finalized state, we don't have the finalized block so this api does not work
+  // chain.forkChoice.getFinalizeBlock().slot
+  const finalizedSlot = chain.forkChoice.getFinalizedCheckpointSlot();
 
   // Finalized range of blocks
   if (startSlot <= finalizedSlot) {

package/src/node/nodejs.ts

@@ -53,6 +53,7 @@ export type BeaconNodeInitModules = {
   dataDir: string;
   peerStoreDir?: string;
   anchorState: BeaconStateAllForks;
+  isAnchorStateFinalized: boolean;
   wsCheckpoint?: phase0.Checkpoint;
   metricsRegistries?: Registry[];
 };
@@ -154,6 +155,7 @@ export class BeaconNode {
     dataDir,
     peerStoreDir,
     anchorState,
+    isAnchorStateFinalized,
     wsCheckpoint,
     metricsRegistries = [],
   }: BeaconNodeInitModules): Promise<T> {
@@ -217,6 +219,7 @@ export class BeaconNode {
       metrics,
       validatorMonitor,
       anchorState,
+      isAnchorStateFinalized,
       eth1: initializeEth1ForBlockProduction(opts.eth1, {
         config,
         db,

package/src/sync/range/range.ts

@@ -114,13 +114,14 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
    */
   addPeer(peerId: PeerIdStr, localStatus: Status, peerStatus: Status): void {
     // Compute if we should do a Finalized or Head sync with this peer
-    const {syncType, startEpoch, target} = getRangeSyncTarget(localStatus, peerStatus, this.chain.forkChoice);
+    const {syncType, startEpoch, target} = getRangeSyncTarget(localStatus, peerStatus, this.chain);
     this.logger.debug("Sync peer joined", {
       peer: peerId,
       syncType,
       startEpoch,
       targetSlot: target.slot,
       targetRoot: toRootHex(target.root),
+      localHeadSlot: localStatus.headSlot,
       earliestAvailableSlot: (peerStatus as fulu.Status).earliestAvailableSlot ?? Infinity,
     });
 

package/src/sync/utils/remoteSyncType.ts

@@ -1,6 +1,7 @@
 import {IForkChoice} from "@lodestar/fork-choice";
 import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Slot, Status} from "@lodestar/types";
+import {IBeaconChain} from "../../chain/interface.ts";
 import {ChainTarget} from "../range/utils/index.js";
 
 /** The type of peer relative to our current state */
@@ -103,8 +104,11 @@ export function getRangeSyncType(local: Status, remote: Status, forkChoice: IFor
 export function getRangeSyncTarget(
   local: Status,
   remote: Status,
-  forkChoice: IForkChoice
+  chain: IBeaconChain
 ): {syncType: RangeSyncType; startEpoch: Slot; target: ChainTarget} {
+  const forkChoice = chain.forkChoice;
+
+  // finalized sync
   if (remote.finalizedEpoch > local.finalizedEpoch && !forkChoice.hasBlock(remote.finalizedRoot)) {
     return {
       // If  RangeSyncType.Finalized, the range of blocks fetchable from startEpoch and target must allow to switch
@@ -131,11 +135,26 @@ export function getRangeSyncTarget(
       },
     };
   }
+
+  // we don't want to sync from epoch < minEpoch
+  // if we boot from an unfinalized checkpoint state, we don't want to sync before anchorStateLatestBlockSlot
+  // if we boot from a finalized checkpoint state, anchorStateLatestBlockSlot is trusted and we also don't want to sync before it
+  const minEpoch = Math.max(remote.finalizedEpoch, computeEpochAtSlot(chain.anchorStateLatestBlockSlot));
+
+  // head sync
   return {
     syncType: RangeSyncType.Head,
-    // The new peer has the same finalized (earlier filters should prevent a peer with an
-    // earlier finalized chain from reaching here) and local head will always be >= local finalized.
-    startEpoch: computeEpochAtSlot(local.headSlot),
+    // The new peer has the same finalized `remote.finalizedEpoch == local.finalizedEpoch` since
+    // previous filters should prevent a peer with an earlier finalized chain from reaching here.
+    //
+    // By default and during stable network conditions, the head sync always starts from
+    // the finalized epoch (even though it's the head sync) because finalized epoch is < local head.
+    // This is to prevent the issue noted here https://github.com/ChainSafe/lodestar/pull/7509#discussion_r1984353063.
+    //
+    // During non-finality of the network, when starting from an unfinalized checkpoint state, we don't want
+    // to sync before anchorStateLatestBlockSlot as finalized epoch is too far away. Local head will also be
+    // the same to that value at startup, the head sync always starts from anchorStateLatestBlockSlot in this case.
+    startEpoch: Math.min(computeEpochAtSlot(local.headSlot), minEpoch),
     target: {
       slot: remote.headSlot,
       root: remote.headRoot,

package/src/util/sszBytes.ts

@@ -417,6 +417,36 @@ export function getSlotFromDataColumnSidecarSerialized(data: Uint8Array): Slot |
   return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR);
 }
 
+/**
+ * BeaconState of all forks (up until Electra, check with new forks)
+ * class BeaconState(Container):
+ *   genesis_time: uint64                    - 8 bytes
+ *   genesis_validators_root: Root           - 32 bytes
+ *   slot: Slot                              - 8 bytes
+ *   fork: Fork                              - 16 bytes
+ *   latest_block_header: BeaconBlockHeader  - fixed size
+ *     slot: Slot                            - 8 bytes
+ *
+ */
+
+const BLOCK_HEADER_SLOT_BYTES_POSITION_IN_BEACON_STATE = 8 + 32 + 8 + 16;
+export function getLastProcessedSlotFromBeaconStateSerialized(data: Uint8Array): Slot | null {
+  if (data.length < BLOCK_HEADER_SLOT_BYTES_POSITION_IN_BEACON_STATE + SLOT_SIZE) {
+    return null;
+  }
+
+  return getSlotFromOffset(data, BLOCK_HEADER_SLOT_BYTES_POSITION_IN_BEACON_STATE);
+}
+
+const SLOT_BYTES_POSITION_IN_BEACON_STATE = 8 + 32;
+export function getSlotFromBeaconStateSerialized(data: Uint8Array): Slot | null {
+  if (data.length < SLOT_BYTES_POSITION_IN_BEACON_STATE) {
+    return null;
+  }
+
+  return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_BEACON_STATE);
+}
+
 /**
  * Read only the first 4 bytes of Slot, max value is 4,294,967,295 will be reached 1634 years after genesis
  *