@lodestar/beacon-node 1.36.0-dev.f259361847 → 1.36.0-dev.f3703b7882

package/src/sync/utils/downloadByRoot.ts

@@ -9,11 +9,11 @@ import {
   isForkPostFulu,
 } from "@lodestar/params";
 import {BeaconBlockBody, BlobIndex, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types";
-import {LodestarError, fromHex, prettyBytes, prettyPrintIndices, toHex, toRootHex} from "@lodestar/utils";
+import {LodestarError, fromHex, prettyPrintIndices, toHex, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../../chain/blocks/blockInput/types.js";
 import {ChainEventEmitter} from "../../chain/emitter.js";
-import {SeenBlockInput} from "../../chain/seenCache/seenGossipBlockInput.js";
+import {IBeaconChain} from "../../chain/interface.ts";
 import {validateBlockBlobSidecars} from "../../chain/validation/blobSidecar.js";
 import {validateBlockDataColumnSidecars} from "../../chain/validation/dataColumnSidecar.js";
 import {INetwork} from "../../network/interface.js";
@@ -32,6 +32,7 @@ import {
 
 export type FetchByRootCoreProps = {
   config: ChainForkConfig;
+  chain: IBeaconChain | null; // null for testing purposes
   network: INetwork;
   peerMeta: PeerSyncMeta;
 };
@@ -63,13 +64,13 @@ export type FetchByRootResponses = {
 
 export type DownloadByRootProps = FetchByRootCoreProps & {
   cacheItem: BlockInputSyncCacheItem;
-  seenCache: SeenBlockInput;
+  chain: IBeaconChain;
   emitter: ChainEventEmitter;
 };
 
 export async function downloadByRoot({
   config,
-  seenCache,
+  chain,
   network,
   emitter,
   peerMeta,
@@ -84,6 +85,7 @@ export async function downloadByRoot({
     warnings,
   } = await fetchByRoot({
     config,
+    chain,
     network,
     cacheItem,
     blockRoot,
@@ -103,7 +105,7 @@ export async function downloadByRoot({
       });
     }
   } else {
-    blockInput = seenCache.getByBlock({
+    blockInput = chain.seenBlockInputCache.getByBlock({
       block,
       peerIdStr,
       blockRootHex: rootHex,
@@ -119,7 +121,7 @@ export async function downloadByRoot({
     if (!blobSidecars) {
       throw new DownloadByRootError({
         code: DownloadByRootErrorCode.MISSING_BLOB_RESPONSE,
-        blockRoot: prettyBytes(rootHex),
+        blockRoot: rootHex,
         peer: peerIdStr,
       });
     }
@@ -157,7 +159,7 @@ export async function downloadByRoot({
     if (!columnSidecars) {
       throw new DownloadByRootError({
         code: DownloadByRootErrorCode.MISSING_COLUMN_RESPONSE,
-        blockRoot: prettyBytes(rootHex),
+        blockRoot: rootHex,
         peer: peerIdStr,
       });
     }
@@ -210,6 +212,7 @@ export async function downloadByRoot({
 
 export async function fetchByRoot({
   config,
+  chain,
   network,
   peerMeta,
   blockRoot,
@@ -237,6 +240,7 @@ export async function fetchByRoot({
       if (isBlockInputBlobs(cacheItem.blockInput)) {
         blobSidecars = await fetchAndValidateBlobs({
           config,
+          chain,
           network,
           peerIdStr,
           forkName: forkName as ForkPreFulu,
@@ -248,6 +252,7 @@ export async function fetchByRoot({
       if (isBlockInputColumns(cacheItem.blockInput)) {
         columnSidecarResult = await fetchAndValidateColumns({
           config,
+          chain,
           network,
           peerMeta,
           forkName: forkName as ForkPostFulu,
@@ -268,6 +273,7 @@ export async function fetchByRoot({
     if (isForkPostFulu(forkName)) {
       columnSidecarResult = await fetchAndValidateColumns({
         config,
+        chain,
         network,
         peerMeta,
         forkName,
@@ -280,6 +286,7 @@ export async function fetchByRoot({
       const blobCount = commitments.length;
       blobSidecars = await fetchAndValidateBlobs({
         config,
+        chain,
         network,
         peerIdStr,
         forkName: forkName as ForkPreFulu,
@@ -305,14 +312,14 @@ export async function fetchAndValidateBlock({
   network,
   peerIdStr,
   blockRoot,
-}: FetchByRootAndValidateBlockProps): Promise<SignedBeaconBlock> {
+}: Omit<FetchByRootAndValidateBlockProps, "chain">): Promise<SignedBeaconBlock> {
   const response = await network.sendBeaconBlocksByRoot(peerIdStr, [blockRoot]);
   const block = response.at(0)?.data;
   if (!block) {
     throw new DownloadByRootError({
       code: DownloadByRootErrorCode.MISSING_BLOCK_RESPONSE,
       peer: prettyPrintPeerIdStr(peerIdStr),
-      blockRoot: prettyBytes(blockRoot),
+      blockRoot: toRootHex(blockRoot),
     });
   }
   const receivedRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
@@ -321,8 +328,8 @@ export async function fetchAndValidateBlock({
       {
         code: DownloadByRootErrorCode.MISMATCH_BLOCK_ROOT,
         peer: prettyPrintPeerIdStr(peerIdStr),
-        requestedBlockRoot: prettyBytes(blockRoot),
-        receivedBlockRoot: prettyBytes(toRootHex(receivedRoot)),
+        requestedBlockRoot: toRootHex(blockRoot),
+        receivedBlockRoot: toRootHex(receivedRoot),
       },
       "block does not match requested root"
     );
@@ -331,6 +338,7 @@ export async function fetchAndValidateBlock({
 }
 
 export async function fetchAndValidateBlobs({
+  chain,
   network,
   peerIdStr,
   blockRoot,
@@ -344,7 +352,7 @@ export async function fetchAndValidateBlobs({
     missing,
   });
 
-  await validateBlockBlobSidecars(block.message.slot, blockRoot, missing.length, blobSidecars);
+  await validateBlockBlobSidecars(chain, block.message.slot, blockRoot, missing.length, blobSidecars);
 
   return blobSidecars;
 }
@@ -368,6 +376,7 @@ export async function fetchBlobsByRoot({
 }
 
 export async function fetchAndValidateColumns({
+  chain,
   network,
   peerMeta,
   block,
@@ -438,7 +447,7 @@ export async function fetchAndValidateColumns({
     );
   }
 
-  await validateBlockDataColumnSidecars(slot, blockRoot, blobCount, columnSidecars);
+  await validateBlockDataColumnSidecars(chain, slot, blockRoot, blobCount, columnSidecars);
 
   return {result: columnSidecars, warnings: warnings.length > 0 ? warnings : null};
 }
@@ -456,49 +465,6 @@ export async function fetchColumnsByRoot({
   return await network.sendDataColumnSidecarsByRoot(peerMeta.peerId, [{blockRoot, columns: missing}]);
 }
 
-// TODO(fulu) not in use, remove?
-export type ValidateColumnSidecarsProps = Pick<
-  FetchByRootAndValidateColumnsProps,
-  "config" | "peerMeta" | "blockRoot" | "missing"
-> & {
-  slot: number;
-  blobCount: number;
-  needed?: fulu.DataColumnSidecars;
-  needToPublish?: fulu.DataColumnSidecars;
-};
-
-// TODO(fulu) not in use, remove?
-export async function validateColumnSidecars({
-  peerMeta,
-  slot,
-  blockRoot,
-  blobCount,
-  missing,
-  needed = [],
-  needToPublish = [],
-}: ValidateColumnSidecarsProps): Promise<void> {
-  const requestedIndices = missing;
-  const extraIndices: number[] = [];
-  for (const columnSidecar of needed) {
-    if (!requestedIndices.includes(columnSidecar.index)) {
-      extraIndices.push(columnSidecar.index);
-    }
-  }
-  if (extraIndices.length > 0) {
-    throw new DownloadByRootError(
-      {
-        code: DownloadByRootErrorCode.EXTRA_SIDECAR_RECEIVED,
-        peer: prettyPrintPeerIdStr(peerMeta.peerId),
-        slot,
-        blockRoot: prettyBytes(blockRoot),
-        invalidIndices: prettyPrintIndices(extraIndices),
-      },
-      "Received a columnSidecar that was not requested"
-    );
-  }
-  await validateBlockDataColumnSidecars(slot, blockRoot, blobCount, [...needed, ...needToPublish]);
-}
-
 export enum DownloadByRootErrorCode {
   MISMATCH_BLOCK_ROOT = "DOWNLOAD_BY_ROOT_ERROR_MISMATCH_BLOCK_ROOT",
   EXTRA_SIDECAR_RECEIVED = "DOWNLOAD_BY_ROOT_ERROR_EXTRA_SIDECAR_RECEIVED",

package/src/sync/utils/remoteSyncType.ts

@@ -1,6 +1,7 @@
 import {IForkChoice} from "@lodestar/fork-choice";
 import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Slot, Status} from "@lodestar/types";
+import {IBeaconChain} from "../../chain/interface.ts";
 import {ChainTarget} from "../range/utils/index.js";
 
 /** The type of peer relative to our current state */
@@ -103,8 +104,11 @@ export function getRangeSyncType(local: Status, remote: Status, forkChoice: IFor
 export function getRangeSyncTarget(
   local: Status,
   remote: Status,
-  forkChoice: IForkChoice
+  chain: IBeaconChain
 ): {syncType: RangeSyncType; startEpoch: Slot; target: ChainTarget} {
+  const forkChoice = chain.forkChoice;
+
+  // finalized sync
   if (remote.finalizedEpoch > local.finalizedEpoch && !forkChoice.hasBlock(remote.finalizedRoot)) {
     return {
       // If  RangeSyncType.Finalized, the range of blocks fetchable from startEpoch and target must allow to switch
@@ -131,11 +135,26 @@ export function getRangeSyncTarget(
       },
     };
   }
+
+  // we don't want to sync from epoch < minEpoch
+  // if we boot from an unfinalized checkpoint state, we don't want to sync before anchorStateLatestBlockSlot
+  // if we boot from a finalized checkpoint state, anchorStateLatestBlockSlot is trusted and we also don't want to sync before it
+  const minEpoch = Math.max(remote.finalizedEpoch, computeEpochAtSlot(chain.anchorStateLatestBlockSlot));
+
+  // head sync
   return {
     syncType: RangeSyncType.Head,
-    // The new peer has the same finalized (earlier filters should prevent a peer with an
-    // earlier finalized chain from reaching here) and local head will always be >= local finalized.
-    startEpoch: computeEpochAtSlot(local.headSlot),
+    // The new peer has the same finalized `remote.finalizedEpoch == local.finalizedEpoch` since
+    // previous filters should prevent a peer with an earlier finalized chain from reaching here.
+    //
+    // By default and during stable network conditions, the head sync always starts from
+    // the finalized epoch (even though it's the head sync) because finalized epoch is < local head.
+    // This is to prevent the issue noted here https://github.com/ChainSafe/lodestar/pull/7509#discussion_r1984353063.
+    //
+    // During non-finality of the network, when starting from an unfinalized checkpoint state, we don't want
+    // to sync before anchorStateLatestBlockSlot as finalized epoch is too far away. Local head will also be
+    // the same to that value at startup, the head sync always starts from anchorStateLatestBlockSlot in this case.
+    startEpoch: Math.min(computeEpochAtSlot(local.headSlot), minEpoch),
     target: {
       slot: remote.headSlot,
       root: remote.headRoot,

package/src/util/blobs.ts

@@ -149,41 +149,85 @@ export async function dataColumnMatrixRecovery(
  * Reconstruct blobs from a set of data columns, at least 50%+ of all the columns
  * must be provided to allow to reconstruct the full data matrix
  */
-export async function reconstructBlobs(sidecars: fulu.DataColumnSidecars): Promise<deneb.Blobs> {
+export async function reconstructBlobs(sidecars: fulu.DataColumnSidecars, indices?: number[]): Promise<deneb.Blobs> {
   if (sidecars.length < NUMBER_OF_COLUMNS / 2) {
     throw Error(
       `Expected at least ${NUMBER_OF_COLUMNS / 2} data columns to reconstruct blobs, received ${sidecars.length}`
     );
   }
+  const blobCount = sidecars[0].column.length;
 
-  let fullSidecars: fulu.DataColumnSidecars;
-
-  if (sidecars.length === NUMBER_OF_COLUMNS) {
-    // Full columns, no need to recover
-    fullSidecars = sidecars;
-  } else {
-    const sidecarsByIndex = new Map<number, fulu.DataColumnSidecar>(sidecars.map((sc) => [sc.index, sc]));
-    const recoveredSidecars = await dataColumnMatrixRecovery(sidecarsByIndex);
-    if (recoveredSidecars === null) {
-      // Should not happen because we check the column count above
-      throw Error("Failed to reconstruct the full data matrix");
+  for (const index of indices ?? []) {
+    if (index < 0 || index >= blobCount) {
+      throw Error(`Invalid blob index ${index}, must be between 0 and ${blobCount - 1}`);
     }
-    fullSidecars = recoveredSidecars;
+  }
+  const indicesToReconstruct = indices ?? Array.from({length: blobCount}, (_, i) => i);
+
+  const recoveredCells = await recoverBlobCells(sidecars, indicesToReconstruct);
+  if (recoveredCells === null) {
+    // Should not happen because we check the column count above
+    throw Error("Failed to recover cells to reconstruct blobs");
   }
 
-  const blobCount = fullSidecars[0].column.length;
-  const blobs: deneb.Blobs = new Array(blobCount);
+  const blobs: deneb.Blobs = new Array(indicesToReconstruct.length);
 
-  const ordered = fullSidecars.slice().sort((a, b) => a.index - b.index);
-  for (let row = 0; row < blobCount; row++) {
-    // 128 cells that make up one "extended blob" row
-    const cells = ordered.map((col) => col.column[row]);
-    blobs[row] = cellsToBlob(cells);
+  for (let i = 0; i < indicesToReconstruct.length; i++) {
+    const blobIndex = indicesToReconstruct[i];
+    const cells = recoveredCells.get(blobIndex);
+    if (!cells) {
+      throw Error(`Failed to get recovered cells for blob index ${blobIndex}`);
+    }
+    blobs[i] = cellsToBlob(cells);
   }
 
   return blobs;
 }
 
+/**
+ * Recover cells for specific blob indices from a set of data columns
+ */
+async function recoverBlobCells(
+  partialSidecars: fulu.DataColumnSidecar[],
+  blobIndices: number[]
+): Promise<Map<number, fulu.Cell[]> | null> {
+  const columnCount = partialSidecars.length;
+  if (columnCount < NUMBER_OF_COLUMNS / 2) {
+    // We don't have enough columns to recover
+    return null;
+  }
+
+  const recoveredCells = new Map<number, fulu.Cell[]>();
+  // Sort data columns by index in ascending order
+  const partialSidecarsSorted = partialSidecars.slice().sort((a, b) => a.index - b.index);
+
+  if (columnCount === NUMBER_OF_COLUMNS) {
+    // Full columns, no need to recover
+    for (const blobIndex of blobIndices) {
+      // 128 cells that make up one "extended blob" row
+      const cells = partialSidecarsSorted.map((col) => col.column[blobIndex]);
+      recoveredCells.set(blobIndex, cells);
+    }
+    return recoveredCells;
+  }
+
+  await Promise.all(
+    blobIndices.map(async (blobIndex) => {
+      const cellIndices: number[] = [];
+      const cells: fulu.Cell[] = [];
+      for (const dataColumn of partialSidecarsSorted) {
+        cellIndices.push(dataColumn.index);
+        cells.push(dataColumn.column[blobIndex]);
+      }
+      // Recover cells for this specific blob row
+      const recovered = await kzg.asyncRecoverCellsAndKzgProofs(cellIndices, cells);
+      recoveredCells.set(blobIndex, recovered.cells);
+    })
+  );
+
+  return recoveredCells;
+}
+
 /**
  * Concatenate the systematic half (columns 0‑63) of a row of cells into
  * the original 131072 byte blob. The parity half (64‑127) is ignored as

package/src/util/profile.ts

@@ -1,13 +1,32 @@
+import fs from "node:fs";
+import path from "node:path";
 import {sleep} from "@lodestar/utils";
 
+export enum ProfileThread {
+  MAIN = "main",
+  NETWORK = "network",
+  DISC5 = "discv5",
+}
+
+/**
+ * The time to take a Bun profile.
+ * If we increase this time it'll potentiall cause the app to crash.
+ * If we decrease this time, profile recorded will be fragmented and hard to analyze.
+ */
+const BUN_PROFILE_MS = 3 * 1000;
+
+export async function profileThread(thread: ProfileThread, durationMs: number, dirpath: string): Promise<string> {
+  return globalThis.Bun ? profileBun(thread, durationMs) : profileNodeJS(thread, durationMs, dirpath);
+}
+
 /**
- * Take 10m profile of the current thread without promise tracking.
+ * Take `durationMs` profile of the current thread and return the persisted file path.
  */
-export async function profileNodeJS(durationMs: number): Promise<string> {
+async function profileNodeJS(thread: ProfileThread, durationMs: number, dirpath: string): Promise<string> {
   const inspector = await import("node:inspector");
 
   // due to some typing issues, not able to use promisify here
-  return new Promise<string>((resolve, reject) => {
+  const profile = await new Promise<string>((resolve, reject) => {
     // Start the inspector and connect to it
     const session = new inspector.Session();
     session.connect();
@@ -29,6 +48,29 @@ export async function profileNodeJS(durationMs: number): Promise<string> {
       });
     });
   });
+
+  const filePath = path.join(dirpath, `${thread}_thread_${new Date().toISOString()}.cpuprofile`);
+  fs.writeFileSync(filePath, profile);
+  return filePath;
+}
+
+/**
+ * Unlike NodeJS, Bun console.profile() api flush data to the inspector,
+ * so this api returns ms taken of this profile instead of file path.
+ */
+async function profileBun(thread: ProfileThread, durationMs: number): Promise<string> {
+  const start = Date.now();
+  let now = Date.now();
+  while (now - start < durationMs) {
+    // biome-ignore lint/suspicious/noConsole: need to use console api to profile in Bun
+    console.profile(String(now));
+    await sleep(BUN_PROFILE_MS);
+    // biome-ignore lint/suspicious/noConsole: need to use console api to profile in Bun
+    console.profileEnd(String(now));
+    now = Date.now();
+  }
+
+  return `Successfully take Bun ${thread} thread profile in ${now - start}ms. Check your inspector to see the profile.`;
 }
 
 /**

package/src/util/sszBytes.ts

@@ -417,6 +417,36 @@ export function getSlotFromDataColumnSidecarSerialized(data: Uint8Array): Slot |
   return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR);
 }
 
+/**
+ * BeaconState of all forks (up until Electra, check with new forks)
+ * class BeaconState(Container):
+ *   genesis_time: uint64                    - 8 bytes
+ *   genesis_validators_root: Root           - 32 bytes
+ *   slot: Slot                              - 8 bytes
+ *   fork: Fork                              - 16 bytes
+ *   latest_block_header: BeaconBlockHeader  - fixed size
+ *     slot: Slot                            - 8 bytes
+ *
+ */
+
+const BLOCK_HEADER_SLOT_BYTES_POSITION_IN_BEACON_STATE = 8 + 32 + 8 + 16;
+export function getLastProcessedSlotFromBeaconStateSerialized(data: Uint8Array): Slot | null {
+  if (data.length < BLOCK_HEADER_SLOT_BYTES_POSITION_IN_BEACON_STATE + SLOT_SIZE) {
+    return null;
+  }
+
+  return getSlotFromOffset(data, BLOCK_HEADER_SLOT_BYTES_POSITION_IN_BEACON_STATE);
+}
+
+const SLOT_BYTES_POSITION_IN_BEACON_STATE = 8 + 32;
+export function getSlotFromBeaconStateSerialized(data: Uint8Array): Slot | null {
+  if (data.length < SLOT_BYTES_POSITION_IN_BEACON_STATE) {
+    return null;
+  }
+
+  return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_BEACON_STATE);
+}
+
 /**
  * Read only the first 4 bytes of Slot, max value is 4,294,967,295 will be reached 1634 years after genesis
  *