@lodestar/beacon-node 1.36.0-dev.6f46a8bd20 → 1.36.0-dev.797fa46c1b

package/src/sync/utils/remoteSyncType.ts

@@ -1,6 +1,7 @@
 import {IForkChoice} from "@lodestar/fork-choice";
 import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Slot, Status} from "@lodestar/types";
+import {IBeaconChain} from "../../chain/interface.ts";
 import {ChainTarget} from "../range/utils/index.js";
 
 /** The type of peer relative to our current state */
@@ -103,8 +104,11 @@ export function getRangeSyncType(local: Status, remote: Status, forkChoice: IFor
 export function getRangeSyncTarget(
   local: Status,
   remote: Status,
-  forkChoice: IForkChoice
+  chain: IBeaconChain
 ): {syncType: RangeSyncType; startEpoch: Slot; target: ChainTarget} {
+  const forkChoice = chain.forkChoice;
+
+  // finalized sync
   if (remote.finalizedEpoch > local.finalizedEpoch && !forkChoice.hasBlock(remote.finalizedRoot)) {
     return {
       // If  RangeSyncType.Finalized, the range of blocks fetchable from startEpoch and target must allow to switch
@@ -131,11 +135,26 @@ export function getRangeSyncTarget(
       },
     };
   }
+
+  // we don't want to sync from epoch < minEpoch
+  // if we boot from an unfinalized checkpoint state, we don't want to sync before anchorStateLatestBlockSlot
+  // if we boot from a finalized checkpoint state, anchorStateLatestBlockSlot is trusted and we also don't want to sync before it
+  const minEpoch = Math.max(remote.finalizedEpoch, computeEpochAtSlot(chain.anchorStateLatestBlockSlot));
+
+  // head sync
   return {
     syncType: RangeSyncType.Head,
-    // The new peer has the same finalized (earlier filters should prevent a peer with an
-    // earlier finalized chain from reaching here) and local head will always be >= local finalized.
-    startEpoch: computeEpochAtSlot(local.headSlot),
+    // The new peer has the same finalized `remote.finalizedEpoch == local.finalizedEpoch` since
+    // previous filters should prevent a peer with an earlier finalized chain from reaching here.
+    //
+    // By default and during stable network conditions, the head sync always starts from
+    // the finalized epoch (even though it's the head sync) because finalized epoch is < local head.
+    // This is to prevent the issue noted here https://github.com/ChainSafe/lodestar/pull/7509#discussion_r1984353063.
+    //
+    // During non-finality of the network, when starting from an unfinalized checkpoint state, we don't want
+    // to sync before anchorStateLatestBlockSlot as finalized epoch is too far away. Local head will also be
+    // the same to that value at startup, the head sync always starts from anchorStateLatestBlockSlot in this case.
+    startEpoch: Math.min(computeEpochAtSlot(local.headSlot), minEpoch),
     target: {
       slot: remote.headSlot,
       root: remote.headRoot,

package/src/util/blobs.ts

@@ -149,41 +149,85 @@ export async function dataColumnMatrixRecovery(
  * Reconstruct blobs from a set of data columns, at least 50%+ of all the columns
  * must be provided to allow to reconstruct the full data matrix
  */
-export async function reconstructBlobs(sidecars: fulu.DataColumnSidecars): Promise<deneb.Blobs> {
+export async function reconstructBlobs(sidecars: fulu.DataColumnSidecars, indices?: number[]): Promise<deneb.Blobs> {
   if (sidecars.length < NUMBER_OF_COLUMNS / 2) {
     throw Error(
       `Expected at least ${NUMBER_OF_COLUMNS / 2} data columns to reconstruct blobs, received ${sidecars.length}`
     );
   }
+  const blobCount = sidecars[0].column.length;
 
-  let fullSidecars: fulu.DataColumnSidecars;
-
-  if (sidecars.length === NUMBER_OF_COLUMNS) {
-    // Full columns, no need to recover
-    fullSidecars = sidecars;
-  } else {
-    const sidecarsByIndex = new Map<number, fulu.DataColumnSidecar>(sidecars.map((sc) => [sc.index, sc]));
-    const recoveredSidecars = await dataColumnMatrixRecovery(sidecarsByIndex);
-    if (recoveredSidecars === null) {
-      // Should not happen because we check the column count above
-      throw Error("Failed to reconstruct the full data matrix");
+  for (const index of indices ?? []) {
+    if (index < 0 || index >= blobCount) {
+      throw Error(`Invalid blob index ${index}, must be between 0 and ${blobCount - 1}`);
     }
-    fullSidecars = recoveredSidecars;
+  }
+  const indicesToReconstruct = indices ?? Array.from({length: blobCount}, (_, i) => i);
+
+  const recoveredCells = await recoverBlobCells(sidecars, indicesToReconstruct);
+  if (recoveredCells === null) {
+    // Should not happen because we check the column count above
+    throw Error("Failed to recover cells to reconstruct blobs");
   }
 
-  const blobCount = fullSidecars[0].column.length;
-  const blobs: deneb.Blobs = new Array(blobCount);
+  const blobs: deneb.Blobs = new Array(indicesToReconstruct.length);
 
-  const ordered = fullSidecars.slice().sort((a, b) => a.index - b.index);
-  for (let row = 0; row < blobCount; row++) {
-    // 128 cells that make up one "extended blob" row
-    const cells = ordered.map((col) => col.column[row]);
-    blobs[row] = cellsToBlob(cells);
+  for (let i = 0; i < indicesToReconstruct.length; i++) {
+    const blobIndex = indicesToReconstruct[i];
+    const cells = recoveredCells.get(blobIndex);
+    if (!cells) {
+      throw Error(`Failed to get recovered cells for blob index ${blobIndex}`);
+    }
+    blobs[i] = cellsToBlob(cells);
   }
 
   return blobs;
 }
 
+/**
+ * Recover cells for specific blob indices from a set of data columns
+ */
+async function recoverBlobCells(
+  partialSidecars: fulu.DataColumnSidecar[],
+  blobIndices: number[]
+): Promise<Map<number, fulu.Cell[]> | null> {
+  const columnCount = partialSidecars.length;
+  if (columnCount < NUMBER_OF_COLUMNS / 2) {
+    // We don't have enough columns to recover
+    return null;
+  }
+
+  const recoveredCells = new Map<number, fulu.Cell[]>();
+  // Sort data columns by index in ascending order
+  const partialSidecarsSorted = partialSidecars.slice().sort((a, b) => a.index - b.index);
+
+  if (columnCount === NUMBER_OF_COLUMNS) {
+    // Full columns, no need to recover
+    for (const blobIndex of blobIndices) {
+      // 128 cells that make up one "extended blob" row
+      const cells = partialSidecarsSorted.map((col) => col.column[blobIndex]);
+      recoveredCells.set(blobIndex, cells);
+    }
+    return recoveredCells;
+  }
+
+  await Promise.all(
+    blobIndices.map(async (blobIndex) => {
+      const cellIndices: number[] = [];
+      const cells: fulu.Cell[] = [];
+      for (const dataColumn of partialSidecarsSorted) {
+        cellIndices.push(dataColumn.index);
+        cells.push(dataColumn.column[blobIndex]);
+      }
+      // Recover cells for this specific blob row
+      const recovered = await kzg.asyncRecoverCellsAndKzgProofs(cellIndices, cells);
+      recoveredCells.set(blobIndex, recovered.cells);
+    })
+  );
+
+  return recoveredCells;
+}
+
 /**
  * Concatenate the systematic half (columns 0‑63) of a row of cells into
  * the original 131072 byte blob. The parity half (64‑127) is ignored as

package/src/util/sszBytes.ts

@@ -417,6 +417,36 @@ export function getSlotFromDataColumnSidecarSerialized(data: Uint8Array): Slot |
   return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR);
 }
 
+/**
+ * BeaconState of all forks (up until Electra, check with new forks)
+ * class BeaconState(Container):
+ *   genesis_time: uint64                    - 8 bytes
+ *   genesis_validators_root: Root           - 32 bytes
+ *   slot: Slot                              - 8 bytes
+ *   fork: Fork                              - 16 bytes
+ *   latest_block_header: BeaconBlockHeader  - fixed size
+ *     slot: Slot                            - 8 bytes
+ *
+ */
+
+const BLOCK_HEADER_SLOT_BYTES_POSITION_IN_BEACON_STATE = 8 + 32 + 8 + 16;
+export function getLastProcessedSlotFromBeaconStateSerialized(data: Uint8Array): Slot | null {
+  if (data.length < BLOCK_HEADER_SLOT_BYTES_POSITION_IN_BEACON_STATE + SLOT_SIZE) {
+    return null;
+  }
+
+  return getSlotFromOffset(data, BLOCK_HEADER_SLOT_BYTES_POSITION_IN_BEACON_STATE);
+}
+
+const SLOT_BYTES_POSITION_IN_BEACON_STATE = 8 + 32;
+export function getSlotFromBeaconStateSerialized(data: Uint8Array): Slot | null {
+  if (data.length < SLOT_BYTES_POSITION_IN_BEACON_STATE) {
+    return null;
+  }
+
+  return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_BEACON_STATE);
+}
+
 /**
  * Read only the first 4 bytes of Slot, max value is 4,294,967,295 will be reached 1634 years after genesis
  *