@lodestar/beacon-node 1.42.0-dev.c3829113f4 → 1.42.0-dev.c58e92512e

package/src/util/blobs.ts

@@ -13,7 +13,17 @@ import {
   VERSIONED_HASH_VERSION_KZG,
 } from "@lodestar/params";
 import {signedBlockToSignedHeader} from "@lodestar/state-transition";
-import {BeaconBlockBody, DataColumnSidecars, SSZTypesFor, SignedBeaconBlock, deneb, fulu, ssz} from "@lodestar/types";
+import {
+  BeaconBlockBody,
+  DataColumnSidecar,
+  SSZTypesFor,
+  SignedBeaconBlock,
+  deneb,
+  fulu,
+  gloas,
+  isGloasDataColumnSidecar,
+  ssz,
+} from "@lodestar/types";
 import {kzg} from "./kzg.js";
 
 type VersionHash = Uint8Array;
@@ -71,8 +81,8 @@ export function getBlobSidecars(
  * See https://github.com/ethereum/consensus-specs/blob/v1.6.0-alpha.4/specs/fulu/das-core.md#recover_matrix
  */
 export async function dataColumnMatrixRecovery(
-  partialSidecars: Map<number, fulu.DataColumnSidecar>
-): Promise<fulu.DataColumnSidecars | null> {
+  partialSidecars: Map<number, DataColumnSidecar>
+): Promise<DataColumnSidecar[] | null> {
   const columnCount = partialSidecars.size;
   if (columnCount < NUMBER_OF_COLUMNS / 2) {
     // We don't have enough columns to recover
@@ -92,7 +102,7 @@ export async function dataColumnMatrixRecovery(
     // should not happen because we check the size of the cache before this
     throw new Error("No data column found in cache to recover from");
   }
-  const blobCount = firstDataColumn.kzgCommitments.length;
+  const blobCount = firstDataColumn.column.length;
 
   const fullColumns: Array<Uint8Array[]> = Array.from(
     {length: NUMBER_OF_COLUMNS},
@@ -121,7 +131,7 @@ export async function dataColumnMatrixRecovery(
     }
   }
 
-  const result: fulu.DataColumnSidecars = new Array(NUMBER_OF_COLUMNS);
+  const result: DataColumnSidecar[] = new Array(NUMBER_OF_COLUMNS);
 
   for (let columnIndex = 0; columnIndex < NUMBER_OF_COLUMNS; columnIndex++) {
     let sidecar = partialSidecars.get(columnIndex);
@@ -131,14 +141,24 @@ export async function dataColumnMatrixRecovery(
       continue;
     }
 
-    sidecar = {
-      index: columnIndex,
-      column: fullColumns[columnIndex],
-      kzgCommitments: firstDataColumn.kzgCommitments,
-      kzgProofs: Array.from({length: blobCount}, (_, rowIndex) => blobProofs[rowIndex][columnIndex]),
-      signedBlockHeader: firstDataColumn.signedBlockHeader,
-      kzgCommitmentsInclusionProof: firstDataColumn.kzgCommitmentsInclusionProof,
-    };
+    if (isGloasDataColumnSidecar(firstDataColumn)) {
+      sidecar = {
+        index: columnIndex,
+        column: fullColumns[columnIndex],
+        kzgProofs: Array.from({length: blobCount}, (_, rowIndex) => blobProofs[rowIndex][columnIndex]),
+        slot: firstDataColumn.slot,
+        beaconBlockRoot: firstDataColumn.beaconBlockRoot,
+      } satisfies gloas.DataColumnSidecar;
+    } else {
+      sidecar = {
+        index: columnIndex,
+        column: fullColumns[columnIndex],
+        kzgCommitments: firstDataColumn.kzgCommitments,
+        kzgProofs: Array.from({length: blobCount}, (_, rowIndex) => blobProofs[rowIndex][columnIndex]),
+        signedBlockHeader: firstDataColumn.signedBlockHeader,
+        kzgCommitmentsInclusionProof: firstDataColumn.kzgCommitmentsInclusionProof,
+      } satisfies fulu.DataColumnSidecar;
+    }
     result[columnIndex] = sidecar;
   }
 
@@ -149,7 +169,7 @@ export async function dataColumnMatrixRecovery(
  * Reconstruct blobs from a set of data columns, at least 50%+ of all the columns
  * must be provided to allow to reconstruct the full data matrix
  */
-export async function reconstructBlobs(sidecars: DataColumnSidecars, indices?: number[]): Promise<deneb.Blobs> {
+export async function reconstructBlobs(sidecars: DataColumnSidecar[], indices?: number[]): Promise<deneb.Blobs> {
   if (sidecars.length < NUMBER_OF_COLUMNS / 2) {
     throw Error(
       `Expected at least ${NUMBER_OF_COLUMNS / 2} data columns to reconstruct blobs, received ${sidecars.length}`
@@ -188,7 +208,7 @@ export async function reconstructBlobs(sidecars: DataColumnSidecars, indices?: n
  * Recover cells for specific blob indices from a set of data columns
  */
 async function recoverBlobCells(
-  partialSidecars: DataColumnSidecars,
+  partialSidecars: DataColumnSidecar[],
   blobIndices: number[]
 ): Promise<Map<number, fulu.Cell[]> | null> {
   const columnCount = partialSidecars.length;

package/src/util/dataColumns.ts

@@ -16,6 +16,7 @@ import {
   BeaconBlockBody,
   ColumnIndex,
   CustodyIndex,
+  DataColumnSidecar,
   Root,
   SSZTypesFor,
   SignedBeaconBlock,
@@ -24,11 +25,13 @@ import {
   deneb,
   fulu,
   gloas,
+  isGloasDataColumnSidecar,
   ssz,
 } from "@lodestar/types";
 import {bytesToBigInt} from "@lodestar/utils";
 import {BlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource} from "../chain/blocks/blockInput/types.js";
+import {PayloadEnvelopeInput, PayloadEnvelopeInputSource} from "../chain/blocks/payloadEnvelopeInput/index.js";
 import {ChainEvent, ChainEventEmitter} from "../chain/emitter.js";
 import {Metrics} from "../metrics/metrics.js";
 import {NodeId} from "../network/subnets/index.js";
@@ -287,17 +290,17 @@ export function getBlobKzgCommitments(
  * SPEC FUNCTION
  * https://github.com/ethereum/consensus-specs/blob/v1.6.0-alpha.4/specs/fulu/validator.md#get_data_column_sidecars
  */
-export function getDataColumnSidecars(
+export function getFuluDataColumnSidecars(
   signedBlockHeader: SignedBeaconBlockHeader,
   kzgCommitments: deneb.KZGCommitment[],
   kzgCommitmentsInclusionProof: fulu.KzgCommitmentsInclusionProof,
   cellsAndKzgProofs: {cells: Uint8Array[]; proofs: Uint8Array[]}[]
-): fulu.DataColumnSidecars {
+): fulu.DataColumnSidecar[] {
   if (cellsAndKzgProofs.length !== kzgCommitments.length) {
     throw Error("Invalid cellsAndKzgProofs length for getDataColumnSidecars");
   }
 
-  const sidecars: fulu.DataColumnSidecars = [];
+  const sidecars: fulu.DataColumnSidecar[] = [];
   for (let columnIndex = 0; columnIndex < NUMBER_OF_COLUMNS; columnIndex++) {
     const columnCells = [];
     const columnProofs = [];
@@ -322,13 +325,14 @@ export function getDataColumnSidecars(
  * block, assemble the sidecars which can be distributed to peers.
  *
  * SPEC FUNCTION
- * https://github.com/ethereum/consensus-specs/blob/v1.6.0-alpha.4/specs/fulu/validator.md#get_data_column_sidecars_from_block
+ * fulu: https://github.com/ethereum/consensus-specs/blob/v1.6.0-alpha.4/specs/fulu/validator.md#get_data_column_sidecars_from_block
+ * gloas: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.3/specs/gloas/builder.md#modified-get_data_column_sidecars_from_block
  */
 export function getDataColumnSidecarsFromBlock(
   config: ChainForkConfig,
   signedBlock: SignedBeaconBlock<ForkPostFulu>,
   cellsAndKzgProofs: {cells: Uint8Array[]; proofs: Uint8Array[]}[]
-): fulu.DataColumnSidecars {
+): DataColumnSidecar[] {
   const fork = config.getForkName(signedBlock.message.slot);
   const blobKzgCommitments = getBlobKzgCommitments(fork, signedBlock);
 
@@ -336,11 +340,21 @@ export function getDataColumnSidecarsFromBlock(
   if (blobKzgCommitments.length === 0) {
     return [];
   }
-  const signedBlockHeader = signedBlockToSignedHeader(config, signedBlock);
 
+  if (isForkPostGloas(fork)) {
+    const beaconBlockRoot = config.getForkTypes(signedBlock.message.slot).BeaconBlock.hashTreeRoot(signedBlock.message);
+    return getGloasDataColumnSidecars(signedBlock.message.slot, beaconBlockRoot, cellsAndKzgProofs);
+  }
+
+  const signedBlockHeader = signedBlockToSignedHeader(config, signedBlock);
   const kzgCommitmentsInclusionProof = computePostFuluKzgCommitmentsInclusionProof(fork, signedBlock.message.body);
 
-  return getDataColumnSidecars(signedBlockHeader, blobKzgCommitments, kzgCommitmentsInclusionProof, cellsAndKzgProofs);
+  return getFuluDataColumnSidecars(
+    signedBlockHeader,
+    blobKzgCommitments,
+    kzgCommitmentsInclusionProof,
+    cellsAndKzgProofs
+  );
 }
 
 /**
@@ -348,13 +362,18 @@ export function getDataColumnSidecarsFromBlock(
  * to the commitments it contains, assemble all sidecars for distribution to peers.
  *
  * SPEC FUNCTION
- * https://github.com/ethereum/consensus-specs/blob/v1.6.0-alpha.4/specs/fulu/validator.md#get_data_column_sidecars_from_column_sidecar
+ * fulu: https://github.com/ethereum/consensus-specs/blob/v1.6.0-alpha.4/specs/fulu/validator.md#get_data_column_sidecars_from_column_sidecar
+ * gloas: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.3/specs/gloas/validator.md#modified-get_data_column_sidecars_from_column_sidecar
  */
 export function getDataColumnSidecarsFromColumnSidecar(
-  sidecar: fulu.DataColumnSidecar,
+  sidecar: DataColumnSidecar,
   cellsAndKzgProofs: {cells: Uint8Array[]; proofs: Uint8Array[]}[]
-): fulu.DataColumnSidecars {
-  return getDataColumnSidecars(
+): DataColumnSidecar[] {
+  if (isGloasDataColumnSidecar(sidecar)) {
+    return getGloasDataColumnSidecars(sidecar.slot, sidecar.beaconBlockRoot, cellsAndKzgProofs);
+  }
+
+  return getFuluDataColumnSidecars(
     sidecar.signedBlockHeader,
     sidecar.kzgCommitments,
     sidecar.kzgCommitmentsInclusionProof,
@@ -362,21 +381,32 @@ export function getDataColumnSidecarsFromColumnSidecar(
   );
 }
 
+export function getDataColumnSidecarSlot(sidecar: DataColumnSidecar): Slot {
+  if (isGloasDataColumnSidecar(sidecar)) {
+    return sidecar.slot;
+  }
+
+  return sidecar.signedBlockHeader.message.slot;
+}
+
 /**
  * In Gloas, data column sidecars have a simplified structure with `slot` and `beaconBlockRoot`
  * instead of `signedBlockHeader`, `kzgCommitments`, and `kzgCommitmentsInclusionProof`.
+ *
+ * SPEC FUNCTION
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.3/specs/gloas/builder.md#modified-get_data_column_sidecars
  */
-export function getDataColumnSidecarsForGloas(
+export function getGloasDataColumnSidecars(
   slot: Slot,
   beaconBlockRoot: Root,
   cellsAndKzgProofs: {cells: Uint8Array[]; proofs: Uint8Array[]}[]
-): gloas.DataColumnSidecars {
+): gloas.DataColumnSidecar[] {
   // No need to create data column sidecars if there are no blobs
   if (cellsAndKzgProofs.length === 0) {
     return [];
   }
 
-  const sidecars: gloas.DataColumnSidecars = [];
+  const sidecars: gloas.DataColumnSidecar[] = [];
   for (let columnIndex = 0; columnIndex < NUMBER_OF_COLUMNS; columnIndex++) {
     const column: Uint8Array[] = [];
     const kzgProofs: Uint8Array[] = [];
@@ -399,11 +429,11 @@ export function getDataColumnSidecarsForGloas(
  * If we receive more than half of NUMBER_OF_COLUMNS (64) we should recover all remaining columns
  */
 export async function recoverDataColumnSidecars(
-  blockInput: BlockInputColumns,
+  input: BlockInputColumns | PayloadEnvelopeInput,
   emitter: ChainEventEmitter,
   metrics: Metrics | null
 ): Promise<DataColumnReconstructionCode> {
-  const existingColumns = blockInput.getAllColumns();
+  const existingColumns = input.getAllColumns();
   const columnCount = existingColumns.length;
   if (columnCount >= NUMBER_OF_COLUMNS) {
     // We have all columns
@@ -416,7 +446,7 @@ export async function recoverDataColumnSidecars(
   }
 
   metrics?.recoverDataColumnSidecars.custodyBeforeReconstruction.set(columnCount);
-  const partialSidecars = new Map<number, fulu.DataColumnSidecar>();
+  const partialSidecars = new Map<number, DataColumnSidecar>();
   for (const columnSidecar of existingColumns) {
     // the more columns we put, the slower the recover
     if (partialSidecars.size >= NUMBER_OF_COLUMNS / 2) {
@@ -434,7 +464,7 @@ export async function recoverDataColumnSidecars(
     return DataColumnReconstructionCode.NullReturned;
   }
 
-  if (blockInput.getAllColumns().length === NUMBER_OF_COLUMNS) {
+  if (input.getAllColumns().length === NUMBER_OF_COLUMNS) {
     // either gossip or getBlobsV2 resolved availability while we were recovering
     metrics?.dataColumns.alreadyAdded.inc(fullSidecars.length);
     return DataColumnReconstructionCode.SuccessLate;
@@ -450,13 +480,27 @@ export async function recoverDataColumnSidecars(
   // the node MAY delete the DataColumnSidecar if it is not part of the node's custody requirement.
   const sidecarsToPublish = [];
   for (const columnSidecar of fullSidecars) {
-    if (!blockInput.hasColumn(columnSidecar.index)) {
-      blockInput.addColumn({
-        blockRootHex: blockInput.blockRootHex,
-        columnSidecar,
-        seenTimestampSec: Date.now() / 1000,
-        source: BlockInputSource.recovery,
-      });
+    if (!input.hasColumn(columnSidecar.index)) {
+      if (input instanceof PayloadEnvelopeInput) {
+        if (!isGloasDataColumnSidecar(columnSidecar)) {
+          throw new Error(`Expected gloas DataColumnSidecar for block ${input.blockRootHex}`);
+        }
+        input.addColumn({
+          columnSidecar,
+          seenTimestampSec: Date.now() / 1000,
+          source: PayloadEnvelopeInputSource.recovery,
+        });
+      } else {
+        if (isGloasDataColumnSidecar(columnSidecar)) {
+          throw new Error(`Expected fulu DataColumnSidecar for block ${input.blockRootHex}`);
+        }
+        input.addColumn({
+          blockRootHex: input.blockRootHex,
+          columnSidecar,
+          seenTimestampSec: Date.now() / 1000,
+          source: BlockInputSource.recovery,
+        });
+      }
       sidecarsToPublish.push(columnSidecar);
     }
   }

package/src/util/execution.ts

@@ -2,10 +2,11 @@ import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkPostFulu, ForkPreFulu} from "@lodestar/params";
 import {signedBlockToSignedHeader} from "@lodestar/state-transition";
-import {deneb, fulu} from "@lodestar/types";
-import {toHex} from "@lodestar/utils";
+import {DataColumnSidecar, SignedBeaconBlock, deneb, isGloasDataColumnSidecar} from "@lodestar/types";
+import {fromHex, toHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../chain/blocks/blockInput/types.js";
+import {PayloadEnvelopeInput, PayloadEnvelopeInputSource} from "../chain/blocks/payloadEnvelopeInput/index.js";
 import {ChainEvent, ChainEventEmitter} from "../chain/emitter.js";
 import {IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/index.js";
@@ -14,6 +15,7 @@ import {
   getCellsAndProofs,
   getDataColumnSidecarsFromBlock,
   getDataColumnSidecarsFromColumnSidecar,
+  getGloasDataColumnSidecars,
 } from "./dataColumns.js";
 
 export enum DataColumnEngineResult {
@@ -124,27 +126,32 @@ export async function getBlobSidecarsFromExecution(
 }
 
 /**
- * Post fulu, call getBlobsV2 from execution engine once per slot whenever we see either beacon_block or data_column_sidecar gossip message
+ * Call getBlobsV2 from execution engine once per slot to fetch blobs and compute data columns.
+ *
+ * Post fulu, whenever we see either beacon_block or data_column_sidecar gossip message and data isn't complete.
+ * Post gloas, immediately when beacon block is successfully imported and PayloadEnvelopeInput is created.
  */
 export async function getDataColumnSidecarsFromExecution(
   config: ChainForkConfig,
   executionEngine: IExecutionEngine,
   emitter: ChainEventEmitter,
-  blockInput: IBlockInput,
+  input: IBlockInput | PayloadEnvelopeInput,
   metrics: Metrics | null,
   blobAndProofBuffers?: Uint8Array[]
 ): Promise<DataColumnEngineResult> {
-  // If its not a column block input, exit
-  if (!isBlockInputColumns(blockInput)) {
+  const isPayloadInput = input instanceof PayloadEnvelopeInput;
+
+  // Pre gloas, ensure it's a column block input
+  if (!isPayloadInput && !isBlockInputColumns(input)) {
     return DataColumnEngineResult.PreFulu;
   }
 
   // If already have all columns, exit
-  if (blockInput.hasAllData()) {
+  if (input.hasAllData()) {
     return DataColumnEngineResult.NotAttemptedFull;
   }
 
-  const versionedHashes = blockInput.getVersionedHashes();
+  const versionedHashes = input.getVersionedHashes();
 
   // If there are no blobs in this block, exit
   if (versionedHashes.length === 0) {
@@ -154,11 +161,7 @@ export async function getDataColumnSidecarsFromExecution(
   // Get blobs from execution engine
   metrics?.peerDas.getBlobsV2Requests.inc();
   const timer = metrics?.peerDas.getBlobsV2RequestDuration.startTimer();
-  const blobs = await executionEngine.getBlobs(
-    blockInput.forkName as ForkPostFulu,
-    versionedHashes,
-    blobAndProofBuffers
-  );
+  const blobs = await executionEngine.getBlobs(input.forkName as ForkPostFulu, versionedHashes, blobAndProofBuffers);
   timer?.();
 
   // Execution engine was unable to find one or more blobs
@@ -168,22 +171,24 @@ export async function getDataColumnSidecarsFromExecution(
   metrics?.peerDas.getBlobsV2Responses.inc();
 
   // Return if we received all data columns while waiting for getBlobs
-  if (blockInput.hasAllData()) {
+  if (input.hasAllData()) {
     return DataColumnEngineResult.SuccessLate;
   }
 
-  let dataColumnSidecars: fulu.DataColumnSidecars;
+  let dataColumnSidecars: DataColumnSidecar[];
   const compTimer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer();
   try {
     const cellsAndProofs = await getCellsAndProofs(blobs);
-    if (blockInput.hasBlock()) {
+    if (isPayloadInput) {
+      dataColumnSidecars = getGloasDataColumnSidecars(input.slot, fromHex(input.blockRootHex), cellsAndProofs);
+    } else if (input.hasBlock()) {
       dataColumnSidecars = getDataColumnSidecarsFromBlock(
         config,
-        blockInput.getBlock() as fulu.SignedBeaconBlock,
+        input.getBlock() as SignedBeaconBlock<ForkPostFulu>,
         cellsAndProofs
       );
     } else {
-      const firstSidecar = blockInput.getAllColumns()[0];
+      const firstSidecar = input.getAllColumns()[0];
       dataColumnSidecars = getDataColumnSidecarsFromColumnSidecar(firstSidecar, cellsAndProofs);
     }
   } finally {
@@ -191,36 +196,50 @@ export async function getDataColumnSidecarsFromExecution(
   }
 
   // Publish columns if and only if subscribed to them
-  const previouslyMissingColumns = blockInput.getMissingSampledColumnMeta().missing;
+  const previouslyMissingColumns = input.getMissingSampledColumnMeta().missing;
   const sampledColumns = previouslyMissingColumns.map((columnIndex) => dataColumnSidecars[columnIndex]);
 
   // for columns that we already seen, it will be ignored through `ignoreDuplicatePublishError` gossip option
   emitter.emit(ChainEvent.publishDataColumns, sampledColumns);
   // TODO: Can we record dataColumns.sentPeersPerSubnet metric here somehow
 
-  // add all sampled columns to the block input, even if we didn't sample them
+  // add all sampled columns to the input, even if we didn't sample them
   const seenTimestampSec = Date.now() / 1000;
   let alreadyAddedColumnsCount = 0;
   for (const columnSidecar of sampledColumns) {
-    if (blockInput.hasColumn(columnSidecar.index)) {
+    if (input.hasColumn(columnSidecar.index)) {
       // columns may have been added while waiting
       alreadyAddedColumnsCount++;
       continue;
     }
 
-    blockInput.addColumn({
-      columnSidecar,
-      blockRootHex: blockInput.blockRootHex,
-      source: BlockInputSource.engine,
-      seenTimestampSec,
-    });
+    if (isPayloadInput) {
+      if (!isGloasDataColumnSidecar(columnSidecar)) {
+        throw new Error(`Expected gloas DataColumnSidecar for block ${input.blockRootHex}`);
+      }
+      input.addColumn({
+        columnSidecar,
+        source: PayloadEnvelopeInputSource.engine,
+        seenTimestampSec,
+      });
+    } else {
+      if (isGloasDataColumnSidecar(columnSidecar)) {
+        throw new Error(`Expected fulu DataColumnSidecar for block ${input.blockRootHex}`);
+      }
+      input.addColumn({
+        columnSidecar,
+        blockRootHex: input.blockRootHex,
+        source: BlockInputSource.engine,
+        seenTimestampSec,
+      });
+    }
 
     if (emitter.listenerCount(routes.events.EventType.dataColumnSidecar)) {
       emitter.emit(routes.events.EventType.dataColumnSidecar, {
-        blockRoot: blockInput.blockRootHex,
-        slot: blockInput.slot,
+        blockRoot: input.blockRootHex,
+        slot: input.slot,
         index: columnSidecar.index,
-        kzgCommitments: columnSidecar.kzgCommitments.map(toHex),
+        kzgCommitments: !isGloasDataColumnSidecar(columnSidecar) ? columnSidecar.kzgCommitments.map(toHex) : undefined,
       });
     }
   }