@lodestar/beacon-node 1.42.0-dev.8961934298 → 1.42.0-dev.8b81ee6c3e

package/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts

@@ -1,8 +1,9 @@
-import {NUMBER_OF_COLUMNS} from "@lodestar/params";
-import {ColumnIndex, DataColumnSidecars, RootHex, Slot, ValidatorIndex, deneb, gloas} from "@lodestar/types";
+import {ForkName, NUMBER_OF_COLUMNS} from "@lodestar/params";
+import {ColumnIndex, RootHex, Slot, ValidatorIndex, deneb, gloas} from "@lodestar/types";
 import {toRootHex, withTimeout} from "@lodestar/utils";
 import {VersionedHashes} from "../../../execution/index.js";
 import {kzgCommitmentToVersionedHash} from "../../../util/blobs.js";
+import {MissingColumnMeta} from "../blockInput/types.js";
 import {AddPayloadEnvelopeProps, ColumnWithSource, CreateFromBlockProps, SourceMeta} from "./types.js";
 
 export type PayloadEnvelopeInputState =
@@ -59,6 +60,7 @@ function createPromise<T>(): PromiseParts<T> {
 export class PayloadEnvelopeInput {
   readonly blockRootHex: RootHex;
   readonly slot: Slot;
+  readonly forkName: ForkName;
   readonly proposerIndex: ValidatorIndex;
   readonly bid: gloas.ExecutionPayloadBid;
   readonly versionedHashes: VersionedHashes;
@@ -71,13 +73,14 @@ export class PayloadEnvelopeInput {
   private timeCreatedSec: number;
 
   private readonly payloadEnvelopeDataPromise: PromiseParts<gloas.SignedExecutionPayloadEnvelope>;
-  private readonly columnsDataPromise: PromiseParts<DataColumnSidecars>;
+  private readonly columnsDataPromise: PromiseParts<gloas.DataColumnSidecar[]>;
 
   state: PayloadEnvelopeInputState;
 
   private constructor(props: {
     blockRootHex: RootHex;
     slot: Slot;
+    forkName: ForkName;
     proposerIndex: ValidatorIndex;
     bid: gloas.ExecutionPayloadBid;
     sampledColumns: ColumnIndex[];
@@ -86,6 +89,7 @@ export class PayloadEnvelopeInput {
   }) {
     this.blockRootHex = props.blockRootHex;
     this.slot = props.slot;
+    this.forkName = props.forkName;
     this.proposerIndex = props.proposerIndex;
     this.bid = props.bid;
     this.versionedHashes = props.bid.blobKzgCommitments.map(kzgCommitmentToVersionedHash);
@@ -112,6 +116,7 @@ export class PayloadEnvelopeInput {
     return new PayloadEnvelopeInput({
       blockRootHex: props.blockRootHex,
       slot: props.block.message.slot,
+      forkName: props.forkName,
       proposerIndex: props.block.message.proposerIndex,
       bid,
       sampledColumns: props.sampledColumns,
@@ -173,8 +178,12 @@ export class PayloadEnvelopeInput {
     }
   }
 
-  addColumn(columnWithSource: ColumnWithSource): void {
+  addColumn(columnWithSource: ColumnWithSource): boolean {
     const {columnSidecar, seenTimestampSec} = columnWithSource;
+    if (this.columnsCache.has(columnSidecar.index)) {
+      return false;
+    }
+
     this.columnsCache.set(columnSidecar.index, columnWithSource);
 
     const sampledColumns = this.getSampledColumns();
@@ -191,7 +200,7 @@ export class PayloadEnvelopeInput {
       sampledColumns.length === this.sampledColumns.length;
 
     if (!hasAllData) {
-      return;
+      return true;
     }
 
     if (hasComputedAllData) {
@@ -217,6 +226,20 @@ export class PayloadEnvelopeInput {
         hasComputedAllData: hasComputedAllData || this.state.hasComputedAllData,
       };
     }
+
+    return true;
+  }
+
+  hasColumn(index: ColumnIndex): boolean {
+    return this.columnsCache.has(index);
+  }
+
+  getColumn(index: ColumnIndex): gloas.DataColumnSidecar | undefined {
+    return this.columnsCache.get(index)?.columnSidecar;
+  }
+
+  getAllColumns(): gloas.DataColumnSidecar[] {
+    return [...this.columnsCache.values()].map(({columnSidecar}) => columnSidecar);
   }
 
   getVersionedHashes(): VersionedHashes {
@@ -237,8 +260,8 @@ export class PayloadEnvelopeInput {
     return this.state.payloadEnvelopeSource;
   }
 
-  getSampledColumns(): gloas.DataColumnSidecars {
-    const columns: gloas.DataColumnSidecars = [];
+  getSampledColumns(): gloas.DataColumnSidecar[] {
+    const columns: gloas.DataColumnSidecar[] = [];
     for (const index of this.sampledColumns) {
       const column = this.columnsCache.get(index);
       if (column) {
@@ -259,8 +282,8 @@ export class PayloadEnvelopeInput {
     return columns;
   }
 
-  getCustodyColumns(): gloas.DataColumnSidecars {
-    const columns: gloas.DataColumnSidecars = [];
+  getCustodyColumns(): gloas.DataColumnSidecar[] {
+    const columns: gloas.DataColumnSidecar[] = [];
     for (const index of this.custodyColumns) {
       const column = this.columnsCache.get(index);
       if (column) {
@@ -270,11 +293,29 @@ export class PayloadEnvelopeInput {
     return columns;
   }
 
+  hasAllData(): boolean {
+    return this.state.hasAllData;
+  }
+
+  getMissingSampledColumnMeta(): MissingColumnMeta {
+    if (this.state.hasAllData) {
+      return {missing: [], versionedHashes: this.versionedHashes};
+    }
+
+    const missing: ColumnIndex[] = [];
+    for (const index of this.sampledColumns) {
+      if (!this.columnsCache.has(index)) {
+        missing.push(index);
+      }
+    }
+    return {missing, versionedHashes: this.versionedHashes};
+  }
+
   hasComputedAllData(): boolean {
     return this.state.hasComputedAllData;
   }
 
-  waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<DataColumnSidecars> {
+  waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<gloas.DataColumnSidecar[]> {
     if (this.state.hasComputedAllData) {
       return Promise.resolve(this.getSampledColumns());
     }
@@ -319,7 +360,7 @@ export class PayloadEnvelopeInput {
     hasAllData: boolean;
     hasComputedAllData: boolean;
     isComplete: boolean;
-    columnsCount: number;
+    receivedColumns: number;
     sampledColumnsCount: number;
   } {
     return {
@@ -329,7 +370,7 @@ export class PayloadEnvelopeInput {
       hasAllData: this.state.hasAllData,
       hasComputedAllData: this.state.hasComputedAllData,
       isComplete: this.isComplete(),
-      columnsCount: this.columnsCache.size,
+      receivedColumns: this.columnsCache.size,
       sampledColumnsCount: this.sampledColumns.length,
     };
   }

package/src/chain/blocks/payloadEnvelopeInput/types.ts

@@ -1,4 +1,4 @@
-import {ForkPostGloas} from "@lodestar/params";
+import {ForkName, ForkPostGloas} from "@lodestar/params";
 import {ColumnIndex, RootHex, SignedBeaconBlock, gloas} from "@lodestar/types";
 
 export enum PayloadEnvelopeInputSource {
@@ -23,6 +23,7 @@ export type ColumnWithSource = SourceMeta & {
 export type CreateFromBlockProps = {
   blockRootHex: RootHex;
   block: SignedBeaconBlock<ForkPostGloas>;
+  forkName: ForkName;
   sampledColumns: ColumnIndex[];
   custodyColumns: ColumnIndex[];
   timeCreatedSec: number;

package/src/chain/chain.ts

@@ -15,6 +15,7 @@ import {
   BUILDER_INDEX_SELF_BUILD,
   EFFECTIVE_BALANCE_INCREMENT,
   type ForkPostFulu,
+  type ForkPostGloas,
   GENESIS_SLOT,
   SLOTS_PER_EPOCH,
   isForkPostElectra,
@@ -34,7 +35,7 @@ import {
   BeaconBlock,
   BlindedBeaconBlock,
   BlindedBeaconBlockBody,
-  DataColumnSidecars,
+  DataColumnSidecar,
   Epoch,
   Root,
   RootHex,
@@ -873,20 +874,32 @@ export class BeaconChain implements IBeaconChain {
     );
   }
 
-  async getDataColumnSidecars(blockSlot: Slot, blockRootHex: string): Promise<DataColumnSidecars> {
-    const blockInput = this.seenBlockInputCache.get(blockRootHex);
-    if (blockInput) {
-      if (!isBlockInputColumns(blockInput)) {
-        throw new Error(`Expected block input to have columns: slot=${blockSlot} root=${blockRootHex}`);
+  async getDataColumnSidecars(blockSlot: Slot, blockRootHex: string): Promise<DataColumnSidecar[]> {
+    const fork = this.config.getForkName(blockSlot);
+
+    if (isForkPostGloas(fork)) {
+      // After gloas, columns are tracked in PayloadEnvelopeInput
+      const payloadInput = this.seenPayloadEnvelopeInputCache.get(blockRootHex);
+      if (payloadInput) {
+        return payloadInput.getAllColumns();
+      }
+    } else {
+      // Before gloas, columns are tracked in BlockInput
+      const blockInput = this.seenBlockInputCache.get(blockRootHex);
+      if (blockInput) {
+        if (!isBlockInputColumns(blockInput)) {
+          throw new Error(`Expected block input to have columns: slot=${blockSlot} root=${blockRootHex}`);
+        }
+        return blockInput.getAllColumns();
       }
-      return blockInput.getAllColumns();
     }
+
     const sidecarsUnfinalized = await this.db.dataColumnSidecar.values(fromHex(blockRootHex));
     if (sidecarsUnfinalized.length > 0) {
-      return sidecarsUnfinalized as DataColumnSidecars;
+      return sidecarsUnfinalized;
     }
     const sidecarsFinalized = await this.db.dataColumnSidecarArchive.values(blockSlot);
-    return sidecarsFinalized as DataColumnSidecars;
+    return sidecarsFinalized;
   }
 
   async getSerializedDataColumnSidecars(
@@ -894,23 +907,45 @@ export class BeaconChain implements IBeaconChain {
     blockRootHex: string,
     indices: number[]
   ): Promise<(Uint8Array | undefined)[]> {
-    const blockInput = this.seenBlockInputCache.get(blockRootHex);
-    if (blockInput) {
-      if (!isBlockInputColumns(blockInput)) {
-        throw new Error(`Expected block input to have columns: slot=${blockSlot} root=${blockRootHex}`);
+    const fork = this.config.getForkName(blockSlot);
+
+    if (isForkPostGloas(fork)) {
+      // After gloas, columns are tracked in PayloadEnvelopeInput
+      const payloadInput = this.seenPayloadEnvelopeInputCache.get(blockRootHex);
+      if (payloadInput) {
+        return indices.map((index) => {
+          const sidecar = payloadInput.getColumn(index);
+          if (!sidecar) {
+            return undefined;
+          }
+          const serialized = this.serializedCache.get(sidecar);
+          if (serialized) {
+            return serialized;
+          }
+          return sszTypesFor(fork as ForkPostGloas).DataColumnSidecar.serialize(sidecar);
+        });
       }
-      return indices.map((index) => {
-        const sidecar = blockInput.getColumn(index);
-        if (!sidecar) {
-          return undefined;
-        }
-        const serialized = this.serializedCache.get(sidecar);
-        if (serialized) {
-          return serialized;
+    } else {
+      // Before gloas, columns are tracked in BlockInput
+      const blockInput = this.seenBlockInputCache.get(blockRootHex);
+      if (blockInput) {
+        if (!isBlockInputColumns(blockInput)) {
+          throw new Error(`Expected block input to have columns: slot=${blockSlot} root=${blockRootHex}`);
         }
-        return sszTypesFor(blockInput.forkName as ForkPostFulu).DataColumnSidecar.serialize(sidecar);
-      });
+        return indices.map((index) => {
+          const sidecar = blockInput.getColumn(index);
+          if (!sidecar) {
+            return undefined;
+          }
+          const serialized = this.serializedCache.get(sidecar);
+          if (serialized) {
+            return serialized;
+          }
+          return sszTypesFor(blockInput.forkName as ForkPostFulu).DataColumnSidecar.serialize(sidecar);
+        });
+      }
     }
+
     const sidecarsUnfinalized = await this.db.dataColumnSidecar.getManyBinary(fromHex(blockRootHex), indices);
     if (sidecarsUnfinalized.some((sidecar) => sidecar != null)) {
       return sidecarsUnfinalized;

package/src/chain/emitter.ts

@@ -3,7 +3,7 @@ import {StrictEventEmitter} from "strict-event-emitter-types";
 import {routes} from "@lodestar/api";
 import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice";
 import {IBeaconStateView} from "@lodestar/state-transition";
-import {DataColumnSidecars, RootHex, deneb, phase0} from "@lodestar/types";
+import {DataColumnSidecar, RootHex, deneb, phase0} from "@lodestar/types";
 import {PeerIdStr} from "../util/peerId.js";
 import {BlockInputSource, IBlockInput} from "./blocks/blockInput/types.js";
 
@@ -88,7 +88,7 @@ export type IChainEvents = ApiEvents & {
 
   [ChainEvent.updateTargetCustodyGroupCount]: (targetGroupCount: number) => void;
 
-  [ChainEvent.publishDataColumns]: (sidecars: DataColumnSidecars) => void;
+  [ChainEvent.publishDataColumns]: (sidecars: DataColumnSidecar[]) => void;
 
   [ChainEvent.publishBlobSidecars]: (sidecars: deneb.BlobSidecar[]) => void;
 

package/src/chain/errors/dataColumnSidecarError.ts

@@ -1,3 +1,4 @@
+import {ForkName} from "@lodestar/params";
 import {RootHex, Slot, SubnetID} from "@lodestar/types";
 import {LodestarError} from "@lodestar/utils";
 import {GossipActionError} from "./gossipValidation.js";
@@ -6,6 +7,7 @@ export enum DataColumnSidecarErrorCode {
   INVALID_INDEX = "DATA_COLUMN_SIDECAR_ERROR_INVALID_INDEX",
   NO_COMMITMENTS = "DATA_COLUMN_SIDECAR_ERROR_NO_COMMITMENTS",
   MISMATCHED_LENGTHS = "DATA_COLUMN_SIDECAR_ERROR_MISMATCHED_LENGTHS",
+  INCORRECT_TYPE = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_TYPE",
   INVALID_SUBNET = "DATA_COLUMN_SIDECAR_ERROR_INVALID_SUBNET",
   INVALID_KZG_PROOF = "DATA_COLUMN_SIDECAR_ERROR_INVALID_KZG_PROOF",
   TOO_MANY_KZG_COMMITMENTS = "DATA_COLUMN_SIDECAR_ERROR_TOO_MANY_KZG_COMMITMENTS",
@@ -18,6 +20,10 @@ export enum DataColumnSidecarErrorCode {
   INCORRECT_SIDECAR_COUNT = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_SIDECAR_COUNT",
   /** Sidecar doesn't match block */
   INCORRECT_BLOCK = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_BLOCK",
+  /** Sidecar slot doesn't match block slot */
+  INCORRECT_SIDECAR_SLOT = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_SIDECAR_SLOT",
+  /** Sidecar referenced block is not the expected block type */
+  INCORRECT_BLOCK_TYPE = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_BLOCK_TYPE",
   /** Sidecar cell count not as expected */
   INCORRECT_CELL_COUNT = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_CELL_COUNT",
   /** Sidecar kzg proof count not as expected */
@@ -32,10 +38,12 @@ export enum DataColumnSidecarErrorCode {
   FUTURE_SLOT = "DATA_COLUMN_SIDECAR_ERROR_FUTURE_SLOT",
   WOULD_REVERT_FINALIZED_SLOT = "DATA_COLUMN_SIDECAR_ERROR_WOULD_REVERT_FINALIZED_SLOT",
   PARENT_UNKNOWN = "DATA_COLUMN_SIDECAR_ERROR_PARENT_UNKNOWN",
+  BLOCK_UNKNOWN = "DATA_COLUMN_SIDECAR_ERROR_BLOCK_UNKNOWN",
   NOT_LATER_THAN_PARENT = "DATA_COLUMN_SIDECAR_ERROR_NOT_LATER_THAN_PARENT",
   PROPOSAL_SIGNATURE_INVALID = "DATA_COLUMN_SIDECAR_ERROR_PROPOSAL_SIGNATURE_INVALID",
   INCLUSION_PROOF_INVALID = "DATA_COLUMN_SIDECAR_ERROR_INCLUSION_PROOF_INVALID",
   INCORRECT_PROPOSER = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_PROPOSER",
+  PAYLOAD_ENVELOPE_INPUT_MISSING = "DATA_COLUMN_SIDECAR_ERROR_PAYLOAD_ENVELOPE_INPUT_MISSING",
 }
 
 export type DataColumnSidecarErrorType =
@@ -47,6 +55,12 @@ export type DataColumnSidecarErrorType =
       commitmentsLength: number;
       proofsLength: number;
     }
+  | {
+      code: DataColumnSidecarErrorCode.INCORRECT_TYPE;
+      slot: Slot;
+      columnIndex: number;
+      fork: ForkName;
+    }
   | {code: DataColumnSidecarErrorCode.INVALID_SUBNET; columnIndex: number; gossipSubnet: SubnetID}
   | {
       code: DataColumnSidecarErrorCode.TOO_MANY_KZG_COMMITMENTS;
@@ -63,6 +77,11 @@ export type DataColumnSidecarErrorType =
       parentRoot: RootHex;
       slot: Slot;
     }
+  | {
+      code: DataColumnSidecarErrorCode.BLOCK_UNKNOWN;
+      blockRoot: RootHex;
+      slot: Slot;
+    }
   | {
       code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID;
       slot: Slot;
@@ -80,6 +99,17 @@ export type DataColumnSidecarErrorType =
       expected: string;
       actual: string;
     }
+  | {
+      code: DataColumnSidecarErrorCode.INCORRECT_BLOCK_TYPE;
+      slot: Slot;
+      columnIndex: number;
+    }
+  | {
+      code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_SLOT;
+      columnIndex: number;
+      expected: Slot;
+      actual: Slot;
+    }
   | {
       code: DataColumnSidecarErrorCode.INCORRECT_HEADER_ROOT;
       slot: number;
@@ -97,7 +127,8 @@ export type DataColumnSidecarErrorType =
       actual: number;
     }
   | {code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF_BATCH; slot: number; reason: string}
-  | {code: DataColumnSidecarErrorCode.INCORRECT_PROPOSER; actualProposerIndex: number; expectedProposerIndex: number};
+  | {code: DataColumnSidecarErrorCode.INCORRECT_PROPOSER; actualProposerIndex: number; expectedProposerIndex: number}
+  | {code: DataColumnSidecarErrorCode.PAYLOAD_ENVELOPE_INPUT_MISSING; slot: Slot; blockRoot: RootHex};
 
 export class DataColumnSidecarGossipError extends GossipActionError<DataColumnSidecarErrorType> {}
 export class DataColumnSidecarValidationError extends LodestarError<DataColumnSidecarErrorType> {}

package/src/chain/interface.ts

@@ -5,7 +5,7 @@ import {EpochShuffling, IBeaconStateView, PubkeyCache} from "@lodestar/state-tra
 import {
   BeaconBlock,
   BlindedBeaconBlock,
-  DataColumnSidecars,
+  DataColumnSidecar,
   Epoch,
   Root,
   RootHex,
@@ -217,7 +217,7 @@ export interface IBeaconChain {
   ): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean} | null>;
   getBlobSidecars(blockSlot: Slot, blockRootHex: string): Promise<deneb.BlobSidecars | null>;
   getSerializedBlobSidecars(blockSlot: Slot, blockRootHex: string): Promise<Uint8Array | null>;
-  getDataColumnSidecars(blockSlot: Slot, blockRootHex: string): Promise<DataColumnSidecars>;
+  getDataColumnSidecars(blockSlot: Slot, blockRootHex: string): Promise<DataColumnSidecar[]>;
   getSerializedDataColumnSidecars(
     blockSlot: Slot,
     blockRootHex: string,