@lodestar/beacon-node 1.51.0-dev.fac22802f6 → 1.51.0-dev.fb216e169a

package/src/chain/GetBlobsTracker.ts

@@ -4,12 +4,8 @@ import {Logger, pruneSetToMax} from "@lodestar/utils";
 import {BLOB_AND_PROOF_V2_RPC_BYTES} from "../execution/engine/types.js";
 import {IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/metrics.js";
-import {
-  DataColumnEngineResult,
-  getBlobSidecarsFromExecution,
-  getDataColumnSidecarsFromExecution,
-} from "../util/execution.js";
-import {IBlockInput, isBlockInputBlobs} from "./blocks/blockInput/index.js";
+import {DataColumnEngineResult, getDataColumnSidecarsFromExecution} from "../util/execution.js";
+import {IBlockInput, isBlockInputColumns} from "./blocks/blockInput/index.js";
 import {PayloadEnvelopeInput} from "./blocks/payloadEnvelopeInput/index.js";
 import {ChainEventEmitter} from "./emitter.js";
 
@@ -20,7 +16,7 @@ const MAX_TRACKED_BLOCK_ROOTS = 64;
 
 export type GetBlobsTrackerInit = {
   logger: Logger;
-  executionEngine: IExecutionEngine;
+  executionEngine: Pick<IExecutionEngine, "getBlobs">;
   emitter: ChainEventEmitter;
   metrics: Metrics | null;
   config: ChainForkConfig;
@@ -31,7 +27,7 @@ export type GetBlobsTrackerInit = {
  */
 export class GetBlobsTracker {
   logger: Logger;
-  executionEngine: IExecutionEngine;
+  executionEngine: Pick<IExecutionEngine, "getBlobs">;
   emitter: ChainEventEmitter;
   metrics: Metrics | null;
   config: ChainForkConfig;
@@ -51,6 +47,10 @@ export class GetBlobsTracker {
   }
 
   triggerGetBlobs(input: IBlockInput | PayloadEnvelopeInput): void {
+    if (!(input instanceof PayloadEnvelopeInput) && !isBlockInputColumns(input)) {
+      return;
+    }
+
     if (this.activeReconstructions.has(input.blockRootHex)) {
       this.pendingTriggers.add(input.blockRootHex);
       return;
@@ -74,37 +74,6 @@ export class GetBlobsTracker {
       }
     }
 
-    // The request is sent right away, before block processing issues newPayload: ELs that drop a
-    // transaction's blobs once the payload is validated can only answer a request that gets there first
-    if (!(input instanceof PayloadEnvelopeInput) && isBlockInputBlobs(input)) {
-      // there is not preallocation for blob sidecars like there is for columns sidecars so no need to
-      // store the index for the preallocated buffers
-      this.activeReconstructions.add(input.blockRootHex);
-      const logCtx = {slot: input.slot, root: input.blockRootHex};
-      this.logger.verbose("Trigger getBlobsV1 for block", logCtx);
-      let requestFailed = false;
-      getBlobSidecarsFromExecution(this.config, this.executionEngine, this.metrics, this.emitter, input)
-        .then(() => {
-          // EL errors and misses are handled inside the helper, blobs still missing means the attempt failed
-          if (!input.hasAllData()) {
-            requestFailed = true;
-            this.recordFailedAttempt(input.blockRootHex);
-          }
-        })
-        .catch((error) => {
-          this.logger.debug("Error during getBlobsV1 for block", logCtx, error as Error);
-          requestFailed = true;
-          this.recordFailedAttempt(input.blockRootHex);
-        })
-        .finally(() => {
-          this.logger.verbose("Completed getBlobsV1 for block", logCtx);
-          this.activeReconstructions.delete(input.blockRootHex);
-          this.retryPendingTrigger(input, requestFailed);
-        });
-
-      return;
-    }
-
     let freeIndex = this.blobsAndProofsBuffers.findIndex(({inUse}) => !inUse);
     if (freeIndex === -1) {
       freeIndex = this.blobsAndProofsBuffers.length;
@@ -126,6 +95,8 @@ export class GetBlobsTracker {
     const logCtx = {slot: input.slot, root: input.blockRootHex};
     this.logger.verbose("Trigger getBlobsV2 for block", logCtx);
     let requestFailed = false;
+    // The request is sent right away, before block processing issues newPayload: ELs that drop a
+    // transaction's blobs once the payload is validated can only answer a request that gets there first
     getDataColumnSidecarsFromExecution(
       this.config,
       this.executionEngine,

package/src/chain/blocks/blockInput/blockInput.ts

@@ -1,15 +1,12 @@
-import {ForkName, ForkPostFulu, ForkPostGloas, ForkPreDeneb, ForkPreGloas, NUMBER_OF_COLUMNS} from "@lodestar/params";
-import {BeaconBlockBody, BlobIndex, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu, gloas} from "@lodestar/types";
-import {byteArrayEquals, fromHex, prettyBytes, toRootHex, withTimeout} from "@lodestar/utils";
+import {ForkName, ForkPostFulu, ForkPostGloas, ForkPreFulu, ForkPreGloas, NUMBER_OF_COLUMNS} from "@lodestar/params";
+import {BeaconBlockBody, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu, gloas} from "@lodestar/types";
+import {prettyBytes, toRootHex, withTimeout} from "@lodestar/utils";
 import {VersionedHashes} from "../../../execution/index.js";
 import {kzgCommitmentToVersionedHash} from "../../../util/blobs.js";
 import {BlockInputError, BlockInputErrorCode} from "./errors.js";
 import {
-  AddBlob,
   AddBlock,
   AddColumn,
-  BlobMeta,
-  BlobWithSource,
   BlockInputInit,
   ColumnWithSource,
   CreateBlockInputMeta,
@@ -17,22 +14,17 @@ import {
   DAType,
   IBlockInput,
   LogMetaBasic,
-  LogMetaBlobs,
   LogMetaColumns,
   MissingColumnMeta,
   PromiseParts,
   SourceMeta,
 } from "./types.js";
 
-export type BlockInput = BlockInputPreData | BlockInputBlobs | BlockInputColumns | BlockInputNoData;
+export type BlockInput = BlockInputPreData | BlockInputColumns | BlockInputNoData;
 
-export function isBlockInputPreDeneb(blockInput: IBlockInput): blockInput is BlockInputPreData {
+export function isBlockInputPreData(blockInput: IBlockInput): blockInput is BlockInputPreData {
   return blockInput.type === DAType.PreData;
 }
-export function isBlockInputBlobs(blockInput: IBlockInput): blockInput is BlockInputBlobs {
-  return blockInput.type === DAType.Blobs;
-}
-
 export function isBlockInputColumns(blockInput: IBlockInput): blockInput is BlockInputColumns {
   return blockInput.type === DAType.Columns;
 }
@@ -189,16 +181,18 @@ abstract class AbstractBlockInput<F extends ForkName = ForkName, TData extends D
 type BlockInputPreDataState = {
   hasBlock: true;
   hasAllData: true;
-  block: SignedBeaconBlock<ForkPreDeneb>;
+  block: SignedBeaconBlock<ForkPreFulu>;
   source: SourceMeta;
   timeCompleteSec: number;
 };
 
 /**
- * Pre-DA, BlockInput only has a single state.
+ * Pre-fulu BlockInput, it only has a single state.
  * - the block simply exists
+ *
+ * Legacy deneb..electra blob sidecars are not tracked.
  */
-export class BlockInputPreData extends AbstractBlockInput<ForkPreDeneb, null> {
+export class BlockInputPreData extends AbstractBlockInput<ForkPreFulu, null> {
   type = DAType.PreData as const;
 
   state: BlockInputPreDataState;
@@ -210,7 +204,7 @@ export class BlockInputPreData extends AbstractBlockInput<ForkPreDeneb, null> {
     this.blockPromise.resolve(state.block);
   }
 
-  static createFromBlock(props: AddBlock<ForkPreDeneb> & CreateBlockInputMeta): BlockInputPreData {
+  static createFromBlock(props: AddBlock<ForkPreFulu> & CreateBlockInputMeta): BlockInputPreData {
     const init: BlockInputInit = {
       daOutOfRange: props.daOutOfRange,
       timeCreated: props.seenTimestampSec,
@@ -250,332 +244,6 @@ export class BlockInputPreData extends AbstractBlockInput<ForkPreDeneb, null> {
   }
 }
 
-// Blobs DA
-
-export type ForkBlobsDA = ForkName.deneb | ForkName.electra;
-
-type BlockInputBlobsState =
-  | {
-      hasBlock: true;
-      hasAllData: true;
-      versionedHashes: VersionedHashes;
-      block: SignedBeaconBlock<ForkBlobsDA>;
-      source: SourceMeta;
-      timeCompleteSec: number;
-    }
-  | {
-      hasBlock: true;
-      hasAllData: false;
-      versionedHashes: VersionedHashes;
-      block: SignedBeaconBlock<ForkBlobsDA>;
-      source: SourceMeta;
-    }
-  | {
-      hasBlock: false;
-      hasAllData: false;
-    };
-
-/**
- * With blobs, BlockInput has several states:
- * - The block is seen and all blobs are seen
- * - The block is seen and all blobs are not yet seen
- * - The block is yet not seen and its unknown if all blobs are seen
- */
-export class BlockInputBlobs extends AbstractBlockInput<ForkBlobsDA, deneb.BlobSidecars> {
-  type = DAType.Blobs as const;
-
-  state: BlockInputBlobsState;
-  private blobsCache = new Map<BlobIndex, BlobWithSource>();
-
-  private constructor(init: BlockInputInit, state: BlockInputBlobsState) {
-    super(init);
-    this.state = state;
-  }
-
-  static createFromBlock(props: AddBlock<ForkBlobsDA> & CreateBlockInputMeta): BlockInputBlobs {
-    const hasAllData = props.daOutOfRange || props.block.message.body.blobKzgCommitments.length === 0;
-
-    const state = {
-      hasBlock: true,
-      hasAllData,
-      versionedHashes: props.block.message.body.blobKzgCommitments.map(kzgCommitmentToVersionedHash),
-      block: props.block,
-      source: {
-        source: props.source,
-        seenTimestampSec: props.seenTimestampSec,
-        peerIdStr: props.peerIdStr,
-      },
-      timeCompleteSec: hasAllData ? props.seenTimestampSec : undefined,
-    } as BlockInputBlobsState;
-    const init: BlockInputInit = {
-      daOutOfRange: props.daOutOfRange,
-      timeCreated: props.seenTimestampSec,
-      forkName: props.forkName,
-      slot: props.block.message.slot,
-      blockRootHex: props.blockRootHex,
-      parentRootHex: toRootHex(props.block.message.parentRoot),
-    };
-    const blockInput = new BlockInputBlobs(init, state);
-    blockInput.blockPromise.resolve(props.block);
-    if (hasAllData) {
-      blockInput.dataPromise.resolve([]);
-    }
-    return blockInput;
-  }
-
-  static createFromBlob(props: AddBlob & CreateBlockInputMeta): BlockInputBlobs {
-    const state: BlockInputBlobsState = {
-      hasBlock: false,
-      hasAllData: false,
-    };
-    const init: BlockInputInit = {
-      daOutOfRange: props.daOutOfRange,
-      timeCreated: props.seenTimestampSec,
-      forkName: props.forkName,
-      blockRootHex: props.blockRootHex,
-      parentRootHex: toRootHex(props.blobSidecar.signedBlockHeader.message.parentRoot),
-      slot: props.blobSidecar.signedBlockHeader.message.slot,
-    };
-    const blockInput = new BlockInputBlobs(init, state);
-    blockInput.blobsCache.set(props.blobSidecar.index, {
-      blobSidecar: props.blobSidecar,
-      source: props.source,
-      seenTimestampSec: props.seenTimestampSec,
-      peerIdStr: props.peerIdStr,
-    });
-    return blockInput;
-  }
-
-  getLogMeta(): LogMetaBlobs {
-    return {
-      slot: this.slot,
-      blockRoot: prettyBytes(this.blockRootHex),
-      timeCreatedSec: this.timeCreatedSec,
-      expectedBlobs: this.state.hasBlock ? this.state.block.message.body.blobKzgCommitments.length : "unknown",
-      receivedBlobs: this.blobsCache.size,
-    };
-  }
-
-  addBlock(
-    {blockRootHex, block, source, seenTimestampSec, peerIdStr}: AddBlock<ForkBlobsDA>,
-    opts = {throwOnDuplicateAdd: true}
-  ): void {
-    // this check suffices for checking slot, parentRoot, and forkName
-    if (blockRootHex !== this.blockRootHex) {
-      throw new BlockInputError(
-        {
-          code: BlockInputErrorCode.MISMATCHED_ROOT_HEX,
-          blockInputRoot: this.blockRootHex,
-          mismatchedRoot: blockRootHex,
-          source,
-          peerId: `${peerIdStr}`,
-        },
-        "addBlock blockRootHex does not match BlockInput.blockRootHex"
-      );
-    }
-
-    if (!opts.throwOnDuplicateAdd) {
-      return;
-    }
-
-    if (this.state.hasBlock) {
-      throw new BlockInputError(
-        {
-          code: BlockInputErrorCode.INVALID_CONSTRUCTION,
-          blockRoot: this.blockRootHex,
-        },
-        "Cannot addBlock to BlockInputBlobs after it already has a block"
-      );
-    }
-
-    for (const {blobSidecar} of this.blobsCache.values()) {
-      if (!blockAndBlobArePaired(block, blobSidecar)) {
-        this.blobsCache.delete(blobSidecar.index);
-        // TODO: (@matthewkeil) spec says to ignore invalid blobs but should we downscore the peer maybe?
-        // this.logger?.error(`Removing blobIndex=${blobSidecar.index} from BlockInput`, {}, err);
-      }
-    }
-
-    const hasAllData = this.blobsCache.size === block.message.body.blobKzgCommitments.length;
-
-    this.state = {
-      ...this.state,
-      hasBlock: true,
-      hasAllData,
-      block,
-      versionedHashes: block.message.body.blobKzgCommitments.map(kzgCommitmentToVersionedHash),
-      source: {
-        source,
-        seenTimestampSec,
-        peerIdStr,
-      },
-      timeCompleteSec: hasAllData ? seenTimestampSec : undefined,
-    } as BlockInputBlobsState;
-    this.blockPromise.resolve(block);
-    if (hasAllData) {
-      this.dataPromise.resolve(this.getBlobs());
-    }
-  }
-
-  hasBlob(blobIndex: BlobIndex): boolean {
-    return this.blobsCache.has(blobIndex);
-  }
-
-  getBlob(blobIndex: BlobIndex): deneb.BlobSidecar | undefined {
-    return this.blobsCache.get(blobIndex)?.blobSidecar;
-  }
-
-  addBlob(
-    {blockRootHex, blobSidecar, source, peerIdStr, seenTimestampSec}: AddBlob,
-    opts = {throwOnDuplicateAdd: true}
-  ): void {
-    // this check suffices for checking slot, parentRoot, and forkName
-    if (blockRootHex !== this.blockRootHex) {
-      throw new BlockInputError(
-        {
-          code: BlockInputErrorCode.MISMATCHED_ROOT_HEX,
-          blockInputRoot: this.blockRootHex,
-          mismatchedRoot: blockRootHex,
-          source: source,
-          peerId: `${peerIdStr}`,
-        },
-        "Blob BeaconBlockHeader blockRootHex does not match BlockInput.blockRootHex"
-      );
-    }
-
-    const isDuplicate = this.blobsCache.has(blobSidecar.index);
-    if (isDuplicate && opts.throwOnDuplicateAdd) {
-      throw new BlockInputError(
-        {
-          code: BlockInputErrorCode.INVALID_CONSTRUCTION,
-          blockRoot: this.blockRootHex,
-        },
-        "Cannot addBlob to BlockInputBlobs with duplicate blobIndex"
-      );
-    }
-
-    if (this.state.hasBlock) {
-      assertBlockAndBlobArePaired(this.blockRootHex, this.state.block, blobSidecar);
-    }
-
-    if (isDuplicate) {
-      return;
-    }
-
-    this.blobsCache.set(blobSidecar.index, {blobSidecar, source, seenTimestampSec, peerIdStr});
-
-    if (this.state.hasBlock && this.blobsCache.size === this.state.block.message.body.blobKzgCommitments.length) {
-      this.state = {
-        ...this.state,
-        hasAllData: true,
-        timeCompleteSec: seenTimestampSec,
-      };
-      this.dataPromise.resolve([...this.blobsCache.values()].map(({blobSidecar}) => blobSidecar));
-    }
-  }
-
-  getVersionedHashes(): VersionedHashes {
-    if (!this.state.hasBlock) {
-      throw new BlockInputError(
-        {
-          code: BlockInputErrorCode.INCOMPLETE_DATA,
-          ...this.getLogMeta(),
-        },
-        "Cannot get versioned hashes. Block is unknown"
-      );
-    }
-    return this.state.versionedHashes;
-  }
-
-  getMissingBlobMeta(): BlobMeta[] {
-    if (!this.state.hasBlock) {
-      throw new BlockInputError(
-        {
-          code: BlockInputErrorCode.INCOMPLETE_DATA,
-          ...this.getLogMeta(),
-        },
-        "Cannot get missing blobs. Block is unknown"
-      );
-    }
-    if (this.state.hasAllData) {
-      return [];
-    }
-
-    const blobMeta: BlobMeta[] = [];
-    const versionedHashes = this.state.versionedHashes;
-    for (let index = 0; index < versionedHashes.length; index++) {
-      if (!this.blobsCache.has(index)) {
-        blobMeta.push({
-          index,
-          blockRoot: fromHex(this.blockRootHex),
-          versionedHash: versionedHashes[index],
-        });
-      }
-    }
-    return blobMeta;
-  }
-
-  getAllBlobsWithSource(): BlobWithSource[] {
-    if (!this.state.hasAllData) {
-      throw new BlockInputError(
-        {
-          code: BlockInputErrorCode.INCOMPLETE_DATA,
-          ...this.getLogMeta(),
-        },
-        "Cannot get all blobs. DA status is not complete"
-      );
-    }
-    return [...this.blobsCache.values()];
-  }
-
-  getBlobs(): deneb.BlobSidecars {
-    return this.getAllBlobsWithSource().map(({blobSidecar}) => blobSidecar);
-  }
-
-  getSerializedCacheKeys(): object[] {
-    const objects: object[] = [];
-
-    if (this.state.hasBlock) {
-      objects.push(this.state.block);
-    }
-
-    for (const {blobSidecar} of this.blobsCache.values()) {
-      objects.push(blobSidecar);
-    }
-
-    return objects;
-  }
-}
-
-function blockAndBlobArePaired(block: SignedBeaconBlock<ForkBlobsDA>, blobSidecar: deneb.BlobSidecar): boolean {
-  const blockCommitment = block.message.body.blobKzgCommitments[blobSidecar.index];
-  if (!blockCommitment || !blobSidecar.kzgCommitment) {
-    return false;
-  }
-  return byteArrayEquals(blockCommitment, blobSidecar.kzgCommitment);
-}
-
-function assertBlockAndBlobArePaired(
-  blockRootHex: string,
-  block: SignedBeaconBlock<ForkBlobsDA>,
-  blobSidecar: deneb.BlobSidecar
-): void {
-  if (!blockAndBlobArePaired(block, blobSidecar)) {
-    // TODO: (@matthewkeil) should this eject the bad blob instead? No way to tell if the blob or the block
-    //       has the invalid commitment. Guessing it would be the blob though because we match via block
-    //       hashTreeRoot and we do not take a hashTreeRoot of the BlobSidecar
-    throw new BlockInputError(
-      {
-        code: BlockInputErrorCode.MISMATCHED_KZG_COMMITMENT,
-        blockRoot: blockRootHex,
-        slot: block.message.slot,
-        sidecarIndex: blobSidecar.index,
-      },
-      "BlobSidecar commitment does not match block commitment"
-    );
-  }
-}
-
 // Columns DA
 
 export type ForkColumnsDA = ForkName.fulu;

package/src/chain/blocks/blockInput/errors.ts

@@ -1,7 +1,7 @@
 import {Slot} from "@lodestar/types";
 import {LodestarError} from "@lodestar/utils";
 import {PeerIdStr} from "../../../util/peerId.js";
-import {BlockInputSource, LogMetaBlobs, LogMetaColumns} from "./types.js";
+import {BlockInputSource, LogMetaColumns} from "./types.js";
 
 export enum BlockInputErrorCode {
   // Bad Arguments
@@ -42,7 +42,6 @@ export type BlockInputErrorType =
       sidecarIndex: number;
       commitmentIndex?: number;
     }
-  | (LogMetaBlobs & {code: BlockInputErrorCode.INCOMPLETE_DATA})
   | (LogMetaColumns & {code: BlockInputErrorCode.INCOMPLETE_DATA});
 
 export class BlockInputError extends LodestarError<BlockInputErrorType> {}

package/src/chain/blocks/blockInput/types.ts

@@ -1,15 +1,14 @@
 import {ForkName} from "@lodestar/params";
-import {ColumnIndex, DataColumnSidecar, RootHex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types";
+import {ColumnIndex, DataColumnSidecar, RootHex, SignedBeaconBlock, Slot, fulu} from "@lodestar/types";
 import {VersionedHashes} from "../../../execution/index.js";
 
 export enum DAType {
   PreData = "pre-data",
-  Blobs = "blobs",
   Columns = "columns",
   NoData = "no-data",
 }
 
-export type DAData = null | deneb.BlobSidecars | fulu.DataColumnSidecar[];
+export type DAData = null | fulu.DataColumnSidecar[];
 
 /**
  * Represents were input originated. Blocks and Data can come from different
@@ -37,11 +36,6 @@ export type LogMetaBasic = {
   timeCreatedSec: number;
 };
 
-export type LogMetaBlobs = LogMetaBasic & {
-  expectedBlobs: number | string;
-  receivedBlobs: number;
-};
-
 export type LogMetaColumns = LogMetaBasic & {
   expectedColumns: number;
   receivedColumns: number;
@@ -55,8 +49,6 @@ export type SourceMeta = {
 
 export type BlockWithSource = SourceMeta & {block: SignedBeaconBlock; blockRootHex: RootHex};
 
-export type BlobWithSource = SourceMeta & {blobSidecar: deneb.BlobSidecar};
-
 export type ColumnWithSource = SourceMeta & {columnSidecar: fulu.DataColumnSidecar};
 
 export type BlockHeaderMeta = {
@@ -82,20 +74,10 @@ export type AddBlock<F extends ForkName = ForkName> = SourceMeta & {
   blockRootHex: string;
 };
 
-export type AddBlob = BlobWithSource & {
-  blockRootHex: RootHex;
-};
-
 export type AddColumn = ColumnWithSource & {
   blockRootHex: RootHex;
 };
 
-export type BlobMeta = {
-  index: number;
-  blockRoot: Uint8Array;
-  versionedHash: Uint8Array;
-};
-
 export type MissingColumnMeta = {
   missing: ColumnIndex[];
   versionedHashes: VersionedHashes;