@lodestar/beacon-node 1.41.0-dev.aeb5a213ee → 1.41.0-dev.bb273175f2

package/src/chain/archiveStore/archiveStore.ts

@@ -1,4 +1,4 @@
-import {CheckpointWithHex} from "@lodestar/fork-choice";
+import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice";
 import {LoggerNode} from "@lodestar/logger/node";
 import {ForkSeq} from "@lodestar/params";
 import {Checkpoint} from "@lodestar/types/phase0";
@@ -44,7 +44,7 @@ export enum ArchiveStoreTask {
  */
 export class ArchiveStore {
   private archiveMode: ArchiveMode;
-  private jobQueue: JobItemQueue<[CheckpointWithHex], void>;
+  private jobQueue: JobItemQueue<[CheckpointWithPayloadStatus], void>;
 
   private archiveDataEpochs?: number;
   private readonly statesArchiverStrategy: StateArchiveStrategy;
@@ -67,7 +67,7 @@ export class ArchiveStore {
     this.archiveMode = opts.archiveMode;
     this.archiveDataEpochs = opts.archiveDataEpochs;
 
-    this.jobQueue = new JobItemQueue<[CheckpointWithHex], void>(this.processFinalizedCheckpoint, {
+    this.jobQueue = new JobItemQueue<[CheckpointWithPayloadStatus], void>(this.processFinalizedCheckpoint, {
       maxLength: PROCESS_FINALIZED_CHECKPOINT_QUEUE_LENGTH,
       signal,
     });
@@ -168,7 +168,7 @@ export class ArchiveStore {
   //-------------------------------------------------------------------------
   // Event handlers
   //-------------------------------------------------------------------------
-  private onFinalizedCheckpoint = (finalized: CheckpointWithHex): void => {
+  private onFinalizedCheckpoint = (finalized: CheckpointWithPayloadStatus): void => {
     this.jobQueue.push(finalized).catch((e) => {
       if (!isQueueErrorAborted(e)) {
         this.logger.error("Error queuing finalized checkpoint", {epoch: finalized.epoch}, e as Error);
@@ -189,7 +189,7 @@ export class ArchiveStore {
     });
   };
 
-  private processFinalizedCheckpoint = async (finalized: CheckpointWithHex): Promise<void> => {
+  private processFinalizedCheckpoint = async (finalized: CheckpointWithPayloadStatus): Promise<void> => {
     try {
       const finalizedEpoch = finalized.epoch;
       const finalizedFork = this.chain.config.getForkSeqAtEpoch(finalizedEpoch);

package/src/chain/archiveStore/historicalState/getHistoricalState.ts

@@ -1,10 +1,10 @@
-import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map";
 import {BeaconConfig} from "@lodestar/config";
 import {
   BeaconStateAllForks,
   CachedBeaconStateAllForks,
   DataAvailabilityStatus,
   ExecutionPayloadStatus,
+  PubkeyCache,
   createCachedBeaconState,
   stateTransition,
 } from "@lodestar/state-transition";
@@ -15,16 +15,16 @@ import {HistoricalStateRegenMetrics} from "./metrics.js";
 import {RegenErrorType} from "./types.js";
 
 /**
- * Populate a PubkeyIndexMap with any new entries based on a BeaconState
+ * Populate a PubkeyCache with any new entries based on a BeaconState
  */
-export function syncPubkeyCache(state: BeaconStateAllForks, pubkey2index: PubkeyIndexMap): void {
+export function syncPubkeyCache(state: BeaconStateAllForks, pubkeyCache: PubkeyCache): void {
   // Get the validators sub tree once for all the loop
   const validators = state.validators;
 
   const newCount = state.validators.length;
-  for (let i = pubkey2index.size; i < newCount; i++) {
+  for (let i = pubkeyCache.size; i < newCount; i++) {
     const pubkey = validators.getReadonly(i).pubkey;
-    pubkey2index.set(pubkey, i);
+    pubkeyCache.set(i, pubkey);
   }
 }
 
@@ -35,7 +35,7 @@ export async function getNearestState(
   slot: number,
   config: BeaconConfig,
   db: IBeaconDb,
-  pubkey2index: PubkeyIndexMap
+  pubkeyCache: PubkeyCache
 ): Promise<CachedBeaconStateAllForks> {
   const stateBytesArr = await db.stateArchive.binaries({limit: 1, lte: slot, reverse: true});
   if (!stateBytesArr.length) {
@@ -44,14 +44,13 @@ export async function getNearestState(
 
   const stateBytes = stateBytesArr[0];
   const state = getStateTypeFromBytes(config, stateBytes).deserializeToViewDU(stateBytes);
-  syncPubkeyCache(state, pubkey2index);
+  syncPubkeyCache(state, pubkeyCache);
 
   return createCachedBeaconState(
     state,
     {
       config,
-      pubkey2index,
-      index2pubkey: [],
+      pubkeyCache,
     },
     {
       skipSyncPubkeys: true,
@@ -66,13 +65,13 @@ export async function getHistoricalState(
   slot: number,
   config: BeaconConfig,
   db: IBeaconDb,
-  pubkey2index: PubkeyIndexMap,
+  pubkeyCache: PubkeyCache,
   metrics?: HistoricalStateRegenMetrics
 ): Promise<Uint8Array> {
   const regenTimer = metrics?.regenTime.startTimer();
 
   const loadStateTimer = metrics?.loadStateTime.startTimer();
-  let state = await getNearestState(slot, config, db, pubkey2index).catch((e) => {
+  let state = await getNearestState(slot, config, db, pubkeyCache).catch((e) => {
     metrics?.regenErrorCount.inc({reason: RegenErrorType.loadState});
     throw e;
   });

package/src/chain/archiveStore/historicalState/worker.ts

@@ -1,9 +1,9 @@
 import worker from "node:worker_threads";
-import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map";
 import {Transfer, expose} from "@chainsafe/threads/worker";
 import {chainConfigFromJson, createBeaconConfig} from "@lodestar/config";
 import {LevelDbController} from "@lodestar/db/controller/level";
 import {getNodeLogger} from "@lodestar/logger/node";
+import {createPubkeyCache} from "@lodestar/state-transition";
 import {BeaconDb} from "../../../db/index.js";
 import {RegistryMetricCreator, collectNodeJSMetrics} from "../../../metrics/index.js";
 import {JobFnQueue} from "../../../util/queue/fnQueue.js";
@@ -52,7 +52,7 @@ const queue = new JobFnQueue(
   queueMetrics
 );
 
-const pubkey2index = new PubkeyIndexMap();
+const pubkeyCache = createPubkeyCache();
 
 const api: HistoricalStateWorkerApi = {
   async close() {
@@ -65,7 +65,7 @@ const api: HistoricalStateWorkerApi = {
     historicalStateRegenMetrics?.regenRequestCount.inc();
 
     const stateBytes = await queue.push<Uint8Array>(() =>
-      getHistoricalState(slot, config, db, pubkey2index, historicalStateRegenMetrics)
+      getHistoricalState(slot, config, db, pubkeyCache, historicalStateRegenMetrics)
     );
     const result = Transfer(stateBytes, [stateBytes.buffer]) as unknown as Uint8Array;
 

package/src/chain/archiveStore/utils/archiveBlocks.ts

@@ -1,10 +1,10 @@
 import path from "node:path";
 import {ChainForkConfig} from "@lodestar/config";
 import {KeyValue} from "@lodestar/db";
-import {IForkChoice} from "@lodestar/fork-choice";
+import {CheckpointWithPayloadStatus, IForkChoice} from "@lodestar/fork-choice";
 import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
-import {Epoch, RootHex, Slot} from "@lodestar/types";
+import {Epoch, Slot} from "@lodestar/types";
 import {Logger, fromAsync, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/index.js";
 import {BlockArchiveBatchPutBinaryItem} from "../../../db/repositories/index.js";
@@ -19,7 +19,6 @@ const BLOCK_BATCH_SIZE = 256;
 const BLOB_SIDECAR_BATCH_SIZE = 32;
 
 type BlockRootSlot = {slot: Slot; root: Uint8Array};
-type CheckpointHex = {epoch: Epoch; rootHex: RootHex};
 
 /**
  * Persist orphaned block to disk
@@ -53,7 +52,7 @@ export async function archiveBlocks(
   forkChoice: IForkChoice,
   lightclientServer: LightClientServer | undefined,
   logger: Logger,
-  finalizedCheckpoint: CheckpointHex,
+  finalizedCheckpoint: CheckpointWithPayloadStatus,
   currentEpoch: Epoch,
   archiveDataEpochs?: number,
   persistOrphanedBlocks?: boolean,
@@ -62,7 +61,7 @@ export async function archiveBlocks(
   // Use fork choice to determine the blocks to archive and delete
   // getAllAncestorBlocks response includes the finalized block, so it's also moved to the cold db
   const {ancestors: finalizedCanonicalBlocks, nonAncestors: finalizedNonCanonicalBlocks} =
-    forkChoice.getAllAncestorAndNonAncestorBlocks(finalizedCheckpoint.rootHex);
+    forkChoice.getAllAncestorAndNonAncestorBlocks(finalizedCheckpoint.rootHex, finalizedCheckpoint.payloadStatus);
 
   // NOTE: The finalized block will be exactly the first block of `epoch` or previous
   const finalizedPostDeneb = finalizedCheckpoint.epoch >= config.DENEB_FORK_EPOCH;

package/src/chain/archiveStore/utils/updateBackfillRange.ts

@@ -20,7 +20,7 @@ export async function updateBackfillRange(
   try {
     // Mark the sequence in backfill db from finalized block's slot till anchor slot as
     // filled.
-    const finalizedBlockFC = chain.forkChoice.getBlockHex(finalized.rootHex);
+    const finalizedBlockFC = chain.forkChoice.getBlockHexDefaultStatus(finalized.rootHex);
     if (finalizedBlockFC && finalizedBlockFC.slot > chain.anchorStateLatestBlockSlot) {
       await db.backfilledRanges.put(finalizedBlockFC.slot, chain.anchorStateLatestBlockSlot);
 

package/src/chain/blocks/blockInput/blockInput.ts

@@ -1,5 +1,5 @@
-import {ForkName, ForkPostFulu, ForkPreDeneb, ForkPreGloas, NUMBER_OF_COLUMNS} from "@lodestar/params";
-import {BeaconBlockBody, BlobIndex, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types";
+import {ForkName, ForkPostFulu, ForkPostGloas, ForkPreDeneb, ForkPreGloas, NUMBER_OF_COLUMNS} from "@lodestar/params";
+import {BeaconBlockBody, BlobIndex, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu, gloas} from "@lodestar/types";
 import {byteArrayEquals, fromHex, prettyBytes, toRootHex, withTimeout} from "@lodestar/utils";
 import {VersionedHashes} from "../../../execution/index.js";
 import {kzgCommitmentToVersionedHash} from "../../../util/blobs.js";
@@ -24,7 +24,7 @@ import {
   SourceMeta,
 } from "./types.js";
 
-export type BlockInput = BlockInputPreData | BlockInputBlobs | BlockInputColumns;
+export type BlockInput = BlockInputPreData | BlockInputBlobs | BlockInputColumns | BlockInputNoData;
 
 export function isBlockInputPreDeneb(blockInput: IBlockInput): blockInput is BlockInputPreData {
   return blockInput.type === DAType.PreData;
@@ -37,6 +37,10 @@ export function isBlockInputColumns(blockInput: IBlockInput): blockInput is Bloc
   return blockInput.type === DAType.Columns;
 }
 
+export function isBlockInputNoData(blockInput: IBlockInput): blockInput is BlockInputNoData {
+  return blockInput.type === DAType.NoData;
+}
+
 function createPromise<T>(): PromiseParts<T> {
   let resolve!: (value: T) => void;
   let reject!: (e: Error) => void;
@@ -903,3 +907,64 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
     return Promise.resolve(this.getSampledColumns());
   }
 }
+
+type BlockInputNoDataState = {
+  hasBlock: true;
+  hasAllData: true;
+  block: SignedBeaconBlock<ForkPostGloas>;
+  source: SourceMeta;
+  timeCompleteSec: number;
+};
+
+export class BlockInputNoData extends AbstractBlockInput<ForkPostGloas, null> {
+  type = DAType.NoData as const;
+
+  state: BlockInputNoDataState;
+
+  private constructor(init: BlockInputInit, state: BlockInputNoDataState) {
+    super(init);
+    this.state = state;
+    this.dataPromise.resolve(null);
+    this.blockPromise.resolve(state.block);
+  }
+
+  static createFromBlock(props: AddBlock<ForkPostGloas> & CreateBlockInputMeta): BlockInputNoData {
+    const init: BlockInputInit = {
+      daOutOfRange: props.daOutOfRange,
+      timeCreated: props.seenTimestampSec,
+      forkName: props.forkName,
+      slot: props.block.message.slot,
+      blockRootHex: props.blockRootHex,
+      parentRootHex: toRootHex(props.block.message.parentRoot),
+    };
+    const state: BlockInputNoDataState = {
+      hasBlock: true,
+      hasAllData: true,
+      block: props.block,
+      source: {
+        source: props.source,
+        seenTimestampSec: props.seenTimestampSec,
+        peerIdStr: props.peerIdStr,
+      },
+      timeCompleteSec: props.seenTimestampSec,
+    };
+    return new BlockInputNoData(init, state);
+  }
+
+  addBlock(_: AddBlock<ForkPostGloas>, opts = {throwOnDuplicateAdd: true}): void {
+    if (opts.throwOnDuplicateAdd) {
+      throw new BlockInputError(
+        {
+          code: BlockInputErrorCode.INVALID_CONSTRUCTION,
+          blockRoot: this.blockRootHex,
+        },
+        "Cannot addBlock to BlockInputNoData - block already exists"
+      );
+    }
+  }
+
+  getBlobKzgCommitments(): deneb.BlobKzgCommitments {
+    return (this.state.block.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message
+      .blobKzgCommitments;
+  }
+}

package/src/chain/blocks/blockInput/types.ts

@@ -6,6 +6,7 @@ export enum DAType {
   PreData = "pre-data",
   Blobs = "blobs",
   Columns = "columns",
+  NoData = "no-data",
 }
 
 export type DAData = null | deneb.BlobSidecars | fulu.DataColumnSidecars;
@@ -99,6 +100,18 @@ export type MissingColumnMeta = {
   versionedHashes: VersionedHashes;
 };
 
+/**
+ * Minimal interface required to write data columns to the DB.
+ * Used by `writeDataColumnsToDb` and designed to be reusable across forks (e.g. Fulu, Gloas).
+ */
+export interface IDataColumnsInput {
+  readonly slot: Slot;
+  readonly blockRootHex: string;
+  getCustodyColumns(): fulu.DataColumnSidecars;
+  hasComputedAllData(): boolean;
+  waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<fulu.DataColumnSidecars>;
+}
+
 /**
  * This is used to validate that BlockInput implementations follow some minimal subset of operations
  * and that adding a new implementation won't break consumers that rely on this subset.

package/src/chain/blocks/importBlock.ts

@@ -20,7 +20,7 @@ import {
   isStartSlotOfEpoch,
   isStateValidatorsNodesPopulated,
 } from "@lodestar/state-transition";
-import {Attestation, BeaconBlock, altair, capella, electra, phase0, ssz} from "@lodestar/types";
+import {Attestation, BeaconBlock, altair, capella, electra, isGloasBeaconBlock, phase0, ssz} from "@lodestar/types";
 import {isErrorAborted, toRootHex} from "@lodestar/utils";
 import {ZERO_HASH_HEX} from "../../constants/index.js";
 import {callInNextEventLoop} from "../../util/eventLoop.js";
@@ -95,7 +95,7 @@ export async function importBlock(
   // Without this, a supernode syncing from behind can accumulate many blocks worth of column
   // data in memory (up to 128 columns per block) causing OOM before persistence catches up.
   await this.unfinalizedBlockWrites.waitForSpace();
-  this.unfinalizedBlockWrites.push([blockInput]).catch((e) => {
+  this.unfinalizedBlockWrites.push(blockInput).catch((e) => {
     if (!isQueueErrorAborted(e)) {
       this.logger.error("Error pushing block to unfinalized write queue", {slot: blockSlot}, e as Error);
     }
@@ -230,6 +230,32 @@ export async function importBlock(
     }
   }
 
+  // 4.5. Import payload attestations to fork choice (Gloas)
+  //
+  if (isGloasBeaconBlock(block.message)) {
+    for (const payloadAttestation of block.message.body.payloadAttestations) {
+      try {
+        // Extract PTC indices from aggregation bits
+        const ptcIndices: number[] = [];
+        for (let i = 0; i < payloadAttestation.aggregationBits.bitLen; i++) {
+          if (payloadAttestation.aggregationBits.get(i)) {
+            ptcIndices.push(i);
+          }
+        }
+
+        if (ptcIndices.length > 0) {
+          this.forkChoice.notifyPtcMessages(
+            toRootHex(payloadAttestation.data.beaconBlockRoot),
+            ptcIndices,
+            payloadAttestation.data.payloadPresent
+          );
+        }
+      } catch (e) {
+        this.logger.warn("Error processing PayloadAttestation from block", {slot: blockSlot}, e as Error);
+      }
+    }
+  }
+
   // 5. Compute head. If new head, immediately stateCache.setHeadState()
 
   const oldHead = this.forkChoice.getHead();
@@ -345,7 +371,7 @@ export async function importBlock(
         //  3) Proposer boost reorg related flag is turned on (this is checked inside the function)
         //  4) Block meets the criteria of being re-orged out (this is also checked inside the function)
         const result = this.forkChoice.shouldOverrideForkChoiceUpdate(
-          blockSummary.blockRoot,
+          blockSummary,
           this.clock.secFromSlot(currentSlot),
           currentSlot
         );
@@ -441,7 +467,12 @@ export async function importBlock(
     this.metrics?.currentActiveValidators.set(activeValidatorsCount);
     this.metrics?.currentValidators.set({status: "active"}, activeValidatorsCount);
 
-    const parentBlockSummary = this.forkChoice.getBlock(checkpointState.latestBlockHeader.parentRoot);
+    const parentBlockSummary = isGloasBeaconBlock(block.message)
+      ? this.forkChoice.getBlockHexAndBlockHash(
+          toRootHex(checkpointState.latestBlockHeader.parentRoot),
+          toRootHex(block.message.body.signedExecutionPayloadBid.message.parentBlockHash)
+        )
+      : this.forkChoice.getBlockDefaultStatus(checkpointState.latestBlockHeader.parentRoot);
 
     if (parentBlockSummary) {
       const justifiedCheckpoint = checkpointState.currentJustifiedCheckpoint;

package/src/chain/blocks/verifyBlocksDataAvailability.ts

@@ -29,6 +29,9 @@ export async function verifyBlocksDataAvailability(
 
   const availableTime = Math.max(0, Math.max(...blocks.map((blockInput) => blockInput.getTimeComplete())));
   const dataAvailabilityStatuses: DataAvailabilityStatus[] = blocks.map((blockInput) => {
+    if (blockInput.type === DAType.NoData) {
+      return DataAvailabilityStatus.NotRequired;
+    }
     if (blockInput.type === DAType.PreData) {
       return DataAvailabilityStatus.PreData;
     }

package/src/chain/blocks/verifyBlocksExecutionPayloads.ts

@@ -21,7 +21,7 @@ import {Metrics} from "../../metrics/metrics.js";
 import {IClock} from "../../util/clock.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {BlockProcessOpts} from "../options.js";
-import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
+import {isBlockInputBlobs, isBlockInputColumns, isBlockInputNoData} from "./blockInput/blockInput.js";
 import {IBlockInput} from "./blockInput/types.js";
 import {ImportBlockOpts} from "./types.js";
 
@@ -51,7 +51,8 @@ type VerifyBlockExecutionResponse =
   | VerifyExecutionErrorResponse
   | {executionStatus: ExecutionStatus.Valid; lvhResponse: LVHValidResponse; execError: null}
   | {executionStatus: ExecutionStatus.Syncing; lvhResponse?: LVHValidResponse; execError: null}
-  | {executionStatus: ExecutionStatus.PreMerge; lvhResponse: undefined; execError: null};
+  | {executionStatus: ExecutionStatus.PreMerge; lvhResponse: undefined; execError: null}
+  | {executionStatus: ExecutionStatus.PayloadSeparated; lvhResponse: undefined; execError: null};
 
 /**
  * Verifies 1 or more execution payloads from a linear sequence of blocks.
@@ -148,6 +149,12 @@ export async function verifyBlockExecutionPayload(
   preState0: CachedBeaconStateAllForks
 ): Promise<VerifyBlockExecutionResponse> {
   const block = blockInput.getBlock();
+
+  // Gloas block doesn't have execution payload. Return right away
+  if (isBlockInputNoData(blockInput)) {
+    return {executionStatus: ExecutionStatus.PayloadSeparated, lvhResponse: undefined, execError: null};
+  }
+
   /** Not null if execution is enabled */
   const executionPayloadEnabled =
     isExecutionStateType(preState0) &&

package/src/chain/blocks/verifyBlocksSanityChecks.ts

@@ -1,7 +1,7 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
-import {RootHex, Slot} from "@lodestar/types";
+import {RootHex, Slot, isGloasBeaconBlock} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
 import {IClock} from "../../util/clock.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
@@ -90,7 +90,12 @@ export function verifyBlocksSanityChecks(
     } else {
       // When importing a block segment, only the first NON-IGNORED block must be known to the fork-choice.
       const parentRoot = toRootHex(block.message.parentRoot);
-      parentBlock = chain.forkChoice.getBlockHex(parentRoot);
+      parentBlock = isGloasBeaconBlock(block.message)
+        ? chain.forkChoice.getBlockHexAndBlockHash(
+            parentRoot,
+            toRootHex(block.message.body.signedExecutionPayloadBid.message.parentBlockHash)
+          )
+        : chain.forkChoice.getBlockHexDefaultStatus(parentRoot);
       if (!parentBlock) {
         throw new BlockError(block, {code: BlockErrorCode.PARENT_UNKNOWN, parentRoot});
       }