@lodestar/beacon-node 1.51.0-dev.596284206a → 1.51.0-dev.5a82190416

package/src/chain/blocks/importExecutionPayload.ts

@@ -2,6 +2,9 @@ import {SerializedList} from "@chainsafe/ssz";
 import {routes} from "@lodestar/api";
 import {
   ExecutionStatus,
+  ForkChoiceError,
+  ForkChoiceErrorCode,
+  InvalidBlockCode,
   PayloadExecutionStatus,
   getFinalizedExecutionBlockHash,
   getSafeExecutionBlockHash,
@@ -33,6 +36,7 @@ export enum PayloadErrorCode {
   EXECUTION_ENGINE_INVALID = "PAYLOAD_ERROR_EXECUTION_ENGINE_INVALID",
   EXECUTION_ENGINE_ERROR = "PAYLOAD_ERROR_EXECUTION_ENGINE_ERROR",
   BLOCK_NOT_IN_FORK_CHOICE = "PAYLOAD_ERROR_BLOCK_NOT_IN_FORK_CHOICE",
+  BLOCK_EXECUTION_INVALID = "PAYLOAD_ERROR_BLOCK_EXECUTION_INVALID",
   MISS_BLOCK_STATE = "PAYLOAD_ERROR_MISS_BLOCK_STATE",
   ENVELOPE_VERIFICATION_ERROR = "PAYLOAD_ERROR_ENVELOPE_VERIFICATION_ERROR",
   INVALID_SIGNATURE = "PAYLOAD_ERROR_INVALID_SIGNATURE",
@@ -54,6 +58,10 @@ export type PayloadErrorType =
       code: PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE;
       blockRootHex: string;
     }
+  | {
+      code: PayloadErrorCode.BLOCK_EXECUTION_INVALID;
+      blockRootHex: string;
+    }
   | {
       code: PayloadErrorCode.MISS_BLOCK_STATE;
       blockRootHex: string;
@@ -144,6 +152,13 @@ export async function importExecutionPayload(
       blockRootHex,
     });
   }
+  // The payload chain this block builds on was invalidated, a VALID verdict here would contradict it
+  if (protoBlock.executionStatus === ExecutionStatus.Invalid) {
+    throw new PayloadError(payloadInput, {
+      code: PayloadErrorCode.BLOCK_EXECUTION_INVALID,
+      blockRootHex,
+    });
+  }
 
   // 3. Regenerate state for envelope verification
   const blockState = await this.regen
@@ -322,14 +337,26 @@ export async function importExecutionPayload(
 
   // 6. Update fork choice, transitions the block's PENDING variant to FULL
   const execStatus = toForkChoiceExecutionStatus(execResult.status);
-  this.forkChoice.onExecutionPayload(
-    blockRootHex,
-    blockHashHex,
-    envelope.payload.blockNumber,
-    envelope.payload.gasLimit,
-    execStatus,
-    dataAvailabilityStatus
-  );
+  try {
+    this.forkChoice.onExecutionPayload(
+      blockRootHex,
+      blockHashHex,
+      envelope.payload.blockNumber,
+      envelope.payload.gasLimit,
+      execStatus,
+      dataAvailabilityStatus
+    );
+  } catch (e) {
+    // The payload chain can be invalidated by a forkchoiceUpdated verdict while the engine was awaited
+    if (
+      e instanceof ForkChoiceError &&
+      e.type.code === ForkChoiceErrorCode.INVALID_BLOCK &&
+      e.type.err.code === InvalidBlockCode.PARENT_EXECUTION_INVALID
+    ) {
+      throw new PayloadError(payloadInput, {code: PayloadErrorCode.BLOCK_EXECUTION_INVALID, blockRootHex});
+    }
+    throw e;
+  }
 
   // 7. Queue notifyForkchoiceUpdate to engine api
   const head = this.forkChoice.getHead();
@@ -342,11 +369,13 @@ export async function importExecutionPayload(
   ) {
     const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice, this.logger);
     const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice);
-    this.executionEngine.notifyForkchoiceUpdate(fork, blockHashHex, safeBlockHash, finalizedBlockHash).catch((e) => {
-      if (!isErrorAborted(e) && !isQueueErrorAborted(e)) {
-        this.logger.error("Error pushing notifyForkchoiceUpdate()", {blockHashHex, finalizedBlockHash}, e);
-      }
-    });
+    this.executionEngine
+      .notifyForkchoiceUpdate(fork, blockRootHex, blockHashHex, safeBlockHash, finalizedBlockHash)
+      .catch((e) => {
+        if (!isErrorAborted(e) && !isQueueErrorAborted(e)) {
+          this.logger.error("Error pushing notifyForkchoiceUpdate()", {blockHashHex, finalizedBlockHash}, e);
+        }
+      });
   }
 
   // 8. Record metrics for payload envelope and column sources

package/src/chain/blocks/index.ts

@@ -7,7 +7,7 @@ import {nextEventLoop} from "../../util/eventLoop.js";
 import {JobItemQueue, isQueueErrorAborted} from "../../util/queue/index.js";
 import type {BeaconChain} from "../chain.js";
 import {BlockError, BlockErrorCode, isBlockErrorAborted} from "../errors/index.js";
-import {ForkchoiceCaller} from "../forkChoice/index.js";
+import {ExecutionInvalidSource, ForkchoiceCaller} from "../forkChoice/index.js";
 import {BlockProcessOpts} from "../options.js";
 import {IBlockInput} from "./blockInput/types.js";
 import {importBlock, importsBlockAttestations} from "./importBlock.js";
@@ -123,7 +123,10 @@ export async function processBlocks(
     // and we need to further propagate
     if (segmentExecStatus.execAborted !== null) {
       if (segmentExecStatus.invalidSegmentLVH !== undefined) {
-        this.forkChoice.validateLatestHash(segmentExecStatus.invalidSegmentLVH);
+        this.executionEngine.invalidateExecutionPayloads(
+          segmentExecStatus.invalidSegmentLVH,
+          ExecutionInvalidSource.blockSegment
+        );
       }
       throw segmentExecStatus.execAborted.execError;
     }

package/src/chain/blocks/types.ts

@@ -1,17 +1,9 @@
-import type {ChainForkConfig} from "@lodestar/config";
 import type {BlockExecutionStatus, PayloadExecutionStatus} from "@lodestar/fork-choice";
-import {ForkSeq} from "@lodestar/params";
-import {DataAvailabilityStatus, IBeaconStateView, computeEpochAtSlot} from "@lodestar/state-transition";
+import {DataAvailabilityStatus, IBeaconStateView} from "@lodestar/state-transition";
 import type {IndexedAttestation, Slot, fulu} from "@lodestar/types";
 import {IBlockInput} from "./blockInput/types.js";
 import {OrphanedPayloadEnvelope} from "./utils/chainSegment.js";
 
-export enum GossipedInputType {
-  block = "block",
-  blob = "blob",
-  dataColumn = "data_column",
-}
-
 export enum InclusionListSource {
   gossip = "gossip",
   api = "api",
@@ -23,30 +15,11 @@ type DataColumnData = {
 };
 export type DataColumnsCacheMap = Map<number, DataColumnData>;
 
-export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clockSlot: Slot): boolean {
-  return (
-    config.getForkSeq(blockSlot) >= ForkSeq.deneb &&
-    // Only request blobs if they are recent enough
-    computeEpochAtSlot(blockSlot) >= computeEpochAtSlot(clockSlot) - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS
-  );
-}
-
 export enum AttestationImportOpt {
   Skip,
   Force,
 }
 
-export enum BlobSidecarValidation {
-  /** When recieved in gossip the blobs are individually verified before import */
-  Individual,
-  /**
-   * Blobs when recieved in req/resp can be fully verified before import
-   * but currently used in spec tests where blobs come without proofs and assumed
-   * to be valid
-   */
-  Full,
-}
-
 export type ImportPayloadOpts = {
   /**
    * Set to true when the envelope was already validated upstream (e.g., gossip/API validation):
@@ -89,8 +62,6 @@ export type ImportBlockOpts = {
    * Metadata: `true` if all the signatures including the proposer signature have been verified
    */
   validSignatures?: boolean;
-  /** Set to true if already run `validateBlobSidecars()` sucessfully on the blobs */
-  validBlobSidecars?: BlobSidecarValidation;
   /** Seen timestamp seconds */
   seenTimestampSec?: number;
 };
@@ -109,7 +80,10 @@ export type FullyVerifiedBlock = {
   parentBlockSlot: Slot;
   proposerBalanceDelta: number;
   dataAvailabilityStatus: DataAvailabilityStatus;
-  /** Pre-computed indexed attestations from signature verification to avoid duplicate work */
+  /**
+   * Pre-computed indexed attestations from signature verification to avoid duplicate work.
+   * Their `attestingIndices` are in committee order, not sorted, so they are not spec-valid `IndexedAttestation`s.
+   */
   indexedAttestations: IndexedAttestation[];
   /** Seen timestamp seconds */
   seenTimestampSec: number;

package/src/chain/blocks/verifyBlock.ts

@@ -4,7 +4,7 @@ import {
   DataAvailabilityStatus,
   IBeaconStateView,
   computeEpochAtSlot,
-  getIndexedAttestation,
+  getAttestingIndices,
   signedBlockToSignedHeader,
 } from "@lodestar/state-transition";
 import {IndexedAttestation, Slot, deneb} from "@lodestar/types";
@@ -119,7 +119,13 @@ export async function verifyBlocksInEpoch(
         const shuffling =
           this.shufflingCache.getSync(attEpoch, preState0.getShufflingDecisionRoot(attEpoch)) ??
           preState0.getShufflingAtEpoch(attEpoch);
-        return getIndexedAttestation(shuffling, fork, attestation);
+        // Unlike getIndexedAttestation, keep attestingIndices in committee order: sorting them dominates building
+        // these on mainnet blocks. See FullyVerifiedBlock.indexedAttestations.
+        return {
+          attestingIndices: getAttestingIndices(shuffling, fork, attestation),
+          data: attestation.data,
+          signature: attestation.signature,
+        };
       });
     }
 
@@ -175,16 +181,29 @@ export async function verifyBlocksInEpoch(
     const [
       segmentExecStatus,
       {blockDAStatuses, payloadDAStatuses, availableTime},
-      {postStates, proposerBalanceDeltas, verifyStateTime},
       {verifySignaturesTime},
+      {postStates, proposerBalanceDeltas, verifyStateTime},
     ] = await Promise.all([
       verifyExecutionPayloadsPromise,
 
       // data availability (fork-specific; see daAvailabilityPromise above)
       daAvailabilityPromise,
 
+      // All signatures at once. Must be called before verifyBlocksStateTransitionOnly()
+      // so we hand the BLS job to the worker pool before the main thread
+      // is busy with the state transition.
+      verifyBlocksSignatures(
+        this.config,
+        this.bls,
+        this.logger,
+        this.metrics,
+        preState0,
+        blocks,
+        indexedAttestationsByBlock,
+        opts
+      ),
+
       // Run state transition only
-      // TODO: Ensure it yields to allow flushing to workers and engine API
       verifyBlocksStateTransitionOnly(
         preState0,
         blockInputs,
@@ -198,22 +217,17 @@ export async function verifyBlocksInEpoch(
         opts
       ),
 
-      // All signatures at once
-      verifyBlocksSignatures(
-        this.config,
-        this.bls,
-        this.logger,
-        this.metrics,
-        preState0,
-        blocks,
-        indexedAttestationsByBlock,
-        opts
-      ),
-
       // TODO GLOAS: can verify payload signatures in batch too
       // maybe chain with the above verifyBlocksSignatures()
     ]);
 
+    if (blockInputs.length === 1 && opts.seenTimestampSec !== undefined) {
+      const consensusVerifiedTime = Math.max(verifyStateTime, verifySignaturesTime);
+      this.metrics?.gossipBlock.receivedToConsensusVerifiedTime.observe(
+        consensusVerifiedTime / 1000 - opts.seenTimestampSec
+      );
+    }
+
     for (const blockInput of blockInputs) {
       const block = blockInput.getBlock();
       const {slot, proposerIndex} = block.message;

package/src/chain/blocks/verifyBlocksDataAvailability.ts

@@ -1,3 +1,4 @@
+import {ForkSeq} from "@lodestar/params";
 import {DataAvailabilityStatus} from "@lodestar/state-transition";
 import {TimeoutError} from "@lodestar/utils";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
@@ -42,7 +43,11 @@ export async function verifyBlocksDataAvailability(
       return DataAvailabilityStatus.NotRequired;
     }
     if (blockInput.type === DAType.PreData) {
-      return DataAvailabilityStatus.PreData;
+      // Legacy deneb..electra blob sidecars are not tracked, so these blocks are imported without a DA check.
+      // PreData cannot represent them because the state transition rejects it for post-deneb forks.
+      return ForkSeq[blockInput.forkName] >= ForkSeq.deneb
+        ? DataAvailabilityStatus.NotRequired
+        : DataAvailabilityStatus.PreData;
     }
     if (blockInput.daOutOfRange) {
       return DataAvailabilityStatus.OutOfRange;

package/src/chain/blocks/verifyBlocksExecutionPayloads.ts

@@ -7,21 +7,23 @@ import {
   LVHValidResponse,
   ProtoBlock,
 } from "@lodestar/fork-choice";
-import {ForkSeq} from "@lodestar/params";
+import {ForkPostDeneb, ForkSeq, isForkPostDeneb} from "@lodestar/params";
 import {IBeaconStateView, isExecutionBlockBodyType, isStatePostBellatrix} from "@lodestar/state-transition";
-import {bellatrix, electra} from "@lodestar/types";
+import {SignedBeaconBlock, bellatrix, electra} from "@lodestar/types";
 import {ErrorAborted, Logger, toRootHex} from "@lodestar/utils";
 import {ExecutionPayloadStatus, IExecutionEngine} from "../../execution/engine/interface.js";
 import {Metrics} from "../../metrics/metrics.js";
+import {kzgCommitmentToVersionedHash} from "../../util/blobs.js";
 import {IClock} from "../../util/clock.js";
+import {getBlobKzgCommitments} from "../../util/dataColumns.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {BlockProcessOpts} from "../options.js";
-import {isBlockInputBlobs, isBlockInputColumns, isBlockInputNoData} from "./blockInput/blockInput.js";
+import {isBlockInputNoData} from "./blockInput/blockInput.js";
 import {IBlockInput} from "./blockInput/types.js";
 import {ImportBlockOpts} from "./types.js";
 
 export type VerifyBlockExecutionPayloadModules = {
-  executionEngine: IExecutionEngine;
+  executionEngine: Pick<IExecutionEngine, "notifyNewPayload">;
   clock: IClock;
   logger: Logger;
   metrics: Metrics | null;
@@ -166,8 +168,9 @@ export async function verifyBlockExecutionPayload(
 
   // TODO: Handle better notifyNewPayload() returning error is syncing
   const fork = blockInput.forkName;
-  const versionedHashes =
-    isBlockInputBlobs(blockInput) || isBlockInputColumns(blockInput) ? blockInput.getVersionedHashes() : undefined;
+  const versionedHashes = isForkPostDeneb(fork)
+    ? getBlobKzgCommitments(fork, block as SignedBeaconBlock<ForkPostDeneb>).map(kzgCommitmentToVersionedHash)
+    : undefined;
   const parentBlockRoot = ForkSeq[fork] >= ForkSeq.deneb ? block.message.parentRoot : undefined;
   const executionRequests =
     ForkSeq[fork] >= ForkSeq.electra ? (block.message.body as electra.BeaconBlockBody).executionRequests : undefined;

package/src/chain/blocks/verifyBlocksSanityChecks.ts

@@ -1,5 +1,5 @@
 import {ChainForkConfig} from "@lodestar/config";
-import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
+import {ExecutionStatus, IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {RootHex, Slot, isGloasBeaconBlock} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
@@ -118,6 +118,9 @@ export function verifyBlocksSanityChecks(
           parentBlock = parentBlockWithPayload;
         }
       }
+      if (parentBlock.executionStatus === ExecutionStatus.Invalid) {
+        throw new BlockError(block, {code: BlockErrorCode.PARENT_EXECUTION_INVALID, parentRoot});
+      }
       // Parent is known to the fork-choice
       parentBlockSlot = parentBlock.slot;
     }

package/src/chain/blocks/verifyBlocksSignatures.ts

@@ -36,10 +36,9 @@ export async function verifyBlocksSignatures(
     ? preState0.currentSyncCommitteeIndexed
     : new SyncCommitteeCacheEmpty();
 
-  // Verifies signatures after running state transition, so all SyncCommittee signed roots are known at this point.
-  // We must ensure block.slot <= state.slot before running getAllBlockSignatureSets().
-  // NOTE: If in the future multiple blocks signatures are verified at once, all blocks must be in the same epoch
-  // so the attester and proposer shufflings are correct.
+  // Runs concurrently with the state transition. Signature sets are built from preState0, which
+  // verifyBlocksInEpoch() guarantees is in the same epoch as every block, so the proposer and attester shufflings
+  // and the sync committee are the ones the blocks were signed under.
   for (const [i, block] of blocks.entries()) {
     // Use [i] to make clear that the index has to be correct to blame the right block below on BlockError()
     isValidPromises[i] = opts.validSignatures
@@ -50,7 +49,10 @@ export async function verifyBlocksSignatures(
         bls.verifySignatureSets(
           getBlockSignatureSets(config, currentSyncCommitteeIndexed, preState0, block, indexedAttestationsByBlock[i], {
             skipProposerSignature: opts.validProposerSignature,
-          })
+          }),
+          // Prioritize block verification over attestations, since
+          // attestations already buffer up to `MAX_BUFFER_WAIT_MS`
+          {priority: true}
         );
 
     // getBlockSignatureSets() takes 45ms in benchmarks for 2022Q2 mainnet blocks (100 sigs). When syncing a 32 blocks

package/src/chain/blocks/writeBlockInputToDb.ts

@@ -3,7 +3,7 @@ import {SignedBeaconBlock} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
 import {getBlobKzgCommitments} from "../../util/dataColumns.js";
 import {BeaconChain} from "../chain.js";
-import {IBlockInput, IDataColumnsInput, isBlockInputBlobs, isBlockInputColumns} from "./blockInput/index.js";
+import {IBlockInput, IDataColumnsInput, isBlockInputColumns} from "./blockInput/index.js";
 import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js";
 
 /**
@@ -12,11 +12,9 @@ import {BLOB_AVAILABILITY_TIMEOUT} from "./verifyBlocksDataAvailability.js";
  *
  * This operation may be performed before, during or after importing to the fork-choice. As long as errors
  * are handled properly for eventual consistency.
- *
- * Block+blobs (pre-fulu) and data columns (fulu+) are written in parallel.
  */
 export async function writeBlockInputToDb(this: BeaconChain, blockInput: IBlockInput): Promise<void> {
-  const promises: Promise<void>[] = [writeBlockAndBlobsToDb.call(this, blockInput)];
+  const promises: Promise<void>[] = [writeBlockToDb.call(this, blockInput)];
 
   if (isBlockInputColumns(blockInput)) {
     promises.push(writeDataColumnsToDb.call(this, blockInput));
@@ -26,7 +24,7 @@ export async function writeBlockInputToDb(this: BeaconChain, blockInput: IBlockI
   this.logger.debug("Persisted blockInput to db", {slot: blockInput.slot, root: blockInput.blockRootHex});
 }
 
-async function writeBlockAndBlobsToDb(this: BeaconChain, blockInput: IBlockInput): Promise<void> {
+async function writeBlockToDb(this: BeaconChain, blockInput: IBlockInput): Promise<void> {
   const block = blockInput.getBlock();
   const slot = block.message.slot;
   const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message);
@@ -34,38 +32,17 @@ async function writeBlockAndBlobsToDb(this: BeaconChain, blockInput: IBlockInput
   const numBlobs = isForkPostDeneb(blockInput.forkName)
     ? getBlobKzgCommitments(blockInput.forkName, block as SignedBeaconBlock<ForkPostDeneb>).length
     : undefined;
-  const fnPromises: Promise<void>[] = [];
+  this.logger.debug("Persist block to hot DB", {slot, root: blockRootHex, inputType: blockInput.type, numBlobs});
 
   const blockBytes = this.serializedCache.get(block);
   if (blockBytes) {
     // skip serializing data if we already have it
     this.metrics?.importBlock.persistBlockWithSerializedDataCount.inc();
-    fnPromises.push(this.db.block.putBinary(this.db.block.getId(block), blockBytes));
+    await this.db.block.putBinary(this.db.block.getId(block), blockBytes);
   } else {
     this.metrics?.importBlock.persistBlockNoSerializedDataCount.inc();
-    fnPromises.push(this.db.block.add(block));
+    await this.db.block.add(block);
   }
-
-  this.logger.debug("Persist block to hot DB", {slot, root: blockRootHex, inputType: blockInput.type, numBlobs});
-
-  if (isBlockInputBlobs(blockInput)) {
-    fnPromises.push(
-      (async () => {
-        if (!blockInput.hasAllData()) {
-          await blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT);
-        }
-        const blobSidecars = blockInput.getBlobs();
-        await this.db.blobSidecars.add({blockRoot, slot, blobSidecars});
-        this.logger.debug("Persisted blobSidecars to hot DB", {
-          slot,
-          root: blockRootHex,
-          numBlobs: blobSidecars.length,
-        });
-      })()
-    );
-  }
-
-  await Promise.all(fnPromises);
 }
 
 /**

package/src/chain/bls/interface.ts

@@ -22,6 +22,8 @@ export type VerifySignatureOpts = {
   verifyOnMainThread?: boolean;
   /**
    * Some signature sets are more important than others, and should be verified first.
+   * Non-batchable priority jobs are posted to an idle worker before this call returns, so the caller
+   * can run synchronous work right after without delaying the verification.
    */
   priority?: boolean;
 };

package/src/chain/bls/multithread/index.ts

@@ -333,15 +333,19 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
       }
     }
 
+    // First hand work to an idle worker before returning to the caller.
+    // Deferring the post to the next macrotask would make the worker
+    // sit idle while waiting for the next synchronous work
+    else if (job.opts.priority) {
+      this.jobs.unshift(job);
+      void this.runJob();
+    }
+
     // Push job and schedule to call `runJob` in the next macro event loop cycle.
     // This is useful to allow batching job submitted from a synchronous for loop,
     // and to prevent large stacks since runJob may be called recursively.
     else {
-      if (job.opts.priority) {
-        this.jobs.unshift(job);
-      } else {
-        this.jobs.push(job);
-      }
+      this.jobs.push(job);
       callInNextEventLoop(() => void this.runJob());
     }
   }