@lodestar/beacon-node 1.51.0-dev.c99cf49add → 1.51.0-dev.cda05d18f1

package/src/api/impl/beacon/blocks/index.ts

@@ -37,7 +37,7 @@ import {
   sszTypesFor,
 } from "@lodestar/types";
 import {fromHex, prettyGweiToEth, sleep, toHex, toRootHex} from "@lodestar/utils";
-import {BlockInputSource, isBlockInputBlobs, isBlockInputColumns} from "../../../../chain/blocks/blockInput/index.js";
+import {BlockInputSource, isBlockInputColumns} from "../../../../chain/blocks/blockInput/index.js";
 import {PayloadEnvelopeInputSource} from "../../../../chain/blocks/payloadEnvelopeInput/index.js";
 import {ImportBlockOpts} from "../../../../chain/blocks/types.js";
 import {verifyBlocksInEpoch} from "../../../../chain/blocks/verifyBlock.js";
@@ -62,12 +62,11 @@ import {
 } from "../../../../chain/produceBlock/index.js";
 import {RegenCaller} from "../../../../chain/regen/index.js";
 import {validateGossipBlock, verifyBlockProposerSignature} from "../../../../chain/validation/block.js";
-import {validateApiExecutionPayloadBid} from "../../../../chain/validation/executionPayloadBid.js";
+import {validateGossipExecutionPayloadBid} from "../../../../chain/validation/executionPayloadBid.js";
 import {validateApiExecutionPayloadEnvelope} from "../../../../chain/validation/executionPayloadEnvelope.js";
 import {OpSource} from "../../../../chain/validatorMonitor.js";
 import {
   computePreFuluKzgCommitmentsInclusionProof,
-  getBlobSidecars,
   kzgCommitmentToVersionedHash,
   reconstructBlobs,
 } from "../../../../util/blobs.js";
@@ -79,6 +78,7 @@ import {
 import {isOptimisticBlock} from "../../../../util/forkChoice.js";
 import {kzg} from "../../../../util/kzg.js";
 import {promiseAllMaybeAsync} from "../../../../util/promises.js";
+import {ApiOptions} from "../../../options.js";
 import {ApiModules} from "../../types.js";
 import {assertUniqueItems} from "../../utils.js";
 import {getBlockResponse, toBeaconHeaderResponse} from "./utils.js";
@@ -96,16 +96,10 @@ const MAX_API_CLOCK_DISPARITY_MS = 1000;
  */
 const IDENTITY_PEER_ID = ""; // TODO: Compute identity keypair
 
-export function getBeaconBlockApi({
-  chain,
-  config,
-  metrics,
-  network,
-  db,
-}: Pick<
-  ApiModules,
-  "chain" | "config" | "metrics" | "network" | "db"
->): ApplicationMethods<routes.beacon.block.Endpoints> {
+export function getBeaconBlockApi(
+  opts: ApiOptions,
+  {chain, config, metrics, network, db}: Pick<ApiModules, "chain" | "config" | "metrics" | "network" | "db">
+): ApplicationMethods<routes.beacon.block.Endpoints> {
   const publishBlockV2: ApplicationMethods<routes.beacon.block.Endpoints>["publishBlockV2"] = async (
     {signedBlockContents, broadcastValidation, builderUrl},
     context,
@@ -115,6 +109,9 @@ export function getBeaconBlockApi({
     const signedBlock = signedBlockContents.signedBlock;
     const slot = signedBlock.message.slot;
     const fork = config.getForkName(slot);
+    if (!isForkPostFulu(fork)) {
+      throw new ApiError(400, `Publishing blocks is not supported for pre-fulu fork=${fork}`);
+    }
     const blockRoot = toRootHex(chain.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(signedBlock.message));
 
     const blockForImport = chain.seenBlockInputCache.getByBlock({
@@ -136,40 +133,26 @@ export function getBeaconBlockApi({
       });
     }
 
-    let blobSidecars: deneb.BlobSidecars, dataColumnSidecars: fulu.DataColumnSidecar[];
-
-    if (isDenebBlockContents(signedBlockContents)) {
-      if (isForkPostGloas(fork)) {
-        // After gloas, data columns are not published with the block but when publishing the execution payload envelope
-        blobSidecars = [];
-        dataColumnSidecars = [];
-      } else if (isForkPostFulu(fork)) {
-        const timer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer();
-        // If the block was produced by this node, we will already have computed cells
-        // Otherwise, we will compute them from the blobs in this function
-        const cells =
-          (chain.blockProductionCache.get(blockRoot) as ProduceFullFulu)?.cells ??
-          signedBlockContents.blobs.map((blob) => kzg.computeCells(blob));
-        const cellsAndProofs = cells.map((rowCells, rowIndex) => ({
-          cells: rowCells,
-          proofs: signedBlockContents.kzgProofs.slice(rowIndex * NUMBER_OF_COLUMNS, (rowIndex + 1) * NUMBER_OF_COLUMNS),
-        }));
-        dataColumnSidecars = getDataColumnSidecarsFromBlock(
-          config,
-          signedBlock as SignedBeaconBlock<ForkPostFulu>,
-          cellsAndProofs
-        ) as fulu.DataColumnSidecar[];
-        timer?.();
-        blobSidecars = [];
-      } else if (isForkPostDeneb(fork)) {
-        blobSidecars = getBlobSidecars(config, signedBlock, signedBlockContents.blobs, signedBlockContents.kzgProofs);
-        dataColumnSidecars = [];
-      } else {
-        throw Error(`Invalid data fork=${fork} for publish`);
-      }
-    } else {
-      blobSidecars = [];
-      dataColumnSidecars = [];
+    // After gloas, data columns are not published with the block but when publishing the execution payload envelope.
+    let dataColumnSidecars: fulu.DataColumnSidecar[] = [];
+
+    if (isDenebBlockContents(signedBlockContents) && isForkPostFulu(fork) && !isForkPostGloas(fork)) {
+      const timer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer();
+      // If the block was produced by this node, we will already have computed cells
+      // Otherwise, we will compute them from the blobs in this function
+      const cells =
+        (chain.blockProductionCache.get(blockRoot) as ProduceFullFulu)?.cells ??
+        signedBlockContents.blobs.map((blob) => kzg.computeCells(blob));
+      const cellsAndProofs = cells.map((rowCells, rowIndex) => ({
+        cells: rowCells,
+        proofs: signedBlockContents.kzgProofs.slice(rowIndex * NUMBER_OF_COLUMNS, (rowIndex + 1) * NUMBER_OF_COLUMNS),
+      }));
+      dataColumnSidecars = getDataColumnSidecarsFromBlock(
+        config,
+        signedBlock as SignedBeaconBlock<ForkPostFulu>,
+        cellsAndProofs
+      ) as fulu.DataColumnSidecar[];
+      timer?.();
     }
 
     if (isBlockInputColumns(blockForImport)) {
@@ -187,19 +170,6 @@ export function getBeaconBlockApi({
           {throwOnDuplicateAdd: false}
         );
       }
-    } else if (isBlockInputBlobs(blockForImport)) {
-      for (const blobSidecar of blobSidecars) {
-        blockForImport.addBlob(
-          {
-            blockRootHex: blockRoot,
-            blobSidecar,
-            source: BlockInputSource.api,
-            seenTimestampSec,
-          },
-          // Same as above for columns
-          {throwOnDuplicateAdd: false}
-        );
-      }
     }
 
     // check what validations have been requested before broadcasting and publishing the block
@@ -321,18 +291,8 @@ export function getBeaconBlockApi({
         break;
       }
 
-      case routes.beacon.BroadcastValidation.none: {
-        chain.logger.debug("Skipping broadcast validation", valLogMeta);
-        break;
-      }
-
       default: {
-        // error or log warning we do not support this validation
-        const message = `Broadcast validation of ${broadcastValidation} type not implemented yet`;
-        if (chain.opts.broadcastValidationStrictness === "error") {
-          throw Error(message);
-        }
-        chain.logger.warn(message, valLogMeta);
+        throw new ApiError(400, `Invalid broadcast validation level ${broadcastValidation}`);
       }
     }
 
@@ -399,29 +359,26 @@ export function getBeaconBlockApi({
       //
       () => network.publishBeaconBlock(signedBlock),
       ...dataColumnSidecars.map((dataColumnSidecar) => () => network.publishDataColumnSidecar(dataColumnSidecar)),
-      ...blobSidecars.map((blobSidecar) => () => network.publishBlobSidecar(blobSidecar)),
       () =>
         // there is no rush to persist block since we published it to gossip anyway
-        chain
-          .processBlock(blockForImport, opts)
-          .catch((e) => {
-            if (e instanceof BlockError) {
-              switch (e.type.code) {
-                case BlockErrorCode.ALREADY_KNOWN:
-                  // Block was imported while publishing, e.g. received via gossip
-                  chain.logger.debug("Ignoring already-known block during publishing", valLogMeta);
-                  return;
-                case BlockErrorCode.PARENT_BLOCK_UNKNOWN:
-                case BlockErrorCode.PARENT_PAYLOAD_UNKNOWN:
-                  chain.emitter.emit(ChainEvent.blockUnknownParent, {
-                    blockInput: blockForImport,
-                    peer: IDENTITY_PEER_ID,
-                    source: BlockInputSource.api,
-                  });
-              }
+        chain.processBlock(blockForImport, opts).catch((e) => {
+          if (e instanceof BlockError) {
+            switch (e.type.code) {
+              case BlockErrorCode.ALREADY_KNOWN:
+                // Block was imported while publishing, e.g. received via gossip
+                chain.logger.debug("Ignoring already-known block during publishing", valLogMeta);
+                return;
+              case BlockErrorCode.PARENT_BLOCK_UNKNOWN:
+              case BlockErrorCode.PARENT_PAYLOAD_UNKNOWN:
+                chain.emitter.emit(ChainEvent.blockUnknownParent, {
+                  blockInput: blockForImport,
+                  peer: IDENTITY_PEER_ID,
+                  source: BlockInputSource.api,
+                });
             }
-            throw e;
-          }),
+          }
+          throw e;
+        }),
     ];
     const sentPeersArr = await promiseAllMaybeAsync<{sentPeers: number; alreadyPublished: boolean} | number | void>(
       publishPromises
@@ -471,20 +428,6 @@ export function getBeaconBlockApi({
           });
         }
       }
-    } else if (isBlockInputBlobs(blockForImport) && chain.emitter.listenerCount(routes.events.EventType.blobSidecar)) {
-      const blobSidecars = blockForImport.getBlobs();
-      const versionedHashes = blockForImport.getVersionedHashes();
-
-      for (const blobSidecar of blobSidecars) {
-        const {index, kzgCommitment} = blobSidecar;
-        chain.emitter.emit(routes.events.EventType.blobSidecar, {
-          blockRoot,
-          slot,
-          index,
-          kzgCommitment: toHex(kzgCommitment),
-          versionedHash: toHex(versionedHashes[index]),
-        });
-      }
     }
   };
 
@@ -809,11 +752,6 @@ export function getBeaconBlockApi({
       };
       try {
         switch (broadcastValidation) {
-          case routes.beacon.BroadcastValidation.none: {
-            chain.logger.debug("Skipping broadcast validation of execution payload envelope", valLogMeta);
-            break;
-          }
-
           case routes.beacon.BroadcastValidation.gossip: {
             await validateApiExecutionPayloadEnvelope(chain, signedExecutionPayloadEnvelope);
             break;
@@ -852,11 +790,7 @@ export function getBeaconBlockApi({
           }
 
           default: {
-            const message = `Broadcast validation of ${broadcastValidation} type not implemented yet`;
-            if (chain.opts.broadcastValidationStrictness === "error") {
-              throw Error(message);
-            }
-            chain.logger.warn(message, valLogMeta);
+            throw new ApiError(400, `Invalid broadcast validation level ${broadcastValidation}`);
           }
         }
       } catch (error) {
@@ -1008,9 +942,13 @@ export function getBeaconBlockApi({
         () => network.publishSignedExecutionPayloadEnvelope(signedExecutionPayloadEnvelope),
         // Publish all data column sidecars
         ...dataColumnSidecars.map((dataColumnSidecar) => () => network.publishDataColumnSidecar(dataColumnSidecar)),
-        // Import execution payload. Signature is verified during broadcast validation, an already
-        // known envelope was verified by its original source and `none` deliberately skips validation
-        () => chain.processExecutionPayload(payloadInput, {validSignature: true}),
+        // Import execution payload
+        () =>
+          chain.processExecutionPayload(payloadInput, {
+            // Signature is verified during broadcast validation only for the submitted envelope,
+            // an envelope cached by another source is verified at import
+            validSignature: payloadInput.getPayloadEnvelope() === signedExecutionPayloadEnvelope,
+          }),
       ];
 
       const publishPromise = promiseAllMaybeAsync<{sentPeers: number; alreadyPublished: boolean} | number | void>(
@@ -1077,15 +1015,25 @@ export function getBeaconBlockApi({
         throw new ApiError(400, `publishExecutionPayloadBid not supported for pre-gloas fork=${fork}`);
       }
 
-      const validationTimer = metrics?.opPool.executionPayloadBidPool.apiValidationTime.startTimer();
-      // TODO: once the builder is tested and can be trusted, skip this local validation
-      // to publish faster, accepting peer-score risk if it ever emits an invalid bid.
-      await validateApiExecutionPayloadBid(chain, signedExecutionPayloadBid);
-      validationTimer?.();
-
       const elapsedSec = chain.clock.secFromSlot(slot, seenTimestampSec);
       metrics?.gossipExecutionPayloadBid.elapsedTimeTillReceived.observe({source: OpSource.api}, elapsedSec);
 
+      if (!opts.disableBidValidation) {
+        const validationTimer = metrics?.opPool.executionPayloadBidPool.apiValidationTime.startTimer();
+        const {proposerIndex} = await validateGossipExecutionPayloadBid(chain, signedExecutionPayloadBid);
+        validationTimer?.();
+
+        // Gossip does not deliver our own messages back to us, so a validated bid is added to the
+        // pool here to allow a proposer on this node to select it
+        const insertOutcome = chain.executionPayloadBidPool.add(
+          signedExecutionPayloadBid,
+          Math.floor(elapsedSec * 1000)
+        );
+        metrics?.opPool.executionPayloadBidPool.apiInsertOutcome.inc({insertOutcome});
+
+        chain.validatorMonitor?.registerExecutionPayloadBid(OpSource.api, proposerIndex, bid);
+      }
+
       const sentPeers = await network.publishSignedExecutionPayloadBid(signedExecutionPayloadBid);
 
       chain.emitter.emit(routes.events.EventType.executionPayloadBid, {
@@ -1104,7 +1052,7 @@ export function getBeaconBlockApi({
     },
 
     async getSignedExecutionPayloadEnvelope({blockId}, context) {
-      const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId);
+      const {block, finalized} = await getBlockResponse(chain, blockId);
       const slot = block.message.slot;
       const fork = config.getForkName(slot);
 
@@ -1118,6 +1066,13 @@ export function getBeaconBlockApi({
       const blockRoot = config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message);
       const blockRootHex = toRootHex(blockRoot);
 
+      // The block's own flag is the status of the payload it builds on, the envelope's is that of its
+      // FULL variant. An unfinalized block without one has not imported its envelope yet.
+      const payload = finalized ? null : chain.forkChoice.getBlockHex(blockRootHex, PayloadStatus.FULL);
+      if (!finalized && payload === null) {
+        throw new ApiError(404, `Execution payload envelope not found for slot=${slot}, blockRoot=${blockRootHex}`);
+      }
+
       let data: Uint8Array | gloas.SignedExecutionPayloadEnvelope | null;
       try {
         data = context?.returnBytes
@@ -1142,7 +1097,7 @@ export function getBeaconBlockApi({
 
       return {
         data,
-        meta: {executionOptimistic, finalized, version: fork},
+        meta: {executionOptimistic: payload !== null && isOptimisticBlock(payload), finalized, version: fork},
       };
     },
 

package/src/api/impl/beacon/index.ts

@@ -3,6 +3,7 @@ import {ApplicationMethods} from "@lodestar/api/server";
 import {isForkPostGloas} from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {toRootHex} from "@lodestar/utils";
+import {ApiOptions} from "../../options.js";
 import {ApiError} from "../errors.js";
 import {ApiModules} from "../types.js";
 import {getBeaconBlockApi} from "./blocks/index.js";
@@ -11,9 +12,10 @@ import {getBeaconRewardsApi} from "./rewards/index.js";
 import {getBeaconStateApi} from "./state/index.js";
 
 export function getBeaconApi(
+  opts: ApiOptions,
   modules: Pick<ApiModules, "chain" | "config" | "logger" | "metrics" | "network" | "db" | "sync">
 ): ApplicationMethods<routes.beacon.Endpoints> {
-  const block = getBeaconBlockApi(modules);
+  const block = getBeaconBlockApi(opts, modules);
   const pool = getBeaconPoolApi(modules);
   const state = getBeaconStateApi(modules);
   const rewards = getBeaconRewardsApi(modules);

package/src/api/impl/beacon/pool/index.ts

@@ -17,6 +17,7 @@ import {
   PayloadAttestationError,
   PayloadAttestationErrorCode,
   SyncCommitteeError,
+  SyncCommitteeErrorCode,
 } from "../../../../chain/errors/index.js";
 import {validateApiAttesterSlashing} from "../../../../chain/validation/attesterSlashing.js";
 import {validateApiBlsToExecutionChange} from "../../../../chain/validation/blsToExecutionChange.js";
@@ -122,6 +123,32 @@ export function getBeaconPoolApi({
               logger.debug("Error adding unaggregated attestation to pool", {subnet}, e as Error);
             }
 
+            // Collected only for vote slots that an upcoming block of a connected validator can include
+            if (isForkPostElectra(fork) && chain.proposerTracker.shouldCollect(slot)) {
+              try {
+                const insertOutcome = chain.singleAttestationPool.add(
+                  attestation as SingleAttestation<ForkPostElectra>,
+                  attDataRootHex,
+                  validatorCommitteeIndex,
+                  committeeSize
+                );
+                metrics?.opPool.singleAttestationPool.apiInsertOutcome.inc({insertOutcome});
+              } catch (e) {
+                logger.debug(
+                  "Error adding single attestation to pool",
+                  {
+                    slot,
+                    committeeIndex,
+                    validatorCommitteeIndex,
+                    validatorIndex: (attestation as SingleAttestation<ForkPostElectra>).attesterIndex,
+                    subnet,
+                    attDataRoot: attDataRootHex,
+                  },
+                  e as Error
+                );
+              }
+            }
+
             if (isForkPostElectra(fork)) {
               chain.emitter.emit(
                 routes.events.EventType.singleAttestation,
@@ -340,7 +367,10 @@ export function getBeaconPoolApi({
             const synCommittee = state.getIndexedSyncCommittee(signature.slot);
             const indexesInCommittee = synCommittee.validatorIndexMap.get(signature.validatorIndex);
             if (indexesInCommittee === undefined || indexesInCommittee.length === 0) {
-              return; // Not a sync committee member
+              throw new SyncCommitteeError(GossipAction.REJECT, {
+                code: SyncCommitteeErrorCode.VALIDATOR_NOT_IN_SYNC_COMMITTEE,
+                validatorIndex: signature.validatorIndex,
+              });
             }
 
             // Verify signature only, all other data is very likely to be correct, since the `signature` object is created by this node.

package/src/api/impl/beacon/state/index.ts

@@ -17,6 +17,7 @@ import {
   isStatePostGloas,
 } from "@lodestar/state-transition";
 import {
+  BuilderIndex,
   ValidatorIndex,
   getBuilderStatus,
   getValidatorStatus,
@@ -29,6 +30,7 @@ import {ApiModules} from "../../types.js";
 import {assertUniqueItems} from "../../utils.js";
 import {
   filterStateValidatorsByStatus,
+  getBuilderIndexByPubkey,
   getStateBuilderIndex,
   getStateResponseWithRegen,
   getStateValidatorIndex,
@@ -170,8 +172,15 @@ export function getBeaconStateApi({
       if (builderIds.length) {
         assertUniqueItems(builderIds, "Duplicate builder IDs provided");
 
+        // There is no builder pubkey cache, scan the registry at most once per request
+        let builderIndexByPubkey: Map<string, BuilderIndex> | null = null;
+        const getBuilderIndexByPubkeyMap = () => {
+          builderIndexByPubkey ??= getBuilderIndexByPubkey(state);
+          return builderIndexByPubkey;
+        };
+
         for (const id of builderIds) {
-          const resp = getStateBuilderIndex(id, state);
+          const resp = getStateBuilderIndex(id, state, getBuilderIndexByPubkeyMap);
           if (resp.valid) {
             const builderIndex = resp.builderIndex;
             const builder = state.getBuilder(builderIndex);

package/src/api/impl/beacon/state/utils.ts

@@ -1,6 +1,6 @@
-import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
 import {routes} from "@lodestar/api";
 import {CheckpointWithHex, IForkChoice} from "@lodestar/fork-choice";
+import {type PubkeyCache} from "@lodestar/native/pubkeys";
 import {GENESIS_SLOT} from "@lodestar/params";
 import {IBeaconStateView, IBeaconStateViewGloas} from "@lodestar/state-transition";
 import {
@@ -13,7 +13,7 @@ import {
   getValidatorStatus,
   phase0,
 } from "@lodestar/types";
-import {byteArrayEquals, fromHex} from "@lodestar/utils";
+import {fromHex, toHex} from "@lodestar/utils";
 import {IBeaconChain} from "../../../../chain/index.js";
 import {GENESIS_EPOCH} from "../../../../constants/index.js";
 import {IBeaconSync} from "../../../../sync/index.js";
@@ -118,9 +118,22 @@ type StateBuilderIndexResponse =
   | {valid: true; builderIndex: BuilderIndex}
   | {valid: false; code: number; reason: string};
 
+export function getBuilderIndexByPubkey(state: IBeaconStateViewGloas): Map<string, BuilderIndex> {
+  const builderIndexByPubkey = new Map<string, BuilderIndex>();
+  const buildersLength = state.getBuildersLength();
+  for (let builderIndex = 0; builderIndex < buildersLength; builderIndex++) {
+    const pubkeyHex = toHex(state.getBuilder(builderIndex).pubkey);
+    if (!builderIndexByPubkey.has(pubkeyHex)) {
+      builderIndexByPubkey.set(pubkeyHex, builderIndex);
+    }
+  }
+  return builderIndexByPubkey;
+}
+
 export function getStateBuilderIndex(
   id: routes.beacon.BuilderId | BLSPubkey,
-  state: IBeaconStateViewGloas
+  state: IBeaconStateViewGloas,
+  getBuilderIndexByPubkeyMap: () => Map<string, BuilderIndex>
 ): StateBuilderIndexResponse {
   if (typeof id === "string") {
     // mutate `id` and fallthrough to below
@@ -148,14 +161,11 @@ export function getStateBuilderIndex(
   }
 
   // typeof id === Uint8Array
-  // There is no builder pubkey cache, linear scan over the registry
-  const buildersLength = state.getBuildersLength();
-  for (let builderIndex = 0; builderIndex < buildersLength; builderIndex++) {
-    if (byteArrayEquals(state.getBuilder(builderIndex).pubkey, id)) {
-      return {valid: true, builderIndex};
-    }
+  const builderIndex = getBuilderIndexByPubkeyMap().get(toHex(id));
+  if (builderIndex === undefined) {
+    return {valid: false, code: 404, reason: "Builder pubkey not found in state"};
   }
-  return {valid: false, code: 404, reason: "Builder pubkey not found in state"};
+  return {valid: true, builderIndex};
 }
 
 export function filterStateValidatorsByStatus(

package/src/api/impl/debug/index.ts

@@ -114,6 +114,10 @@ export function getDebugApi({
                 unrealizedJustifiedRoot: node.unrealizedJustifiedRoot,
                 unrealizedFinalizedEpoch: node.unrealizedFinalizedEpoch,
                 unrealizedFinalizedRoot: node.unrealizedFinalizedRoot,
+                inclusionListSatisfied:
+                  node.payloadStatus === PayloadStatus.FULL
+                    ? forkChoice.getPayloadInclusionListSatisfaction(node.blockRoot)
+                    : null,
               },
             };
           }),