@lodestar/beacon-node 1.47.0-dev.0bcaaaebb5 → 1.47.0-dev.157b2b2a96

package/src/api/impl/beacon/blocks/index.ts

@@ -220,12 +220,6 @@ export function getBeaconBlockApi({
                   // Block has already been seen, e.g. via gossip racing the publish API. Benign.
                   chain.logger.debug("Ignoring already-known block during publishing", valLogMeta);
                   return;
-                case BlockErrorCode.REPEAT_PROPOSAL:
-                  // The proposer already produced a block for this slot. For a solo setup this is a
-                  // notable signal (duplicate-proposal attempt). For fallback / DVT setups it is
-                  // expected on every block where another node published first.
-                  chain.logger.warn("Ignoring repeat-proposal block during publishing", valLogMeta);
-                  return;
               }
             }
 
@@ -407,7 +401,7 @@ export function getBeaconBlockApi({
       for (let i = 0; i < dataColumnSidecars.length; i++) {
         // + 1 because we publish to beacon_block first
         const {sentPeers, alreadyPublished} = sentPeersArr[i + 1] as {sentPeers: number; alreadyPublished: boolean};
-        // sent peers could be 0 as we set `allowPublishToZeroTopicPeers=true` in network.publishDataColumnSidecar()
+        // sent peers could be 0, data_column_sidecar opts out of the zero peers publish error, see gossipTopicAllowPublishToZeroPeers
         metrics?.dataColumns.sentPeersPerSubnet.observe(sentPeers);
         // A duplicate publish (alreadyPublished=true) means the column is already propagating on the network —
         // expected in self-build flows where peers gossip columns back to us before we publish the envelope.
@@ -1032,6 +1026,7 @@ export function getBeaconBlockApi({
     },
 
     async publishExecutionPayloadBid({signedExecutionPayloadBid}) {
+      const seenTimestampSec = Date.now() / 1000;
       const bid = signedExecutionPayloadBid.message;
       const slot = bid.slot;
       const fork = config.getForkName(slot);
@@ -1042,6 +1037,9 @@ export function getBeaconBlockApi({
 
       await validateApiExecutionPayloadBid(chain, signedExecutionPayloadBid);
 
+      const elapsedSec = chain.clock.secFromSlot(slot, seenTimestampSec);
+      metrics?.gossipExecutionPayloadBid.elapsedTimeTillReceived.observe({source: OpSource.api}, elapsedSec);
+
       try {
         const insertOutcome = chain.executionPayloadBidPool.add(signedExecutionPayloadBid);
         metrics?.opPool.executionPayloadBidPool.apiInsertOutcome.inc({insertOutcome});

package/src/api/impl/beacon/pool/index.ts

@@ -25,6 +25,7 @@ import {validateApiPayloadAttestationMessage} from "../../../../chain/validation
 import {validateApiProposerSlashing} from "../../../../chain/validation/proposerSlashing.js";
 import {validateApiSyncCommittee} from "../../../../chain/validation/syncCommittee.js";
 import {validateApiVoluntaryExit} from "../../../../chain/validation/voluntaryExit.js";
+import {OpSource} from "../../../../chain/validatorMonitor.js";
 import {validateGossipFnRetryUnknownRoot} from "../../../../network/processor/gossipHandlers.js";
 import {ApiError, FailureList, IndexedError} from "../../errors.js";
 import {ApiModules} from "../../types.js";
@@ -99,7 +100,9 @@ export function getBeaconPoolApi({
             const {indexedAttestation, subnet, attDataRootHex, committeeIndex, validatorCommitteeIndex, committeeSize} =
               await validateGossipFnRetryUnknownRoot(validateFn, network, chain, slot, beaconBlockRoot);
 
-            if (network.shouldAggregate(subnet, slot)) {
+            try {
+              // `is_aggregator` only covers aggregating what we receive on the subnet via gossip, not what
+              // a validator client hands us directly, see https://github.com/ChainSafe/lodestar/issues/7548
               const insertOutcome = chain.attestationPool.add(
                 committeeIndex,
                 attestation,
@@ -109,6 +112,10 @@ export function getBeaconPoolApi({
                 priority
               );
               metrics?.opPool.attestationPool.apiInsertOutcome.inc({insertOutcome});
+            } catch (e) {
+              // The pool is a local optimization, failing to insert must not stop us from publishing a
+              // validated attestation, which the route requires us to do. Same handling as the gossip path
+              logger.debug("Error adding unaggregated attestation to pool", {subnet}, e as Error);
             }
 
             if (isForkPostElectra(fork)) {
@@ -211,6 +218,7 @@ export function getBeaconPoolApi({
     },
 
     async submitPayloadAttestationMessages({payloadAttestationMessages}) {
+      const seenTimestampSec = Date.now() / 1000;
       const failures: FailureList = [];
 
       await Promise.all(
@@ -226,6 +234,9 @@ export function getBeaconPoolApi({
               beaconBlockRoot
             );
 
+            const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
+            metrics?.gossipPayloadAttestationMessage.elapsedTimeTillReceived.observe({source: OpSource.api}, delaySec);
+
             const insertOutcome = chain.payloadAttestationPool.add(
               payloadAttestationMessage,
               attDataRootHex,

package/src/api/impl/beacon/state/utils.ts

@@ -1,7 +1,8 @@
+import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
 import {routes} from "@lodestar/api";
 import {CheckpointWithHex, IForkChoice} from "@lodestar/fork-choice";
 import {GENESIS_SLOT} from "@lodestar/params";
-import {IBeaconStateView, IBeaconStateViewGloas, PubkeyCache} from "@lodestar/state-transition";
+import {IBeaconStateView, IBeaconStateViewGloas} from "@lodestar/state-transition";
 import {
   BLSPubkey,
   BuilderIndex,

package/src/api/impl/events/index.ts

@@ -4,7 +4,8 @@ import {ApiModules} from "../types.js";
 
 export function getEventsApi({
   chain,
-}: Pick<ApiModules, "chain" | "config">): ApplicationMethods<routes.events.Endpoints> {
+  logger,
+}: Pick<ApiModules, "chain" | "config" | "logger">): ApplicationMethods<routes.events.Endpoints> {
   return {
     async eventstream({topics, signal, onEvent}) {
       const onAbortFns: (() => void)[] = [];
@@ -12,9 +13,16 @@ export function getEventsApi({
       for (const topic of topics) {
         // biome-ignore lint/suspicious/noExplicitAny: We need to use `any` type here
         const handler = (data: any): void => {
-          // TODO: What happens if this handler throws? Does it break the other chain.emitter listeners?
-
-          onEvent({type: topic, message: data});
+          try {
+            // TS only relates `{type, message}` to the `BeaconEvent` discriminated union while
+            // `EventType` stays within its ~25-member cap; without the cast, adding events trips TS2345.
+            onEvent({type: topic, message: data} as routes.events.BeaconEvent);
+          } catch (e) {
+            // `chain.emitter` emits synchronously, a throwing listener would propagate the error to
+            // whoever emitted the event, e.g. a gossip handler, and skip the remaining listeners.
+            // A single misbehaving event or client must not affect the node or other subscribers.
+            logger.warn("Error sending event to client", {topic}, e as Error);
+          }
         };
 
         chain.emitter.on(topic, handler);

package/src/api/impl/validator/index.ts

@@ -895,7 +895,7 @@ export function getValidatorApi(
       // TODO GLOAS: add external builder api support when it is implemented
       const isBuildingOnFull = chain.forkChoice.shouldBuildOnFull(parentBlock, slot);
       const bidParentBlockHash = isBuildingOnFull ? parentBlock.executionPayloadBlockHash : parentBlock.parentBlockHash;
-      const circuitBreakerActive = chain.builderCircuitBreaker.isActive(slot);
+      const circuitBreakerActive = chain.builderCircuitBreaker.isActive(slot, parentBlock);
       // Keep a builder bid as fallback unless the circuit breaker is active
       const builderBid = circuitBreakerActive
         ? null

package/src/chain/archiveStore/historicalState/metrics.ts

@@ -1,3 +1,4 @@
+import {ForkName} from "@lodestar/params";
 import {
   BeaconStateTransitionMetrics,
   EpochTransitionStep,
@@ -45,6 +46,27 @@ export function createHistoricalStateTransitionMetrics(
       labelNames: ["step"],
       buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1],
     }),
+    forkUpgradeTime: metricsRegister.histogram<{fork: ForkName}>({
+      name: "lodestar_historical_state_stfn_fork_upgrade_seconds",
+      help: "Time to upgrade the state at a fork boundary in seconds",
+      labelNames: ["fork"],
+      buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30],
+    }),
+    onboardBuildersTime: metricsRegister.histogram({
+      name: "lodestar_historical_state_stfn_gloas_onboard_builders_seconds",
+      help: "Time spent in onboardBuildersFromPendingDeposits at the gloas fork transition",
+      buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30],
+    }),
+    onboardBuildersDeposits: metricsRegister.gauge<{outcome: "onboarded" | "topup" | "kept" | "dropped"}>({
+      name: "lodestar_historical_state_stfn_gloas_onboard_builders_deposits",
+      help: "Pending deposits handled at the gloas fork transition, by outcome",
+      labelNames: ["outcome"],
+    }),
+    onboardBuildersSignatureChecks: metricsRegister.gauge<{source: "cache" | "verified"}>({
+      name: "lodestar_historical_state_stfn_gloas_onboard_builders_signature_checks",
+      help: "Builder deposit signature checks at the gloas fork transition, by whether the pre-verify cache served them",
+      labelNames: ["source"],
+    }),
     processBlockTime: metricsRegister.histogram({
       name: "lodestar_historical_state_stfn_process_block_seconds",
       help: "Time to process a single block in seconds",

package/src/chain/blocks/importBlock.ts

@@ -22,7 +22,6 @@ import {
   RootCache,
   computeEpochAtSlot,
   computeTimeAtSlot,
-  isStartSlotOfEpoch,
   isStatePostAltair,
   isStatePostBellatrix,
 } from "@lodestar/state-transition";
@@ -404,11 +403,7 @@ export async function importBlock(
         notOverrideFcuReason = NotReorgedReason.NotProposerOfNextSlot;
       }
     } catch (e) {
-      if (isStartSlotOfEpoch(proposalSlot)) {
-        notOverrideFcuReason = NotReorgedReason.AtEpochBoundary;
-      } else {
-        this.logger.warn("Unable to get beacon proposer. Do not override fcu.", {proposalSlot}, e as Error);
-      }
+      this.logger.warn("Unable to get beacon proposer. Do not override fcu.", {proposalSlot}, e as Error);
     }
 
     if (shouldOverrideFcu) {

package/src/chain/blocks/importExecutionPayload.ts

@@ -58,11 +58,12 @@ export type PayloadErrorType =
     };
 
 export class PayloadError extends Error {
-  type: PayloadErrorType;
-
-  constructor(type: PayloadErrorType, message?: string) {
+  constructor(
+    readonly payloadInput: PayloadEnvelopeInput,
+    readonly type: PayloadErrorType,
+    message?: string
+  ) {
     super(message ?? type.code);
-    this.type = type;
   }
 }
 
@@ -125,7 +126,7 @@ export async function importExecutionPayload(
   // 2. Get ProtoBlock for parent root lookup
   const protoBlock = this.forkChoice.getBlockHexDefaultStatus(blockRootHex);
   if (!protoBlock) {
-    throw new PayloadError({
+    throw new PayloadError(payloadInput, {
       code: PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE,
       blockRootHex,
     });
@@ -140,13 +141,13 @@ export async function importExecutionPayload(
     );
 
   if (blockState == null) {
-    throw new PayloadError({
+    throw new PayloadError(payloadInput, {
       code: PayloadErrorCode.MISS_BLOCK_STATE,
       blockRootHex: protoBlock.blockRoot,
     });
   }
   if (!isStatePostGloas(blockState)) {
-    throw new PayloadError({
+    throw new PayloadError(payloadInput, {
       code: PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR,
       message: `Expected gloas+ state for payload import, got fork=${blockState.forkName}`,
     });
@@ -161,6 +162,7 @@ export async function importExecutionPayload(
     });
   } catch (e) {
     throw new PayloadError(
+      payloadInput,
       {
         code: PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR,
         message: (e as Error).message,
@@ -184,7 +186,6 @@ export async function importExecutionPayload(
       : verifyExecutionPayloadEnvelopeSignature(
           this.config,
           blockState,
-          this.pubkeyCache,
           signedEnvelope,
           payloadInput.proposerIndex,
           this.bls
@@ -193,7 +194,7 @@ export async function importExecutionPayload(
 
   // 4b. Check signature verification result
   if (!signatureValid) {
-    throw new PayloadError({code: PayloadErrorCode.INVALID_SIGNATURE});
+    throw new PayloadError(payloadInput, {code: PayloadErrorCode.INVALID_SIGNATURE});
   }
 
   // 4c. Handle EL response
@@ -202,7 +203,7 @@ export async function importExecutionPayload(
       break;
 
     case ExecutionPayloadStatus.INVALID:
-      throw new PayloadError({
+      throw new PayloadError(payloadInput, {
         code: PayloadErrorCode.EXECUTION_ENGINE_INVALID,
         execStatus: execResult.status,
         errorMessage: execResult.validationError ?? "",
@@ -215,7 +216,7 @@ export async function importExecutionPayload(
     case ExecutionPayloadStatus.INVALID_BLOCK_HASH:
     case ExecutionPayloadStatus.ELERROR:
     case ExecutionPayloadStatus.UNAVAILABLE:
-      throw new PayloadError({
+      throw new PayloadError(payloadInput, {
         code: PayloadErrorCode.EXECUTION_ENGINE_ERROR,
         execStatus: execResult.status,
         errorMessage: execResult.validationError ?? "",

package/src/chain/blocks/index.ts

@@ -8,7 +8,7 @@ import {BlockError, BlockErrorCode, isBlockErrorAborted} from "../errors/index.j
 import {BlockProcessOpts} from "../options.js";
 import {IBlockInput} from "./blockInput/types.js";
 import {importBlock} from "./importBlock.js";
-import {importExecutionPayload} from "./importExecutionPayload.js";
+import {PayloadError, importExecutionPayload} from "./importExecutionPayload.js";
 import {PayloadEnvelopeInput} from "./payloadEnvelopeInput/payloadEnvelopeInput.js";
 import {FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
 import {assertLinearChainSegment} from "./utils/chainSegment.js";
@@ -164,13 +164,21 @@ export async function processBlocks(
       return; // Ignore
     }
 
-    // above functions should only throw BlockError
-    const err = getBlockError(e, blocks[0].getBlock());
+    // above functions should only throw BlockError, or PayloadError from the gloas payload import
+    const err = getBlockOrPayloadError(e, blocks[0].getBlock());
 
     // TODO: De-duplicate with logic above
     // ChainEvent.errorBlock
-    if (!(err instanceof BlockError)) {
-      this.logger.debug("Non BlockError received", {}, err);
+    if (!(err instanceof BlockError) && !(err instanceof PayloadError)) {
+      this.logger.debug("Neither BlockError nor PayloadError received", {}, err);
+    } else if (err instanceof PayloadError) {
+      if (!opts.disableOnBlockError) {
+        this.logger.debug(
+          "Payload error",
+          {slot: err.payloadInput.slot, blockRoot: err.payloadInput.blockRootHex},
+          err
+        );
+      }
     } else if (!opts.disableOnBlockError) {
       this.logger.debug("Block error", {slot: err.signedBlock.message.slot}, err);
 
@@ -201,11 +209,15 @@ export async function processBlocks(
   }
 }
 
-function getBlockError(e: unknown, block: SignedBeaconBlock): BlockError {
+function getBlockOrPayloadError(e: unknown, block: SignedBeaconBlock): BlockError | PayloadError {
   if (e instanceof BlockError) {
     return e;
   }
 
+  if (e instanceof PayloadError) {
+    return e;
+  }
+
   if (e instanceof Error) {
     const blockError = new BlockError(block, {code: BlockErrorCode.BEACON_CHAIN_ERROR, error: e});
     blockError.stack = e.stack;

package/src/chain/blocks/verifyBlocksStateTransitionOnly.ts

@@ -44,22 +44,31 @@ export async function verifyBlocksStateTransitionOnly(
     // STFN - per_slot_processing() + per_block_processing()
     // NOTE: `regen.getPreState()` should have dialed forward the state already caching checkpoint states
     const useBlsBatchVerify = !opts?.disableBlsBatchVerify;
-    const postState = preState.stateTransition(
-      block,
-      {
-        // NOTE: Assume valid for now while sending payload to execution engine in parallel
-        // Latter verifyBlocksInEpoch() will make sure that payload is indeed valid
-        executionPayloadStatus: ExecutionPayloadStatus.valid,
-        dataAvailabilityStatus,
-        // false because it's verified below with better error typing
-        verifyStateRoot: false,
-        // if block is trusted don't verify proposer or op signature
-        verifyProposer: !useBlsBatchVerify && !validSignatures && !validProposerSignature,
-        verifySignatures: !useBlsBatchVerify && !validSignatures,
-        dontTransferCache: false,
-      },
-      {metrics, validatorMonitor}
-    );
+    let postState: IBeaconStateView;
+    try {
+      postState = preState.stateTransition(
+        block,
+        {
+          // NOTE: Assume valid for now while sending payload to execution engine in parallel
+          // Latter verifyBlocksInEpoch() will make sure that payload is indeed valid
+          executionPayloadStatus: ExecutionPayloadStatus.valid,
+          dataAvailabilityStatus,
+          // false because it's verified below with better error typing
+          verifyStateRoot: false,
+          // if block is trusted don't verify proposer or op signature
+          verifyProposer: !useBlsBatchVerify && !validSignatures && !validProposerSignature,
+          verifySignatures: !useBlsBatchVerify && !validSignatures,
+          dontTransferCache: false,
+        },
+        {metrics, validatorMonitor}
+      );
+    } catch (e) {
+      throw new BlockError(block, {
+        code: BlockErrorCode.PER_BLOCK_PROCESSING_ERROR,
+        blockRoot: blocks[i].blockRootHex,
+        error: e as Error,
+      });
+    }
 
     const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({
       source: StateHashTreeRootSource.blockTransition,

package/src/chain/blocks/verifyExecutionPayloadEnvelope.ts

@@ -1,7 +1,6 @@
 import {BeaconConfig} from "@lodestar/config";
 import {
   type IBeaconStateViewGloas,
-  type PubkeyCache,
   computeTimeAtSlot,
   getExecutionPayloadEnvelopeSignatureSet,
 } from "@lodestar/state-transition";
@@ -119,17 +118,10 @@ export function verifyExecutionPayloadEnvelope(
 export async function verifyExecutionPayloadEnvelopeSignature(
   config: BeaconConfig,
   state: IBeaconStateViewGloas,
-  pubkeyCache: PubkeyCache,
   signedEnvelope: gloas.SignedExecutionPayloadEnvelope,
   proposerIndex: number,
   bls: IBlsVerifier
 ): Promise<boolean> {
-  const signatureSet = getExecutionPayloadEnvelopeSignatureSet(
-    config,
-    pubkeyCache,
-    state,
-    signedEnvelope,
-    proposerIndex
-  );
+  const signatureSet = getExecutionPayloadEnvelopeSignatureSet(config, state, signedEnvelope, proposerIndex);
   return bls.verifySignatureSets([signatureSet]);
 }

package/src/chain/bls/index.ts

@@ -1,4 +1,4 @@
-export type {IBlsVerifier} from "./interface.js";
+export type {IBlsVerifier, SameMessageSignatureSet} from "./interface.js";
 export type {BlsMultiThreadWorkerPoolModules, JobQueueItemType} from "./multithread/index.js";
 export {BlsMultiThreadWorkerPool} from "./multithread/index.js";
 export {BlsSingleThreadVerifier} from "./singleThread.js";

package/src/chain/bls/interface.ts

@@ -1,6 +1,10 @@
-import {PublicKey} from "@chainsafe/blst";
 import {ISignatureSet} from "@lodestar/state-transition";
 
+export type SameMessageSignatureSet = {
+  index: number;
+  signature: Uint8Array;
+};
+
 export type VerifySignatureOpts = {
   /**
    * A batchable set MAY be verified with more sets to reduce the verification costs.
@@ -53,8 +57,8 @@ export interface IBlsVerifier {
    *   - only support `batchable` option
    */
   verifySignatureSetsSameMessage(
-    sets: {publicKey: PublicKey; signature: Uint8Array}[],
-    messsage: Uint8Array,
+    sets: SameMessageSignatureSet[],
+    message: Uint8Array,
     opts?: Omit<VerifySignatureOpts, "verifyOnMainThread">
   ): Promise<boolean[]>;