@lodestar/beacon-node 1.51.0-dev.a1662c7de7 → 1.51.0-dev.a6892012c4

package/src/api/impl/beacon/blocks/index.ts

@@ -62,7 +62,7 @@ import {
 } from "../../../../chain/produceBlock/index.js";
 import {RegenCaller} from "../../../../chain/regen/index.js";
 import {validateGossipBlock, verifyBlockProposerSignature} from "../../../../chain/validation/block.js";
-import {validateApiExecutionPayloadBid} from "../../../../chain/validation/executionPayloadBid.js";
+import {validateGossipExecutionPayloadBid} from "../../../../chain/validation/executionPayloadBid.js";
 import {validateApiExecutionPayloadEnvelope} from "../../../../chain/validation/executionPayloadEnvelope.js";
 import {OpSource} from "../../../../chain/validatorMonitor.js";
 import {
@@ -79,6 +79,7 @@ import {
 import {isOptimisticBlock} from "../../../../util/forkChoice.js";
 import {kzg} from "../../../../util/kzg.js";
 import {promiseAllMaybeAsync} from "../../../../util/promises.js";
+import {ApiOptions} from "../../../options.js";
 import {ApiModules} from "../../types.js";
 import {assertUniqueItems} from "../../utils.js";
 import {getBlockResponse, toBeaconHeaderResponse} from "./utils.js";
@@ -96,16 +97,10 @@ const MAX_API_CLOCK_DISPARITY_MS = 1000;
  */
 const IDENTITY_PEER_ID = ""; // TODO: Compute identity keypair
 
-export function getBeaconBlockApi({
-  chain,
-  config,
-  metrics,
-  network,
-  db,
-}: Pick<
-  ApiModules,
-  "chain" | "config" | "metrics" | "network" | "db"
->): ApplicationMethods<routes.beacon.block.Endpoints> {
+export function getBeaconBlockApi(
+  opts: ApiOptions,
+  {chain, config, metrics, network, db}: Pick<ApiModules, "chain" | "config" | "metrics" | "network" | "db">
+): ApplicationMethods<routes.beacon.block.Endpoints> {
   const publishBlockV2: ApplicationMethods<routes.beacon.block.Endpoints>["publishBlockV2"] = async (
     {signedBlockContents, broadcastValidation, builderUrl},
     context,
@@ -1077,15 +1072,25 @@ export function getBeaconBlockApi({
         throw new ApiError(400, `publishExecutionPayloadBid not supported for pre-gloas fork=${fork}`);
       }
 
-      const validationTimer = metrics?.opPool.executionPayloadBidPool.apiValidationTime.startTimer();
-      // TODO: once the builder is tested and can be trusted, skip this local validation
-      // to publish faster, accepting peer-score risk if it ever emits an invalid bid.
-      await validateApiExecutionPayloadBid(chain, signedExecutionPayloadBid);
-      validationTimer?.();
-
       const elapsedSec = chain.clock.secFromSlot(slot, seenTimestampSec);
       metrics?.gossipExecutionPayloadBid.elapsedTimeTillReceived.observe({source: OpSource.api}, elapsedSec);
 
+      if (!opts.disableBidValidation) {
+        const validationTimer = metrics?.opPool.executionPayloadBidPool.apiValidationTime.startTimer();
+        const {proposerIndex} = await validateGossipExecutionPayloadBid(chain, signedExecutionPayloadBid);
+        validationTimer?.();
+
+        // Gossip does not deliver our own messages back to us, so a validated bid is added to the
+        // pool here to allow a proposer on this node to select it
+        const insertOutcome = chain.executionPayloadBidPool.add(
+          signedExecutionPayloadBid,
+          Math.floor(elapsedSec * 1000)
+        );
+        metrics?.opPool.executionPayloadBidPool.apiInsertOutcome.inc({insertOutcome});
+
+        chain.validatorMonitor?.registerExecutionPayloadBid(OpSource.api, proposerIndex, bid);
+      }
+
       const sentPeers = await network.publishSignedExecutionPayloadBid(signedExecutionPayloadBid);
 
       chain.emitter.emit(routes.events.EventType.executionPayloadBid, {

package/src/api/impl/beacon/index.ts

@@ -3,6 +3,7 @@ import {ApplicationMethods} from "@lodestar/api/server";
 import {isForkPostGloas} from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {toRootHex} from "@lodestar/utils";
+import {ApiOptions} from "../../options.js";
 import {ApiError} from "../errors.js";
 import {ApiModules} from "../types.js";
 import {getBeaconBlockApi} from "./blocks/index.js";
@@ -11,9 +12,10 @@ import {getBeaconRewardsApi} from "./rewards/index.js";
 import {getBeaconStateApi} from "./state/index.js";
 
 export function getBeaconApi(
+  opts: ApiOptions,
   modules: Pick<ApiModules, "chain" | "config" | "logger" | "metrics" | "network" | "db" | "sync">
 ): ApplicationMethods<routes.beacon.Endpoints> {
-  const block = getBeaconBlockApi(modules);
+  const block = getBeaconBlockApi(opts, modules);
   const pool = getBeaconPoolApi(modules);
   const state = getBeaconStateApi(modules);
   const rewards = getBeaconRewardsApi(modules);

package/src/api/impl/beacon/state/index.ts

@@ -17,6 +17,7 @@ import {
   isStatePostGloas,
 } from "@lodestar/state-transition";
 import {
+  BuilderIndex,
   ValidatorIndex,
   getBuilderStatus,
   getValidatorStatus,
@@ -29,6 +30,7 @@ import {ApiModules} from "../../types.js";
 import {assertUniqueItems} from "../../utils.js";
 import {
   filterStateValidatorsByStatus,
+  getBuilderIndexByPubkey,
   getStateBuilderIndex,
   getStateResponseWithRegen,
   getStateValidatorIndex,
@@ -170,8 +172,15 @@ export function getBeaconStateApi({
       if (builderIds.length) {
         assertUniqueItems(builderIds, "Duplicate builder IDs provided");
 
+        // There is no builder pubkey cache, scan the registry at most once per request
+        let builderIndexByPubkey: Map<string, BuilderIndex> | null = null;
+        const getBuilderIndexByPubkeyMap = () => {
+          builderIndexByPubkey ??= getBuilderIndexByPubkey(state);
+          return builderIndexByPubkey;
+        };
+
         for (const id of builderIds) {
-          const resp = getStateBuilderIndex(id, state);
+          const resp = getStateBuilderIndex(id, state, getBuilderIndexByPubkeyMap);
           if (resp.valid) {
             const builderIndex = resp.builderIndex;
             const builder = state.getBuilder(builderIndex);

package/src/api/impl/beacon/state/utils.ts

@@ -13,7 +13,7 @@ import {
   getValidatorStatus,
   phase0,
 } from "@lodestar/types";
-import {byteArrayEquals, fromHex} from "@lodestar/utils";
+import {fromHex, toHex} from "@lodestar/utils";
 import {IBeaconChain} from "../../../../chain/index.js";
 import {GENESIS_EPOCH} from "../../../../constants/index.js";
 import {IBeaconSync} from "../../../../sync/index.js";
@@ -118,9 +118,22 @@ type StateBuilderIndexResponse =
   | {valid: true; builderIndex: BuilderIndex}
   | {valid: false; code: number; reason: string};
 
+export function getBuilderIndexByPubkey(state: IBeaconStateViewGloas): Map<string, BuilderIndex> {
+  const builderIndexByPubkey = new Map<string, BuilderIndex>();
+  const buildersLength = state.getBuildersLength();
+  for (let builderIndex = 0; builderIndex < buildersLength; builderIndex++) {
+    const pubkeyHex = toHex(state.getBuilder(builderIndex).pubkey);
+    if (!builderIndexByPubkey.has(pubkeyHex)) {
+      builderIndexByPubkey.set(pubkeyHex, builderIndex);
+    }
+  }
+  return builderIndexByPubkey;
+}
+
 export function getStateBuilderIndex(
   id: routes.beacon.BuilderId | BLSPubkey,
-  state: IBeaconStateViewGloas
+  state: IBeaconStateViewGloas,
+  getBuilderIndexByPubkeyMap: () => Map<string, BuilderIndex>
 ): StateBuilderIndexResponse {
   if (typeof id === "string") {
     // mutate `id` and fallthrough to below
@@ -148,14 +161,11 @@ export function getStateBuilderIndex(
   }
 
   // typeof id === Uint8Array
-  // There is no builder pubkey cache, linear scan over the registry
-  const buildersLength = state.getBuildersLength();
-  for (let builderIndex = 0; builderIndex < buildersLength; builderIndex++) {
-    if (byteArrayEquals(state.getBuilder(builderIndex).pubkey, id)) {
-      return {valid: true, builderIndex};
-    }
+  const builderIndex = getBuilderIndexByPubkeyMap().get(toHex(id));
+  if (builderIndex === undefined) {
+    return {valid: false, code: 404, reason: "Builder pubkey not found in state"};
   }
-  return {valid: false, code: 404, reason: "Builder pubkey not found in state"};
+  return {valid: true, builderIndex};
 }
 
 export function filterStateValidatorsByStatus(

package/src/api/impl/validator/index.ts

@@ -254,13 +254,13 @@ export function getValidatorApi(
         genesisBlockRoot = state.computeAnchorCheckpoint().checkpoint.root;
       } else if (state.slot < SLOTS_PER_HISTORICAL_ROOT) {
         genesisBlockRoot = state.getBlockRootAtSlot(GENESIS_SLOT);
-      }
-
-      const blockRes = await chain.getCanonicalBlockAtSlot(GENESIS_SLOT);
-      if (blockRes) {
-        genesisBlockRoot = config
-          .getForkTypes(blockRes.block.message.slot)
-          .SignedBeaconBlock.hashTreeRoot(blockRes.block);
+      } else {
+        const blockRes = await chain.getCanonicalBlockAtSlot(GENESIS_SLOT);
+        if (blockRes) {
+          genesisBlockRoot = config
+            .getForkTypes(blockRes.block.message.slot)
+            .BeaconBlock.hashTreeRoot(blockRes.block.message);
+        }
       }
     }
 
@@ -1756,7 +1756,10 @@ export function getValidatorApi(
         }
       }
 
-      const dependentRoot = fromHex(state.getShufflingDecisionRoot(epoch));
+      // The fork epoch PTC is computed at the fork from effective balances updated at the end of the
+      // last pre-fork epoch, so its dependent root is one epoch later than for other epochs
+      const dependentRootEpoch = epoch === config.GLOAS_FORK_EPOCH ? epoch + 1 : epoch;
+      const dependentRoot = fromHex(state.getShufflingDecisionRoot(dependentRootEpoch));
 
       return {
         data: duties,

package/src/api/options.ts

@@ -2,6 +2,8 @@ import {BeaconRestApiServerOpts, beaconRestApiServerOpts} from "./rest/index.js"
 
 export type ApiOptions = {
   maxGindicesInProof?: number;
+  /** Broadcast execution payload bids submitted via the API without running gossip validation */
+  disableBidValidation?: boolean;
   rest: BeaconRestApiServerOpts;
   commit?: string;
   version?: string;
@@ -10,6 +12,7 @@ export type ApiOptions = {
 
 export const defaultApiOptions: ApiOptions = {
   maxGindicesInProof: 512,
+  disableBidValidation: false,
   rest: beaconRestApiServerOpts,
   version: "dev",
   private: false,

package/src/chain/GetBlobsTracker.ts

@@ -1,10 +1,9 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
-import {Logger} from "@lodestar/utils";
+import {Logger, pruneSetToMax} from "@lodestar/utils";
 import {BLOB_AND_PROOF_V2_RPC_BYTES} from "../execution/engine/types.js";
 import {IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/metrics.js";
-import {callInNextEventLoop} from "../util/eventLoop.js";
 import {
   DataColumnEngineResult,
   getBlobSidecarsFromExecution,
@@ -14,6 +13,11 @@ import {IBlockInput, isBlockInputBlobs} from "./blocks/blockInput/index.js";
 import {PayloadEnvelopeInput} from "./blocks/payloadEnvelopeInput/index.js";
 import {ChainEventEmitter} from "./emitter.js";
 
+/** A `null` answer only changes if the EL receives the transactions later, so retries are few and spaced out */
+export const MAX_GET_BLOBS_ATTEMPTS = 3;
+export const GET_BLOBS_RETRY_INTERVAL_MS = 1000;
+const MAX_TRACKED_BLOCK_ROOTS = 64;
+
 export type GetBlobsTrackerInit = {
   logger: Logger;
   executionEngine: IExecutionEngine;
@@ -32,6 +36,8 @@ export class GetBlobsTracker {
   metrics: Metrics | null;
   config: ChainForkConfig;
   activeReconstructions = new Set<string>();
+  failedAttempts = new Map<string, {count: number; lastAttemptMs: number; retryTimer?: NodeJS.Timeout}>();
+  pendingTriggers = new Set<string>();
   // Preallocate buffers for getBlobsV2 RPC calls
   // See https://github.com/ChainSafe/lodestar/pull/8282 for context
   blobsAndProofsBuffers: {buffers: Uint8Array[]; inUse: boolean}[] = [];
@@ -46,23 +52,55 @@ export class GetBlobsTracker {
 
   triggerGetBlobs(input: IBlockInput | PayloadEnvelopeInput): void {
     if (this.activeReconstructions.has(input.blockRootHex)) {
+      this.pendingTriggers.add(input.blockRootHex);
       return;
     }
 
+    const failed = this.failedAttempts.get(input.blockRootHex);
+    if (failed) {
+      if (failed.count >= MAX_GET_BLOBS_ATTEMPTS) {
+        return;
+      }
+      const waitMs = GET_BLOBS_RETRY_INTERVAL_MS - (Date.now() - failed.lastAttemptMs);
+      if (waitMs > 0) {
+        // Columns tend to arrive in one burst, so keep a single trailing retry instead of relying on a later trigger
+        failed.retryTimer ??= setTimeout(() => {
+          failed.retryTimer = undefined;
+          if (this.failedAttempts.get(input.blockRootHex) === failed && !input.hasAllData()) {
+            this.triggerGetBlobs(input);
+          }
+        }, waitMs).unref();
+        return;
+      }
+    }
+
+    // The request is sent right away, before block processing issues newPayload: ELs that drop a
+    // transaction's blobs once the payload is validated can only answer a request that gets there first
     if (!(input instanceof PayloadEnvelopeInput) && isBlockInputBlobs(input)) {
       // there is not preallocation for blob sidecars like there is for columns sidecars so no need to
       // store the index for the preallocated buffers
       this.activeReconstructions.add(input.blockRootHex);
-      callInNextEventLoop(() => {
-        const logCtx = {slot: input.slot, root: input.blockRootHex};
-        this.logger.verbose("Trigger getBlobsV1 for block", logCtx);
-        getBlobSidecarsFromExecution(this.config, this.executionEngine, this.metrics, this.emitter, input).finally(
-          () => {
-            this.logger.verbose("Completed getBlobsV1 for block", logCtx);
-            this.activeReconstructions.delete(input.blockRootHex);
+      const logCtx = {slot: input.slot, root: input.blockRootHex};
+      this.logger.verbose("Trigger getBlobsV1 for block", logCtx);
+      let requestFailed = false;
+      getBlobSidecarsFromExecution(this.config, this.executionEngine, this.metrics, this.emitter, input)
+        .then(() => {
+          // EL errors and misses are handled inside the helper, blobs still missing means the attempt failed
+          if (!input.hasAllData()) {
+            requestFailed = true;
+            this.recordFailedAttempt(input.blockRootHex);
           }
-        );
-      });
+        })
+        .catch((error) => {
+          this.logger.debug("Error during getBlobsV1 for block", logCtx, error as Error);
+          requestFailed = true;
+          this.recordFailedAttempt(input.blockRootHex);
+        })
+        .finally(() => {
+          this.logger.verbose("Completed getBlobsV1 for block", logCtx);
+          this.activeReconstructions.delete(input.blockRootHex);
+          this.retryPendingTrigger(input, requestFailed);
+        });
 
       return;
     }
@@ -83,34 +121,53 @@ export class GetBlobsTracker {
     }
     timer?.();
 
-    // We don't care about the outcome of this call,
-    // just that it has been triggered for this block root.
     this.activeReconstructions.add(input.blockRootHex);
     this.blobsAndProofsBuffers[freeIndex].inUse = true;
-    callInNextEventLoop(() => {
-      const logCtx = {slot: input.slot, root: input.blockRootHex};
-      this.logger.verbose("Trigger getBlobsV2 for block", logCtx);
-      getDataColumnSidecarsFromExecution(
-        this.config,
-        this.executionEngine,
-        this.emitter,
-        input,
-        this.metrics,
-        this.blobsAndProofsBuffers[freeIndex].buffers
-      )
-        .then((result) => {
-          this.logger.debug("getBlobsV2 result for block", {...logCtx, result});
-          this.metrics?.dataColumns.dataColumnEngineResult.inc({result});
-        })
-        .catch((error) => {
-          this.logger.debug("Error during getBlobsV2 for block", logCtx, error as Error);
-          this.metrics?.dataColumns.dataColumnEngineResult.inc({result: DataColumnEngineResult.Failed});
-        })
-        .finally(() => {
-          this.logger.verbose("Completed getBlobsV2 for block", logCtx);
-          this.activeReconstructions.delete(input.blockRootHex);
-          this.blobsAndProofsBuffers[freeIndex].inUse = false;
-        });
-    });
+    const logCtx = {slot: input.slot, root: input.blockRootHex};
+    this.logger.verbose("Trigger getBlobsV2 for block", logCtx);
+    let requestFailed = false;
+    getDataColumnSidecarsFromExecution(
+      this.config,
+      this.executionEngine,
+      this.emitter,
+      input,
+      this.metrics,
+      this.blobsAndProofsBuffers[freeIndex].buffers
+    )
+      .then((result) => {
+        this.logger.debug("getBlobsV2 result for block", {...logCtx, result});
+        this.metrics?.dataColumns.dataColumnEngineResult.inc({result});
+        if (result === DataColumnEngineResult.NullResponse) {
+          requestFailed = true;
+          this.recordFailedAttempt(input.blockRootHex);
+        }
+      })
+      .catch((error) => {
+        this.logger.debug("Error during getBlobsV2 for block", logCtx, error as Error);
+        this.metrics?.dataColumns.dataColumnEngineResult.inc({result: DataColumnEngineResult.Failed});
+        requestFailed = true;
+        this.recordFailedAttempt(input.blockRootHex);
+      })
+      .finally(() => {
+        this.logger.verbose("Completed getBlobsV2 for block", logCtx);
+        this.activeReconstructions.delete(input.blockRootHex);
+        this.blobsAndProofsBuffers[freeIndex].inUse = false;
+        this.retryPendingTrigger(input, requestFailed);
+      });
+  }
+
+  private retryPendingTrigger(input: IBlockInput | PayloadEnvelopeInput, requestFailed: boolean): void {
+    // Triggers dropped while the request was in flight re-enter once it failed, the attempt that was just
+    // recorded makes this arm the trailing retry instead of sending another request right away
+    if (this.pendingTriggers.delete(input.blockRootHex) && requestFailed) {
+      this.triggerGetBlobs(input);
+    }
+  }
+
+  private recordFailedAttempt(blockRootHex: string): void {
+    const failed = this.failedAttempts.get(blockRootHex);
+    clearTimeout(failed?.retryTimer);
+    this.failedAttempts.set(blockRootHex, {count: (failed?.count ?? 0) + 1, lastAttemptMs: Date.now()});
+    pruneSetToMax(this.failedAttempts, MAX_TRACKED_BLOCK_ROOTS);
   }
 }