@lodestar/beacon-node 1.47.0-dev.f591cb177e → 1.48.0-dev.67344272c0

package/src/execution/builder/apiClient.ts

@@ -0,0 +1,264 @@
+import {routes} from "@lodestar/api";
+import {ApiClient as BuilderApi, getClient} from "@lodestar/api/builder";
+import {ChainForkConfig} from "@lodestar/config";
+import {Logger} from "@lodestar/logger";
+import {DOMAIN_BUILDER_REQUEST_AUTH, ForkPostGloas} from "@lodestar/params";
+import {
+  ZERO_HASH,
+  computeDomain,
+  computeSigningRoot,
+  createSingleSignatureSetFromComponents,
+} from "@lodestar/state-transition";
+import {BLSPubkey, Root, SignedBeaconBlock, Slot, WithOptionalBytes, gloas, ssz} from "@lodestar/types";
+import {isValidAsciiHttpUrl, toHex, toPrintableUrl} from "@lodestar/utils";
+import type {IBlsVerifier} from "../../chain/bls/index.js";
+import {Metrics} from "../../metrics/metrics.js";
+import {IClock} from "../../util/clock.js";
+
+/**
+ * Additional duration to account for potential event loop lag which causes
+ * builder bids to be rejected even though the response was sent in time.
+ */
+const EVENT_LOOP_LAG_BUFFER = 250;
+
+/**
+ * Duration given to a builder to provide a `SignedExecutionPayloadBid` before the deadline
+ * is reached, advertised on each bid request via the `X-Timeout-Ms` header. The p2p bid is
+ * selected after the same duration, only considering bids received up to that point.
+ */
+export const BUILDER_BID_DEADLINE_MS = 500;
+
+/** Local timeout for bid requests, event loop lag must not discard bids that arrived in time */
+export const BUILDER_BID_REQUEST_TIMEOUT_MS = BUILDER_BID_DEADLINE_MS + EVENT_LOOP_LAG_BUFFER;
+
+export type BuilderApiClientOpts = {
+  /** Timeout for builder api requests, bid requests always use `BUILDER_BID_REQUEST_TIMEOUT_MS` */
+  timeout?: number;
+  // Add User-Agent header to all requests
+  userAgent?: string;
+};
+
+type BuilderUrl = string;
+
+/** Decode the SSZ URL bytes without allowing replacement characters or unsafe header values. */
+export function decodeBuilderUrl(value: Uint8Array): BuilderUrl {
+  let url: string;
+  try {
+    url = new TextDecoder("utf8", {fatal: true}).decode(value);
+  } catch {
+    throw Error("Builder url must be valid UTF-8");
+  }
+  if (!isValidAsciiHttpUrl(url)) {
+    throw Error("Invalid builder url");
+  }
+  return url;
+}
+
+export type BuilderApiBid = {
+  url: BuilderUrl;
+  entry: routes.validator.BuilderEntry;
+  signedBid: gloas.SignedExecutionPayloadBid;
+  /** Time in milliseconds from the slot start when the bid was received */
+  receivedMs: number;
+};
+
+/**
+ * External builder integration post-gloas
+ *
+ * The builder set is driven by the resolved `BuilderConfig` the validator client supplies on
+ * each block production request, clients are dialed on demand based on the entry `url`.
+ */
+export class BuilderApiClient {
+  private readonly clients = new Map<BuilderUrl, BuilderApi>();
+
+  constructor(
+    private readonly opts: BuilderApiClientOpts,
+    private readonly config: ChainForkConfig,
+    private readonly clock: IClock,
+    private readonly bls: IBlsVerifier,
+    private readonly metrics: Metrics | null = null,
+    private readonly logger?: Logger
+  ) {}
+
+  /**
+   * Fan out bid requests to the builders named by the entries, one request per unique
+   * `(url, auth.message.data)` pair. Errors and empty responses (204) are logged and filtered
+   * out so a single unresponsive builder never fails block production.
+   */
+  async getExecutionPayloadBids(
+    entries: routes.validator.BuilderEntry[],
+    slot: Slot,
+    parentHash: Root,
+    parentRoot: Root,
+    proposerPubkey: BLSPubkey
+  ): Promise<BuilderApiBid[]> {
+    const seenRequests = new Set<string>();
+    const requests: {url: BuilderUrl; entry: routes.validator.BuilderEntry}[] = [];
+
+    for (const entry of entries) {
+      const urlForLog = Buffer.from(entry.url).toString("utf8");
+      let url: BuilderUrl;
+      try {
+        url = decodeBuilderUrl(entry.url);
+      } catch {
+        this.logger?.warn("Ignoring builder entry with invalid url", {slot, url: urlForLog});
+        continue;
+      }
+
+      const requestKey = `${url}-${toHex(entry.auth.message.data)}`;
+      if (seenRequests.has(requestKey)) {
+        continue;
+      }
+      seenRequests.add(requestKey);
+
+      // The builder rejects a mismatch, an entry naming a different slot is not used for a bid request
+      if (entry.auth.message.slot !== slot) {
+        this.logger?.warn("Ignoring builder entry with auth for different slot", {
+          slot,
+          authSlot: entry.auth.message.slot,
+          builder: toPrintableUrl(url),
+        });
+        continue;
+      }
+
+      requests.push({url, entry});
+    }
+
+    // Collected in arrival order so an earlier received bid wins ties during candidate ranking
+    const bids: BuilderApiBid[] = [];
+    await Promise.all(
+      requests.map(async ({url, entry}) => {
+        this.metrics?.builderApi.bidRequests.inc();
+        try {
+          const client = await this.getOrCreateClient(url, proposerPubkey, entry.auth);
+          const res = await client.getExecutionPayloadBid(
+            {
+              slot,
+              parentHash,
+              parentRoot,
+              proposerPubkey,
+              requestAuth: entry.auth,
+              dateMilliseconds: Date.now(),
+              timeoutMs: BUILDER_BID_DEADLINE_MS,
+            },
+            {timeoutMs: BUILDER_BID_REQUEST_TIMEOUT_MS}
+          );
+          const signedBid = res.value();
+          if (signedBid === undefined) {
+            this.logger?.debug("No bid received from builder", {slot, builder: toPrintableUrl(url)});
+            return;
+          }
+          this.metrics?.builderApi.bidsReceived.inc();
+          bids.push({url, entry, signedBid, receivedMs: this.clock.msFromSlot(slot)});
+        } catch (e) {
+          this.metrics?.builderApi.bidRequestErrors.inc();
+          this.logger?.warn("Failed to get bid from builder", {slot, builder: toPrintableUrl(url)}, e as Error);
+        }
+      })
+    );
+
+    return bids;
+  }
+
+  /** Forward a proposer's builder preferences to the builder at the given url */
+  async submitBuilderPreferences(
+    url: BuilderUrl,
+    proposerPubkey: BLSPubkey,
+    request: gloas.BuilderPreferencesRequest
+  ): Promise<void> {
+    try {
+      const client = await this.getOrCreateClient(url, proposerPubkey, request.auth);
+      (await client.submitBuilderPreferences({proposerPubkey, request})).assertOk();
+      this.metrics?.builderApi.preferencesForwarded.inc({status: "success"});
+    } catch (e) {
+      this.metrics?.builderApi.preferencesForwarded.inc({status: "error"});
+      throw e;
+    }
+  }
+
+  /**
+   * Submit the signed beacon block to the builder whose bid was included. The builder is
+   * then responsible for constructing and broadcasting the corresponding
+   * `SignedExecutionPayloadEnvelope`, no further action is required by the proposer.
+   */
+  async submitSignedBeaconBlock(
+    url: BuilderUrl,
+    signedBlock: WithOptionalBytes<SignedBeaconBlock<ForkPostGloas>>
+  ): Promise<void> {
+    const client = this.clients.get(url);
+    if (client === undefined) {
+      this.metrics?.builderApi.blockSubmissions.inc({status: "error"});
+      this.logger?.warn("Ignoring signed block submission to unauthenticated builder", {url});
+      return;
+    }
+
+    try {
+      (await client.submitSignedBeaconBlock({signedBlock}, {redirect: "manual"})).assertOk();
+      this.metrics?.builderApi.blockSubmissions.inc({status: "success"});
+    } catch (e) {
+      this.metrics?.builderApi.blockSubmissions.inc({status: "error"});
+      throw e;
+    }
+  }
+
+  /**
+   * Establish the connection to every known builder outside the bid deadline, a cold TCP/TLS handshake
+   * to a distant builder can exhaust `BUILDER_BID_DEADLINE_MS` on its own, and a long idle connection
+   * may be half-open and only fail once written to.
+   */
+  async checkStatus(): Promise<void> {
+    await Promise.all(
+      Array.from(this.clients.entries()).map(async ([url, client]) => {
+        try {
+          (await client.status()).assertOk();
+          this.metrics?.builderApi.statusChecks.inc({status: "success"});
+          this.logger?.debug("Builder is ready", {builder: toPrintableUrl(url)});
+        } catch (e) {
+          this.metrics?.builderApi.statusChecks.inc({status: "error"});
+          this.logger?.warn("Builder status check failed", {builder: toPrintableUrl(url)}, e as Error);
+        }
+      })
+    );
+  }
+
+  private async getOrCreateClient(
+    url: BuilderUrl,
+    proposerPubkey: BLSPubkey,
+    auth: gloas.SignedBuilderRequestAuth
+  ): Promise<BuilderApi> {
+    if (auth.message.data.length === 0) {
+      throw Error("Builder request auth data must not be empty");
+    }
+
+    let client = this.clients.get(url);
+    if (client === undefined) {
+      const domain = computeDomain(DOMAIN_BUILDER_REQUEST_AUTH, this.config.GENESIS_FORK_VERSION, ZERO_HASH);
+      const signingRoot = computeSigningRoot(ssz.gloas.BuilderRequestAuth, auth.message, domain);
+      const signatureSet = createSingleSignatureSetFromComponents(proposerPubkey, signingRoot, auth.signature);
+
+      let isValid = false;
+      try {
+        isValid = await this.bls.verifySignatureSets([signatureSet]);
+      } catch {
+        // Malformed signatures are invalid request authentication.
+      }
+      if (!isValid) {
+        throw Error("Invalid builder request auth");
+      }
+
+      client = getClient(
+        {
+          baseUrl: url,
+          globalInit: {
+            timeoutMs: this.opts.timeout,
+            headers: this.opts.userAgent ? {"User-Agent": this.opts.userAgent} : undefined,
+          },
+        },
+        {config: this.config, metrics: this.metrics?.builderHttpClient, logger: this.logger}
+      );
+      this.clients.set(url, client);
+      this.logger?.debug("External builder registered", {url: toPrintableUrl(url)});
+    }
+    return client;
+  }
+}

package/src/execution/builder/validateBid.ts

@@ -0,0 +1,181 @@
+import {routes} from "@lodestar/api";
+import {ProtoBlock} from "@lodestar/fork-choice";
+import {MAX_EXECUTION_PAYMENT, PAYLOAD_BUILDER_VERSION} from "@lodestar/params";
+import {
+  computeEpochAtSlot,
+  createSingleSignatureSetFromComponents,
+  getExecutionPayloadBidSigningRoot,
+  isActiveBuilder,
+  isGasLimitTargetCompatible,
+  isStatePostGloas,
+} from "@lodestar/state-transition";
+import {RootHex, Slot, gloas} from "@lodestar/types";
+import {bigIntMin, byteArrayEquals, prettyGweiToEth, toHex, toRootHex} from "@lodestar/utils";
+import {IBeaconChain} from "../../chain/index.js";
+import {RegenCaller} from "../../chain/regen/index.js";
+import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
+
+/** Return the counted bid payment in Gwei, saturated at uint64 max. */
+export function getBuilderBidTotalGwei(bid: gloas.ExecutionPayloadBid, maxExecutionPayment: bigint): bigint {
+  return bigIntMin(MAX_EXECUTION_PAYMENT, BigInt(bid.value) + bigIntMin(bid.executionPayment, maxExecutionPayment));
+}
+
+/**
+ * Validate a bid received from a builder over the builder API in response to a bid request
+ * made during block production. Unlike gossip validation, the bid must match the requested
+ * slot and parent exactly, may carry a non-zero `executionPayment` which is counted at most at
+ * the entry's `maxExecutionPayment`, and is not subject to gossip anti-spam rules.
+ *
+ * Throws with a description of the failure, the caller drops the bid.
+ */
+export async function validateBuilderApiExecutionPayloadBid(
+  chain: IBeaconChain,
+  signedExecutionPayloadBid: gloas.SignedExecutionPayloadBid,
+  request: {
+    slot: Slot;
+    parentBlock: ProtoBlock;
+    parentBlockHash: RootHex;
+    parentBlockRoot: RootHex;
+    entry: routes.validator.BuilderEntry;
+    getParentExecutionRequests: () => Promise<gloas.ExecutionRequests>;
+  }
+): Promise<void> {
+  const bid = signedExecutionPayloadBid.message;
+  const {slot, parentBlock, parentBlockHash, parentBlockRoot, entry, getParentExecutionRequests} = request;
+
+  if (bid.slot !== slot) {
+    throw Error(`Bid slot=${bid.slot} does not match requested slot=${slot}`);
+  }
+
+  const bidParentBlockHash = toRootHex(bid.parentBlockHash);
+  const bidParentBlockRoot = toRootHex(bid.parentBlockRoot);
+  if (bidParentBlockHash !== parentBlockHash || bidParentBlockRoot !== parentBlockRoot) {
+    throw Error(
+      `Bid parent parentBlockHash=${bidParentBlockHash} parentBlockRoot=${bidParentBlockRoot} does not match ` +
+        `requested parentBlockHash=${parentBlockHash} parentBlockRoot=${parentBlockRoot}`
+    );
+  }
+
+  const totalPayment = getBuilderBidTotalGwei(bid, entry.maxExecutionPayment);
+  if (totalPayment < entry.minBid) {
+    throw Error(
+      `Bid total payment=${prettyGweiToEth(totalPayment)} ` +
+        `(value=${prettyGweiToEth(bid.value)} ` +
+        `executionPayment=${prettyGweiToEth(bid.executionPayment)}) ` +
+        `is below minBid=${prettyGweiToEth(entry.minBid)}`
+    );
+  }
+
+  const state = await chain.regen
+    .getBlockSlotState(parentBlock, slot, {dontTransferCache: true}, RegenCaller.validateGossipExecutionPayloadBid)
+    .catch((e: Error) => {
+      throw Error(`Unable to regenerate state to validate bid: ${e.message}`);
+    });
+
+  if (!isStatePostGloas(state)) {
+    throw Error(`Expected gloas+ state for execution payload bid validation, got fork=${state.forkName}`);
+  }
+
+  if (bid.builderIndex >= state.getBuildersLength()) {
+    throw Error(`Bid builderIndex=${bid.builderIndex} is out of bounds`);
+  }
+
+  const builder = state.getBuilder(bid.builderIndex);
+  if (!isActiveBuilder(builder, state.finalizedCheckpoint.epoch)) {
+    throw Error(`Bid builderIndex=${bid.builderIndex} is not an active builder`);
+  }
+
+  if (builder.version !== PAYLOAD_BUILDER_VERSION) {
+    throw Error(`Invalid builder version=${builder.version} expected=${PAYLOAD_BUILDER_VERSION}`);
+  }
+
+  // A bid not signed by one of the builder pubkeys the entry accepts bids from must not be accepted
+  if (
+    entry.builderPubkeys.length > 0 &&
+    !entry.builderPubkeys.some((pubkey) => byteArrayEquals(pubkey, builder.pubkey))
+  ) {
+    throw Error(
+      `Bid builder pubkey=${toHex(builder.pubkey)} is not in the entry's ` +
+        `builderPubkeys=${entry.builderPubkeys.map(toHex).join(",")}`
+    );
+  }
+
+  const blobKzgCommitmentsLen = bid.blobKzgCommitments.length;
+  const maxBlobsPerBlock = chain.config.getMaxBlobsPerBlock(computeEpochAtSlot(bid.slot));
+  if (blobKzgCommitmentsLen > maxBlobsPerBlock) {
+    throw Error(`Bid has too many KZG commitments len=${blobKzgCommitmentsLen} limit=${maxBlobsPerBlock}`);
+  }
+
+  // The coverage check only applies to the staked collateral payment, a pure execution
+  // layer payment bid has nothing to cover on-chain
+  if (bid.value > 0 && !state.canBuilderCoverBid(bid.builderIndex, bid.value)) {
+    throw Error(
+      `Builder cannot cover bid value=${prettyGweiToEth(bid.value)} ` + `balance=${prettyGweiToEth(builder.balance)}`
+    );
+  }
+
+  const randaoMix = state.getRandaoMix(computeEpochAtSlot(state.slot));
+  if (!byteArrayEquals(bid.prevRandao, randaoMix)) {
+    throw Error(`Invalid bid prevRandao=${toHex(bid.prevRandao)} expected=${toHex(randaoMix)}`);
+  }
+
+  // The parent's payload does not try to exit the builder
+  if (byteArrayEquals(bid.parentBlockHash, state.latestExecutionPayloadBid.blockHash)) {
+    const requests = await getParentExecutionRequests();
+    if (
+      requests.builderExits.some(
+        (request) =>
+          byteArrayEquals(request.pubkey, builder.pubkey) &&
+          byteArrayEquals(request.sourceAddress, builder.executionAddress)
+      )
+    ) {
+      throw Error(`Bid builderIndex=${bid.builderIndex} may exit in the parent payload`);
+    }
+  }
+
+  // The builder must honor the proposer preferences it learned over gossip
+  const bidEpoch = computeEpochAtSlot(bid.slot);
+  const dependentRootHex = (() => {
+    try {
+      return getShufflingDependentRoot(chain.forkChoice, bidEpoch, computeEpochAtSlot(parentBlock.slot), parentBlock);
+    } catch {
+      return null;
+    }
+  })();
+  if (dependentRootHex === null) {
+    throw Error(`Unable to resolve proposer preferences dependent root for bid slot=${bid.slot}`);
+  }
+  const proposerPreferences = chain.proposerPreferencesPool.get(bid.slot, dependentRootHex);
+  if (proposerPreferences === null) {
+    throw Error(`No proposer preferences found for bid slot=${bid.slot} dependentRoot=${dependentRootHex}`);
+  }
+  if (!byteArrayEquals(bid.feeRecipient, proposerPreferences.message.feeRecipient)) {
+    throw Error(
+      `Bid feeRecipient=${toHex(bid.feeRecipient)} does not match ` +
+        `proposer preferences feeRecipient=${toHex(proposerPreferences.message.feeRecipient)}`
+    );
+  }
+
+  const parentPayloadVariant = chain.forkChoice.getBlockHexAndBlockHash(bidParentBlockRoot, bidParentBlockHash);
+  if (parentPayloadVariant === null || parentPayloadVariant.executionPayloadBlockHash === null) {
+    throw Error(`Unable to resolve parent payload gas limit for bid parentBlockHash=${bidParentBlockHash}`);
+  }
+  const parentGasLimit = BigInt(parentPayloadVariant.executionPayloadGasLimit);
+  const targetGasLimit = proposerPreferences.message.targetGasLimit;
+  if (!isGasLimitTargetCompatible(parentGasLimit, bid.gasLimit, targetGasLimit)) {
+    throw Error(
+      `Bid gasLimit=${bid.gasLimit} is not compatible with ` +
+        `parentGasLimit=${parentGasLimit} targetGasLimit=${targetGasLimit}`
+    );
+  }
+
+  const signatureSet = createSingleSignatureSetFromComponents(
+    builder.pubkey,
+    getExecutionPayloadBidSigningRoot(chain.config, bid),
+    signedExecutionPayloadBid.signature
+  );
+
+  if (!(await chain.bls.verifySignatureSets([signatureSet]))) {
+    throw Error(`Invalid bid signature builderIndex=${bid.builderIndex}`);
+  }
+}