@lodestar/builder 1.51.0-dev.e91a5043ed → 1.51.0-dev.ec1447da07

package/src/defaults.ts

@@ -1,5 +1,19 @@
+import {MIN_DEPOSIT_AMOUNT} from "@lodestar/params";
+
 export const defaultOptions = {
   // Source beacon node the builder connects to
   beaconNodeUrl: "http://127.0.0.1:9596",
   requestTimeout: 10_000,
+
+  bidding: {
+    /** Share of the payload value offered to the proposer, the full value by default */
+    shareBps: 10_000,
+    fixedCostGwei: 0,
+    minValueGwei: 0,
+    /** Retrieve the payload and publish the bid at 95% of the slot before the proposal slot */
+    deadlineBps: 9500,
+    getPayloadTimeout: 1000,
+    /** Do not bid when the builder balance falls below this, bids are only covered by the balance above the deposit minimum */
+    minOperatingBalanceGwei: MIN_DEPOSIT_AMOUNT,
+  },
 };

package/src/identity.ts

@@ -99,11 +99,10 @@ async function waitForBuilder(
     }
     if (builder?.status === "pending") {
       logger.info("Waiting for builder deposit to be finalized", {id, slot: clock.getCurrentSlot()});
-      await sleep(msToNextEpochPoll(clock), signal);
     } else {
       logger.info("Waiting for builder to be known to the beacon node", {id, slot: clock.getCurrentSlot()});
-      await sleep(msToNextSlotPoll(clock), signal);
     }
+    await sleep(msToNextSlotPoll(clock), signal);
   }
   throw new ErrorAborted("waitForBuilder");
 }
@@ -145,8 +144,9 @@ async function fetchBuilder(
   }
 }
 
+/** Polls at the second slot of the next epoch, the head state has gone through the epoch transition by then */
 function msToNextEpochPoll(clock: IClock): number {
-  return clock.msToSlot(computeStartSlotAtEpoch(clock.getCurrentEpoch() + 1));
+  return clock.msToSlot(computeStartSlotAtEpoch(clock.getCurrentEpoch() + 1) + 1);
 }
 
 function msToNextSlotPoll(clock: IClock): number {

package/src/index.ts

@@ -2,3 +2,9 @@ export {Builder, type BuilderOptions} from "./builder.js";
 export {defaultOptions} from "./defaults.js";
 export {type Metrics, getMetrics} from "./metrics.js";
 export {type Keypair} from "./services/builderSigner.js";
+export {
+  type EnginePayloadResult,
+  EnginePayloadSource,
+  type PayloadSource,
+  type PayloadSourceEngine,
+} from "./services/payloadSource.js";

package/src/metrics.ts

@@ -14,6 +14,22 @@ export type LodestarGitData = {
 
 export const builderStatusValue: Record<BuilderStatus, number> = {pending: 0, active: 1, exited: 2};
 
+export enum BidResult {
+  published = "published",
+  noProposerPreferences = "no_proposer_preferences",
+  inactive = "inactive",
+  lowBalance = "low_balance",
+  policyDeclined = "policy_declined",
+  error = "error",
+}
+
+export enum RevealResult {
+  published = "published",
+  unknownPayload = "unknown_payload",
+  late = "late",
+  error = "error",
+}
+
 export function getMetrics(register: MetricsRegisterExtra, gitData: LodestarGitData) {
   register
     .gauge<LodestarGitData>({
@@ -34,6 +50,26 @@ export function getMetrics(register: MetricsRegisterExtra, gitData: LodestarGitD
       help: "Current builder balance in gwei",
     }),
 
+    bids: {
+      total: register.gauge<{result: BidResult}>({
+        name: "bc_builder_bids_total",
+        help: "Total count of bid attempts by result",
+        labelNames: ["result"],
+      }),
+      won: register.gauge({
+        name: "bc_builder_bids_won_total",
+        help: "Total count of imported blocks that selected one of our bids",
+      }),
+    },
+
+    reveals: {
+      total: register.gauge<{result: RevealResult}>({
+        name: "bc_builder_reveals_total",
+        help: "Total count of payload reveal attempts by result",
+        labelNames: ["result"],
+      }),
+    },
+
     // REST API client
 
     restApiClient: {

package/src/services/bidLedger.ts

@@ -190,6 +190,21 @@ export class BidLedger {
     }
   }
 
+  /** Marks won bids before the given slot as settled, by then their payment is reflected in the builder balance. */
+  settlePaymentsBefore(slot: Slot): void {
+    for (const [bidSlot, bidsForSlot] of this.bidsBySlot) {
+      if (bidSlot >= slot) {
+        continue;
+      }
+
+      for (const record of bidsForSlot.values()) {
+        if (record.wonBlockRoots.size > 0) {
+          record.paymentSettled = true;
+        }
+      }
+    }
+  }
+
   getUnsettledValueGwei(currentEpoch: Epoch): number {
     let total = 0;
     for (const bidsForSlot of this.bidsBySlot.values()) {

package/src/services/bidder.ts

@@ -0,0 +1,191 @@
+import {ApiClient, routes} from "@lodestar/api";
+import {ChainForkConfig} from "@lodestar/config";
+import {ForkName, MIN_DEPOSIT_AMOUNT, isForkPostGloas} from "@lodestar/params";
+import {IClock, computeEpochAtSlot} from "@lodestar/state-transition";
+import {BuilderIndex, BuilderStatus, Epoch, ExecutionAddress, RootHex, gloas} from "@lodestar/types";
+import {GWEI_TO_WEI, Logger, isErrorAborted, prettyGweiToEth, toHex, toRootHex} from "@lodestar/utils";
+import {BidResult, Metrics} from "../metrics.js";
+import {BidLedger} from "./bidLedger.js";
+import {BidPolicy} from "./bidPolicy.js";
+import {BidPublisher} from "./bidPublisher.js";
+import {createExecutionPayloadBid} from "./executionPayloadBid.js";
+import {PayloadOrchestrator} from "./payloadOrchestrator.js";
+import {PayloadStore} from "./payloadStore.js";
+import {ProposerPreferencesTracker} from "./proposerPreferencesTracker.js";
+
+export type PayloadAttributesEvent = routes.events.EventData[routes.events.EventType.payloadAttributes];
+
+export type BidderOptions = {
+  /** Point within the slot before the proposal slot at which the payload is retrieved and bid on, in basis points */
+  deadlineBps: number;
+  /** Do not bid while the builder balance is below this value */
+  minOperatingBalanceGwei: number;
+};
+
+export type BidderModules = {
+  config: ChainForkConfig;
+  logger: Logger;
+  clock: IClock;
+  api: ApiClient;
+  orchestrator: PayloadOrchestrator;
+  payloadStore: PayloadStore;
+  ledger: BidLedger;
+  policy: BidPolicy;
+  bidPublisher: BidPublisher;
+  proposerPreferencesTracker: ProposerPreferencesTracker;
+  getBuilderStatus: () => {status: BuilderStatus | undefined; balance: number | undefined};
+  builderIndex: BuilderIndex;
+  executionFeeRecipient: ExecutionAddress;
+  metrics: Metrics | null;
+  signal: AbortSignal;
+};
+
+/**
+ * Bids once per proposal slot and parent: starts a payload build when the beacon node emits payload attributes,
+ * retrieves the payload at the deadline, prices it and publishes the signed bid.
+ */
+export class Bidder {
+  private readonly activeBuilds = new Set<string>();
+
+  constructor(
+    private readonly modules: BidderModules,
+    private readonly opts: BidderOptions
+  ) {}
+
+  async onPayloadAttributes(event: PayloadAttributesEvent): Promise<void> {
+    const {clock, config, ledger, logger, metrics, orchestrator, proposerPreferencesTracker} = this.modules;
+    if (!isForkPostGloas(event.version)) {
+      return;
+    }
+    const data = event.data as gloas.SSEPayloadAttributes;
+    const slot = data.proposalSlot;
+    const parentBlockRoot = toRootHex(data.parentBlockRoot);
+    const parentBlockHash = toRootHex(data.parentBlockHash);
+    const logCtx = {slot, parentBlockRoot, parentBlockHash};
+
+    if (slot <= clock.getCurrentSlot()) {
+      logger.debug("Ignoring payload attributes for past slot", logCtx);
+      return;
+    }
+
+    // TODO: the beacon node emits payload attributes once per slot, late in the slot and only for the parent
+    // payload variant it would build on, revisit to start the build earlier and to bid on both variants
+    const buildId = `${slot}:${parentBlockRoot}:${parentBlockHash}`;
+    if (this.activeBuilds.has(buildId) || ledger.hasSubmitted(slot, parentBlockHash, parentBlockRoot)) {
+      return;
+    }
+    this.activeBuilds.add(buildId);
+
+    try {
+      // Heze bids require inclusion list bits which are not tracked yet
+      if (event.version !== ForkName.gloas) {
+        logger.warn("Bidding is not supported for fork", {...logCtx, fork: event.version});
+        return;
+      }
+
+      const dependentRoot = await this.getDependentRoot(computeEpochAtSlot(slot), this.modules.signal);
+      const proposerPreferences = proposerPreferencesTracker.get(slot, dependentRoot);
+      if (proposerPreferences === null) {
+        logger.debug("No proposer preferences known for slot, not bidding", {...logCtx, dependentRoot});
+        metrics?.bids.total.inc({result: BidResult.noProposerPreferences});
+        return;
+      }
+      const {feeRecipient, targetGasLimit} = proposerPreferences.message;
+
+      // The payload is retrieved in the slot before the proposal slot, use the timing of that slot
+      const buildFork = config.getForkName(slot - 1);
+      const msToDeadline =
+        clock.msToSlot(slot) -
+        config.getSlotDurationMs(buildFork) +
+        config.getSlotComponentDurationMs(buildFork, this.opts.deadlineBps);
+      logger.verbose("Preparing payload build", {...logCtx, msToDeadline, targetGasLimit});
+
+      const payload = await orchestrator.run({
+        id: buildId,
+        request: {
+          fork: event.version,
+          forkchoiceState: {
+            headBlockHash: parentBlockHash,
+            safeBlockHash: toRootHex(data.safeBlockHash),
+            finalizedBlockHash: toRootHex(data.finalizedBlockHash),
+          },
+          payloadAttributes: {
+            ...data.payloadAttributes,
+            suggestedFeeRecipient: toHex(this.modules.executionFeeRecipient),
+            targetGasLimit,
+          },
+        },
+        getPayloadAt: Date.now() + msToDeadline,
+      });
+
+      const {status, balance} = this.modules.getBuilderStatus();
+      if (status !== "active" || balance === undefined) {
+        logger.warn("Builder is not active, not bidding", {...logCtx, status});
+        metrics?.bids.total.inc({result: BidResult.inactive});
+        return;
+      }
+      if (balance < this.opts.minOperatingBalanceGwei) {
+        logger.warn("Builder balance below operating minimum, not bidding", {
+          ...logCtx,
+          balance: prettyGweiToEth(balance),
+          minOperatingBalance: prettyGweiToEth(this.opts.minOperatingBalanceGwei),
+        });
+        metrics?.bids.total.inc({result: BidResult.lowBalance});
+        return;
+      }
+
+      const unsettledGwei = ledger.getUnsettledValueGwei(computeEpochAtSlot(slot));
+      const coverableGwei = Math.max(balance - MIN_DEPOSIT_AMOUNT - unsettledGwei, 0);
+      const payloadValueGwei = Number(payload.executionPayloadValue / GWEI_TO_WEI);
+      const value = this.modules.policy.computeValue({payloadValueGwei, coverableGwei});
+      if (value === null) {
+        logger.info("Bid policy declined to bid", {...logCtx, payloadValueGwei, coverableGwei, unsettledGwei});
+        metrics?.bids.total.inc({result: BidResult.policyDeclined});
+        return;
+      }
+
+      const {executionPayload} = payload;
+      const blockHash = toRootHex(executionPayload.blockHash);
+      this.modules.payloadStore.add({slot, parentBlockRoot: data.parentBlockRoot, blockHash, payload});
+
+      const bid = createExecutionPayloadBid({
+        slot,
+        parentBlockRoot: data.parentBlockRoot,
+        prevRandao: data.payloadAttributes.prevRandao,
+        builderIndex: this.modules.builderIndex,
+        feeRecipient,
+        value,
+        payload: {...payload, fork: ForkName.gloas},
+      });
+      await this.modules.bidPublisher.publish(bid, this.modules.signal);
+
+      metrics?.bids.total.inc({result: BidResult.published});
+      logger.info("Published execution payload bid", {
+        ...logCtx,
+        blockHash,
+        value: prettyGweiToEth(value),
+        payloadValue: prettyGweiToEth(payloadValueGwei),
+        feeRecipient: toHex(feeRecipient),
+        gasLimit: executionPayload.gasLimit,
+        transactions: executionPayload.transactions.length,
+        blobs: bid.blobKzgCommitments.length,
+        secFromSlot: clock.secFromSlot(slot),
+      });
+    } catch (e) {
+      if (!isErrorAborted(e)) {
+        metrics?.bids.total.inc({result: BidResult.error});
+        logger.error("Failed to bid", logCtx, e as Error);
+      }
+    } finally {
+      this.activeBuilds.delete(buildId);
+    }
+  }
+
+  /**
+   * Dependent root of the proposer duties in the given epoch, identifies the proposer preferences to bid against.
+   * Not cached, a reorg can change the dependent root within an epoch. Active builds already coalesce repeated attributes.
+   */
+  private async getDependentRoot(epoch: Epoch, signal: AbortSignal): Promise<RootHex> {
+    return (await this.modules.api.validator.getProposerDutiesV2({epoch}, {signal})).meta().dependentRoot;
+  }
+}

package/src/services/payloadOrchestrator.ts

@@ -35,15 +35,20 @@ export type PayloadOrchestratorErrorType =
       getPayloadAt: number;
     }
   | {
-      code: PayloadOrchestratorErrorCode.PREPARE_TIMEOUT | PayloadOrchestratorErrorCode.GET_PAYLOAD_TIMEOUT;
+      code: PayloadOrchestratorErrorCode.PREPARE_TIMEOUT;
+      jobId: string;
+      lastError: string | null;
+    }
+  | {
+      code: PayloadOrchestratorErrorCode.GET_PAYLOAD_TIMEOUT;
       jobId: string;
     };
 
 export class PayloadOrchestratorError extends LodestarError<PayloadOrchestratorErrorType> {}
 
 /**
- * Prepares a payload and retrieves it at the requested time. Preparation retries missing payload IDs until the
- * retrieval time. Transport-level retries remain owned by the source.
+ * Prepares a payload and retrieves it at the requested time. A missing payload ID is retried until the
+ * retrieval time, the execution client may not have imported the parent payload yet. Other errors fail the build.
  */
 export class PayloadOrchestrator {
   private readonly jobs = new Map<string, Promise<BuiltPayload>>();
@@ -73,6 +78,7 @@ export class PayloadOrchestrator {
       );
     }
 
+    let lastError: Error | null = null;
     const handle = await withTimeout(
       (signal = this.signal) =>
         retry(() => this.source.prepare(request, signal), {
@@ -80,13 +86,19 @@ export class PayloadOrchestrator {
           retryDelay: PREPARE_RETRY_DELAY,
           shouldRetry: (error) =>
             error instanceof PayloadSourceError && error.type.code === PayloadSourceErrorCode.NO_PAYLOAD_ID,
+          onRetry: (error) => {
+            lastError = error;
+          },
           signal,
         }),
       prepareTimeout,
       this.signal
     ).catch((error: unknown) => {
       if (error instanceof TimeoutError) {
-        throw new PayloadOrchestratorError({code: PayloadOrchestratorErrorCode.PREPARE_TIMEOUT, jobId: id});
+        throw new PayloadOrchestratorError(
+          {code: PayloadOrchestratorErrorCode.PREPARE_TIMEOUT, jobId: id, lastError: lastError?.message ?? null},
+          `Payload preparation timed out jobId=${id} lastError=${lastError?.message}`
+        );
       }
       throw error;
     });

package/src/services/revealer.ts

@@ -0,0 +1,88 @@
+import {ChainForkConfig} from "@lodestar/config";
+import {IClock} from "@lodestar/state-transition";
+import {BuilderIndex} from "@lodestar/types";
+import {Logger} from "@lodestar/utils";
+import {Metrics, RevealResult} from "../metrics.js";
+import {BidSelector} from "./bidSelector.js";
+import {ObservedBlock} from "./blockObserver.js";
+import {EnvelopePublisher} from "./envelopePublisher.js";
+import {createExecutionPayloadEnvelopeContents} from "./executionPayloadEnvelope.js";
+import {PayloadStore} from "./payloadStore.js";
+
+export type RevealerOptions = {
+  /**
+   * Latest point within the block's slot at which a reveal may start, in basis points.
+   * Defaults to the network's PAYLOAD_ATTESTATION_DUE_BPS
+   */
+  cutoffBps: number;
+};
+
+export type RevealerModules = {
+  config: ChainForkConfig;
+  logger: Logger;
+  clock: IClock;
+  bidSelector: BidSelector;
+  payloadStore: PayloadStore;
+  envelopePublisher: EnvelopePublisher;
+  builderIndex: BuilderIndex;
+  metrics: Metrics | null;
+  signal: AbortSignal;
+};
+
+/** Reveals the payload once a block committing to one of our bids is imported by the beacon node. */
+export class Revealer {
+  constructor(
+    private readonly modules: RevealerModules,
+    private readonly opts: RevealerOptions
+  ) {}
+
+  async onBlock(observed: ObservedBlock): Promise<void> {
+    const {bidSelector, clock, config, envelopePublisher, logger, metrics, payloadStore} = this.modules;
+    const selection = bidSelector.match(observed);
+    if (selection.status !== "selected") {
+      return;
+    }
+    metrics?.bids.won.inc();
+
+    const {blockRoot, slot} = observed;
+    const {blockHash} = selection.bid;
+    const logCtx = {slot, blockRoot, blockHash};
+
+    const storedPayload = payloadStore.get(blockHash);
+    if (storedPayload === null) {
+      logger.error("Payload of selected bid is not retained, cannot reveal", logCtx);
+      metrics?.reveals.total.inc({result: RevealResult.unknownPayload});
+      return;
+    }
+
+    const msFromSlot = clock.msFromSlot(slot);
+    const cutoffMs = config.getSlotComponentDurationMs(config.getForkName(slot), this.opts.cutoffBps);
+    if (msFromSlot > cutoffMs) {
+      logger.warn("Block with our bid arrived after reveal cutoff, not revealing", {...logCtx, msFromSlot, cutoffMs});
+      metrics?.reveals.total.inc({result: RevealResult.late});
+      return;
+    }
+
+    const contents = createExecutionPayloadEnvelopeContents({
+      blockRoot,
+      builderIndex: this.modules.builderIndex,
+      selectedBid: observed.signedBid.message,
+      storedPayload,
+    });
+
+    try {
+      await envelopePublisher.publish(contents, this.modules.signal);
+    } catch (e) {
+      metrics?.reveals.total.inc({result: RevealResult.error});
+      throw e;
+    }
+
+    metrics?.reveals.total.inc({result: RevealResult.published});
+    logger.info("Revealed execution payload", {
+      ...logCtx,
+      blobs: contents.blobs.length,
+      transactions: contents.envelope.payload.transactions.length,
+      secFromSlot: clock.secFromSlot(slot),
+    });
+  }
+}