@lodestar/builder 1.50.0-dev.ece0239bd3 → 1.50.0-dev.f23b93e9eb

package/src/services/blockObserver.ts

@@ -47,55 +47,17 @@ export class BlockObserver {
     this.retries = retries;
     this.retryDelay = retryDelay;
     this.maxSeenBlockRoots = maxSeenBlockRoots;
+    this.logger.info("Block observer initialized", {retries, retryDelay, maxSeenBlockRoots});
   }
 
   /**
-   * Register a block consumer. Consumers that require complete observation must register before {@link start}, because
+   * Register a block consumer. Consumers that require complete observation must register before event delivery, because
    * roots observed before registration remain consumed until normal FIFO eviction.
    */
   runOnBlock(fn: RunOnBlockFn): void {
     this.fns.push(fn);
   }
 
-  start(signal: AbortSignal): void {
-    this.logger.info("Subscribing to block events", {
-      retries: this.retries,
-      retryDelay: this.retryDelay,
-      maxSeenBlockRoots: this.maxSeenBlockRoots,
-    });
-
-    this.api.events
-      .eventstream({
-        topics: [EventType.block],
-        signal,
-        onEvent: (event) => {
-          if (event.type !== EventType.block) {
-            this.logger.debug("Ignoring unexpected beacon event", {eventType: event.type});
-            return;
-          }
-
-          void this.processBlockEvent(event.message, signal);
-        },
-        onError: (error) => {
-          this.logger.error("Failed to receive block event", {}, error);
-        },
-        onClose: () => {
-          if (signal.aborted) {
-            this.logger.debug("Closed stream for block events");
-          } else {
-            this.logger.error("Block event stream closed unexpectedly", {});
-          }
-        },
-      })
-      .catch((error: unknown) => {
-        this.logger.error(
-          "Failed to subscribe to block events",
-          {},
-          error instanceof Error ? error : Error(String(error))
-        );
-      });
-  }
-
   async processBlockEvent(event: BlockEvent, signal: AbortSignal): Promise<void> {
     try {
       const {slot, block: blockRoot, executionOptimistic} = event;
@@ -145,7 +107,7 @@ export class BlockObserver {
         return null;
       });
 
-      if (response === null) {
+      if (response === null || signal.aborted) {
         return;
       }
 

package/src/services/builderSigner.ts

@@ -1,7 +1,7 @@
 import {PublicKey, SecretKey} from "@chainsafe/lodestar-z/blst";
 import {BeaconConfig} from "@lodestar/config";
 import {getExecutionPayloadBidSigningRoot, getExecutionPayloadEnvelopeSigningRoot} from "@lodestar/state-transition";
-import {gloas} from "@lodestar/types";
+import {gloas, heze} from "@lodestar/types";
 
 export type Keypair = {publicKey: PublicKey; secretKey: SecretKey};
 
@@ -23,6 +23,8 @@ export class BuilderSigner {
     };
   }
 
+  signExecutionPayloadBid(bid: heze.ExecutionPayloadBid): heze.SignedExecutionPayloadBid;
+  signExecutionPayloadBid(bid: gloas.ExecutionPayloadBid): gloas.SignedExecutionPayloadBid;
   signExecutionPayloadBid(bid: gloas.ExecutionPayloadBid): gloas.SignedExecutionPayloadBid {
     const signingRoot = getExecutionPayloadBidSigningRoot(this.config, bid);
 

package/src/services/envelopePublisher.ts

@@ -0,0 +1,154 @@
+import {type ApiClient, routes} from "@lodestar/api";
+import {type BuilderIndex, type RootHex, type gloas, ssz} from "@lodestar/types";
+import {LodestarError, defer, toRootHex} from "@lodestar/utils";
+import type {BidIdentity, BidLedger} from "./bidLedger.js";
+import type {BuilderSigner} from "./builderSigner.js";
+import type {ExecutionPayloadEnvelopeContents} from "./executionPayloadEnvelope.js";
+
+export type EnvelopeSelectionIdentity = BidIdentity & {blockRoot: RootHex};
+
+export type EnvelopePublisherModules = {
+  api: ApiClient;
+  signer: BuilderSigner;
+  ledger: BidLedger;
+  builderIndex: BuilderIndex;
+};
+
+export enum EnvelopePublisherErrorCode {
+  BUILDER_INDEX_MISMATCH = "ENVELOPE_PUBLISHER_ERROR_BUILDER_INDEX_MISMATCH",
+  SELECTION_NOT_RECORDED = "ENVELOPE_PUBLISHER_ERROR_SELECTION_NOT_RECORDED",
+}
+
+export type EnvelopePublisherErrorType =
+  | {
+      code: EnvelopePublisherErrorCode.BUILDER_INDEX_MISMATCH;
+      builderIndex: BuilderIndex;
+      envelopeBuilderIndex: BuilderIndex;
+    }
+  | ({code: EnvelopePublisherErrorCode.SELECTION_NOT_RECORDED} & EnvelopeSelectionIdentity);
+
+export class EnvelopePublisherError extends LodestarError<EnvelopePublisherErrorType> {}
+
+export type EnvelopePublicationResult =
+  | {status: "published"; signedEnvelope: gloas.SignedExecutionPayloadEnvelope}
+  | {status: "duplicate"};
+
+type ActivePublication = {
+  controller: AbortController;
+  promise: Promise<EnvelopePublicationResult>;
+  waiters: number;
+};
+
+/** Signs and submits an envelope for a recorded local selection. */
+export class EnvelopePublisher {
+  private readonly activePublications = new Map<RootHex, ActivePublication>();
+
+  constructor(private readonly modules: EnvelopePublisherModules) {}
+
+  async publish(contents: ExecutionPayloadEnvelopeContents, signal: AbortSignal): Promise<EnvelopePublicationResult> {
+    signal.throwIfAborted();
+
+    const {api, builderIndex, ledger, signer} = this.modules;
+    const {envelope} = contents;
+    if (envelope.builderIndex !== builderIndex) {
+      throw new EnvelopePublisherError(
+        {
+          code: EnvelopePublisherErrorCode.BUILDER_INDEX_MISMATCH,
+          builderIndex,
+          envelopeBuilderIndex: envelope.builderIndex,
+        },
+        `Envelope builder index does not match local builder index builderIndex=${builderIndex} envelopeBuilderIndex=${envelope.builderIndex}`
+      );
+    }
+
+    const identity: EnvelopeSelectionIdentity = {
+      slot: envelope.payload.slotNumber,
+      parentBlockHash: toRootHex(envelope.payload.parentHash),
+      parentBlockRoot: toRootHex(envelope.parentBeaconBlockRoot),
+      blockHash: toRootHex(envelope.payload.blockHash),
+      blockRoot: toRootHex(envelope.beaconBlockRoot),
+    };
+    if (!ledger.hasWon(identity, identity.blockRoot)) {
+      throw new EnvelopePublisherError(
+        {code: EnvelopePublisherErrorCode.SELECTION_NOT_RECORDED, ...identity},
+        `Envelope selection is not recorded slot=${identity.slot} blockRoot=${identity.blockRoot} blockHash=${identity.blockHash}`
+      );
+    }
+
+    const envelopeRoot = toRootHex(ssz.gloas.ExecutionPayloadEnvelope.hashTreeRoot(envelope));
+    ledger.recordReveal(identity.slot, identity.blockRoot, identity.blockHash, envelopeRoot);
+    if (ledger.hasPublishedReveal(identity.blockRoot)) {
+      return {status: "duplicate"};
+    }
+
+    let publication = this.activePublications.get(identity.blockRoot);
+    if (publication === undefined) {
+      const controller = new AbortController();
+      publication = {
+        controller,
+        waiters: 0,
+        promise: this.publishEnvelope(contents, identity, envelopeRoot, api, ledger, signer, controller.signal).finally(
+          () => {
+            if (this.activePublications.get(identity.blockRoot)?.controller === controller) {
+              this.activePublications.delete(identity.blockRoot);
+            }
+          }
+        ),
+      };
+      this.activePublications.set(identity.blockRoot, publication);
+    }
+    return this.waitForPublication(identity.blockRoot, publication, signal);
+  }
+
+  private async waitForPublication(
+    blockRoot: RootHex,
+    publication: ActivePublication,
+    signal: AbortSignal
+  ): Promise<EnvelopePublicationResult> {
+    const aborted = defer<never>();
+    const onAbort = (): void => aborted.reject(signal.reason);
+    publication.waiters++;
+    signal.addEventListener("abort", onAbort, {once: true});
+    if (signal.aborted) onAbort();
+
+    try {
+      return await Promise.race([publication.promise, aborted.promise]);
+    } finally {
+      signal.removeEventListener("abort", onAbort);
+      publication.waiters--;
+      if (publication.waiters === 0 && signal.aborted) {
+        if (this.activePublications.get(blockRoot) === publication) {
+          this.activePublications.delete(blockRoot);
+        }
+        publication.controller.abort(signal.reason);
+      }
+    }
+  }
+
+  private async publishEnvelope(
+    contents: ExecutionPayloadEnvelopeContents,
+    identity: EnvelopeSelectionIdentity,
+    envelopeRoot: RootHex,
+    api: ApiClient,
+    ledger: BidLedger,
+    signer: BuilderSigner,
+    signal: AbortSignal
+  ): Promise<EnvelopePublicationResult> {
+    const signedEnvelope = signer.signExecutionPayloadEnvelope(contents.envelope);
+
+    const response = await api.beacon.publishExecutionPayloadEnvelope(
+      {
+        signedEnvelopeOrContents: {
+          signedExecutionPayloadEnvelope: signedEnvelope,
+          kzgProofs: contents.kzgProofs,
+          blobs: contents.blobs,
+        },
+        broadcastValidation: routes.beacon.BroadcastValidation.gossip,
+      },
+      {signal}
+    );
+    response.assertOk();
+    ledger.recordRevealPublished(identity.slot, identity.blockRoot, identity.blockHash, envelopeRoot);
+    return {status: "published", signedEnvelope};
+  }
+}

package/src/services/executionPayloadBid.ts

@@ -0,0 +1,125 @@
+import {ForkName, type ForkPostGloas, INCLUSION_LIST_COMMITTEE_SIZE, isForkPostHeze} from "@lodestar/params";
+import type {
+  BuilderIndex,
+  Bytes32,
+  ExecutionAddress,
+  ExecutionPayloadBid,
+  Root,
+  RootHex,
+  Slot,
+  gloas,
+  heze,
+} from "@lodestar/types";
+import {sszTypesFor} from "@lodestar/types";
+import {LodestarError, byteArrayEquals, toRootHex} from "@lodestar/utils";
+import type {BuiltPayload} from "./payloadSource.js";
+
+type CommonBidInput<F extends ForkPostGloas> = {
+  slot: Slot;
+  parentBlockRoot: Root;
+  /** Expected value from the BN payload attributes used for this build. */
+  prevRandao: Bytes32;
+  builderIndex: BuilderIndex;
+  feeRecipient: ExecutionAddress;
+  /** Bid value in gwei */
+  value: number;
+  payload: BuiltPayload & {fork: F};
+};
+
+export type GloasBidInput = CommonBidInput<ForkName.gloas>;
+
+export type HezeBidInput = CommonBidInput<ForkName.heze> & {
+  inclusionListBits: heze.ExecutionPayloadBid["inclusionListBits"];
+};
+
+export type ExecutionPayloadBidInput = GloasBidInput | HezeBidInput;
+
+export enum ExecutionPayloadBidErrorCode {
+  INVALID_VALUE = "EXECUTION_PAYLOAD_BID_ERROR_INVALID_VALUE",
+  INVALID_GAS_LIMIT = "EXECUTION_PAYLOAD_BID_ERROR_INVALID_GAS_LIMIT",
+  SLOT_MISMATCH = "EXECUTION_PAYLOAD_BID_ERROR_SLOT_MISMATCH",
+  BLOCK_HASH_EQUALS_PARENT = "EXECUTION_PAYLOAD_BID_ERROR_BLOCK_HASH_EQUALS_PARENT",
+  INVALID_INCLUSION_LIST_BITS = "EXECUTION_PAYLOAD_BID_ERROR_INVALID_INCLUSION_LIST_BITS",
+  PREV_RANDAO_MISMATCH = "EXECUTION_PAYLOAD_BID_ERROR_PREV_RANDAO_MISMATCH",
+}
+
+export type ExecutionPayloadBidErrorType =
+  | {code: ExecutionPayloadBidErrorCode.INVALID_GAS_LIMIT; gasLimit: number}
+  | {code: ExecutionPayloadBidErrorCode.SLOT_MISMATCH; slot: Slot; payloadSlot: Slot}
+  | {code: ExecutionPayloadBidErrorCode.BLOCK_HASH_EQUALS_PARENT}
+  | {code: ExecutionPayloadBidErrorCode.INVALID_INCLUSION_LIST_BITS; bitLen: number | undefined}
+  | {
+      code: ExecutionPayloadBidErrorCode.PREV_RANDAO_MISMATCH;
+      expectedPrevRandao: RootHex;
+      payloadPrevRandao: RootHex;
+    }
+  | {
+      code: ExecutionPayloadBidErrorCode.INVALID_VALUE;
+      value: number;
+    };
+
+export class ExecutionPayloadBidError extends LodestarError<ExecutionPayloadBidErrorType> {}
+
+export function createExecutionPayloadBid(input: GloasBidInput): gloas.ExecutionPayloadBid;
+export function createExecutionPayloadBid(input: HezeBidInput): heze.ExecutionPayloadBid;
+export function createExecutionPayloadBid(input: ExecutionPayloadBidInput): ExecutionPayloadBid;
+export function createExecutionPayloadBid(input: ExecutionPayloadBidInput): ExecutionPayloadBid {
+  if (!Number.isSafeInteger(input.value) || input.value < 0) {
+    throw new ExecutionPayloadBidError(
+      {code: ExecutionPayloadBidErrorCode.INVALID_VALUE, value: input.value},
+      `Invalid bid value value=${input.value}`
+    );
+  }
+
+  const {fork, executionPayload, executionRequests, blobsBundle} = input.payload;
+  if (executionPayload.gasLimit > Number.MAX_SAFE_INTEGER) {
+    throw new ExecutionPayloadBidError({
+      code: ExecutionPayloadBidErrorCode.INVALID_GAS_LIMIT,
+      gasLimit: executionPayload.gasLimit,
+    });
+  }
+  if (executionPayload.slotNumber !== input.slot) {
+    throw new ExecutionPayloadBidError({
+      code: ExecutionPayloadBidErrorCode.SLOT_MISMATCH,
+      slot: input.slot,
+      payloadSlot: executionPayload.slotNumber,
+    });
+  }
+  if (byteArrayEquals(executionPayload.blockHash, executionPayload.parentHash)) {
+    throw new ExecutionPayloadBidError({code: ExecutionPayloadBidErrorCode.BLOCK_HASH_EQUALS_PARENT});
+  }
+  if (!byteArrayEquals(executionPayload.prevRandao, input.prevRandao)) {
+    throw new ExecutionPayloadBidError({
+      code: ExecutionPayloadBidErrorCode.PREV_RANDAO_MISMATCH,
+      expectedPrevRandao: toRootHex(input.prevRandao),
+      payloadPrevRandao: toRootHex(executionPayload.prevRandao),
+    });
+  }
+  const bid: gloas.ExecutionPayloadBid = {
+    parentBlockHash: executionPayload.parentHash,
+    parentBlockRoot: input.parentBlockRoot,
+    blockHash: executionPayload.blockHash,
+    prevRandao: input.prevRandao,
+    feeRecipient: input.feeRecipient,
+    gasLimit: BigInt(executionPayload.gasLimit),
+    builderIndex: input.builderIndex,
+    slot: input.slot,
+    value: input.value,
+    executionPayment: 0n,
+    blobKzgCommitments: blobsBundle.commitments,
+    executionRequestsRoot: sszTypesFor(fork).ExecutionRequests.hashTreeRoot(executionRequests),
+  };
+
+  if (isForkPostHeze(fork)) {
+    const inclusionListBits = "inclusionListBits" in input ? input.inclusionListBits : undefined;
+    if (inclusionListBits?.bitLen !== INCLUSION_LIST_COMMITTEE_SIZE) {
+      throw new ExecutionPayloadBidError({
+        code: ExecutionPayloadBidErrorCode.INVALID_INCLUSION_LIST_BITS,
+        bitLen: inclusionListBits?.bitLen,
+      });
+    }
+    return {...bid, inclusionListBits};
+  }
+
+  return bid;
+}

package/src/services/executionPayloadEnvelope.ts

@@ -0,0 +1,156 @@
+import type {BuilderIndex, RootHex, Slot, deneb, gloas} from "@lodestar/types";
+import {ssz} from "@lodestar/types";
+import {LodestarError, byteArrayEquals, fromHex, toRootHex} from "@lodestar/utils";
+import type {StoredPayload} from "./payloadStore.js";
+
+export type ExecutionPayloadEnvelopeInput = {
+  blockRoot: RootHex;
+  builderIndex: BuilderIndex;
+  selectedBid: gloas.ExecutionPayloadBid;
+  storedPayload: StoredPayload;
+};
+
+export type ExecutionPayloadEnvelopeContents = {
+  envelope: gloas.ExecutionPayloadEnvelope;
+  kzgProofs: deneb.KZGProofs;
+  blobs: deneb.Blobs;
+};
+
+export enum ExecutionPayloadEnvelopeErrorCode {
+  SLOT_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_SLOT_MISMATCH",
+  PARENT_BLOCK_ROOT_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_PARENT_BLOCK_ROOT_MISMATCH",
+  PARENT_BLOCK_HASH_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_PARENT_BLOCK_HASH_MISMATCH",
+  BLOCK_HASH_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BLOCK_HASH_MISMATCH",
+  BUILDER_INDEX_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BUILDER_INDEX_MISMATCH",
+  BLOB_KZG_COMMITMENTS_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BLOB_KZG_COMMITMENTS_MISMATCH",
+  EXECUTION_REQUESTS_ROOT_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_EXECUTION_REQUESTS_ROOT_MISMATCH",
+}
+
+export type ExecutionPayloadEnvelopeErrorType =
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.BUILDER_INDEX_MISMATCH;
+      builderIndex: BuilderIndex;
+      bidBuilderIndex: BuilderIndex;
+    }
+  | {code: ExecutionPayloadEnvelopeErrorCode.BLOB_KZG_COMMITMENTS_MISMATCH}
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.EXECUTION_REQUESTS_ROOT_MISMATCH;
+      bidExecutionRequestsRoot: RootHex;
+      payloadExecutionRequestsRoot: RootHex;
+    }
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.PARENT_BLOCK_ROOT_MISMATCH;
+      bidParentBlockRoot: RootHex;
+      storedParentBlockRoot: RootHex;
+    }
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.SLOT_MISMATCH;
+      bidSlot: Slot;
+      payloadSlot: Slot;
+    }
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.PARENT_BLOCK_HASH_MISMATCH;
+      bidParentBlockHash: RootHex;
+      payloadParentBlockHash: RootHex;
+    }
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.BLOCK_HASH_MISMATCH;
+      bidBlockHash: RootHex;
+      payloadBlockHash: RootHex;
+    };
+
+export class ExecutionPayloadEnvelopeError extends LodestarError<ExecutionPayloadEnvelopeErrorType> {}
+
+/** The caller must have matched the selected bid to its local ledger record. */
+export function createExecutionPayloadEnvelopeContents({
+  blockRoot,
+  builderIndex,
+  selectedBid,
+  storedPayload,
+}: ExecutionPayloadEnvelopeInput): ExecutionPayloadEnvelopeContents {
+  if (builderIndex !== selectedBid.builderIndex) {
+    throw new ExecutionPayloadEnvelopeError({
+      code: ExecutionPayloadEnvelopeErrorCode.BUILDER_INDEX_MISMATCH,
+      builderIndex,
+      bidBuilderIndex: selectedBid.builderIndex,
+    });
+  }
+
+  const bidParentBlockRoot = toRootHex(selectedBid.parentBlockRoot);
+  const storedParentBlockRoot = toRootHex(storedPayload.parentBlockRoot);
+  if (storedParentBlockRoot !== bidParentBlockRoot) {
+    throw new ExecutionPayloadEnvelopeError(
+      {
+        code: ExecutionPayloadEnvelopeErrorCode.PARENT_BLOCK_ROOT_MISMATCH,
+        bidParentBlockRoot,
+        storedParentBlockRoot,
+      },
+      `Selected bid beacon parent does not match retained payload bidParentBlockRoot=${bidParentBlockRoot} storedParentBlockRoot=${storedParentBlockRoot}`
+    );
+  }
+
+  const {payload} = storedPayload;
+  const payloadSlot = payload.executionPayload.slotNumber;
+  if (payloadSlot !== selectedBid.slot) {
+    throw new ExecutionPayloadEnvelopeError(
+      {code: ExecutionPayloadEnvelopeErrorCode.SLOT_MISMATCH, bidSlot: selectedBid.slot, payloadSlot},
+      `Selected bid slot does not match payload slot bidSlot=${selectedBid.slot} payloadSlot=${payloadSlot}`
+    );
+  }
+
+  const payloadParentBlockHash = toRootHex(payload.executionPayload.parentHash);
+  const bidParentBlockHash = toRootHex(selectedBid.parentBlockHash);
+  if (payloadParentBlockHash !== bidParentBlockHash) {
+    throw new ExecutionPayloadEnvelopeError(
+      {
+        code: ExecutionPayloadEnvelopeErrorCode.PARENT_BLOCK_HASH_MISMATCH,
+        bidParentBlockHash,
+        payloadParentBlockHash,
+      },
+      `Selected bid parent does not match payload parent bidParentBlockHash=${bidParentBlockHash} payloadParentBlockHash=${payloadParentBlockHash}`
+    );
+  }
+
+  const payloadBlockHash = toRootHex(payload.executionPayload.blockHash);
+  const bidBlockHash = toRootHex(selectedBid.blockHash);
+  if (payloadBlockHash !== bidBlockHash) {
+    throw new ExecutionPayloadEnvelopeError(
+      {
+        code: ExecutionPayloadEnvelopeErrorCode.BLOCK_HASH_MISMATCH,
+        bidBlockHash,
+        payloadBlockHash,
+      },
+      `Selected bid block hash does not match payload bidBlockHash=${bidBlockHash} payloadBlockHash=${payloadBlockHash}`
+    );
+  }
+
+  if (
+    !ssz.gloas.ExecutionPayloadBid.fields.blobKzgCommitments.equals(
+      selectedBid.blobKzgCommitments,
+      payload.blobsBundle.commitments
+    )
+  ) {
+    throw new ExecutionPayloadEnvelopeError({code: ExecutionPayloadEnvelopeErrorCode.BLOB_KZG_COMMITMENTS_MISMATCH});
+  }
+
+  const executionRequestsRoot = ssz.gloas.ExecutionRequests.hashTreeRoot(payload.executionRequests);
+  if (!byteArrayEquals(executionRequestsRoot, selectedBid.executionRequestsRoot)) {
+    throw new ExecutionPayloadEnvelopeError({
+      code: ExecutionPayloadEnvelopeErrorCode.EXECUTION_REQUESTS_ROOT_MISMATCH,
+      bidExecutionRequestsRoot: toRootHex(selectedBid.executionRequestsRoot),
+      payloadExecutionRequestsRoot: toRootHex(executionRequestsRoot),
+    });
+  }
+
+  return {
+    envelope: {
+      payload: payload.executionPayload,
+      executionRequests: payload.executionRequests,
+      builderIndex,
+      beaconBlockRoot: fromHex(blockRoot),
+      parentBeaconBlockRoot: selectedBid.parentBlockRoot,
+    },
+    kzgProofs: payload.blobsBundle.proofs,
+    blobs: payload.blobsBundle.blobs,
+  };
+}

package/src/services/payloadOrchestrator.ts

@@ -0,0 +1,107 @@
+import {LodestarError, TimeoutError, retry, sleep, withTimeout} from "@lodestar/utils";
+import {
+  type BuildRequest,
+  type BuiltPayload,
+  type PayloadSource,
+  PayloadSourceError,
+  PayloadSourceErrorCode,
+} from "./payloadSource.js";
+
+const PREPARE_RETRY_DELAY = 100;
+
+export type PayloadBuildJob = {
+  /** Jobs with the same ID share one build and its result. */
+  id: string;
+  request: BuildRequest;
+  /** Unix timestamp in milliseconds at which the prepared payload should be retrieved. */
+  getPayloadAt: number;
+};
+
+export type PayloadOrchestratorOptions = {
+  /** Maximum time in milliseconds to wait for payload retrieval. */
+  getPayloadTimeout: number;
+};
+
+export enum PayloadOrchestratorErrorCode {
+  PREPARE_DEADLINE_REACHED = "PAYLOAD_ORCHESTRATOR_ERROR_PREPARE_DEADLINE_REACHED",
+  PREPARE_TIMEOUT = "PAYLOAD_ORCHESTRATOR_ERROR_PREPARE_TIMEOUT",
+  GET_PAYLOAD_TIMEOUT = "PAYLOAD_ORCHESTRATOR_ERROR_GET_PAYLOAD_TIMEOUT",
+}
+
+export type PayloadOrchestratorErrorType =
+  | {
+      code: PayloadOrchestratorErrorCode.PREPARE_DEADLINE_REACHED;
+      jobId: string;
+      getPayloadAt: number;
+    }
+  | {
+      code: PayloadOrchestratorErrorCode.PREPARE_TIMEOUT | PayloadOrchestratorErrorCode.GET_PAYLOAD_TIMEOUT;
+      jobId: string;
+    };
+
+export class PayloadOrchestratorError extends LodestarError<PayloadOrchestratorErrorType> {}
+
+/**
+ * Prepares a payload and retrieves it at the requested time. Preparation retries missing payload IDs until the
+ * retrieval time. Transport-level retries remain owned by the source.
+ */
+export class PayloadOrchestrator {
+  private readonly jobs = new Map<string, Promise<BuiltPayload>>();
+
+  constructor(
+    private readonly source: PayloadSource,
+    private readonly options: PayloadOrchestratorOptions,
+    private readonly signal: AbortSignal
+  ) {}
+
+  run(job: PayloadBuildJob): Promise<BuiltPayload> {
+    const {id} = job;
+    let promise = this.jobs.get(id);
+    if (promise === undefined) {
+      promise = this.runJob(job).finally(() => this.jobs.delete(id));
+      this.jobs.set(id, promise);
+    }
+    return promise;
+  }
+
+  private async runJob({id, request, getPayloadAt}: PayloadBuildJob): Promise<BuiltPayload> {
+    const prepareTimeout = getPayloadAt - Date.now();
+    if (!(prepareTimeout > 0)) {
+      throw new PayloadOrchestratorError(
+        {code: PayloadOrchestratorErrorCode.PREPARE_DEADLINE_REACHED, jobId: id, getPayloadAt},
+        `Payload preparation deadline reached jobId=${id} getPayloadAt=${getPayloadAt}`
+      );
+    }
+
+    const handle = await withTimeout(
+      (signal = this.signal) =>
+        retry(() => this.source.prepare(request, signal), {
+          retries: Infinity,
+          retryDelay: PREPARE_RETRY_DELAY,
+          shouldRetry: (error) =>
+            error instanceof PayloadSourceError && error.type.code === PayloadSourceErrorCode.NO_PAYLOAD_ID,
+          signal,
+        }),
+      prepareTimeout,
+      this.signal
+    ).catch((error: unknown) => {
+      if (error instanceof TimeoutError) {
+        throw new PayloadOrchestratorError({code: PayloadOrchestratorErrorCode.PREPARE_TIMEOUT, jobId: id});
+      }
+      throw error;
+    });
+
+    await sleep(getPayloadAt - Date.now(), this.signal);
+
+    return withTimeout(
+      (signal = this.signal) => this.source.getPayload(handle, signal),
+      this.options.getPayloadTimeout,
+      this.signal
+    ).catch((error: unknown) => {
+      if (error instanceof TimeoutError) {
+        throw new PayloadOrchestratorError({code: PayloadOrchestratorErrorCode.GET_PAYLOAD_TIMEOUT, jobId: id});
+      }
+      throw error;
+    });
+  }
+}

package/src/services/proposerPreferencesTracker.ts

@@ -1,6 +1,5 @@
-import {ApiClient, routes} from "@lodestar/api";
 import type {RootHex, Slot, gloas} from "@lodestar/types";
-import {Logger, MapDef, toRootHex} from "@lodestar/utils";
+import {MapDef, toRootHex} from "@lodestar/utils";
 
 /** Retains validated proposer preferences by the branch-specific identity used for bid validation. */
 export class ProposerPreferencesTracker {
@@ -8,40 +7,6 @@ export class ProposerPreferencesTracker {
     () => new Map()
   );
 
-  constructor(
-    private readonly api: ApiClient,
-    private readonly logger: Logger
-  ) {}
-
-  start(signal: AbortSignal): void {
-    if (signal.aborted) return;
-
-    this.logger.verbose("Subscribing to proposer preferences");
-    this.api.events
-      .eventstream({
-        topics: [routes.events.EventType.proposerPreferences],
-        signal,
-        onEvent: (event) => {
-          if (!signal.aborted && event.type === routes.events.EventType.proposerPreferences) {
-            this.onProposerPreferences(event.message.data);
-          }
-        },
-        onError: (error) => {
-          if (!signal.aborted) this.logger.error("Failed to receive proposer preferences", {}, error);
-        },
-        onClose: () => {
-          if (signal.aborted) {
-            this.logger.verbose("Closed proposer preferences stream");
-          } else {
-            this.logger.error("Proposer preferences stream closed unexpectedly", {});
-          }
-        },
-      })
-      .catch((error: Error) => {
-        if (!signal.aborted) this.logger.error("Failed to subscribe to proposer preferences", {}, error);
-      });
-  }
-
   onProposerPreferences(signedProposerPreferences: gloas.SignedProposerPreferences): boolean {
     const {proposalSlot, dependentRoot} = signedProposerPreferences.message;
     const dependentRootHex = toRootHex(dependentRoot);