@lodestar/builder 1.50.0-dev.1f38fe79fb → 1.50.0-dev.20062ed60e

package/src/services/bidSelector.ts

@@ -0,0 +1,138 @@
+import type {ChainForkConfig} from "@lodestar/config";
+import type {ForkPostGloas} from "@lodestar/params";
+import type {BuilderIndex, RootHex, SignedBeaconBlock, Slot} from "@lodestar/types";
+import {sszTypesFor} from "@lodestar/types";
+import {LodestarError, toRootHex} from "@lodestar/utils";
+import type {BidIdentity, BidLedger, BidLedgerRecord} from "./bidLedger.js";
+
+export type ObservedPostGloasBlock = {
+  blockRoot: RootHex;
+  slot: Slot;
+  version: ForkPostGloas;
+  block: SignedBeaconBlock<ForkPostGloas>;
+};
+
+export type BidSelectorModules = {
+  config: ChainForkConfig;
+  ledger: BidLedger;
+  builderIndex: BuilderIndex;
+};
+
+export enum BidSelectionIgnoreReason {
+  FOREIGN_BUILDER = "foreign_builder",
+}
+
+export type BidSelectionResult =
+  | {
+      status: "selected";
+      blockRoot: RootHex;
+      bid: BidLedgerRecord;
+    }
+  | {
+      status: "ignored";
+      reason: BidSelectionIgnoreReason;
+    };
+
+export enum BidSelectorErrorCode {
+  BLOCK_ROOT_MISMATCH = "BID_SELECTOR_ERROR_BLOCK_ROOT_MISMATCH",
+  BLOCK_SLOT_MISMATCH = "BID_SELECTOR_ERROR_BLOCK_SLOT_MISMATCH",
+  BLOCK_FORK_MISMATCH = "BID_SELECTOR_ERROR_BLOCK_FORK_MISMATCH",
+  BID_SLOT_MISMATCH = "BID_SELECTOR_ERROR_BID_SLOT_MISMATCH",
+  UNKNOWN_BID = "BID_SELECTOR_ERROR_UNKNOWN_BID",
+}
+
+export type BidSelectorErrorType =
+  | {
+      code: BidSelectorErrorCode.BLOCK_ROOT_MISMATCH;
+      blockRoot: RootHex;
+      computedBlockRoot: RootHex;
+    }
+  | {
+      code: BidSelectorErrorCode.BLOCK_SLOT_MISMATCH;
+      slot: Slot;
+      blockSlot: Slot;
+    }
+  | {
+      code: BidSelectorErrorCode.BLOCK_FORK_MISMATCH;
+      version: ForkPostGloas;
+      blockFork: string;
+    }
+  | {
+      code: BidSelectorErrorCode.BID_SLOT_MISMATCH;
+      slot: Slot;
+      bidSlot: Slot;
+    }
+  | {
+      code: BidSelectorErrorCode.UNKNOWN_BID;
+      slot: Slot;
+      parentBlockHash: RootHex;
+      parentBlockRoot: RootHex;
+      blockHash: RootHex;
+      signedBidRoot: RootHex;
+    };
+
+export class BidSelectorError extends LodestarError<BidSelectorErrorType> {}
+
+/** Matches an imported post-Gloas block to an exact local bid and records the win. */
+export class BidSelector {
+  constructor(private readonly modules: BidSelectorModules) {}
+
+  match(observed: ObservedPostGloasBlock): BidSelectionResult {
+    const {block, blockRoot, slot, version} = observed;
+    const {builderIndex, config, ledger} = this.modules;
+    const blockSlot = block.message.slot;
+
+    if (blockSlot !== slot) {
+      throw new BidSelectorError(
+        {code: BidSelectorErrorCode.BLOCK_SLOT_MISMATCH, slot, blockSlot},
+        `Observed block slot does not match fetched block slot=${slot} blockSlot=${blockSlot}`
+      );
+    }
+
+    const blockFork = config.getForkName(slot);
+    if (blockFork !== version) {
+      throw new BidSelectorError(
+        {code: BidSelectorErrorCode.BLOCK_FORK_MISMATCH, version, blockFork},
+        `Observed block fork does not match configured fork version=${version} blockFork=${blockFork}`
+      );
+    }
+
+    const computedBlockRoot = toRootHex(config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message));
+    if (computedBlockRoot !== blockRoot) {
+      throw new BidSelectorError(
+        {code: BidSelectorErrorCode.BLOCK_ROOT_MISMATCH, blockRoot, computedBlockRoot},
+        `Observed block root does not match fetched block blockRoot=${blockRoot} computedBlockRoot=${computedBlockRoot}`
+      );
+    }
+
+    const bid = block.message.body.signedExecutionPayloadBid.message;
+    if (bid.slot !== slot) {
+      throw new BidSelectorError(
+        {code: BidSelectorErrorCode.BID_SLOT_MISMATCH, slot, bidSlot: bid.slot},
+        `Selected bid slot does not match block slot=${slot} bidSlot=${bid.slot}`
+      );
+    }
+    if (bid.builderIndex !== builderIndex) {
+      return {status: "ignored", reason: BidSelectionIgnoreReason.FOREIGN_BUILDER};
+    }
+
+    const identity: BidIdentity = {
+      slot,
+      parentBlockHash: toRootHex(bid.parentBlockHash),
+      parentBlockRoot: toRootHex(bid.parentBlockRoot),
+      blockHash: toRootHex(bid.blockHash),
+    };
+    const signedBidRoot = toRootHex(
+      sszTypesFor(version, "SignedExecutionPayloadBid").hashTreeRoot(block.message.body.signedExecutionPayloadBid)
+    );
+    const localBid = ledger.recordWin({...identity, signedBidRoot}, blockRoot);
+    if (localBid === null) {
+      throw new BidSelectorError(
+        {code: BidSelectorErrorCode.UNKNOWN_BID, ...identity, signedBidRoot},
+        `Selected bid is not in the local ledger slot=${slot} blockHash=${identity.blockHash} signedBidRoot=${signedBidRoot}`
+      );
+    }
+
+    return {status: "selected", blockRoot, bid: localBid};
+  }
+}

package/src/services/blockObserver.ts

@@ -47,55 +47,17 @@ export class BlockObserver {
     this.retries = retries;
     this.retryDelay = retryDelay;
     this.maxSeenBlockRoots = maxSeenBlockRoots;
+    this.logger.info("Block observer initialized", {retries, retryDelay, maxSeenBlockRoots});
   }
 
   /**
-   * Register a block consumer. Consumers that require complete observation must register before {@link start}, because
+   * Register a block consumer. Consumers that require complete observation must register before event delivery, because
    * roots observed before registration remain consumed until normal FIFO eviction.
    */
   runOnBlock(fn: RunOnBlockFn): void {
     this.fns.push(fn);
   }
 
-  start(signal: AbortSignal): void {
-    this.logger.info("Subscribing to block events", {
-      retries: this.retries,
-      retryDelay: this.retryDelay,
-      maxSeenBlockRoots: this.maxSeenBlockRoots,
-    });
-
-    this.api.events
-      .eventstream({
-        topics: [EventType.block],
-        signal,
-        onEvent: (event) => {
-          if (event.type !== EventType.block) {
-            this.logger.debug("Ignoring unexpected beacon event", {eventType: event.type});
-            return;
-          }
-
-          void this.processBlockEvent(event.message, signal);
-        },
-        onError: (error) => {
-          this.logger.error("Failed to receive block event", {}, error);
-        },
-        onClose: () => {
-          if (signal.aborted) {
-            this.logger.debug("Closed stream for block events");
-          } else {
-            this.logger.error("Block event stream closed unexpectedly", {});
-          }
-        },
-      })
-      .catch((error: unknown) => {
-        this.logger.error(
-          "Failed to subscribe to block events",
-          {},
-          error instanceof Error ? error : Error(String(error))
-        );
-      });
-  }
-
   async processBlockEvent(event: BlockEvent, signal: AbortSignal): Promise<void> {
     try {
       const {slot, block: blockRoot, executionOptimistic} = event;
@@ -145,7 +107,7 @@ export class BlockObserver {
         return null;
       });
 
-      if (response === null) {
+      if (response === null || signal.aborted) {
         return;
       }
 

package/src/services/envelopePublisher.ts

@@ -0,0 +1,154 @@
+import {type ApiClient, routes} from "@lodestar/api";
+import {type BuilderIndex, type RootHex, type gloas, ssz} from "@lodestar/types";
+import {LodestarError, defer, toRootHex} from "@lodestar/utils";
+import type {BidIdentity, BidLedger} from "./bidLedger.js";
+import type {BuilderSigner} from "./builderSigner.js";
+import type {ExecutionPayloadEnvelopeContents} from "./executionPayloadEnvelope.js";
+
+export type EnvelopeSelectionIdentity = BidIdentity & {blockRoot: RootHex};
+
+export type EnvelopePublisherModules = {
+  api: ApiClient;
+  signer: BuilderSigner;
+  ledger: BidLedger;
+  builderIndex: BuilderIndex;
+};
+
+export enum EnvelopePublisherErrorCode {
+  BUILDER_INDEX_MISMATCH = "ENVELOPE_PUBLISHER_ERROR_BUILDER_INDEX_MISMATCH",
+  SELECTION_NOT_RECORDED = "ENVELOPE_PUBLISHER_ERROR_SELECTION_NOT_RECORDED",
+}
+
+export type EnvelopePublisherErrorType =
+  | {
+      code: EnvelopePublisherErrorCode.BUILDER_INDEX_MISMATCH;
+      builderIndex: BuilderIndex;
+      envelopeBuilderIndex: BuilderIndex;
+    }
+  | ({code: EnvelopePublisherErrorCode.SELECTION_NOT_RECORDED} & EnvelopeSelectionIdentity);
+
+export class EnvelopePublisherError extends LodestarError<EnvelopePublisherErrorType> {}
+
+export type EnvelopePublicationResult =
+  | {status: "published"; signedEnvelope: gloas.SignedExecutionPayloadEnvelope}
+  | {status: "duplicate"};
+
+type ActivePublication = {
+  controller: AbortController;
+  promise: Promise<EnvelopePublicationResult>;
+  waiters: number;
+};
+
+/** Signs and submits an envelope for a recorded local selection. */
+export class EnvelopePublisher {
+  private readonly activePublications = new Map<RootHex, ActivePublication>();
+
+  constructor(private readonly modules: EnvelopePublisherModules) {}
+
+  async publish(contents: ExecutionPayloadEnvelopeContents, signal: AbortSignal): Promise<EnvelopePublicationResult> {
+    signal.throwIfAborted();
+
+    const {api, builderIndex, ledger, signer} = this.modules;
+    const {envelope} = contents;
+    if (envelope.builderIndex !== builderIndex) {
+      throw new EnvelopePublisherError(
+        {
+          code: EnvelopePublisherErrorCode.BUILDER_INDEX_MISMATCH,
+          builderIndex,
+          envelopeBuilderIndex: envelope.builderIndex,
+        },
+        `Envelope builder index does not match local builder index builderIndex=${builderIndex} envelopeBuilderIndex=${envelope.builderIndex}`
+      );
+    }
+
+    const identity: EnvelopeSelectionIdentity = {
+      slot: envelope.payload.slotNumber,
+      parentBlockHash: toRootHex(envelope.payload.parentHash),
+      parentBlockRoot: toRootHex(envelope.parentBeaconBlockRoot),
+      blockHash: toRootHex(envelope.payload.blockHash),
+      blockRoot: toRootHex(envelope.beaconBlockRoot),
+    };
+    if (!ledger.hasWon(identity, identity.blockRoot)) {
+      throw new EnvelopePublisherError(
+        {code: EnvelopePublisherErrorCode.SELECTION_NOT_RECORDED, ...identity},
+        `Envelope selection is not recorded slot=${identity.slot} blockRoot=${identity.blockRoot} blockHash=${identity.blockHash}`
+      );
+    }
+
+    const envelopeRoot = toRootHex(ssz.gloas.ExecutionPayloadEnvelope.hashTreeRoot(envelope));
+    ledger.recordReveal(identity.slot, identity.blockRoot, identity.blockHash, envelopeRoot);
+    if (ledger.hasPublishedReveal(identity.blockRoot)) {
+      return {status: "duplicate"};
+    }
+
+    let publication = this.activePublications.get(identity.blockRoot);
+    if (publication === undefined) {
+      const controller = new AbortController();
+      publication = {
+        controller,
+        waiters: 0,
+        promise: this.publishEnvelope(contents, identity, envelopeRoot, api, ledger, signer, controller.signal).finally(
+          () => {
+            if (this.activePublications.get(identity.blockRoot)?.controller === controller) {
+              this.activePublications.delete(identity.blockRoot);
+            }
+          }
+        ),
+      };
+      this.activePublications.set(identity.blockRoot, publication);
+    }
+    return this.waitForPublication(identity.blockRoot, publication, signal);
+  }
+
+  private async waitForPublication(
+    blockRoot: RootHex,
+    publication: ActivePublication,
+    signal: AbortSignal
+  ): Promise<EnvelopePublicationResult> {
+    const aborted = defer<never>();
+    const onAbort = (): void => aborted.reject(signal.reason);
+    publication.waiters++;
+    signal.addEventListener("abort", onAbort, {once: true});
+    if (signal.aborted) onAbort();
+
+    try {
+      return await Promise.race([publication.promise, aborted.promise]);
+    } finally {
+      signal.removeEventListener("abort", onAbort);
+      publication.waiters--;
+      if (publication.waiters === 0 && signal.aborted) {
+        if (this.activePublications.get(blockRoot) === publication) {
+          this.activePublications.delete(blockRoot);
+        }
+        publication.controller.abort(signal.reason);
+      }
+    }
+  }
+
+  private async publishEnvelope(
+    contents: ExecutionPayloadEnvelopeContents,
+    identity: EnvelopeSelectionIdentity,
+    envelopeRoot: RootHex,
+    api: ApiClient,
+    ledger: BidLedger,
+    signer: BuilderSigner,
+    signal: AbortSignal
+  ): Promise<EnvelopePublicationResult> {
+    const signedEnvelope = signer.signExecutionPayloadEnvelope(contents.envelope);
+
+    const response = await api.beacon.publishExecutionPayloadEnvelope(
+      {
+        signedEnvelopeOrContents: {
+          signedExecutionPayloadEnvelope: signedEnvelope,
+          kzgProofs: contents.kzgProofs,
+          blobs: contents.blobs,
+        },
+        broadcastValidation: routes.beacon.BroadcastValidation.gossip,
+      },
+      {signal}
+    );
+    response.assertOk();
+    ledger.recordRevealPublished(identity.slot, identity.blockRoot, identity.blockHash, envelopeRoot);
+    return {status: "published", signedEnvelope};
+  }
+}

package/src/services/executionPayloadEnvelope.ts

@@ -0,0 +1,156 @@
+import type {BuilderIndex, RootHex, Slot, deneb, gloas} from "@lodestar/types";
+import {ssz} from "@lodestar/types";
+import {LodestarError, byteArrayEquals, fromHex, toRootHex} from "@lodestar/utils";
+import type {StoredPayload} from "./payloadStore.js";
+
+export type ExecutionPayloadEnvelopeInput = {
+  blockRoot: RootHex;
+  builderIndex: BuilderIndex;
+  selectedBid: gloas.ExecutionPayloadBid;
+  storedPayload: StoredPayload;
+};
+
+export type ExecutionPayloadEnvelopeContents = {
+  envelope: gloas.ExecutionPayloadEnvelope;
+  kzgProofs: deneb.KZGProofs;
+  blobs: deneb.Blobs;
+};
+
+export enum ExecutionPayloadEnvelopeErrorCode {
+  SLOT_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_SLOT_MISMATCH",
+  PARENT_BLOCK_ROOT_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_PARENT_BLOCK_ROOT_MISMATCH",
+  PARENT_BLOCK_HASH_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_PARENT_BLOCK_HASH_MISMATCH",
+  BLOCK_HASH_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BLOCK_HASH_MISMATCH",
+  BUILDER_INDEX_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BUILDER_INDEX_MISMATCH",
+  BLOB_KZG_COMMITMENTS_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BLOB_KZG_COMMITMENTS_MISMATCH",
+  EXECUTION_REQUESTS_ROOT_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_EXECUTION_REQUESTS_ROOT_MISMATCH",
+}
+
+export type ExecutionPayloadEnvelopeErrorType =
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.BUILDER_INDEX_MISMATCH;
+      builderIndex: BuilderIndex;
+      bidBuilderIndex: BuilderIndex;
+    }
+  | {code: ExecutionPayloadEnvelopeErrorCode.BLOB_KZG_COMMITMENTS_MISMATCH}
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.EXECUTION_REQUESTS_ROOT_MISMATCH;
+      bidExecutionRequestsRoot: RootHex;
+      payloadExecutionRequestsRoot: RootHex;
+    }
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.PARENT_BLOCK_ROOT_MISMATCH;
+      bidParentBlockRoot: RootHex;
+      storedParentBlockRoot: RootHex;
+    }
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.SLOT_MISMATCH;
+      bidSlot: Slot;
+      payloadSlot: Slot;
+    }
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.PARENT_BLOCK_HASH_MISMATCH;
+      bidParentBlockHash: RootHex;
+      payloadParentBlockHash: RootHex;
+    }
+  | {
+      code: ExecutionPayloadEnvelopeErrorCode.BLOCK_HASH_MISMATCH;
+      bidBlockHash: RootHex;
+      payloadBlockHash: RootHex;
+    };
+
+export class ExecutionPayloadEnvelopeError extends LodestarError<ExecutionPayloadEnvelopeErrorType> {}
+
+/** The caller must have matched the selected bid to its local ledger record. */
+export function createExecutionPayloadEnvelopeContents({
+  blockRoot,
+  builderIndex,
+  selectedBid,
+  storedPayload,
+}: ExecutionPayloadEnvelopeInput): ExecutionPayloadEnvelopeContents {
+  if (builderIndex !== selectedBid.builderIndex) {
+    throw new ExecutionPayloadEnvelopeError({
+      code: ExecutionPayloadEnvelopeErrorCode.BUILDER_INDEX_MISMATCH,
+      builderIndex,
+      bidBuilderIndex: selectedBid.builderIndex,
+    });
+  }
+
+  const bidParentBlockRoot = toRootHex(selectedBid.parentBlockRoot);
+  const storedParentBlockRoot = toRootHex(storedPayload.parentBlockRoot);
+  if (storedParentBlockRoot !== bidParentBlockRoot) {
+    throw new ExecutionPayloadEnvelopeError(
+      {
+        code: ExecutionPayloadEnvelopeErrorCode.PARENT_BLOCK_ROOT_MISMATCH,
+        bidParentBlockRoot,
+        storedParentBlockRoot,
+      },
+      `Selected bid beacon parent does not match retained payload bidParentBlockRoot=${bidParentBlockRoot} storedParentBlockRoot=${storedParentBlockRoot}`
+    );
+  }
+
+  const {payload} = storedPayload;
+  const payloadSlot = payload.executionPayload.slotNumber;
+  if (payloadSlot !== selectedBid.slot) {
+    throw new ExecutionPayloadEnvelopeError(
+      {code: ExecutionPayloadEnvelopeErrorCode.SLOT_MISMATCH, bidSlot: selectedBid.slot, payloadSlot},
+      `Selected bid slot does not match payload slot bidSlot=${selectedBid.slot} payloadSlot=${payloadSlot}`
+    );
+  }
+
+  const payloadParentBlockHash = toRootHex(payload.executionPayload.parentHash);
+  const bidParentBlockHash = toRootHex(selectedBid.parentBlockHash);
+  if (payloadParentBlockHash !== bidParentBlockHash) {
+    throw new ExecutionPayloadEnvelopeError(
+      {
+        code: ExecutionPayloadEnvelopeErrorCode.PARENT_BLOCK_HASH_MISMATCH,
+        bidParentBlockHash,
+        payloadParentBlockHash,
+      },
+      `Selected bid parent does not match payload parent bidParentBlockHash=${bidParentBlockHash} payloadParentBlockHash=${payloadParentBlockHash}`
+    );
+  }
+
+  const payloadBlockHash = toRootHex(payload.executionPayload.blockHash);
+  const bidBlockHash = toRootHex(selectedBid.blockHash);
+  if (payloadBlockHash !== bidBlockHash) {
+    throw new ExecutionPayloadEnvelopeError(
+      {
+        code: ExecutionPayloadEnvelopeErrorCode.BLOCK_HASH_MISMATCH,
+        bidBlockHash,
+        payloadBlockHash,
+      },
+      `Selected bid block hash does not match payload bidBlockHash=${bidBlockHash} payloadBlockHash=${payloadBlockHash}`
+    );
+  }
+
+  if (
+    !ssz.gloas.ExecutionPayloadBid.fields.blobKzgCommitments.equals(
+      selectedBid.blobKzgCommitments,
+      payload.blobsBundle.commitments
+    )
+  ) {
+    throw new ExecutionPayloadEnvelopeError({code: ExecutionPayloadEnvelopeErrorCode.BLOB_KZG_COMMITMENTS_MISMATCH});
+  }
+
+  const executionRequestsRoot = ssz.gloas.ExecutionRequests.hashTreeRoot(payload.executionRequests);
+  if (!byteArrayEquals(executionRequestsRoot, selectedBid.executionRequestsRoot)) {
+    throw new ExecutionPayloadEnvelopeError({
+      code: ExecutionPayloadEnvelopeErrorCode.EXECUTION_REQUESTS_ROOT_MISMATCH,
+      bidExecutionRequestsRoot: toRootHex(selectedBid.executionRequestsRoot),
+      payloadExecutionRequestsRoot: toRootHex(executionRequestsRoot),
+    });
+  }
+
+  return {
+    envelope: {
+      payload: payload.executionPayload,
+      executionRequests: payload.executionRequests,
+      builderIndex,
+      beaconBlockRoot: fromHex(blockRoot),
+      parentBeaconBlockRoot: selectedBid.parentBlockRoot,
+    },
+    kzgProofs: payload.blobsBundle.proofs,
+    blobs: payload.blobsBundle.blobs,
+  };
+}

package/src/services/payloadOrchestrator.ts

@@ -0,0 +1,107 @@
+import {LodestarError, TimeoutError, retry, sleep, withTimeout} from "@lodestar/utils";
+import {
+  type BuildRequest,
+  type BuiltPayload,
+  type PayloadSource,
+  PayloadSourceError,
+  PayloadSourceErrorCode,
+} from "./payloadSource.js";
+
+const PREPARE_RETRY_DELAY = 100;
+
+export type PayloadBuildJob = {
+  /** Jobs with the same ID share one build and its result. */
+  id: string;
+  request: BuildRequest;
+  /** Unix timestamp in milliseconds at which the prepared payload should be retrieved. */
+  getPayloadAt: number;
+};
+
+export type PayloadOrchestratorOptions = {
+  /** Maximum time in milliseconds to wait for payload retrieval. */
+  getPayloadTimeout: number;
+};
+
+export enum PayloadOrchestratorErrorCode {
+  PREPARE_DEADLINE_REACHED = "PAYLOAD_ORCHESTRATOR_ERROR_PREPARE_DEADLINE_REACHED",
+  PREPARE_TIMEOUT = "PAYLOAD_ORCHESTRATOR_ERROR_PREPARE_TIMEOUT",
+  GET_PAYLOAD_TIMEOUT = "PAYLOAD_ORCHESTRATOR_ERROR_GET_PAYLOAD_TIMEOUT",
+}
+
+export type PayloadOrchestratorErrorType =
+  | {
+      code: PayloadOrchestratorErrorCode.PREPARE_DEADLINE_REACHED;
+      jobId: string;
+      getPayloadAt: number;
+    }
+  | {
+      code: PayloadOrchestratorErrorCode.PREPARE_TIMEOUT | PayloadOrchestratorErrorCode.GET_PAYLOAD_TIMEOUT;
+      jobId: string;
+    };
+
+export class PayloadOrchestratorError extends LodestarError<PayloadOrchestratorErrorType> {}
+
+/**
+ * Prepares a payload and retrieves it at the requested time. Preparation retries missing payload IDs until the
+ * retrieval time. Transport-level retries remain owned by the source.
+ */
+export class PayloadOrchestrator {
+  private readonly jobs = new Map<string, Promise<BuiltPayload>>();
+
+  constructor(
+    private readonly source: PayloadSource,
+    private readonly options: PayloadOrchestratorOptions,
+    private readonly signal: AbortSignal
+  ) {}
+
+  run(job: PayloadBuildJob): Promise<BuiltPayload> {
+    const {id} = job;
+    let promise = this.jobs.get(id);
+    if (promise === undefined) {
+      promise = this.runJob(job).finally(() => this.jobs.delete(id));
+      this.jobs.set(id, promise);
+    }
+    return promise;
+  }
+
+  private async runJob({id, request, getPayloadAt}: PayloadBuildJob): Promise<BuiltPayload> {
+    const prepareTimeout = getPayloadAt - Date.now();
+    if (!(prepareTimeout > 0)) {
+      throw new PayloadOrchestratorError(
+        {code: PayloadOrchestratorErrorCode.PREPARE_DEADLINE_REACHED, jobId: id, getPayloadAt},
+        `Payload preparation deadline reached jobId=${id} getPayloadAt=${getPayloadAt}`
+      );
+    }
+
+    const handle = await withTimeout(
+      (signal = this.signal) =>
+        retry(() => this.source.prepare(request, signal), {
+          retries: Infinity,
+          retryDelay: PREPARE_RETRY_DELAY,
+          shouldRetry: (error) =>
+            error instanceof PayloadSourceError && error.type.code === PayloadSourceErrorCode.NO_PAYLOAD_ID,
+          signal,
+        }),
+      prepareTimeout,
+      this.signal
+    ).catch((error: unknown) => {
+      if (error instanceof TimeoutError) {
+        throw new PayloadOrchestratorError({code: PayloadOrchestratorErrorCode.PREPARE_TIMEOUT, jobId: id});
+      }
+      throw error;
+    });
+
+    await sleep(getPayloadAt - Date.now(), this.signal);
+
+    return withTimeout(
+      (signal = this.signal) => this.source.getPayload(handle, signal),
+      this.options.getPayloadTimeout,
+      this.signal
+    ).catch((error: unknown) => {
+      if (error instanceof TimeoutError) {
+        throw new PayloadOrchestratorError({code: PayloadOrchestratorErrorCode.GET_PAYLOAD_TIMEOUT, jobId: id});
+      }
+      throw error;
+    });
+  }
+}

package/src/services/proposerPreferencesTracker.ts

@@ -1,6 +1,5 @@
-import {ApiClient, routes} from "@lodestar/api";
 import type {RootHex, Slot, gloas} from "@lodestar/types";
-import {Logger, MapDef, toRootHex} from "@lodestar/utils";
+import {MapDef, toRootHex} from "@lodestar/utils";
 
 /** Retains validated proposer preferences by the branch-specific identity used for bid validation. */
 export class ProposerPreferencesTracker {
@@ -8,40 +7,6 @@ export class ProposerPreferencesTracker {
     () => new Map()
   );
 
-  constructor(
-    private readonly api: ApiClient,
-    private readonly logger: Logger
-  ) {}
-
-  start(signal: AbortSignal): void {
-    if (signal.aborted) return;
-
-    this.logger.verbose("Subscribing to proposer preferences");
-    this.api.events
-      .eventstream({
-        topics: [routes.events.EventType.proposerPreferences],
-        signal,
-        onEvent: (event) => {
-          if (!signal.aborted && event.type === routes.events.EventType.proposerPreferences) {
-            this.onProposerPreferences(event.message.data);
-          }
-        },
-        onError: (error) => {
-          if (!signal.aborted) this.logger.error("Failed to receive proposer preferences", {}, error);
-        },
-        onClose: () => {
-          if (signal.aborted) {
-            this.logger.verbose("Closed proposer preferences stream");
-          } else {
-            this.logger.error("Proposer preferences stream closed unexpectedly", {});
-          }
-        },
-      })
-      .catch((error: Error) => {
-        if (!signal.aborted) this.logger.error("Failed to subscribe to proposer preferences", {}, error);
-      });
-  }
-
   onProposerPreferences(signedProposerPreferences: gloas.SignedProposerPreferences): boolean {
     const {proposalSlot, dependentRoot} = signedProposerPreferences.message;
     const dependentRootHex = toRootHex(dependentRoot);