@lodestar/beacon-node 1.50.0-dev.c04a499311 → 1.50.0-dev.c24762bc60

package/src/execution/engine/index.ts

@@ -1,17 +1,24 @@
-import {fromHex, toPrintableUrl} from "@lodestar/utils";
+import {LodestarError, fromHex, toPrintableUrl} from "@lodestar/utils";
+import {getLodestarClientVersion} from "../../util/metadata.js";
 import {ExecutionEngineDisabled} from "./disabled.js";
 import {
+  EngineApiMode,
   ExecutionEngineHttp,
   ExecutionEngineHttpOpts,
   ExecutionEngineModules,
   defaultExecutionEngineHttpOpts,
+  engineApiModes,
 } from "./http.js";
 import {IExecutionEngine} from "./interface.js";
 import {JsonRpcHttpClient} from "./jsonRpcHttpClient.js";
+import {JsonRpcEngineTransport} from "./jsonRpcTransport.js";
 import {ExecutionEngineMockBackend, ExecutionEngineMockOpts} from "./mock.js";
+import {EngineRestHttpClient} from "./restHttpClient.js";
+import {RestEngineTransport, formatClientVersionHeader} from "./restTransport.js";
 import {ExecutionEngineMockJsonRpcClient, JsonRpcBackend} from "./utils.js";
 
-export {ExecutionEngineHttp, ExecutionEngineDisabled, defaultExecutionEngineHttpOpts};
+export {ExecutionEngineHttp, ExecutionEngineDisabled, defaultExecutionEngineHttpOpts, engineApiModes};
+export type {EngineApiMode};
 
 export type ExecutionEngineOpts =
   | ({mode?: "http"} & ExecutionEngineHttpOpts)
@@ -24,23 +31,50 @@ export function getExecutionEngineFromBackend(
   modules: ExecutionEngineModules
 ): IExecutionEngine {
   const rpc = new ExecutionEngineMockJsonRpcClient(backend);
-  return new ExecutionEngineHttp(rpc, modules);
+  return new ExecutionEngineHttp({jsonRpc: new JsonRpcEngineTransport(rpc)}, modules);
 }
 
 export function getExecutionEngineHttp(
   opts: ExecutionEngineHttpOpts,
   modules: ExecutionEngineModules
 ): IExecutionEngine {
+  const jwtSecret = opts.jwtSecretHex ? fromHex(opts.jwtSecretHex) : undefined;
+  const metrics = modules.metrics?.executionEnginerHttpClient;
+  const engineApi = opts.engineApi ?? "auto";
+  if (engineApi === "ssz" && opts.urls.length !== 1) {
+    throw new LodestarError({code: "ENGINE_REST_REQUIRES_SINGLE_URL", count: opts.urls.length});
+  }
+  if (engineApi === "auto" && opts.urls.length > 1) {
+    modules.logger.debug("Using JSON-RPC for multiple execution URLs", {count: opts.urls.length});
+  }
+
   const rpc = new JsonRpcHttpClient(opts.urls, {
     ...opts,
     signal: modules.signal,
-    metrics: modules.metrics?.executionEnginerHttpClient,
-    jwtSecret: opts.jwtSecretHex ? fromHex(opts.jwtSecretHex) : undefined,
+    metrics,
+    jwtSecret,
     jwtId: opts.jwtId,
     jwtVersion: opts.jwtVersion,
   });
-  modules.logger.info("Execution client", {urls: opts.urls.map(toPrintableUrl).toString()});
-  return new ExecutionEngineHttp(rpc, modules, opts);
+
+  const rest =
+    engineApi === "json-rpc" || opts.urls.length !== 1
+      ? undefined
+      : new RestEngineTransport(
+          new EngineRestHttpClient(opts.urls, {
+            signal: modules.signal,
+            timeout: opts.timeout,
+            retries: opts.retries,
+            retryDelay: opts.retryDelay,
+            jwtSecret,
+            jwtId: opts.jwtId,
+            clientVersion: formatClientVersionHeader(getLodestarClientVersion(opts)),
+            metrics,
+          })
+        );
+
+  modules.logger.info("Execution client", {urls: opts.urls.map(toPrintableUrl).toString(), engineApi});
+  return new ExecutionEngineHttp({jsonRpc: new JsonRpcEngineTransport(rpc), rest}, modules, opts);
 }
 
 export function initializeExecutionEngine(

package/src/execution/engine/interface.ts

@@ -170,7 +170,7 @@ export interface IExecutionEngine {
     shouldOverrideBuilder?: boolean;
   }>;
 
-  /** Amsterdam: payload bodies including the block access list (null if the EL no longer has it) */
+  /** Amsterdam: includes the block access list; REST returns null for bodies outside the Amsterdam era. */
   getPayloadBodiesByHashV2(blockHash: DATA[]): Promise<(ExecutionPayloadBodyV2 | null)[]>;
 
   getBlobs(

package/src/execution/engine/jsonRpcTransport.ts

@@ -0,0 +1,247 @@
+import {ForkName, ForkSeq} from "@lodestar/params";
+import {ExecutionPayload, ExecutionRequests, Root, RootHex} from "@lodestar/types";
+import {BlobAndProof} from "@lodestar/types/deneb";
+import {BlobAndProofV2} from "@lodestar/types/fulu";
+import {strip0xPrefix} from "@lodestar/utils";
+import {ClientCode, ClientVersion, PayloadAttributes, PayloadId, VersionedHashes} from "./interface.js";
+import {IJsonRpcHttpClient, JsonRpcHttpClientEventEmitter, ReqOpts} from "./jsonRpcHttpClient.js";
+import {ForkchoiceUpdatedResult, GetPayloadResult, IEngineTransport, PayloadStatusResult} from "./transport.js";
+import {
+  BLOB_AND_PROOF_V2_RPC_BYTES,
+  EngineApiRpcParamTypes,
+  EngineApiRpcReturnTypes,
+  ExecutionPayloadBodyV2,
+  assertReqSizeLimit,
+  deserializeBlobAndProofs,
+  deserializeBlobAndProofsV2,
+  deserializeBlobAndProofsV2IntoBytes,
+  deserializeExecutionPayloadBodyV2,
+  parseExecutionPayload,
+  serializeBeaconBlockRoot,
+  serializeExecutionPayload,
+  serializeExecutionRequests,
+  serializePayloadAttributes,
+  serializeVersionedHashes,
+} from "./types.js";
+import {bytesToData} from "./utils.js";
+
+/**
+ * Maximum number of version hashes that can be sent in a getBlobs request
+ * Clients must support at least 128 versionedHashes, so we avoid sending more
+ * https://github.com/ethereum/execution-apis/blob/main/src/engine/cancun.md#specification-3
+ */
+const MAX_VERSIONED_HASHES = 128;
+
+// Define static options once to prevent extra allocations
+const notifyNewPayloadOpts: ReqOpts = {routeId: "notifyNewPayload"};
+const forkchoiceUpdatedOpts: ReqOpts = {routeId: "forkchoiceUpdated"};
+const getPayloadOpts: ReqOpts = {routeId: "getPayload"};
+const getPayloadBodiesByHashOpts: ReqOpts = {routeId: "getPayloadBodiesByHash"};
+const getBlobsV1Opts: ReqOpts = {routeId: "getBlobsV1"};
+const getBlobsV2Opts: ReqOpts = {routeId: "getBlobsV2"};
+const getClientVersionOpts: ReqOpts = {routeId: "getClientVersion"};
+
+/**
+ * Legacy `engine_*` JSON-RPC methods
+ * https://github.com/ethereum/execution-apis/tree/main/src/engine
+ */
+export class JsonRpcEngineTransport implements IEngineTransport {
+  constructor(private readonly rpc: IJsonRpcHttpClient) {}
+
+  get emitter(): JsonRpcHttpClientEventEmitter {
+    return this.rpc.emitter;
+  }
+
+  async newPayload(
+    fork: ForkName,
+    executionPayload: ExecutionPayload,
+    versionedHashes?: VersionedHashes,
+    parentBeaconBlockRoot?: Root,
+    executionRequests?: ExecutionRequests
+  ): Promise<PayloadStatusResult> {
+    const serializedExecutionPayload = serializeExecutionPayload(fork, executionPayload);
+
+    if (ForkSeq[fork] >= ForkSeq.deneb) {
+      if (versionedHashes === undefined) {
+        throw Error(`versionedHashes required in notifyNewPayload for fork=${fork}`);
+      }
+      if (parentBeaconBlockRoot === undefined) {
+        throw Error(`parentBlockRoot required in notifyNewPayload for fork=${fork}`);
+      }
+
+      const serializedVersionedHashes = serializeVersionedHashes(versionedHashes);
+      const serializedParentBeaconBlockRoot = serializeBeaconBlockRoot(parentBeaconBlockRoot);
+
+      if (ForkSeq[fork] >= ForkSeq.electra) {
+        if (executionRequests === undefined) {
+          throw Error(`executionRequests required in notifyNewPayload for fork=${fork}`);
+        }
+        const method = ForkSeq[fork] >= ForkSeq.gloas ? "engine_newPayloadV5" : "engine_newPayloadV4";
+        return this.rpc.fetchWithRetries<EngineApiRpcReturnTypes[typeof method], EngineApiRpcParamTypes[typeof method]>(
+          {
+            method,
+            params: [
+              serializedExecutionPayload,
+              serializedVersionedHashes,
+              serializedParentBeaconBlockRoot,
+              serializeExecutionRequests(fork, executionRequests),
+            ],
+          },
+          notifyNewPayloadOpts
+        );
+      }
+
+      const method = "engine_newPayloadV3";
+      return this.rpc.fetchWithRetries<EngineApiRpcReturnTypes[typeof method], EngineApiRpcParamTypes[typeof method]>(
+        {method, params: [serializedExecutionPayload, serializedVersionedHashes, serializedParentBeaconBlockRoot]},
+        notifyNewPayloadOpts
+      );
+    }
+
+    const method = ForkSeq[fork] >= ForkSeq.capella ? "engine_newPayloadV2" : "engine_newPayloadV1";
+    return this.rpc.fetchWithRetries<EngineApiRpcReturnTypes[typeof method], EngineApiRpcParamTypes[typeof method]>(
+      {method, params: [serializedExecutionPayload]},
+      notifyNewPayloadOpts
+    );
+  }
+
+  async forkchoiceUpdated(
+    fork: ForkName,
+    headBlockHash: RootHex,
+    safeBlockHash: RootHex,
+    finalizedBlockHash: RootHex,
+    payloadAttributes?: PayloadAttributes
+  ): Promise<ForkchoiceUpdatedResult> {
+    const method =
+      ForkSeq[fork] >= ForkSeq.gloas
+        ? "engine_forkchoiceUpdatedV4"
+        : ForkSeq[fork] >= ForkSeq.deneb
+          ? "engine_forkchoiceUpdatedV3"
+          : ForkSeq[fork] >= ForkSeq.capella
+            ? "engine_forkchoiceUpdatedV2"
+            : "engine_forkchoiceUpdatedV1";
+    const payloadAttributesRpc = payloadAttributes ? serializePayloadAttributes(payloadAttributes) : undefined;
+    // If we are just fcUing and not asking execution for payload, retry is not required
+    // and we can move on, as the next fcU will be issued soon on the new slot
+    const fcUReqOpts = payloadAttributes !== undefined ? forkchoiceUpdatedOpts : {...forkchoiceUpdatedOpts, retries: 0};
+
+    const {payloadStatus, payloadId} = await this.rpc.fetchWithRetries<
+      EngineApiRpcReturnTypes[typeof method],
+      EngineApiRpcParamTypes[typeof method]
+    >({method, params: [{headBlockHash, safeBlockHash, finalizedBlockHash}, payloadAttributesRpc]}, fcUReqOpts);
+
+    return {payloadStatus, payloadId: payloadId && payloadId !== "0x" ? payloadId : null};
+  }
+
+  async getPayload(fork: ForkName, payloadId: PayloadId): Promise<GetPayloadResult> {
+    let method: keyof EngineApiRpcReturnTypes;
+    switch (fork) {
+      case ForkName.phase0:
+      case ForkName.altair:
+      case ForkName.bellatrix:
+        method = "engine_getPayloadV1";
+        break;
+      case ForkName.capella:
+        method = "engine_getPayloadV2";
+        break;
+      case ForkName.deneb:
+        method = "engine_getPayloadV3";
+        break;
+      case ForkName.electra:
+        method = "engine_getPayloadV4";
+        break;
+      case ForkName.fulu:
+        method = "engine_getPayloadV5";
+        break;
+      default:
+        method = "engine_getPayloadV6";
+        break;
+    }
+    const payloadResponse = await this.rpc.fetchWithRetries<
+      EngineApiRpcReturnTypes[typeof method],
+      EngineApiRpcParamTypes[typeof method]
+    >({method, params: [payloadId]}, getPayloadOpts);
+    return parseExecutionPayload(fork, payloadResponse);
+  }
+
+  async getPayloadBodiesByHashV2(blockHashes: RootHex[]): Promise<(ExecutionPayloadBodyV2 | null)[]> {
+    const method = "engine_getPayloadBodiesByHashV2";
+    assertReqSizeLimit(blockHashes.length, 32);
+    const response = await this.rpc.fetchWithRetries<
+      EngineApiRpcReturnTypes[typeof method],
+      EngineApiRpcParamTypes[typeof method]
+    >({method, params: [blockHashes]}, getPayloadBodiesByHashOpts);
+    return response.map(deserializeExecutionPayloadBodyV2);
+  }
+
+  async getBlobsV1(versionedHashes: VersionedHashes): Promise<(BlobAndProof | null)[]> {
+    assertReqSizeLimit(versionedHashes.length, MAX_VERSIONED_HASHES);
+    const versionedHashesHex = versionedHashes.map(bytesToData);
+    const response = await this.rpc.fetchWithRetries<
+      EngineApiRpcReturnTypes["engine_getBlobsV1"],
+      EngineApiRpcParamTypes["engine_getBlobsV1"]
+    >({method: "engine_getBlobsV1", params: [versionedHashesHex]}, getBlobsV1Opts);
+
+    if (response.length !== versionedHashesHex.length) {
+      throw Error(
+        `Invalid engine_getBlobsV1 response length=${response.length} versionedHashes=${versionedHashesHex.length}`
+      );
+    }
+
+    return response.map(deserializeBlobAndProofs);
+  }
+
+  async getBlobsV2(versionedHashes: VersionedHashes, buffers?: Uint8Array[]): Promise<BlobAndProofV2[] | null> {
+    assertReqSizeLimit(versionedHashes.length, MAX_VERSIONED_HASHES);
+    const versionedHashesHex = versionedHashes.map(bytesToData);
+    if (buffers) {
+      // Callers preallocate one buffer per max blobs of the epoch, only the first entries are used
+      if (buffers.length < versionedHashesHex.length) {
+        throw Error(`Not enough buffers length=${buffers.length} versionedHashes=${versionedHashesHex.length}`);
+      }
+
+      for (const [i, buffer] of buffers.entries()) {
+        if (buffer.length !== BLOB_AND_PROOF_V2_RPC_BYTES) {
+          throw Error(`Invalid buffer[${i}] length=${buffer.length} expected=${BLOB_AND_PROOF_V2_RPC_BYTES}`);
+        }
+      }
+    }
+
+    const response = await this.rpc.fetchWithRetries<
+      EngineApiRpcReturnTypes["engine_getBlobsV2"],
+      EngineApiRpcParamTypes["engine_getBlobsV2"]
+    >({method: "engine_getBlobsV2", params: [versionedHashesHex]}, getBlobsV2Opts);
+
+    // engine_getBlobsV2 does not return partial responses. It returns null if any blob is not found
+    if (response == null) {
+      return null;
+    }
+
+    if (response.length !== versionedHashesHex.length) {
+      throw Error(
+        `Invalid engine_getBlobsV2 response length=${response.length} versionedHashes=${versionedHashesHex.length}`
+      );
+    }
+
+    if (buffers) {
+      // getBlobsV2() is designed to called once per slot so we expect to have buffers
+      return response.map((data, i) => deserializeBlobAndProofsV2IntoBytes(data, buffers[i]));
+    }
+
+    return response.map(deserializeBlobAndProofsV2);
+  }
+
+  async getClientVersion(clientVersion: ClientVersion): Promise<ClientVersion[]> {
+    const method = "engine_getClientVersionV1";
+
+    const response = await this.rpc.fetchWithRetries<
+      EngineApiRpcReturnTypes[typeof method],
+      EngineApiRpcParamTypes[typeof method]
+    >({method, params: [{...clientVersion, commit: `0x${clientVersion.commit}`}]}, getClientVersionOpts);
+
+    return response.map((cv) => {
+      const code = cv.code in ClientCode ? ClientCode[cv.code as keyof typeof ClientCode] : ClientCode.XX;
+      return {code, name: cv.name, version: cv.version, commit: strip0xPrefix(cv.commit)};
+    });
+  }
+}

package/src/execution/engine/restHttpClient.ts

@@ -0,0 +1,269 @@
+import {ErrorAborted, FetchError, LodestarError, TimeoutError, fetch, isValidHttpUrl, retry} from "@lodestar/utils";
+import {
+  JsonRpcHttpClientEvent,
+  JsonRpcHttpClientEventEmitter,
+  JsonRpcHttpClientMetrics,
+  ReqOpts,
+} from "./jsonRpcHttpClient.js";
+import {JwtClaim, encodeJwtToken} from "./jwt.js";
+import {ExecutionForkName} from "./sszTypes.js";
+
+export const ENGINE_REST_BASE_PATH = "/engine/v1";
+export const EXECUTION_VERSION_HEADER = "Eth-Execution-Version";
+export const CLIENT_VERSION_HEADER = "X-Engine-Client-Version";
+export const MEDIA_TYPE_SSZ = "application/octet-stream";
+export const MEDIA_TYPE_JSON = "application/json";
+
+const REQUEST_TIMEOUT = 30 * 1000;
+/** Limits the amount of error detail printed with REST errors */
+const MAX_ERROR_DETAIL_LENGTH = 500;
+
+export type EngineRestRequest = {
+  method: "GET" | "POST";
+  /** Path relative to the `/engine/v1` base path */
+  path: string;
+  query?: Record<string, string | number>;
+  /** Sets the `Eth-Execution-Version` header on fork-scoped endpoints */
+  executionFork?: ExecutionForkName;
+  /** SSZ encoded request body */
+  body?: Uint8Array;
+  responseType: "ssz" | "json";
+};
+
+export type EngineRestResponse = {
+  status: number;
+  /** Empty on `204 No Content` */
+  body: Uint8Array;
+};
+
+/** Non-2xx response, `type` and `detail` are taken from the problem+json body when present */
+export class EngineRestError extends Error {
+  constructor(
+    readonly status: number,
+    readonly type: string | null,
+    readonly detail: string | null,
+    routeId: string
+  ) {
+    super(
+      `Engine REST error: status=${status} type=${type ?? "unknown"}${detail ? ` detail=${detail}` : ""}, ${routeId}`
+    );
+  }
+}
+
+export class EngineRestResponseError extends LodestarError<{
+  code: "ENGINE_REST_INVALID_RESPONSE";
+  routeId: string;
+  reason: string;
+}> {
+  constructor(routeId: string, reason: string) {
+    super(
+      {code: "ENGINE_REST_INVALID_RESPONSE", routeId, reason},
+      `Invalid engine REST response: ${reason}, ${routeId}`
+    );
+  }
+}
+
+export type EngineRestHttpClientOpts = {
+  signal?: AbortSignal;
+  timeout?: number;
+  retries?: number;
+  retryDelay?: number;
+  /** HS256 secret, a fresh token is generated for every request as ELs check `iat` freshness */
+  jwtSecret?: Uint8Array;
+  /** Included as `id` claim if `jwtSecret` is provided */
+  jwtId?: string;
+  /** Value of the `X-Engine-Client-Version` header, the `clv` JWT claim is removed in the REST API */
+  clientVersion?: string;
+  metrics?: JsonRpcHttpClientMetrics | null;
+};
+
+/**
+ * HTTP/1.1 client for the REST engine API. Only transport concerns live here, encoding of
+ * request and response bodies is up to the caller.
+ */
+export class EngineRestHttpClient {
+  readonly emitter = new JsonRpcHttpClientEventEmitter();
+  private readonly metrics: JsonRpcHttpClientMetrics | null;
+
+  constructor(
+    private readonly urls: string[],
+    private readonly opts: EngineRestHttpClientOpts = {}
+  ) {
+    if (urls.length === 0) {
+      throw Error("No urls provided to EngineRestHttpClient");
+    }
+    for (const [i, url] of urls.entries()) {
+      if (!isValidHttpUrl(url)) {
+        throw Error(`EngineRestHttpClient.urls[${i}] must be a valid URL: ${url}`);
+      }
+    }
+    this.metrics = opts.metrics ?? null;
+  }
+
+  /** Perform request with retries, emitting response and error events consumed by the engine state tracking */
+  async request(req: EngineRestRequest, opts?: ReqOpts): Promise<EngineRestResponse> {
+    try {
+      const response = await this.requestWithRetries(req, opts);
+      this.emitter.emit(JsonRpcHttpClientEvent.RESPONSE, {payload: req, response});
+      return response;
+    } catch (error) {
+      this.emitter.emit(JsonRpcHttpClientEvent.ERROR, {payload: req, error: error as Error});
+      throw error;
+    }
+  }
+
+  /** Perform request with retries without emitting events, used to probe support for the REST API */
+  async requestWithRetries(req: EngineRestRequest, opts?: ReqOpts): Promise<EngineRestResponse> {
+    const routeId = opts?.routeId ?? "unknown";
+    return retry(() => this.requestAnyUrl(req, opts), {
+      retries: opts?.retries ?? this.opts.retries ?? 0,
+      retryDelay: opts?.retryDelay ?? this.opts.retryDelay,
+      shouldRetry: opts?.shouldRetry ?? isRetryableEngineRestError,
+      signal: this.opts.signal,
+      onRetry: () => {
+        this.metrics?.retryCount.inc({routeId});
+      },
+    });
+  }
+
+  private async requestAnyUrl(req: EngineRestRequest, opts?: ReqOpts): Promise<EngineRestResponse> {
+    const routeId = opts?.routeId ?? "unknown";
+    let lastError: Error | null = null;
+
+    for (let i = 0; i < this.urls.length; i++) {
+      if (i > 0) {
+        this.metrics?.requestUsedFallbackUrl.inc({routeId});
+      }
+
+      try {
+        return await this.requestOneUrl(this.urls[i], req, opts);
+      } catch (e) {
+        if (!isRetryableEngineRestError(e as Error)) {
+          throw e;
+        }
+        lastError = e as Error;
+      }
+    }
+    throw lastError ?? Error("Unknown error");
+  }
+
+  private async requestOneUrl(baseUrl: string, req: EngineRestRequest, opts?: ReqOpts): Promise<EngineRestResponse> {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), opts?.timeout ?? this.opts.timeout ?? REQUEST_TIMEOUT);
+
+    // Not AbortSignal.any(), Node retains every composite until the long-lived parent signal aborts
+    const onParentSignalAbort = (): void => controller.abort();
+    this.opts.signal?.addEventListener("abort", onParentSignalAbort, {once: true});
+    if (this.opts.signal?.aborted) {
+      controller.abort();
+    }
+
+    const routeId = opts?.routeId ?? "unknown";
+    const timer = this.metrics?.requestTime.startTimer({routeId});
+    this.metrics?.activeRequests.inc({routeId}, 1);
+
+    try {
+      const url = new URL(`${ENGINE_REST_BASE_PATH}${req.path}`, baseUrl);
+      for (const [key, value] of Object.entries(req.query ?? {})) {
+        url.searchParams.set(key, String(value));
+      }
+
+      const headers: Record<string, string> = {
+        Accept: req.responseType === "ssz" ? MEDIA_TYPE_SSZ : MEDIA_TYPE_JSON,
+      };
+      if (req.body !== undefined) {
+        headers["Content-Type"] = MEDIA_TYPE_SSZ;
+      }
+      if (req.executionFork !== undefined) {
+        headers[EXECUTION_VERSION_HEADER] = req.executionFork;
+      }
+      if (this.opts.clientVersion !== undefined) {
+        headers[CLIENT_VERSION_HEADER] = this.opts.clientVersion;
+      }
+      if (this.opts.jwtSecret) {
+        const jwtClaim: JwtClaim = {iat: Math.floor(Date.now() / 1000), id: this.opts.jwtId};
+        headers.Authorization = `Bearer ${encodeJwtToken(jwtClaim, this.opts.jwtSecret)}`;
+      }
+
+      this.metrics?.requestBytes.inc({routeId}, req.body?.length ?? 0);
+
+      const res = await fetch(url, {
+        method: req.method,
+        body: req.body as BodyInit | undefined,
+        headers,
+        signal: controller.signal,
+      });
+
+      const streamTimer = this.metrics?.streamTime.startTimer({routeId});
+      let body: Uint8Array;
+      try {
+        body = new Uint8Array(await res.arrayBuffer());
+      } catch (e) {
+        throw new FetchError(url, e);
+      } finally {
+        streamTimer?.();
+      }
+      this.metrics?.responseBytes.inc({routeId}, body.length);
+
+      if (!res.ok) {
+        const {type, detail} = parseProblemBody(body);
+        throw new EngineRestError(res.status, type, detail, routeId);
+      }
+
+      if (res.status !== 200 && res.status !== 204) {
+        throw new EngineRestResponseError(routeId, `Unexpected status ${res.status}`);
+      }
+      const contentType = res.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase();
+      const expectedContentType = req.responseType === "ssz" ? MEDIA_TYPE_SSZ : MEDIA_TYPE_JSON;
+      if (res.status !== 204 && contentType !== expectedContentType) {
+        throw new EngineRestResponseError(routeId, `Unexpected content type ${contentType}`);
+      }
+
+      return {status: res.status, body};
+    } catch (e) {
+      this.metrics?.requestErrors.inc({routeId});
+      if (controller.signal.aborted) {
+        if (this.opts.signal?.aborted) {
+          throw new ErrorAborted("request");
+        }
+        throw new TimeoutError("request");
+      }
+      throw e;
+    } finally {
+      timer?.();
+      this.metrics?.activeRequests.dec({routeId}, 1);
+
+      clearTimeout(timeout);
+      this.opts.signal?.removeEventListener("abort", onParentSignalAbort);
+    }
+  }
+}
+
+/** Client errors are deterministic apart from timeouts and rate limits, transport failures and server errors are retried */
+export function isRetryableEngineRestError(e: Error): boolean {
+  return (
+    (e instanceof EngineRestError && (e.status >= 500 || e.status === 408 || e.status === 429)) ||
+    e instanceof TimeoutError ||
+    (e instanceof FetchError && e.type !== "input" && e.type !== "aborted")
+  );
+}
+
+function parseProblemBody(body: Uint8Array): {type: string | null; detail: string | null} {
+  const text = new TextDecoder().decode(body);
+  try {
+    const problem: unknown = JSON.parse(text);
+    if (typeof problem === "object" && problem !== null) {
+      return {
+        type: "type" in problem && typeof problem.type === "string" ? problem.type : null,
+        detail:
+          ("detail" in problem && typeof problem.detail === "string" ? problem.detail : text).slice(
+            0,
+            MAX_ERROR_DETAIL_LENGTH
+          ) || null,
+      };
+    }
+  } catch {
+    // Legacy endpoints and proxies may return plain text instead of a problem document.
+  }
+  return {type: null, detail: text.slice(0, MAX_ERROR_DETAIL_LENGTH) || null};
+}