@lodestar/reqresp 1.35.0-dev.f80d2d52da → 1.35.0-dev.fcf8d024ea

package/src/rate_limiter/rateLimiterGRCA.ts

@@ -0,0 +1,92 @@
+type MiliSeconds = number;
+
+export interface RateLimiterQuota {
+  /** How often are `max_tokens` fully replenished. */
+  quotaTimeMs: MiliSeconds;
+  /** Token limit. This translates on how large can an instantaneous batch of tokens be. */
+  quota: number;
+}
+
+/**
+ * Generic Cell Rate Algorithm is a leaky bucket-type scheduling algorithm.
+ *
+ * Most rate-limit implementations are either time-bucket or leaky-bucket based. The time-bucket requires the storage
+ * of two values and does not enforce a rate, while the leaky-bucket approach requires a separate process to
+ * continually refill the bucket. GCRA only storing a value (the TAT) while still being simple. GCRA may be rarely
+ * used because of its perceived complexity.
+ *
+ * GCRA aims to limit requests to `R = L/P`, where this implementation sets `L = 1` for simplicity. The target rate
+ * then is `R = 1/P` so request separated by at least `P` are not limited. Define the Theoretical Arrival Time (TAT)
+ * of the next request to be equal
+ */
+export class RateLimiterGRCA<Key> {
+  /** Time when the bucket will be full for each peer. TAT (theoretical arrival time) from GCRA */
+  private readonly tatPerKey = new Map<Key, MiliSeconds>();
+  private readonly startTimeMs = Date.now();
+
+  constructor(
+    /** After how long is the bucket considered full via replenishing 1T every `t`. */
+    private readonly msPerBucket: MiliSeconds,
+    /** How often is 1 token replenished */
+    private readonly msPerToken: MiliSeconds
+  ) {}
+
+  static fromQuota<Key>(quota: RateLimiterQuota): RateLimiterGRCA<Key> {
+    if (quota.quota === 0) {
+      throw Error("Max number of tokens should be positive");
+    }
+    const msPerBucket = quota.quotaTimeMs;
+    if (msPerBucket === 0) {
+      throw Error("Replenish time must be positive");
+    }
+    const msPerToken = msPerBucket / quota.quota;
+    return new RateLimiterGRCA(msPerBucket, msPerToken);
+  }
+
+  allows(key: Key, tokens: number): boolean {
+    if (tokens <= 0) {
+      throw new Error(`Token value should always be positive. Given: ${tokens}.`);
+    }
+
+    const msSinceStart = Date.now() - this.startTimeMs;
+
+    /** how long does it take to replenish these tokens */
+    const additionalTime = this.msPerToken * tokens;
+
+    if (additionalTime > this.msPerBucket) {
+      // the time required to process this amount of tokens is longer than the time that makes the bucket full.
+      return false;
+    }
+
+    // If the key is new, we consider their bucket full (which means, their request will be allowed)
+    let resetTimeForKey = this.tatPerKey.get(key);
+    if (resetTimeForKey === undefined) {
+      resetTimeForKey = msSinceStart;
+      this.tatPerKey.set(key, resetTimeForKey);
+    }
+
+    // check how soon could the request be made
+    const earliestTime = resetTimeForKey + additionalTime - this.msPerBucket;
+    if (msSinceStart < earliestTime) {
+      return false;
+    }
+
+    // calculate the new TAT
+    this.tatPerKey.set(key, Math.max(msSinceStart, resetTimeForKey) + additionalTime);
+    return true;
+  }
+
+  /** Removes keys for which their bucket is full by `time_limit` */
+  pruneByTime(timeLimit: MiliSeconds): void {
+    for (const entry of this.tatPerKey.entries()) {
+      // remove those for which tat < lim
+      if (entry[1] < timeLimit) {
+        this.tatPerKey.delete(entry[0]);
+      }
+    }
+  }
+
+  pruneByKey(key: Key): void {
+    this.tatPerKey.delete(key);
+  }
+}

package/src/rate_limiter/selfRateLimiter.ts

@@ -0,0 +1,112 @@
+import {Logger, MapDef} from "@lodestar/utils";
+
+type PeerIdStr = string;
+type ProtocolID = string;
+/** https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/p2p-interface.md#constants */
+const MAX_CONCURRENT_REQUESTS = 2;
+
+/** Sometimes a peer request comes AFTER libp2p disconnect event, check for such peers every 2 minutes */
+export const CHECK_DISCONNECTED_PEERS_INTERVAL_MS = 2 * 60 * 1000;
+
+/** Given PING_INTERVAL constants of 15s/20s, we consider a peer is disconnected if there is no request in 1 minute */
+const DISCONNECTED_TIMEOUT_MS = 60 * 1000;
+
+/**
+ * Timeout to consider a request is no longer in progress
+ * this is to cover the case where `requestCompleted()` is not called due to unexpected errors
+ * for example https://github.com/ChainSafe/lodestar/issues/8256
+ **/
+export const REQUEST_TIMEOUT_MS = 30 * 1000;
+
+type RequestId = number;
+type RequestIdMs = number;
+
+/**
+ * Simple rate limiter that allows a maximum of 2 concurrent requests per protocol per peer.
+ * The consumer should either prevent requests from being sent when the limit is reached or handle the case when the request is not allowed.
+ */
+export class SelfRateLimiter {
+  private readonly rateLimitersPerPeer: MapDef<PeerIdStr, MapDef<ProtocolID, Map<RequestId, RequestIdMs>>>;
+  /**
+   * It's not convenient to handle a peer disconnected event so we track the last seen requests by peer.
+   * This is the same design to `ReqRespRateLimiter`.
+   **/
+  private lastSeenRequestsByPeer: Map<string, number>;
+  /** Interval to check lastSeenMessagesByPeer */
+  private cleanupInterval: NodeJS.Timeout | undefined = undefined;
+
+  constructor(private readonly logger?: Logger) {
+    this.rateLimitersPerPeer = new MapDef<PeerIdStr, MapDef<ProtocolID, Map<RequestId, RequestIdMs>>>(
+      () => new MapDef<ProtocolID, Map<RequestId, RequestIdMs>>(() => new Map())
+    );
+    this.lastSeenRequestsByPeer = new Map();
+  }
+
+  start(): void {
+    this.cleanupInterval = setInterval(this.checkDisconnectedPeers.bind(this), CHECK_DISCONNECTED_PEERS_INTERVAL_MS);
+  }
+
+  stop(): void {
+    if (this.cleanupInterval !== undefined) {
+      clearInterval(this.cleanupInterval);
+      this.cleanupInterval = undefined;
+    }
+  }
+
+  /**
+   * called before we send a request to a peer.
+   */
+  allows(peerId: PeerIdStr, protocolId: ProtocolID, requestId: RequestId): boolean {
+    const now = Date.now();
+    const peerRateLimiter = this.rateLimitersPerPeer.getOrDefault(peerId);
+    const trackedRequests = peerRateLimiter.getOrDefault(protocolId);
+    this.lastSeenRequestsByPeer.set(peerId, now);
+
+    let inProgressRequests = 0;
+    for (const [trackedRequestId, trackedRequestTimeMs] of trackedRequests.entries()) {
+      if (trackedRequestTimeMs + REQUEST_TIMEOUT_MS <= now) {
+        // request timed out, remove it
+        trackedRequests.delete(trackedRequestId);
+        this.logger?.debug("SelfRateLimiter: request timed out, removing it", {
+          requestId: trackedRequestId,
+          requestTime: trackedRequestTimeMs,
+          peerId,
+          protocolId,
+        });
+      } else {
+        inProgressRequests++;
+      }
+    }
+
+    if (inProgressRequests >= MAX_CONCURRENT_REQUESTS) {
+      return false;
+    }
+
+    trackedRequests.set(requestId, now);
+    return true;
+  }
+
+  /**
+   * called when a request to a peer is completed, regardless of success or failure.
+   * This should NOT be called when the request was not allowed
+   */
+  requestCompleted(peerId: PeerIdStr, protocolId: ProtocolID, requestId: RequestId): void {
+    const peerRateLimiter = this.rateLimitersPerPeer.getOrDefault(peerId);
+    const trackedRequests = peerRateLimiter.getOrDefault(protocolId);
+    trackedRequests.delete(requestId);
+  }
+
+  getPeerCount(): number {
+    return this.rateLimitersPerPeer.size;
+  }
+
+  private checkDisconnectedPeers(): void {
+    const now = Date.now();
+    for (const [peerIdStr, lastSeenTime] of this.lastSeenRequestsByPeer.entries()) {
+      if (now - lastSeenTime >= DISCONNECTED_TIMEOUT_MS) {
+        this.rateLimitersPerPeer.delete(peerIdStr);
+        this.lastSeenRequestsByPeer.delete(peerIdStr);
+      }
+    }
+  }
+}

package/src/request/errors.ts

@@ -0,0 +1,119 @@
+import {LodestarError, LodestarErrorObject} from "@lodestar/utils";
+import {RespStatus, RpcResponseStatusError} from "../interface.js";
+import {ResponseError} from "../response/index.js";
+
+export enum RequestErrorCode {
+  // Declaring specific values of RpcResponseStatusError for error clarity downstream
+  /** `<response_chunk>` had `<result>` === INVALID_REQUEST */
+  INVALID_REQUEST = "REQUEST_ERROR_INVALID_REQUEST",
+  INVALID_RESPONSE_SSZ = "REQUEST_ERROR_INVALID_RESPONSE_SSZ",
+  /** `<response_chunk>` had `<result>` === SERVER_ERROR */
+  SERVER_ERROR = "REQUEST_ERROR_SERVER_ERROR",
+  /** `<response_chunk>` had `<result>` === RESOURCE_UNAVAILABLE */
+  RESOURCE_UNAVAILABLE = "RESOURCE_UNAVAILABLE_ERROR",
+  /** `<response_chunk>` had a `<result>` not known in the current spec */
+  UNKNOWN_ERROR_STATUS = "REQUEST_ERROR_UNKNOWN_ERROR_STATUS",
+  /** Could not open a stream with peer before DIAL_TIMEOUT */
+  DIAL_TIMEOUT = "REQUEST_ERROR_DIAL_TIMEOUT",
+  /** Error opening a stream with peer */
+  DIAL_ERROR = "REQUEST_ERROR_DIAL_ERROR",
+  /** Reponder did not close write stream before REQUEST_TIMEOUT */
+  REQUEST_TIMEOUT = "REQUEST_ERROR_REQUEST_TIMEOUT",
+  /** Error when sending request to responder */
+  REQUEST_ERROR = "REQUEST_ERROR_REQUEST_ERROR",
+  /** Reponder did not deliver a full reponse before max maxTotalResponseTimeout() */
+  RESPONSE_TIMEOUT = "REQUEST_ERROR_RESPONSE_TIMEOUT",
+  /** A single-response method returned 0 chunks */
+  EMPTY_RESPONSE = "REQUEST_ERROR_EMPTY_RESPONSE",
+  /** Time to first byte timeout */
+  TTFB_TIMEOUT = "REQUEST_ERROR_TTFB_TIMEOUT",
+  /** Timeout between `<response_chunk>` exceed */
+  RESP_TIMEOUT = "REQUEST_ERROR_RESP_TIMEOUT",
+  /** Request rate limited */
+  REQUEST_RATE_LIMITED = "REQUEST_ERROR_RATE_LIMITED",
+  /** Request self rate limited */
+  REQUEST_SELF_RATE_LIMITED = "REQUEST_ERROR_SELF_RATE_LIMITED",
+  /** Response rate limited */
+  RESP_RATE_LIMITED = "RESPONSE_ERROR_RATE_LIMITED",
+  /** For malformed SSZ (metadata) responses */
+  SSZ_OVER_MAX_SIZE = "SSZ_SNAPPY_ERROR_OVER_SSZ_MAX_SIZE",
+}
+
+type RequestErrorType =
+  | {code: RequestErrorCode.INVALID_REQUEST; errorMessage: string}
+  | {code: RequestErrorCode.INVALID_RESPONSE_SSZ; errorMessage: string}
+  | {code: RequestErrorCode.SERVER_ERROR; errorMessage: string}
+  | {code: RequestErrorCode.RESOURCE_UNAVAILABLE; errorMessage: string}
+  | {code: RequestErrorCode.UNKNOWN_ERROR_STATUS; status: RpcResponseStatusError; errorMessage: string}
+  | {code: RequestErrorCode.DIAL_TIMEOUT}
+  | {code: RequestErrorCode.DIAL_ERROR; error: Error}
+  | {code: RequestErrorCode.REQUEST_TIMEOUT}
+  | {code: RequestErrorCode.REQUEST_ERROR; error: Error}
+  | {code: RequestErrorCode.EMPTY_RESPONSE}
+  | {code: RequestErrorCode.TTFB_TIMEOUT}
+  | {code: RequestErrorCode.RESP_TIMEOUT}
+  | {code: RequestErrorCode.REQUEST_RATE_LIMITED}
+  | {code: RequestErrorCode.REQUEST_SELF_RATE_LIMITED}
+  | {code: RequestErrorCode.RESP_RATE_LIMITED}
+  | {code: RequestErrorCode.SSZ_OVER_MAX_SIZE};
+
+export const REQUEST_ERROR_CLASS_NAME = "RequestError";
+
+export class RequestError extends LodestarError<RequestErrorType> {
+  constructor(type: RequestErrorType, message?: string, stack?: string) {
+    super(type, message ?? renderErrorMessage(type), stack);
+  }
+
+  static fromObject(obj: LodestarErrorObject): RequestError {
+    if (obj.className !== "RequestError") {
+      throw new Error(`Expected className to be RequestError, but got ${obj.className}`);
+    }
+
+    return new RequestError(obj.type as RequestErrorType, obj.message, obj.stack);
+  }
+}
+
+/**
+ * Parse response status errors into detailed request errors for each status code for easier debugging
+ */
+export function responseStatusErrorToRequestError(e: ResponseError): RequestErrorType {
+  const {errorMessage, status} = e;
+  // rate limited error from clients have different status, for example: lighthouse responds with 139, teku responds with 1
+  // but all of them has "rate limit" in the error message
+  // refer to https://github.com/ChainSafe/lodestar/issues/8065#issuecomment-3157266196
+  const errorMessageLowercase = errorMessage.toLowerCase();
+  if (errorMessageLowercase.includes("rate limit")) {
+    return {code: RequestErrorCode.RESP_RATE_LIMITED};
+  }
+
+  // Grandine may return this without standard RespStatus, see https://github.com/ChainSafe/lodestar/issues/8110
+  if (errorMessageLowercase.includes("wait ")) {
+    return {code: RequestErrorCode.RESP_TIMEOUT};
+  }
+
+  switch (status) {
+    case RespStatus.INVALID_REQUEST:
+      return {code: RequestErrorCode.INVALID_REQUEST, errorMessage};
+    case RespStatus.SERVER_ERROR:
+      return {code: RequestErrorCode.SERVER_ERROR, errorMessage};
+    case RespStatus.RESOURCE_UNAVAILABLE:
+      return {code: RequestErrorCode.RESOURCE_UNAVAILABLE, errorMessage};
+    default:
+      return {code: RequestErrorCode.UNKNOWN_ERROR_STATUS, errorMessage, status};
+  }
+}
+
+/**
+ * Render responder's errorMessage directly in main's error.message for easier debugging
+ */
+function renderErrorMessage(type: RequestErrorType): string | undefined {
+  switch (type.code) {
+    case RequestErrorCode.INVALID_REQUEST:
+    case RequestErrorCode.SERVER_ERROR:
+    case RequestErrorCode.RESOURCE_UNAVAILABLE:
+    case RequestErrorCode.UNKNOWN_ERROR_STATUS:
+      return `${type.code}: ${type.errorMessage}`;
+    default:
+      return type.code;
+  }
+}

package/src/request/index.ts

@@ -0,0 +1,225 @@
+import {PeerId} from "@libp2p/interface";
+import {pipe} from "it-pipe";
+import type {Libp2p} from "libp2p";
+import {Uint8ArrayList} from "uint8arraylist";
+import {ErrorAborted, Logger, TimeoutError, withTimeout} from "@lodestar/utils";
+import {requestEncode} from "../encoders/requestEncode.js";
+import {responseDecode} from "../encoders/responseDecode.js";
+import {Metrics} from "../metrics.js";
+import {ResponseError} from "../response/index.js";
+import {MixedProtocol, ResponseIncoming} from "../types.js";
+import {abortableSource, prettyPrintPeerId} from "../utils/index.js";
+import {RequestError, RequestErrorCode, responseStatusErrorToRequestError} from "./errors.js";
+
+export {RequestError, RequestErrorCode};
+
+// Default spec values from https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#configuration
+export const DEFAULT_DIAL_TIMEOUT = 5 * 1000; // 5 sec
+export const DEFAULT_REQUEST_TIMEOUT = 5 * 1000; // 5 sec
+export const DEFAULT_TTFB_TIMEOUT = 5 * 1000; // 5 sec
+export const DEFAULT_RESP_TIMEOUT = 10 * 1000; // 10 sec
+
+export interface SendRequestOpts {
+  /** The maximum time for complete response transfer. */
+  respTimeoutMs?: number;
+  /** Non-spec timeout from sending request until write stream closed by responder */
+  requestTimeoutMs?: number;
+  /** The maximum time to wait for first byte of request response (time-to-first-byte). */
+  ttfbTimeoutMs?: number;
+  /** Non-spec timeout from dialing protocol until stream opened */
+  dialTimeoutMs?: number;
+}
+
+type SendRequestModules = {
+  logger: Logger;
+  libp2p: Libp2p;
+  metrics: Metrics | null;
+  peerClient?: string;
+};
+
+/**
+ * Sends ReqResp request to a peer. Throws on error. Logs each step of the request lifecycle.
+ *
+ * 1. Dial peer, establish duplex stream
+ * 2. Encoded and write request to peer. Expect the responder to close the stream's write side
+ * 3. Read and decode reponse(s) from peer. Will close the read stream if:
+ *    - An error result is received in one of the chunks. Reads the error_message and throws.
+ *    - The responder closes the stream. If at the end or start of a <response_chunk>, return. Otherwise throws
+ *    - Any part of the response_chunk fails validation. Throws a typed error (see `SszSnappyError`)
+ *    - The maximum number of requested chunks are read. Does not throw, returns read chunks only.
+ */
+export async function* sendRequest(
+  {logger, libp2p, metrics, peerClient}: SendRequestModules,
+  peerId: PeerId,
+  protocols: MixedProtocol[],
+  protocolIDs: string[],
+  requestBody: Uint8Array,
+  signal?: AbortSignal,
+  opts?: SendRequestOpts,
+  requestId = 0
+): AsyncIterable<ResponseIncoming> {
+  if (protocols.length === 0) {
+    throw Error("sendRequest must set > 0 protocols");
+  }
+
+  const DIAL_TIMEOUT = opts?.dialTimeoutMs ?? DEFAULT_DIAL_TIMEOUT;
+  const REQUEST_TIMEOUT = opts?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT;
+  const TTFB_TIMEOUT = opts?.ttfbTimeoutMs ?? DEFAULT_TTFB_TIMEOUT;
+  const RESP_TIMEOUT = opts?.respTimeoutMs ?? DEFAULT_RESP_TIMEOUT;
+
+  const peerIdStrShort = prettyPrintPeerId(peerId);
+  const {method, encoding, version} = protocols[0];
+  const logCtx = {method, version, encoding, client: peerClient, peer: peerIdStrShort, requestId};
+
+  if (signal?.aborted) {
+    throw new ErrorAborted("sendRequest");
+  }
+
+  logger.debug("Req  dialing peer", logCtx);
+
+  try {
+    // From Altair block query methods have V1 and V2. Both protocols should be requested.
+    // On stream negotiation `libp2p.dialProtocol` will pick the available protocol and return
+    // the picked protocol in `connection.protocol`
+    const protocolsMap = new Map<string, MixedProtocol>(protocols.map((protocol, i) => [protocolIDs[i], protocol]));
+
+    // As of October 2020 we can't rely on libp2p.dialProtocol timeout to work so
+    // this function wraps the dialProtocol promise with an extra timeout
+    //
+    // > The issue might be: you add the peer's addresses to the AddressBook,
+    //   which will result in autoDial to kick in and dial your peer. In parallel,
+    //   you do a manual dial and it will wait for the previous one without using
+    //   the abort signal:
+    //
+    // https://github.com/ChainSafe/lodestar/issues/1597#issuecomment-703394386
+
+    // DIAL_TIMEOUT: Non-spec timeout from dialing protocol until stream opened
+    const stream = await withTimeout(
+      async (timeoutAndParentSignal) => {
+        const protocolIds = Array.from(protocolsMap.keys());
+        const conn = await libp2p.dialProtocol(peerId, protocolIds, {signal: timeoutAndParentSignal});
+        if (!conn) throw Error("dialProtocol timeout");
+        return conn;
+      },
+      DIAL_TIMEOUT,
+      signal
+    ).catch((e: Error) => {
+      if (e instanceof TimeoutError) {
+        throw new RequestError({code: RequestErrorCode.DIAL_TIMEOUT});
+      }
+      throw new RequestError({code: RequestErrorCode.DIAL_ERROR, error: e});
+    });
+
+    metrics?.outgoingOpenedStreams?.inc({method});
+
+    // TODO: Does the TTFB timer start on opening stream or after receiving request
+    const timerTTFB = metrics?.outgoingResponseTTFB.startTimer({method});
+
+    // Parse protocol selected by the responder
+    const protocolId = stream.protocol ?? "unknown";
+    const protocol = protocolsMap.get(protocolId);
+    if (!protocol) throw Error(`dialProtocol selected unknown protocolId ${protocolId}`);
+
+    // Override with actual version that was negotiated
+    logCtx.version = protocol.version;
+
+    logger.debug("Req  sending request", logCtx);
+
+    // Spec: The requester MUST close the write side of the stream once it finishes writing the request message
+    // Impl: stream.sink is closed automatically by js-libp2p-mplex when piped source is exhausted
+
+    // REQUEST_TIMEOUT: Non-spec timeout from sending request until write stream closed by responder
+    // Note: libp2p.stop() will close all connections, so not necessary to abort this pipe on parent stop
+    await withTimeout(() => pipe(requestEncode(protocol, requestBody), stream.sink), REQUEST_TIMEOUT, signal).catch(
+      (e) => {
+        // Must close the stream read side (stream.source) manually AND the write side
+        stream.abort(e);
+
+        if (e instanceof TimeoutError) {
+          throw new RequestError({code: RequestErrorCode.REQUEST_TIMEOUT});
+        }
+        throw new RequestError({code: RequestErrorCode.REQUEST_ERROR, error: e as Error});
+      }
+    );
+
+    logger.debug("Req  request sent", logCtx);
+
+    // For goodbye method peers may disconnect before completing the response and trigger multiple errors.
+    // Do not expect them to reply and successfully return early
+    if (protocol.ignoreResponse) {
+      return;
+    }
+
+    // - TTFB_TIMEOUT: The requester MUST wait a maximum of TTFB_TIMEOUT for the first response byte to arrive
+    // - RESP_TIMEOUT: Requester allows a further RESP_TIMEOUT for each subsequent response_chunk
+    // - Max total timeout: This timeout is not required by the spec. It may not be necessary, but it's kept as
+    //   safe-guard to close. streams in case of bugs on other timeout mechanisms.
+    const ttfbTimeoutController = new AbortController();
+    const respTimeoutController = new AbortController();
+
+    let timeoutRESP: NodeJS.Timeout | null = null;
+
+    const timeoutTTFB = setTimeout(() => {
+      // If we abort on first byte delay, don't need to abort for response delay
+      if (timeoutRESP) clearTimeout(timeoutRESP);
+      ttfbTimeoutController.abort();
+    }, TTFB_TIMEOUT);
+
+    const restartRespTimeout = (): void => {
+      if (timeoutRESP) clearTimeout(timeoutRESP);
+      timeoutRESP = setTimeout(() => respTimeoutController.abort(), RESP_TIMEOUT);
+    };
+
+    try {
+      // Note: libp2p.stop() will close all connections, so not necessary to abort this pipe on parent stop
+      yield* pipe(
+        abortableSource(stream.source as AsyncIterable<Uint8ArrayList>, [
+          {
+            signal: ttfbTimeoutController.signal,
+            getError: () => new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}),
+          },
+          {
+            signal: respTimeoutController.signal,
+            getError: () => new RequestError({code: RequestErrorCode.RESP_TIMEOUT}),
+          },
+        ]),
+
+        // Transforms `Buffer` chunks to yield `ResponseBody` chunks
+        responseDecode(protocol, {
+          onFirstHeader() {
+            // On first byte, cancel the single use TTFB_TIMEOUT, and start RESP_TIMEOUT
+            clearTimeout(timeoutTTFB);
+            timerTTFB?.();
+            restartRespTimeout();
+          },
+          onFirstResponseChunk() {
+            // On <response_chunk>, cancel this chunk's RESP_TIMEOUT and start next's
+            restartRespTimeout();
+          },
+        })
+      );
+
+      // NOTE: Only log once per request to verbose, intermediate steps to debug
+      // NOTE: Do not log the response, logs get extremely cluttered
+      // NOTE: add double space after "Req  " to align log with the "Resp " log
+      logger.verbose("Req  done", logCtx);
+    } finally {
+      clearTimeout(timeoutTTFB);
+      if (timeoutRESP !== null) clearTimeout(timeoutRESP);
+
+      // Necessary to call `stream.close()` since collectResponses() may break out of the source before exhausting it
+      // `stream.close()` libp2p-mplex will .end() the source (it-pushable instance)
+      // If collectResponses() exhausts the source, it-pushable.end() can be safely called multiple times
+      await stream.close();
+      metrics?.outgoingClosedStreams?.inc({method});
+      logger.verbose("Req  stream closed", logCtx);
+    }
+  } catch (e) {
+    logger.verbose("Req  error", logCtx, e as Error);
+
+    if (e instanceof ResponseError) {
+      throw new RequestError(responseStatusErrorToRequestError(e));
+    }
+    throw e;
+  }
+}

package/src/response/errors.ts

@@ -0,0 +1,50 @@
+import {LodestarError, LodestarErrorMetaData, LodestarErrorObject} from "@lodestar/utils";
+import {RespStatus, RpcResponseStatusError} from "../interface.js";
+
+type RpcResponseStatusNotSuccess = Exclude<RespStatus, RespStatus.SUCCESS>;
+
+export enum ResponseErrorCode {
+  RESPONSE_STATUS_ERROR = "RESPONSE_STATUS_ERROR",
+}
+
+type RequestErrorType = {
+  code: ResponseErrorCode;
+  status: RpcResponseStatusError;
+  errorMessage: string;
+};
+
+export const RESPONSE_ERROR_CLASS_NAME = "ResponseError";
+
+/**
+ * Used internally only to signal a response status error. Since the error should never bubble up to the user,
+ * the error code and error message does not matter much.
+ */
+export class ResponseError extends LodestarError<RequestErrorType> {
+  status: RpcResponseStatusNotSuccess;
+  errorMessage: string;
+  constructor(status: RpcResponseStatusNotSuccess, errorMessage: string, stack?: string) {
+    const type = {code: ResponseErrorCode.RESPONSE_STATUS_ERROR, status, errorMessage};
+    super(type, `RESPONSE_ERROR_${RespStatus[status]}: ${errorMessage}`, stack);
+    this.status = status;
+    this.errorMessage = errorMessage;
+  }
+
+  getMetadata(): LodestarErrorMetaData {
+    return {
+      status: this.status,
+      errorMessage: this.errorMessage,
+    };
+  }
+
+  static fromObject(obj: LodestarErrorObject): ResponseError {
+    if (obj.className !== RESPONSE_ERROR_CLASS_NAME) {
+      throw new Error(`Expected className to be ResponseError, but got ${obj.className}`);
+    }
+
+    return new ResponseError(
+      obj.type.status as RpcResponseStatusNotSuccess,
+      obj.type.errorMessage as string,
+      obj.stack
+    );
+  }
+}