@lodestar/reqresp 1.35.0-dev.f80d2d52da → 1.35.0-dev.fd1dac853d

package/src/response/index.ts

@@ -0,0 +1,147 @@
+import {PeerId, Stream} from "@libp2p/interface";
+import {pipe} from "it-pipe";
+import {Uint8ArrayList} from "uint8arraylist";
+import {Logger, TimeoutError, withTimeout} from "@lodestar/utils";
+import {requestDecode} from "../encoders/requestDecode.js";
+import {responseEncodeError, responseEncodeSuccess} from "../encoders/responseEncode.js";
+import {RespStatus} from "../interface.js";
+import {Metrics} from "../metrics.js";
+import {ReqRespRateLimiter} from "../rate_limiter/ReqRespRateLimiter.js";
+import {RequestError, RequestErrorCode} from "../request/errors.js";
+import {Protocol, ReqRespRequest} from "../types.js";
+import {prettyPrintPeerId} from "../utils/index.js";
+import {ResponseError} from "./errors.js";
+
+export {ResponseError};
+
+// Default spec values from https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#configuration
+export const DEFAULT_REQUEST_TIMEOUT = 5 * 1000; // 5 sec
+
+export interface HandleRequestOpts {
+  logger: Logger;
+  metrics: Metrics | null;
+  stream: Stream;
+  peerId: PeerId;
+  protocol: Protocol;
+  protocolID: string;
+  rateLimiter: ReqRespRateLimiter;
+  signal?: AbortSignal;
+  requestId?: number;
+  /** Peer client type for logging and metrics: 'prysm' | 'lighthouse' */
+  peerClient?: string;
+  /** Non-spec timeout from sending request until write stream closed by responder */
+  requestTimeoutMs?: number;
+}
+
+/**
+ * Handles a ReqResp request from a peer. Throws on error. Logs each step of the response lifecycle.
+ *
+ * 1. A duplex `stream` with the peer is already available
+ * 2. Read and decode request from peer
+ * 3. Delegate to `performRequestHandler()` to perform the request job and expect
+ *    to yield zero or more `<response_chunks>`
+ * 4a. Encode and write `<response_chunks>` to peer
+ * 4b. On error, encode and write an error `<response_chunk>` and stop
+ */
+export async function handleRequest({
+  logger,
+  metrics,
+  stream,
+  peerId,
+  protocol,
+  protocolID,
+  rateLimiter,
+  signal,
+  requestId = 0,
+  peerClient = "unknown",
+  requestTimeoutMs,
+}: HandleRequestOpts): Promise<void> {
+  const REQUEST_TIMEOUT = requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT;
+
+  const logCtx = {
+    method: protocol.method,
+    version: protocol.version,
+    client: peerClient,
+    peer: prettyPrintPeerId(peerId),
+    requestId,
+  };
+  metrics?.incomingOpenedStreams.inc({method: protocol.method});
+
+  let responseError: Error | null = null;
+  await pipe(
+    // Yields success chunks and error chunks in the same generator
+    // This syntax allows to recycle stream.sink to send success and error chunks without returning
+    // in case request whose body is a List fails at chunk_i > 0, without breaking out of the for..await..of
+    (async function* requestHandlerSource() {
+      try {
+        // TODO: Does the TTFB timer start on opening stream or after receiving request
+        const timerTTFB = metrics?.outgoingResponseTTFB.startTimer({method: protocol.method});
+
+        const requestBody = await withTimeout(
+          () => pipe(stream.source as AsyncIterable<Uint8ArrayList>, requestDecode(protocol)),
+          REQUEST_TIMEOUT,
+          signal
+        ).catch((e: unknown) => {
+          if (e instanceof TimeoutError) {
+            throw e; // Let outter catch (_e) {} re-type the error as SERVER_ERROR
+          }
+          throw new ResponseError(RespStatus.INVALID_REQUEST, (e as Error).message);
+        });
+
+        logger.debug("Req  received", logCtx);
+
+        // Max count by request for byRange and byRoot
+        const requestCount = protocol?.inboundRateLimits?.getRequestCount?.(requestBody) ?? 1;
+
+        if (!rateLimiter.allows(peerId, protocolID, requestCount)) {
+          throw new RequestError({code: RequestErrorCode.REQUEST_RATE_LIMITED});
+        }
+
+        const requestChunk: ReqRespRequest = {
+          data: requestBody,
+          version: protocol.version,
+        };
+
+        yield* pipe(
+          // TODO: Debug the reason for type conversion here
+          protocol.handler(requestChunk, peerId),
+          // NOTE: Do not log the resp chunk contents, logs get extremely cluttered
+          // Note: Not logging on each chunk since after 1 year it hasn't add any value when debugging
+          // onChunk(() => logger.debug("Resp sending chunk", logCtx)),
+          responseEncodeSuccess(protocol, {
+            onChunk(chunkIndex) {
+              if (chunkIndex === 0) timerTTFB?.();
+            },
+          })
+        );
+      } catch (e) {
+        const status = e instanceof ResponseError ? e.status : RespStatus.SERVER_ERROR;
+        yield* responseEncodeError(protocol, status, (e as Error).message);
+
+        // Should not throw an error here or libp2p-mplex throws with 'AbortError: stream reset'
+        // throw e;
+        responseError = e as Error;
+      }
+    })(),
+    stream.sink
+  );
+
+  // If streak.sink throws, libp2p-mplex will close stream.source
+  // If `requestDecode()` throws the stream.source must be closed manually
+  // To ensure the stream.source it-pushable instance is always closed, stream.close() is called always
+  await stream.close();
+  metrics?.incomingClosedStreams.inc({method: protocol.method});
+
+  // TODO: It may happen that stream.sink returns before returning stream.source first,
+  // so you never see "Resp received request" in the logs and the response ends without
+  // sending any chunk, triggering EMPTY_RESPONSE error on the requesting side
+  // It has only happened when doing a request too fast upon immediate connection on inbound peer
+  // investigate a potential race condition there
+
+  if (responseError !== null) {
+    logger.verbose("Resp error", logCtx, responseError);
+    throw responseError;
+  }
+  // NOTE: Only log once per request to verbose, intermediate steps to debug
+  logger.verbose("Resp done", logCtx);
+}

package/src/types.ts

@@ -0,0 +1,158 @@
+import {PeerId} from "@libp2p/interface";
+import {BeaconConfig, ForkBoundary} from "@lodestar/config";
+import {ForkName} from "@lodestar/params";
+import {LodestarError} from "@lodestar/utils";
+import {RateLimiterQuota} from "./rate_limiter/rateLimiterGRCA.js";
+
+export const protocolPrefix = "/eth2/beacon_chain/req";
+
+/**
+ * Available request/response encoding strategies:
+ * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#encoding-strategies
+ */
+export enum Encoding {
+  SSZ_SNAPPY = "ssz_snappy",
+}
+
+export const CONTEXT_BYTES_FORK_DIGEST_LENGTH = 4;
+
+/**
+ * Wrapper for the request/response payload
+ */
+export type ResponseIncoming = {
+  data: Uint8Array;
+  fork: ForkName;
+  protocolVersion: number;
+};
+
+export type ResponseOutgoing = {
+  data: Uint8Array;
+  /**
+   * Reason why outgoing needs fork boundary but incoming only needs fork is because we can deserialize incoming data
+   * with only fork info, but for outgoing we also need to compute fork digest, which requires fork boundary.
+   */
+  boundary: ForkBoundary;
+};
+
+/**
+ * Rate limiter options for the requests
+ */
+export interface ReqRespRateLimiterOpts {
+  rateLimitMultiplier?: number;
+  onRateLimit?: (peer: PeerId, method: string) => void;
+}
+
+/**
+ * Inbound rate limiter quota for the protocol
+ */
+export interface InboundRateLimitQuota {
+  /**
+   * Will be tracked for the protocol per peer
+   */
+  byPeer?: RateLimiterQuota;
+  /**
+   * Will be tracked regardless of the peer
+   */
+  total?: RateLimiterQuota;
+  /**
+   * Some requests may be counted multiple e.g. getBlocksByRange
+   * for such implement this method else `1` will be used default
+   */
+  getRequestCount?: (req: Uint8Array) => number;
+}
+
+export type ReqRespRequest = {
+  data: Uint8Array;
+  version: number;
+};
+
+/**
+ * Request handler
+ */
+export type ProtocolHandler = (req: ReqRespRequest, peerId: PeerId) => AsyncIterable<ResponseOutgoing>;
+
+/**
+ * ReqResp Protocol Deceleration
+ */
+export interface ProtocolAttributes {
+  readonly protocolPrefix: string;
+  /** Protocol name identifier `beacon_blocks_by_range` or `status` */
+  readonly method: string;
+  /** Version counter: `1`, `2` etc */
+  readonly version: number;
+  readonly encoding: Encoding;
+}
+
+export interface ProtocolDescriptor extends Omit<ProtocolAttributes, "protocolPrefix"> {
+  contextBytes: ContextBytesFactory;
+  ignoreResponse?: boolean;
+}
+
+// `protocolPrefix` is added runtime so not part of definition
+/**
+ * ReqResp Protocol definition for full duplex protocols
+ */
+export interface Protocol extends ProtocolDescriptor {
+  handler: ProtocolHandler;
+  inboundRateLimits?: InboundRateLimitQuota;
+  requestSizes: TypeSizes | null;
+  responseSizes: (fork: ForkName) => TypeSizes;
+}
+
+/**
+ * ReqResp Protocol definition for dial only protocols
+ */
+export interface DialOnlyProtocol extends Omit<Protocol, "handler" | "inboundRateLimits" | "renderRequestBody"> {
+  handler?: never;
+  inboundRateLimits?: never;
+  renderRequestBody?: never;
+}
+
+/**
+ * ReqResp Protocol definition for full duplex and dial only protocols
+ */
+export type MixedProtocol = DialOnlyProtocol | Protocol;
+
+/**
+ * ReqResp protocol definition descriptor for full duplex and dial only protocols
+ * If handler is not provided, the protocol will be dial only
+ * If handler is provided, the protocol will be full duplex
+ */
+export type MixedProtocolGenerators = <H extends ProtocolHandler | undefined = undefined>(
+  // "inboundRateLimiter" is available only on handler context not on generator
+  modules: {config: BeaconConfig},
+  handler?: H
+) => H extends undefined ? DialOnlyProtocol : Protocol;
+
+/**
+ * ReqResp protocol definition descriptor for full duplex protocols
+ */
+export type ProtocolGenerator = (modules: {config: BeaconConfig}, handler: ProtocolHandler) => Protocol;
+
+export type HandlerTypeFromMessage<T> = T extends ProtocolGenerator ? ProtocolHandler : never;
+
+export type ContextBytesFactory =
+  | {type: ContextBytesType.Empty}
+  | {type: ContextBytesType.ForkDigest; config: BeaconConfig};
+
+export type ContextBytes = {type: ContextBytesType.Empty} | {type: ContextBytesType.ForkDigest; fork: ForkName};
+
+export enum ContextBytesType {
+  /** 0 bytes chunk, can be ignored */
+  Empty,
+  /** A fixed-width 4 byte <context-bytes>, set to the ForkDigest matching the chunk: compute_fork_digest(fork_version, genesis_validators_root) */
+  ForkDigest,
+}
+
+export enum LightClientServerErrorCode {
+  RESOURCE_UNAVAILABLE = "RESOURCE_UNAVAILABLE",
+}
+
+export type LightClientServerErrorType = {code: LightClientServerErrorCode.RESOURCE_UNAVAILABLE};
+
+export class LightClientServerError extends LodestarError<LightClientServerErrorType> {}
+
+export type TypeSizes = {
+  maxSize: number;
+  minSize: number;
+};

package/src/utils/abortableSource.ts

@@ -0,0 +1,80 @@
+/**
+ * Wraps an AsyncIterable and rejects early if any signal aborts.
+ * Throws the error returned by `getError()` of each signal options.
+ *
+ * Simplified fork of `"abortable-iterator"`.
+ * Read function's source for reasoning of the fork.
+ */
+export function abortableSource<T>(
+  sourceArg: AsyncIterable<T>,
+  signals: {
+    signal: AbortSignal;
+    getError: () => Error;
+  }[]
+): AsyncIterable<T> {
+  const source = sourceArg as AsyncGenerator<T>;
+
+  async function* abortable(): AsyncIterable<T> {
+    // Handler that will hold a reference to the `abort()` promise,
+    // necessary for the signal abort listeners to reject the iterable promise
+    let nextAbortHandler: ((error: Error) => void) | null = null;
+
+    // For each signal register an abortHandler(), and prepare clean-up with `onDoneCbs`
+    const onDoneCbs: (() => void)[] = [];
+    for (const {signal, getError} of signals) {
+      const abortHandler = (): void => {
+        if (nextAbortHandler) nextAbortHandler(getError());
+      };
+      signal.addEventListener("abort", abortHandler);
+      onDoneCbs.push(() => {
+        signal.removeEventListener("abort", abortHandler);
+      });
+    }
+
+    try {
+      while (true) {
+        // Abort early if any signal is aborted
+        for (const {signal, getError} of signals) {
+          if (signal.aborted) {
+            throw getError();
+          }
+        }
+
+        // Race the iterator and the abort signals
+        const result = await Promise.race([
+          new Promise<never>((_, reject) => {
+            nextAbortHandler = (error) => reject(error);
+          }),
+          source.next(),
+        ]);
+
+        // source.next() resolved first
+        nextAbortHandler = null;
+
+        if (result.done) {
+          return;
+        }
+
+        yield result.value;
+      }
+    } catch (err) {
+      // End the iterator if it is a generator
+      if (typeof source.return === "function") {
+        // This source.return() function may never resolve depending on the source AsyncGenerator implementation.
+        // This is the main reason to fork "abortable-iterator", which caused our node to get stuck during Sync.
+        // We choose to call .return() but not await it. In general, source.return should never throw. If it does,
+        // it a problem of the source implementor, and thus logged as an unhandled rejection. If that happens,
+        // the source implementor should fix the upstream code.
+        void source.return(null);
+      }
+
+      throw err;
+    } finally {
+      for (const cb of onDoneCbs) {
+        cb();
+      }
+    }
+  }
+
+  return abortable();
+}

package/src/utils/bufferedSource.ts

@@ -0,0 +1,46 @@
+import {Uint8ArrayList} from "uint8arraylist";
+
+/**
+ * Wraps a buffer chunk stream source with another async iterable
+ * so it can be reused in multiple for..of statements.
+ *
+ * Uses a BufferList internally to make sure all chunks are consumed
+ * when switching consumers
+ */
+export class BufferedSource {
+  isDone = false;
+  private buffer: Uint8ArrayList;
+  private source: AsyncGenerator<Uint8ArrayList | Uint8Array>;
+
+  constructor(source: AsyncGenerator<Uint8ArrayList | Uint8Array>) {
+    this.buffer = new Uint8ArrayList();
+    this.source = source;
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<Uint8ArrayList> {
+    const that = this;
+
+    let firstNext = true;
+
+    return {
+      async next() {
+        // Prevent fetching a new chunk if there are pending bytes
+        // not processed by a previous consumer of this BufferedSource
+        if (firstNext && that.buffer.length > 0) {
+          firstNext = false;
+          return {done: false, value: that.buffer};
+        }
+
+        const {done, value: chunk} = await that.source.next();
+        if (done === true) {
+          that.isDone = true;
+          return {done: true, value: undefined};
+        }
+
+        // Concat new chunk and return a reference to its BufferList instance
+        that.buffer.append(chunk);
+        return {done: false, value: that.buffer};
+      },
+    };
+  }
+}

package/src/utils/collectExactOne.ts

@@ -0,0 +1,15 @@
+import {RequestError, RequestErrorCode} from "../request/errors.js";
+
+/**
+ * Sink for `<response_chunk>*`, from
+ * ```bnf
+ * response ::= <response_chunk>*
+ * ```
+ * Expects exactly one response
+ */
+export async function collectExactOne<T>(source: AsyncIterable<T>): Promise<T> {
+  for await (const response of source) {
+    return response;
+  }
+  throw new RequestError({code: RequestErrorCode.EMPTY_RESPONSE});
+}

package/src/utils/collectMaxResponse.ts

@@ -0,0 +1,19 @@
+/**
+ * Sink for `<response_chunk>*`, from
+ * ```bnf
+ * response ::= <response_chunk>*
+ * ```
+ * Collects a bounded list of responses up to `maxResponses`
+ */
+export async function collectMaxResponse<T>(source: AsyncIterable<T>, maxResponses: number): Promise<T[]> {
+  // else: zero or more responses
+  const responses: T[] = [];
+  for await (const response of source) {
+    responses.push(response);
+
+    if (maxResponses !== undefined && responses.length >= maxResponses) {
+      break;
+    }
+  }
+  return responses;
+}

package/src/utils/errorMessage.ts

@@ -0,0 +1,51 @@
+import {decode as varintDecode, encodingLength as varintEncodingLength} from "uint8-varint";
+import {Uint8ArrayList} from "uint8arraylist";
+import {writeSszSnappyPayload} from "../encodingStrategies/sszSnappy/encode.js";
+import {SnappyFramesUncompress} from "../encodingStrategies/sszSnappy/snappyFrames/uncompress.js";
+import {Encoding} from "../types.js";
+
+// ErrorMessage schema:
+//
+// (
+//   error_message: List[byte, 256]
+// )
+//
+// By convention, the error_message is a sequence of bytes that MAY be interpreted as a
+// UTF-8 string (for debugging purposes). Clients MUST treat as valid any byte sequences
+//
+// Spec v1.1.10 https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#responding-side
+
+/**
+ * Encodes a UTF-8 string to 256 bytes max
+ */
+export async function* encodeErrorMessage(errorMessage: string, encoding: Encoding): AsyncGenerator<Buffer> {
+  const encoder = new TextEncoder();
+  const bytes = encoder.encode(errorMessage).slice(0, 256);
+
+  switch (encoding) {
+    case Encoding.SSZ_SNAPPY:
+      yield* writeSszSnappyPayload(bytes);
+  }
+}
+
+/**
+ * Decodes error message from network bytes and removes non printable, non ascii characters.
+ */
+export async function decodeErrorMessage(encodedErrorMessage: Uint8Array): Promise<string> {
+  const encoder = new TextDecoder();
+  let sszDataLength: number;
+  try {
+    sszDataLength = varintDecode(encodedErrorMessage);
+    const decompressor = new SnappyFramesUncompress();
+    const varintBytes = varintEncodingLength(sszDataLength);
+    const errorMessage = decompressor.uncompress(new Uint8ArrayList(encodedErrorMessage.subarray(varintBytes)));
+    if (errorMessage == null || errorMessage.length !== sszDataLength) {
+      throw new Error("Malformed input: data length mismatch");
+    }
+    // remove non ascii characters from string
+    return encoder.decode(errorMessage.subarray(0)).replace(/[^\x20-\x7F]/g, "");
+  } catch (_e) {
+    // remove non ascii characters from string
+    return encoder.decode(encodedErrorMessage.slice(0, 256)).replace(/[^\x20-\x7F]/g, "");
+  }
+}

package/src/utils/index.ts

@@ -0,0 +1,8 @@
+export * from "./abortableSource.js";
+export * from "./bufferedSource.js";
+export * from "./collectExactOne.js";
+export * from "./collectMaxResponse.js";
+export * from "./errorMessage.js";
+export * from "./onChunk.js";
+export * from "./peerId.js";
+export * from "./protocolId.js";

package/src/utils/onChunk.ts

@@ -0,0 +1,12 @@
+/**
+ * Calls `callback` with each `chunk` received from the `source` AsyncIterable
+ * Useful for logging, or cancelling timeouts
+ */
+export function onChunk<T>(callback: (chunk: T) => void): (source: AsyncIterable<T>) => AsyncIterable<T> {
+  return async function* onChunkTransform(source) {
+    for await (const chunk of source) {
+      callback(chunk);
+      yield chunk;
+    }
+  };
+}

package/src/utils/peerId.ts

@@ -0,0 +1,6 @@
+import {PeerId} from "@libp2p/interface";
+
+export function prettyPrintPeerId(peerId: PeerId): string {
+  const id = peerId.toString();
+  return `${id.substr(0, 2)}...${id.substr(id.length - 6, id.length)}`;
+}

package/src/utils/protocolId.ts

@@ -0,0 +1,44 @@
+import {Encoding, ProtocolAttributes} from "../types.js";
+
+/**
+ * https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#protocol-identification
+ */
+export function formatProtocolID(protocolPrefix: string, method: string, version: number, encoding: Encoding): string {
+  return `${protocolPrefix}/${method}/${version}/${encoding}`;
+}
+
+/**
+ * https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#protocol-identification
+ */
+export function parseProtocolID(protocolId: string): ProtocolAttributes {
+  const result = protocolId.split("/");
+  if (result.length < 4) {
+    throw new Error(`Invalid protocol id: ${protocolId}`);
+  }
+
+  const encoding = result.at(-1) as Encoding;
+  if (!Object.values(Encoding).includes(encoding)) {
+    throw new Error(`Invalid protocol encoding: ${encoding}`);
+  }
+
+  const versionStr = result.at(-2) as string;
+  if (!/^-?[0-9]+$/.test(versionStr)) {
+    throw new Error(`Invalid protocol version: ${versionStr}`);
+  }
+
+  // an ordinal version number (e.g. 1, 2, 3…).
+  const version = parseInt(versionStr);
+
+  // each request is identified by a name consisting of English alphabet, digits and underscores (_).
+  const method = result.at(-3) as string;
+
+  // messages are grouped into families identified by a shared libp2p protocol name prefix
+  const protocolPrefix = result.slice(0, result.length - 3).join("/");
+
+  return {
+    protocolPrefix,
+    method,
+    version,
+    encoding,
+  };
+}