@lodestar/reqresp 1.41.0-dev.9fa839a030 → 1.41.0-dev.aeab9f930d

package/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.41.0-dev.9fa839a030",
+  "version": "1.41.0-dev.aeab9f930d",
   "type": "module",
   "exports": {
     ".": {
@@ -32,11 +32,11 @@
   ],
   "scripts": {
     "clean": "rm -rf lib && rm -f *.tsbuildinfo",
-    "build": "tsc -p tsconfig.build.json",
+    "build": "tsgo -p tsconfig.build.json",
     "build:watch": "pnpm run build --watch",
     "build:release": "pnpm clean && pnpm run build",
     "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"",
-    "check-types": "tsc",
+    "check-types": "tsgo",
     "lint": "biome check src/ test/",
     "lint:fix": "pnpm run lint --write",
     "test": "pnpm test:unit",
@@ -45,12 +45,11 @@
   },
   "dependencies": {
     "@chainsafe/fast-crc32c": "^4.2.0",
-    "@libp2p/interface": "^2.7.0",
-    "@lodestar/config": "^1.41.0-dev.9fa839a030",
-    "@lodestar/params": "^1.41.0-dev.9fa839a030",
-    "@lodestar/utils": "^1.41.0-dev.9fa839a030",
-    "it-all": "^3.0.4",
-    "it-pipe": "^3.0.1",
+    "@libp2p/interface": "^3.1.0",
+    "@libp2p/utils": "^7.0.12",
+    "@lodestar/config": "^1.41.0-dev.aeab9f930d",
+    "@lodestar/params": "^1.41.0-dev.aeab9f930d",
+    "@lodestar/utils": "^1.41.0-dev.aeab9f930d",
     "snappy": "^7.2.2",
     "snappyjs": "^0.7.0",
     "uint8-varint": "^2.0.2",
@@ -58,16 +57,15 @@
   },
   "devDependencies": {
     "@chainsafe/ssz": "^1.2.2",
-    "@libp2p/crypto": "^5.1.7",
-    "@libp2p/logger": "^5.1.21",
-    "@libp2p/peer-id": "^5.1.8",
-    "@lodestar/logger": "^1.41.0-dev.9fa839a030",
-    "@lodestar/types": "^1.41.0-dev.9fa839a030",
-    "it-stream-types": "^2.0.2",
-    "libp2p": "2.9.0"
+    "@libp2p/crypto": "^5.1.13",
+    "@libp2p/logger": "^6.2.2",
+    "@libp2p/peer-id": "^6.0.4",
+    "@lodestar/logger": "^1.41.0-dev.aeab9f930d",
+    "@lodestar/types": "^1.41.0-dev.aeab9f930d",
+    "libp2p": "3.1.5"
   },
   "peerDependencies": {
-    "libp2p": "~2.9.0"
+    "libp2p": "^3.1.4"
   },
   "keywords": [
     "ethereum",
@@ -77,5 +75,5 @@
     "reqresp",
     "blockchain"
   ],
-  "gitHead": "1ba8fa66f8fc0890153a0ecc8cd5b3be617ce410"
+  "gitHead": "89f360d7bf466b84570b2b831517069776a24643"
 }

package/src/ReqResp.ts

@@ -37,8 +37,8 @@ export interface ReqRespOpts extends SendRequestOpts, ReqRespRateLimiterOpts {
 /**
  * Implementation of Ethereum Consensus p2p Req/Resp domain.
  * For the spec that this code is based on, see:
- * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-reqresp-domain
- * https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/p2p-interface.md#the-reqresp-domain
+ * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#the-reqresp-domain
+ * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/light-client/p2p-interface.md#the-reqresp-domain
  */
 export class ReqResp {
   // protected to be usable by extending class
@@ -221,7 +221,7 @@ export class ReqResp {
   }
 
   private getRequestHandler(protocol: MixedProtocol, protocolID: string) {
-    return async ({connection, stream}: {connection: Connection; stream: Stream}) => {
+    return async (stream: Stream, connection: Connection) => {
       if (this.dialOnlyProtocols.get(protocolID)) {
         throw new Error(`Received request on dial only protocol '${protocolID}'`);
       }
@@ -281,7 +281,7 @@ export class ReqResp {
    * ```
    * /ProtocolPrefix/MessageName/SchemaVersion/Encoding
    * ```
-   * https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#protocol-identification
+   * https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#protocol-identification
    */
   protected formatProtocolID(protocol: Pick<MixedProtocol, "method" | "version" | "encoding">): string {
     return formatProtocolID(this.protocolPrefix, protocol.method, protocol.version, protocol.encoding);

package/src/encoders/requestDecode.ts

@@ -1,8 +1,8 @@
-import type {Sink} from "it-stream-types";
-import {Uint8ArrayList} from "uint8arraylist";
+import type {Stream} from "@libp2p/interface";
+import {byteStream} from "@libp2p/utils";
 import {readEncodedPayload} from "../encodingStrategies/index.js";
 import {MixedProtocol} from "../types.js";
-import {BufferedSource} from "../utils/index.js";
+import {drainByteStream} from "../utils/stream.js";
 
 const EMPTY_DATA = new Uint8Array();
 
@@ -12,18 +12,34 @@ const EMPTY_DATA = new Uint8Array();
  * request  ::= <encoding-dependent-header> | <encoded-payload>
  * ```
  */
-export function requestDecode(
-  protocol: MixedProtocol
-): Sink<AsyncIterable<Uint8Array | Uint8ArrayList>, Promise<Uint8Array>> {
-  return async function requestDecodeSink(source) {
-    const type = protocol.requestSizes;
-    if (type === null) {
-      // method has no body
-      return EMPTY_DATA;
-    }
+export async function requestDecode(
+  protocol: MixedProtocol,
+  stream: Stream,
+  signal?: AbortSignal
+): Promise<Uint8Array> {
+  const type = protocol.requestSizes;
+  if (type === null) {
+    // method has no body
+    return EMPTY_DATA;
+  }
 
-    // Request has a single payload, so return immediately
-    const bufferedSource = new BufferedSource(source as AsyncGenerator<Uint8ArrayList>);
-    return readEncodedPayload(bufferedSource, protocol.encoding, type);
-  };
+  // Request has a single payload, so return immediately
+  const bytes = byteStream(stream);
+  let requestReadDone = false;
+  try {
+    const requestBody = await readEncodedPayload(bytes, protocol.encoding, type, signal);
+    requestReadDone = true;
+    return requestBody;
+  } finally {
+    try {
+      if (!requestReadDone) {
+        // Do not push partial bytes back into the stream on decode failure/abort.
+        // This stream is consumed by req/resp only once.
+        drainByteStream(bytes);
+      }
+      bytes.unwrap();
+    } catch {
+      // Ignore unwrap errors - stream may already be closed
+    }
+  }
 }

package/src/encoders/requestEncode.ts

@@ -9,7 +9,7 @@ import {MixedProtocol} from "../types.js";
  * Requests may contain no payload (e.g. /eth2/beacon_chain/req/metadata/1/)
  * if so, it would yield no byte chunks
  */
-export async function* requestEncode(protocol: MixedProtocol, requestBody: Uint8Array): AsyncGenerator<Buffer> {
+export function* requestEncode(protocol: MixedProtocol, requestBody: Uint8Array): Generator<Buffer> {
   const type = protocol.requestSizes;
 
   if (type && requestBody !== null) {

package/src/encoders/responseDecode.ts

@@ -1,4 +1,6 @@
-import {Uint8ArrayList} from "uint8arraylist";
+import type {Stream} from "@libp2p/interface";
+import type {ByteStream} from "@libp2p/utils";
+import {byteStream} from "@libp2p/utils";
 import {ForkName} from "@lodestar/params";
 import {readEncodedPayload} from "../encodingStrategies/index.js";
 import {RespStatus} from "../interface.js";
@@ -10,7 +12,7 @@ import {
   MixedProtocol,
   ResponseIncoming,
 } from "../types.js";
-import {BufferedSource, decodeErrorMessage} from "../utils/index.js";
+import {decodeErrorMessage, drainByteStream} from "../utils/index.js";
 
 /**
  * Internal helper type to signal stream ended early
@@ -27,22 +29,17 @@ enum StreamStatus {
  * result          ::= "0" | "1" | "2" | ["128" ... "255"]
  * ```
  */
-export function responseDecode(
+export async function* responseDecode(
   protocol: MixedProtocol,
-  cbs: {
-    onFirstHeader: () => void;
-    onFirstResponseChunk: () => void;
-  }
-): (source: AsyncIterable<Uint8Array | Uint8ArrayList>) => AsyncIterable<ResponseIncoming> {
-  return async function* responseDecodeSink(source) {
-    const bufferedSource = new BufferedSource(source as AsyncGenerator<Uint8ArrayList>);
-
-    let readFirstHeader = false;
-    let readFirstResponseChunk = false;
+  stream: Stream,
+  opts: {signal?: AbortSignal; getError?: () => Error} = {}
+): AsyncIterable<ResponseIncoming> {
+  const bytes = byteStream(stream);
+  let responseReadDone = false;
 
-    // Consumers of `responseDecode()` may limit the number of <response_chunk> and break out of the while loop
-    while (!bufferedSource.isDone) {
-      const status = await readResultHeader(bufferedSource);
+  try {
+    while (true) {
+      const status = await readResultHeader(bytes, opts.signal);
 
       // Stream is only allowed to end at the start of a <response_chunk> block
       // The happens when source ends before readResultHeader() can fetch 1 byte
@@ -50,34 +47,41 @@ export function responseDecode(
         break;
       }
 
-      if (!readFirstHeader) {
-        cbs.onFirstHeader();
-        readFirstHeader = true;
-      }
-
       // For multiple chunks, only the last chunk is allowed to have a non-zero error
       // code (i.e. The chunk stream is terminated once an error occurs
       if (status !== RespStatus.SUCCESS) {
-        const errorMessage = await readErrorMessage(bufferedSource);
+        const errorMessage = await readErrorMessage(bytes, opts.signal);
         throw new ResponseError(status, errorMessage);
       }
 
-      const forkName = await readContextBytes(protocol.contextBytes, bufferedSource);
+      const forkName = await readContextBytes(protocol.contextBytes, bytes, opts.signal);
       const typeSizes = protocol.responseSizes(forkName);
-      const chunkData = await readEncodedPayload(bufferedSource, protocol.encoding, typeSizes);
+      const chunkData = await readEncodedPayload(bytes, protocol.encoding, typeSizes, opts.signal);
 
       yield {
         data: chunkData,
         fork: forkName,
         protocolVersion: protocol.version,
       };
-
-      if (!readFirstResponseChunk) {
-        cbs.onFirstResponseChunk();
-        readFirstResponseChunk = true;
+    }
+    responseReadDone = true;
+  } catch (e) {
+    if (opts.signal?.aborted && opts.getError) {
+      throw opts.getError();
+    }
+    throw e;
+  } finally {
+    try {
+      if (!responseReadDone) {
+        // Do not push partial bytes back into the stream on decode failure/abort.
+        // This stream is consumed by req/resp only once.
+        drainByteStream(bytes);
       }
+      bytes.unwrap();
+    } catch {
+      // Ignore unwrap errors - stream may already be closed
     }
-  };
+  }
 }
 
 /**
@@ -87,18 +91,17 @@ export function responseDecode(
  * ```
  * `<response_chunk>` starts with a single-byte response code which determines the contents of the response_chunk
  */
-export async function readResultHeader(bufferedSource: BufferedSource): Promise<RespStatus | StreamStatus> {
-  for await (const buffer of bufferedSource) {
-    const status = buffer.get(0);
-    buffer.consume(1);
-
-    // If first chunk had zero bytes status === null, get next
-    if (status !== null) {
-      return status;
-    }
-  }
-
-  return StreamStatus.Ended;
+export async function readResultHeader(
+  bytes: ByteStream<Stream>,
+  signal?: AbortSignal
+): Promise<RespStatus | StreamStatus> {
+  const chunk = await bytes.read({bytes: 1, signal}).catch((e) => {
+    if ((e as Error).name === "UnexpectedEOFError") return null;
+    throw e;
+  });
+  if (chunk === null) return StreamStatus.Ended;
+
+  return chunk.get(0);
 }
 
 /**
@@ -108,28 +111,29 @@ export async function readResultHeader(bufferedSource: BufferedSource): Promise<
  * result          ::= "1" | "2" | ["128" ... "255"]
  * ```
  */
-export async function readErrorMessage(bufferedSource: BufferedSource): Promise<string> {
-  // Read at least 256 or wait for the stream to end
-  let length: number | undefined;
-  for await (const buffer of bufferedSource) {
-    // Wait for next chunk with bytes or for the stream to end
-    // Note: The entire <error_message> is expected to be in the same chunk
-    if (buffer.length >= 256) {
-      length = 256;
+export async function readErrorMessage(bytes: ByteStream<Stream>, signal?: AbortSignal): Promise<string> {
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+
+  while (total < 256) {
+    const chunk = await bytes.read({signal}).catch((e) => {
+      if ((e as Error).name === "UnexpectedEOFError") return null;
+      throw e;
+    });
+    if (chunk === null) {
+      // If EOF is reached while satisfying a larger read, libp2p v3 may still have
+      // buffered bytes available. Drain them so error_message matches pre-v3 behavior.
+      const remaining = drainByteStream(bytes);
+      if (remaining) {
+        chunks.push(remaining);
+      }
       break;
     }
-    length = buffer.length;
+    chunks.push(chunk.subarray());
+    total += chunk.byteLength;
   }
 
-  // biome-ignore lint/complexity/useLiteralKeys: It is a private attribute
-  const bytes = bufferedSource["buffer"].slice(0, length);
-
-  try {
-    return decodeErrorMessage(bytes);
-  } catch (_e) {
-    // Error message is optional and may not be included in the response stream
-    return Buffer.prototype.toString.call(bytes, "hex");
-  }
+  return decodeErrorMessage(Buffer.concat(chunks).subarray(0, 256));
 }
 
 /**
@@ -139,14 +143,15 @@ export async function readErrorMessage(bufferedSource: BufferedSource): Promise<
  */
 export async function readContextBytes(
   contextBytes: ContextBytesFactory,
-  bufferedSource: BufferedSource
+  bytes: ByteStream<Stream>,
+  signal?: AbortSignal
 ): Promise<ForkName> {
   switch (contextBytes.type) {
     case ContextBytesType.Empty:
       return ForkName.phase0;
 
     case ContextBytesType.ForkDigest: {
-      const forkDigest = await readContextBytesForkDigest(bufferedSource);
+      const forkDigest = await readContextBytesForkDigest(bytes, signal);
       return contextBytes.config.forkDigest2ForkBoundary(forkDigest).fork;
     }
   }
@@ -155,15 +160,6 @@ export async function readContextBytes(
 /**
  * Consumes a stream source to read `<context-bytes>`, where it's a fixed-width 4 byte
  */
-export async function readContextBytesForkDigest(bufferedSource: BufferedSource): Promise<Uint8Array> {
-  for await (const buffer of bufferedSource) {
-    if (buffer.length >= CONTEXT_BYTES_FORK_DIGEST_LENGTH) {
-      const bytes = buffer.slice(0, CONTEXT_BYTES_FORK_DIGEST_LENGTH);
-      buffer.consume(CONTEXT_BYTES_FORK_DIGEST_LENGTH);
-      return bytes;
-    }
-  }
-
-  // TODO: Use typed error
-  throw Error("Source ended while reading context bytes");
+export async function readContextBytesForkDigest(bytes: ByteStream<Stream>, signal?: AbortSignal): Promise<Uint8Array> {
+  return (await bytes.read({bytes: CONTEXT_BYTES_FORK_DIGEST_LENGTH, signal})).subarray();
 }

package/src/encoders/responseEncode.ts

@@ -1,7 +1,7 @@
 import {writeEncodedPayload} from "../encodingStrategies/index.js";
 import {RespStatus, RpcResponseStatusError} from "../interface.js";
 import {ContextBytesFactory, ContextBytesType, MixedProtocol, Protocol, ResponseOutgoing} from "../types.js";
-import {encodeErrorMessage} from "../utils/index.js";
+import {encodeErrorMessageToBuffer} from "../utils/index.js";
 
 const SUCCESS_BUFFER = Buffer.from([RespStatus.SUCCESS]);
 
@@ -14,30 +14,23 @@ const SUCCESS_BUFFER = Buffer.from([RespStatus.SUCCESS]);
  * ```
  * Note: `response` has zero or more chunks (denoted by `<>*`)
  */
-export function responseEncodeSuccess(
+export async function* responseEncodeSuccess(
   protocol: Protocol,
-  cbs: {onChunk: (chunkIndex: number) => void}
-): (source: AsyncIterable<ResponseOutgoing>) => AsyncIterable<Buffer> {
-  return async function* responseEncodeSuccessTransform(source) {
-    let chunkIndex = 0;
+  source: AsyncIterable<ResponseOutgoing>
+): AsyncIterable<Uint8Array> {
+  for await (const chunk of source) {
+    // <result>
+    yield SUCCESS_BUFFER;
 
-    for await (const chunk of source) {
-      // Postfix increment, return 0 as first chunk
-      cbs.onChunk(chunkIndex++);
-
-      // <result>
-      yield SUCCESS_BUFFER;
-
-      // <context-bytes> - from altair
-      const contextBytes = getContextBytes(protocol.contextBytes, chunk);
-      if (contextBytes) {
-        yield contextBytes as Buffer;
-      }
-
-      // <encoding-dependent-header> | <encoded-payload>
-      yield* writeEncodedPayload(chunk.data, protocol.encoding);
+    // <context-bytes> - from altair
+    const contextBytes = getContextBytes(protocol.contextBytes, chunk);
+    if (contextBytes) {
+      yield contextBytes;
     }
-  };
+
+    // <encoding-dependent-header> | <encoded-payload>
+    yield* writeEncodedPayload(chunk.data, protocol.encoding);
+  }
 }
 
 /**
@@ -50,18 +43,16 @@ export function responseEncodeSuccess(
  * Only the last `<response_chunk>` is allowed to have a non-zero error code, so this
  * fn yields exactly one `<error_response>` and afterwards the stream must be terminated
  */
-export async function* responseEncodeError(
+export function* responseEncodeError(
   protocol: Pick<MixedProtocol, "encoding">,
   status: RpcResponseStatusError,
   errorMessage: string
-): AsyncGenerator<Buffer> {
-  // <result>
-  yield Buffer.from([status]);
-
-  // <error_message>? is optional
-  if (errorMessage) {
-    yield* encodeErrorMessage(errorMessage, protocol.encoding);
-  }
+): Generator<Buffer> {
+  // Combine <result> and <error_message> into a single chunk for atomic delivery.
+  // Yielding them separately causes a race condition where the stream closes after the
+  // status byte but before the error message arrives on the reader side.
+  const errorMessageBuffer = encodeErrorMessageToBuffer(errorMessage, protocol.encoding);
+  yield Buffer.concat([Buffer.from([status]), errorMessageBuffer]);
 }
 
 /**

package/src/encodingStrategies/index.ts

@@ -1,10 +1,11 @@
+import type {Stream} from "@libp2p/interface";
+import type {ByteStream} from "@libp2p/utils";
 import {Encoding, TypeSizes} from "../types.js";
-import {BufferedSource} from "../utils/index.js";
 import {readSszSnappyPayload} from "./sszSnappy/decode.js";
 import {writeSszSnappyPayload} from "./sszSnappy/encode.js";
 
 // For more info about Ethereum Consensus request/response encoding strategies, see:
-// https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#encoding-strategies
+// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#encoding-strategies
 // Supported encoding strategies:
 // - ssz_snappy
 
@@ -15,13 +16,14 @@ import {writeSszSnappyPayload} from "./sszSnappy/encode.js";
  * ```
  */
 export async function readEncodedPayload(
-  bufferedSource: BufferedSource,
+  stream: ByteStream<Stream>,
   encoding: Encoding,
-  type: TypeSizes
+  type: TypeSizes,
+  signal?: AbortSignal
 ): Promise<Uint8Array> {
   switch (encoding) {
     case Encoding.SSZ_SNAPPY:
-      return readSszSnappyPayload(bufferedSource, type);
+      return readSszSnappyPayload(stream, type, signal);
 
     default:
       throw Error("Unsupported encoding");
@@ -34,7 +36,7 @@ export async function readEncodedPayload(
  * <encoding-dependent-header> | <encoded-payload>
  * ```
  */
-export async function* writeEncodedPayload(chunkData: Uint8Array, encoding: Encoding): AsyncGenerator<Buffer> {
+export function* writeEncodedPayload(chunkData: Uint8Array, encoding: Encoding): Generator<Buffer> {
   switch (encoding) {
     case Encoding.SSZ_SNAPPY:
       yield* writeSszSnappyPayload(chunkData);