@lodestar/reqresp 1.41.0-dev.983b1a457b → 1.41.0-dev.9939b12b53

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/README.md +1 -1
  2. package/lib/ReqResp.d.ts +3 -3
  3. package/lib/ReqResp.d.ts.map +1 -1
  4. package/lib/ReqResp.js +4 -4
  5. package/lib/ReqResp.js.map +1 -1
  6. package/lib/encoders/requestDecode.d.ts +2 -3
  7. package/lib/encoders/requestDecode.d.ts.map +1 -1
  8. package/lib/encoders/requestDecode.js +28 -11
  9. package/lib/encoders/requestDecode.js.map +1 -1
  10. package/lib/encoders/requestEncode.d.ts +1 -1
  11. package/lib/encoders/requestEncode.d.ts.map +1 -1
  12. package/lib/encoders/requestEncode.js +1 -1
  13. package/lib/encoders/requestEncode.js.map +1 -1
  14. package/lib/encoders/responseDecode.d.ts +10 -10
  15. package/lib/encoders/responseDecode.d.ts.map +1 -1
  16. package/lib/encoders/responseDecode.js +63 -60
  17. package/lib/encoders/responseDecode.js.map +1 -1
  18. package/lib/encoders/responseEncode.d.ts +2 -4
  19. package/lib/encoders/responseEncode.d.ts.map +1 -1
  20. package/lib/encoders/responseEncode.js +13 -22
  21. package/lib/encoders/responseEncode.js.map +1 -1
  22. package/lib/encodingStrategies/index.d.ts +4 -3
  23. package/lib/encodingStrategies/index.d.ts.map +1 -1
  24. package/lib/encodingStrategies/index.js +4 -4
  25. package/lib/encodingStrategies/index.js.map +1 -1
  26. package/lib/encodingStrategies/sszSnappy/decode.d.ts +5 -4
  27. package/lib/encodingStrategies/sszSnappy/decode.d.ts.map +1 -1
  28. package/lib/encodingStrategies/sszSnappy/decode.js +83 -52
  29. package/lib/encodingStrategies/sszSnappy/decode.js.map +1 -1
  30. package/lib/encodingStrategies/sszSnappy/encode.d.ts +2 -2
  31. package/lib/encodingStrategies/sszSnappy/encode.d.ts.map +1 -1
  32. package/lib/encodingStrategies/sszSnappy/encode.js +1 -1
  33. package/lib/encodingStrategies/sszSnappy/encode.js.map +1 -1
  34. package/lib/encodingStrategies/sszSnappy/errors.d.ts +0 -8
  35. package/lib/encodingStrategies/sszSnappy/errors.d.ts.map +1 -1
  36. package/lib/encodingStrategies/sszSnappy/errors.js +2 -3
  37. package/lib/encodingStrategies/sszSnappy/errors.js.map +1 -1
  38. package/lib/encodingStrategies/sszSnappy/index.d.ts +0 -1
  39. package/lib/encodingStrategies/sszSnappy/index.d.ts.map +1 -1
  40. package/lib/encodingStrategies/sszSnappy/index.js +0 -1
  41. package/lib/encodingStrategies/sszSnappy/index.js.map +1 -1
  42. package/lib/encodingStrategies/sszSnappy/utils.js.map +1 -1
  43. package/lib/interface.js +2 -1
  44. package/lib/interface.js.map +1 -1
  45. package/lib/metrics.d.ts +1 -7
  46. package/lib/metrics.d.ts.map +1 -1
  47. package/lib/metrics.js +1 -17
  48. package/lib/metrics.js.map +1 -1
  49. package/lib/rate_limiter/ReqRespRateLimiter.d.ts.map +1 -1
  50. package/lib/rate_limiter/ReqRespRateLimiter.js.map +1 -1
  51. package/lib/rate_limiter/rateLimiterGRCA.d.ts.map +1 -1
  52. package/lib/rate_limiter/rateLimiterGRCA.js.map +1 -1
  53. package/lib/rate_limiter/selfRateLimiter.d.ts.map +1 -1
  54. package/lib/rate_limiter/selfRateLimiter.js.map +1 -1
  55. package/lib/request/errors.d.ts +1 -7
  56. package/lib/request/errors.d.ts.map +1 -1
  57. package/lib/request/errors.js +3 -6
  58. package/lib/request/errors.js.map +1 -1
  59. package/lib/request/index.d.ts +0 -3
  60. package/lib/request/index.d.ts.map +1 -1
  61. package/lib/request/index.js +96 -70
  62. package/lib/request/index.js.map +1 -1
  63. package/lib/response/errors.d.ts.map +1 -1
  64. package/lib/response/errors.js +2 -1
  65. package/lib/response/errors.js.map +1 -1
  66. package/lib/response/index.d.ts +2 -2
  67. package/lib/response/index.d.ts.map +1 -1
  68. package/lib/response/index.js +46 -50
  69. package/lib/response/index.js.map +1 -1
  70. package/lib/types.d.ts +1 -2
  71. package/lib/types.d.ts.map +1 -1
  72. package/lib/types.js +7 -5
  73. package/lib/types.js.map +1 -1
  74. package/lib/utils/collectExactOne.js.map +1 -1
  75. package/lib/utils/collectMaxResponse.d.ts.map +1 -1
  76. package/lib/utils/collectMaxResponse.js +1 -2
  77. package/lib/utils/collectMaxResponse.js.map +1 -1
  78. package/lib/utils/errorMessage.d.ts +3 -3
  79. package/lib/utils/errorMessage.d.ts.map +1 -1
  80. package/lib/utils/errorMessage.js +14 -13
  81. package/lib/utils/errorMessage.js.map +1 -1
  82. package/lib/utils/index.d.ts +1 -3
  83. package/lib/utils/index.d.ts.map +1 -1
  84. package/lib/utils/index.js +1 -3
  85. package/lib/utils/index.js.map +1 -1
  86. package/lib/utils/peerId.js.map +1 -1
  87. package/lib/utils/protocolId.d.ts +2 -2
  88. package/lib/utils/protocolId.js +2 -2
  89. package/lib/utils/protocolId.js.map +1 -1
  90. package/lib/utils/snappyCommon.js +2 -1
  91. package/lib/utils/snappyCommon.js.map +1 -1
  92. package/lib/utils/snappyCompress.d.ts +1 -1
  93. package/lib/utils/snappyCompress.d.ts.map +1 -1
  94. package/lib/utils/snappyCompress.js +1 -1
  95. package/lib/utils/snappyCompress.js.map +1 -1
  96. package/lib/utils/snappyIndex.d.ts +1 -1
  97. package/lib/utils/snappyIndex.d.ts.map +1 -1
  98. package/lib/utils/snappyIndex.js +1 -1
  99. package/lib/utils/snappyIndex.js.map +1 -1
  100. package/lib/utils/snappyUncompress.d.ts +7 -11
  101. package/lib/utils/snappyUncompress.d.ts.map +1 -1
  102. package/lib/utils/snappyUncompress.js +68 -68
  103. package/lib/utils/snappyUncompress.js.map +1 -1
  104. package/lib/utils/stream.d.ts +6 -0
  105. package/lib/utils/stream.d.ts.map +1 -0
  106. package/lib/utils/stream.js +21 -0
  107. package/lib/utils/stream.js.map +1 -0
  108. package/package.json +16 -18
  109. package/src/ReqResp.ts +4 -4
  110. package/src/encoders/requestDecode.ts +32 -16
  111. package/src/encoders/requestEncode.ts +1 -1
  112. package/src/encoders/responseDecode.ts +68 -72
  113. package/src/encoders/responseEncode.ts +17 -29
  114. package/src/encodingStrategies/index.ts +8 -6
  115. package/src/encodingStrategies/sszSnappy/decode.ts +111 -53
  116. package/src/encodingStrategies/sszSnappy/encode.ts +2 -2
  117. package/src/encodingStrategies/sszSnappy/errors.ts +0 -4
  118. package/src/encodingStrategies/sszSnappy/index.ts +0 -1
  119. package/src/metrics.ts +1 -17
  120. package/src/request/errors.ts +1 -6
  121. package/src/request/index.ts +119 -85
  122. package/src/response/index.ts +55 -61
  123. package/src/types.ts +1 -3
  124. package/src/utils/collectMaxResponse.ts +1 -2
  125. package/src/utils/errorMessage.ts +14 -13
  126. package/src/utils/index.ts +1 -3
  127. package/src/utils/protocolId.ts +2 -2
  128. package/src/utils/snappyCompress.ts +1 -1
  129. package/src/utils/snappyIndex.ts +1 -1
  130. package/src/utils/snappyUncompress.ts +73 -75
  131. package/src/utils/stream.ts +34 -0
  132. package/lib/utils/abortableSource.d.ts +0 -12
  133. package/lib/utils/abortableSource.d.ts.map +0 -1
  134. package/lib/utils/abortableSource.js +0 -69
  135. package/lib/utils/abortableSource.js.map +0 -1
  136. package/lib/utils/bufferedSource.d.ts +0 -16
  137. package/lib/utils/bufferedSource.d.ts.map +0 -1
  138. package/lib/utils/bufferedSource.js +0 -40
  139. package/lib/utils/bufferedSource.js.map +0 -1
  140. package/lib/utils/onChunk.d.ts +0 -6
  141. package/lib/utils/onChunk.d.ts.map +0 -1
  142. package/lib/utils/onChunk.js +0 -13
  143. package/lib/utils/onChunk.js.map +0 -1
  144. package/lib/utils/snappy.d.ts +0 -3
  145. package/lib/utils/snappy.d.ts.map +0 -1
  146. package/lib/utils/snappy.js +0 -3
  147. package/lib/utils/snappy.js.map +0 -1
  148. package/src/utils/abortableSource.ts +0 -80
  149. package/src/utils/bufferedSource.ts +0 -46
  150. package/src/utils/onChunk.ts +0 -12
  151. package/src/utils/snappy.ts +0 -2
@@ -1,8 +1,9 @@
1
+ import type {Stream} from "@libp2p/interface";
2
+ import type {ByteStream} from "@libp2p/utils";
1
3
  import {decode as varintDecode, encodingLength as varintEncodingLength} from "uint8-varint";
2
4
  import {Uint8ArrayList} from "uint8arraylist";
3
5
  import {TypeSizes} from "../../types.js";
4
- import {BufferedSource} from "../../utils/index.js";
5
- import {SnappyFramesUncompress} from "../../utils/snappyIndex.js";
6
+ import {ChunkType, decodeSnappyFrameData, parseSnappyFrameHeader} from "../../utils/snappyIndex.js";
6
7
  import {SszSnappyError, SszSnappyErrorCode} from "./errors.js";
7
8
  import {maxEncodedLen} from "./utils.js";
8
9
 
@@ -15,76 +16,115 @@ export const MAX_VARINT_BYTES = 10;
15
16
  * <encoding-dependent-header> | <encoded-payload>
16
17
  * ```
17
18
  */
18
- export async function readSszSnappyPayload(bufferedSource: BufferedSource, type: TypeSizes): Promise<Uint8Array> {
19
- const sszDataLength = await readSszSnappyHeader(bufferedSource, type);
20
-
21
- return readSszSnappyBody(bufferedSource, sszDataLength);
19
+ export async function readSszSnappyPayload(
20
+ stream: ByteStream<Stream>,
21
+ type: TypeSizes,
22
+ signal?: AbortSignal
23
+ ): Promise<Uint8Array> {
24
+ const sszDataLength = await readSszSnappyHeader(stream, type, signal);
25
+
26
+ return readSszSnappyBody(stream, sszDataLength, signal);
22
27
  }
23
28
 
24
29
  /**
25
30
  * Reads `<encoding-dependent-header>` for ssz-snappy.
26
31
  * encoding-header ::= the length of the raw SSZ bytes, encoded as an unsigned protobuf varint
27
32
  */
28
- export async function readSszSnappyHeader(bufferedSource: BufferedSource, type: TypeSizes): Promise<number> {
29
- for await (const buffer of bufferedSource) {
30
- // Get next bytes if empty
31
- if (buffer.length === 0) {
32
- continue;
33
- }
33
+ export async function readSszSnappyHeader(
34
+ stream: ByteStream<Stream>,
35
+ type: TypeSizes,
36
+ signal?: AbortSignal
37
+ ): Promise<number> {
38
+ const varintBytes: number[] = [];
34
39
 
35
- let sszDataLength: number;
36
- try {
37
- sszDataLength = varintDecode(buffer.subarray());
38
- } catch (_e) {
39
- throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity});
40
- }
40
+ while (true) {
41
+ const byte = await readExactOrSourceAborted(stream, 1, signal);
41
42
 
42
- // MUST validate: the unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes
43
- // encodingLength function only returns 1-8 inclusive
44
- const varintBytes = varintEncodingLength(sszDataLength);
45
- buffer.consume(varintBytes);
43
+ const value = byte.get(0);
46
44
 
47
- // MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type.
48
- const minSize = type.minSize;
49
- const maxSize = type.maxSize;
50
- if (sszDataLength < minSize) {
51
- throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength});
52
- }
53
- if (sszDataLength > maxSize) {
54
- throw new SszSnappyError({code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE, maxSize, sszDataLength});
45
+ varintBytes.push(value);
46
+ if (varintBytes.length > MAX_VARINT_BYTES) {
47
+ throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintBytes.length});
55
48
  }
56
49
 
57
- return sszDataLength;
50
+ // MSB not set => varint terminated
51
+ if ((value & 0x80) === 0) break;
52
+ }
53
+
54
+ let sszDataLength: number;
55
+ try {
56
+ sszDataLength = varintDecode(Uint8Array.from(varintBytes));
57
+ } catch {
58
+ throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity});
58
59
  }
59
60
 
60
- throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED});
61
+ // MUST validate: the unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes
62
+ // encodingLength function only returns 1-8 inclusive
63
+ const varintByteLength = varintEncodingLength(sszDataLength);
64
+ if (varintByteLength > MAX_VARINT_BYTES) {
65
+ throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintByteLength});
66
+ }
67
+
68
+ // MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type.
69
+ const minSize = type.minSize;
70
+ const maxSize = type.maxSize;
71
+ if (sszDataLength < minSize) {
72
+ throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength});
73
+ }
74
+ if (sszDataLength > maxSize) {
75
+ throw new SszSnappyError({code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE, maxSize, sszDataLength});
76
+ }
77
+
78
+ return sszDataLength;
61
79
  }
62
80
 
63
81
  /**
64
82
  * Reads `<encoded-payload>` for ssz-snappy and decompress.
65
83
  * The returned bytes can be SSZ deseralized
66
84
  */
67
- export async function readSszSnappyBody(bufferedSource: BufferedSource, sszDataLength: number): Promise<Uint8Array> {
68
- const decompressor = new SnappyFramesUncompress();
85
+ export async function readSszSnappyBody(
86
+ stream: ByteStream<Stream>,
87
+ sszDataLength: number,
88
+ signal?: AbortSignal
89
+ ): Promise<Uint8Array> {
69
90
  const uncompressedData = new Uint8ArrayList();
70
- let readBytes = 0;
91
+ let encodedBytesRead = 0;
92
+ const maxBytes = maxEncodedLen(sszDataLength);
93
+ let foundIdentifier = false;
94
+
95
+ while (uncompressedData.length < sszDataLength) {
96
+ const header = await readExactOrSourceAborted(stream, 4, signal);
71
97
 
72
- for await (const buffer of bufferedSource) {
73
98
  // SHOULD NOT read more than max_encoded_len(n) bytes after reading the SSZ length-prefix n from the header
74
- readBytes += buffer.length;
75
- if (readBytes > maxEncodedLen(sszDataLength)) {
76
- throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, readBytes, sszDataLength});
99
+ encodedBytesRead = addEncodedBytesReadOrThrow(encodedBytesRead, header.length, maxBytes, sszDataLength);
100
+
101
+ let headerParsed: {type: ChunkType; frameSize: number};
102
+ try {
103
+ headerParsed = parseSnappyFrameHeader(header.subarray());
104
+ if (!foundIdentifier && headerParsed.type !== ChunkType.IDENTIFIER) {
105
+ throw new Error("malformed input: must begin with an identifier");
106
+ }
107
+ } catch (e) {
108
+ throw new SszSnappyError({code: SszSnappyErrorCode.DECOMPRESSOR_ERROR, decompressorError: e as Error});
77
109
  }
78
110
 
79
- // No bytes left to consume, get next
80
- if (buffer.length === 0) {
81
- continue;
111
+ if (headerParsed.frameSize > maxBytes - encodedBytesRead) {
112
+ throw new SszSnappyError({
113
+ code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ,
114
+ readBytes: encodedBytesRead + headerParsed.frameSize,
115
+ sszDataLength,
116
+ });
82
117
  }
118
+ const frame = await readExactOrSourceAborted(stream, headerParsed.frameSize, signal);
119
+
120
+ encodedBytesRead = addEncodedBytesReadOrThrow(encodedBytesRead, frame.length, maxBytes, sszDataLength);
83
121
 
84
- // stream contents can be passed through a buffered Snappy reader to decompress frame by frame
85
122
  try {
86
- const uncompressed = decompressor.uncompress(buffer);
87
- buffer.consume(buffer.length);
123
+ if (headerParsed.type === ChunkType.IDENTIFIER) {
124
+ foundIdentifier = true;
125
+ }
126
+
127
+ const uncompressed = decodeSnappyFrameData(headerParsed.type, frame.subarray());
88
128
  if (uncompressed !== null) {
89
129
  uncompressedData.append(uncompressed);
90
130
  }
@@ -96,16 +136,34 @@ export async function readSszSnappyBody(bufferedSource: BufferedSource, sszDataL
96
136
  if (uncompressedData.length > sszDataLength) {
97
137
  throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MANY_BYTES, sszDataLength});
98
138
  }
139
+ }
99
140
 
100
- // Keep reading chunks until `n` SSZ bytes
101
- if (uncompressedData.length < sszDataLength) {
102
- continue;
103
- }
141
+ // buffer.length === n
142
+ return uncompressedData.subarray(0, sszDataLength);
143
+ }
104
144
 
105
- // buffer.length === n
106
- return uncompressedData.subarray(0, sszDataLength);
145
+ function addEncodedBytesReadOrThrow(
146
+ encodedBytesRead: number,
147
+ bytesToAdd: number,
148
+ maxBytes: number,
149
+ sszDataLength: number
150
+ ): number {
151
+ const nextReadBytes = encodedBytesRead + bytesToAdd;
152
+ if (nextReadBytes > maxBytes) {
153
+ throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, readBytes: nextReadBytes, sszDataLength});
107
154
  }
155
+ return nextReadBytes;
156
+ }
108
157
 
109
- // SHOULD consider invalid: An early EOF before fully reading the declared length-prefix worth of SSZ bytes
110
- throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED});
158
+ async function readExactOrSourceAborted(
159
+ stream: ByteStream<Stream>,
160
+ bytes: number,
161
+ signal?: AbortSignal
162
+ ): Promise<Uint8ArrayList> {
163
+ return stream.read({bytes, signal}).catch((e) => {
164
+ if ((e as Error).name === "UnexpectedEOFError") {
165
+ throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED});
166
+ }
167
+ throw e;
168
+ });
111
169
  }
@@ -8,12 +8,12 @@ import {encodeSnappy} from "../../utils/snappyIndex.js";
8
8
  * <encoding-dependent-header> | <encoded-payload>
9
9
  * ```
10
10
  */
11
- export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => AsyncGenerator<Buffer>;
11
+ export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => Generator<Buffer>;
12
12
 
13
13
  /**
14
14
  * Buffered Snappy writer
15
15
  */
16
- export async function* encodeSszSnappy(bytes: Buffer): AsyncGenerator<Buffer> {
16
+ export function* encodeSszSnappy(bytes: Buffer): Generator<Buffer> {
17
17
  // MUST encode the length of the raw SSZ bytes, encoded as an unsigned protobuf varint
18
18
  const varint = varintEncode(bytes.length);
19
19
  yield Buffer.from(varint.buffer, varint.byteOffset, varint.byteLength);
@@ -9,8 +9,6 @@ export enum SszSnappyErrorCode {
9
9
  OVER_SSZ_MAX_SIZE = "SSZ_SNAPPY_ERROR_OVER_SSZ_MAX_SIZE",
10
10
  TOO_MUCH_BYTES_READ = "SSZ_SNAPPY_ERROR_TOO_MUCH_BYTES_READ",
11
11
  DECOMPRESSOR_ERROR = "SSZ_SNAPPY_ERROR_DECOMPRESSOR_ERROR",
12
- DESERIALIZE_ERROR = "SSZ_SNAPPY_ERROR_DESERIALIZE_ERROR",
13
- SERIALIZE_ERROR = "SSZ_SNAPPY_ERROR_SERIALIZE_ERROR",
14
12
  /** Received more bytes than specified sszDataLength */
15
13
  TOO_MANY_BYTES = "SSZ_SNAPPY_ERROR_TOO_MANY_BYTES",
16
14
  /** Source aborted before reading sszDataLength bytes */
@@ -23,8 +21,6 @@ type SszSnappyErrorType =
23
21
  | {code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE; maxSize: number; sszDataLength: number}
24
22
  | {code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ; readBytes: number; sszDataLength: number}
25
23
  | {code: SszSnappyErrorCode.DECOMPRESSOR_ERROR; decompressorError: Error}
26
- | {code: SszSnappyErrorCode.DESERIALIZE_ERROR; deserializeError: Error}
27
- | {code: SszSnappyErrorCode.SERIALIZE_ERROR; serializeError: Error}
28
24
  | {code: SszSnappyErrorCode.TOO_MANY_BYTES; sszDataLength: number}
29
25
  | {code: SszSnappyErrorCode.SOURCE_ABORTED};
30
26
 
@@ -1,4 +1,3 @@
1
- export {SnappyFramesUncompress, encodeSnappy} from "../../utils/snappyIndex.js";
2
1
  export * from "./decode.js";
3
2
  export * from "./encode.js";
4
3
  export * from "./errors.js";
package/src/metrics.ts CHANGED
@@ -4,7 +4,7 @@ import {RequestErrorCode} from "./request/errors.js";
4
4
  export type Metrics = ReturnType<typeof getMetrics>;
5
5
 
6
6
  /**
7
- * A collection of metrics used throughout the Gossipsub behaviour.
7
+ * A collection of metrics used throughout the Req/Resp domain.
8
8
  */
9
9
  export function getMetrics(register: MetricsRegisterExtra) {
10
10
  // Using function style instead of class to prevent having to re-declare all MetricsPrometheus types.
@@ -29,7 +29,6 @@ export function getMetrics(register: MetricsRegisterExtra) {
29
29
  name: "beacon_reqresp_outgoing_request_roundtrip_time_seconds",
30
30
  help: "Histogram of outgoing requests round-trip time",
31
31
  labelNames: ["method"],
32
- // Spec sets RESP_TIMEOUT = 10 sec
33
32
  buckets: [0.1, 0.2, 0.5, 1, 5, 10, 15, 60],
34
33
  }),
35
34
  outgoingErrors: register.gauge<{method: string}>({
@@ -61,7 +60,6 @@ export function getMetrics(register: MetricsRegisterExtra) {
61
60
  name: "beacon_reqresp_incoming_request_handler_time_seconds",
62
61
  help: "Histogram of incoming requests internal handling time",
63
62
  labelNames: ["method"],
64
- // Spec sets RESP_TIMEOUT = 10 sec
65
63
  buckets: [0.1, 0.2, 0.5, 1, 5, 10],
66
64
  }),
67
65
  incomingErrors: register.gauge<{method: string}>({
@@ -69,20 +67,6 @@ export function getMetrics(register: MetricsRegisterExtra) {
69
67
  help: "Counts total failed responses handled per method",
70
68
  labelNames: ["method"],
71
69
  }),
72
- outgoingResponseTTFB: register.histogram<{method: string}>({
73
- name: "beacon_reqresp_outgoing_response_ttfb_seconds",
74
- help: "Time to first byte (TTFB) for outgoing responses",
75
- labelNames: ["method"],
76
- // Spec sets TTFB_TIMEOUT = 5 sec
77
- buckets: [0.1, 1, 5],
78
- }),
79
- incomingResponseTTFB: register.histogram<{method: string}>({
80
- name: "beacon_reqresp_incoming_response_ttfb_seconds",
81
- help: "Time to first byte (TTFB) for incoming responses",
82
- labelNames: ["method"],
83
- // Spec sets TTFB_TIMEOUT = 5 sec
84
- buckets: [0.1, 1, 5],
85
- }),
86
70
  dialErrors: register.gauge({
87
71
  name: "beacon_reqresp_dial_errors_total",
88
72
  help: "Count total dial errors",
@@ -21,13 +21,9 @@ export enum RequestErrorCode {
21
21
  REQUEST_TIMEOUT = "REQUEST_ERROR_REQUEST_TIMEOUT",
22
22
  /** Error when sending request to responder */
23
23
  REQUEST_ERROR = "REQUEST_ERROR_REQUEST_ERROR",
24
- /** Reponder did not deliver a full reponse before max maxTotalResponseTimeout() */
25
- RESPONSE_TIMEOUT = "REQUEST_ERROR_RESPONSE_TIMEOUT",
26
24
  /** A single-response method returned 0 chunks */
27
25
  EMPTY_RESPONSE = "REQUEST_ERROR_EMPTY_RESPONSE",
28
- /** Time to first byte timeout */
29
- TTFB_TIMEOUT = "REQUEST_ERROR_TTFB_TIMEOUT",
30
- /** Timeout between `<response_chunk>` exceed */
26
+ /** Response transfer timeout exceeded */
31
27
  RESP_TIMEOUT = "REQUEST_ERROR_RESP_TIMEOUT",
32
28
  /** Request rate limited */
33
29
  REQUEST_RATE_LIMITED = "REQUEST_ERROR_RATE_LIMITED",
@@ -50,7 +46,6 @@ type RequestErrorType =
50
46
  | {code: RequestErrorCode.REQUEST_TIMEOUT}
51
47
  | {code: RequestErrorCode.REQUEST_ERROR; error: Error}
52
48
  | {code: RequestErrorCode.EMPTY_RESPONSE}
53
- | {code: RequestErrorCode.TTFB_TIMEOUT}
54
49
  | {code: RequestErrorCode.RESP_TIMEOUT}
55
50
  | {code: RequestErrorCode.REQUEST_RATE_LIMITED}
56
51
  | {code: RequestErrorCode.REQUEST_SELF_RATE_LIMITED}
@@ -1,31 +1,93 @@
1
+ import type {Stream} from "@libp2p/interface";
1
2
  import {PeerId} from "@libp2p/interface";
2
- import {pipe} from "it-pipe";
3
3
  import type {Libp2p} from "libp2p";
4
- import {Uint8ArrayList} from "uint8arraylist";
5
4
  import {ErrorAborted, Logger, TimeoutError, withTimeout} from "@lodestar/utils";
6
5
  import {requestEncode} from "../encoders/requestEncode.js";
7
6
  import {responseDecode} from "../encoders/responseDecode.js";
8
7
  import {Metrics} from "../metrics.js";
9
8
  import {ResponseError} from "../response/index.js";
10
9
  import {MixedProtocol, ResponseIncoming} from "../types.js";
11
- import {abortableSource, prettyPrintPeerId} from "../utils/index.js";
10
+ import {prettyPrintPeerId, sendChunks} from "../utils/index.js";
12
11
  import {RequestError, RequestErrorCode, responseStatusErrorToRequestError} from "./errors.js";
13
12
 
14
13
  export {RequestError, RequestErrorCode};
15
14
 
16
- // Default spec values from https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#configuration
15
+ // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#the-reqresp-domain
17
16
  export const DEFAULT_DIAL_TIMEOUT = 5 * 1000; // 5 sec
18
17
  export const DEFAULT_REQUEST_TIMEOUT = 5 * 1000; // 5 sec
19
- export const DEFAULT_TTFB_TIMEOUT = 5 * 1000; // 5 sec
20
18
  export const DEFAULT_RESP_TIMEOUT = 10 * 1000; // 10 sec
21
19
 
20
+ function getStreamNotFullyConsumedError(): Error {
21
+ return new Error("ReqResp stream was not fully consumed");
22
+ }
23
+
24
+ function scheduleStreamAbortIfNotClosed(stream: Stream, timeoutMs: number): void {
25
+ const onClose = (): void => {
26
+ clearTimeout(timeout);
27
+ };
28
+
29
+ const timeout = setTimeout(() => {
30
+ stream.removeEventListener("close", onClose);
31
+ if (stream.status === "open" && stream.remoteWriteStatus === "writable") {
32
+ stream.abort(getStreamNotFullyConsumedError());
33
+ }
34
+ }, timeoutMs);
35
+
36
+ stream.addEventListener("close", onClose, {once: true});
37
+ }
38
+
39
+ type ClearableSignal = AbortSignal & {clear: () => void};
40
+
41
+ /**
42
+ * Compose an abort signal from an optional parent signal and a timeout, with explicit cleanup.
43
+ *
44
+ * This replaces a plain `AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)])` to work
45
+ * around a memory leak in Node.js where AbortSignal.any() retains references to all source
46
+ * signals for the lifetime of the longest-lived signal (the timeout). In a long-running
47
+ * req/resp workload this causes unbounded growth of the dependent-signal set.
48
+ *
49
+ * Upstream issue: https://github.com/nodejs/node/issues/54614
50
+ * Lodestar investigation: https://github.com/ChainSafe/lodestar/issues/8969
51
+ */
52
+ function createRespSignal(signal: AbortSignal | undefined, timeoutMs: number): ClearableSignal {
53
+ const timeoutSignal = AbortSignal.timeout(timeoutMs);
54
+ const signals = signal ? [signal, timeoutSignal] : [timeoutSignal];
55
+ const controller = new AbortController();
56
+
57
+ const clear = (): void => {
58
+ for (const entry of signals) {
59
+ entry.removeEventListener("abort", onAbort);
60
+ }
61
+ };
62
+
63
+ const onAbort = (): void => {
64
+ if (controller.signal.aborted) {
65
+ return;
66
+ }
67
+ const reason = signals.find((entry) => entry.aborted)?.reason;
68
+ controller.abort(reason);
69
+ clear();
70
+ };
71
+
72
+ for (const entry of signals) {
73
+ if (entry.aborted) {
74
+ onAbort();
75
+ break;
76
+ }
77
+ entry.addEventListener("abort", onAbort);
78
+ }
79
+
80
+ const respSignal = controller.signal as ClearableSignal;
81
+ respSignal.clear = clear;
82
+
83
+ return respSignal;
84
+ }
85
+
22
86
  export interface SendRequestOpts {
23
87
  /** The maximum time for complete response transfer. */
24
88
  respTimeoutMs?: number;
25
89
  /** Non-spec timeout from sending request until write stream closed by responder */
26
90
  requestTimeoutMs?: number;
27
- /** The maximum time to wait for first byte of request response (time-to-first-byte). */
28
- ttfbTimeoutMs?: number;
29
91
  /** Non-spec timeout from dialing protocol until stream opened */
30
92
  dialTimeoutMs?: number;
31
93
  }
@@ -64,7 +126,6 @@ export async function* sendRequest(
64
126
 
65
127
  const DIAL_TIMEOUT = opts?.dialTimeoutMs ?? DEFAULT_DIAL_TIMEOUT;
66
128
  const REQUEST_TIMEOUT = opts?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT;
67
- const TTFB_TIMEOUT = opts?.ttfbTimeoutMs ?? DEFAULT_TTFB_TIMEOUT;
68
129
  const RESP_TIMEOUT = opts?.respTimeoutMs ?? DEFAULT_RESP_TIMEOUT;
69
130
 
70
131
  const peerIdStrShort = prettyPrintPeerId(peerId);
@@ -83,17 +144,6 @@ export async function* sendRequest(
83
144
  // the picked protocol in `connection.protocol`
84
145
  const protocolsMap = new Map<string, MixedProtocol>(protocols.map((protocol, i) => [protocolIDs[i], protocol]));
85
146
 
86
- // As of October 2020 we can't rely on libp2p.dialProtocol timeout to work so
87
- // this function wraps the dialProtocol promise with an extra timeout
88
- //
89
- // > The issue might be: you add the peer's addresses to the AddressBook,
90
- // which will result in autoDial to kick in and dial your peer. In parallel,
91
- // you do a manual dial and it will wait for the previous one without using
92
- // the abort signal:
93
- //
94
- // https://github.com/ChainSafe/lodestar/issues/1597#issuecomment-703394386
95
-
96
- // DIAL_TIMEOUT: Non-spec timeout from dialing protocol until stream opened
97
147
  const stream = await withTimeout(
98
148
  async (timeoutAndParentSignal) => {
99
149
  const protocolIds = Array.from(protocolsMap.keys());
@@ -112,9 +162,6 @@ export async function* sendRequest(
112
162
 
113
163
  metrics?.outgoingOpenedStreams?.inc({method});
114
164
 
115
- // TODO: Does the TTFB timer start on opening stream or after receiving request
116
- const timerTTFB = metrics?.outgoingResponseTTFB.startTimer({method});
117
-
118
165
  // Parse protocol selected by the responder
119
166
  const protocolId = stream.protocol ?? "unknown";
120
167
  const protocol = protocolsMap.get(protocolId);
@@ -126,21 +173,24 @@ export async function* sendRequest(
126
173
  logger.debug("Req sending request", logCtx);
127
174
 
128
175
  // Spec: The requester MUST close the write side of the stream once it finishes writing the request message
129
- // Impl: stream.sink is closed automatically by js-libp2p-mplex when piped source is exhausted
130
176
 
131
177
  // REQUEST_TIMEOUT: Non-spec timeout from sending request until write stream closed by responder
132
- // Note: libp2p.stop() will close all connections, so not necessary to abort this pipe on parent stop
133
- await withTimeout(() => pipe(requestEncode(protocol, requestBody), stream.sink), REQUEST_TIMEOUT, signal).catch(
134
- (e) => {
135
- // Must close the stream read side (stream.source) manually AND the write side
136
- stream.abort(e);
137
-
138
- if (e instanceof TimeoutError) {
139
- throw new RequestError({code: RequestErrorCode.REQUEST_TIMEOUT});
140
- }
141
- throw new RequestError({code: RequestErrorCode.REQUEST_ERROR, error: e as Error});
178
+ // Note: libp2p.stop() will close all connections, so not necessary to abort this send on parent stop
179
+ await withTimeout(
180
+ async (timeoutAndParentSignal) => {
181
+ await sendChunks(stream, requestEncode(protocol, requestBody), timeoutAndParentSignal);
182
+ await stream.close({signal: timeoutAndParentSignal});
183
+ },
184
+ REQUEST_TIMEOUT,
185
+ signal
186
+ ).catch((e) => {
187
+ stream.abort(e as Error);
188
+
189
+ if (e instanceof TimeoutError) {
190
+ throw new RequestError({code: RequestErrorCode.REQUEST_TIMEOUT});
142
191
  }
143
- );
192
+ throw new RequestError({code: RequestErrorCode.REQUEST_ERROR, error: e as Error});
193
+ });
144
194
 
145
195
  logger.debug("Req request sent", logCtx);
146
196
 
@@ -150,67 +200,51 @@ export async function* sendRequest(
150
200
  return;
151
201
  }
152
202
 
153
- // - TTFB_TIMEOUT: The requester MUST wait a maximum of TTFB_TIMEOUT for the first response byte to arrive
154
- // - RESP_TIMEOUT: Requester allows a further RESP_TIMEOUT for each subsequent response_chunk
155
- // - Max total timeout: This timeout is not required by the spec. It may not be necessary, but it's kept as
156
- // safe-guard to close. streams in case of bugs on other timeout mechanisms.
157
- const ttfbTimeoutController = new AbortController();
158
- const respTimeoutController = new AbortController();
159
-
160
- let timeoutRESP: NodeJS.Timeout | null = null;
203
+ // RESP_TIMEOUT: Maximum time for complete response transfer
204
+ const respSignal = createRespSignal(signal, RESP_TIMEOUT);
161
205
 
162
- const timeoutTTFB = setTimeout(() => {
163
- // If we abort on first byte delay, don't need to abort for response delay
164
- if (timeoutRESP) clearTimeout(timeoutRESP);
165
- ttfbTimeoutController.abort();
166
- }, TTFB_TIMEOUT);
167
-
168
- const restartRespTimeout = (): void => {
169
- if (timeoutRESP) clearTimeout(timeoutRESP);
170
- timeoutRESP = setTimeout(() => respTimeoutController.abort(), RESP_TIMEOUT);
171
- };
206
+ let responseError: Error | null = null;
207
+ let responseFullyConsumed = false;
172
208
 
173
209
  try {
174
- // Note: libp2p.stop() will close all connections, so not necessary to abort this pipe on parent stop
175
- yield* pipe(
176
- abortableSource(stream.source as AsyncIterable<Uint8ArrayList>, [
177
- {
178
- signal: ttfbTimeoutController.signal,
179
- getError: () => new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}),
180
- },
181
- {
182
- signal: respTimeoutController.signal,
183
- getError: () => new RequestError({code: RequestErrorCode.RESP_TIMEOUT}),
184
- },
185
- ]),
186
-
187
- // Transforms `Buffer` chunks to yield `ResponseBody` chunks
188
- responseDecode(protocol, {
189
- onFirstHeader() {
190
- // On first byte, cancel the single use TTFB_TIMEOUT, and start RESP_TIMEOUT
191
- clearTimeout(timeoutTTFB);
192
- timerTTFB?.();
193
- restartRespTimeout();
194
- },
195
- onFirstResponseChunk() {
196
- // On <response_chunk>, cancel this chunk's RESP_TIMEOUT and start next's
197
- restartRespTimeout();
198
- },
199
- })
200
- );
210
+ yield* responseDecode(protocol, stream, {
211
+ signal: respSignal,
212
+ getError: () =>
213
+ signal?.aborted ? new ErrorAborted("sendRequest") : new RequestError({code: RequestErrorCode.RESP_TIMEOUT}),
214
+ });
215
+ responseFullyConsumed = true;
201
216
 
202
217
  // NOTE: Only log once per request to verbose, intermediate steps to debug
203
218
  // NOTE: Do not log the response, logs get extremely cluttered
204
219
  // NOTE: add double space after "Req " to align log with the "Resp " log
205
220
  logger.verbose("Req done", logCtx);
221
+ } catch (e) {
222
+ responseError = e as Error;
223
+ throw e;
206
224
  } finally {
207
- clearTimeout(timeoutTTFB);
208
- if (timeoutRESP !== null) clearTimeout(timeoutRESP);
225
+ // On decode/timeout failures abort immediately so mplex can reclaim stream state.
226
+ // On normal early consumer exit, close gracefully to avoid stream-id desync with peers.
227
+ if (responseError !== null || signal?.aborted) {
228
+ stream.abort(responseError ?? new ErrorAborted("sendRequest"));
229
+ } else {
230
+ await stream.close().catch((e) => {
231
+ stream.abort(e as Error);
232
+ });
233
+
234
+ if (!responseFullyConsumed) {
235
+ // Stop buffering unread inbound data after caller exits early.
236
+ // mplex does not support propagating closeRead to the remote, so still
237
+ // abort later if the remote never closes write.
238
+ await stream.closeRead().catch(() => {
239
+ // Ignore closeRead errors - close/abort path below will reclaim stream.
240
+ });
209
241
 
210
- // Necessary to call `stream.close()` since collectResponses() may break out of the source before exhausting it
211
- // `stream.close()` libp2p-mplex will .end() the source (it-pushable instance)
212
- // If collectResponses() exhausts the source, it-pushable.end() can be safely called multiple times
213
- await stream.close();
242
+ if (stream.remoteWriteStatus === "writable") {
243
+ scheduleStreamAbortIfNotClosed(stream, RESP_TIMEOUT);
244
+ }
245
+ }
246
+ }
247
+ respSignal.clear();
214
248
  metrics?.outgoingClosedStreams?.inc({method});
215
249
  logger.verbose("Req stream closed", logCtx);
216
250
  }