@lodestar/reqresp 1.41.0-dev.a35cbde8b3 → 1.41.0-dev.aeab9f930d

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/README.md +1 -1
  2. package/lib/ReqResp.d.ts +3 -3
  3. package/lib/ReqResp.d.ts.map +1 -1
  4. package/lib/ReqResp.js +4 -4
  5. package/lib/ReqResp.js.map +1 -1
  6. package/lib/encoders/requestDecode.d.ts +2 -3
  7. package/lib/encoders/requestDecode.d.ts.map +1 -1
  8. package/lib/encoders/requestDecode.js +28 -11
  9. package/lib/encoders/requestDecode.js.map +1 -1
  10. package/lib/encoders/requestEncode.d.ts +1 -1
  11. package/lib/encoders/requestEncode.d.ts.map +1 -1
  12. package/lib/encoders/requestEncode.js +1 -1
  13. package/lib/encoders/requestEncode.js.map +1 -1
  14. package/lib/encoders/responseDecode.d.ts +10 -10
  15. package/lib/encoders/responseDecode.d.ts.map +1 -1
  16. package/lib/encoders/responseDecode.js +63 -60
  17. package/lib/encoders/responseDecode.js.map +1 -1
  18. package/lib/encoders/responseEncode.d.ts +2 -4
  19. package/lib/encoders/responseEncode.d.ts.map +1 -1
  20. package/lib/encoders/responseEncode.js +13 -22
  21. package/lib/encoders/responseEncode.js.map +1 -1
  22. package/lib/encodingStrategies/index.d.ts +4 -3
  23. package/lib/encodingStrategies/index.d.ts.map +1 -1
  24. package/lib/encodingStrategies/index.js +4 -4
  25. package/lib/encodingStrategies/index.js.map +1 -1
  26. package/lib/encodingStrategies/sszSnappy/decode.d.ts +5 -4
  27. package/lib/encodingStrategies/sszSnappy/decode.d.ts.map +1 -1
  28. package/lib/encodingStrategies/sszSnappy/decode.js +83 -52
  29. package/lib/encodingStrategies/sszSnappy/decode.js.map +1 -1
  30. package/lib/encodingStrategies/sszSnappy/encode.d.ts +2 -2
  31. package/lib/encodingStrategies/sszSnappy/encode.d.ts.map +1 -1
  32. package/lib/encodingStrategies/sszSnappy/encode.js +1 -1
  33. package/lib/encodingStrategies/sszSnappy/encode.js.map +1 -1
  34. package/lib/encodingStrategies/sszSnappy/errors.d.ts +0 -8
  35. package/lib/encodingStrategies/sszSnappy/errors.d.ts.map +1 -1
  36. package/lib/encodingStrategies/sszSnappy/errors.js +2 -3
  37. package/lib/encodingStrategies/sszSnappy/errors.js.map +1 -1
  38. package/lib/encodingStrategies/sszSnappy/index.d.ts +0 -1
  39. package/lib/encodingStrategies/sszSnappy/index.d.ts.map +1 -1
  40. package/lib/encodingStrategies/sszSnappy/index.js +0 -1
  41. package/lib/encodingStrategies/sszSnappy/index.js.map +1 -1
  42. package/lib/encodingStrategies/sszSnappy/utils.js.map +1 -1
  43. package/lib/interface.js +2 -1
  44. package/lib/interface.js.map +1 -1
  45. package/lib/metrics.d.ts +1 -7
  46. package/lib/metrics.d.ts.map +1 -1
  47. package/lib/metrics.js +1 -17
  48. package/lib/metrics.js.map +1 -1
  49. package/lib/rate_limiter/ReqRespRateLimiter.d.ts.map +1 -1
  50. package/lib/rate_limiter/ReqRespRateLimiter.js.map +1 -1
  51. package/lib/rate_limiter/rateLimiterGRCA.d.ts.map +1 -1
  52. package/lib/rate_limiter/rateLimiterGRCA.js.map +1 -1
  53. package/lib/rate_limiter/selfRateLimiter.d.ts.map +1 -1
  54. package/lib/rate_limiter/selfRateLimiter.js.map +1 -1
  55. package/lib/request/errors.d.ts +1 -7
  56. package/lib/request/errors.d.ts.map +1 -1
  57. package/lib/request/errors.js +3 -6
  58. package/lib/request/errors.js.map +1 -1
  59. package/lib/request/index.d.ts +0 -3
  60. package/lib/request/index.d.ts.map +1 -1
  61. package/lib/request/index.js +85 -70
  62. package/lib/request/index.js.map +1 -1
  63. package/lib/response/errors.d.ts.map +1 -1
  64. package/lib/response/errors.js +2 -1
  65. package/lib/response/errors.js.map +1 -1
  66. package/lib/response/index.d.ts +2 -2
  67. package/lib/response/index.d.ts.map +1 -1
  68. package/lib/response/index.js +46 -50
  69. package/lib/response/index.js.map +1 -1
  70. package/lib/types.d.ts +1 -2
  71. package/lib/types.d.ts.map +1 -1
  72. package/lib/types.js +7 -5
  73. package/lib/types.js.map +1 -1
  74. package/lib/utils/collectExactOne.js.map +1 -1
  75. package/lib/utils/collectMaxResponse.d.ts.map +1 -1
  76. package/lib/utils/collectMaxResponse.js +1 -2
  77. package/lib/utils/collectMaxResponse.js.map +1 -1
  78. package/lib/utils/errorMessage.d.ts +3 -3
  79. package/lib/utils/errorMessage.d.ts.map +1 -1
  80. package/lib/utils/errorMessage.js +14 -13
  81. package/lib/utils/errorMessage.js.map +1 -1
  82. package/lib/utils/index.d.ts +1 -3
  83. package/lib/utils/index.d.ts.map +1 -1
  84. package/lib/utils/index.js +1 -3
  85. package/lib/utils/index.js.map +1 -1
  86. package/lib/utils/peerId.js.map +1 -1
  87. package/lib/utils/protocolId.d.ts +2 -2
  88. package/lib/utils/protocolId.js +2 -2
  89. package/lib/utils/protocolId.js.map +1 -1
  90. package/lib/utils/snappyCommon.js +2 -1
  91. package/lib/utils/snappyCommon.js.map +1 -1
  92. package/lib/utils/snappyCompress.d.ts +1 -1
  93. package/lib/utils/snappyCompress.d.ts.map +1 -1
  94. package/lib/utils/snappyCompress.js +1 -1
  95. package/lib/utils/snappyCompress.js.map +1 -1
  96. package/lib/utils/snappyIndex.d.ts +1 -1
  97. package/lib/utils/snappyIndex.d.ts.map +1 -1
  98. package/lib/utils/snappyIndex.js +1 -1
  99. package/lib/utils/snappyIndex.js.map +1 -1
  100. package/lib/utils/snappyUncompress.d.ts +7 -11
  101. package/lib/utils/snappyUncompress.d.ts.map +1 -1
  102. package/lib/utils/snappyUncompress.js +68 -68
  103. package/lib/utils/snappyUncompress.js.map +1 -1
  104. package/lib/utils/stream.d.ts +6 -0
  105. package/lib/utils/stream.d.ts.map +1 -0
  106. package/lib/utils/stream.js +21 -0
  107. package/lib/utils/stream.js.map +1 -0
  108. package/package.json +16 -18
  109. package/src/ReqResp.ts +4 -4
  110. package/src/encoders/requestDecode.ts +32 -16
  111. package/src/encoders/requestEncode.ts +1 -1
  112. package/src/encoders/responseDecode.ts +68 -72
  113. package/src/encoders/responseEncode.ts +17 -29
  114. package/src/encodingStrategies/index.ts +8 -6
  115. package/src/encodingStrategies/sszSnappy/decode.ts +111 -53
  116. package/src/encodingStrategies/sszSnappy/encode.ts +2 -2
  117. package/src/encodingStrategies/sszSnappy/errors.ts +0 -4
  118. package/src/encodingStrategies/sszSnappy/index.ts +0 -1
  119. package/src/metrics.ts +1 -17
  120. package/src/request/errors.ts +1 -6
  121. package/src/request/index.ts +109 -86
  122. package/src/response/index.ts +55 -61
  123. package/src/types.ts +1 -3
  124. package/src/utils/collectMaxResponse.ts +1 -2
  125. package/src/utils/errorMessage.ts +14 -13
  126. package/src/utils/index.ts +1 -3
  127. package/src/utils/protocolId.ts +2 -2
  128. package/src/utils/snappyCompress.ts +1 -1
  129. package/src/utils/snappyIndex.ts +1 -1
  130. package/src/utils/snappyUncompress.ts +73 -75
  131. package/src/utils/stream.ts +34 -0
  132. package/lib/utils/abortableSource.d.ts +0 -12
  133. package/lib/utils/abortableSource.d.ts.map +0 -1
  134. package/lib/utils/abortableSource.js +0 -69
  135. package/lib/utils/abortableSource.js.map +0 -1
  136. package/lib/utils/bufferedSource.d.ts +0 -16
  137. package/lib/utils/bufferedSource.d.ts.map +0 -1
  138. package/lib/utils/bufferedSource.js +0 -40
  139. package/lib/utils/bufferedSource.js.map +0 -1
  140. package/lib/utils/onChunk.d.ts +0 -6
  141. package/lib/utils/onChunk.d.ts.map +0 -1
  142. package/lib/utils/onChunk.js +0 -13
  143. package/lib/utils/onChunk.js.map +0 -1
  144. package/lib/utils/snappy.d.ts +0 -3
  145. package/lib/utils/snappy.d.ts.map +0 -1
  146. package/lib/utils/snappy.js +0 -3
  147. package/lib/utils/snappy.js.map +0 -1
  148. package/src/utils/abortableSource.ts +0 -80
  149. package/src/utils/bufferedSource.ts +0 -46
  150. package/src/utils/onChunk.ts +0 -12
  151. package/src/utils/snappy.ts +0 -2
@@ -1,8 +1,9 @@
1
+ import type {Stream} from "@libp2p/interface";
2
+ import type {ByteStream} from "@libp2p/utils";
1
3
  import {decode as varintDecode, encodingLength as varintEncodingLength} from "uint8-varint";
2
4
  import {Uint8ArrayList} from "uint8arraylist";
3
5
  import {TypeSizes} from "../../types.js";
4
- import {BufferedSource} from "../../utils/index.js";
5
- import {SnappyFramesUncompress} from "../../utils/snappyIndex.js";
6
+ import {ChunkType, decodeSnappyFrameData, parseSnappyFrameHeader} from "../../utils/snappyIndex.js";
6
7
  import {SszSnappyError, SszSnappyErrorCode} from "./errors.js";
7
8
  import {maxEncodedLen} from "./utils.js";
8
9
 
@@ -15,76 +16,115 @@ export const MAX_VARINT_BYTES = 10;
15
16
  * <encoding-dependent-header> | <encoded-payload>
16
17
  * ```
17
18
  */
18
- export async function readSszSnappyPayload(bufferedSource: BufferedSource, type: TypeSizes): Promise<Uint8Array> {
19
- const sszDataLength = await readSszSnappyHeader(bufferedSource, type);
20
-
21
- return readSszSnappyBody(bufferedSource, sszDataLength);
19
+ export async function readSszSnappyPayload(
20
+ stream: ByteStream<Stream>,
21
+ type: TypeSizes,
22
+ signal?: AbortSignal
23
+ ): Promise<Uint8Array> {
24
+ const sszDataLength = await readSszSnappyHeader(stream, type, signal);
25
+
26
+ return readSszSnappyBody(stream, sszDataLength, signal);
22
27
  }
23
28
 
24
29
  /**
25
30
  * Reads `<encoding-dependent-header>` for ssz-snappy.
26
31
  * encoding-header ::= the length of the raw SSZ bytes, encoded as an unsigned protobuf varint
27
32
  */
28
- export async function readSszSnappyHeader(bufferedSource: BufferedSource, type: TypeSizes): Promise<number> {
29
- for await (const buffer of bufferedSource) {
30
- // Get next bytes if empty
31
- if (buffer.length === 0) {
32
- continue;
33
- }
33
+ export async function readSszSnappyHeader(
34
+ stream: ByteStream<Stream>,
35
+ type: TypeSizes,
36
+ signal?: AbortSignal
37
+ ): Promise<number> {
38
+ const varintBytes: number[] = [];
34
39
 
35
- let sszDataLength: number;
36
- try {
37
- sszDataLength = varintDecode(buffer.subarray());
38
- } catch (_e) {
39
- throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity});
40
- }
40
+ while (true) {
41
+ const byte = await readExactOrSourceAborted(stream, 1, signal);
41
42
 
42
- // MUST validate: the unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes
43
- // encodingLength function only returns 1-8 inclusive
44
- const varintBytes = varintEncodingLength(sszDataLength);
45
- buffer.consume(varintBytes);
43
+ const value = byte.get(0);
46
44
 
47
- // MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type.
48
- const minSize = type.minSize;
49
- const maxSize = type.maxSize;
50
- if (sszDataLength < minSize) {
51
- throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength});
52
- }
53
- if (sszDataLength > maxSize) {
54
- throw new SszSnappyError({code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE, maxSize, sszDataLength});
45
+ varintBytes.push(value);
46
+ if (varintBytes.length > MAX_VARINT_BYTES) {
47
+ throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintBytes.length});
55
48
  }
56
49
 
57
- return sszDataLength;
50
+ // MSB not set => varint terminated
51
+ if ((value & 0x80) === 0) break;
52
+ }
53
+
54
+ let sszDataLength: number;
55
+ try {
56
+ sszDataLength = varintDecode(Uint8Array.from(varintBytes));
57
+ } catch {
58
+ throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity});
58
59
  }
59
60
 
60
- throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED});
61
+ // MUST validate: the unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes
62
+ // encodingLength function only returns 1-8 inclusive
63
+ const varintByteLength = varintEncodingLength(sszDataLength);
64
+ if (varintByteLength > MAX_VARINT_BYTES) {
65
+ throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintByteLength});
66
+ }
67
+
68
+ // MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type.
69
+ const minSize = type.minSize;
70
+ const maxSize = type.maxSize;
71
+ if (sszDataLength < minSize) {
72
+ throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength});
73
+ }
74
+ if (sszDataLength > maxSize) {
75
+ throw new SszSnappyError({code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE, maxSize, sszDataLength});
76
+ }
77
+
78
+ return sszDataLength;
61
79
  }
62
80
 
63
81
  /**
64
82
  * Reads `<encoded-payload>` for ssz-snappy and decompress.
65
83
  * The returned bytes can be SSZ deseralized
66
84
  */
67
- export async function readSszSnappyBody(bufferedSource: BufferedSource, sszDataLength: number): Promise<Uint8Array> {
68
- const decompressor = new SnappyFramesUncompress();
85
+ export async function readSszSnappyBody(
86
+ stream: ByteStream<Stream>,
87
+ sszDataLength: number,
88
+ signal?: AbortSignal
89
+ ): Promise<Uint8Array> {
69
90
  const uncompressedData = new Uint8ArrayList();
70
- let readBytes = 0;
91
+ let encodedBytesRead = 0;
92
+ const maxBytes = maxEncodedLen(sszDataLength);
93
+ let foundIdentifier = false;
94
+
95
+ while (uncompressedData.length < sszDataLength) {
96
+ const header = await readExactOrSourceAborted(stream, 4, signal);
71
97
 
72
- for await (const buffer of bufferedSource) {
73
98
  // SHOULD NOT read more than max_encoded_len(n) bytes after reading the SSZ length-prefix n from the header
74
- readBytes += buffer.length;
75
- if (readBytes > maxEncodedLen(sszDataLength)) {
76
- throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, readBytes, sszDataLength});
99
+ encodedBytesRead = addEncodedBytesReadOrThrow(encodedBytesRead, header.length, maxBytes, sszDataLength);
100
+
101
+ let headerParsed: {type: ChunkType; frameSize: number};
102
+ try {
103
+ headerParsed = parseSnappyFrameHeader(header.subarray());
104
+ if (!foundIdentifier && headerParsed.type !== ChunkType.IDENTIFIER) {
105
+ throw new Error("malformed input: must begin with an identifier");
106
+ }
107
+ } catch (e) {
108
+ throw new SszSnappyError({code: SszSnappyErrorCode.DECOMPRESSOR_ERROR, decompressorError: e as Error});
77
109
  }
78
110
 
79
- // No bytes left to consume, get next
80
- if (buffer.length === 0) {
81
- continue;
111
+ if (headerParsed.frameSize > maxBytes - encodedBytesRead) {
112
+ throw new SszSnappyError({
113
+ code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ,
114
+ readBytes: encodedBytesRead + headerParsed.frameSize,
115
+ sszDataLength,
116
+ });
82
117
  }
118
+ const frame = await readExactOrSourceAborted(stream, headerParsed.frameSize, signal);
119
+
120
+ encodedBytesRead = addEncodedBytesReadOrThrow(encodedBytesRead, frame.length, maxBytes, sszDataLength);
83
121
 
84
- // stream contents can be passed through a buffered Snappy reader to decompress frame by frame
85
122
  try {
86
- const uncompressed = decompressor.uncompress(buffer);
87
- buffer.consume(buffer.length);
123
+ if (headerParsed.type === ChunkType.IDENTIFIER) {
124
+ foundIdentifier = true;
125
+ }
126
+
127
+ const uncompressed = decodeSnappyFrameData(headerParsed.type, frame.subarray());
88
128
  if (uncompressed !== null) {
89
129
  uncompressedData.append(uncompressed);
90
130
  }
@@ -96,16 +136,34 @@ export async function readSszSnappyBody(bufferedSource: BufferedSource, sszDataL
96
136
  if (uncompressedData.length > sszDataLength) {
97
137
  throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MANY_BYTES, sszDataLength});
98
138
  }
139
+ }
99
140
 
100
- // Keep reading chunks until `n` SSZ bytes
101
- if (uncompressedData.length < sszDataLength) {
102
- continue;
103
- }
141
+ // buffer.length === n
142
+ return uncompressedData.subarray(0, sszDataLength);
143
+ }
104
144
 
105
- // buffer.length === n
106
- return uncompressedData.subarray(0, sszDataLength);
145
+ function addEncodedBytesReadOrThrow(
146
+ encodedBytesRead: number,
147
+ bytesToAdd: number,
148
+ maxBytes: number,
149
+ sszDataLength: number
150
+ ): number {
151
+ const nextReadBytes = encodedBytesRead + bytesToAdd;
152
+ if (nextReadBytes > maxBytes) {
153
+ throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, readBytes: nextReadBytes, sszDataLength});
107
154
  }
155
+ return nextReadBytes;
156
+ }
108
157
 
109
- // SHOULD consider invalid: An early EOF before fully reading the declared length-prefix worth of SSZ bytes
110
- throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED});
158
+ async function readExactOrSourceAborted(
159
+ stream: ByteStream<Stream>,
160
+ bytes: number,
161
+ signal?: AbortSignal
162
+ ): Promise<Uint8ArrayList> {
163
+ return stream.read({bytes, signal}).catch((e) => {
164
+ if ((e as Error).name === "UnexpectedEOFError") {
165
+ throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED});
166
+ }
167
+ throw e;
168
+ });
111
169
  }
@@ -8,12 +8,12 @@ import {encodeSnappy} from "../../utils/snappyIndex.js";
8
8
  * <encoding-dependent-header> | <encoded-payload>
9
9
  * ```
10
10
  */
11
- export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => AsyncGenerator<Buffer>;
11
+ export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => Generator<Buffer>;
12
12
 
13
13
  /**
14
14
  * Buffered Snappy writer
15
15
  */
16
- export async function* encodeSszSnappy(bytes: Buffer): AsyncGenerator<Buffer> {
16
+ export function* encodeSszSnappy(bytes: Buffer): Generator<Buffer> {
17
17
  // MUST encode the length of the raw SSZ bytes, encoded as an unsigned protobuf varint
18
18
  const varint = varintEncode(bytes.length);
19
19
  yield Buffer.from(varint.buffer, varint.byteOffset, varint.byteLength);
@@ -9,8 +9,6 @@ export enum SszSnappyErrorCode {
9
9
  OVER_SSZ_MAX_SIZE = "SSZ_SNAPPY_ERROR_OVER_SSZ_MAX_SIZE",
10
10
  TOO_MUCH_BYTES_READ = "SSZ_SNAPPY_ERROR_TOO_MUCH_BYTES_READ",
11
11
  DECOMPRESSOR_ERROR = "SSZ_SNAPPY_ERROR_DECOMPRESSOR_ERROR",
12
- DESERIALIZE_ERROR = "SSZ_SNAPPY_ERROR_DESERIALIZE_ERROR",
13
- SERIALIZE_ERROR = "SSZ_SNAPPY_ERROR_SERIALIZE_ERROR",
14
12
  /** Received more bytes than specified sszDataLength */
15
13
  TOO_MANY_BYTES = "SSZ_SNAPPY_ERROR_TOO_MANY_BYTES",
16
14
  /** Source aborted before reading sszDataLength bytes */
@@ -23,8 +21,6 @@ type SszSnappyErrorType =
23
21
  | {code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE; maxSize: number; sszDataLength: number}
24
22
  | {code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ; readBytes: number; sszDataLength: number}
25
23
  | {code: SszSnappyErrorCode.DECOMPRESSOR_ERROR; decompressorError: Error}
26
- | {code: SszSnappyErrorCode.DESERIALIZE_ERROR; deserializeError: Error}
27
- | {code: SszSnappyErrorCode.SERIALIZE_ERROR; serializeError: Error}
28
24
  | {code: SszSnappyErrorCode.TOO_MANY_BYTES; sszDataLength: number}
29
25
  | {code: SszSnappyErrorCode.SOURCE_ABORTED};
30
26
 
@@ -1,4 +1,3 @@
1
- export {SnappyFramesUncompress, encodeSnappy} from "../../utils/snappyIndex.js";
2
1
  export * from "./decode.js";
3
2
  export * from "./encode.js";
4
3
  export * from "./errors.js";
package/src/metrics.ts CHANGED
@@ -4,7 +4,7 @@ import {RequestErrorCode} from "./request/errors.js";
4
4
  export type Metrics = ReturnType<typeof getMetrics>;
5
5
 
6
6
  /**
7
- * A collection of metrics used throughout the Gossipsub behaviour.
7
+ * A collection of metrics used throughout the Req/Resp domain.
8
8
  */
9
9
  export function getMetrics(register: MetricsRegisterExtra) {
10
10
  // Using function style instead of class to prevent having to re-declare all MetricsPrometheus types.
@@ -29,7 +29,6 @@ export function getMetrics(register: MetricsRegisterExtra) {
29
29
  name: "beacon_reqresp_outgoing_request_roundtrip_time_seconds",
30
30
  help: "Histogram of outgoing requests round-trip time",
31
31
  labelNames: ["method"],
32
- // Spec sets RESP_TIMEOUT = 10 sec
33
32
  buckets: [0.1, 0.2, 0.5, 1, 5, 10, 15, 60],
34
33
  }),
35
34
  outgoingErrors: register.gauge<{method: string}>({
@@ -61,7 +60,6 @@ export function getMetrics(register: MetricsRegisterExtra) {
61
60
  name: "beacon_reqresp_incoming_request_handler_time_seconds",
62
61
  help: "Histogram of incoming requests internal handling time",
63
62
  labelNames: ["method"],
64
- // Spec sets RESP_TIMEOUT = 10 sec
65
63
  buckets: [0.1, 0.2, 0.5, 1, 5, 10],
66
64
  }),
67
65
  incomingErrors: register.gauge<{method: string}>({
@@ -69,20 +67,6 @@ export function getMetrics(register: MetricsRegisterExtra) {
69
67
  help: "Counts total failed responses handled per method",
70
68
  labelNames: ["method"],
71
69
  }),
72
- outgoingResponseTTFB: register.histogram<{method: string}>({
73
- name: "beacon_reqresp_outgoing_response_ttfb_seconds",
74
- help: "Time to first byte (TTFB) for outgoing responses",
75
- labelNames: ["method"],
76
- // Spec sets TTFB_TIMEOUT = 5 sec
77
- buckets: [0.1, 1, 5],
78
- }),
79
- incomingResponseTTFB: register.histogram<{method: string}>({
80
- name: "beacon_reqresp_incoming_response_ttfb_seconds",
81
- help: "Time to first byte (TTFB) for incoming responses",
82
- labelNames: ["method"],
83
- // Spec sets TTFB_TIMEOUT = 5 sec
84
- buckets: [0.1, 1, 5],
85
- }),
86
70
  dialErrors: register.gauge({
87
71
  name: "beacon_reqresp_dial_errors_total",
88
72
  help: "Count total dial errors",
@@ -21,13 +21,9 @@ export enum RequestErrorCode {
21
21
  REQUEST_TIMEOUT = "REQUEST_ERROR_REQUEST_TIMEOUT",
22
22
  /** Error when sending request to responder */
23
23
  REQUEST_ERROR = "REQUEST_ERROR_REQUEST_ERROR",
24
- /** Reponder did not deliver a full reponse before max maxTotalResponseTimeout() */
25
- RESPONSE_TIMEOUT = "REQUEST_ERROR_RESPONSE_TIMEOUT",
26
24
  /** A single-response method returned 0 chunks */
27
25
  EMPTY_RESPONSE = "REQUEST_ERROR_EMPTY_RESPONSE",
28
- /** Time to first byte timeout */
29
- TTFB_TIMEOUT = "REQUEST_ERROR_TTFB_TIMEOUT",
30
- /** Timeout between `<response_chunk>` exceed */
26
+ /** Response transfer timeout exceeded */
31
27
  RESP_TIMEOUT = "REQUEST_ERROR_RESP_TIMEOUT",
32
28
  /** Request rate limited */
33
29
  REQUEST_RATE_LIMITED = "REQUEST_ERROR_RATE_LIMITED",
@@ -50,7 +46,6 @@ type RequestErrorType =
50
46
  | {code: RequestErrorCode.REQUEST_TIMEOUT}
51
47
  | {code: RequestErrorCode.REQUEST_ERROR; error: Error}
52
48
  | {code: RequestErrorCode.EMPTY_RESPONSE}
53
- | {code: RequestErrorCode.TTFB_TIMEOUT}
54
49
  | {code: RequestErrorCode.RESP_TIMEOUT}
55
50
  | {code: RequestErrorCode.REQUEST_RATE_LIMITED}
56
51
  | {code: RequestErrorCode.REQUEST_SELF_RATE_LIMITED}
@@ -1,31 +1,82 @@
1
+ import type {Stream} from "@libp2p/interface";
1
2
  import {PeerId} from "@libp2p/interface";
2
- import {pipe} from "it-pipe";
3
3
  import type {Libp2p} from "libp2p";
4
- import {Uint8ArrayList} from "uint8arraylist";
5
4
  import {ErrorAborted, Logger, TimeoutError, withTimeout} from "@lodestar/utils";
6
5
  import {requestEncode} from "../encoders/requestEncode.js";
7
6
  import {responseDecode} from "../encoders/responseDecode.js";
8
7
  import {Metrics} from "../metrics.js";
9
8
  import {ResponseError} from "../response/index.js";
10
9
  import {MixedProtocol, ResponseIncoming} from "../types.js";
11
- import {abortableSource, prettyPrintPeerId} from "../utils/index.js";
10
+ import {prettyPrintPeerId, sendChunks} from "../utils/index.js";
12
11
  import {RequestError, RequestErrorCode, responseStatusErrorToRequestError} from "./errors.js";
13
12
 
14
13
  export {RequestError, RequestErrorCode};
15
14
 
16
- // Default spec values from https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#configuration
15
+ // https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#the-reqresp-domain
17
16
  export const DEFAULT_DIAL_TIMEOUT = 5 * 1000; // 5 sec
18
17
  export const DEFAULT_REQUEST_TIMEOUT = 5 * 1000; // 5 sec
19
- export const DEFAULT_TTFB_TIMEOUT = 5 * 1000; // 5 sec
20
18
  export const DEFAULT_RESP_TIMEOUT = 10 * 1000; // 10 sec
21
19
 
20
+ function getStreamNotFullyConsumedError(): Error {
21
+ return new Error("ReqResp stream was not fully consumed");
22
+ }
23
+
24
+ function scheduleStreamAbortIfNotClosed(stream: Stream, timeoutMs: number): void {
25
+ const onClose = (): void => {
26
+ clearTimeout(timeout);
27
+ };
28
+
29
+ const timeout = setTimeout(() => {
30
+ stream.removeEventListener("close", onClose);
31
+ if (stream.status === "open" && stream.remoteWriteStatus === "writable") {
32
+ stream.abort(getStreamNotFullyConsumedError());
33
+ }
34
+ }, timeoutMs);
35
+
36
+ stream.addEventListener("close", onClose, {once: true});
37
+ }
38
+
39
+ type ClearableSignal = AbortSignal & {clear: () => void};
40
+
41
+ function createRespSignal(signal: AbortSignal | undefined, timeoutMs: number): ClearableSignal {
42
+ const timeoutSignal = AbortSignal.timeout(timeoutMs);
43
+ const signals = signal ? [signal, timeoutSignal] : [timeoutSignal];
44
+ const controller = new AbortController();
45
+
46
+ const clear = (): void => {
47
+ for (const entry of signals) {
48
+ entry.removeEventListener("abort", onAbort);
49
+ }
50
+ };
51
+
52
+ const onAbort = (): void => {
53
+ if (controller.signal.aborted) {
54
+ return;
55
+ }
56
+ const reason = signals.find((entry) => entry.aborted)?.reason;
57
+ controller.abort(reason);
58
+ clear();
59
+ };
60
+
61
+ for (const entry of signals) {
62
+ if (entry.aborted) {
63
+ onAbort();
64
+ break;
65
+ }
66
+ entry.addEventListener("abort", onAbort);
67
+ }
68
+
69
+ const respSignal = controller.signal as ClearableSignal;
70
+ respSignal.clear = clear;
71
+
72
+ return respSignal;
73
+ }
74
+
22
75
  export interface SendRequestOpts {
23
76
  /** The maximum time for complete response transfer. */
24
77
  respTimeoutMs?: number;
25
78
  /** Non-spec timeout from sending request until write stream closed by responder */
26
79
  requestTimeoutMs?: number;
27
- /** The maximum time to wait for first byte of request response (time-to-first-byte). */
28
- ttfbTimeoutMs?: number;
29
80
  /** Non-spec timeout from dialing protocol until stream opened */
30
81
  dialTimeoutMs?: number;
31
82
  }
@@ -64,7 +115,6 @@ export async function* sendRequest(
64
115
 
65
116
  const DIAL_TIMEOUT = opts?.dialTimeoutMs ?? DEFAULT_DIAL_TIMEOUT;
66
117
  const REQUEST_TIMEOUT = opts?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT;
67
- const TTFB_TIMEOUT = opts?.ttfbTimeoutMs ?? DEFAULT_TTFB_TIMEOUT;
68
118
  const RESP_TIMEOUT = opts?.respTimeoutMs ?? DEFAULT_RESP_TIMEOUT;
69
119
 
70
120
  const peerIdStrShort = prettyPrintPeerId(peerId);
@@ -83,17 +133,6 @@ export async function* sendRequest(
83
133
  // the picked protocol in `connection.protocol`
84
134
  const protocolsMap = new Map<string, MixedProtocol>(protocols.map((protocol, i) => [protocolIDs[i], protocol]));
85
135
 
86
- // As of October 2020 we can't rely on libp2p.dialProtocol timeout to work so
87
- // this function wraps the dialProtocol promise with an extra timeout
88
- //
89
- // > The issue might be: you add the peer's addresses to the AddressBook,
90
- // which will result in autoDial to kick in and dial your peer. In parallel,
91
- // you do a manual dial and it will wait for the previous one without using
92
- // the abort signal:
93
- //
94
- // https://github.com/ChainSafe/lodestar/issues/1597#issuecomment-703394386
95
-
96
- // DIAL_TIMEOUT: Non-spec timeout from dialing protocol until stream opened
97
136
  const stream = await withTimeout(
98
137
  async (timeoutAndParentSignal) => {
99
138
  const protocolIds = Array.from(protocolsMap.keys());
@@ -112,9 +151,6 @@ export async function* sendRequest(
112
151
 
113
152
  metrics?.outgoingOpenedStreams?.inc({method});
114
153
 
115
- // TODO: Does the TTFB timer start on opening stream or after receiving request
116
- const timerTTFB = metrics?.outgoingResponseTTFB.startTimer({method});
117
-
118
154
  // Parse protocol selected by the responder
119
155
  const protocolId = stream.protocol ?? "unknown";
120
156
  const protocol = protocolsMap.get(protocolId);
@@ -126,21 +162,24 @@ export async function* sendRequest(
126
162
  logger.debug("Req sending request", logCtx);
127
163
 
128
164
  // Spec: The requester MUST close the write side of the stream once it finishes writing the request message
129
- // Impl: stream.sink is closed automatically by js-libp2p-mplex when piped source is exhausted
130
165
 
131
166
  // REQUEST_TIMEOUT: Non-spec timeout from sending request until write stream closed by responder
132
- // Note: libp2p.stop() will close all connections, so not necessary to abort this pipe on parent stop
133
- await withTimeout(() => pipe(requestEncode(protocol, requestBody), stream.sink), REQUEST_TIMEOUT, signal).catch(
134
- (e) => {
135
- // Must close the stream read side (stream.source) manually AND the write side
136
- stream.abort(e);
137
-
138
- if (e instanceof TimeoutError) {
139
- throw new RequestError({code: RequestErrorCode.REQUEST_TIMEOUT});
140
- }
141
- throw new RequestError({code: RequestErrorCode.REQUEST_ERROR, error: e as Error});
167
+ // Note: libp2p.stop() will close all connections, so not necessary to abort this send on parent stop
168
+ await withTimeout(
169
+ async (timeoutAndParentSignal) => {
170
+ await sendChunks(stream, requestEncode(protocol, requestBody), timeoutAndParentSignal);
171
+ await stream.close({signal: timeoutAndParentSignal});
172
+ },
173
+ REQUEST_TIMEOUT,
174
+ signal
175
+ ).catch((e) => {
176
+ stream.abort(e as Error);
177
+
178
+ if (e instanceof TimeoutError) {
179
+ throw new RequestError({code: RequestErrorCode.REQUEST_TIMEOUT});
142
180
  }
143
- );
181
+ throw new RequestError({code: RequestErrorCode.REQUEST_ERROR, error: e as Error});
182
+ });
144
183
 
145
184
  logger.debug("Req request sent", logCtx);
146
185
 
@@ -150,67 +189,51 @@ export async function* sendRequest(
150
189
  return;
151
190
  }
152
191
 
153
- // - TTFB_TIMEOUT: The requester MUST wait a maximum of TTFB_TIMEOUT for the first response byte to arrive
154
- // - RESP_TIMEOUT: Requester allows a further RESP_TIMEOUT for each subsequent response_chunk
155
- // - Max total timeout: This timeout is not required by the spec. It may not be necessary, but it's kept as
156
- // safe-guard to close. streams in case of bugs on other timeout mechanisms.
157
- const ttfbTimeoutController = new AbortController();
158
- const respTimeoutController = new AbortController();
159
-
160
- let timeoutRESP: NodeJS.Timeout | null = null;
192
+ // RESP_TIMEOUT: Maximum time for complete response transfer
193
+ const respSignal = createRespSignal(signal, RESP_TIMEOUT);
161
194
 
162
- const timeoutTTFB = setTimeout(() => {
163
- // If we abort on first byte delay, don't need to abort for response delay
164
- if (timeoutRESP) clearTimeout(timeoutRESP);
165
- ttfbTimeoutController.abort();
166
- }, TTFB_TIMEOUT);
167
-
168
- const restartRespTimeout = (): void => {
169
- if (timeoutRESP) clearTimeout(timeoutRESP);
170
- timeoutRESP = setTimeout(() => respTimeoutController.abort(), RESP_TIMEOUT);
171
- };
195
+ let responseError: Error | null = null;
196
+ let responseFullyConsumed = false;
172
197
 
173
198
  try {
174
- // Note: libp2p.stop() will close all connections, so not necessary to abort this pipe on parent stop
175
- yield* pipe(
176
- abortableSource(stream.source as AsyncIterable<Uint8ArrayList>, [
177
- {
178
- signal: ttfbTimeoutController.signal,
179
- getError: () => new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}),
180
- },
181
- {
182
- signal: respTimeoutController.signal,
183
- getError: () => new RequestError({code: RequestErrorCode.RESP_TIMEOUT}),
184
- },
185
- ]),
186
-
187
- // Transforms `Buffer` chunks to yield `ResponseBody` chunks
188
- responseDecode(protocol, {
189
- onFirstHeader() {
190
- // On first byte, cancel the single use TTFB_TIMEOUT, and start RESP_TIMEOUT
191
- clearTimeout(timeoutTTFB);
192
- timerTTFB?.();
193
- restartRespTimeout();
194
- },
195
- onFirstResponseChunk() {
196
- // On <response_chunk>, cancel this chunk's RESP_TIMEOUT and start next's
197
- restartRespTimeout();
198
- },
199
- })
200
- );
199
+ yield* responseDecode(protocol, stream, {
200
+ signal: respSignal,
201
+ getError: () =>
202
+ signal?.aborted ? new ErrorAborted("sendRequest") : new RequestError({code: RequestErrorCode.RESP_TIMEOUT}),
203
+ });
204
+ responseFullyConsumed = true;
201
205
 
202
206
  // NOTE: Only log once per request to verbose, intermediate steps to debug
203
207
  // NOTE: Do not log the response, logs get extremely cluttered
204
208
  // NOTE: add double space after "Req " to align log with the "Resp " log
205
209
  logger.verbose("Req done", logCtx);
210
+ } catch (e) {
211
+ responseError = e as Error;
212
+ throw e;
206
213
  } finally {
207
- clearTimeout(timeoutTTFB);
208
- if (timeoutRESP !== null) clearTimeout(timeoutRESP);
209
-
210
- // Necessary to call `stream.close()` since collectResponses() may break out of the source before exhausting it
211
- // `stream.close()` libp2p-mplex will .end() the source (it-pushable instance)
212
- // If collectResponses() exhausts the source, it-pushable.end() can be safely called multiple times
213
- await stream.close();
214
+ // On decode/timeout failures abort immediately so mplex can reclaim stream state.
215
+ // On normal early consumer exit, close gracefully to avoid stream-id desync with peers.
216
+ if (responseError !== null || signal?.aborted) {
217
+ stream.abort(responseError ?? new ErrorAborted("sendRequest"));
218
+ } else {
219
+ await stream.close().catch((e) => {
220
+ stream.abort(e as Error);
221
+ });
222
+
223
+ if (!responseFullyConsumed) {
224
+ // Stop buffering unread inbound data after caller exits early.
225
+ // mplex does not support propagating closeRead to the remote, so still
226
+ // abort later if the remote never closes write.
227
+ await stream.closeRead().catch(() => {
228
+ // Ignore closeRead errors - close/abort path below will reclaim stream.
229
+ });
230
+
231
+ if (stream.remoteWriteStatus === "writable") {
232
+ scheduleStreamAbortIfNotClosed(stream, RESP_TIMEOUT);
233
+ }
234
+ }
235
+ }
236
+ respSignal.clear();
214
237
  metrics?.outgoingClosedStreams?.inc({method});
215
238
  logger.verbose("Req stream closed", logCtx);
216
239
  }