@lodestar/reqresp 1.41.0-dev.983b1a457b → 1.41.0-dev.9939b12b53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/ReqResp.d.ts +3 -3
- package/lib/ReqResp.d.ts.map +1 -1
- package/lib/ReqResp.js +4 -4
- package/lib/ReqResp.js.map +1 -1
- package/lib/encoders/requestDecode.d.ts +2 -3
- package/lib/encoders/requestDecode.d.ts.map +1 -1
- package/lib/encoders/requestDecode.js +28 -11
- package/lib/encoders/requestDecode.js.map +1 -1
- package/lib/encoders/requestEncode.d.ts +1 -1
- package/lib/encoders/requestEncode.d.ts.map +1 -1
- package/lib/encoders/requestEncode.js +1 -1
- package/lib/encoders/requestEncode.js.map +1 -1
- package/lib/encoders/responseDecode.d.ts +10 -10
- package/lib/encoders/responseDecode.d.ts.map +1 -1
- package/lib/encoders/responseDecode.js +63 -60
- package/lib/encoders/responseDecode.js.map +1 -1
- package/lib/encoders/responseEncode.d.ts +2 -4
- package/lib/encoders/responseEncode.d.ts.map +1 -1
- package/lib/encoders/responseEncode.js +13 -22
- package/lib/encoders/responseEncode.js.map +1 -1
- package/lib/encodingStrategies/index.d.ts +4 -3
- package/lib/encodingStrategies/index.d.ts.map +1 -1
- package/lib/encodingStrategies/index.js +4 -4
- package/lib/encodingStrategies/index.js.map +1 -1
- package/lib/encodingStrategies/sszSnappy/decode.d.ts +5 -4
- package/lib/encodingStrategies/sszSnappy/decode.d.ts.map +1 -1
- package/lib/encodingStrategies/sszSnappy/decode.js +83 -52
- package/lib/encodingStrategies/sszSnappy/decode.js.map +1 -1
- package/lib/encodingStrategies/sszSnappy/encode.d.ts +2 -2
- package/lib/encodingStrategies/sszSnappy/encode.d.ts.map +1 -1
- package/lib/encodingStrategies/sszSnappy/encode.js +1 -1
- package/lib/encodingStrategies/sszSnappy/encode.js.map +1 -1
- package/lib/encodingStrategies/sszSnappy/errors.d.ts +0 -8
- package/lib/encodingStrategies/sszSnappy/errors.d.ts.map +1 -1
- package/lib/encodingStrategies/sszSnappy/errors.js +2 -3
- package/lib/encodingStrategies/sszSnappy/errors.js.map +1 -1
- package/lib/encodingStrategies/sszSnappy/index.d.ts +0 -1
- package/lib/encodingStrategies/sszSnappy/index.d.ts.map +1 -1
- package/lib/encodingStrategies/sszSnappy/index.js +0 -1
- package/lib/encodingStrategies/sszSnappy/index.js.map +1 -1
- package/lib/encodingStrategies/sszSnappy/utils.js.map +1 -1
- package/lib/interface.js +2 -1
- package/lib/interface.js.map +1 -1
- package/lib/metrics.d.ts +1 -7
- package/lib/metrics.d.ts.map +1 -1
- package/lib/metrics.js +1 -17
- package/lib/metrics.js.map +1 -1
- package/lib/rate_limiter/ReqRespRateLimiter.d.ts.map +1 -1
- package/lib/rate_limiter/ReqRespRateLimiter.js.map +1 -1
- package/lib/rate_limiter/rateLimiterGRCA.d.ts.map +1 -1
- package/lib/rate_limiter/rateLimiterGRCA.js.map +1 -1
- package/lib/rate_limiter/selfRateLimiter.d.ts.map +1 -1
- package/lib/rate_limiter/selfRateLimiter.js.map +1 -1
- package/lib/request/errors.d.ts +1 -7
- package/lib/request/errors.d.ts.map +1 -1
- package/lib/request/errors.js +3 -6
- package/lib/request/errors.js.map +1 -1
- package/lib/request/index.d.ts +0 -3
- package/lib/request/index.d.ts.map +1 -1
- package/lib/request/index.js +96 -70
- package/lib/request/index.js.map +1 -1
- package/lib/response/errors.d.ts.map +1 -1
- package/lib/response/errors.js +2 -1
- package/lib/response/errors.js.map +1 -1
- package/lib/response/index.d.ts +2 -2
- package/lib/response/index.d.ts.map +1 -1
- package/lib/response/index.js +46 -50
- package/lib/response/index.js.map +1 -1
- package/lib/types.d.ts +1 -2
- package/lib/types.d.ts.map +1 -1
- package/lib/types.js +7 -5
- package/lib/types.js.map +1 -1
- package/lib/utils/collectExactOne.js.map +1 -1
- package/lib/utils/collectMaxResponse.d.ts.map +1 -1
- package/lib/utils/collectMaxResponse.js +1 -2
- package/lib/utils/collectMaxResponse.js.map +1 -1
- package/lib/utils/errorMessage.d.ts +3 -3
- package/lib/utils/errorMessage.d.ts.map +1 -1
- package/lib/utils/errorMessage.js +14 -13
- package/lib/utils/errorMessage.js.map +1 -1
- package/lib/utils/index.d.ts +1 -3
- package/lib/utils/index.d.ts.map +1 -1
- package/lib/utils/index.js +1 -3
- package/lib/utils/index.js.map +1 -1
- package/lib/utils/peerId.js.map +1 -1
- package/lib/utils/protocolId.d.ts +2 -2
- package/lib/utils/protocolId.js +2 -2
- package/lib/utils/protocolId.js.map +1 -1
- package/lib/utils/snappyCommon.js +2 -1
- package/lib/utils/snappyCommon.js.map +1 -1
- package/lib/utils/snappyCompress.d.ts +1 -1
- package/lib/utils/snappyCompress.d.ts.map +1 -1
- package/lib/utils/snappyCompress.js +1 -1
- package/lib/utils/snappyCompress.js.map +1 -1
- package/lib/utils/snappyIndex.d.ts +1 -1
- package/lib/utils/snappyIndex.d.ts.map +1 -1
- package/lib/utils/snappyIndex.js +1 -1
- package/lib/utils/snappyIndex.js.map +1 -1
- package/lib/utils/snappyUncompress.d.ts +7 -11
- package/lib/utils/snappyUncompress.d.ts.map +1 -1
- package/lib/utils/snappyUncompress.js +68 -68
- package/lib/utils/snappyUncompress.js.map +1 -1
- package/lib/utils/stream.d.ts +6 -0
- package/lib/utils/stream.d.ts.map +1 -0
- package/lib/utils/stream.js +21 -0
- package/lib/utils/stream.js.map +1 -0
- package/package.json +16 -18
- package/src/ReqResp.ts +4 -4
- package/src/encoders/requestDecode.ts +32 -16
- package/src/encoders/requestEncode.ts +1 -1
- package/src/encoders/responseDecode.ts +68 -72
- package/src/encoders/responseEncode.ts +17 -29
- package/src/encodingStrategies/index.ts +8 -6
- package/src/encodingStrategies/sszSnappy/decode.ts +111 -53
- package/src/encodingStrategies/sszSnappy/encode.ts +2 -2
- package/src/encodingStrategies/sszSnappy/errors.ts +0 -4
- package/src/encodingStrategies/sszSnappy/index.ts +0 -1
- package/src/metrics.ts +1 -17
- package/src/request/errors.ts +1 -6
- package/src/request/index.ts +119 -85
- package/src/response/index.ts +55 -61
- package/src/types.ts +1 -3
- package/src/utils/collectMaxResponse.ts +1 -2
- package/src/utils/errorMessage.ts +14 -13
- package/src/utils/index.ts +1 -3
- package/src/utils/protocolId.ts +2 -2
- package/src/utils/snappyCompress.ts +1 -1
- package/src/utils/snappyIndex.ts +1 -1
- package/src/utils/snappyUncompress.ts +73 -75
- package/src/utils/stream.ts +34 -0
- package/lib/utils/abortableSource.d.ts +0 -12
- package/lib/utils/abortableSource.d.ts.map +0 -1
- package/lib/utils/abortableSource.js +0 -69
- package/lib/utils/abortableSource.js.map +0 -1
- package/lib/utils/bufferedSource.d.ts +0 -16
- package/lib/utils/bufferedSource.d.ts.map +0 -1
- package/lib/utils/bufferedSource.js +0 -40
- package/lib/utils/bufferedSource.js.map +0 -1
- package/lib/utils/onChunk.d.ts +0 -6
- package/lib/utils/onChunk.d.ts.map +0 -1
- package/lib/utils/onChunk.js +0 -13
- package/lib/utils/onChunk.js.map +0 -1
- package/lib/utils/snappy.d.ts +0 -3
- package/lib/utils/snappy.d.ts.map +0 -1
- package/lib/utils/snappy.js +0 -3
- package/lib/utils/snappy.js.map +0 -1
- package/src/utils/abortableSource.ts +0 -80
- package/src/utils/bufferedSource.ts +0 -46
- package/src/utils/onChunk.ts +0 -12
- package/src/utils/snappy.ts +0 -2
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import type {Stream} from "@libp2p/interface";
|
|
2
|
+
import type {ByteStream} from "@libp2p/utils";
|
|
1
3
|
import {decode as varintDecode, encodingLength as varintEncodingLength} from "uint8-varint";
|
|
2
4
|
import {Uint8ArrayList} from "uint8arraylist";
|
|
3
5
|
import {TypeSizes} from "../../types.js";
|
|
4
|
-
import {
|
|
5
|
-
import {SnappyFramesUncompress} from "../../utils/snappyIndex.js";
|
|
6
|
+
import {ChunkType, decodeSnappyFrameData, parseSnappyFrameHeader} from "../../utils/snappyIndex.js";
|
|
6
7
|
import {SszSnappyError, SszSnappyErrorCode} from "./errors.js";
|
|
7
8
|
import {maxEncodedLen} from "./utils.js";
|
|
8
9
|
|
|
@@ -15,76 +16,115 @@ export const MAX_VARINT_BYTES = 10;
|
|
|
15
16
|
* <encoding-dependent-header> | <encoded-payload>
|
|
16
17
|
* ```
|
|
17
18
|
*/
|
|
18
|
-
export async function readSszSnappyPayload(
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
export async function readSszSnappyPayload(
|
|
20
|
+
stream: ByteStream<Stream>,
|
|
21
|
+
type: TypeSizes,
|
|
22
|
+
signal?: AbortSignal
|
|
23
|
+
): Promise<Uint8Array> {
|
|
24
|
+
const sszDataLength = await readSszSnappyHeader(stream, type, signal);
|
|
25
|
+
|
|
26
|
+
return readSszSnappyBody(stream, sszDataLength, signal);
|
|
22
27
|
}
|
|
23
28
|
|
|
24
29
|
/**
|
|
25
30
|
* Reads `<encoding-dependent-header>` for ssz-snappy.
|
|
26
31
|
* encoding-header ::= the length of the raw SSZ bytes, encoded as an unsigned protobuf varint
|
|
27
32
|
*/
|
|
28
|
-
export async function readSszSnappyHeader(
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
33
|
+
export async function readSszSnappyHeader(
|
|
34
|
+
stream: ByteStream<Stream>,
|
|
35
|
+
type: TypeSizes,
|
|
36
|
+
signal?: AbortSignal
|
|
37
|
+
): Promise<number> {
|
|
38
|
+
const varintBytes: number[] = [];
|
|
34
39
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
sszDataLength = varintDecode(buffer.subarray());
|
|
38
|
-
} catch (_e) {
|
|
39
|
-
throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity});
|
|
40
|
-
}
|
|
40
|
+
while (true) {
|
|
41
|
+
const byte = await readExactOrSourceAborted(stream, 1, signal);
|
|
41
42
|
|
|
42
|
-
|
|
43
|
-
// encodingLength function only returns 1-8 inclusive
|
|
44
|
-
const varintBytes = varintEncodingLength(sszDataLength);
|
|
45
|
-
buffer.consume(varintBytes);
|
|
43
|
+
const value = byte.get(0);
|
|
46
44
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
if (sszDataLength < minSize) {
|
|
51
|
-
throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength});
|
|
52
|
-
}
|
|
53
|
-
if (sszDataLength > maxSize) {
|
|
54
|
-
throw new SszSnappyError({code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE, maxSize, sszDataLength});
|
|
45
|
+
varintBytes.push(value);
|
|
46
|
+
if (varintBytes.length > MAX_VARINT_BYTES) {
|
|
47
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintBytes.length});
|
|
55
48
|
}
|
|
56
49
|
|
|
57
|
-
|
|
50
|
+
// MSB not set => varint terminated
|
|
51
|
+
if ((value & 0x80) === 0) break;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
let sszDataLength: number;
|
|
55
|
+
try {
|
|
56
|
+
sszDataLength = varintDecode(Uint8Array.from(varintBytes));
|
|
57
|
+
} catch {
|
|
58
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity});
|
|
58
59
|
}
|
|
59
60
|
|
|
60
|
-
|
|
61
|
+
// MUST validate: the unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes
|
|
62
|
+
// encodingLength function only returns 1-8 inclusive
|
|
63
|
+
const varintByteLength = varintEncodingLength(sszDataLength);
|
|
64
|
+
if (varintByteLength > MAX_VARINT_BYTES) {
|
|
65
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintByteLength});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type.
|
|
69
|
+
const minSize = type.minSize;
|
|
70
|
+
const maxSize = type.maxSize;
|
|
71
|
+
if (sszDataLength < minSize) {
|
|
72
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength});
|
|
73
|
+
}
|
|
74
|
+
if (sszDataLength > maxSize) {
|
|
75
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE, maxSize, sszDataLength});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return sszDataLength;
|
|
61
79
|
}
|
|
62
80
|
|
|
63
81
|
/**
|
|
64
82
|
* Reads `<encoded-payload>` for ssz-snappy and decompress.
|
|
65
83
|
* The returned bytes can be SSZ deseralized
|
|
66
84
|
*/
|
|
67
|
-
export async function readSszSnappyBody(
|
|
68
|
-
|
|
85
|
+
export async function readSszSnappyBody(
|
|
86
|
+
stream: ByteStream<Stream>,
|
|
87
|
+
sszDataLength: number,
|
|
88
|
+
signal?: AbortSignal
|
|
89
|
+
): Promise<Uint8Array> {
|
|
69
90
|
const uncompressedData = new Uint8ArrayList();
|
|
70
|
-
let
|
|
91
|
+
let encodedBytesRead = 0;
|
|
92
|
+
const maxBytes = maxEncodedLen(sszDataLength);
|
|
93
|
+
let foundIdentifier = false;
|
|
94
|
+
|
|
95
|
+
while (uncompressedData.length < sszDataLength) {
|
|
96
|
+
const header = await readExactOrSourceAborted(stream, 4, signal);
|
|
71
97
|
|
|
72
|
-
for await (const buffer of bufferedSource) {
|
|
73
98
|
// SHOULD NOT read more than max_encoded_len(n) bytes after reading the SSZ length-prefix n from the header
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
99
|
+
encodedBytesRead = addEncodedBytesReadOrThrow(encodedBytesRead, header.length, maxBytes, sszDataLength);
|
|
100
|
+
|
|
101
|
+
let headerParsed: {type: ChunkType; frameSize: number};
|
|
102
|
+
try {
|
|
103
|
+
headerParsed = parseSnappyFrameHeader(header.subarray());
|
|
104
|
+
if (!foundIdentifier && headerParsed.type !== ChunkType.IDENTIFIER) {
|
|
105
|
+
throw new Error("malformed input: must begin with an identifier");
|
|
106
|
+
}
|
|
107
|
+
} catch (e) {
|
|
108
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.DECOMPRESSOR_ERROR, decompressorError: e as Error});
|
|
77
109
|
}
|
|
78
110
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
111
|
+
if (headerParsed.frameSize > maxBytes - encodedBytesRead) {
|
|
112
|
+
throw new SszSnappyError({
|
|
113
|
+
code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ,
|
|
114
|
+
readBytes: encodedBytesRead + headerParsed.frameSize,
|
|
115
|
+
sszDataLength,
|
|
116
|
+
});
|
|
82
117
|
}
|
|
118
|
+
const frame = await readExactOrSourceAborted(stream, headerParsed.frameSize, signal);
|
|
119
|
+
|
|
120
|
+
encodedBytesRead = addEncodedBytesReadOrThrow(encodedBytesRead, frame.length, maxBytes, sszDataLength);
|
|
83
121
|
|
|
84
|
-
// stream contents can be passed through a buffered Snappy reader to decompress frame by frame
|
|
85
122
|
try {
|
|
86
|
-
|
|
87
|
-
|
|
123
|
+
if (headerParsed.type === ChunkType.IDENTIFIER) {
|
|
124
|
+
foundIdentifier = true;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const uncompressed = decodeSnappyFrameData(headerParsed.type, frame.subarray());
|
|
88
128
|
if (uncompressed !== null) {
|
|
89
129
|
uncompressedData.append(uncompressed);
|
|
90
130
|
}
|
|
@@ -96,16 +136,34 @@ export async function readSszSnappyBody(bufferedSource: BufferedSource, sszDataL
|
|
|
96
136
|
if (uncompressedData.length > sszDataLength) {
|
|
97
137
|
throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MANY_BYTES, sszDataLength});
|
|
98
138
|
}
|
|
139
|
+
}
|
|
99
140
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
}
|
|
141
|
+
// buffer.length === n
|
|
142
|
+
return uncompressedData.subarray(0, sszDataLength);
|
|
143
|
+
}
|
|
104
144
|
|
|
105
|
-
|
|
106
|
-
|
|
145
|
+
function addEncodedBytesReadOrThrow(
|
|
146
|
+
encodedBytesRead: number,
|
|
147
|
+
bytesToAdd: number,
|
|
148
|
+
maxBytes: number,
|
|
149
|
+
sszDataLength: number
|
|
150
|
+
): number {
|
|
151
|
+
const nextReadBytes = encodedBytesRead + bytesToAdd;
|
|
152
|
+
if (nextReadBytes > maxBytes) {
|
|
153
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, readBytes: nextReadBytes, sszDataLength});
|
|
107
154
|
}
|
|
155
|
+
return nextReadBytes;
|
|
156
|
+
}
|
|
108
157
|
|
|
109
|
-
|
|
110
|
-
|
|
158
|
+
async function readExactOrSourceAborted(
|
|
159
|
+
stream: ByteStream<Stream>,
|
|
160
|
+
bytes: number,
|
|
161
|
+
signal?: AbortSignal
|
|
162
|
+
): Promise<Uint8ArrayList> {
|
|
163
|
+
return stream.read({bytes, signal}).catch((e) => {
|
|
164
|
+
if ((e as Error).name === "UnexpectedEOFError") {
|
|
165
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED});
|
|
166
|
+
}
|
|
167
|
+
throw e;
|
|
168
|
+
});
|
|
111
169
|
}
|
|
@@ -8,12 +8,12 @@ import {encodeSnappy} from "../../utils/snappyIndex.js";
|
|
|
8
8
|
* <encoding-dependent-header> | <encoded-payload>
|
|
9
9
|
* ```
|
|
10
10
|
*/
|
|
11
|
-
export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) =>
|
|
11
|
+
export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => Generator<Buffer>;
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* Buffered Snappy writer
|
|
15
15
|
*/
|
|
16
|
-
export
|
|
16
|
+
export function* encodeSszSnappy(bytes: Buffer): Generator<Buffer> {
|
|
17
17
|
// MUST encode the length of the raw SSZ bytes, encoded as an unsigned protobuf varint
|
|
18
18
|
const varint = varintEncode(bytes.length);
|
|
19
19
|
yield Buffer.from(varint.buffer, varint.byteOffset, varint.byteLength);
|
|
@@ -9,8 +9,6 @@ export enum SszSnappyErrorCode {
|
|
|
9
9
|
OVER_SSZ_MAX_SIZE = "SSZ_SNAPPY_ERROR_OVER_SSZ_MAX_SIZE",
|
|
10
10
|
TOO_MUCH_BYTES_READ = "SSZ_SNAPPY_ERROR_TOO_MUCH_BYTES_READ",
|
|
11
11
|
DECOMPRESSOR_ERROR = "SSZ_SNAPPY_ERROR_DECOMPRESSOR_ERROR",
|
|
12
|
-
DESERIALIZE_ERROR = "SSZ_SNAPPY_ERROR_DESERIALIZE_ERROR",
|
|
13
|
-
SERIALIZE_ERROR = "SSZ_SNAPPY_ERROR_SERIALIZE_ERROR",
|
|
14
12
|
/** Received more bytes than specified sszDataLength */
|
|
15
13
|
TOO_MANY_BYTES = "SSZ_SNAPPY_ERROR_TOO_MANY_BYTES",
|
|
16
14
|
/** Source aborted before reading sszDataLength bytes */
|
|
@@ -23,8 +21,6 @@ type SszSnappyErrorType =
|
|
|
23
21
|
| {code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE; maxSize: number; sszDataLength: number}
|
|
24
22
|
| {code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ; readBytes: number; sszDataLength: number}
|
|
25
23
|
| {code: SszSnappyErrorCode.DECOMPRESSOR_ERROR; decompressorError: Error}
|
|
26
|
-
| {code: SszSnappyErrorCode.DESERIALIZE_ERROR; deserializeError: Error}
|
|
27
|
-
| {code: SszSnappyErrorCode.SERIALIZE_ERROR; serializeError: Error}
|
|
28
24
|
| {code: SszSnappyErrorCode.TOO_MANY_BYTES; sszDataLength: number}
|
|
29
25
|
| {code: SszSnappyErrorCode.SOURCE_ABORTED};
|
|
30
26
|
|
package/src/metrics.ts
CHANGED
|
@@ -4,7 +4,7 @@ import {RequestErrorCode} from "./request/errors.js";
|
|
|
4
4
|
export type Metrics = ReturnType<typeof getMetrics>;
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* A collection of metrics used throughout the
|
|
7
|
+
* A collection of metrics used throughout the Req/Resp domain.
|
|
8
8
|
*/
|
|
9
9
|
export function getMetrics(register: MetricsRegisterExtra) {
|
|
10
10
|
// Using function style instead of class to prevent having to re-declare all MetricsPrometheus types.
|
|
@@ -29,7 +29,6 @@ export function getMetrics(register: MetricsRegisterExtra) {
|
|
|
29
29
|
name: "beacon_reqresp_outgoing_request_roundtrip_time_seconds",
|
|
30
30
|
help: "Histogram of outgoing requests round-trip time",
|
|
31
31
|
labelNames: ["method"],
|
|
32
|
-
// Spec sets RESP_TIMEOUT = 10 sec
|
|
33
32
|
buckets: [0.1, 0.2, 0.5, 1, 5, 10, 15, 60],
|
|
34
33
|
}),
|
|
35
34
|
outgoingErrors: register.gauge<{method: string}>({
|
|
@@ -61,7 +60,6 @@ export function getMetrics(register: MetricsRegisterExtra) {
|
|
|
61
60
|
name: "beacon_reqresp_incoming_request_handler_time_seconds",
|
|
62
61
|
help: "Histogram of incoming requests internal handling time",
|
|
63
62
|
labelNames: ["method"],
|
|
64
|
-
// Spec sets RESP_TIMEOUT = 10 sec
|
|
65
63
|
buckets: [0.1, 0.2, 0.5, 1, 5, 10],
|
|
66
64
|
}),
|
|
67
65
|
incomingErrors: register.gauge<{method: string}>({
|
|
@@ -69,20 +67,6 @@ export function getMetrics(register: MetricsRegisterExtra) {
|
|
|
69
67
|
help: "Counts total failed responses handled per method",
|
|
70
68
|
labelNames: ["method"],
|
|
71
69
|
}),
|
|
72
|
-
outgoingResponseTTFB: register.histogram<{method: string}>({
|
|
73
|
-
name: "beacon_reqresp_outgoing_response_ttfb_seconds",
|
|
74
|
-
help: "Time to first byte (TTFB) for outgoing responses",
|
|
75
|
-
labelNames: ["method"],
|
|
76
|
-
// Spec sets TTFB_TIMEOUT = 5 sec
|
|
77
|
-
buckets: [0.1, 1, 5],
|
|
78
|
-
}),
|
|
79
|
-
incomingResponseTTFB: register.histogram<{method: string}>({
|
|
80
|
-
name: "beacon_reqresp_incoming_response_ttfb_seconds",
|
|
81
|
-
help: "Time to first byte (TTFB) for incoming responses",
|
|
82
|
-
labelNames: ["method"],
|
|
83
|
-
// Spec sets TTFB_TIMEOUT = 5 sec
|
|
84
|
-
buckets: [0.1, 1, 5],
|
|
85
|
-
}),
|
|
86
70
|
dialErrors: register.gauge({
|
|
87
71
|
name: "beacon_reqresp_dial_errors_total",
|
|
88
72
|
help: "Count total dial errors",
|
package/src/request/errors.ts
CHANGED
|
@@ -21,13 +21,9 @@ export enum RequestErrorCode {
|
|
|
21
21
|
REQUEST_TIMEOUT = "REQUEST_ERROR_REQUEST_TIMEOUT",
|
|
22
22
|
/** Error when sending request to responder */
|
|
23
23
|
REQUEST_ERROR = "REQUEST_ERROR_REQUEST_ERROR",
|
|
24
|
-
/** Reponder did not deliver a full reponse before max maxTotalResponseTimeout() */
|
|
25
|
-
RESPONSE_TIMEOUT = "REQUEST_ERROR_RESPONSE_TIMEOUT",
|
|
26
24
|
/** A single-response method returned 0 chunks */
|
|
27
25
|
EMPTY_RESPONSE = "REQUEST_ERROR_EMPTY_RESPONSE",
|
|
28
|
-
/**
|
|
29
|
-
TTFB_TIMEOUT = "REQUEST_ERROR_TTFB_TIMEOUT",
|
|
30
|
-
/** Timeout between `<response_chunk>` exceed */
|
|
26
|
+
/** Response transfer timeout exceeded */
|
|
31
27
|
RESP_TIMEOUT = "REQUEST_ERROR_RESP_TIMEOUT",
|
|
32
28
|
/** Request rate limited */
|
|
33
29
|
REQUEST_RATE_LIMITED = "REQUEST_ERROR_RATE_LIMITED",
|
|
@@ -50,7 +46,6 @@ type RequestErrorType =
|
|
|
50
46
|
| {code: RequestErrorCode.REQUEST_TIMEOUT}
|
|
51
47
|
| {code: RequestErrorCode.REQUEST_ERROR; error: Error}
|
|
52
48
|
| {code: RequestErrorCode.EMPTY_RESPONSE}
|
|
53
|
-
| {code: RequestErrorCode.TTFB_TIMEOUT}
|
|
54
49
|
| {code: RequestErrorCode.RESP_TIMEOUT}
|
|
55
50
|
| {code: RequestErrorCode.REQUEST_RATE_LIMITED}
|
|
56
51
|
| {code: RequestErrorCode.REQUEST_SELF_RATE_LIMITED}
|
package/src/request/index.ts
CHANGED
|
@@ -1,31 +1,93 @@
|
|
|
1
|
+
import type {Stream} from "@libp2p/interface";
|
|
1
2
|
import {PeerId} from "@libp2p/interface";
|
|
2
|
-
import {pipe} from "it-pipe";
|
|
3
3
|
import type {Libp2p} from "libp2p";
|
|
4
|
-
import {Uint8ArrayList} from "uint8arraylist";
|
|
5
4
|
import {ErrorAborted, Logger, TimeoutError, withTimeout} from "@lodestar/utils";
|
|
6
5
|
import {requestEncode} from "../encoders/requestEncode.js";
|
|
7
6
|
import {responseDecode} from "../encoders/responseDecode.js";
|
|
8
7
|
import {Metrics} from "../metrics.js";
|
|
9
8
|
import {ResponseError} from "../response/index.js";
|
|
10
9
|
import {MixedProtocol, ResponseIncoming} from "../types.js";
|
|
11
|
-
import {
|
|
10
|
+
import {prettyPrintPeerId, sendChunks} from "../utils/index.js";
|
|
12
11
|
import {RequestError, RequestErrorCode, responseStatusErrorToRequestError} from "./errors.js";
|
|
13
12
|
|
|
14
13
|
export {RequestError, RequestErrorCode};
|
|
15
14
|
|
|
16
|
-
//
|
|
15
|
+
// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#the-reqresp-domain
|
|
17
16
|
export const DEFAULT_DIAL_TIMEOUT = 5 * 1000; // 5 sec
|
|
18
17
|
export const DEFAULT_REQUEST_TIMEOUT = 5 * 1000; // 5 sec
|
|
19
|
-
export const DEFAULT_TTFB_TIMEOUT = 5 * 1000; // 5 sec
|
|
20
18
|
export const DEFAULT_RESP_TIMEOUT = 10 * 1000; // 10 sec
|
|
21
19
|
|
|
20
|
+
function getStreamNotFullyConsumedError(): Error {
|
|
21
|
+
return new Error("ReqResp stream was not fully consumed");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function scheduleStreamAbortIfNotClosed(stream: Stream, timeoutMs: number): void {
|
|
25
|
+
const onClose = (): void => {
|
|
26
|
+
clearTimeout(timeout);
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
const timeout = setTimeout(() => {
|
|
30
|
+
stream.removeEventListener("close", onClose);
|
|
31
|
+
if (stream.status === "open" && stream.remoteWriteStatus === "writable") {
|
|
32
|
+
stream.abort(getStreamNotFullyConsumedError());
|
|
33
|
+
}
|
|
34
|
+
}, timeoutMs);
|
|
35
|
+
|
|
36
|
+
stream.addEventListener("close", onClose, {once: true});
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
type ClearableSignal = AbortSignal & {clear: () => void};
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Compose an abort signal from an optional parent signal and a timeout, with explicit cleanup.
|
|
43
|
+
*
|
|
44
|
+
* This replaces a plain `AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)])` to work
|
|
45
|
+
* around a memory leak in Node.js where AbortSignal.any() retains references to all source
|
|
46
|
+
* signals for the lifetime of the longest-lived signal (the timeout). In a long-running
|
|
47
|
+
* req/resp workload this causes unbounded growth of the dependent-signal set.
|
|
48
|
+
*
|
|
49
|
+
* Upstream issue: https://github.com/nodejs/node/issues/54614
|
|
50
|
+
* Lodestar investigation: https://github.com/ChainSafe/lodestar/issues/8969
|
|
51
|
+
*/
|
|
52
|
+
function createRespSignal(signal: AbortSignal | undefined, timeoutMs: number): ClearableSignal {
|
|
53
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
54
|
+
const signals = signal ? [signal, timeoutSignal] : [timeoutSignal];
|
|
55
|
+
const controller = new AbortController();
|
|
56
|
+
|
|
57
|
+
const clear = (): void => {
|
|
58
|
+
for (const entry of signals) {
|
|
59
|
+
entry.removeEventListener("abort", onAbort);
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
const onAbort = (): void => {
|
|
64
|
+
if (controller.signal.aborted) {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
const reason = signals.find((entry) => entry.aborted)?.reason;
|
|
68
|
+
controller.abort(reason);
|
|
69
|
+
clear();
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
for (const entry of signals) {
|
|
73
|
+
if (entry.aborted) {
|
|
74
|
+
onAbort();
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
entry.addEventListener("abort", onAbort);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const respSignal = controller.signal as ClearableSignal;
|
|
81
|
+
respSignal.clear = clear;
|
|
82
|
+
|
|
83
|
+
return respSignal;
|
|
84
|
+
}
|
|
85
|
+
|
|
22
86
|
export interface SendRequestOpts {
|
|
23
87
|
/** The maximum time for complete response transfer. */
|
|
24
88
|
respTimeoutMs?: number;
|
|
25
89
|
/** Non-spec timeout from sending request until write stream closed by responder */
|
|
26
90
|
requestTimeoutMs?: number;
|
|
27
|
-
/** The maximum time to wait for first byte of request response (time-to-first-byte). */
|
|
28
|
-
ttfbTimeoutMs?: number;
|
|
29
91
|
/** Non-spec timeout from dialing protocol until stream opened */
|
|
30
92
|
dialTimeoutMs?: number;
|
|
31
93
|
}
|
|
@@ -64,7 +126,6 @@ export async function* sendRequest(
|
|
|
64
126
|
|
|
65
127
|
const DIAL_TIMEOUT = opts?.dialTimeoutMs ?? DEFAULT_DIAL_TIMEOUT;
|
|
66
128
|
const REQUEST_TIMEOUT = opts?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT;
|
|
67
|
-
const TTFB_TIMEOUT = opts?.ttfbTimeoutMs ?? DEFAULT_TTFB_TIMEOUT;
|
|
68
129
|
const RESP_TIMEOUT = opts?.respTimeoutMs ?? DEFAULT_RESP_TIMEOUT;
|
|
69
130
|
|
|
70
131
|
const peerIdStrShort = prettyPrintPeerId(peerId);
|
|
@@ -83,17 +144,6 @@ export async function* sendRequest(
|
|
|
83
144
|
// the picked protocol in `connection.protocol`
|
|
84
145
|
const protocolsMap = new Map<string, MixedProtocol>(protocols.map((protocol, i) => [protocolIDs[i], protocol]));
|
|
85
146
|
|
|
86
|
-
// As of October 2020 we can't rely on libp2p.dialProtocol timeout to work so
|
|
87
|
-
// this function wraps the dialProtocol promise with an extra timeout
|
|
88
|
-
//
|
|
89
|
-
// > The issue might be: you add the peer's addresses to the AddressBook,
|
|
90
|
-
// which will result in autoDial to kick in and dial your peer. In parallel,
|
|
91
|
-
// you do a manual dial and it will wait for the previous one without using
|
|
92
|
-
// the abort signal:
|
|
93
|
-
//
|
|
94
|
-
// https://github.com/ChainSafe/lodestar/issues/1597#issuecomment-703394386
|
|
95
|
-
|
|
96
|
-
// DIAL_TIMEOUT: Non-spec timeout from dialing protocol until stream opened
|
|
97
147
|
const stream = await withTimeout(
|
|
98
148
|
async (timeoutAndParentSignal) => {
|
|
99
149
|
const protocolIds = Array.from(protocolsMap.keys());
|
|
@@ -112,9 +162,6 @@ export async function* sendRequest(
|
|
|
112
162
|
|
|
113
163
|
metrics?.outgoingOpenedStreams?.inc({method});
|
|
114
164
|
|
|
115
|
-
// TODO: Does the TTFB timer start on opening stream or after receiving request
|
|
116
|
-
const timerTTFB = metrics?.outgoingResponseTTFB.startTimer({method});
|
|
117
|
-
|
|
118
165
|
// Parse protocol selected by the responder
|
|
119
166
|
const protocolId = stream.protocol ?? "unknown";
|
|
120
167
|
const protocol = protocolsMap.get(protocolId);
|
|
@@ -126,21 +173,24 @@ export async function* sendRequest(
|
|
|
126
173
|
logger.debug("Req sending request", logCtx);
|
|
127
174
|
|
|
128
175
|
// Spec: The requester MUST close the write side of the stream once it finishes writing the request message
|
|
129
|
-
// Impl: stream.sink is closed automatically by js-libp2p-mplex when piped source is exhausted
|
|
130
176
|
|
|
131
177
|
// REQUEST_TIMEOUT: Non-spec timeout from sending request until write stream closed by responder
|
|
132
|
-
// Note: libp2p.stop() will close all connections, so not necessary to abort this
|
|
133
|
-
await withTimeout(
|
|
134
|
-
(
|
|
135
|
-
|
|
136
|
-
stream.
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
178
|
+
// Note: libp2p.stop() will close all connections, so not necessary to abort this send on parent stop
|
|
179
|
+
await withTimeout(
|
|
180
|
+
async (timeoutAndParentSignal) => {
|
|
181
|
+
await sendChunks(stream, requestEncode(protocol, requestBody), timeoutAndParentSignal);
|
|
182
|
+
await stream.close({signal: timeoutAndParentSignal});
|
|
183
|
+
},
|
|
184
|
+
REQUEST_TIMEOUT,
|
|
185
|
+
signal
|
|
186
|
+
).catch((e) => {
|
|
187
|
+
stream.abort(e as Error);
|
|
188
|
+
|
|
189
|
+
if (e instanceof TimeoutError) {
|
|
190
|
+
throw new RequestError({code: RequestErrorCode.REQUEST_TIMEOUT});
|
|
142
191
|
}
|
|
143
|
-
|
|
192
|
+
throw new RequestError({code: RequestErrorCode.REQUEST_ERROR, error: e as Error});
|
|
193
|
+
});
|
|
144
194
|
|
|
145
195
|
logger.debug("Req request sent", logCtx);
|
|
146
196
|
|
|
@@ -150,67 +200,51 @@ export async function* sendRequest(
|
|
|
150
200
|
return;
|
|
151
201
|
}
|
|
152
202
|
|
|
153
|
-
//
|
|
154
|
-
|
|
155
|
-
// - Max total timeout: This timeout is not required by the spec. It may not be necessary, but it's kept as
|
|
156
|
-
// safe-guard to close. streams in case of bugs on other timeout mechanisms.
|
|
157
|
-
const ttfbTimeoutController = new AbortController();
|
|
158
|
-
const respTimeoutController = new AbortController();
|
|
159
|
-
|
|
160
|
-
let timeoutRESP: NodeJS.Timeout | null = null;
|
|
203
|
+
// RESP_TIMEOUT: Maximum time for complete response transfer
|
|
204
|
+
const respSignal = createRespSignal(signal, RESP_TIMEOUT);
|
|
161
205
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
if (timeoutRESP) clearTimeout(timeoutRESP);
|
|
165
|
-
ttfbTimeoutController.abort();
|
|
166
|
-
}, TTFB_TIMEOUT);
|
|
167
|
-
|
|
168
|
-
const restartRespTimeout = (): void => {
|
|
169
|
-
if (timeoutRESP) clearTimeout(timeoutRESP);
|
|
170
|
-
timeoutRESP = setTimeout(() => respTimeoutController.abort(), RESP_TIMEOUT);
|
|
171
|
-
};
|
|
206
|
+
let responseError: Error | null = null;
|
|
207
|
+
let responseFullyConsumed = false;
|
|
172
208
|
|
|
173
209
|
try {
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
{
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
},
|
|
181
|
-
{
|
|
182
|
-
signal: respTimeoutController.signal,
|
|
183
|
-
getError: () => new RequestError({code: RequestErrorCode.RESP_TIMEOUT}),
|
|
184
|
-
},
|
|
185
|
-
]),
|
|
186
|
-
|
|
187
|
-
// Transforms `Buffer` chunks to yield `ResponseBody` chunks
|
|
188
|
-
responseDecode(protocol, {
|
|
189
|
-
onFirstHeader() {
|
|
190
|
-
// On first byte, cancel the single use TTFB_TIMEOUT, and start RESP_TIMEOUT
|
|
191
|
-
clearTimeout(timeoutTTFB);
|
|
192
|
-
timerTTFB?.();
|
|
193
|
-
restartRespTimeout();
|
|
194
|
-
},
|
|
195
|
-
onFirstResponseChunk() {
|
|
196
|
-
// On <response_chunk>, cancel this chunk's RESP_TIMEOUT and start next's
|
|
197
|
-
restartRespTimeout();
|
|
198
|
-
},
|
|
199
|
-
})
|
|
200
|
-
);
|
|
210
|
+
yield* responseDecode(protocol, stream, {
|
|
211
|
+
signal: respSignal,
|
|
212
|
+
getError: () =>
|
|
213
|
+
signal?.aborted ? new ErrorAborted("sendRequest") : new RequestError({code: RequestErrorCode.RESP_TIMEOUT}),
|
|
214
|
+
});
|
|
215
|
+
responseFullyConsumed = true;
|
|
201
216
|
|
|
202
217
|
// NOTE: Only log once per request to verbose, intermediate steps to debug
|
|
203
218
|
// NOTE: Do not log the response, logs get extremely cluttered
|
|
204
219
|
// NOTE: add double space after "Req " to align log with the "Resp " log
|
|
205
220
|
logger.verbose("Req done", logCtx);
|
|
221
|
+
} catch (e) {
|
|
222
|
+
responseError = e as Error;
|
|
223
|
+
throw e;
|
|
206
224
|
} finally {
|
|
207
|
-
|
|
208
|
-
|
|
225
|
+
// On decode/timeout failures abort immediately so mplex can reclaim stream state.
|
|
226
|
+
// On normal early consumer exit, close gracefully to avoid stream-id desync with peers.
|
|
227
|
+
if (responseError !== null || signal?.aborted) {
|
|
228
|
+
stream.abort(responseError ?? new ErrorAborted("sendRequest"));
|
|
229
|
+
} else {
|
|
230
|
+
await stream.close().catch((e) => {
|
|
231
|
+
stream.abort(e as Error);
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
if (!responseFullyConsumed) {
|
|
235
|
+
// Stop buffering unread inbound data after caller exits early.
|
|
236
|
+
// mplex does not support propagating closeRead to the remote, so still
|
|
237
|
+
// abort later if the remote never closes write.
|
|
238
|
+
await stream.closeRead().catch(() => {
|
|
239
|
+
// Ignore closeRead errors - close/abort path below will reclaim stream.
|
|
240
|
+
});
|
|
209
241
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
242
|
+
if (stream.remoteWriteStatus === "writable") {
|
|
243
|
+
scheduleStreamAbortIfNotClosed(stream, RESP_TIMEOUT);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
respSignal.clear();
|
|
214
248
|
metrics?.outgoingClosedStreams?.inc({method});
|
|
215
249
|
logger.verbose("Req stream closed", logCtx);
|
|
216
250
|
}
|