@lodestar/reqresp 1.41.0-dev.a35cbde8b3 → 1.41.0-dev.aeab9f930d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/ReqResp.d.ts +3 -3
- package/lib/ReqResp.d.ts.map +1 -1
- package/lib/ReqResp.js +4 -4
- package/lib/ReqResp.js.map +1 -1
- package/lib/encoders/requestDecode.d.ts +2 -3
- package/lib/encoders/requestDecode.d.ts.map +1 -1
- package/lib/encoders/requestDecode.js +28 -11
- package/lib/encoders/requestDecode.js.map +1 -1
- package/lib/encoders/requestEncode.d.ts +1 -1
- package/lib/encoders/requestEncode.d.ts.map +1 -1
- package/lib/encoders/requestEncode.js +1 -1
- package/lib/encoders/requestEncode.js.map +1 -1
- package/lib/encoders/responseDecode.d.ts +10 -10
- package/lib/encoders/responseDecode.d.ts.map +1 -1
- package/lib/encoders/responseDecode.js +63 -60
- package/lib/encoders/responseDecode.js.map +1 -1
- package/lib/encoders/responseEncode.d.ts +2 -4
- package/lib/encoders/responseEncode.d.ts.map +1 -1
- package/lib/encoders/responseEncode.js +13 -22
- package/lib/encoders/responseEncode.js.map +1 -1
- package/lib/encodingStrategies/index.d.ts +4 -3
- package/lib/encodingStrategies/index.d.ts.map +1 -1
- package/lib/encodingStrategies/index.js +4 -4
- package/lib/encodingStrategies/index.js.map +1 -1
- package/lib/encodingStrategies/sszSnappy/decode.d.ts +5 -4
- package/lib/encodingStrategies/sszSnappy/decode.d.ts.map +1 -1
- package/lib/encodingStrategies/sszSnappy/decode.js +83 -52
- package/lib/encodingStrategies/sszSnappy/decode.js.map +1 -1
- package/lib/encodingStrategies/sszSnappy/encode.d.ts +2 -2
- package/lib/encodingStrategies/sszSnappy/encode.d.ts.map +1 -1
- package/lib/encodingStrategies/sszSnappy/encode.js +1 -1
- package/lib/encodingStrategies/sszSnappy/encode.js.map +1 -1
- package/lib/encodingStrategies/sszSnappy/errors.d.ts +0 -8
- package/lib/encodingStrategies/sszSnappy/errors.d.ts.map +1 -1
- package/lib/encodingStrategies/sszSnappy/errors.js +2 -3
- package/lib/encodingStrategies/sszSnappy/errors.js.map +1 -1
- package/lib/encodingStrategies/sszSnappy/index.d.ts +0 -1
- package/lib/encodingStrategies/sszSnappy/index.d.ts.map +1 -1
- package/lib/encodingStrategies/sszSnappy/index.js +0 -1
- package/lib/encodingStrategies/sszSnappy/index.js.map +1 -1
- package/lib/encodingStrategies/sszSnappy/utils.js.map +1 -1
- package/lib/interface.js +2 -1
- package/lib/interface.js.map +1 -1
- package/lib/metrics.d.ts +1 -7
- package/lib/metrics.d.ts.map +1 -1
- package/lib/metrics.js +1 -17
- package/lib/metrics.js.map +1 -1
- package/lib/rate_limiter/ReqRespRateLimiter.d.ts.map +1 -1
- package/lib/rate_limiter/ReqRespRateLimiter.js.map +1 -1
- package/lib/rate_limiter/rateLimiterGRCA.d.ts.map +1 -1
- package/lib/rate_limiter/rateLimiterGRCA.js.map +1 -1
- package/lib/rate_limiter/selfRateLimiter.d.ts.map +1 -1
- package/lib/rate_limiter/selfRateLimiter.js.map +1 -1
- package/lib/request/errors.d.ts +1 -7
- package/lib/request/errors.d.ts.map +1 -1
- package/lib/request/errors.js +3 -6
- package/lib/request/errors.js.map +1 -1
- package/lib/request/index.d.ts +0 -3
- package/lib/request/index.d.ts.map +1 -1
- package/lib/request/index.js +85 -70
- package/lib/request/index.js.map +1 -1
- package/lib/response/errors.d.ts.map +1 -1
- package/lib/response/errors.js +2 -1
- package/lib/response/errors.js.map +1 -1
- package/lib/response/index.d.ts +2 -2
- package/lib/response/index.d.ts.map +1 -1
- package/lib/response/index.js +46 -50
- package/lib/response/index.js.map +1 -1
- package/lib/types.d.ts +1 -2
- package/lib/types.d.ts.map +1 -1
- package/lib/types.js +7 -5
- package/lib/types.js.map +1 -1
- package/lib/utils/collectExactOne.js.map +1 -1
- package/lib/utils/collectMaxResponse.d.ts.map +1 -1
- package/lib/utils/collectMaxResponse.js +1 -2
- package/lib/utils/collectMaxResponse.js.map +1 -1
- package/lib/utils/errorMessage.d.ts +3 -3
- package/lib/utils/errorMessage.d.ts.map +1 -1
- package/lib/utils/errorMessage.js +14 -13
- package/lib/utils/errorMessage.js.map +1 -1
- package/lib/utils/index.d.ts +1 -3
- package/lib/utils/index.d.ts.map +1 -1
- package/lib/utils/index.js +1 -3
- package/lib/utils/index.js.map +1 -1
- package/lib/utils/peerId.js.map +1 -1
- package/lib/utils/protocolId.d.ts +2 -2
- package/lib/utils/protocolId.js +2 -2
- package/lib/utils/protocolId.js.map +1 -1
- package/lib/utils/snappyCommon.js +2 -1
- package/lib/utils/snappyCommon.js.map +1 -1
- package/lib/utils/snappyCompress.d.ts +1 -1
- package/lib/utils/snappyCompress.d.ts.map +1 -1
- package/lib/utils/snappyCompress.js +1 -1
- package/lib/utils/snappyCompress.js.map +1 -1
- package/lib/utils/snappyIndex.d.ts +1 -1
- package/lib/utils/snappyIndex.d.ts.map +1 -1
- package/lib/utils/snappyIndex.js +1 -1
- package/lib/utils/snappyIndex.js.map +1 -1
- package/lib/utils/snappyUncompress.d.ts +7 -11
- package/lib/utils/snappyUncompress.d.ts.map +1 -1
- package/lib/utils/snappyUncompress.js +68 -68
- package/lib/utils/snappyUncompress.js.map +1 -1
- package/lib/utils/stream.d.ts +6 -0
- package/lib/utils/stream.d.ts.map +1 -0
- package/lib/utils/stream.js +21 -0
- package/lib/utils/stream.js.map +1 -0
- package/package.json +16 -18
- package/src/ReqResp.ts +4 -4
- package/src/encoders/requestDecode.ts +32 -16
- package/src/encoders/requestEncode.ts +1 -1
- package/src/encoders/responseDecode.ts +68 -72
- package/src/encoders/responseEncode.ts +17 -29
- package/src/encodingStrategies/index.ts +8 -6
- package/src/encodingStrategies/sszSnappy/decode.ts +111 -53
- package/src/encodingStrategies/sszSnappy/encode.ts +2 -2
- package/src/encodingStrategies/sszSnappy/errors.ts +0 -4
- package/src/encodingStrategies/sszSnappy/index.ts +0 -1
- package/src/metrics.ts +1 -17
- package/src/request/errors.ts +1 -6
- package/src/request/index.ts +109 -86
- package/src/response/index.ts +55 -61
- package/src/types.ts +1 -3
- package/src/utils/collectMaxResponse.ts +1 -2
- package/src/utils/errorMessage.ts +14 -13
- package/src/utils/index.ts +1 -3
- package/src/utils/protocolId.ts +2 -2
- package/src/utils/snappyCompress.ts +1 -1
- package/src/utils/snappyIndex.ts +1 -1
- package/src/utils/snappyUncompress.ts +73 -75
- package/src/utils/stream.ts +34 -0
- package/lib/utils/abortableSource.d.ts +0 -12
- package/lib/utils/abortableSource.d.ts.map +0 -1
- package/lib/utils/abortableSource.js +0 -69
- package/lib/utils/abortableSource.js.map +0 -1
- package/lib/utils/bufferedSource.d.ts +0 -16
- package/lib/utils/bufferedSource.d.ts.map +0 -1
- package/lib/utils/bufferedSource.js +0 -40
- package/lib/utils/bufferedSource.js.map +0 -1
- package/lib/utils/onChunk.d.ts +0 -6
- package/lib/utils/onChunk.d.ts.map +0 -1
- package/lib/utils/onChunk.js +0 -13
- package/lib/utils/onChunk.js.map +0 -1
- package/lib/utils/snappy.d.ts +0 -3
- package/lib/utils/snappy.d.ts.map +0 -1
- package/lib/utils/snappy.js +0 -3
- package/lib/utils/snappy.js.map +0 -1
- package/src/utils/abortableSource.ts +0 -80
- package/src/utils/bufferedSource.ts +0 -46
- package/src/utils/onChunk.ts +0 -12
- package/src/utils/snappy.ts +0 -2
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import type {Stream} from "@libp2p/interface";
|
|
2
|
+
import type {ByteStream} from "@libp2p/utils";
|
|
1
3
|
import {decode as varintDecode, encodingLength as varintEncodingLength} from "uint8-varint";
|
|
2
4
|
import {Uint8ArrayList} from "uint8arraylist";
|
|
3
5
|
import {TypeSizes} from "../../types.js";
|
|
4
|
-
import {
|
|
5
|
-
import {SnappyFramesUncompress} from "../../utils/snappyIndex.js";
|
|
6
|
+
import {ChunkType, decodeSnappyFrameData, parseSnappyFrameHeader} from "../../utils/snappyIndex.js";
|
|
6
7
|
import {SszSnappyError, SszSnappyErrorCode} from "./errors.js";
|
|
7
8
|
import {maxEncodedLen} from "./utils.js";
|
|
8
9
|
|
|
@@ -15,76 +16,115 @@ export const MAX_VARINT_BYTES = 10;
|
|
|
15
16
|
* <encoding-dependent-header> | <encoded-payload>
|
|
16
17
|
* ```
|
|
17
18
|
*/
|
|
18
|
-
export async function readSszSnappyPayload(
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
export async function readSszSnappyPayload(
|
|
20
|
+
stream: ByteStream<Stream>,
|
|
21
|
+
type: TypeSizes,
|
|
22
|
+
signal?: AbortSignal
|
|
23
|
+
): Promise<Uint8Array> {
|
|
24
|
+
const sszDataLength = await readSszSnappyHeader(stream, type, signal);
|
|
25
|
+
|
|
26
|
+
return readSszSnappyBody(stream, sszDataLength, signal);
|
|
22
27
|
}
|
|
23
28
|
|
|
24
29
|
/**
|
|
25
30
|
* Reads `<encoding-dependent-header>` for ssz-snappy.
|
|
26
31
|
* encoding-header ::= the length of the raw SSZ bytes, encoded as an unsigned protobuf varint
|
|
27
32
|
*/
|
|
28
|
-
export async function readSszSnappyHeader(
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
33
|
+
export async function readSszSnappyHeader(
|
|
34
|
+
stream: ByteStream<Stream>,
|
|
35
|
+
type: TypeSizes,
|
|
36
|
+
signal?: AbortSignal
|
|
37
|
+
): Promise<number> {
|
|
38
|
+
const varintBytes: number[] = [];
|
|
34
39
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
sszDataLength = varintDecode(buffer.subarray());
|
|
38
|
-
} catch (_e) {
|
|
39
|
-
throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity});
|
|
40
|
-
}
|
|
40
|
+
while (true) {
|
|
41
|
+
const byte = await readExactOrSourceAborted(stream, 1, signal);
|
|
41
42
|
|
|
42
|
-
|
|
43
|
-
// encodingLength function only returns 1-8 inclusive
|
|
44
|
-
const varintBytes = varintEncodingLength(sszDataLength);
|
|
45
|
-
buffer.consume(varintBytes);
|
|
43
|
+
const value = byte.get(0);
|
|
46
44
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
if (sszDataLength < minSize) {
|
|
51
|
-
throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength});
|
|
52
|
-
}
|
|
53
|
-
if (sszDataLength > maxSize) {
|
|
54
|
-
throw new SszSnappyError({code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE, maxSize, sszDataLength});
|
|
45
|
+
varintBytes.push(value);
|
|
46
|
+
if (varintBytes.length > MAX_VARINT_BYTES) {
|
|
47
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintBytes.length});
|
|
55
48
|
}
|
|
56
49
|
|
|
57
|
-
|
|
50
|
+
// MSB not set => varint terminated
|
|
51
|
+
if ((value & 0x80) === 0) break;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
let sszDataLength: number;
|
|
55
|
+
try {
|
|
56
|
+
sszDataLength = varintDecode(Uint8Array.from(varintBytes));
|
|
57
|
+
} catch {
|
|
58
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: Infinity});
|
|
58
59
|
}
|
|
59
60
|
|
|
60
|
-
|
|
61
|
+
// MUST validate: the unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes
|
|
62
|
+
// encodingLength function only returns 1-8 inclusive
|
|
63
|
+
const varintByteLength = varintEncodingLength(sszDataLength);
|
|
64
|
+
if (varintByteLength > MAX_VARINT_BYTES) {
|
|
65
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.INVALID_VARINT_BYTES_COUNT, bytes: varintByteLength});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// MUST validate: the length-prefix is within the expected size bounds derived from the payload SSZ type.
|
|
69
|
+
const minSize = type.minSize;
|
|
70
|
+
const maxSize = type.maxSize;
|
|
71
|
+
if (sszDataLength < minSize) {
|
|
72
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.UNDER_SSZ_MIN_SIZE, minSize, sszDataLength});
|
|
73
|
+
}
|
|
74
|
+
if (sszDataLength > maxSize) {
|
|
75
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE, maxSize, sszDataLength});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return sszDataLength;
|
|
61
79
|
}
|
|
62
80
|
|
|
63
81
|
/**
|
|
64
82
|
* Reads `<encoded-payload>` for ssz-snappy and decompress.
|
|
65
83
|
* The returned bytes can be SSZ deseralized
|
|
66
84
|
*/
|
|
67
|
-
export async function readSszSnappyBody(
|
|
68
|
-
|
|
85
|
+
export async function readSszSnappyBody(
|
|
86
|
+
stream: ByteStream<Stream>,
|
|
87
|
+
sszDataLength: number,
|
|
88
|
+
signal?: AbortSignal
|
|
89
|
+
): Promise<Uint8Array> {
|
|
69
90
|
const uncompressedData = new Uint8ArrayList();
|
|
70
|
-
let
|
|
91
|
+
let encodedBytesRead = 0;
|
|
92
|
+
const maxBytes = maxEncodedLen(sszDataLength);
|
|
93
|
+
let foundIdentifier = false;
|
|
94
|
+
|
|
95
|
+
while (uncompressedData.length < sszDataLength) {
|
|
96
|
+
const header = await readExactOrSourceAborted(stream, 4, signal);
|
|
71
97
|
|
|
72
|
-
for await (const buffer of bufferedSource) {
|
|
73
98
|
// SHOULD NOT read more than max_encoded_len(n) bytes after reading the SSZ length-prefix n from the header
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
99
|
+
encodedBytesRead = addEncodedBytesReadOrThrow(encodedBytesRead, header.length, maxBytes, sszDataLength);
|
|
100
|
+
|
|
101
|
+
let headerParsed: {type: ChunkType; frameSize: number};
|
|
102
|
+
try {
|
|
103
|
+
headerParsed = parseSnappyFrameHeader(header.subarray());
|
|
104
|
+
if (!foundIdentifier && headerParsed.type !== ChunkType.IDENTIFIER) {
|
|
105
|
+
throw new Error("malformed input: must begin with an identifier");
|
|
106
|
+
}
|
|
107
|
+
} catch (e) {
|
|
108
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.DECOMPRESSOR_ERROR, decompressorError: e as Error});
|
|
77
109
|
}
|
|
78
110
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
111
|
+
if (headerParsed.frameSize > maxBytes - encodedBytesRead) {
|
|
112
|
+
throw new SszSnappyError({
|
|
113
|
+
code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ,
|
|
114
|
+
readBytes: encodedBytesRead + headerParsed.frameSize,
|
|
115
|
+
sszDataLength,
|
|
116
|
+
});
|
|
82
117
|
}
|
|
118
|
+
const frame = await readExactOrSourceAborted(stream, headerParsed.frameSize, signal);
|
|
119
|
+
|
|
120
|
+
encodedBytesRead = addEncodedBytesReadOrThrow(encodedBytesRead, frame.length, maxBytes, sszDataLength);
|
|
83
121
|
|
|
84
|
-
// stream contents can be passed through a buffered Snappy reader to decompress frame by frame
|
|
85
122
|
try {
|
|
86
|
-
|
|
87
|
-
|
|
123
|
+
if (headerParsed.type === ChunkType.IDENTIFIER) {
|
|
124
|
+
foundIdentifier = true;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const uncompressed = decodeSnappyFrameData(headerParsed.type, frame.subarray());
|
|
88
128
|
if (uncompressed !== null) {
|
|
89
129
|
uncompressedData.append(uncompressed);
|
|
90
130
|
}
|
|
@@ -96,16 +136,34 @@ export async function readSszSnappyBody(bufferedSource: BufferedSource, sszDataL
|
|
|
96
136
|
if (uncompressedData.length > sszDataLength) {
|
|
97
137
|
throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MANY_BYTES, sszDataLength});
|
|
98
138
|
}
|
|
139
|
+
}
|
|
99
140
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
}
|
|
141
|
+
// buffer.length === n
|
|
142
|
+
return uncompressedData.subarray(0, sszDataLength);
|
|
143
|
+
}
|
|
104
144
|
|
|
105
|
-
|
|
106
|
-
|
|
145
|
+
function addEncodedBytesReadOrThrow(
|
|
146
|
+
encodedBytesRead: number,
|
|
147
|
+
bytesToAdd: number,
|
|
148
|
+
maxBytes: number,
|
|
149
|
+
sszDataLength: number
|
|
150
|
+
): number {
|
|
151
|
+
const nextReadBytes = encodedBytesRead + bytesToAdd;
|
|
152
|
+
if (nextReadBytes > maxBytes) {
|
|
153
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ, readBytes: nextReadBytes, sszDataLength});
|
|
107
154
|
}
|
|
155
|
+
return nextReadBytes;
|
|
156
|
+
}
|
|
108
157
|
|
|
109
|
-
|
|
110
|
-
|
|
158
|
+
async function readExactOrSourceAborted(
|
|
159
|
+
stream: ByteStream<Stream>,
|
|
160
|
+
bytes: number,
|
|
161
|
+
signal?: AbortSignal
|
|
162
|
+
): Promise<Uint8ArrayList> {
|
|
163
|
+
return stream.read({bytes, signal}).catch((e) => {
|
|
164
|
+
if ((e as Error).name === "UnexpectedEOFError") {
|
|
165
|
+
throw new SszSnappyError({code: SszSnappyErrorCode.SOURCE_ABORTED});
|
|
166
|
+
}
|
|
167
|
+
throw e;
|
|
168
|
+
});
|
|
111
169
|
}
|
|
@@ -8,12 +8,12 @@ import {encodeSnappy} from "../../utils/snappyIndex.js";
|
|
|
8
8
|
* <encoding-dependent-header> | <encoded-payload>
|
|
9
9
|
* ```
|
|
10
10
|
*/
|
|
11
|
-
export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) =>
|
|
11
|
+
export const writeSszSnappyPayload = encodeSszSnappy as (bytes: Uint8Array) => Generator<Buffer>;
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* Buffered Snappy writer
|
|
15
15
|
*/
|
|
16
|
-
export
|
|
16
|
+
export function* encodeSszSnappy(bytes: Buffer): Generator<Buffer> {
|
|
17
17
|
// MUST encode the length of the raw SSZ bytes, encoded as an unsigned protobuf varint
|
|
18
18
|
const varint = varintEncode(bytes.length);
|
|
19
19
|
yield Buffer.from(varint.buffer, varint.byteOffset, varint.byteLength);
|
|
@@ -9,8 +9,6 @@ export enum SszSnappyErrorCode {
|
|
|
9
9
|
OVER_SSZ_MAX_SIZE = "SSZ_SNAPPY_ERROR_OVER_SSZ_MAX_SIZE",
|
|
10
10
|
TOO_MUCH_BYTES_READ = "SSZ_SNAPPY_ERROR_TOO_MUCH_BYTES_READ",
|
|
11
11
|
DECOMPRESSOR_ERROR = "SSZ_SNAPPY_ERROR_DECOMPRESSOR_ERROR",
|
|
12
|
-
DESERIALIZE_ERROR = "SSZ_SNAPPY_ERROR_DESERIALIZE_ERROR",
|
|
13
|
-
SERIALIZE_ERROR = "SSZ_SNAPPY_ERROR_SERIALIZE_ERROR",
|
|
14
12
|
/** Received more bytes than specified sszDataLength */
|
|
15
13
|
TOO_MANY_BYTES = "SSZ_SNAPPY_ERROR_TOO_MANY_BYTES",
|
|
16
14
|
/** Source aborted before reading sszDataLength bytes */
|
|
@@ -23,8 +21,6 @@ type SszSnappyErrorType =
|
|
|
23
21
|
| {code: SszSnappyErrorCode.OVER_SSZ_MAX_SIZE; maxSize: number; sszDataLength: number}
|
|
24
22
|
| {code: SszSnappyErrorCode.TOO_MUCH_BYTES_READ; readBytes: number; sszDataLength: number}
|
|
25
23
|
| {code: SszSnappyErrorCode.DECOMPRESSOR_ERROR; decompressorError: Error}
|
|
26
|
-
| {code: SszSnappyErrorCode.DESERIALIZE_ERROR; deserializeError: Error}
|
|
27
|
-
| {code: SszSnappyErrorCode.SERIALIZE_ERROR; serializeError: Error}
|
|
28
24
|
| {code: SszSnappyErrorCode.TOO_MANY_BYTES; sszDataLength: number}
|
|
29
25
|
| {code: SszSnappyErrorCode.SOURCE_ABORTED};
|
|
30
26
|
|
package/src/metrics.ts
CHANGED
|
@@ -4,7 +4,7 @@ import {RequestErrorCode} from "./request/errors.js";
|
|
|
4
4
|
export type Metrics = ReturnType<typeof getMetrics>;
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* A collection of metrics used throughout the
|
|
7
|
+
* A collection of metrics used throughout the Req/Resp domain.
|
|
8
8
|
*/
|
|
9
9
|
export function getMetrics(register: MetricsRegisterExtra) {
|
|
10
10
|
// Using function style instead of class to prevent having to re-declare all MetricsPrometheus types.
|
|
@@ -29,7 +29,6 @@ export function getMetrics(register: MetricsRegisterExtra) {
|
|
|
29
29
|
name: "beacon_reqresp_outgoing_request_roundtrip_time_seconds",
|
|
30
30
|
help: "Histogram of outgoing requests round-trip time",
|
|
31
31
|
labelNames: ["method"],
|
|
32
|
-
// Spec sets RESP_TIMEOUT = 10 sec
|
|
33
32
|
buckets: [0.1, 0.2, 0.5, 1, 5, 10, 15, 60],
|
|
34
33
|
}),
|
|
35
34
|
outgoingErrors: register.gauge<{method: string}>({
|
|
@@ -61,7 +60,6 @@ export function getMetrics(register: MetricsRegisterExtra) {
|
|
|
61
60
|
name: "beacon_reqresp_incoming_request_handler_time_seconds",
|
|
62
61
|
help: "Histogram of incoming requests internal handling time",
|
|
63
62
|
labelNames: ["method"],
|
|
64
|
-
// Spec sets RESP_TIMEOUT = 10 sec
|
|
65
63
|
buckets: [0.1, 0.2, 0.5, 1, 5, 10],
|
|
66
64
|
}),
|
|
67
65
|
incomingErrors: register.gauge<{method: string}>({
|
|
@@ -69,20 +67,6 @@ export function getMetrics(register: MetricsRegisterExtra) {
|
|
|
69
67
|
help: "Counts total failed responses handled per method",
|
|
70
68
|
labelNames: ["method"],
|
|
71
69
|
}),
|
|
72
|
-
outgoingResponseTTFB: register.histogram<{method: string}>({
|
|
73
|
-
name: "beacon_reqresp_outgoing_response_ttfb_seconds",
|
|
74
|
-
help: "Time to first byte (TTFB) for outgoing responses",
|
|
75
|
-
labelNames: ["method"],
|
|
76
|
-
// Spec sets TTFB_TIMEOUT = 5 sec
|
|
77
|
-
buckets: [0.1, 1, 5],
|
|
78
|
-
}),
|
|
79
|
-
incomingResponseTTFB: register.histogram<{method: string}>({
|
|
80
|
-
name: "beacon_reqresp_incoming_response_ttfb_seconds",
|
|
81
|
-
help: "Time to first byte (TTFB) for incoming responses",
|
|
82
|
-
labelNames: ["method"],
|
|
83
|
-
// Spec sets TTFB_TIMEOUT = 5 sec
|
|
84
|
-
buckets: [0.1, 1, 5],
|
|
85
|
-
}),
|
|
86
70
|
dialErrors: register.gauge({
|
|
87
71
|
name: "beacon_reqresp_dial_errors_total",
|
|
88
72
|
help: "Count total dial errors",
|
package/src/request/errors.ts
CHANGED
|
@@ -21,13 +21,9 @@ export enum RequestErrorCode {
|
|
|
21
21
|
REQUEST_TIMEOUT = "REQUEST_ERROR_REQUEST_TIMEOUT",
|
|
22
22
|
/** Error when sending request to responder */
|
|
23
23
|
REQUEST_ERROR = "REQUEST_ERROR_REQUEST_ERROR",
|
|
24
|
-
/** Reponder did not deliver a full reponse before max maxTotalResponseTimeout() */
|
|
25
|
-
RESPONSE_TIMEOUT = "REQUEST_ERROR_RESPONSE_TIMEOUT",
|
|
26
24
|
/** A single-response method returned 0 chunks */
|
|
27
25
|
EMPTY_RESPONSE = "REQUEST_ERROR_EMPTY_RESPONSE",
|
|
28
|
-
/**
|
|
29
|
-
TTFB_TIMEOUT = "REQUEST_ERROR_TTFB_TIMEOUT",
|
|
30
|
-
/** Timeout between `<response_chunk>` exceed */
|
|
26
|
+
/** Response transfer timeout exceeded */
|
|
31
27
|
RESP_TIMEOUT = "REQUEST_ERROR_RESP_TIMEOUT",
|
|
32
28
|
/** Request rate limited */
|
|
33
29
|
REQUEST_RATE_LIMITED = "REQUEST_ERROR_RATE_LIMITED",
|
|
@@ -50,7 +46,6 @@ type RequestErrorType =
|
|
|
50
46
|
| {code: RequestErrorCode.REQUEST_TIMEOUT}
|
|
51
47
|
| {code: RequestErrorCode.REQUEST_ERROR; error: Error}
|
|
52
48
|
| {code: RequestErrorCode.EMPTY_RESPONSE}
|
|
53
|
-
| {code: RequestErrorCode.TTFB_TIMEOUT}
|
|
54
49
|
| {code: RequestErrorCode.RESP_TIMEOUT}
|
|
55
50
|
| {code: RequestErrorCode.REQUEST_RATE_LIMITED}
|
|
56
51
|
| {code: RequestErrorCode.REQUEST_SELF_RATE_LIMITED}
|
package/src/request/index.ts
CHANGED
|
@@ -1,31 +1,82 @@
|
|
|
1
|
+
import type {Stream} from "@libp2p/interface";
|
|
1
2
|
import {PeerId} from "@libp2p/interface";
|
|
2
|
-
import {pipe} from "it-pipe";
|
|
3
3
|
import type {Libp2p} from "libp2p";
|
|
4
|
-
import {Uint8ArrayList} from "uint8arraylist";
|
|
5
4
|
import {ErrorAborted, Logger, TimeoutError, withTimeout} from "@lodestar/utils";
|
|
6
5
|
import {requestEncode} from "../encoders/requestEncode.js";
|
|
7
6
|
import {responseDecode} from "../encoders/responseDecode.js";
|
|
8
7
|
import {Metrics} from "../metrics.js";
|
|
9
8
|
import {ResponseError} from "../response/index.js";
|
|
10
9
|
import {MixedProtocol, ResponseIncoming} from "../types.js";
|
|
11
|
-
import {
|
|
10
|
+
import {prettyPrintPeerId, sendChunks} from "../utils/index.js";
|
|
12
11
|
import {RequestError, RequestErrorCode, responseStatusErrorToRequestError} from "./errors.js";
|
|
13
12
|
|
|
14
13
|
export {RequestError, RequestErrorCode};
|
|
15
14
|
|
|
16
|
-
//
|
|
15
|
+
// https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/p2p-interface.md#the-reqresp-domain
|
|
17
16
|
export const DEFAULT_DIAL_TIMEOUT = 5 * 1000; // 5 sec
|
|
18
17
|
export const DEFAULT_REQUEST_TIMEOUT = 5 * 1000; // 5 sec
|
|
19
|
-
export const DEFAULT_TTFB_TIMEOUT = 5 * 1000; // 5 sec
|
|
20
18
|
export const DEFAULT_RESP_TIMEOUT = 10 * 1000; // 10 sec
|
|
21
19
|
|
|
20
|
+
function getStreamNotFullyConsumedError(): Error {
|
|
21
|
+
return new Error("ReqResp stream was not fully consumed");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function scheduleStreamAbortIfNotClosed(stream: Stream, timeoutMs: number): void {
|
|
25
|
+
const onClose = (): void => {
|
|
26
|
+
clearTimeout(timeout);
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
const timeout = setTimeout(() => {
|
|
30
|
+
stream.removeEventListener("close", onClose);
|
|
31
|
+
if (stream.status === "open" && stream.remoteWriteStatus === "writable") {
|
|
32
|
+
stream.abort(getStreamNotFullyConsumedError());
|
|
33
|
+
}
|
|
34
|
+
}, timeoutMs);
|
|
35
|
+
|
|
36
|
+
stream.addEventListener("close", onClose, {once: true});
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
type ClearableSignal = AbortSignal & {clear: () => void};
|
|
40
|
+
|
|
41
|
+
function createRespSignal(signal: AbortSignal | undefined, timeoutMs: number): ClearableSignal {
|
|
42
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
43
|
+
const signals = signal ? [signal, timeoutSignal] : [timeoutSignal];
|
|
44
|
+
const controller = new AbortController();
|
|
45
|
+
|
|
46
|
+
const clear = (): void => {
|
|
47
|
+
for (const entry of signals) {
|
|
48
|
+
entry.removeEventListener("abort", onAbort);
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const onAbort = (): void => {
|
|
53
|
+
if (controller.signal.aborted) {
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
const reason = signals.find((entry) => entry.aborted)?.reason;
|
|
57
|
+
controller.abort(reason);
|
|
58
|
+
clear();
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
for (const entry of signals) {
|
|
62
|
+
if (entry.aborted) {
|
|
63
|
+
onAbort();
|
|
64
|
+
break;
|
|
65
|
+
}
|
|
66
|
+
entry.addEventListener("abort", onAbort);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const respSignal = controller.signal as ClearableSignal;
|
|
70
|
+
respSignal.clear = clear;
|
|
71
|
+
|
|
72
|
+
return respSignal;
|
|
73
|
+
}
|
|
74
|
+
|
|
22
75
|
export interface SendRequestOpts {
|
|
23
76
|
/** The maximum time for complete response transfer. */
|
|
24
77
|
respTimeoutMs?: number;
|
|
25
78
|
/** Non-spec timeout from sending request until write stream closed by responder */
|
|
26
79
|
requestTimeoutMs?: number;
|
|
27
|
-
/** The maximum time to wait for first byte of request response (time-to-first-byte). */
|
|
28
|
-
ttfbTimeoutMs?: number;
|
|
29
80
|
/** Non-spec timeout from dialing protocol until stream opened */
|
|
30
81
|
dialTimeoutMs?: number;
|
|
31
82
|
}
|
|
@@ -64,7 +115,6 @@ export async function* sendRequest(
|
|
|
64
115
|
|
|
65
116
|
const DIAL_TIMEOUT = opts?.dialTimeoutMs ?? DEFAULT_DIAL_TIMEOUT;
|
|
66
117
|
const REQUEST_TIMEOUT = opts?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT;
|
|
67
|
-
const TTFB_TIMEOUT = opts?.ttfbTimeoutMs ?? DEFAULT_TTFB_TIMEOUT;
|
|
68
118
|
const RESP_TIMEOUT = opts?.respTimeoutMs ?? DEFAULT_RESP_TIMEOUT;
|
|
69
119
|
|
|
70
120
|
const peerIdStrShort = prettyPrintPeerId(peerId);
|
|
@@ -83,17 +133,6 @@ export async function* sendRequest(
|
|
|
83
133
|
// the picked protocol in `connection.protocol`
|
|
84
134
|
const protocolsMap = new Map<string, MixedProtocol>(protocols.map((protocol, i) => [protocolIDs[i], protocol]));
|
|
85
135
|
|
|
86
|
-
// As of October 2020 we can't rely on libp2p.dialProtocol timeout to work so
|
|
87
|
-
// this function wraps the dialProtocol promise with an extra timeout
|
|
88
|
-
//
|
|
89
|
-
// > The issue might be: you add the peer's addresses to the AddressBook,
|
|
90
|
-
// which will result in autoDial to kick in and dial your peer. In parallel,
|
|
91
|
-
// you do a manual dial and it will wait for the previous one without using
|
|
92
|
-
// the abort signal:
|
|
93
|
-
//
|
|
94
|
-
// https://github.com/ChainSafe/lodestar/issues/1597#issuecomment-703394386
|
|
95
|
-
|
|
96
|
-
// DIAL_TIMEOUT: Non-spec timeout from dialing protocol until stream opened
|
|
97
136
|
const stream = await withTimeout(
|
|
98
137
|
async (timeoutAndParentSignal) => {
|
|
99
138
|
const protocolIds = Array.from(protocolsMap.keys());
|
|
@@ -112,9 +151,6 @@ export async function* sendRequest(
|
|
|
112
151
|
|
|
113
152
|
metrics?.outgoingOpenedStreams?.inc({method});
|
|
114
153
|
|
|
115
|
-
// TODO: Does the TTFB timer start on opening stream or after receiving request
|
|
116
|
-
const timerTTFB = metrics?.outgoingResponseTTFB.startTimer({method});
|
|
117
|
-
|
|
118
154
|
// Parse protocol selected by the responder
|
|
119
155
|
const protocolId = stream.protocol ?? "unknown";
|
|
120
156
|
const protocol = protocolsMap.get(protocolId);
|
|
@@ -126,21 +162,24 @@ export async function* sendRequest(
|
|
|
126
162
|
logger.debug("Req sending request", logCtx);
|
|
127
163
|
|
|
128
164
|
// Spec: The requester MUST close the write side of the stream once it finishes writing the request message
|
|
129
|
-
// Impl: stream.sink is closed automatically by js-libp2p-mplex when piped source is exhausted
|
|
130
165
|
|
|
131
166
|
// REQUEST_TIMEOUT: Non-spec timeout from sending request until write stream closed by responder
|
|
132
|
-
// Note: libp2p.stop() will close all connections, so not necessary to abort this
|
|
133
|
-
await withTimeout(
|
|
134
|
-
(
|
|
135
|
-
|
|
136
|
-
stream.
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
167
|
+
// Note: libp2p.stop() will close all connections, so not necessary to abort this send on parent stop
|
|
168
|
+
await withTimeout(
|
|
169
|
+
async (timeoutAndParentSignal) => {
|
|
170
|
+
await sendChunks(stream, requestEncode(protocol, requestBody), timeoutAndParentSignal);
|
|
171
|
+
await stream.close({signal: timeoutAndParentSignal});
|
|
172
|
+
},
|
|
173
|
+
REQUEST_TIMEOUT,
|
|
174
|
+
signal
|
|
175
|
+
).catch((e) => {
|
|
176
|
+
stream.abort(e as Error);
|
|
177
|
+
|
|
178
|
+
if (e instanceof TimeoutError) {
|
|
179
|
+
throw new RequestError({code: RequestErrorCode.REQUEST_TIMEOUT});
|
|
142
180
|
}
|
|
143
|
-
|
|
181
|
+
throw new RequestError({code: RequestErrorCode.REQUEST_ERROR, error: e as Error});
|
|
182
|
+
});
|
|
144
183
|
|
|
145
184
|
logger.debug("Req request sent", logCtx);
|
|
146
185
|
|
|
@@ -150,67 +189,51 @@ export async function* sendRequest(
|
|
|
150
189
|
return;
|
|
151
190
|
}
|
|
152
191
|
|
|
153
|
-
//
|
|
154
|
-
|
|
155
|
-
// - Max total timeout: This timeout is not required by the spec. It may not be necessary, but it's kept as
|
|
156
|
-
// safe-guard to close. streams in case of bugs on other timeout mechanisms.
|
|
157
|
-
const ttfbTimeoutController = new AbortController();
|
|
158
|
-
const respTimeoutController = new AbortController();
|
|
159
|
-
|
|
160
|
-
let timeoutRESP: NodeJS.Timeout | null = null;
|
|
192
|
+
// RESP_TIMEOUT: Maximum time for complete response transfer
|
|
193
|
+
const respSignal = createRespSignal(signal, RESP_TIMEOUT);
|
|
161
194
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
if (timeoutRESP) clearTimeout(timeoutRESP);
|
|
165
|
-
ttfbTimeoutController.abort();
|
|
166
|
-
}, TTFB_TIMEOUT);
|
|
167
|
-
|
|
168
|
-
const restartRespTimeout = (): void => {
|
|
169
|
-
if (timeoutRESP) clearTimeout(timeoutRESP);
|
|
170
|
-
timeoutRESP = setTimeout(() => respTimeoutController.abort(), RESP_TIMEOUT);
|
|
171
|
-
};
|
|
195
|
+
let responseError: Error | null = null;
|
|
196
|
+
let responseFullyConsumed = false;
|
|
172
197
|
|
|
173
198
|
try {
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
{
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
},
|
|
181
|
-
{
|
|
182
|
-
signal: respTimeoutController.signal,
|
|
183
|
-
getError: () => new RequestError({code: RequestErrorCode.RESP_TIMEOUT}),
|
|
184
|
-
},
|
|
185
|
-
]),
|
|
186
|
-
|
|
187
|
-
// Transforms `Buffer` chunks to yield `ResponseBody` chunks
|
|
188
|
-
responseDecode(protocol, {
|
|
189
|
-
onFirstHeader() {
|
|
190
|
-
// On first byte, cancel the single use TTFB_TIMEOUT, and start RESP_TIMEOUT
|
|
191
|
-
clearTimeout(timeoutTTFB);
|
|
192
|
-
timerTTFB?.();
|
|
193
|
-
restartRespTimeout();
|
|
194
|
-
},
|
|
195
|
-
onFirstResponseChunk() {
|
|
196
|
-
// On <response_chunk>, cancel this chunk's RESP_TIMEOUT and start next's
|
|
197
|
-
restartRespTimeout();
|
|
198
|
-
},
|
|
199
|
-
})
|
|
200
|
-
);
|
|
199
|
+
yield* responseDecode(protocol, stream, {
|
|
200
|
+
signal: respSignal,
|
|
201
|
+
getError: () =>
|
|
202
|
+
signal?.aborted ? new ErrorAborted("sendRequest") : new RequestError({code: RequestErrorCode.RESP_TIMEOUT}),
|
|
203
|
+
});
|
|
204
|
+
responseFullyConsumed = true;
|
|
201
205
|
|
|
202
206
|
// NOTE: Only log once per request to verbose, intermediate steps to debug
|
|
203
207
|
// NOTE: Do not log the response, logs get extremely cluttered
|
|
204
208
|
// NOTE: add double space after "Req " to align log with the "Resp " log
|
|
205
209
|
logger.verbose("Req done", logCtx);
|
|
210
|
+
} catch (e) {
|
|
211
|
+
responseError = e as Error;
|
|
212
|
+
throw e;
|
|
206
213
|
} finally {
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
+
// On decode/timeout failures abort immediately so mplex can reclaim stream state.
|
|
215
|
+
// On normal early consumer exit, close gracefully to avoid stream-id desync with peers.
|
|
216
|
+
if (responseError !== null || signal?.aborted) {
|
|
217
|
+
stream.abort(responseError ?? new ErrorAborted("sendRequest"));
|
|
218
|
+
} else {
|
|
219
|
+
await stream.close().catch((e) => {
|
|
220
|
+
stream.abort(e as Error);
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
if (!responseFullyConsumed) {
|
|
224
|
+
// Stop buffering unread inbound data after caller exits early.
|
|
225
|
+
// mplex does not support propagating closeRead to the remote, so still
|
|
226
|
+
// abort later if the remote never closes write.
|
|
227
|
+
await stream.closeRead().catch(() => {
|
|
228
|
+
// Ignore closeRead errors - close/abort path below will reclaim stream.
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
if (stream.remoteWriteStatus === "writable") {
|
|
232
|
+
scheduleStreamAbortIfNotClosed(stream, RESP_TIMEOUT);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
respSignal.clear();
|
|
214
237
|
metrics?.outgoingClosedStreams?.inc({method});
|
|
215
238
|
logger.verbose("Req stream closed", logCtx);
|
|
216
239
|
}
|