@lodestar/reqresp 1.35.0-dev.fcf8d024ea → 1.35.0-dev.feed916580
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +12 -12
- package/lib/ReqResp.d.ts.map +0 -1
- package/lib/encoders/requestDecode.d.ts.map +0 -1
- package/lib/encoders/requestEncode.d.ts.map +0 -1
- package/lib/encoders/responseDecode.d.ts.map +0 -1
- package/lib/encoders/responseEncode.d.ts.map +0 -1
- package/lib/encodingStrategies/index.d.ts.map +0 -1
- package/lib/encodingStrategies/sszSnappy/decode.d.ts.map +0 -1
- package/lib/encodingStrategies/sszSnappy/encode.d.ts.map +0 -1
- package/lib/encodingStrategies/sszSnappy/errors.d.ts.map +0 -1
- package/lib/encodingStrategies/sszSnappy/index.d.ts.map +0 -1
- package/lib/encodingStrategies/sszSnappy/snappyFrames/common.d.ts.map +0 -1
- package/lib/encodingStrategies/sszSnappy/snappyFrames/compress.d.ts.map +0 -1
- package/lib/encodingStrategies/sszSnappy/snappyFrames/uncompress.d.ts.map +0 -1
- package/lib/encodingStrategies/sszSnappy/utils.d.ts.map +0 -1
- package/lib/index.d.ts.map +0 -1
- package/lib/interface.d.ts.map +0 -1
- package/lib/metrics.d.ts.map +0 -1
- package/lib/rate_limiter/ReqRespRateLimiter.d.ts.map +0 -1
- package/lib/rate_limiter/rateLimiterGRCA.d.ts.map +0 -1
- package/lib/rate_limiter/selfRateLimiter.d.ts.map +0 -1
- package/lib/request/errors.d.ts.map +0 -1
- package/lib/request/index.d.ts.map +0 -1
- package/lib/response/errors.d.ts.map +0 -1
- package/lib/response/index.d.ts.map +0 -1
- package/lib/types.d.ts.map +0 -1
- package/lib/utils/abortableSource.d.ts.map +0 -1
- package/lib/utils/bufferedSource.d.ts.map +0 -1
- package/lib/utils/collectExactOne.d.ts.map +0 -1
- package/lib/utils/collectMaxResponse.d.ts.map +0 -1
- package/lib/utils/errorMessage.d.ts.map +0 -1
- package/lib/utils/index.d.ts.map +0 -1
- package/lib/utils/onChunk.d.ts.map +0 -1
- package/lib/utils/peerId.d.ts.map +0 -1
- package/lib/utils/protocolId.d.ts.map +0 -1
- package/src/ReqResp.ts +0 -289
- package/src/encoders/requestDecode.ts +0 -29
- package/src/encoders/requestEncode.ts +0 -18
- package/src/encoders/responseDecode.ts +0 -169
- package/src/encoders/responseEncode.ts +0 -81
- package/src/encodingStrategies/index.ts +0 -46
- package/src/encodingStrategies/sszSnappy/decode.ts +0 -111
- package/src/encodingStrategies/sszSnappy/encode.ts +0 -24
- package/src/encodingStrategies/sszSnappy/errors.ts +0 -31
- package/src/encodingStrategies/sszSnappy/index.ts +0 -3
- package/src/encodingStrategies/sszSnappy/snappyFrames/common.ts +0 -36
- package/src/encodingStrategies/sszSnappy/snappyFrames/compress.ts +0 -25
- package/src/encodingStrategies/sszSnappy/snappyFrames/uncompress.ts +0 -114
- package/src/encodingStrategies/sszSnappy/utils.ts +0 -7
- package/src/index.ts +0 -10
- package/src/interface.ts +0 -26
- package/src/metrics.ts +0 -95
- package/src/rate_limiter/ReqRespRateLimiter.ts +0 -107
- package/src/rate_limiter/rateLimiterGRCA.ts +0 -92
- package/src/rate_limiter/selfRateLimiter.ts +0 -112
- package/src/request/errors.ts +0 -119
- package/src/request/index.ts +0 -225
- package/src/response/errors.ts +0 -50
- package/src/response/index.ts +0 -147
- package/src/types.ts +0 -158
- package/src/utils/abortableSource.ts +0 -80
- package/src/utils/bufferedSource.ts +0 -46
- package/src/utils/collectExactOne.ts +0 -15
- package/src/utils/collectMaxResponse.ts +0 -19
- package/src/utils/errorMessage.ts +0 -51
- package/src/utils/index.ts +0 -8
- package/src/utils/onChunk.ts +0 -12
- package/src/utils/peerId.ts +0 -6
- package/src/utils/protocolId.ts +0 -44
package/src/request/index.ts
DELETED
|
@@ -1,225 +0,0 @@
|
|
|
1
|
-
import {PeerId} from "@libp2p/interface";
|
|
2
|
-
import {pipe} from "it-pipe";
|
|
3
|
-
import type {Libp2p} from "libp2p";
|
|
4
|
-
import {Uint8ArrayList} from "uint8arraylist";
|
|
5
|
-
import {ErrorAborted, Logger, TimeoutError, withTimeout} from "@lodestar/utils";
|
|
6
|
-
import {requestEncode} from "../encoders/requestEncode.js";
|
|
7
|
-
import {responseDecode} from "../encoders/responseDecode.js";
|
|
8
|
-
import {Metrics} from "../metrics.js";
|
|
9
|
-
import {ResponseError} from "../response/index.js";
|
|
10
|
-
import {MixedProtocol, ResponseIncoming} from "../types.js";
|
|
11
|
-
import {abortableSource, prettyPrintPeerId} from "../utils/index.js";
|
|
12
|
-
import {RequestError, RequestErrorCode, responseStatusErrorToRequestError} from "./errors.js";
|
|
13
|
-
|
|
14
|
-
export {RequestError, RequestErrorCode};
|
|
15
|
-
|
|
16
|
-
// Default spec values from https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#configuration
|
|
17
|
-
export const DEFAULT_DIAL_TIMEOUT = 5 * 1000; // 5 sec
|
|
18
|
-
export const DEFAULT_REQUEST_TIMEOUT = 5 * 1000; // 5 sec
|
|
19
|
-
export const DEFAULT_TTFB_TIMEOUT = 5 * 1000; // 5 sec
|
|
20
|
-
export const DEFAULT_RESP_TIMEOUT = 10 * 1000; // 10 sec
|
|
21
|
-
|
|
22
|
-
export interface SendRequestOpts {
|
|
23
|
-
/** The maximum time for complete response transfer. */
|
|
24
|
-
respTimeoutMs?: number;
|
|
25
|
-
/** Non-spec timeout from sending request until write stream closed by responder */
|
|
26
|
-
requestTimeoutMs?: number;
|
|
27
|
-
/** The maximum time to wait for first byte of request response (time-to-first-byte). */
|
|
28
|
-
ttfbTimeoutMs?: number;
|
|
29
|
-
/** Non-spec timeout from dialing protocol until stream opened */
|
|
30
|
-
dialTimeoutMs?: number;
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
type SendRequestModules = {
|
|
34
|
-
logger: Logger;
|
|
35
|
-
libp2p: Libp2p;
|
|
36
|
-
metrics: Metrics | null;
|
|
37
|
-
peerClient?: string;
|
|
38
|
-
};
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* Sends ReqResp request to a peer. Throws on error. Logs each step of the request lifecycle.
|
|
42
|
-
*
|
|
43
|
-
* 1. Dial peer, establish duplex stream
|
|
44
|
-
* 2. Encoded and write request to peer. Expect the responder to close the stream's write side
|
|
45
|
-
* 3. Read and decode reponse(s) from peer. Will close the read stream if:
|
|
46
|
-
* - An error result is received in one of the chunks. Reads the error_message and throws.
|
|
47
|
-
* - The responder closes the stream. If at the end or start of a <response_chunk>, return. Otherwise throws
|
|
48
|
-
* - Any part of the response_chunk fails validation. Throws a typed error (see `SszSnappyError`)
|
|
49
|
-
* - The maximum number of requested chunks are read. Does not throw, returns read chunks only.
|
|
50
|
-
*/
|
|
51
|
-
export async function* sendRequest(
|
|
52
|
-
{logger, libp2p, metrics, peerClient}: SendRequestModules,
|
|
53
|
-
peerId: PeerId,
|
|
54
|
-
protocols: MixedProtocol[],
|
|
55
|
-
protocolIDs: string[],
|
|
56
|
-
requestBody: Uint8Array,
|
|
57
|
-
signal?: AbortSignal,
|
|
58
|
-
opts?: SendRequestOpts,
|
|
59
|
-
requestId = 0
|
|
60
|
-
): AsyncIterable<ResponseIncoming> {
|
|
61
|
-
if (protocols.length === 0) {
|
|
62
|
-
throw Error("sendRequest must set > 0 protocols");
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
const DIAL_TIMEOUT = opts?.dialTimeoutMs ?? DEFAULT_DIAL_TIMEOUT;
|
|
66
|
-
const REQUEST_TIMEOUT = opts?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT;
|
|
67
|
-
const TTFB_TIMEOUT = opts?.ttfbTimeoutMs ?? DEFAULT_TTFB_TIMEOUT;
|
|
68
|
-
const RESP_TIMEOUT = opts?.respTimeoutMs ?? DEFAULT_RESP_TIMEOUT;
|
|
69
|
-
|
|
70
|
-
const peerIdStrShort = prettyPrintPeerId(peerId);
|
|
71
|
-
const {method, encoding, version} = protocols[0];
|
|
72
|
-
const logCtx = {method, version, encoding, client: peerClient, peer: peerIdStrShort, requestId};
|
|
73
|
-
|
|
74
|
-
if (signal?.aborted) {
|
|
75
|
-
throw new ErrorAborted("sendRequest");
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
logger.debug("Req dialing peer", logCtx);
|
|
79
|
-
|
|
80
|
-
try {
|
|
81
|
-
// From Altair block query methods have V1 and V2. Both protocols should be requested.
|
|
82
|
-
// On stream negotiation `libp2p.dialProtocol` will pick the available protocol and return
|
|
83
|
-
// the picked protocol in `connection.protocol`
|
|
84
|
-
const protocolsMap = new Map<string, MixedProtocol>(protocols.map((protocol, i) => [protocolIDs[i], protocol]));
|
|
85
|
-
|
|
86
|
-
// As of October 2020 we can't rely on libp2p.dialProtocol timeout to work so
|
|
87
|
-
// this function wraps the dialProtocol promise with an extra timeout
|
|
88
|
-
//
|
|
89
|
-
// > The issue might be: you add the peer's addresses to the AddressBook,
|
|
90
|
-
// which will result in autoDial to kick in and dial your peer. In parallel,
|
|
91
|
-
// you do a manual dial and it will wait for the previous one without using
|
|
92
|
-
// the abort signal:
|
|
93
|
-
//
|
|
94
|
-
// https://github.com/ChainSafe/lodestar/issues/1597#issuecomment-703394386
|
|
95
|
-
|
|
96
|
-
// DIAL_TIMEOUT: Non-spec timeout from dialing protocol until stream opened
|
|
97
|
-
const stream = await withTimeout(
|
|
98
|
-
async (timeoutAndParentSignal) => {
|
|
99
|
-
const protocolIds = Array.from(protocolsMap.keys());
|
|
100
|
-
const conn = await libp2p.dialProtocol(peerId, protocolIds, {signal: timeoutAndParentSignal});
|
|
101
|
-
if (!conn) throw Error("dialProtocol timeout");
|
|
102
|
-
return conn;
|
|
103
|
-
},
|
|
104
|
-
DIAL_TIMEOUT,
|
|
105
|
-
signal
|
|
106
|
-
).catch((e: Error) => {
|
|
107
|
-
if (e instanceof TimeoutError) {
|
|
108
|
-
throw new RequestError({code: RequestErrorCode.DIAL_TIMEOUT});
|
|
109
|
-
}
|
|
110
|
-
throw new RequestError({code: RequestErrorCode.DIAL_ERROR, error: e});
|
|
111
|
-
});
|
|
112
|
-
|
|
113
|
-
metrics?.outgoingOpenedStreams?.inc({method});
|
|
114
|
-
|
|
115
|
-
// TODO: Does the TTFB timer start on opening stream or after receiving request
|
|
116
|
-
const timerTTFB = metrics?.outgoingResponseTTFB.startTimer({method});
|
|
117
|
-
|
|
118
|
-
// Parse protocol selected by the responder
|
|
119
|
-
const protocolId = stream.protocol ?? "unknown";
|
|
120
|
-
const protocol = protocolsMap.get(protocolId);
|
|
121
|
-
if (!protocol) throw Error(`dialProtocol selected unknown protocolId ${protocolId}`);
|
|
122
|
-
|
|
123
|
-
// Override with actual version that was negotiated
|
|
124
|
-
logCtx.version = protocol.version;
|
|
125
|
-
|
|
126
|
-
logger.debug("Req sending request", logCtx);
|
|
127
|
-
|
|
128
|
-
// Spec: The requester MUST close the write side of the stream once it finishes writing the request message
|
|
129
|
-
// Impl: stream.sink is closed automatically by js-libp2p-mplex when piped source is exhausted
|
|
130
|
-
|
|
131
|
-
// REQUEST_TIMEOUT: Non-spec timeout from sending request until write stream closed by responder
|
|
132
|
-
// Note: libp2p.stop() will close all connections, so not necessary to abort this pipe on parent stop
|
|
133
|
-
await withTimeout(() => pipe(requestEncode(protocol, requestBody), stream.sink), REQUEST_TIMEOUT, signal).catch(
|
|
134
|
-
(e) => {
|
|
135
|
-
// Must close the stream read side (stream.source) manually AND the write side
|
|
136
|
-
stream.abort(e);
|
|
137
|
-
|
|
138
|
-
if (e instanceof TimeoutError) {
|
|
139
|
-
throw new RequestError({code: RequestErrorCode.REQUEST_TIMEOUT});
|
|
140
|
-
}
|
|
141
|
-
throw new RequestError({code: RequestErrorCode.REQUEST_ERROR, error: e as Error});
|
|
142
|
-
}
|
|
143
|
-
);
|
|
144
|
-
|
|
145
|
-
logger.debug("Req request sent", logCtx);
|
|
146
|
-
|
|
147
|
-
// For goodbye method peers may disconnect before completing the response and trigger multiple errors.
|
|
148
|
-
// Do not expect them to reply and successfully return early
|
|
149
|
-
if (protocol.ignoreResponse) {
|
|
150
|
-
return;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
// - TTFB_TIMEOUT: The requester MUST wait a maximum of TTFB_TIMEOUT for the first response byte to arrive
|
|
154
|
-
// - RESP_TIMEOUT: Requester allows a further RESP_TIMEOUT for each subsequent response_chunk
|
|
155
|
-
// - Max total timeout: This timeout is not required by the spec. It may not be necessary, but it's kept as
|
|
156
|
-
// safe-guard to close. streams in case of bugs on other timeout mechanisms.
|
|
157
|
-
const ttfbTimeoutController = new AbortController();
|
|
158
|
-
const respTimeoutController = new AbortController();
|
|
159
|
-
|
|
160
|
-
let timeoutRESP: NodeJS.Timeout | null = null;
|
|
161
|
-
|
|
162
|
-
const timeoutTTFB = setTimeout(() => {
|
|
163
|
-
// If we abort on first byte delay, don't need to abort for response delay
|
|
164
|
-
if (timeoutRESP) clearTimeout(timeoutRESP);
|
|
165
|
-
ttfbTimeoutController.abort();
|
|
166
|
-
}, TTFB_TIMEOUT);
|
|
167
|
-
|
|
168
|
-
const restartRespTimeout = (): void => {
|
|
169
|
-
if (timeoutRESP) clearTimeout(timeoutRESP);
|
|
170
|
-
timeoutRESP = setTimeout(() => respTimeoutController.abort(), RESP_TIMEOUT);
|
|
171
|
-
};
|
|
172
|
-
|
|
173
|
-
try {
|
|
174
|
-
// Note: libp2p.stop() will close all connections, so not necessary to abort this pipe on parent stop
|
|
175
|
-
yield* pipe(
|
|
176
|
-
abortableSource(stream.source as AsyncIterable<Uint8ArrayList>, [
|
|
177
|
-
{
|
|
178
|
-
signal: ttfbTimeoutController.signal,
|
|
179
|
-
getError: () => new RequestError({code: RequestErrorCode.TTFB_TIMEOUT}),
|
|
180
|
-
},
|
|
181
|
-
{
|
|
182
|
-
signal: respTimeoutController.signal,
|
|
183
|
-
getError: () => new RequestError({code: RequestErrorCode.RESP_TIMEOUT}),
|
|
184
|
-
},
|
|
185
|
-
]),
|
|
186
|
-
|
|
187
|
-
// Transforms `Buffer` chunks to yield `ResponseBody` chunks
|
|
188
|
-
responseDecode(protocol, {
|
|
189
|
-
onFirstHeader() {
|
|
190
|
-
// On first byte, cancel the single use TTFB_TIMEOUT, and start RESP_TIMEOUT
|
|
191
|
-
clearTimeout(timeoutTTFB);
|
|
192
|
-
timerTTFB?.();
|
|
193
|
-
restartRespTimeout();
|
|
194
|
-
},
|
|
195
|
-
onFirstResponseChunk() {
|
|
196
|
-
// On <response_chunk>, cancel this chunk's RESP_TIMEOUT and start next's
|
|
197
|
-
restartRespTimeout();
|
|
198
|
-
},
|
|
199
|
-
})
|
|
200
|
-
);
|
|
201
|
-
|
|
202
|
-
// NOTE: Only log once per request to verbose, intermediate steps to debug
|
|
203
|
-
// NOTE: Do not log the response, logs get extremely cluttered
|
|
204
|
-
// NOTE: add double space after "Req " to align log with the "Resp " log
|
|
205
|
-
logger.verbose("Req done", logCtx);
|
|
206
|
-
} finally {
|
|
207
|
-
clearTimeout(timeoutTTFB);
|
|
208
|
-
if (timeoutRESP !== null) clearTimeout(timeoutRESP);
|
|
209
|
-
|
|
210
|
-
// Necessary to call `stream.close()` since collectResponses() may break out of the source before exhausting it
|
|
211
|
-
// `stream.close()` libp2p-mplex will .end() the source (it-pushable instance)
|
|
212
|
-
// If collectResponses() exhausts the source, it-pushable.end() can be safely called multiple times
|
|
213
|
-
await stream.close();
|
|
214
|
-
metrics?.outgoingClosedStreams?.inc({method});
|
|
215
|
-
logger.verbose("Req stream closed", logCtx);
|
|
216
|
-
}
|
|
217
|
-
} catch (e) {
|
|
218
|
-
logger.verbose("Req error", logCtx, e as Error);
|
|
219
|
-
|
|
220
|
-
if (e instanceof ResponseError) {
|
|
221
|
-
throw new RequestError(responseStatusErrorToRequestError(e));
|
|
222
|
-
}
|
|
223
|
-
throw e;
|
|
224
|
-
}
|
|
225
|
-
}
|
package/src/response/errors.ts
DELETED
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
import {LodestarError, LodestarErrorMetaData, LodestarErrorObject} from "@lodestar/utils";
|
|
2
|
-
import {RespStatus, RpcResponseStatusError} from "../interface.js";
|
|
3
|
-
|
|
4
|
-
type RpcResponseStatusNotSuccess = Exclude<RespStatus, RespStatus.SUCCESS>;
|
|
5
|
-
|
|
6
|
-
export enum ResponseErrorCode {
|
|
7
|
-
RESPONSE_STATUS_ERROR = "RESPONSE_STATUS_ERROR",
|
|
8
|
-
}
|
|
9
|
-
|
|
10
|
-
type RequestErrorType = {
|
|
11
|
-
code: ResponseErrorCode;
|
|
12
|
-
status: RpcResponseStatusError;
|
|
13
|
-
errorMessage: string;
|
|
14
|
-
};
|
|
15
|
-
|
|
16
|
-
export const RESPONSE_ERROR_CLASS_NAME = "ResponseError";
|
|
17
|
-
|
|
18
|
-
/**
|
|
19
|
-
* Used internally only to signal a response status error. Since the error should never bubble up to the user,
|
|
20
|
-
* the error code and error message does not matter much.
|
|
21
|
-
*/
|
|
22
|
-
export class ResponseError extends LodestarError<RequestErrorType> {
|
|
23
|
-
status: RpcResponseStatusNotSuccess;
|
|
24
|
-
errorMessage: string;
|
|
25
|
-
constructor(status: RpcResponseStatusNotSuccess, errorMessage: string, stack?: string) {
|
|
26
|
-
const type = {code: ResponseErrorCode.RESPONSE_STATUS_ERROR, status, errorMessage};
|
|
27
|
-
super(type, `RESPONSE_ERROR_${RespStatus[status]}: ${errorMessage}`, stack);
|
|
28
|
-
this.status = status;
|
|
29
|
-
this.errorMessage = errorMessage;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
getMetadata(): LodestarErrorMetaData {
|
|
33
|
-
return {
|
|
34
|
-
status: this.status,
|
|
35
|
-
errorMessage: this.errorMessage,
|
|
36
|
-
};
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
static fromObject(obj: LodestarErrorObject): ResponseError {
|
|
40
|
-
if (obj.className !== RESPONSE_ERROR_CLASS_NAME) {
|
|
41
|
-
throw new Error(`Expected className to be ResponseError, but got ${obj.className}`);
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
return new ResponseError(
|
|
45
|
-
obj.type.status as RpcResponseStatusNotSuccess,
|
|
46
|
-
obj.type.errorMessage as string,
|
|
47
|
-
obj.stack
|
|
48
|
-
);
|
|
49
|
-
}
|
|
50
|
-
}
|
package/src/response/index.ts
DELETED
|
@@ -1,147 +0,0 @@
|
|
|
1
|
-
import {PeerId, Stream} from "@libp2p/interface";
|
|
2
|
-
import {pipe} from "it-pipe";
|
|
3
|
-
import {Uint8ArrayList} from "uint8arraylist";
|
|
4
|
-
import {Logger, TimeoutError, withTimeout} from "@lodestar/utils";
|
|
5
|
-
import {requestDecode} from "../encoders/requestDecode.js";
|
|
6
|
-
import {responseEncodeError, responseEncodeSuccess} from "../encoders/responseEncode.js";
|
|
7
|
-
import {RespStatus} from "../interface.js";
|
|
8
|
-
import {Metrics} from "../metrics.js";
|
|
9
|
-
import {ReqRespRateLimiter} from "../rate_limiter/ReqRespRateLimiter.js";
|
|
10
|
-
import {RequestError, RequestErrorCode} from "../request/errors.js";
|
|
11
|
-
import {Protocol, ReqRespRequest} from "../types.js";
|
|
12
|
-
import {prettyPrintPeerId} from "../utils/index.js";
|
|
13
|
-
import {ResponseError} from "./errors.js";
|
|
14
|
-
|
|
15
|
-
export {ResponseError};
|
|
16
|
-
|
|
17
|
-
// Default spec values from https://github.com/ethereum/consensus-specs/blob/v1.2.0/specs/phase0/p2p-interface.md#configuration
|
|
18
|
-
export const DEFAULT_REQUEST_TIMEOUT = 5 * 1000; // 5 sec
|
|
19
|
-
|
|
20
|
-
export interface HandleRequestOpts {
|
|
21
|
-
logger: Logger;
|
|
22
|
-
metrics: Metrics | null;
|
|
23
|
-
stream: Stream;
|
|
24
|
-
peerId: PeerId;
|
|
25
|
-
protocol: Protocol;
|
|
26
|
-
protocolID: string;
|
|
27
|
-
rateLimiter: ReqRespRateLimiter;
|
|
28
|
-
signal?: AbortSignal;
|
|
29
|
-
requestId?: number;
|
|
30
|
-
/** Peer client type for logging and metrics: 'prysm' | 'lighthouse' */
|
|
31
|
-
peerClient?: string;
|
|
32
|
-
/** Non-spec timeout from sending request until write stream closed by responder */
|
|
33
|
-
requestTimeoutMs?: number;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
/**
|
|
37
|
-
* Handles a ReqResp request from a peer. Throws on error. Logs each step of the response lifecycle.
|
|
38
|
-
*
|
|
39
|
-
* 1. A duplex `stream` with the peer is already available
|
|
40
|
-
* 2. Read and decode request from peer
|
|
41
|
-
* 3. Delegate to `performRequestHandler()` to perform the request job and expect
|
|
42
|
-
* to yield zero or more `<response_chunks>`
|
|
43
|
-
* 4a. Encode and write `<response_chunks>` to peer
|
|
44
|
-
* 4b. On error, encode and write an error `<response_chunk>` and stop
|
|
45
|
-
*/
|
|
46
|
-
export async function handleRequest({
|
|
47
|
-
logger,
|
|
48
|
-
metrics,
|
|
49
|
-
stream,
|
|
50
|
-
peerId,
|
|
51
|
-
protocol,
|
|
52
|
-
protocolID,
|
|
53
|
-
rateLimiter,
|
|
54
|
-
signal,
|
|
55
|
-
requestId = 0,
|
|
56
|
-
peerClient = "unknown",
|
|
57
|
-
requestTimeoutMs,
|
|
58
|
-
}: HandleRequestOpts): Promise<void> {
|
|
59
|
-
const REQUEST_TIMEOUT = requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT;
|
|
60
|
-
|
|
61
|
-
const logCtx = {
|
|
62
|
-
method: protocol.method,
|
|
63
|
-
version: protocol.version,
|
|
64
|
-
client: peerClient,
|
|
65
|
-
peer: prettyPrintPeerId(peerId),
|
|
66
|
-
requestId,
|
|
67
|
-
};
|
|
68
|
-
metrics?.incomingOpenedStreams.inc({method: protocol.method});
|
|
69
|
-
|
|
70
|
-
let responseError: Error | null = null;
|
|
71
|
-
await pipe(
|
|
72
|
-
// Yields success chunks and error chunks in the same generator
|
|
73
|
-
// This syntax allows to recycle stream.sink to send success and error chunks without returning
|
|
74
|
-
// in case request whose body is a List fails at chunk_i > 0, without breaking out of the for..await..of
|
|
75
|
-
(async function* requestHandlerSource() {
|
|
76
|
-
try {
|
|
77
|
-
// TODO: Does the TTFB timer start on opening stream or after receiving request
|
|
78
|
-
const timerTTFB = metrics?.outgoingResponseTTFB.startTimer({method: protocol.method});
|
|
79
|
-
|
|
80
|
-
const requestBody = await withTimeout(
|
|
81
|
-
() => pipe(stream.source as AsyncIterable<Uint8ArrayList>, requestDecode(protocol)),
|
|
82
|
-
REQUEST_TIMEOUT,
|
|
83
|
-
signal
|
|
84
|
-
).catch((e: unknown) => {
|
|
85
|
-
if (e instanceof TimeoutError) {
|
|
86
|
-
throw e; // Let outter catch (_e) {} re-type the error as SERVER_ERROR
|
|
87
|
-
}
|
|
88
|
-
throw new ResponseError(RespStatus.INVALID_REQUEST, (e as Error).message);
|
|
89
|
-
});
|
|
90
|
-
|
|
91
|
-
logger.debug("Req received", logCtx);
|
|
92
|
-
|
|
93
|
-
// Max count by request for byRange and byRoot
|
|
94
|
-
const requestCount = protocol?.inboundRateLimits?.getRequestCount?.(requestBody) ?? 1;
|
|
95
|
-
|
|
96
|
-
if (!rateLimiter.allows(peerId, protocolID, requestCount)) {
|
|
97
|
-
throw new RequestError({code: RequestErrorCode.REQUEST_RATE_LIMITED});
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
const requestChunk: ReqRespRequest = {
|
|
101
|
-
data: requestBody,
|
|
102
|
-
version: protocol.version,
|
|
103
|
-
};
|
|
104
|
-
|
|
105
|
-
yield* pipe(
|
|
106
|
-
// TODO: Debug the reason for type conversion here
|
|
107
|
-
protocol.handler(requestChunk, peerId),
|
|
108
|
-
// NOTE: Do not log the resp chunk contents, logs get extremely cluttered
|
|
109
|
-
// Note: Not logging on each chunk since after 1 year it hasn't add any value when debugging
|
|
110
|
-
// onChunk(() => logger.debug("Resp sending chunk", logCtx)),
|
|
111
|
-
responseEncodeSuccess(protocol, {
|
|
112
|
-
onChunk(chunkIndex) {
|
|
113
|
-
if (chunkIndex === 0) timerTTFB?.();
|
|
114
|
-
},
|
|
115
|
-
})
|
|
116
|
-
);
|
|
117
|
-
} catch (e) {
|
|
118
|
-
const status = e instanceof ResponseError ? e.status : RespStatus.SERVER_ERROR;
|
|
119
|
-
yield* responseEncodeError(protocol, status, (e as Error).message);
|
|
120
|
-
|
|
121
|
-
// Should not throw an error here or libp2p-mplex throws with 'AbortError: stream reset'
|
|
122
|
-
// throw e;
|
|
123
|
-
responseError = e as Error;
|
|
124
|
-
}
|
|
125
|
-
})(),
|
|
126
|
-
stream.sink
|
|
127
|
-
);
|
|
128
|
-
|
|
129
|
-
// If streak.sink throws, libp2p-mplex will close stream.source
|
|
130
|
-
// If `requestDecode()` throws the stream.source must be closed manually
|
|
131
|
-
// To ensure the stream.source it-pushable instance is always closed, stream.close() is called always
|
|
132
|
-
await stream.close();
|
|
133
|
-
metrics?.incomingClosedStreams.inc({method: protocol.method});
|
|
134
|
-
|
|
135
|
-
// TODO: It may happen that stream.sink returns before returning stream.source first,
|
|
136
|
-
// so you never see "Resp received request" in the logs and the response ends without
|
|
137
|
-
// sending any chunk, triggering EMPTY_RESPONSE error on the requesting side
|
|
138
|
-
// It has only happened when doing a request too fast upon immediate connection on inbound peer
|
|
139
|
-
// investigate a potential race condition there
|
|
140
|
-
|
|
141
|
-
if (responseError !== null) {
|
|
142
|
-
logger.verbose("Resp error", logCtx, responseError);
|
|
143
|
-
throw responseError;
|
|
144
|
-
}
|
|
145
|
-
// NOTE: Only log once per request to verbose, intermediate steps to debug
|
|
146
|
-
logger.verbose("Resp done", logCtx);
|
|
147
|
-
}
|
package/src/types.ts
DELETED
|
@@ -1,158 +0,0 @@
|
|
|
1
|
-
import {PeerId} from "@libp2p/interface";
|
|
2
|
-
import {BeaconConfig, ForkBoundary} from "@lodestar/config";
|
|
3
|
-
import {ForkName} from "@lodestar/params";
|
|
4
|
-
import {LodestarError} from "@lodestar/utils";
|
|
5
|
-
import {RateLimiterQuota} from "./rate_limiter/rateLimiterGRCA.js";
|
|
6
|
-
|
|
7
|
-
export const protocolPrefix = "/eth2/beacon_chain/req";
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* Available request/response encoding strategies:
|
|
11
|
-
* https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#encoding-strategies
|
|
12
|
-
*/
|
|
13
|
-
export enum Encoding {
|
|
14
|
-
SSZ_SNAPPY = "ssz_snappy",
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export const CONTEXT_BYTES_FORK_DIGEST_LENGTH = 4;
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* Wrapper for the request/response payload
|
|
21
|
-
*/
|
|
22
|
-
export type ResponseIncoming = {
|
|
23
|
-
data: Uint8Array;
|
|
24
|
-
fork: ForkName;
|
|
25
|
-
protocolVersion: number;
|
|
26
|
-
};
|
|
27
|
-
|
|
28
|
-
export type ResponseOutgoing = {
|
|
29
|
-
data: Uint8Array;
|
|
30
|
-
/**
|
|
31
|
-
* Reason why outgoing needs fork boundary but incoming only needs fork is because we can deserialize incoming data
|
|
32
|
-
* with only fork info, but for outgoing we also need to compute fork digest, which requires fork boundary.
|
|
33
|
-
*/
|
|
34
|
-
boundary: ForkBoundary;
|
|
35
|
-
};
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Rate limiter options for the requests
|
|
39
|
-
*/
|
|
40
|
-
export interface ReqRespRateLimiterOpts {
|
|
41
|
-
rateLimitMultiplier?: number;
|
|
42
|
-
onRateLimit?: (peer: PeerId, method: string) => void;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* Inbound rate limiter quota for the protocol
|
|
47
|
-
*/
|
|
48
|
-
export interface InboundRateLimitQuota {
|
|
49
|
-
/**
|
|
50
|
-
* Will be tracked for the protocol per peer
|
|
51
|
-
*/
|
|
52
|
-
byPeer?: RateLimiterQuota;
|
|
53
|
-
/**
|
|
54
|
-
* Will be tracked regardless of the peer
|
|
55
|
-
*/
|
|
56
|
-
total?: RateLimiterQuota;
|
|
57
|
-
/**
|
|
58
|
-
* Some requests may be counted multiple e.g. getBlocksByRange
|
|
59
|
-
* for such implement this method else `1` will be used default
|
|
60
|
-
*/
|
|
61
|
-
getRequestCount?: (req: Uint8Array) => number;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
export type ReqRespRequest = {
|
|
65
|
-
data: Uint8Array;
|
|
66
|
-
version: number;
|
|
67
|
-
};
|
|
68
|
-
|
|
69
|
-
/**
|
|
70
|
-
* Request handler
|
|
71
|
-
*/
|
|
72
|
-
export type ProtocolHandler = (req: ReqRespRequest, peerId: PeerId) => AsyncIterable<ResponseOutgoing>;
|
|
73
|
-
|
|
74
|
-
/**
|
|
75
|
-
* ReqResp Protocol Deceleration
|
|
76
|
-
*/
|
|
77
|
-
export interface ProtocolAttributes {
|
|
78
|
-
readonly protocolPrefix: string;
|
|
79
|
-
/** Protocol name identifier `beacon_blocks_by_range` or `status` */
|
|
80
|
-
readonly method: string;
|
|
81
|
-
/** Version counter: `1`, `2` etc */
|
|
82
|
-
readonly version: number;
|
|
83
|
-
readonly encoding: Encoding;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
export interface ProtocolDescriptor extends Omit<ProtocolAttributes, "protocolPrefix"> {
|
|
87
|
-
contextBytes: ContextBytesFactory;
|
|
88
|
-
ignoreResponse?: boolean;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
// `protocolPrefix` is added runtime so not part of definition
|
|
92
|
-
/**
|
|
93
|
-
* ReqResp Protocol definition for full duplex protocols
|
|
94
|
-
*/
|
|
95
|
-
export interface Protocol extends ProtocolDescriptor {
|
|
96
|
-
handler: ProtocolHandler;
|
|
97
|
-
inboundRateLimits?: InboundRateLimitQuota;
|
|
98
|
-
requestSizes: TypeSizes | null;
|
|
99
|
-
responseSizes: (fork: ForkName) => TypeSizes;
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/**
|
|
103
|
-
* ReqResp Protocol definition for dial only protocols
|
|
104
|
-
*/
|
|
105
|
-
export interface DialOnlyProtocol extends Omit<Protocol, "handler" | "inboundRateLimits" | "renderRequestBody"> {
|
|
106
|
-
handler?: never;
|
|
107
|
-
inboundRateLimits?: never;
|
|
108
|
-
renderRequestBody?: never;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
/**
|
|
112
|
-
* ReqResp Protocol definition for full duplex and dial only protocols
|
|
113
|
-
*/
|
|
114
|
-
export type MixedProtocol = DialOnlyProtocol | Protocol;
|
|
115
|
-
|
|
116
|
-
/**
|
|
117
|
-
* ReqResp protocol definition descriptor for full duplex and dial only protocols
|
|
118
|
-
* If handler is not provided, the protocol will be dial only
|
|
119
|
-
* If handler is provided, the protocol will be full duplex
|
|
120
|
-
*/
|
|
121
|
-
export type MixedProtocolGenerators = <H extends ProtocolHandler | undefined = undefined>(
|
|
122
|
-
// "inboundRateLimiter" is available only on handler context not on generator
|
|
123
|
-
modules: {config: BeaconConfig},
|
|
124
|
-
handler?: H
|
|
125
|
-
) => H extends undefined ? DialOnlyProtocol : Protocol;
|
|
126
|
-
|
|
127
|
-
/**
|
|
128
|
-
* ReqResp protocol definition descriptor for full duplex protocols
|
|
129
|
-
*/
|
|
130
|
-
export type ProtocolGenerator = (modules: {config: BeaconConfig}, handler: ProtocolHandler) => Protocol;
|
|
131
|
-
|
|
132
|
-
export type HandlerTypeFromMessage<T> = T extends ProtocolGenerator ? ProtocolHandler : never;
|
|
133
|
-
|
|
134
|
-
export type ContextBytesFactory =
|
|
135
|
-
| {type: ContextBytesType.Empty}
|
|
136
|
-
| {type: ContextBytesType.ForkDigest; config: BeaconConfig};
|
|
137
|
-
|
|
138
|
-
export type ContextBytes = {type: ContextBytesType.Empty} | {type: ContextBytesType.ForkDigest; fork: ForkName};
|
|
139
|
-
|
|
140
|
-
export enum ContextBytesType {
|
|
141
|
-
/** 0 bytes chunk, can be ignored */
|
|
142
|
-
Empty,
|
|
143
|
-
/** A fixed-width 4 byte <context-bytes>, set to the ForkDigest matching the chunk: compute_fork_digest(fork_version, genesis_validators_root) */
|
|
144
|
-
ForkDigest,
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
export enum LightClientServerErrorCode {
|
|
148
|
-
RESOURCE_UNAVAILABLE = "RESOURCE_UNAVAILABLE",
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
export type LightClientServerErrorType = {code: LightClientServerErrorCode.RESOURCE_UNAVAILABLE};
|
|
152
|
-
|
|
153
|
-
export class LightClientServerError extends LodestarError<LightClientServerErrorType> {}
|
|
154
|
-
|
|
155
|
-
export type TypeSizes = {
|
|
156
|
-
maxSize: number;
|
|
157
|
-
minSize: number;
|
|
158
|
-
};
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Wraps an AsyncIterable and rejects early if any signal aborts.
|
|
3
|
-
* Throws the error returned by `getError()` of each signal options.
|
|
4
|
-
*
|
|
5
|
-
* Simplified fork of `"abortable-iterator"`.
|
|
6
|
-
* Read function's source for reasoning of the fork.
|
|
7
|
-
*/
|
|
8
|
-
export function abortableSource<T>(
|
|
9
|
-
sourceArg: AsyncIterable<T>,
|
|
10
|
-
signals: {
|
|
11
|
-
signal: AbortSignal;
|
|
12
|
-
getError: () => Error;
|
|
13
|
-
}[]
|
|
14
|
-
): AsyncIterable<T> {
|
|
15
|
-
const source = sourceArg as AsyncGenerator<T>;
|
|
16
|
-
|
|
17
|
-
async function* abortable(): AsyncIterable<T> {
|
|
18
|
-
// Handler that will hold a reference to the `abort()` promise,
|
|
19
|
-
// necessary for the signal abort listeners to reject the iterable promise
|
|
20
|
-
let nextAbortHandler: ((error: Error) => void) | null = null;
|
|
21
|
-
|
|
22
|
-
// For each signal register an abortHandler(), and prepare clean-up with `onDoneCbs`
|
|
23
|
-
const onDoneCbs: (() => void)[] = [];
|
|
24
|
-
for (const {signal, getError} of signals) {
|
|
25
|
-
const abortHandler = (): void => {
|
|
26
|
-
if (nextAbortHandler) nextAbortHandler(getError());
|
|
27
|
-
};
|
|
28
|
-
signal.addEventListener("abort", abortHandler);
|
|
29
|
-
onDoneCbs.push(() => {
|
|
30
|
-
signal.removeEventListener("abort", abortHandler);
|
|
31
|
-
});
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
try {
|
|
35
|
-
while (true) {
|
|
36
|
-
// Abort early if any signal is aborted
|
|
37
|
-
for (const {signal, getError} of signals) {
|
|
38
|
-
if (signal.aborted) {
|
|
39
|
-
throw getError();
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
// Race the iterator and the abort signals
|
|
44
|
-
const result = await Promise.race([
|
|
45
|
-
new Promise<never>((_, reject) => {
|
|
46
|
-
nextAbortHandler = (error) => reject(error);
|
|
47
|
-
}),
|
|
48
|
-
source.next(),
|
|
49
|
-
]);
|
|
50
|
-
|
|
51
|
-
// source.next() resolved first
|
|
52
|
-
nextAbortHandler = null;
|
|
53
|
-
|
|
54
|
-
if (result.done) {
|
|
55
|
-
return;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
yield result.value;
|
|
59
|
-
}
|
|
60
|
-
} catch (err) {
|
|
61
|
-
// End the iterator if it is a generator
|
|
62
|
-
if (typeof source.return === "function") {
|
|
63
|
-
// This source.return() function may never resolve depending on the source AsyncGenerator implementation.
|
|
64
|
-
// This is the main reason to fork "abortable-iterator", which caused our node to get stuck during Sync.
|
|
65
|
-
// We choose to call .return() but not await it. In general, source.return should never throw. If it does,
|
|
66
|
-
// it a problem of the source implementor, and thus logged as an unhandled rejection. If that happens,
|
|
67
|
-
// the source implementor should fix the upstream code.
|
|
68
|
-
void source.return(null);
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
throw err;
|
|
72
|
-
} finally {
|
|
73
|
-
for (const cb of onDoneCbs) {
|
|
74
|
-
cb();
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
return abortable();
|
|
80
|
-
}
|