@aztec/p2p 0.71.0 → 0.72.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/p2p_client.d.ts.map +1 -1
- package/dest/client/p2p_client.js +6 -6
- package/dest/mem_pools/attestation_pool/mocks.d.ts +2 -1
- package/dest/mem_pools/attestation_pool/mocks.d.ts.map +1 -1
- package/dest/mem_pools/attestation_pool/mocks.js +1 -1
- package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.d.ts.map +1 -1
- package/dest/mem_pools/tx_pool/aztec_kv_tx_pool.js +2 -2
- package/dest/mocks/index.d.ts +3 -3
- package/dest/mocks/index.d.ts.map +1 -1
- package/dest/mocks/index.js +19 -18
- package/dest/services/dummy_service.d.ts +7 -0
- package/dest/services/dummy_service.d.ts.map +1 -1
- package/dest/services/dummy_service.js +10 -1
- package/dest/services/libp2p/libp2p_service.d.ts +17 -13
- package/dest/services/libp2p/libp2p_service.d.ts.map +1 -1
- package/dest/services/libp2p/libp2p_service.js +109 -101
- package/dest/services/peer-manager/metrics.d.ts +12 -0
- package/dest/services/peer-manager/metrics.d.ts.map +1 -0
- package/dest/services/peer-manager/metrics.js +26 -0
- package/dest/services/{peer_manager.d.ts → peer-manager/peer_manager.d.ts} +23 -8
- package/dest/services/peer-manager/peer_manager.d.ts.map +1 -0
- package/dest/services/peer-manager/peer_manager.js +392 -0
- package/dest/services/{peer-scoring → peer-manager}/peer_scoring.d.ts +3 -0
- package/dest/services/peer-manager/peer_scoring.d.ts.map +1 -0
- package/dest/services/peer-manager/peer_scoring.js +84 -0
- package/dest/services/reqresp/connection-sampler/batch_connection_sampler.d.ts +45 -0
- package/dest/services/reqresp/connection-sampler/batch_connection_sampler.d.ts.map +1 -0
- package/dest/services/reqresp/connection-sampler/batch_connection_sampler.js +81 -0
- package/dest/services/reqresp/connection-sampler/connection_sampler.d.ts +61 -0
- package/dest/services/reqresp/connection-sampler/connection_sampler.d.ts.map +1 -0
- package/dest/services/reqresp/connection-sampler/connection_sampler.js +175 -0
- package/dest/services/reqresp/interface.d.ts +17 -4
- package/dest/services/reqresp/interface.d.ts.map +1 -1
- package/dest/services/reqresp/interface.js +34 -11
- package/dest/services/reqresp/metrics.d.ts +15 -0
- package/dest/services/reqresp/metrics.d.ts.map +1 -0
- package/dest/services/reqresp/metrics.js +42 -0
- package/dest/services/reqresp/protocols/block.d.ts +4 -0
- package/dest/services/reqresp/protocols/block.d.ts.map +1 -0
- package/dest/services/reqresp/protocols/block.js +9 -0
- package/dest/services/reqresp/protocols/goodbye.d.ts +51 -0
- package/dest/services/reqresp/protocols/goodbye.d.ts.map +1 -0
- package/dest/services/reqresp/protocols/goodbye.js +92 -0
- package/dest/services/reqresp/protocols/index.d.ts +9 -0
- package/dest/services/reqresp/protocols/index.d.ts.map +1 -0
- package/dest/services/reqresp/protocols/index.js +9 -0
- package/dest/services/reqresp/protocols/ping.d.ts +9 -0
- package/dest/services/reqresp/protocols/ping.d.ts.map +1 -0
- package/dest/services/reqresp/protocols/ping.js +9 -0
- package/dest/services/reqresp/{handlers.d.ts → protocols/status.d.ts} +1 -7
- package/dest/services/reqresp/protocols/status.d.ts.map +1 -0
- package/dest/services/reqresp/protocols/status.js +9 -0
- package/dest/services/reqresp/protocols/tx.d.ts +13 -0
- package/dest/services/reqresp/protocols/tx.d.ts.map +1 -0
- package/dest/services/reqresp/protocols/tx.js +23 -0
- package/dest/services/reqresp/rate-limiter/index.d.ts.map +1 -0
- package/dest/services/reqresp/{rate_limiter → rate-limiter}/index.js +1 -1
- package/dest/services/reqresp/{rate_limiter → rate-limiter}/rate_limiter.d.ts +3 -3
- package/dest/services/reqresp/{rate_limiter → rate-limiter}/rate_limiter.d.ts.map +1 -1
- package/dest/services/reqresp/{rate_limiter → rate-limiter}/rate_limiter.js +4 -4
- package/dest/services/reqresp/rate-limiter/rate_limits.d.ts.map +1 -0
- package/dest/services/reqresp/rate-limiter/rate_limits.js +55 -0
- package/dest/services/reqresp/reqresp.d.ts +33 -6
- package/dest/services/reqresp/reqresp.d.ts.map +1 -1
- package/dest/services/reqresp/reqresp.js +414 -249
- package/dest/services/service.d.ts +8 -0
- package/dest/services/service.d.ts.map +1 -1
- package/dest/util.d.ts +4 -0
- package/dest/util.d.ts.map +1 -1
- package/dest/util.js +1 -1
- package/package.json +8 -8
- package/src/client/p2p_client.ts +5 -5
- package/src/mem_pools/attestation_pool/mocks.ts +2 -2
- package/src/mem_pools/tx_pool/aztec_kv_tx_pool.ts +0 -1
- package/src/mocks/index.ts +19 -20
- package/src/services/dummy_service.ts +13 -0
- package/src/services/libp2p/libp2p_service.ts +143 -128
- package/src/services/peer-manager/metrics.ts +41 -0
- package/src/services/{peer_manager.ts → peer-manager/peer_manager.ts} +67 -22
- package/src/services/{peer-scoring → peer-manager}/peer_scoring.ts +16 -3
- package/src/services/reqresp/connection-sampler/batch_connection_sampler.ts +94 -0
- package/src/services/reqresp/connection-sampler/connection_sampler.ts +211 -0
- package/src/services/reqresp/interface.ts +39 -16
- package/src/services/reqresp/metrics.ts +57 -0
- package/src/services/reqresp/protocols/block.ts +15 -0
- package/src/services/reqresp/protocols/goodbye.ts +101 -0
- package/src/services/reqresp/protocols/index.ts +8 -0
- package/src/services/reqresp/protocols/ping.ts +8 -0
- package/src/services/reqresp/{handlers.ts → protocols/status.ts} +0 -9
- package/src/services/reqresp/protocols/tx.ts +29 -0
- package/src/services/reqresp/{rate_limiter → rate-limiter}/rate_limiter.ts +3 -3
- package/src/services/reqresp/{rate_limiter → rate-limiter}/rate_limits.ts +24 -4
- package/src/services/reqresp/reqresp.ts +224 -25
- package/src/services/service.ts +12 -0
- package/src/util.ts +4 -0
- package/dest/services/peer-scoring/peer_scoring.d.ts.map +0 -1
- package/dest/services/peer-scoring/peer_scoring.js +0 -75
- package/dest/services/peer_manager.d.ts.map +0 -1
- package/dest/services/peer_manager.js +0 -358
- package/dest/services/reqresp/handlers.d.ts.map +0 -1
- package/dest/services/reqresp/handlers.js +0 -17
- package/dest/services/reqresp/rate_limiter/index.d.ts.map +0 -1
- package/dest/services/reqresp/rate_limiter/rate_limits.d.ts.map +0 -1
- package/dest/services/reqresp/rate_limiter/rate_limits.js +0 -35
- /package/dest/services/reqresp/{rate_limiter → rate-limiter}/index.d.ts +0 -0
- /package/dest/services/reqresp/{rate_limiter → rate-limiter}/rate_limits.d.ts +0 -0
- /package/src/services/reqresp/{rate_limiter → rate-limiter}/index.ts +0 -0
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { PeerErrorSeverity } from '@aztec/circuit-types';
|
|
3
3
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
4
4
|
import { executeTimeout } from '@aztec/foundation/timer';
|
|
5
|
+
import { Attributes, type TelemetryClient, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
|
|
5
6
|
|
|
6
7
|
import { type IncomingStreamData, type PeerId, type Stream } from '@libp2p/interface';
|
|
7
8
|
import { pipe } from 'it-pipe';
|
|
@@ -14,8 +15,10 @@ import {
|
|
|
14
15
|
InvalidResponseError,
|
|
15
16
|
} from '../../errors/reqresp.error.js';
|
|
16
17
|
import { SnappyTransform } from '../encoding.js';
|
|
17
|
-
import { type
|
|
18
|
+
import { type PeerScoring } from '../peer-manager/peer_scoring.js';
|
|
18
19
|
import { type P2PReqRespConfig } from './config.js';
|
|
20
|
+
import { BatchConnectionSampler } from './connection-sampler/batch_connection_sampler.js';
|
|
21
|
+
import { ConnectionSampler } from './connection-sampler/connection_sampler.js';
|
|
19
22
|
import {
|
|
20
23
|
DEFAULT_SUB_PROTOCOL_HANDLERS,
|
|
21
24
|
DEFAULT_SUB_PROTOCOL_VALIDATORS,
|
|
@@ -25,7 +28,8 @@ import {
|
|
|
25
28
|
type SubProtocolMap,
|
|
26
29
|
subProtocolMap,
|
|
27
30
|
} from './interface.js';
|
|
28
|
-
import {
|
|
31
|
+
import { ReqRespMetrics } from './metrics.js';
|
|
32
|
+
import { RequestResponseRateLimiter } from './rate-limiter/rate_limiter.js';
|
|
29
33
|
|
|
30
34
|
/**
|
|
31
35
|
* The Request Response Service
|
|
@@ -51,18 +55,35 @@ export class ReqResp {
|
|
|
51
55
|
private subProtocolHandlers: ReqRespSubProtocolHandlers = DEFAULT_SUB_PROTOCOL_HANDLERS;
|
|
52
56
|
private subProtocolValidators: ReqRespSubProtocolValidators = DEFAULT_SUB_PROTOCOL_VALIDATORS;
|
|
53
57
|
|
|
58
|
+
private connectionSampler: ConnectionSampler;
|
|
54
59
|
private rateLimiter: RequestResponseRateLimiter;
|
|
55
60
|
|
|
56
61
|
private snappyTransform: SnappyTransform;
|
|
57
62
|
|
|
58
|
-
|
|
63
|
+
private metrics: ReqRespMetrics;
|
|
64
|
+
|
|
65
|
+
constructor(
|
|
66
|
+
config: P2PReqRespConfig,
|
|
67
|
+
private libp2p: Libp2p,
|
|
68
|
+
private peerScoring: PeerScoring,
|
|
69
|
+
telemetryClient: TelemetryClient = getTelemetryClient(),
|
|
70
|
+
) {
|
|
59
71
|
this.logger = createLogger('p2p:reqresp');
|
|
60
72
|
|
|
61
73
|
this.overallRequestTimeoutMs = config.overallRequestTimeoutMs;
|
|
62
74
|
this.individualRequestTimeoutMs = config.individualRequestTimeoutMs;
|
|
63
75
|
|
|
64
|
-
this.rateLimiter = new RequestResponseRateLimiter(
|
|
76
|
+
this.rateLimiter = new RequestResponseRateLimiter(peerScoring);
|
|
77
|
+
|
|
78
|
+
// Connection sampler is used to sample our connected peers
|
|
79
|
+
this.connectionSampler = new ConnectionSampler(libp2p);
|
|
80
|
+
|
|
65
81
|
this.snappyTransform = new SnappyTransform();
|
|
82
|
+
this.metrics = new ReqRespMetrics(telemetryClient);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
get tracer() {
|
|
86
|
+
return this.metrics.tracer;
|
|
66
87
|
}
|
|
67
88
|
|
|
68
89
|
/**
|
|
@@ -83,12 +104,15 @@ export class ReqResp {
|
|
|
83
104
|
* Stop the reqresp service
|
|
84
105
|
*/
|
|
85
106
|
async stop() {
|
|
86
|
-
// Unregister
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
107
|
+
// Unregister handlers in parallel
|
|
108
|
+
const unregisterPromises = Object.keys(this.subProtocolHandlers).map(protocol => this.libp2p.unhandle(protocol));
|
|
109
|
+
await Promise.all(unregisterPromises);
|
|
110
|
+
|
|
111
|
+
// Close connection sampler
|
|
112
|
+
await this.connectionSampler.stop();
|
|
113
|
+
this.logger.debug('ReqResp: Connection sampler stopped');
|
|
90
114
|
|
|
91
|
-
// Close
|
|
115
|
+
// Close streams in parallel
|
|
92
116
|
const closeStreamPromises = this.libp2p.getConnections().map(connection => connection.close());
|
|
93
117
|
await Promise.all(closeStreamPromises);
|
|
94
118
|
this.logger.debug('ReqResp: All active streams closed');
|
|
@@ -113,8 +137,8 @@ export class ReqResp {
|
|
|
113
137
|
* If no response is received from any peer, it returns undefined.
|
|
114
138
|
*
|
|
115
139
|
* The method performs the following steps:
|
|
116
|
-
* -
|
|
117
|
-
* - Opens a stream with
|
|
140
|
+
* - Sample a peer to send the request to.
|
|
141
|
+
* - Opens a stream with the peer using the specified sub-protocol.
|
|
118
142
|
*
|
|
119
143
|
* When a response is received, it is validated using the given sub protocols response validator.
|
|
120
144
|
* To see the interface for the response validator - see `interface.ts`
|
|
@@ -132,15 +156,32 @@ export class ReqResp {
|
|
|
132
156
|
subProtocol: SubProtocol,
|
|
133
157
|
request: InstanceType<SubProtocolMap[SubProtocol]['request']>,
|
|
134
158
|
): Promise<InstanceType<SubProtocolMap[SubProtocol]['response']> | undefined> {
|
|
159
|
+
const responseValidator = this.subProtocolValidators[subProtocol];
|
|
160
|
+
const requestBuffer = request.toBuffer();
|
|
161
|
+
|
|
135
162
|
const requestFunction = async () => {
|
|
136
|
-
|
|
137
|
-
|
|
163
|
+
// Attempt to ask all of our peers, but sampled in a random order
|
|
164
|
+
// This function is wrapped in a timeout, so we will exit the loop if we have not received a response
|
|
165
|
+
const numberOfPeers = this.libp2p.getPeers().length;
|
|
138
166
|
|
|
139
|
-
|
|
140
|
-
|
|
167
|
+
if (numberOfPeers === 0) {
|
|
168
|
+
this.logger.debug('No active peers to send requests to');
|
|
169
|
+
return undefined;
|
|
170
|
+
}
|
|
141
171
|
|
|
142
|
-
|
|
143
|
-
for (
|
|
172
|
+
const attemptedPeers: Map<string, boolean> = new Map();
|
|
173
|
+
for (let i = 0; i < numberOfPeers; i++) {
|
|
174
|
+
// Sample a peer to make a request to
|
|
175
|
+
const peer = this.connectionSampler.getPeer(attemptedPeers);
|
|
176
|
+
this.logger.trace(`Attempting to send request to peer: ${peer?.toString()}`);
|
|
177
|
+
if (!peer) {
|
|
178
|
+
this.logger.debug('No peers available to send requests to');
|
|
179
|
+
return undefined;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
attemptedPeers.set(peer.toString(), true);
|
|
183
|
+
|
|
184
|
+
this.logger.trace(`Sending request to peer: ${peer.toString()}`);
|
|
144
185
|
const response = await this.sendRequestToPeer(peer, subProtocol, requestBuffer);
|
|
145
186
|
|
|
146
187
|
// If we get a response, return it, otherwise we iterate onto the next peer
|
|
@@ -155,7 +196,6 @@ export class ReqResp {
|
|
|
155
196
|
return object;
|
|
156
197
|
}
|
|
157
198
|
}
|
|
158
|
-
return undefined;
|
|
159
199
|
};
|
|
160
200
|
|
|
161
201
|
try {
|
|
@@ -170,6 +210,152 @@ export class ReqResp {
|
|
|
170
210
|
}
|
|
171
211
|
}
|
|
172
212
|
|
|
213
|
+
/**
|
|
214
|
+
* Request multiple messages over the same sub protocol, balancing the requests across peers.
|
|
215
|
+
*
|
|
216
|
+
* @devnote
|
|
217
|
+
* - The function prioritizes sending requests to free peers using a batch sampling strategy.
|
|
218
|
+
* - If a peer fails to respond or returns an invalid response, it is removed from the sampling pool and replaced.
|
|
219
|
+
* - The function stops retrying once all requests are processed, no active peers remain, or the maximum retry attempts are reached.
|
|
220
|
+
* - Responses are validated using a custom validator for the sub-protocol.*
|
|
221
|
+
*
|
|
222
|
+
* Requests are sent in parallel to each peer, but multiple requests are sent to the same peer in series
|
|
223
|
+
* - If a peer fails to respond or returns an invalid response, it is removed from the sampling pool and replaced.
|
|
224
|
+
* - The function stops retrying once all requests are processed, no active peers remain, or the maximum retry attempts are reached.
|
|
225
|
+
* - Responses are validated using a custom validator for the sub-protocol.*
|
|
226
|
+
*
|
|
227
|
+
* @param subProtocol
|
|
228
|
+
* @param requests
|
|
229
|
+
* @param timeoutMs
|
|
230
|
+
* @param maxPeers
|
|
231
|
+
* @returns
|
|
232
|
+
*
|
|
233
|
+
* @throws {CollectiveReqRespTimeoutError} - If the request batch exceeds the specified timeout (`timeoutMs`).
|
|
234
|
+
*/
|
|
235
|
+
@trackSpan(
|
|
236
|
+
'ReqResp.sendBatchRequest',
|
|
237
|
+
(subProtocol: ReqRespSubProtocol, requests: InstanceType<SubProtocolMap[ReqRespSubProtocol]['request']>[]) => ({
|
|
238
|
+
[Attributes.P2P_REQ_RESP_PROTOCOL]: subProtocol,
|
|
239
|
+
[Attributes.P2P_REQ_RESP_BATCH_REQUESTS_COUNT]: requests.length,
|
|
240
|
+
}),
|
|
241
|
+
)
|
|
242
|
+
async sendBatchRequest<SubProtocol extends ReqRespSubProtocol>(
|
|
243
|
+
subProtocol: SubProtocol,
|
|
244
|
+
requests: InstanceType<SubProtocolMap[SubProtocol]['request']>[],
|
|
245
|
+
timeoutMs = 10000,
|
|
246
|
+
maxPeers = Math.min(10, requests.length),
|
|
247
|
+
maxRetryAttempts = 3,
|
|
248
|
+
): Promise<InstanceType<SubProtocolMap[SubProtocol]['response']>[]> {
|
|
249
|
+
const responseValidator = this.subProtocolValidators[subProtocol];
|
|
250
|
+
const responses: InstanceType<SubProtocolMap[SubProtocol]['response']>[] = new Array(requests.length);
|
|
251
|
+
const requestBuffers = requests.map(req => req.toBuffer());
|
|
252
|
+
|
|
253
|
+
const requestFunction = async () => {
|
|
254
|
+
// Track which requests still need to be processed
|
|
255
|
+
const pendingRequestIndices = new Set(requestBuffers.map((_, i) => i));
|
|
256
|
+
|
|
257
|
+
// Create batch sampler with the total number of requests and max peers
|
|
258
|
+
const batchSampler = new BatchConnectionSampler(this.connectionSampler, requests.length, maxPeers);
|
|
259
|
+
|
|
260
|
+
if (batchSampler.activePeerCount === 0) {
|
|
261
|
+
this.logger.debug('No active peers to send requests to');
|
|
262
|
+
return [];
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// This is where it gets fun
|
|
266
|
+
// The outer loop is the retry loop, we will continue to retry until we process all indices we have
|
|
267
|
+
// not received a response for, or we have reached the max retry attempts
|
|
268
|
+
|
|
269
|
+
// The inner loop is the batch loop, we will process all requests for each peer in parallel
|
|
270
|
+
// We will then process the results of the requests, and resample any peers that failed to respond
|
|
271
|
+
// We will continue to retry until we have processed all indices, or we have reached the max retry attempts
|
|
272
|
+
|
|
273
|
+
let retryAttempts = 0;
|
|
274
|
+
while (pendingRequestIndices.size > 0 && batchSampler.activePeerCount > 0 && retryAttempts < maxRetryAttempts) {
|
|
275
|
+
// Process requests in parallel for each available peer
|
|
276
|
+
const requestBatches = new Map<PeerId, number[]>();
|
|
277
|
+
|
|
278
|
+
// Group requests by peer
|
|
279
|
+
for (const requestIndex of pendingRequestIndices) {
|
|
280
|
+
const peer = batchSampler.getPeerForRequest(requestIndex);
|
|
281
|
+
if (!peer) {
|
|
282
|
+
break;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
if (!requestBatches.has(peer)) {
|
|
286
|
+
requestBatches.set(peer, []);
|
|
287
|
+
}
|
|
288
|
+
requestBatches.get(peer)!.push(requestIndex);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// Make parallel requests for each peer's batch
|
|
292
|
+
// A batch entry will look something like this:
|
|
293
|
+
// PeerId0: [0, 1, 2, 3]
|
|
294
|
+
// PeerId1: [4, 5, 6, 7]
|
|
295
|
+
|
|
296
|
+
// Peer Id 0 will send requests 0, 1, 2, 3 in serial
|
|
297
|
+
// while simultaneously Peer Id 1 will send requests 4, 5, 6, 7 in serial
|
|
298
|
+
|
|
299
|
+
const batchResults = await Promise.all(
|
|
300
|
+
Array.from(requestBatches.entries()).map(async ([peer, indices]) => {
|
|
301
|
+
try {
|
|
302
|
+
// Requests all going to the same peer are sent synchronously
|
|
303
|
+
const peerResults: { index: number; response: InstanceType<SubProtocolMap[SubProtocol]['response']> }[] =
|
|
304
|
+
[];
|
|
305
|
+
for (const index of indices) {
|
|
306
|
+
const response = await this.sendRequestToPeer(peer, subProtocol, requestBuffers[index]);
|
|
307
|
+
|
|
308
|
+
if (response && response.length > 0) {
|
|
309
|
+
const object = subProtocolMap[subProtocol].response.fromBuffer(response);
|
|
310
|
+
const isValid = await responseValidator(requests[index], object, peer);
|
|
311
|
+
|
|
312
|
+
if (isValid) {
|
|
313
|
+
peerResults.push({ index, response: object });
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
return { peer, results: peerResults };
|
|
319
|
+
} catch (error) {
|
|
320
|
+
this.logger.debug(`Failed batch request to peer ${peer.toString()}:`, error);
|
|
321
|
+
batchSampler.removePeerAndReplace(peer);
|
|
322
|
+
return { peer, results: [] };
|
|
323
|
+
}
|
|
324
|
+
}),
|
|
325
|
+
);
|
|
326
|
+
|
|
327
|
+
// Process results
|
|
328
|
+
for (const { results } of batchResults) {
|
|
329
|
+
for (const { index, response } of results) {
|
|
330
|
+
if (response) {
|
|
331
|
+
responses[index] = response;
|
|
332
|
+
pendingRequestIndices.delete(index);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
retryAttempts++;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
if (retryAttempts >= maxRetryAttempts) {
|
|
341
|
+
this.logger.debug(`Max retry attempts ${maxRetryAttempts} reached for batch request`);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
return responses;
|
|
345
|
+
};
|
|
346
|
+
|
|
347
|
+
try {
|
|
348
|
+
return await executeTimeout<InstanceType<SubProtocolMap[SubProtocol]['response']>[]>(
|
|
349
|
+
requestFunction,
|
|
350
|
+
timeoutMs,
|
|
351
|
+
() => new CollectiveReqRespTimeoutError(),
|
|
352
|
+
);
|
|
353
|
+
} catch (e: any) {
|
|
354
|
+
this.logger.debug(`${e.message} | subProtocol: ${subProtocol}`);
|
|
355
|
+
return [];
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
173
359
|
/**
|
|
174
360
|
* Sends a request to a specific peer
|
|
175
361
|
*
|
|
@@ -194,15 +380,20 @@ export class ReqResp {
|
|
|
194
380
|
* If the stream is not closed by the dialled peer, and a timeout occurs, then
|
|
195
381
|
* the stream is closed on the requester's end and sender (us) updates its peer score
|
|
196
382
|
*/
|
|
197
|
-
|
|
383
|
+
@trackSpan('ReqResp.sendRequestToPeer', (peerId: PeerId, subProtocol: ReqRespSubProtocol, _: Buffer) => ({
|
|
384
|
+
[Attributes.P2P_ID]: peerId.toString(),
|
|
385
|
+
[Attributes.P2P_REQ_RESP_PROTOCOL]: subProtocol,
|
|
386
|
+
}))
|
|
387
|
+
public async sendRequestToPeer(
|
|
198
388
|
peerId: PeerId,
|
|
199
389
|
subProtocol: ReqRespSubProtocol,
|
|
200
390
|
payload: Buffer,
|
|
201
391
|
): Promise<Buffer | undefined> {
|
|
202
392
|
let stream: Stream | undefined;
|
|
203
393
|
try {
|
|
204
|
-
|
|
205
|
-
|
|
394
|
+
this.metrics.recordRequestSent(subProtocol);
|
|
395
|
+
|
|
396
|
+
stream = await this.connectionSampler.dialProtocol(peerId, subProtocol);
|
|
206
397
|
|
|
207
398
|
// Open the stream with a timeout
|
|
208
399
|
const result = await executeTimeout<Buffer>(
|
|
@@ -213,12 +404,13 @@ export class ReqResp {
|
|
|
213
404
|
|
|
214
405
|
return result;
|
|
215
406
|
} catch (e: any) {
|
|
407
|
+
this.metrics.recordRequestError(subProtocol);
|
|
216
408
|
this.handleResponseError(e, peerId, subProtocol);
|
|
217
409
|
} finally {
|
|
410
|
+
// Only close the stream if we created it
|
|
218
411
|
if (stream) {
|
|
219
412
|
try {
|
|
220
|
-
await
|
|
221
|
-
this.logger.trace(`Stream closed with ${peerId.toString()} for ${subProtocol}`);
|
|
413
|
+
await this.connectionSampler.close(stream.id);
|
|
222
414
|
} catch (closeError) {
|
|
223
415
|
this.logger.error(
|
|
224
416
|
`Error closing stream: ${closeError instanceof Error ? closeError.message : 'Unknown error'}`,
|
|
@@ -241,7 +433,7 @@ export class ReqResp {
|
|
|
241
433
|
private handleResponseError(e: any, peerId: PeerId, subProtocol: ReqRespSubProtocol): void {
|
|
242
434
|
const severity = this.categorizeError(e, peerId, subProtocol);
|
|
243
435
|
if (severity) {
|
|
244
|
-
this.
|
|
436
|
+
this.peerScoring.penalizePeer(peerId, severity);
|
|
245
437
|
}
|
|
246
438
|
}
|
|
247
439
|
|
|
@@ -320,7 +512,13 @@ export class ReqResp {
|
|
|
320
512
|
* We check rate limits for each peer, note the peer will be penalised within the rate limiter implementation
|
|
321
513
|
* if they exceed their peer specific limits.
|
|
322
514
|
*/
|
|
515
|
+
@trackSpan('ReqResp.streamHandler', (protocol: ReqRespSubProtocol, { connection }: IncomingStreamData) => ({
|
|
516
|
+
[Attributes.P2P_REQ_RESP_PROTOCOL]: protocol,
|
|
517
|
+
[Attributes.P2P_ID]: connection.remotePeer.toString(),
|
|
518
|
+
}))
|
|
323
519
|
private async streamHandler(protocol: ReqRespSubProtocol, { stream, connection }: IncomingStreamData) {
|
|
520
|
+
this.metrics.recordRequestReceived(protocol);
|
|
521
|
+
|
|
324
522
|
// Store a reference to from this for the async generator
|
|
325
523
|
if (!this.rateLimiter.allow(protocol, connection.remotePeer)) {
|
|
326
524
|
this.logger.warn(`Rate limit exceeded for ${protocol} from ${connection.remotePeer}`);
|
|
@@ -339,7 +537,7 @@ export class ReqResp {
|
|
|
339
537
|
async function* (source: any) {
|
|
340
538
|
for await (const chunkList of source) {
|
|
341
539
|
const msg = Buffer.from(chunkList.subarray());
|
|
342
|
-
const response = await handler(msg);
|
|
540
|
+
const response = await handler(connection.remotePeer, msg);
|
|
343
541
|
yield new Uint8Array(transform.outboundTransformNoTopic(response));
|
|
344
542
|
}
|
|
345
543
|
},
|
|
@@ -347,6 +545,7 @@ export class ReqResp {
|
|
|
347
545
|
);
|
|
348
546
|
} catch (e: any) {
|
|
349
547
|
this.logger.warn(e);
|
|
548
|
+
this.metrics.recordResponseError(protocol);
|
|
350
549
|
} finally {
|
|
351
550
|
await stream.close();
|
|
352
551
|
}
|
package/src/services/service.ts
CHANGED
|
@@ -45,6 +45,18 @@ export interface P2PService {
|
|
|
45
45
|
request: InstanceType<SubProtocolMap[Protocol]['request']>,
|
|
46
46
|
): Promise<InstanceType<SubProtocolMap[Protocol]['response']> | undefined>;
|
|
47
47
|
|
|
48
|
+
/**
|
|
49
|
+
* Send a batch of requests to peers, and return the responses
|
|
50
|
+
*
|
|
51
|
+
* @param protocol - The request response protocol to use
|
|
52
|
+
* @param requests - The requests to send to the peers
|
|
53
|
+
* @returns The responses to the requests
|
|
54
|
+
*/
|
|
55
|
+
sendBatchRequest<Protocol extends ReqRespSubProtocol>(
|
|
56
|
+
protocol: Protocol,
|
|
57
|
+
requests: InstanceType<SubProtocolMap[Protocol]['request']>[],
|
|
58
|
+
): Promise<InstanceType<SubProtocolMap[Protocol]['response']>[] | undefined>;
|
|
59
|
+
|
|
48
60
|
// Leaky abstraction: fix https://github.com/AztecProtocol/aztec-packages/issues/7963
|
|
49
61
|
registerBlockReceivedCallback(callback: (block: BlockProposal) => Promise<BlockAttestation | undefined>): void;
|
|
50
62
|
|
package/src/util.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { type DataStoreConfig } from '@aztec/kv-store/config';
|
|
|
4
4
|
import type { GossipSub } from '@chainsafe/libp2p-gossipsub';
|
|
5
5
|
import { generateKeyPair, marshalPrivateKey, unmarshalPrivateKey } from '@libp2p/crypto/keys';
|
|
6
6
|
import { type PeerId, type PrivateKey } from '@libp2p/interface';
|
|
7
|
+
import { type ConnectionManager } from '@libp2p/interface-internal';
|
|
7
8
|
import { createFromPrivKey } from '@libp2p/peer-id-factory';
|
|
8
9
|
import { resolve } from 'dns/promises';
|
|
9
10
|
import type { Libp2p } from 'libp2p';
|
|
@@ -13,6 +14,9 @@ import { type P2PConfig } from './config.js';
|
|
|
13
14
|
export interface PubSubLibp2p extends Libp2p {
|
|
14
15
|
services: {
|
|
15
16
|
pubsub: GossipSub;
|
|
17
|
+
components: {
|
|
18
|
+
connectionManager: ConnectionManager;
|
|
19
|
+
};
|
|
16
20
|
};
|
|
17
21
|
}
|
|
18
22
|
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"peer_scoring.d.ts","sourceRoot":"","sources":["../../../src/services/peer-scoring/peer_scoring.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAQjD,oBAAY,cAAc;IACxB,MAAM,IAAA;IACN,UAAU,IAAA;IACV,OAAO,IAAA;CACR;AAMD,qBAAa,WAAW;IACtB,OAAO,CAAC,MAAM,CAAkC;IAChD,OAAO,CAAC,cAAc,CAAkC;IACxD,OAAO,CAAC,aAAa,CAAa;IAClC,OAAO,CAAC,WAAW,CAAO;IAC1B,aAAa,EAAE;SAAG,GAAG,IAAI,iBAAiB,GAAG,MAAM;KAAE,CAAC;gBAE1C,MAAM,EAAE,SAAS;IAY7B,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM;IAmBvD,cAAc,IAAI,IAAI;IActB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAIhC,aAAa,CAAC,MAAM,EAAE,MAAM;IAW5B,QAAQ,IAAI;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE;CAGpC"}
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
import { PeerErrorSeverity } from '@aztec/circuit-types';
|
|
2
|
-
import { median } from '@aztec/foundation/collection';
|
|
3
|
-
const DefaultPeerPenalties = {
|
|
4
|
-
[PeerErrorSeverity.LowToleranceError]: 50,
|
|
5
|
-
[PeerErrorSeverity.MidToleranceError]: 10,
|
|
6
|
-
[PeerErrorSeverity.HighToleranceError]: 2,
|
|
7
|
-
};
|
|
8
|
-
export var PeerScoreState;
|
|
9
|
-
(function (PeerScoreState) {
|
|
10
|
-
PeerScoreState[PeerScoreState["Banned"] = 0] = "Banned";
|
|
11
|
-
PeerScoreState[PeerScoreState["Disconnect"] = 1] = "Disconnect";
|
|
12
|
-
PeerScoreState[PeerScoreState["Healthy"] = 2] = "Healthy";
|
|
13
|
-
})(PeerScoreState || (PeerScoreState = {}));
|
|
14
|
-
// TODO: move into config / constants
|
|
15
|
-
const MIN_SCORE_BEFORE_BAN = -100;
|
|
16
|
-
const MIN_SCORE_BEFORE_DISCONNECT = -50;
|
|
17
|
-
export class PeerScoring {
|
|
18
|
-
constructor(config) {
|
|
19
|
-
this.scores = new Map();
|
|
20
|
-
this.lastUpdateTime = new Map();
|
|
21
|
-
this.decayInterval = 1000 * 60; // 1 minute
|
|
22
|
-
this.decayFactor = 0.9;
|
|
23
|
-
const orderedValues = config.peerPenaltyValues?.sort((a, b) => a - b);
|
|
24
|
-
this.peerPenalties = {
|
|
25
|
-
[PeerErrorSeverity.HighToleranceError]: orderedValues?.[0] ?? DefaultPeerPenalties[PeerErrorSeverity.HighToleranceError],
|
|
26
|
-
[PeerErrorSeverity.MidToleranceError]: orderedValues?.[1] ?? DefaultPeerPenalties[PeerErrorSeverity.MidToleranceError],
|
|
27
|
-
[PeerErrorSeverity.LowToleranceError]: orderedValues?.[2] ?? DefaultPeerPenalties[PeerErrorSeverity.LowToleranceError],
|
|
28
|
-
};
|
|
29
|
-
}
|
|
30
|
-
updateScore(peerId, scoreDelta) {
|
|
31
|
-
const currentTime = Date.now();
|
|
32
|
-
const lastUpdate = this.lastUpdateTime.get(peerId) || currentTime;
|
|
33
|
-
const timePassed = currentTime - lastUpdate;
|
|
34
|
-
const decayPeriods = Math.floor(timePassed / this.decayInterval);
|
|
35
|
-
let currentScore = this.scores.get(peerId) || 0;
|
|
36
|
-
// Apply decay
|
|
37
|
-
currentScore *= Math.pow(this.decayFactor, decayPeriods);
|
|
38
|
-
// Apply new score delta
|
|
39
|
-
currentScore += scoreDelta;
|
|
40
|
-
this.scores.set(peerId, currentScore);
|
|
41
|
-
this.lastUpdateTime.set(peerId, currentTime);
|
|
42
|
-
return currentScore;
|
|
43
|
-
}
|
|
44
|
-
decayAllScores() {
|
|
45
|
-
const currentTime = Date.now();
|
|
46
|
-
for (const [peerId, lastUpdate] of this.lastUpdateTime.entries()) {
|
|
47
|
-
const timePassed = currentTime - lastUpdate;
|
|
48
|
-
const decayPeriods = Math.floor(timePassed / this.decayInterval);
|
|
49
|
-
if (decayPeriods > 0) {
|
|
50
|
-
let score = this.scores.get(peerId) || 0;
|
|
51
|
-
score *= Math.pow(this.decayFactor, decayPeriods);
|
|
52
|
-
this.scores.set(peerId, score);
|
|
53
|
-
this.lastUpdateTime.set(peerId, currentTime);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
getScore(peerId) {
|
|
58
|
-
return this.scores.get(peerId) || 0;
|
|
59
|
-
}
|
|
60
|
-
getScoreState(peerId) {
|
|
61
|
-
// TODO: permanently store banned peers???
|
|
62
|
-
const score = this.getScore(peerId);
|
|
63
|
-
if (score < MIN_SCORE_BEFORE_BAN) {
|
|
64
|
-
return PeerScoreState.Banned;
|
|
65
|
-
}
|
|
66
|
-
else if (score < MIN_SCORE_BEFORE_DISCONNECT) {
|
|
67
|
-
return PeerScoreState.Disconnect;
|
|
68
|
-
}
|
|
69
|
-
return PeerScoreState.Healthy;
|
|
70
|
-
}
|
|
71
|
-
getStats() {
|
|
72
|
-
return { medianScore: median(Array.from(this.scores.values())) ?? 0 };
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGVlcl9zY29yaW5nLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL3NlcnZpY2VzL3BlZXItc2NvcmluZy9wZWVyX3Njb3JpbmcudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLGlCQUFpQixFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDekQsT0FBTyxFQUFFLE1BQU0sRUFBRSxNQUFNLDhCQUE4QixDQUFDO0FBSXRELE1BQU0sb0JBQW9CLEdBQUc7SUFDM0IsQ0FBQyxpQkFBaUIsQ0FBQyxpQkFBaUIsQ0FBQyxFQUFFLEVBQUU7SUFDekMsQ0FBQyxpQkFBaUIsQ0FBQyxpQkFBaUIsQ0FBQyxFQUFFLEVBQUU7SUFDekMsQ0FBQyxpQkFBaUIsQ0FBQyxrQkFBa0IsQ0FBQyxFQUFFLENBQUM7Q0FDMUMsQ0FBQztBQUVGLE1BQU0sQ0FBTixJQUFZLGNBSVg7QUFKRCxXQUFZLGNBQWM7SUFDeEIsdURBQU0sQ0FBQTtJQUNOLCtEQUFVLENBQUE7SUFDVix5REFBTyxDQUFBO0FBQ1QsQ0FBQyxFQUpXLGNBQWMsS0FBZCxjQUFjLFFBSXpCO0FBRUQscUNBQXFDO0FBQ3JDLE1BQU0sb0JBQW9CLEdBQUcsQ0FBQyxHQUFHLENBQUM7QUFDbEMsTUFBTSwyQkFBMkIsR0FBRyxDQUFDLEVBQUUsQ0FBQztBQUV4QyxNQUFNLE9BQU8sV0FBVztJQU90QixZQUFZLE1BQWlCO1FBTnJCLFdBQU0sR0FBd0IsSUFBSSxHQUFHLEVBQUUsQ0FBQztRQUN4QyxtQkFBYyxHQUF3QixJQUFJLEdBQUcsRUFBRSxDQUFDO1FBQ2hELGtCQUFhLEdBQUcsSUFBSSxHQUFHLEVBQUUsQ0FBQyxDQUFDLFdBQVc7UUFDdEMsZ0JBQVcsR0FBRyxHQUFHLENBQUM7UUFJeEIsTUFBTSxhQUFhLEdBQUcsTUFBTSxDQUFDLGlCQUFpQixFQUFFLElBQUksQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQztRQUN0RSxJQUFJLENBQUMsYUFBYSxHQUFHO1lBQ25CLENBQUMsaUJBQWlCLENBQUMsa0JBQWtCLENBQUMsRUFDcEMsYUFBYSxFQUFFLENBQUMsQ0FBQyxDQUFDLElBQUksb0JBQW9CLENBQUMsaUJBQWlCLENBQUMsa0JBQWtCLENBQUM7WUFDbEYsQ0FBQyxpQkFBaUIsQ0FBQyxpQkFBaUIsQ0FBQyxFQUNuQyxhQUFhLEVBQUUsQ0FBQyxDQUFDLENBQUMsSUFBSSxvQkFBb0IsQ0FBQyxpQkFBaUIsQ0FBQyxpQkFBaUIsQ0FBQztZQUNqRixDQUFDLGlCQUFpQixDQUFDLGlCQUFpQixDQUFDLEVBQ25DLGFBQWEsRUFBRSxDQUFDLENBQUMsQ0FBQyxJQUFJLG9CQUFvQixDQUFDLGlCQUFpQixDQUFDLGlCQUFpQixDQUFDO1NBQ2xGLENBQUM7SUFDSixDQUFDO0lBRUQsV0FBVyxDQUFDLE1BQWMsRUFBRSxVQUFrQjtRQUM1QyxNQUFNLFdBQVcsR0FBRyxJQUFJLENBQUMsR0FBRyxFQUFFLENBQUM7UUFDL0IsTUFBTSxVQUFVLEdBQUcsSUFBSSxDQUFDLGNBQWMsQ0FBQyxHQUFHLENBQUMsTUFBTSxDQUFDLElBQUksV0FBVyxDQUFDO1FBQ2xFLE1BQU0sVUFBVSxHQUFHLFdBQVcsR0FBRyxVQUFVLENBQUM7UUFDNUMsTUFBTSxZQUFZLEdBQUcsSUFBSSxDQUFDLEtBQUssQ0FBQyxVQUFVLEdBQUcsSUFBSSxDQUFDLGFBQWEsQ0FBQyxDQUFDO1FBRWpFLElBQUksWUFBWSxHQUFHLElBQUksQ0FBQyxNQUFNLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUVoRCxjQUFjO1FBQ2QsWUFBWSxJQUFJLElBQUksQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLFdBQVcsRUFBRSxZQUFZLENBQUMsQ0FBQztRQUV6RCx3QkFBd0I7UUFDeEIsWUFBWSxJQUFJLFVBQVUsQ0FBQztRQUUzQixJQUFJLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxNQUFNLEVBQUUsWUFBWSxDQUFDLENBQUM7UUFDdEMsSUFBSSxDQUFDLGNBQWMsQ0FBQyxHQUFHLENBQUMsTUFBTSxFQUFFLFdBQVcsQ0FBQyxDQUFDO1FBQzdDLE9BQU8sWUFBWSxDQUFDO0lBQ3RCLENBQUM7SUFFRCxjQUFjO1FBQ1osTUFBTSxXQUFXLEdBQUcsSUFBSSxDQUFDLEdBQUcsRUFBRSxDQUFDO1FBQy9CLEtBQUssTUFBTSxDQUFDLE1BQU0sRUFBRSxVQUFVLENBQUMsSUFBSSxJQUFJLENBQUMsY0FBYyxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUM7WUFDakUsTUFBTSxVQUFVLEdBQUcsV0FBVyxHQUFHLFVBQVUsQ0FBQztZQUM1QyxNQUFNLFlBQVksR0FBRyxJQUFJLENBQUMsS0FBSyxDQUFDLFVBQVUsR0FBRyxJQUFJLENBQUMsYUFBYSxDQUFDLENBQUM7WUFDakUsSUFBSSxZQUFZLEdBQUcsQ0FBQyxFQUFFLENBQUM7Z0JBQ3JCLElBQUksS0FBSyxHQUFHLElBQUksQ0FBQyxNQUFNLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsQ0FBQztnQkFDekMsS0FBSyxJQUFJLElBQUksQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLFdBQVcsRUFBRSxZQUFZLENBQUMsQ0FBQztnQkFDbEQsSUFBSSxDQUFDLE1BQU0sQ0FBQyxHQUFHLENBQUMsTUFBTSxFQUFFLEtBQUssQ0FBQyxDQUFDO2dCQUMvQixJQUFJLENBQUMsY0FBYyxDQUFDLEdBQUcsQ0FBQyxNQUFNLEVBQUUsV0FBVyxDQUFDLENBQUM7WUFDL0MsQ0FBQztRQUNILENBQUM7SUFDSCxDQUFDO0lBRUQsUUFBUSxDQUFDLE1BQWM7UUFDckIsT0FBTyxJQUFJLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDdEMsQ0FBQztJQUVELGFBQWEsQ0FBQyxNQUFjO1FBQzFCLDBDQUEwQztRQUMxQyxNQUFNLEtBQUssR0FBRyxJQUFJLENBQUMsUUFBUSxDQUFDLE1BQU0sQ0FBQyxDQUFDO1FBQ3BDLElBQUksS0FBSyxHQUFHLG9CQUFvQixFQUFFLENBQUM7WUFDakMsT0FBTyxjQUFjLENBQUMsTUFBTSxDQUFDO1FBQy9CLENBQUM7YUFBTSxJQUFJLEtBQUssR0FBRywyQkFBMkIsRUFBRSxDQUFDO1lBQy9DLE9BQU8sY0FBYyxDQUFDLFVBQVUsQ0FBQztRQUNuQyxDQUFDO1FBQ0QsT0FBTyxjQUFjLENBQUMsT0FBTyxDQUFDO0lBQ2hDLENBQUM7SUFFRCxRQUFRO1FBQ04sT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsTUFBTSxDQUFDLE1BQU0sRUFBRSxDQUFDLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQztJQUN4RSxDQUFDO0NBQ0YifQ==
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"peer_manager.d.ts","sourceRoot":"","sources":["../../src/services/peer_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,iBAAiB,EAAE,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAE7E,OAAO,EAAE,KAAK,eAAe,EAAE,UAAU,EAAa,MAAM,yBAAyB,CAAC;AAGtF,OAAO,EAAmB,KAAK,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAIjE,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,OAAO,EAAE,KAAK,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAqBzD,qBAAa,WAAY,SAAQ,UAAU;IAQvC,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,oBAAoB;IAC5B,OAAO,CAAC,MAAM;IAEd,OAAO,CAAC,MAAM;IAXhB,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,gBAAgB,CAAa;IACrC,OAAO,CAAC,8BAA8B,CAAa;IACnD,OAAO,CAAC,aAAa,CAAwC;gBAGnD,UAAU,EAAE,YAAY,EACxB,oBAAoB,EAAE,oBAAoB,EAC1C,MAAM,EAAE,SAAS,EACzB,eAAe,EAAE,eAAe,EACxB,MAAM,yCAAmC;IAkB5C,SAAS;IAShB;;;;;;OAMG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IAShC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAS5B,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB;IAOvD,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAIpC,QAAQ,CAAC,cAAc,UAAQ,GAAG,QAAQ,EAAE;IAiCnD;;OAEG;IACH,OAAO,CAAC,QAAQ;IA6DhB,OAAO,CAAC,mBAAmB;YAoBb,cAAc;IAK5B;;;OAGG;YACW,oBAAoB;YA8DpB,QAAQ;IA2BtB,OAAO,CAAC,cAAc;IAWtB,OAAO,CAAC,gBAAgB;IAiBxB;;;OAGG;IACI,IAAI;CAKZ"}
|