@lodestar/beacon-node 1.50.0 → 1.51.0-dev.7d2ef8139c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
- package/lib/api/impl/beacon/blocks/index.js +7 -2
- package/lib/api/impl/beacon/blocks/index.js.map +1 -1
- package/lib/api/impl/validator/index.d.ts.map +1 -1
- package/lib/api/impl/validator/index.js +31 -17
- package/lib/api/impl/validator/index.js.map +1 -1
- package/lib/chain/ColumnReconstructionTracker.d.ts +0 -2
- package/lib/chain/ColumnReconstructionTracker.d.ts.map +1 -1
- package/lib/chain/ColumnReconstructionTracker.js +4 -5
- package/lib/chain/ColumnReconstructionTracker.js.map +1 -1
- package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map +1 -1
- package/lib/chain/archiveStore/utils/archiveBlocks.js +4 -5
- package/lib/chain/archiveStore/utils/archiveBlocks.js.map +1 -1
- package/lib/chain/blocks/blockInput/blockInput.d.ts +1 -1
- package/lib/chain/blocks/blockInput/blockInput.d.ts.map +1 -1
- package/lib/chain/blocks/blockInput/blockInput.js.map +1 -1
- package/lib/chain/blocks/blockInput/utils.js +2 -2
- package/lib/chain/blocks/blockInput/utils.js.map +1 -1
- package/lib/chain/blocks/importBlock.d.ts.map +1 -1
- package/lib/chain/blocks/importBlock.js +7 -4
- package/lib/chain/blocks/importBlock.js.map +1 -1
- package/lib/chain/chain.d.ts.map +1 -1
- package/lib/chain/chain.js +3 -2
- package/lib/chain/chain.js.map +1 -1
- package/lib/chain/interface.d.ts +1 -1
- package/lib/chain/interface.d.ts.map +1 -1
- package/lib/chain/prepareNextSlot.d.ts.map +1 -1
- package/lib/chain/prepareNextSlot.js +4 -3
- package/lib/chain/prepareNextSlot.js.map +1 -1
- package/lib/chain/produceBlock/computeNewStateRoot.d.ts.map +1 -1
- package/lib/chain/produceBlock/computeNewStateRoot.js.map +1 -1
- package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
- package/lib/chain/produceBlock/produceBlockBody.js +8 -6
- package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
- package/lib/chain/reprocess.d.ts +2 -1
- package/lib/chain/reprocess.d.ts.map +1 -1
- package/lib/chain/reprocess.js +2 -1
- package/lib/chain/reprocess.js.map +1 -1
- package/lib/chain/seenCache/seenGossipBlockInput.d.ts.map +1 -1
- package/lib/chain/seenCache/seenGossipBlockInput.js +1 -1
- package/lib/chain/seenCache/seenGossipBlockInput.js.map +1 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.d.ts.map +1 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.js +1 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.js.map +1 -1
- package/lib/chain/validatorMonitor.d.ts.map +1 -1
- package/lib/chain/validatorMonitor.js +6 -4
- package/lib/chain/validatorMonitor.js.map +1 -1
- package/lib/constants/constants.d.ts +1 -0
- package/lib/constants/constants.d.ts.map +1 -1
- package/lib/constants/constants.js +1 -0
- package/lib/constants/constants.js.map +1 -1
- package/lib/db/repositories/blockArchive.js +2 -2
- package/lib/db/repositories/blockArchive.js.map +1 -1
- package/lib/db/repositories/blockArchiveIndex.d.ts +1 -1
- package/lib/db/repositories/blockArchiveIndex.d.ts.map +1 -1
- package/lib/db/repositories/blockArchiveIndex.js +1 -2
- package/lib/db/repositories/blockArchiveIndex.js.map +1 -1
- package/lib/execution/engine/index.d.ts.map +1 -1
- package/lib/execution/engine/index.js +1 -1
- package/lib/execution/engine/index.js.map +1 -1
- package/lib/execution/engine/jsonRpcHttpClient.d.ts.map +1 -1
- package/lib/execution/engine/jsonRpcHttpClient.js +14 -4
- package/lib/execution/engine/jsonRpcHttpClient.js.map +1 -1
- package/lib/execution/engine/restTransport.d.ts +3 -1
- package/lib/execution/engine/restTransport.d.ts.map +1 -1
- package/lib/execution/engine/restTransport.js +33 -21
- package/lib/execution/engine/restTransport.js.map +1 -1
- package/lib/execution/engine/transportMetrics.d.ts +5 -1
- package/lib/execution/engine/transportMetrics.d.ts.map +1 -1
- package/lib/execution/engine/types.d.ts.map +1 -1
- package/lib/execution/engine/types.js +12 -11
- package/lib/execution/engine/types.js.map +1 -1
- package/lib/metrics/metrics/lodestar.d.ts +4 -0
- package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
- package/lib/metrics/metrics/lodestar.js +7 -1
- package/lib/metrics/metrics/lodestar.js.map +1 -1
- package/lib/network/gossip/gossipsub.d.ts.map +1 -1
- package/lib/network/gossip/gossipsub.js +3 -1
- package/lib/network/gossip/gossipsub.js.map +1 -1
- package/lib/network/gossip/topic.d.ts +0 -14
- package/lib/network/gossip/topic.d.ts.map +1 -1
- package/lib/network/processor/index.js +3 -3
- package/lib/network/processor/index.js.map +1 -1
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.d.ts.map +1 -1
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.js +4 -4
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.js.map +1 -1
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js +3 -3
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js.map +1 -1
- package/lib/network/reqresp/rateLimit.js +2 -2
- package/lib/network/reqresp/rateLimit.js.map +1 -1
- package/lib/network/subnets/attnetsService.d.ts.map +1 -1
- package/lib/network/subnets/attnetsService.js +3 -1
- package/lib/network/subnets/attnetsService.js.map +1 -1
- package/lib/node/notifier.d.ts.map +1 -1
- package/lib/node/notifier.js +12 -13
- package/lib/node/notifier.js.map +1 -1
- package/lib/sync/unknownBlock.d.ts.map +1 -1
- package/lib/sync/unknownBlock.js +3 -2
- package/lib/sync/unknownBlock.js.map +1 -1
- package/lib/util/clock.d.ts.map +1 -1
- package/lib/util/clock.js +3 -4
- package/lib/util/clock.js.map +1 -1
- package/lib/util/sszBytes.d.ts +2 -2
- package/lib/util/sszBytes.d.ts.map +1 -1
- package/lib/util/sszBytes.js +22 -14
- package/lib/util/sszBytes.js.map +1 -1
- package/package.json +15 -15
- package/src/api/impl/beacon/blocks/index.ts +10 -3
- package/src/api/impl/validator/index.ts +41 -19
- package/src/chain/ColumnReconstructionTracker.ts +4 -6
- package/src/chain/archiveStore/utils/archiveBlocks.ts +6 -7
- package/src/chain/blocks/blockInput/blockInput.ts +1 -1
- package/src/chain/blocks/blockInput/utils.ts +2 -2
- package/src/chain/blocks/importBlock.ts +8 -4
- package/src/chain/chain.ts +3 -2
- package/src/chain/interface.ts +2 -1
- package/src/chain/prepareNextSlot.ts +7 -3
- package/src/chain/produceBlock/computeNewStateRoot.ts +9 -2
- package/src/chain/produceBlock/produceBlockBody.ts +10 -5
- package/src/chain/reprocess.ts +2 -1
- package/src/chain/seenCache/seenGossipBlockInput.ts +2 -1
- package/src/chain/stateCache/persistentCheckpointsCache.ts +4 -1
- package/src/chain/validatorMonitor.ts +7 -4
- package/src/constants/constants.ts +1 -0
- package/src/db/repositories/blockArchive.ts +2 -2
- package/src/db/repositories/blockArchiveIndex.ts +2 -3
- package/src/execution/engine/index.ts +2 -1
- package/src/execution/engine/jsonRpcHttpClient.ts +12 -5
- package/src/execution/engine/restTransport.ts +53 -22
- package/src/execution/engine/transportMetrics.ts +2 -1
- package/src/execution/engine/types.ts +13 -11
- package/src/metrics/metrics/lodestar.ts +7 -1
- package/src/network/gossip/gossipsub.ts +3 -1
- package/src/network/processor/index.ts +3 -3
- package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts +4 -10
- package/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts +3 -3
- package/src/network/reqresp/rateLimit.ts +2 -2
- package/src/network/subnets/attnetsService.ts +8 -3
- package/src/node/notifier.ts +14 -13
- package/src/sync/unknownBlock.ts +3 -2
- package/src/util/clock.ts +3 -4
- package/src/util/sszBytes.ts +25 -12
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {Db, encodeKey} from "@lodestar/db";
|
|
2
2
|
import {ForkAll} from "@lodestar/params";
|
|
3
|
-
import {Root, SSZTypesFor, SignedBeaconBlock, Slot
|
|
3
|
+
import {Root, SSZTypesFor, SignedBeaconBlock, Slot} from "@lodestar/types";
|
|
4
4
|
import {Bucket, getBucketNameByValue} from "../buckets.js";
|
|
5
5
|
|
|
6
6
|
export const rootIndexBucketId = getBucketNameByValue(Bucket.index_blockArchiveRootIndex);
|
|
@@ -16,10 +16,9 @@ export async function getParentRootIndex(db: Db, parentRoot: Root): Promise<Uint
|
|
|
16
16
|
|
|
17
17
|
export async function deleteRootIndex(
|
|
18
18
|
db: Db,
|
|
19
|
-
|
|
19
|
+
beaconBlockType: SSZTypesFor<ForkAll, "BeaconBlock">,
|
|
20
20
|
block: SignedBeaconBlock
|
|
21
21
|
): Promise<void> {
|
|
22
|
-
const beaconBlockType = (signedBeaconBlockType as typeof ssz.phase0.SignedBeaconBlock).fields.message;
|
|
23
22
|
return db.delete(getRootIndexKey(beaconBlockType.hashTreeRoot(block.message)), {bucketId: rootIndexBucketId});
|
|
24
23
|
}
|
|
25
24
|
|
|
@@ -70,7 +70,8 @@ export function getExecutionEngineHttp(
|
|
|
70
70
|
jwtId: opts.jwtId,
|
|
71
71
|
clientVersion: formatClientVersionHeader(getLodestarClientVersion(opts)),
|
|
72
72
|
metrics,
|
|
73
|
-
})
|
|
73
|
+
}),
|
|
74
|
+
metrics
|
|
74
75
|
);
|
|
75
76
|
|
|
76
77
|
modules.logger.info("Execution client", {urls: opts.urls.map(toPrintableUrl).toString(), engineApi});
|
|
@@ -284,7 +284,12 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient {
|
|
|
284
284
|
});
|
|
285
285
|
|
|
286
286
|
const streamTimer = this.metrics?.streamTime.startTimer({routeId});
|
|
287
|
-
|
|
287
|
+
let bodyText: string;
|
|
288
|
+
try {
|
|
289
|
+
bodyText = await res.text();
|
|
290
|
+
} finally {
|
|
291
|
+
streamTimer?.();
|
|
292
|
+
}
|
|
288
293
|
this.metrics?.responseBytes.inc({routeId}, bodyText.length);
|
|
289
294
|
if (!res.ok) {
|
|
290
295
|
// Infura errors:
|
|
@@ -292,10 +297,12 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient {
|
|
|
292
297
|
throw new HttpRpcError(res.status, `${res.statusText}: ${bodyText.slice(0, maxStringLengthToPrint)}`);
|
|
293
298
|
}
|
|
294
299
|
|
|
295
|
-
const
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
300
|
+
const parseTimer = this.metrics?.responseParseTime.startTimer({routeId, encoding: "json"});
|
|
301
|
+
try {
|
|
302
|
+
return parseJson<R>(bodyText);
|
|
303
|
+
} finally {
|
|
304
|
+
parseTimer?.();
|
|
305
|
+
}
|
|
299
306
|
} catch (e) {
|
|
300
307
|
this.metrics?.requestErrors.inc({routeId});
|
|
301
308
|
if (controller.signal.aborted) {
|
|
@@ -3,7 +3,7 @@ import {CELLS_PER_EXT_BLOB, ForkName, ForkSeq, isForkPostBellatrix} from "@lodes
|
|
|
3
3
|
import {ExecutionPayload, ExecutionRequests, Root, RootHex, capella, deneb, electra, gloas} from "@lodestar/types";
|
|
4
4
|
import {BlobAndProof} from "@lodestar/types/deneb";
|
|
5
5
|
import {BlobAndProofV2} from "@lodestar/types/fulu";
|
|
6
|
-
import {LodestarError, fromHex, strip0xPrefix, toHex, toRootHex} from "@lodestar/utils";
|
|
6
|
+
import {EndTimer, LodestarError, fromHex, strip0xPrefix, toHex, toRootHex} from "@lodestar/utils";
|
|
7
7
|
import {isValidBlobVersionedHashes} from "./blobVersionedHashes.js";
|
|
8
8
|
import {
|
|
9
9
|
ClientCode,
|
|
@@ -48,6 +48,7 @@ import {
|
|
|
48
48
|
executionForkName,
|
|
49
49
|
} from "./sszTypes.js";
|
|
50
50
|
import {ForkchoiceUpdatedResult, GetPayloadResult, IEngineTransport, PayloadStatusResult} from "./transport.js";
|
|
51
|
+
import {EngineTransportMetrics} from "./transportMetrics.js";
|
|
51
52
|
import {
|
|
52
53
|
ExecutionPayloadBodyV2,
|
|
53
54
|
assertReqSizeLimit,
|
|
@@ -102,7 +103,10 @@ export function formatClientVersionHeader(clientVersion: ClientVersion): string
|
|
|
102
103
|
export class RestEngineTransport implements IEngineTransport {
|
|
103
104
|
private limits = DEFAULT_LIMITS;
|
|
104
105
|
|
|
105
|
-
constructor(
|
|
106
|
+
constructor(
|
|
107
|
+
private readonly client: EngineRestHttpClient,
|
|
108
|
+
private readonly metrics?: EngineTransportMetrics | null
|
|
109
|
+
) {}
|
|
106
110
|
|
|
107
111
|
get emitter(): JsonRpcHttpClientEventEmitter {
|
|
108
112
|
return this.client.emitter;
|
|
@@ -114,7 +118,7 @@ export class RestEngineTransport implements IEngineTransport {
|
|
|
114
118
|
{method: "GET", path: "/capabilities", responseType: "json"},
|
|
115
119
|
getCapabilitiesOpts
|
|
116
120
|
);
|
|
117
|
-
const capabilities = parseJson(body, "getCapabilities");
|
|
121
|
+
const capabilities = parseJson(body, "getCapabilities", this.metrics);
|
|
118
122
|
if (
|
|
119
123
|
!isRecord(capabilities) ||
|
|
120
124
|
!Array.isArray(capabilities.supported_forks) ||
|
|
@@ -220,7 +224,7 @@ export class RestEngineTransport implements IEngineTransport {
|
|
|
220
224
|
{method: "POST", path: "/payloads", executionFork: toExecutionForkName(fork), body, responseType: "ssz"},
|
|
221
225
|
notifyNewPayloadOpts
|
|
222
226
|
);
|
|
223
|
-
return toPayloadStatusResult(deserializeResponse(PayloadStatus, res.body, "notifyNewPayload"));
|
|
227
|
+
return toPayloadStatusResult(deserializeResponse(PayloadStatus, res.body, "notifyNewPayload", this.metrics));
|
|
224
228
|
}
|
|
225
229
|
|
|
226
230
|
async forkchoiceUpdated(
|
|
@@ -268,7 +272,12 @@ export class RestEngineTransport implements IEngineTransport {
|
|
|
268
272
|
{method: "POST", path: "/forkchoice", executionFork: toExecutionForkName(fork), body, responseType: "ssz"},
|
|
269
273
|
fcUReqOpts
|
|
270
274
|
);
|
|
271
|
-
const {payloadStatus, payloadId} = deserializeResponse(
|
|
275
|
+
const {payloadStatus, payloadId} = deserializeResponse(
|
|
276
|
+
ForkchoiceUpdateResponse,
|
|
277
|
+
res.body,
|
|
278
|
+
"forkchoiceUpdated",
|
|
279
|
+
this.metrics
|
|
280
|
+
);
|
|
272
281
|
return {
|
|
273
282
|
payloadStatus: toPayloadStatusResult(payloadStatus),
|
|
274
283
|
payloadId: payloadId.length > 0 ? toHex(payloadId[0]) : null,
|
|
@@ -289,7 +298,8 @@ export class RestEngineTransport implements IEngineTransport {
|
|
|
289
298
|
const {payload, blockValue, blobsBundle, executionRequests, shouldOverrideBuilder} = deserializeResponse(
|
|
290
299
|
BuiltPayloadGloas,
|
|
291
300
|
res.body,
|
|
292
|
-
"getPayload"
|
|
301
|
+
"getPayload",
|
|
302
|
+
this.metrics
|
|
293
303
|
);
|
|
294
304
|
return {
|
|
295
305
|
executionPayload: payload,
|
|
@@ -303,7 +313,8 @@ export class RestEngineTransport implements IEngineTransport {
|
|
|
303
313
|
const {payload, blockValue, blobsBundle, executionRequests, shouldOverrideBuilder} = deserializeResponse(
|
|
304
314
|
BuiltPayloadFulu,
|
|
305
315
|
res.body,
|
|
306
|
-
"getPayload"
|
|
316
|
+
"getPayload",
|
|
317
|
+
this.metrics
|
|
307
318
|
);
|
|
308
319
|
return {
|
|
309
320
|
executionPayload: payload,
|
|
@@ -317,7 +328,8 @@ export class RestEngineTransport implements IEngineTransport {
|
|
|
317
328
|
const {payload, blockValue, blobsBundle, executionRequests, shouldOverrideBuilder} = deserializeResponse(
|
|
318
329
|
BuiltPayloadElectra,
|
|
319
330
|
res.body,
|
|
320
|
-
"getPayload"
|
|
331
|
+
"getPayload",
|
|
332
|
+
this.metrics
|
|
321
333
|
);
|
|
322
334
|
return {
|
|
323
335
|
executionPayload: payload,
|
|
@@ -331,15 +343,16 @@ export class RestEngineTransport implements IEngineTransport {
|
|
|
331
343
|
const {payload, blockValue, blobsBundle, shouldOverrideBuilder} = deserializeResponse(
|
|
332
344
|
BuiltPayloadDeneb,
|
|
333
345
|
res.body,
|
|
334
|
-
"getPayload"
|
|
346
|
+
"getPayload",
|
|
347
|
+
this.metrics
|
|
335
348
|
);
|
|
336
349
|
return {executionPayload: payload, executionPayloadValue: blockValue, blobsBundle, shouldOverrideBuilder};
|
|
337
350
|
}
|
|
338
351
|
if (ForkSeq[fork] >= ForkSeq.capella) {
|
|
339
|
-
const {payload, blockValue} = deserializeResponse(BuiltPayloadCapella, res.body, "getPayload");
|
|
352
|
+
const {payload, blockValue} = deserializeResponse(BuiltPayloadCapella, res.body, "getPayload", this.metrics);
|
|
340
353
|
return {executionPayload: payload, executionPayloadValue: blockValue, shouldOverrideBuilder: false};
|
|
341
354
|
}
|
|
342
|
-
const {payload, blockValue} = deserializeResponse(BuiltPayloadBellatrix, res.body, "getPayload");
|
|
355
|
+
const {payload, blockValue} = deserializeResponse(BuiltPayloadBellatrix, res.body, "getPayload", this.metrics);
|
|
343
356
|
return {executionPayload: payload, executionPayloadValue: blockValue, shouldOverrideBuilder: false};
|
|
344
357
|
}
|
|
345
358
|
|
|
@@ -356,7 +369,7 @@ export class RestEngineTransport implements IEngineTransport {
|
|
|
356
369
|
},
|
|
357
370
|
getPayloadBodiesByHashOpts
|
|
358
371
|
);
|
|
359
|
-
const bodies = deserializeBodiesResponse(ForkName.gloas, res.body);
|
|
372
|
+
const bodies = deserializeBodiesResponse(ForkName.gloas, res.body, this.metrics);
|
|
360
373
|
if (bodies.length !== blockHashes.length) {
|
|
361
374
|
throw Error(`Invalid bodies response length=${bodies.length} blockHashes=${blockHashes.length}`);
|
|
362
375
|
}
|
|
@@ -374,7 +387,7 @@ export class RestEngineTransport implements IEngineTransport {
|
|
|
374
387
|
return versionedHashes.map(() => null);
|
|
375
388
|
}
|
|
376
389
|
|
|
377
|
-
const {entries} = deserializeResponse(BlobsV1Response, res.body, "getBlobsV1");
|
|
390
|
+
const {entries} = deserializeResponse(BlobsV1Response, res.body, "getBlobsV1", this.metrics);
|
|
378
391
|
if (entries.length !== versionedHashes.length) {
|
|
379
392
|
throw Error(`Invalid blobs/v1 response length=${entries.length} versionedHashes=${versionedHashes.length}`);
|
|
380
393
|
}
|
|
@@ -397,7 +410,7 @@ export class RestEngineTransport implements IEngineTransport {
|
|
|
397
410
|
return null;
|
|
398
411
|
}
|
|
399
412
|
|
|
400
|
-
const {entries} = deserializeResponse(BlobsV2Response, res.body, "getBlobsV2", {reuseBytes: true});
|
|
413
|
+
const {entries} = deserializeResponse(BlobsV2Response, res.body, "getBlobsV2", this.metrics, {reuseBytes: true});
|
|
401
414
|
if (entries.length !== versionedHashes.length) {
|
|
402
415
|
throw Error(`Invalid blobs/v2 response length=${entries.length} versionedHashes=${versionedHashes.length}`);
|
|
403
416
|
}
|
|
@@ -418,7 +431,7 @@ export class RestEngineTransport implements IEngineTransport {
|
|
|
418
431
|
{method: "GET", path: "/identity", responseType: "json"},
|
|
419
432
|
getClientVersionOpts
|
|
420
433
|
);
|
|
421
|
-
const versions = parseJson(body, "getClientVersion");
|
|
434
|
+
const versions = parseJson(body, "getClientVersion", this.metrics);
|
|
422
435
|
if (!Array.isArray(versions)) {
|
|
423
436
|
throw new EngineRestResponseError("getClientVersion", "Expected an array of client versions");
|
|
424
437
|
}
|
|
@@ -502,36 +515,54 @@ function toPayloadStatusResult(payloadStatus: PayloadStatusSsz): PayloadStatusRe
|
|
|
502
515
|
}
|
|
503
516
|
}
|
|
504
517
|
|
|
505
|
-
function deserializeBodiesResponse(
|
|
518
|
+
function deserializeBodiesResponse(
|
|
519
|
+
fork: ForkName,
|
|
520
|
+
data: Uint8Array,
|
|
521
|
+
metrics?: EngineTransportMetrics | null
|
|
522
|
+
): (ExecutionPayloadBodyV2 | null)[] {
|
|
506
523
|
if (ForkSeq[fork] >= ForkSeq.gloas) {
|
|
507
|
-
return deserializeResponse(BodiesResponseGloas, data, "getPayloadBodiesByHash").entries.map((entry) =>
|
|
524
|
+
return deserializeResponse(BodiesResponseGloas, data, "getPayloadBodiesByHash", metrics).entries.map((entry) =>
|
|
508
525
|
entry.available ? entry.body : null
|
|
509
526
|
);
|
|
510
527
|
}
|
|
511
528
|
if (ForkSeq[fork] >= ForkSeq.capella) {
|
|
512
|
-
return deserializeResponse(BodiesResponseCapella, data, "getPayloadBodiesByHash").entries.map((entry) =>
|
|
529
|
+
return deserializeResponse(BodiesResponseCapella, data, "getPayloadBodiesByHash", metrics).entries.map((entry) =>
|
|
513
530
|
entry.available ? {...entry.body, blockAccessList: null} : null
|
|
514
531
|
);
|
|
515
532
|
}
|
|
516
|
-
return deserializeResponse(BodiesResponseBellatrix, data, "getPayloadBodiesByHash").entries.map((entry) =>
|
|
533
|
+
return deserializeResponse(BodiesResponseBellatrix, data, "getPayloadBodiesByHash", metrics).entries.map((entry) =>
|
|
517
534
|
entry.available ? {transactions: entry.body.transactions, withdrawals: null, blockAccessList: null} : null
|
|
518
535
|
);
|
|
519
536
|
}
|
|
520
537
|
|
|
521
538
|
/** A 2xx response that does not decode is an execution client error, not an unreachable client */
|
|
522
|
-
function deserializeResponse<T>(
|
|
539
|
+
function deserializeResponse<T>(
|
|
540
|
+
type: Type<T>,
|
|
541
|
+
body: Uint8Array,
|
|
542
|
+
routeId: string,
|
|
543
|
+
metrics?: EngineTransportMetrics | null,
|
|
544
|
+
opts?: {reuseBytes?: boolean}
|
|
545
|
+
): T {
|
|
546
|
+
const timer = metrics?.responseParseTime.startTimer({routeId, encoding: "ssz"});
|
|
523
547
|
try {
|
|
524
548
|
return type.deserialize(body, opts);
|
|
525
549
|
} catch (e) {
|
|
526
550
|
throw new EngineRestResponseError(routeId, `Invalid SSZ: ${(e as Error).message}`);
|
|
551
|
+
} finally {
|
|
552
|
+
timer?.();
|
|
527
553
|
}
|
|
528
554
|
}
|
|
529
555
|
|
|
530
|
-
function parseJson(body: Uint8Array, routeId: string): unknown {
|
|
556
|
+
function parseJson(body: Uint8Array, routeId: string, metrics?: EngineTransportMetrics | null): unknown {
|
|
557
|
+
let timer: EndTimer | undefined;
|
|
531
558
|
try {
|
|
532
|
-
|
|
559
|
+
const text = textDecoder.decode(body);
|
|
560
|
+
timer = metrics?.responseParseTime.startTimer({routeId, encoding: "json"});
|
|
561
|
+
return JSON.parse(text);
|
|
533
562
|
} catch (e) {
|
|
534
563
|
throw new EngineRestResponseError(routeId, `Invalid JSON: ${(e as Error).message}`);
|
|
564
|
+
} finally {
|
|
565
|
+
timer?.();
|
|
535
566
|
}
|
|
536
567
|
}
|
|
537
568
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {Counter} from "@lodestar/utils";
|
|
1
|
+
import {Counter, Histogram} from "@lodestar/utils";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Common metrics type for both json rpc and ssz transport.
|
|
@@ -6,4 +6,5 @@ import {Counter} from "@lodestar/utils";
|
|
|
6
6
|
export type EngineTransportMetrics = {
|
|
7
7
|
requestBytes: Counter<{routeId: string}>;
|
|
8
8
|
responseBytes: Counter<{routeId: string}>;
|
|
9
|
+
responseParseTime: Histogram<{routeId: string; encoding: "json" | "ssz"}>;
|
|
9
10
|
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import {Type} from "@chainsafe/ssz";
|
|
1
2
|
import {
|
|
2
3
|
BUILDER_DEPOSIT_REQUEST_TYPE,
|
|
3
4
|
BUILDER_EXIT_REQUEST_TYPE,
|
|
@@ -500,14 +501,15 @@ export function deserializeWithdrawal(serialized: WithdrawalRpc): capella.Withdr
|
|
|
500
501
|
}
|
|
501
502
|
|
|
502
503
|
/**
|
|
503
|
-
*
|
|
504
|
+
* Serialize requests prefixed with a single-byte requestType in one allocation
|
|
504
505
|
*/
|
|
505
|
-
function
|
|
506
|
-
const
|
|
507
|
-
|
|
508
|
-
|
|
506
|
+
function serializePrefixedRequests<T>(type: Type<T>, requests: T, requestType: ExecutionRequestType): Uint8Array {
|
|
507
|
+
const uint8Array = new Uint8Array(1 + type.value_serializedSize(requests));
|
|
508
|
+
const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength);
|
|
509
|
+
uint8Array[0] = requestType;
|
|
510
|
+
type.value_serializeToBytes({uint8Array, dataView}, 1, requests);
|
|
509
511
|
|
|
510
|
-
return
|
|
512
|
+
return uint8Array;
|
|
511
513
|
}
|
|
512
514
|
|
|
513
515
|
/**
|
|
@@ -522,16 +524,16 @@ export function serializeExecutionRequestsToBytes(fork: ForkName, executionReque
|
|
|
522
524
|
const result: Uint8Array[] = [];
|
|
523
525
|
|
|
524
526
|
if (deposits.length !== 0) {
|
|
525
|
-
result.push(
|
|
527
|
+
result.push(serializePrefixedRequests(ssz.electra.DepositRequests, deposits, DEPOSIT_REQUEST_TYPE));
|
|
526
528
|
}
|
|
527
529
|
|
|
528
530
|
if (withdrawals.length !== 0) {
|
|
529
|
-
result.push(
|
|
531
|
+
result.push(serializePrefixedRequests(ssz.electra.WithdrawalRequests, withdrawals, WITHDRAWAL_REQUEST_TYPE));
|
|
530
532
|
}
|
|
531
533
|
|
|
532
534
|
if (consolidations.length !== 0) {
|
|
533
535
|
result.push(
|
|
534
|
-
|
|
536
|
+
serializePrefixedRequests(ssz.electra.ConsolidationRequests, consolidations, CONSOLIDATION_REQUEST_TYPE)
|
|
535
537
|
);
|
|
536
538
|
}
|
|
537
539
|
|
|
@@ -540,12 +542,12 @@ export function serializeExecutionRequestsToBytes(fork: ForkName, executionReque
|
|
|
540
542
|
|
|
541
543
|
if (builderDeposits.length !== 0) {
|
|
542
544
|
result.push(
|
|
543
|
-
|
|
545
|
+
serializePrefixedRequests(ssz.gloas.BuilderDepositRequests, builderDeposits, BUILDER_DEPOSIT_REQUEST_TYPE)
|
|
544
546
|
);
|
|
545
547
|
}
|
|
546
548
|
|
|
547
549
|
if (builderExits.length !== 0) {
|
|
548
|
-
result.push(
|
|
550
|
+
result.push(serializePrefixedRequests(ssz.gloas.BuilderExitRequests, builderExits, BUILDER_EXIT_REQUEST_TYPE));
|
|
549
551
|
}
|
|
550
552
|
}
|
|
551
553
|
|
|
@@ -2061,11 +2061,17 @@ export function createLodestarMetrics(
|
|
|
2061
2061
|
}),
|
|
2062
2062
|
streamTime: register.histogram<{routeId: string}>({
|
|
2063
2063
|
name: "lodestar_execution_engine_http_client_stream_time_seconds",
|
|
2064
|
-
help: "ExecutionEngineHttp client -
|
|
2064
|
+
help: "ExecutionEngineHttp client - response body consumption time, including failures and excluding parsing",
|
|
2065
2065
|
labelNames: ["routeId"],
|
|
2066
2066
|
// Provide max resolution on problematic values around 1 second
|
|
2067
2067
|
buckets: [0.1, 0.5, 1, 2, 5, 15],
|
|
2068
2068
|
}),
|
|
2069
|
+
responseParseTime: register.histogram<{routeId: string; encoding: "json" | "ssz"}>({
|
|
2070
|
+
name: "lodestar_execution_engine_http_client_response_parse_time_seconds",
|
|
2071
|
+
help: "ExecutionEngineHttp client - JSON.parse or SSZ deserialize time, including failures, excluding body reads and subsequent value conversion",
|
|
2072
|
+
labelNames: ["routeId", "encoding"],
|
|
2073
|
+
buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1],
|
|
2074
|
+
}),
|
|
2069
2075
|
requestErrors: register.gauge<{routeId: string}>({
|
|
2070
2076
|
name: "lodestar_execution_engine_http_client_request_errors_total",
|
|
2071
2077
|
help: "ExecutionEngineHttp client - total count of request errors",
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
SLOTS_PER_EPOCH,
|
|
22
22
|
SYNC_COMMITTEE_SUBNET_COUNT,
|
|
23
23
|
} from "@lodestar/params";
|
|
24
|
+
import {getSlotDurations} from "@lodestar/state-transition";
|
|
24
25
|
import {SubnetID} from "@lodestar/types";
|
|
25
26
|
import {Logger, Map2d, Map2dArr} from "@lodestar/utils";
|
|
26
27
|
import {RegistryMetricCreator} from "../../metrics/index.js";
|
|
@@ -172,7 +173,8 @@ export class Eth2Gossipsub {
|
|
|
172
173
|
mcacheLength: 6,
|
|
173
174
|
mcacheGossip: 3,
|
|
174
175
|
// this should be in ms
|
|
175
|
-
|
|
176
|
+
// use the largest configured slot duration so the TTL stays conservative across duration changes
|
|
177
|
+
seenTTL: Math.max(...getSlotDurations(config).map(([, slotDurationMs]) => slotDurationMs)) * SLOTS_PER_EPOCH * 2,
|
|
176
178
|
scoreParams,
|
|
177
179
|
scoreThresholds: gossipScoreThresholds,
|
|
178
180
|
// For a single stream, await processing each RPC before processing the next
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {TopicValidatorResult} from "@libp2p/gossipsub";
|
|
2
2
|
import {routes} from "@lodestar/api";
|
|
3
|
-
import {ForkSeq, NUMBER_OF_COLUMNS} from "@lodestar/params";
|
|
3
|
+
import {ForkSeq, NUMBER_OF_COLUMNS, isForkPostGloas} from "@lodestar/params";
|
|
4
4
|
import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
|
|
5
5
|
import {RootHex, Slot, SlotRootHex} from "@lodestar/types";
|
|
6
6
|
import {Logger, MapDef, mapValues, sleep} from "@lodestar/utils";
|
|
@@ -528,9 +528,9 @@ export class NetworkProcessor {
|
|
|
528
528
|
const parentRoot = getParentRootFromSignedBeaconBlockSerialized(message.msg.data);
|
|
529
529
|
if (parentRoot) {
|
|
530
530
|
let search: SearchUnknownRootTarget | null = null;
|
|
531
|
-
if (
|
|
531
|
+
if (isForkPostGloas(fork)) {
|
|
532
532
|
// GLOAS: also check parent envelope, same logic as execution_payload_bid
|
|
533
|
-
const parentBlockHash = getParentBlockHashFromGloasSignedBeaconBlockSerialized(message.msg.data);
|
|
533
|
+
const parentBlockHash = getParentBlockHashFromGloasSignedBeaconBlockSerialized(message.msg.data, fork);
|
|
534
534
|
if (parentBlockHash && !this.chain.forkChoice.getBlockHexAndBlockHash(parentRoot, parentBlockHash)) {
|
|
535
535
|
const protoBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentRoot);
|
|
536
536
|
if (protoBlock === null) {
|
|
@@ -3,7 +3,7 @@ import {ChainConfig} from "@lodestar/config";
|
|
|
3
3
|
import {PayloadStatus} from "@lodestar/fork-choice";
|
|
4
4
|
import {ForkSeq, GENESIS_SLOT, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
|
|
5
5
|
import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
|
|
6
|
-
import {computeEpochAtSlot} from "@lodestar/state-transition";
|
|
6
|
+
import {computeEpochAtSlot, getBlobDataRetentionStartEpoch} from "@lodestar/state-transition";
|
|
7
7
|
import {ColumnIndex, Epoch, RootHex, fulu} from "@lodestar/types";
|
|
8
8
|
import {fromHex, toRootHex} from "@lodestar/utils";
|
|
9
9
|
import {IBeaconChain} from "../../../chain/index.js";
|
|
@@ -216,16 +216,10 @@ export function validateDataColumnSidecarsByRangeRequest(
|
|
|
216
216
|
throw new ResponseError(RespStatus.INVALID_REQUEST, "startSlot < genesis");
|
|
217
217
|
}
|
|
218
218
|
|
|
219
|
-
// Spec: [max(current_epoch
|
|
220
|
-
const minimumRequestEpoch = Math.max(
|
|
221
|
-
currentEpoch - config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS,
|
|
222
|
-
config.FULU_FORK_EPOCH
|
|
223
|
-
);
|
|
219
|
+
// Spec: [max(compute_blob_data_retention_start_epoch(current_epoch), FULU_FORK_EPOCH), current_epoch]
|
|
220
|
+
const minimumRequestEpoch = Math.max(getBlobDataRetentionStartEpoch(config, currentEpoch), config.FULU_FORK_EPOCH);
|
|
224
221
|
if (computeEpochAtSlot(startSlot) < minimumRequestEpoch) {
|
|
225
|
-
throw new ResponseError(
|
|
226
|
-
RespStatus.RESOURCE_UNAVAILABLE,
|
|
227
|
-
"startSlot is before MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS"
|
|
228
|
-
);
|
|
222
|
+
throw new ResponseError(RespStatus.RESOURCE_UNAVAILABLE, "startSlot is before the blob data retention window");
|
|
229
223
|
}
|
|
230
224
|
|
|
231
225
|
if (count > config.MAX_REQUEST_BLOCKS_DENEB) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {PeerId} from "@libp2p/interface";
|
|
2
2
|
import {ResponseOutgoing} from "@lodestar/reqresp";
|
|
3
|
-
import {computeEpochAtSlot} from "@lodestar/state-transition";
|
|
3
|
+
import {computeEpochAtSlot, getBlobDataRetentionStartEpoch} from "@lodestar/state-transition";
|
|
4
4
|
import {ColumnIndex} from "@lodestar/types";
|
|
5
5
|
import {toRootHex} from "@lodestar/utils";
|
|
6
6
|
import {IBeaconChain} from "../../../chain/index.js";
|
|
@@ -19,10 +19,10 @@ export async function* onDataColumnSidecarsByRoot(
|
|
|
19
19
|
peerId: PeerId,
|
|
20
20
|
peerClient: string
|
|
21
21
|
): AsyncIterable<ResponseOutgoing> {
|
|
22
|
-
// SPEC: minimum_request_epoch = max(current_epoch
|
|
22
|
+
// SPEC: minimum_request_epoch = max(compute_blob_data_retention_start_epoch(current_epoch), FULU_FORK_EPOCH)
|
|
23
23
|
const currentEpoch = chain.clock.currentEpoch;
|
|
24
24
|
const minimumRequestEpoch = Math.max(
|
|
25
|
-
|
|
25
|
+
getBlobDataRetentionStartEpoch(chain.config, currentEpoch),
|
|
26
26
|
chain.config.FULU_FORK_EPOCH
|
|
27
27
|
);
|
|
28
28
|
|
|
@@ -107,12 +107,12 @@ export const rateLimitQuotas: (fork: ForkName, config: BeaconConfig) => Record<R
|
|
|
107
107
|
[ReqRespMethod.LightClientFinalityUpdate]: {
|
|
108
108
|
// Finality updates should not be requested more than once per epoch.
|
|
109
109
|
// Allow 2 per slot and a very safe bound until there's more testing of real usage.
|
|
110
|
-
byPeer: {quota: 2, quotaTimeMs:
|
|
110
|
+
byPeer: {quota: 2, quotaTimeMs: config.getSlotDurationMs(fork)},
|
|
111
111
|
},
|
|
112
112
|
[ReqRespMethod.LightClientOptimisticUpdate]: {
|
|
113
113
|
// Optimistic updates should not be requested more than once per slot.
|
|
114
114
|
// Allow 2 per slot and a very safe bound until there's more testing of real usage.
|
|
115
|
-
byPeer: {quota: 2, quotaTimeMs:
|
|
115
|
+
byPeer: {quota: 2, quotaTimeMs: config.getSlotDurationMs(fork)},
|
|
116
116
|
},
|
|
117
117
|
});
|
|
118
118
|
|
|
@@ -3,6 +3,7 @@ import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH} from "@lodestar/params";
|
|
|
3
3
|
import {computeEpochAtSlot} from "@lodestar/state-transition";
|
|
4
4
|
import {Epoch, Slot, SubnetID, ssz} from "@lodestar/types";
|
|
5
5
|
import {Logger, MapDef} from "@lodestar/utils";
|
|
6
|
+
import {HALF_SLOT_BPS} from "../../constants/index.js";
|
|
6
7
|
import {ClockEvent, IClock} from "../../util/clock.js";
|
|
7
8
|
import {NetworkCoreMetrics} from "../core/metrics.js";
|
|
8
9
|
import {getActiveForkBoundaries} from "../forks.js";
|
|
@@ -163,9 +164,13 @@ export class AttnetsService implements IAttnetsService {
|
|
|
163
164
|
*/
|
|
164
165
|
private onSlot = (clockSlot: Slot): void => {
|
|
165
166
|
try {
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
167
|
+
const fork = this.config.getForkName(clockSlot);
|
|
168
|
+
setTimeout(
|
|
169
|
+
() => {
|
|
170
|
+
this.onHalfSlot(clockSlot);
|
|
171
|
+
},
|
|
172
|
+
this.config.getSlotComponentDurationMs(fork, HALF_SLOT_BPS)
|
|
173
|
+
);
|
|
169
174
|
|
|
170
175
|
for (const [dutiedSlot, dutiedInfo] of this.aggregatorSlotSubnet.entries()) {
|
|
171
176
|
if (dutiedSlot === clockSlot + this.opts.slotsToSubscribeBeforeAggregatorDuty) {
|
package/src/node/notifier.ts
CHANGED
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
IBeaconStateView,
|
|
6
6
|
computeEpochAtSlot,
|
|
7
7
|
computeStartSlotAtEpoch,
|
|
8
|
+
computeTimeAtSlot,
|
|
8
9
|
isStatePostBellatrix,
|
|
9
10
|
} from "@lodestar/state-transition";
|
|
10
11
|
import {Epoch} from "@lodestar/types";
|
|
@@ -140,21 +141,21 @@ export async function runNodeNotifier(modules: NodeNotifierModules): Promise<voi
|
|
|
140
141
|
}
|
|
141
142
|
|
|
142
143
|
function timeToNextHalfSlot(config: BeaconConfig, chain: IBeaconChain, isFirstTime: boolean): number {
|
|
143
|
-
const
|
|
144
|
-
const
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
msFromGenesis < 0
|
|
148
|
-
? // For future genesis time, calculate time left in the slot
|
|
149
|
-
-msFromGenesis % msPerSlot
|
|
150
|
-
: // For past genesis time, calculate time until the next slot
|
|
151
|
-
msPerSlot - (msFromGenesis % msPerSlot);
|
|
144
|
+
const currentSlot = chain.clock.currentSlot;
|
|
145
|
+
const nextSlotTimeSec = computeTimeAtSlot(config, currentSlot + 1, chain.genesisTime);
|
|
146
|
+
const nowMs = Date.now();
|
|
147
|
+
|
|
152
148
|
if (isFirstTime) {
|
|
153
|
-
|
|
154
|
-
|
|
149
|
+
const currentSlotTimeSec = computeTimeAtSlot(config, currentSlot, chain.genesisTime);
|
|
150
|
+
const currentMidpointMs = ((currentSlotTimeSec + nextSlotTimeSec) / 2) * 1000;
|
|
151
|
+
if (currentMidpointMs > nowMs) {
|
|
152
|
+
return currentMidpointMs - nowMs;
|
|
153
|
+
}
|
|
155
154
|
}
|
|
156
|
-
|
|
157
|
-
|
|
155
|
+
|
|
156
|
+
const nextSlotEndTimeSec = computeTimeAtSlot(config, currentSlot + 2, chain.genesisTime);
|
|
157
|
+
const nextMidpointMs = ((nextSlotTimeSec + nextSlotEndTimeSec) / 2) * 1000;
|
|
158
|
+
return Math.max(0, nextMidpointMs - nowMs);
|
|
158
159
|
}
|
|
159
160
|
|
|
160
161
|
function getHeadExecutionInfo(
|
package/src/sync/unknownBlock.ts
CHANGED
|
@@ -918,16 +918,17 @@ export class BlockInputSync {
|
|
|
918
918
|
* Poll unknown payloads of the current slot.
|
|
919
919
|
*/
|
|
920
920
|
private async pollPayloadsAtSlot(slot: Slot, signal: AbortSignal): Promise<void> {
|
|
921
|
+
const fork = this.config.getForkName(slot);
|
|
921
922
|
// fixed tick count so the loop cannot outlive its slot: for mainnet, it's 6 ticks
|
|
922
923
|
const tickCount = Math.floor(
|
|
923
|
-
(this.config.
|
|
924
|
+
(this.config.getSlotDurationMs(fork) - this.config.getPayloadDueMs(fork)) / PAYLOAD_POLL_INTERVAL_MS
|
|
924
925
|
);
|
|
925
926
|
// a root still missing on several ticks is searched on each of them, recorded once
|
|
926
927
|
const searchedRoots = new Set<RootHex>();
|
|
927
928
|
let polls = 0;
|
|
928
929
|
|
|
929
930
|
try {
|
|
930
|
-
await sleep(Math.max(0, this.config.getPayloadDueMs() - this.chain.clock.msFromSlot(slot)), signal);
|
|
931
|
+
await sleep(Math.max(0, this.config.getPayloadDueMs(fork) - this.chain.clock.msFromSlot(slot)), signal);
|
|
931
932
|
|
|
932
933
|
for (polls = 0; polls < tickCount; polls++) {
|
|
933
934
|
let importedCount = 0;
|
package/src/util/clock.ts
CHANGED
|
@@ -78,9 +78,9 @@ export class Clock extends EventEmitter implements IClock {
|
|
|
78
78
|
|
|
79
79
|
this.config = config;
|
|
80
80
|
this.genesisTime = genesisTime;
|
|
81
|
-
this.timeoutId = setTimeout(this.onNextSlot, this.msUntilNextSlot());
|
|
82
81
|
this.signal = signal;
|
|
83
82
|
this._currentSlot = getCurrentSlot(this.config, this.genesisTime);
|
|
83
|
+
this.timeoutId = setTimeout(this.onNextSlot, this.msUntilNextSlot());
|
|
84
84
|
this.signal.addEventListener("abort", () => clearTimeout(this.timeoutId), {once: true});
|
|
85
85
|
}
|
|
86
86
|
|
|
@@ -204,8 +204,7 @@ export class Clock extends EventEmitter implements IClock {
|
|
|
204
204
|
};
|
|
205
205
|
|
|
206
206
|
private msUntilNextSlot(): number {
|
|
207
|
-
const
|
|
208
|
-
|
|
209
|
-
return milliSecondsPerSlot - (diffInMilliSeconds % milliSecondsPerSlot);
|
|
207
|
+
const nextSlotTimeSec = computeTimeAtSlot(this.config, this._currentSlot + 1, this.genesisTime);
|
|
208
|
+
return Math.max(0, nextSlotTimeSec * 1000 - Date.now());
|
|
210
209
|
}
|
|
211
210
|
}
|