@lodestar/beacon-node 1.36.0-dev.f259361847 → 1.36.0-dev.f3703b7882
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
- package/lib/api/impl/beacon/blocks/index.js +41 -22
- package/lib/api/impl/beacon/blocks/index.js.map +1 -1
- package/lib/api/impl/lodestar/index.d.ts +5 -0
- package/lib/api/impl/lodestar/index.d.ts.map +1 -1
- package/lib/api/impl/lodestar/index.js +35 -10
- package/lib/api/impl/lodestar/index.js.map +1 -1
- package/lib/api/impl/node/utils.js +1 -1
- package/lib/api/impl/node/utils.js.map +1 -1
- package/lib/chain/chain.d.ts +5 -2
- package/lib/chain/chain.d.ts.map +1 -1
- package/lib/chain/chain.js +32 -16
- package/lib/chain/chain.js.map +1 -1
- package/lib/chain/errors/blobSidecarError.d.ts +5 -0
- package/lib/chain/errors/blobSidecarError.d.ts.map +1 -1
- package/lib/chain/errors/blobSidecarError.js.map +1 -1
- package/lib/chain/errors/dataColumnSidecarError.d.ts +21 -14
- package/lib/chain/errors/dataColumnSidecarError.d.ts.map +1 -1
- package/lib/chain/errors/dataColumnSidecarError.js +4 -0
- package/lib/chain/errors/dataColumnSidecarError.js.map +1 -1
- package/lib/chain/forkChoice/index.d.ts +9 -1
- package/lib/chain/forkChoice/index.d.ts.map +1 -1
- package/lib/chain/forkChoice/index.js +109 -4
- package/lib/chain/forkChoice/index.js.map +1 -1
- package/lib/chain/interface.d.ts +2 -0
- package/lib/chain/interface.d.ts.map +1 -1
- package/lib/chain/options.d.ts +0 -2
- package/lib/chain/options.d.ts.map +1 -1
- package/lib/chain/options.js +2 -2
- package/lib/chain/options.js.map +1 -1
- package/lib/chain/stateCache/datastore/db.d.ts +12 -0
- package/lib/chain/stateCache/datastore/db.d.ts.map +1 -1
- package/lib/chain/stateCache/datastore/db.js +70 -0
- package/lib/chain/stateCache/datastore/db.js.map +1 -1
- package/lib/chain/stateCache/datastore/file.d.ts +1 -0
- package/lib/chain/stateCache/datastore/file.d.ts.map +1 -1
- package/lib/chain/stateCache/datastore/file.js +7 -0
- package/lib/chain/stateCache/datastore/file.js.map +1 -1
- package/lib/chain/stateCache/datastore/types.d.ts +1 -0
- package/lib/chain/stateCache/datastore/types.d.ts.map +1 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.d.ts +16 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.d.ts.map +1 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.js +31 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.js.map +1 -1
- package/lib/chain/validation/blobSidecar.d.ts +4 -1
- package/lib/chain/validation/blobSidecar.d.ts.map +1 -1
- package/lib/chain/validation/blobSidecar.js +46 -11
- package/lib/chain/validation/blobSidecar.js.map +1 -1
- package/lib/chain/validation/dataColumnSidecar.d.ts +4 -1
- package/lib/chain/validation/dataColumnSidecar.d.ts.map +1 -1
- package/lib/chain/validation/dataColumnSidecar.js +64 -19
- package/lib/chain/validation/dataColumnSidecar.js.map +1 -1
- package/lib/index.d.ts +2 -0
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -0
- package/lib/index.js.map +1 -1
- package/lib/metrics/metrics/lodestar.js +1 -1
- package/lib/metrics/metrics/lodestar.js.map +1 -1
- package/lib/network/core/networkCore.d.ts.map +1 -1
- package/lib/network/core/networkCore.js +5 -1
- package/lib/network/core/networkCore.js.map +1 -1
- package/lib/network/core/networkCoreWorker.js +8 -8
- package/lib/network/core/networkCoreWorker.js.map +1 -1
- package/lib/network/core/networkCoreWorkerHandler.js +1 -1
- package/lib/network/core/networkCoreWorkerHandler.js.map +1 -1
- package/lib/network/discv5/worker.js +2 -7
- package/lib/network/discv5/worker.js.map +1 -1
- package/lib/network/events.d.ts +1 -0
- package/lib/network/events.d.ts.map +1 -1
- package/lib/network/gossip/encoding.js +1 -1
- package/lib/network/gossip/encoding.js.map +1 -1
- package/lib/network/gossip/gossipsub.d.ts.map +1 -1
- package/lib/network/gossip/gossipsub.js +6 -1
- package/lib/network/gossip/gossipsub.js.map +1 -1
- package/lib/network/gossip/interface.d.ts +2 -0
- package/lib/network/gossip/interface.d.ts.map +1 -1
- package/lib/network/gossip/snappy_bun.d.ts +3 -0
- package/lib/network/gossip/snappy_bun.d.ts.map +1 -0
- package/lib/network/gossip/snappy_bun.js +3 -0
- package/lib/network/gossip/snappy_bun.js.map +1 -0
- package/lib/network/metadata.d.ts +1 -1
- package/lib/network/metadata.d.ts.map +1 -1
- package/lib/network/metadata.js +1 -0
- package/lib/network/metadata.js.map +1 -1
- package/lib/network/options.d.ts +0 -1
- package/lib/network/options.d.ts.map +1 -1
- package/lib/network/options.js.map +1 -1
- package/lib/network/peers/discover.js +2 -2
- package/lib/network/peers/discover.js.map +1 -1
- package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
- package/lib/network/processor/gossipHandlers.js +15 -1
- package/lib/network/processor/gossipHandlers.js.map +1 -1
- package/lib/network/processor/gossipValidatorFn.d.ts.map +1 -1
- package/lib/network/processor/gossipValidatorFn.js +3 -2
- package/lib/network/processor/gossipValidatorFn.js.map +1 -1
- package/lib/network/processor/types.d.ts +2 -0
- package/lib/network/processor/types.d.ts.map +1 -1
- package/lib/network/reqresp/ReqRespBeaconNode.d.ts.map +1 -1
- package/lib/network/reqresp/ReqRespBeaconNode.js +3 -1
- package/lib/network/reqresp/ReqRespBeaconNode.js.map +1 -1
- package/lib/network/reqresp/handlers/beaconBlocksByRange.d.ts +2 -1
- package/lib/network/reqresp/handlers/beaconBlocksByRange.d.ts.map +1 -1
- package/lib/network/reqresp/handlers/beaconBlocksByRange.js +14 -3
- package/lib/network/reqresp/handlers/beaconBlocksByRange.js.map +1 -1
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.d.ts +2 -1
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.d.ts.map +1 -1
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.js +9 -1
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.js.map +1 -1
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.d.ts +2 -1
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.d.ts.map +1 -1
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js +9 -1
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js.map +1 -1
- package/lib/network/reqresp/handlers/index.js +6 -6
- package/lib/network/reqresp/handlers/index.js.map +1 -1
- package/lib/network/reqresp/types.d.ts +1 -0
- package/lib/network/reqresp/types.d.ts.map +1 -1
- package/lib/node/nodejs.d.ts +2 -1
- package/lib/node/nodejs.d.ts.map +1 -1
- package/lib/node/nodejs.js +2 -1
- package/lib/node/nodejs.js.map +1 -1
- package/lib/sync/range/range.d.ts.map +1 -1
- package/lib/sync/range/range.js +2 -1
- package/lib/sync/range/range.js.map +1 -1
- package/lib/sync/unknownBlock.js +1 -1
- package/lib/sync/unknownBlock.js.map +1 -1
- package/lib/sync/utils/downloadByRange.d.ts +59 -15
- package/lib/sync/utils/downloadByRange.d.ts.map +1 -1
- package/lib/sync/utils/downloadByRange.js +204 -83
- package/lib/sync/utils/downloadByRange.js.map +1 -1
- package/lib/sync/utils/downloadByRoot.d.ts +8 -14
- package/lib/sync/utils/downloadByRoot.d.ts.map +1 -1
- package/lib/sync/utils/downloadByRoot.js +18 -33
- package/lib/sync/utils/downloadByRoot.js.map +1 -1
- package/lib/sync/utils/remoteSyncType.d.ts +2 -1
- package/lib/sync/utils/remoteSyncType.d.ts.map +1 -1
- package/lib/sync/utils/remoteSyncType.js +19 -4
- package/lib/sync/utils/remoteSyncType.js.map +1 -1
- package/lib/util/blobs.d.ts +1 -1
- package/lib/util/blobs.d.ts.map +1 -1
- package/lib/util/blobs.js +53 -20
- package/lib/util/blobs.js.map +1 -1
- package/lib/util/profile.d.ts +6 -4
- package/lib/util/profile.d.ts.map +1 -1
- package/lib/util/profile.js +40 -3
- package/lib/util/profile.js.map +1 -1
- package/lib/util/sszBytes.d.ts +2 -0
- package/lib/util/sszBytes.d.ts.map +1 -1
- package/lib/util/sszBytes.js +25 -0
- package/lib/util/sszBytes.js.map +1 -1
- package/package.json +32 -25
- package/src/api/impl/beacon/blocks/index.ts +47 -25
- package/src/api/impl/lodestar/index.ts +42 -10
- package/src/api/impl/node/utils.ts +1 -1
- package/src/chain/chain.ts +48 -23
- package/src/chain/errors/blobSidecarError.ts +12 -2
- package/src/chain/errors/dataColumnSidecarError.ts +31 -16
- package/src/chain/forkChoice/index.ts +178 -2
- package/src/chain/interface.ts +2 -0
- package/src/chain/options.ts +2 -3
- package/src/chain/stateCache/datastore/db.ts +89 -1
- package/src/chain/stateCache/datastore/file.ts +8 -0
- package/src/chain/stateCache/datastore/types.ts +1 -0
- package/src/chain/stateCache/persistentCheckpointsCache.ts +45 -2
- package/src/chain/validation/blobSidecar.ts +54 -10
- package/src/chain/validation/dataColumnSidecar.ts +76 -19
- package/src/index.ts +2 -0
- package/src/metrics/metrics/lodestar.ts +1 -1
- package/src/network/core/networkCore.ts +5 -1
- package/src/network/core/networkCoreWorker.ts +9 -9
- package/src/network/core/networkCoreWorkerHandler.ts +1 -1
- package/src/network/discv5/worker.ts +2 -7
- package/src/network/events.ts +1 -1
- package/src/network/gossip/encoding.ts +1 -1
- package/src/network/gossip/gossipsub.ts +7 -1
- package/src/network/gossip/interface.ts +2 -0
- package/src/network/gossip/snappy_bun.ts +2 -0
- package/src/network/metadata.ts +3 -1
- package/src/network/options.ts +0 -1
- package/src/network/peers/discover.ts +2 -2
- package/src/network/processor/gossipHandlers.ts +16 -1
- package/src/network/processor/gossipValidatorFn.ts +15 -2
- package/src/network/processor/types.ts +2 -0
- package/src/network/reqresp/ReqRespBeaconNode.ts +3 -1
- package/src/network/reqresp/handlers/beaconBlocksByRange.ts +18 -3
- package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts +13 -1
- package/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts +13 -1
- package/src/network/reqresp/handlers/index.ts +6 -6
- package/src/network/reqresp/types.ts +1 -0
- package/src/node/nodejs.ts +3 -0
- package/src/sync/range/range.ts +2 -1
- package/src/sync/unknownBlock.ts +1 -1
- package/src/sync/utils/downloadByRange.ts +273 -108
- package/src/sync/utils/downloadByRoot.ts +22 -56
- package/src/sync/utils/remoteSyncType.ts +23 -4
- package/src/util/blobs.ts +64 -20
- package/src/util/profile.ts +45 -3
- package/src/util/sszBytes.ts +30 -0
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
import {ChainForkConfig} from "@lodestar/config";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
ForkPostDeneb,
|
|
4
|
+
ForkPostFulu,
|
|
5
|
+
ForkPreFulu,
|
|
6
|
+
ForkPreGloas,
|
|
7
|
+
isForkPostFulu,
|
|
8
|
+
isForkPostGloas,
|
|
9
|
+
} from "@lodestar/params";
|
|
3
10
|
import {SignedBeaconBlock, Slot, deneb, fulu, phase0} from "@lodestar/types";
|
|
4
|
-
import {LodestarError, Logger, fromHex,
|
|
11
|
+
import {LodestarError, Logger, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
|
|
5
12
|
import {
|
|
6
13
|
BlockInputSource,
|
|
7
14
|
DAType,
|
|
@@ -15,7 +22,6 @@ import {validateBlockDataColumnSidecars} from "../../chain/validation/dataColumn
|
|
|
15
22
|
import {INetwork} from "../../network/index.js";
|
|
16
23
|
import {PeerIdStr} from "../../util/peerId.js";
|
|
17
24
|
import {WarnResult} from "../../util/wrapError.js";
|
|
18
|
-
import {DownloadByRootErrorCode} from "./downloadByRoot.js";
|
|
19
25
|
|
|
20
26
|
export type DownloadByRangeRequests = {
|
|
21
27
|
blocksRequest?: phase0.BeaconBlocksByRangeRequest;
|
|
@@ -31,7 +37,6 @@ export type DownloadByRangeResponses = {
|
|
|
31
37
|
|
|
32
38
|
export type DownloadAndCacheByRangeProps = DownloadByRangeRequests & {
|
|
33
39
|
config: ChainForkConfig;
|
|
34
|
-
cache: SeenBlockInput;
|
|
35
40
|
network: INetwork;
|
|
36
41
|
logger: Logger;
|
|
37
42
|
peerIdStr: string;
|
|
@@ -111,7 +116,13 @@ export function cacheByRangeResponses({
|
|
|
111
116
|
}
|
|
112
117
|
|
|
113
118
|
for (const {blockRoot, blobSidecars} of responses.validatedBlobSidecars ?? []) {
|
|
114
|
-
const
|
|
119
|
+
const dataSlot = blobSidecars.at(0)?.signedBlockHeader.message.slot;
|
|
120
|
+
if (dataSlot === undefined) {
|
|
121
|
+
throw new Error(
|
|
122
|
+
`Coding Error: empty blobSidecars returned for blockRoot=${toRootHex(blockRoot)} from validation functions`
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
const existing = updatedBatchBlocks.get(dataSlot);
|
|
115
126
|
const blockRootHex = toRootHex(blockRoot);
|
|
116
127
|
|
|
117
128
|
if (!existing) {
|
|
@@ -122,7 +133,7 @@ export function cacheByRangeResponses({
|
|
|
122
133
|
throw new DownloadByRangeError({
|
|
123
134
|
code: DownloadByRangeErrorCode.MISMATCH_BLOCK_INPUT_TYPE,
|
|
124
135
|
slot: existing.slot,
|
|
125
|
-
blockRoot:
|
|
136
|
+
blockRoot: existing.blockRootHex,
|
|
126
137
|
expected: DAType.Blobs,
|
|
127
138
|
actual: existing.type,
|
|
128
139
|
});
|
|
@@ -143,18 +154,24 @@ export function cacheByRangeResponses({
|
|
|
143
154
|
}
|
|
144
155
|
|
|
145
156
|
for (const {blockRoot, columnSidecars} of responses.validatedColumnSidecars ?? []) {
|
|
146
|
-
const
|
|
157
|
+
const dataSlot = columnSidecars.at(0)?.signedBlockHeader.message.slot;
|
|
158
|
+
if (dataSlot === undefined) {
|
|
159
|
+
throw new Error(
|
|
160
|
+
`Coding Error: empty columnSidecars returned for blockRoot=${toRootHex(blockRoot)} from validation functions`
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
const existing = updatedBatchBlocks.get(dataSlot);
|
|
147
164
|
const blockRootHex = toRootHex(blockRoot);
|
|
148
165
|
|
|
149
166
|
if (!existing) {
|
|
150
|
-
throw new Error("Coding error: blockInput must exist when adding
|
|
167
|
+
throw new Error("Coding error: blockInput must exist when adding columns");
|
|
151
168
|
}
|
|
152
169
|
|
|
153
170
|
if (!isBlockInputColumns(existing)) {
|
|
154
171
|
throw new DownloadByRangeError({
|
|
155
172
|
code: DownloadByRangeErrorCode.MISMATCH_BLOCK_INPUT_TYPE,
|
|
156
173
|
slot: existing.slot,
|
|
157
|
-
blockRoot:
|
|
174
|
+
blockRoot: existing.blockRootHex,
|
|
158
175
|
expected: DAType.Columns,
|
|
159
176
|
actual: existing.type,
|
|
160
177
|
});
|
|
@@ -185,7 +202,7 @@ export async function downloadByRange({
|
|
|
185
202
|
blocksRequest,
|
|
186
203
|
blobsRequest,
|
|
187
204
|
columnsRequest,
|
|
188
|
-
}:
|
|
205
|
+
}: DownloadAndCacheByRangeProps): Promise<WarnResult<ValidatedResponses, DownloadByRangeError>> {
|
|
189
206
|
let response: DownloadByRangeResponses;
|
|
190
207
|
try {
|
|
191
208
|
response = await requestByRange({
|
|
@@ -290,7 +307,7 @@ export async function validateResponses({
|
|
|
290
307
|
if ((blobsRequest || columnsRequest) && !(blocks || batchBlocks)) {
|
|
291
308
|
throw new DownloadByRangeError(
|
|
292
309
|
{
|
|
293
|
-
code: DownloadByRangeErrorCode.
|
|
310
|
+
code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
|
|
294
311
|
...requestsLogMeta({blobsRequest, columnsRequest}),
|
|
295
312
|
},
|
|
296
313
|
"No blocks to validate data requests against"
|
|
@@ -301,24 +318,28 @@ export async function validateResponses({
|
|
|
301
318
|
let warnings: DownloadByRangeError[] | null = null;
|
|
302
319
|
|
|
303
320
|
if (blocksRequest) {
|
|
304
|
-
|
|
321
|
+
const result = validateBlockByRangeResponse(config, blocksRequest, blocks ?? []);
|
|
322
|
+
if (result.warnings?.length) {
|
|
323
|
+
warnings = result.warnings;
|
|
324
|
+
}
|
|
325
|
+
validatedResponses.validatedBlocks = result.result;
|
|
305
326
|
}
|
|
306
327
|
|
|
307
328
|
const dataRequest = blobsRequest ?? columnsRequest;
|
|
308
329
|
if (!dataRequest) {
|
|
309
|
-
return {result: validatedResponses, warnings
|
|
330
|
+
return {result: validatedResponses, warnings};
|
|
310
331
|
}
|
|
311
332
|
|
|
312
|
-
const
|
|
333
|
+
const blocksForDataValidation = getBlocksForDataValidation(
|
|
313
334
|
dataRequest,
|
|
314
335
|
batchBlocks,
|
|
315
|
-
|
|
336
|
+
validatedResponses.validatedBlocks?.length ? validatedResponses.validatedBlocks : undefined
|
|
316
337
|
);
|
|
317
338
|
|
|
318
|
-
if (!
|
|
339
|
+
if (!blocksForDataValidation.length) {
|
|
319
340
|
throw new DownloadByRangeError(
|
|
320
341
|
{
|
|
321
|
-
code: DownloadByRangeErrorCode.
|
|
342
|
+
code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
|
|
322
343
|
...requestsLogMeta({blobsRequest, columnsRequest}),
|
|
323
344
|
},
|
|
324
345
|
"No blocks in data request slot range to validate data response against"
|
|
@@ -336,7 +357,10 @@ export async function validateResponses({
|
|
|
336
357
|
);
|
|
337
358
|
}
|
|
338
359
|
|
|
339
|
-
validatedResponses.validatedBlobSidecars = await validateBlobsByRangeResponse(
|
|
360
|
+
validatedResponses.validatedBlobSidecars = await validateBlobsByRangeResponse(
|
|
361
|
+
blocksForDataValidation,
|
|
362
|
+
blobSidecars
|
|
363
|
+
);
|
|
340
364
|
}
|
|
341
365
|
|
|
342
366
|
if (columnsRequest) {
|
|
@@ -351,8 +375,9 @@ export async function validateResponses({
|
|
|
351
375
|
}
|
|
352
376
|
|
|
353
377
|
const validatedColumnSidecarsResult = await validateColumnsByRangeResponse(
|
|
378
|
+
config,
|
|
354
379
|
columnsRequest,
|
|
355
|
-
|
|
380
|
+
blocksForDataValidation,
|
|
356
381
|
columnSidecars
|
|
357
382
|
);
|
|
358
383
|
validatedResponses.validatedColumnSidecars = validatedColumnSidecarsResult.result;
|
|
@@ -375,20 +400,30 @@ export function validateBlockByRangeResponse(
|
|
|
375
400
|
config: ChainForkConfig,
|
|
376
401
|
blocksRequest: phase0.BeaconBlocksByRangeRequest,
|
|
377
402
|
blocks: SignedBeaconBlock[]
|
|
378
|
-
): ValidatedBlock[] {
|
|
403
|
+
): WarnResult<ValidatedBlock[], DownloadByRangeError> {
|
|
379
404
|
const {startSlot, count} = blocksRequest;
|
|
380
405
|
|
|
381
|
-
//
|
|
382
|
-
//
|
|
383
|
-
//
|
|
384
|
-
//
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
406
|
+
// An error was thrown here by @twoeths in #8150 but it breaks for epochs with 0 blocks during chain
|
|
407
|
+
// liveness issues. See comment https://github.com/ChainSafe/lodestar/issues/8147#issuecomment-3246434697
|
|
408
|
+
// There are instances where clients return no blocks though. Need to monitor this via the warns to see
|
|
409
|
+
// if what the correct behavior should be
|
|
410
|
+
if (!blocks.length) {
|
|
411
|
+
throw new DownloadByRangeError({
|
|
412
|
+
code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
|
|
413
|
+
...requestsLogMeta({blocksRequest}),
|
|
414
|
+
});
|
|
415
|
+
// TODO: this was causing deadlock again. need to come back and fix this so that its possible to process through
|
|
416
|
+
// an empty epoch for periods with poor liveness
|
|
417
|
+
// return {
|
|
418
|
+
// result: [],
|
|
419
|
+
// warnings: [
|
|
420
|
+
// new DownloadByRangeError({
|
|
421
|
+
// code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
|
|
422
|
+
// ...requestsLogMeta({blocksRequest}),
|
|
423
|
+
// }),
|
|
424
|
+
// ],
|
|
425
|
+
// };
|
|
426
|
+
}
|
|
392
427
|
|
|
393
428
|
if (blocks.length > count) {
|
|
394
429
|
throw new DownloadByRangeError(
|
|
@@ -445,8 +480,8 @@ export function validateBlockByRangeResponse(
|
|
|
445
480
|
{
|
|
446
481
|
code: DownloadByRangeErrorCode.PARENT_ROOT_MISMATCH,
|
|
447
482
|
slot: blocks[i].message.slot,
|
|
448
|
-
expected:
|
|
449
|
-
actual:
|
|
483
|
+
expected: toRootHex(blockRoot),
|
|
484
|
+
actual: toRootHex(parentRoot),
|
|
450
485
|
},
|
|
451
486
|
`Block parent root does not match the previous block's root in BeaconBlocksByRange response`
|
|
452
487
|
);
|
|
@@ -454,7 +489,10 @@ export function validateBlockByRangeResponse(
|
|
|
454
489
|
}
|
|
455
490
|
}
|
|
456
491
|
|
|
457
|
-
return
|
|
492
|
+
return {
|
|
493
|
+
result: response,
|
|
494
|
+
warnings: null,
|
|
495
|
+
};
|
|
458
496
|
}
|
|
459
497
|
|
|
460
498
|
/**
|
|
@@ -516,9 +554,13 @@ export async function validateBlobsByRangeResponse(
|
|
|
516
554
|
}
|
|
517
555
|
|
|
518
556
|
validateSidecarsPromises.push(
|
|
519
|
-
validateBlockBlobSidecars(
|
|
520
|
-
|
|
521
|
-
|
|
557
|
+
validateBlockBlobSidecars(
|
|
558
|
+
null, // do not pass chain here so we do not validate header signature
|
|
559
|
+
block.message.slot,
|
|
560
|
+
blockRoot,
|
|
561
|
+
blockKzgCommitments.length,
|
|
562
|
+
blockBlobSidecars
|
|
563
|
+
).then(() => ({blockRoot, blobSidecars: blockBlobSidecars}))
|
|
522
564
|
);
|
|
523
565
|
}
|
|
524
566
|
|
|
@@ -528,76 +570,184 @@ export async function validateBlobsByRangeResponse(
|
|
|
528
570
|
|
|
529
571
|
/**
|
|
530
572
|
* Should not be called directly. Only exported for unit testing purposes
|
|
573
|
+
*
|
|
574
|
+
* Spec states:
|
|
575
|
+
* 1) must be within range [start_slot, start_slot + count]
|
|
576
|
+
* 2) should respond with all columns in the range or and 3:ResourceUnavailable (and potentially get down-scored)
|
|
577
|
+
* 3) must response with at least the sidecars of the first blob-carrying block that exists in the range
|
|
578
|
+
* 4) must include all sidecars from each block from which there are blobs
|
|
579
|
+
* 5) where they exists, sidecars must be sent in (slot, index) order
|
|
580
|
+
* 6) clients may limit the number of sidecars in a response
|
|
581
|
+
* 7) clients may stop responding mid-response if their view of fork-choice changes
|
|
582
|
+
*
|
|
583
|
+
* We will interpret the spec as follows
|
|
584
|
+
* - Errors when validating: 1, 3, 5
|
|
585
|
+
* - Warnings when validating: 2, 4, 6, 7
|
|
586
|
+
*
|
|
587
|
+
* For "warning" cases, where we get a partial response but sidecars are validated and correct with respect to the
|
|
588
|
+
* blocks, then they will be kept. This loosening of the spec is to help ensure sync goes smoothly and we can find
|
|
589
|
+
* the data needed in difficult network situations.
|
|
590
|
+
*
|
|
591
|
+
* Assume for the following two examples we request indices 5, 10, 15 for a range of slots 32-63
|
|
592
|
+
*
|
|
593
|
+
* For slots where we receive no sidecars, example slot 45, but blobs exist we will stop validating subsequent
|
|
594
|
+
* slots, 45-63. The next round of requests will get structured to pull the from the slot that had columns
|
|
595
|
+
* missing to the end of the range for all columns indices that were requested for the current partially failed
|
|
596
|
+
* request (slots 45-63 and indices 5, 10, 15).
|
|
597
|
+
*
|
|
598
|
+
* For slots where only some of the requested sidecars are received we will proceed with validation. For simplicity sake
|
|
599
|
+
* we will assume that if we only get some indices back for a (or several) slot(s) that the indices we get will be
|
|
600
|
+
* consistent. IE if a peer returns only index 5, they will most likely return that same index for subsequent slot
|
|
601
|
+
* (index 5 for slots 34, 35, 36, etc). They will not likely return 5 on slot 34, 10 on slot 35, 15 on slot 36, etc.
|
|
602
|
+
* This assumption makes the code simpler. For both cases the request for the next round will be structured correctly
|
|
603
|
+
* to pull any missing column indices for whatever range remains. The simplification just leads to re-verification
|
|
604
|
+
* of the columns but the number of columns downloaded will be the same regardless of if they are validated twice.
|
|
605
|
+
*
|
|
606
|
+
* validateColumnsByRangeResponse makes some assumptions about the data being passed in
|
|
607
|
+
* blocks are:
|
|
608
|
+
* - slotwise in order
|
|
609
|
+
* - form a chain
|
|
610
|
+
* - non-sparse response (any missing block is a skipped slot not a bad response)
|
|
611
|
+
* - last block is last slot received
|
|
531
612
|
*/
|
|
532
613
|
export async function validateColumnsByRangeResponse(
|
|
614
|
+
config: ChainForkConfig,
|
|
533
615
|
request: fulu.DataColumnSidecarsByRangeRequest,
|
|
534
|
-
|
|
616
|
+
blocks: ValidatedBlock[],
|
|
535
617
|
columnSidecars: fulu.DataColumnSidecars
|
|
536
618
|
): Promise<WarnResult<ValidatedColumnSidecars[], DownloadByRangeError>> {
|
|
537
|
-
// Expected column count considering currently-validated batch blocks
|
|
538
|
-
// TODO GLOAS: Post-gloas's blobKzgCommitments is not in beacon block body. Need to source it from somewhere else.
|
|
539
|
-
const expectedColumnCount = dataRequestBlocks.reduce((acc, {block}) => {
|
|
540
|
-
return (block as SignedBeaconBlock<ForkPostDeneb & ForkPreGloas>).message.body.blobKzgCommitments.length > 0
|
|
541
|
-
? request.columns.length + acc
|
|
542
|
-
: acc;
|
|
543
|
-
}, 0);
|
|
544
|
-
const nextSlot = dataRequestBlocks.length
|
|
545
|
-
? (dataRequestBlocks.at(-1) as ValidatedBlock).block.message.slot + 1
|
|
546
|
-
: request.startSlot;
|
|
547
|
-
const possiblyMissingBlocks = nextSlot - request.startSlot + request.count;
|
|
548
|
-
|
|
549
|
-
// Allow for extra columns if some blocks are missing from the end of a batch
|
|
550
|
-
// Eg: If we requested 10 blocks but only 8 were returned, allow for up to 2 * columns.length extra columns
|
|
551
|
-
const maxColumnCount = expectedColumnCount + possiblyMissingBlocks * request.columns.length;
|
|
552
|
-
|
|
553
|
-
if (columnSidecars.length > maxColumnCount) {
|
|
554
|
-
// this never happens on devnet, so throw error for now
|
|
555
|
-
throw new DownloadByRangeError(
|
|
556
|
-
{
|
|
557
|
-
code: DownloadByRangeErrorCode.OVER_COLUMNS,
|
|
558
|
-
max: maxColumnCount,
|
|
559
|
-
actual: columnSidecars.length,
|
|
560
|
-
},
|
|
561
|
-
"Extra data columns received in DataColumnSidecarsByRange response"
|
|
562
|
-
);
|
|
563
|
-
}
|
|
564
|
-
|
|
565
619
|
const warnings: DownloadByRangeError[] = [];
|
|
566
|
-
// no need to check for columnSidecars.length vs expectedColumnCount here, will be checked per-block below
|
|
567
|
-
const requestedColumns = new Set(request.columns);
|
|
568
|
-
const validateSidecarsPromises: Promise<ValidatedColumnSidecars>[] = [];
|
|
569
|
-
for (let blockIndex = 0, columnSidecarIndex = 0; blockIndex < dataRequestBlocks.length; blockIndex++) {
|
|
570
|
-
const {block, blockRoot} = dataRequestBlocks[blockIndex];
|
|
571
|
-
const slot = block.message.slot;
|
|
572
|
-
const blockRootHex = toRootHex(blockRoot);
|
|
573
|
-
// TODO GLOAS: Post-gloas's blobKzgCommitments is not in beacon block body. Need to source it from somewhere else.
|
|
574
|
-
const blockKzgCommitments = (block as SignedBeaconBlock<ForkPostFulu & ForkPreGloas>).message.body
|
|
575
|
-
.blobKzgCommitments;
|
|
576
|
-
const expectedColumns = blockKzgCommitments.length ? request.columns.length : 0;
|
|
577
620
|
|
|
578
|
-
|
|
621
|
+
const seenColumns = new Map<Slot, Map<number, fulu.DataColumnSidecar>>();
|
|
622
|
+
let currentSlot = -1;
|
|
623
|
+
let currentIndex = -1;
|
|
624
|
+
// Check for duplicates and order
|
|
625
|
+
for (const columnSidecar of columnSidecars) {
|
|
626
|
+
const slot = columnSidecar.signedBlockHeader.message.slot;
|
|
627
|
+
let seenSlotColumns = seenColumns.get(slot);
|
|
628
|
+
if (!seenSlotColumns) {
|
|
629
|
+
seenSlotColumns = new Map();
|
|
630
|
+
seenColumns.set(slot, seenSlotColumns);
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
if (seenSlotColumns.has(columnSidecar.index)) {
|
|
634
|
+
warnings.push(
|
|
635
|
+
new DownloadByRangeError({
|
|
636
|
+
code: DownloadByRangeErrorCode.DUPLICATE_COLUMN,
|
|
637
|
+
slot,
|
|
638
|
+
index: columnSidecar.index,
|
|
639
|
+
})
|
|
640
|
+
);
|
|
641
|
+
|
|
579
642
|
continue;
|
|
580
643
|
}
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
644
|
+
|
|
645
|
+
if (currentSlot > slot) {
|
|
646
|
+
warnings.push(
|
|
647
|
+
new DownloadByRangeError(
|
|
648
|
+
{
|
|
649
|
+
code: DownloadByRangeErrorCode.OUT_OF_ORDER_COLUMNS,
|
|
650
|
+
slot,
|
|
651
|
+
},
|
|
652
|
+
"ColumnSidecars received out of slot order"
|
|
653
|
+
)
|
|
654
|
+
);
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
if (currentSlot === slot && currentIndex > columnSidecar.index) {
|
|
658
|
+
warnings.push(
|
|
659
|
+
new DownloadByRangeError(
|
|
660
|
+
{
|
|
661
|
+
code: DownloadByRangeErrorCode.OUT_OF_ORDER_COLUMNS,
|
|
662
|
+
slot,
|
|
663
|
+
},
|
|
664
|
+
"Column indices out of order within a slot"
|
|
665
|
+
)
|
|
666
|
+
);
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
seenSlotColumns.set(columnSidecar.index, columnSidecar);
|
|
670
|
+
if (currentSlot !== slot) {
|
|
671
|
+
// a new slot has started, reset index
|
|
672
|
+
currentIndex = -1;
|
|
673
|
+
} else {
|
|
674
|
+
currentIndex = columnSidecar.index;
|
|
675
|
+
}
|
|
676
|
+
currentSlot = slot;
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
const validationPromises: Promise<ValidatedColumnSidecars>[] = [];
|
|
680
|
+
|
|
681
|
+
for (const {blockRoot, block} of blocks) {
|
|
682
|
+
const slot = block.message.slot;
|
|
683
|
+
const rootHex = toRootHex(blockRoot);
|
|
684
|
+
const forkName = config.getForkName(slot);
|
|
685
|
+
const columnSidecarsMap: Map<number, fulu.DataColumnSidecar> = seenColumns.get(slot) ?? new Map();
|
|
686
|
+
const columnSidecars = Array.from(columnSidecarsMap.values()).sort((a, b) => a.index - b.index);
|
|
687
|
+
|
|
688
|
+
let blobCount: number;
|
|
689
|
+
if (!isForkPostFulu(forkName)) {
|
|
690
|
+
const dataSlot = columnSidecars.at(0)?.signedBlockHeader.message.slot;
|
|
691
|
+
throw new DownloadByRangeError({
|
|
692
|
+
code: DownloadByRangeErrorCode.MISMATCH_BLOCK_FORK,
|
|
693
|
+
slot,
|
|
694
|
+
blockFork: forkName,
|
|
695
|
+
dataFork: dataSlot ? config.getForkName(dataSlot) : "unknown",
|
|
696
|
+
});
|
|
697
|
+
}
|
|
698
|
+
if (isForkPostGloas(forkName)) {
|
|
699
|
+
// TODO GLOAS: Post-gloas's blobKzgCommitments is not in beacon block body. Need to source it from somewhere else.
|
|
700
|
+
// if block without columns is passed default to zero and throw below
|
|
701
|
+
blobCount = 0;
|
|
702
|
+
} else {
|
|
703
|
+
blobCount = (block as SignedBeaconBlock<ForkPostFulu & ForkPreGloas>).message.body.blobKzgCommitments.length;
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
if (columnSidecars.length === 0) {
|
|
707
|
+
if (!blobCount) {
|
|
708
|
+
// no columns in the slot
|
|
709
|
+
continue;
|
|
587
710
|
}
|
|
588
|
-
|
|
589
|
-
|
|
711
|
+
|
|
712
|
+
/**
|
|
713
|
+
* If no columns are found for a block and there are commitments on the block then stop checking and just
|
|
714
|
+
* return early. Even if there were columns returned for subsequent slots that doesn't matter because
|
|
715
|
+
* we will be re-requesting them again anyway. Leftovers just get ignored
|
|
716
|
+
*/
|
|
717
|
+
warnings.push(
|
|
718
|
+
new DownloadByRangeError({
|
|
719
|
+
code: DownloadByRangeErrorCode.MISSING_COLUMNS,
|
|
720
|
+
slot,
|
|
721
|
+
blockRoot: rootHex,
|
|
722
|
+
missingIndices: prettyPrintIndices(request.columns),
|
|
723
|
+
})
|
|
724
|
+
);
|
|
725
|
+
break;
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
const returnedColumns = Array.from(columnSidecarsMap.keys()).sort();
|
|
729
|
+
if (!blobCount) {
|
|
730
|
+
// columns for a block that does not have blobs
|
|
731
|
+
// TODO(fulu): should this be a hard error with no data retained from peer or just a warning
|
|
732
|
+
throw new DownloadByRangeError(
|
|
733
|
+
{
|
|
734
|
+
code: DownloadByRangeErrorCode.NO_COLUMNS_FOR_BLOCK,
|
|
735
|
+
slot,
|
|
736
|
+
blockRoot: rootHex,
|
|
737
|
+
invalidIndices: prettyPrintIndices(returnedColumns),
|
|
738
|
+
},
|
|
739
|
+
"Block has no blob commitments but data column sidecars were provided"
|
|
740
|
+
);
|
|
590
741
|
}
|
|
591
742
|
|
|
592
|
-
const
|
|
593
|
-
const missingIndices = request.columns.filter((i) => !returnedColumns.has(i));
|
|
743
|
+
const missingIndices = request.columns.filter((i) => !columnSidecarsMap.has(i));
|
|
594
744
|
if (missingIndices.length > 0) {
|
|
595
745
|
warnings.push(
|
|
596
746
|
new DownloadByRangeError(
|
|
597
747
|
{
|
|
598
748
|
code: DownloadByRangeErrorCode.MISSING_COLUMNS,
|
|
599
749
|
slot,
|
|
600
|
-
blockRoot:
|
|
750
|
+
blockRoot: rootHex,
|
|
601
751
|
missingIndices: prettyPrintIndices(missingIndices),
|
|
602
752
|
},
|
|
603
753
|
"Missing data columns in DataColumnSidecarsByRange response"
|
|
@@ -605,14 +755,14 @@ export async function validateColumnsByRangeResponse(
|
|
|
605
755
|
);
|
|
606
756
|
}
|
|
607
757
|
|
|
608
|
-
const extraIndices =
|
|
758
|
+
const extraIndices = returnedColumns.filter((i) => !request.columns.includes(i));
|
|
609
759
|
if (extraIndices.length > 0) {
|
|
610
760
|
warnings.push(
|
|
611
761
|
new DownloadByRangeError(
|
|
612
762
|
{
|
|
613
763
|
code: DownloadByRangeErrorCode.EXTRA_COLUMNS,
|
|
614
764
|
slot,
|
|
615
|
-
blockRoot:
|
|
765
|
+
blockRoot: rootHex,
|
|
616
766
|
invalidIndices: prettyPrintIndices(extraIndices),
|
|
617
767
|
},
|
|
618
768
|
"Data column in not in requested columns in DataColumnSidecarsByRange response"
|
|
@@ -620,17 +770,25 @@ export async function validateColumnsByRangeResponse(
|
|
|
620
770
|
);
|
|
621
771
|
}
|
|
622
772
|
|
|
623
|
-
|
|
624
|
-
validateBlockDataColumnSidecars(
|
|
773
|
+
validationPromises.push(
|
|
774
|
+
validateBlockDataColumnSidecars(
|
|
775
|
+
null, // do not pass chain here so we do not validate header signature
|
|
776
|
+
slot,
|
|
625
777
|
blockRoot,
|
|
626
|
-
|
|
778
|
+
blobCount,
|
|
779
|
+
columnSidecars
|
|
780
|
+
).then(() => ({
|
|
781
|
+
blockRoot,
|
|
782
|
+
columnSidecars,
|
|
627
783
|
}))
|
|
628
784
|
);
|
|
629
785
|
}
|
|
630
786
|
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
787
|
+
const validatedColumns = await Promise.all(validationPromises);
|
|
788
|
+
return {
|
|
789
|
+
result: validatedColumns,
|
|
790
|
+
warnings: warnings.length ? warnings : null,
|
|
791
|
+
};
|
|
634
792
|
}
|
|
635
793
|
|
|
636
794
|
/**
|
|
@@ -697,7 +855,7 @@ function requestsLogMeta({blocksRequest, blobsRequest, columnsRequest}: Download
|
|
|
697
855
|
}
|
|
698
856
|
|
|
699
857
|
export enum DownloadByRangeErrorCode {
|
|
700
|
-
|
|
858
|
+
MISSING_BLOCKS_RESPONSE = "DOWNLOAD_BY_RANGE_ERROR_MISSING_BLOCK_RESPONSE",
|
|
701
859
|
MISSING_BLOBS_RESPONSE = "DOWNLOAD_BY_RANGE_ERROR_MISSING_BLOBS_RESPONSE",
|
|
702
860
|
MISSING_COLUMNS_RESPONSE = "DOWNLOAD_BY_RANGE_ERROR_MISSING_COLUMNS_RESPONSE",
|
|
703
861
|
|
|
@@ -718,19 +876,19 @@ export enum DownloadByRangeErrorCode {
|
|
|
718
876
|
MISSING_COLUMNS = "DOWNLOAD_BY_RANGE_ERROR_MISSING_COLUMNS",
|
|
719
877
|
OVER_COLUMNS = "DOWNLOAD_BY_RANGE_ERROR_OVER_COLUMNS",
|
|
720
878
|
EXTRA_COLUMNS = "DOWNLOAD_BY_RANGE_ERROR_EXTRA_COLUMNS",
|
|
879
|
+
NO_COLUMNS_FOR_BLOCK = "DOWNLOAD_BY_RANGE_ERROR_NO_COLUMNS_FOR_BLOCK",
|
|
880
|
+
DUPLICATE_COLUMN = "DOWNLOAD_BY_RANGE_ERROR_DUPLICATE_COLUMN",
|
|
881
|
+
OUT_OF_ORDER_COLUMNS = "DOWNLOAD_BY_RANGE_OUT_OF_ORDER_COLUMNS",
|
|
721
882
|
|
|
722
883
|
/** Cached block input type mismatches new data */
|
|
884
|
+
MISMATCH_BLOCK_FORK = "DOWNLOAD_BY_RANGE_ERROR_MISMATCH_BLOCK_FORK",
|
|
723
885
|
MISMATCH_BLOCK_INPUT_TYPE = "DOWNLOAD_BY_RANGE_ERROR_MISMATCH_BLOCK_INPUT_TYPE",
|
|
724
886
|
}
|
|
725
887
|
|
|
726
888
|
export type DownloadByRangeErrorType =
|
|
727
|
-
| {
|
|
728
|
-
code: DownloadByRootErrorCode.MISSING_BLOCK_RESPONSE;
|
|
729
|
-
expectedCount: number;
|
|
730
|
-
}
|
|
731
889
|
| {
|
|
732
890
|
code:
|
|
733
|
-
| DownloadByRangeErrorCode.
|
|
891
|
+
| DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE
|
|
734
892
|
| DownloadByRangeErrorCode.MISSING_BLOBS_RESPONSE
|
|
735
893
|
| DownloadByRangeErrorCode.MISSING_COLUMNS_RESPONSE;
|
|
736
894
|
blockStartSlot?: number;
|
|
@@ -741,12 +899,14 @@ export type DownloadByRangeErrorType =
|
|
|
741
899
|
columnCount?: number;
|
|
742
900
|
}
|
|
743
901
|
| {
|
|
744
|
-
code:
|
|
745
|
-
|
|
902
|
+
code: DownloadByRangeErrorCode.OUT_OF_RANGE_BLOCKS;
|
|
903
|
+
slot: number;
|
|
746
904
|
}
|
|
747
905
|
| {
|
|
748
|
-
code: DownloadByRangeErrorCode.
|
|
906
|
+
code: DownloadByRangeErrorCode.MISMATCH_BLOCK_FORK;
|
|
749
907
|
slot: number;
|
|
908
|
+
dataFork: string;
|
|
909
|
+
blockFork: string;
|
|
750
910
|
}
|
|
751
911
|
| {
|
|
752
912
|
code: DownloadByRangeErrorCode.OUT_OF_ORDER_BLOCKS;
|
|
@@ -778,7 +938,7 @@ export type DownloadByRangeErrorType =
|
|
|
778
938
|
actual: number;
|
|
779
939
|
}
|
|
780
940
|
| {
|
|
781
|
-
code: DownloadByRangeErrorCode.OUT_OF_ORDER_BLOBS;
|
|
941
|
+
code: DownloadByRangeErrorCode.OUT_OF_ORDER_BLOBS | DownloadByRangeErrorCode.OUT_OF_ORDER_COLUMNS;
|
|
782
942
|
slot: number;
|
|
783
943
|
}
|
|
784
944
|
| {
|
|
@@ -798,7 +958,12 @@ export type DownloadByRangeErrorType =
|
|
|
798
958
|
missingIndices: string;
|
|
799
959
|
}
|
|
800
960
|
| {
|
|
801
|
-
code: DownloadByRangeErrorCode.
|
|
961
|
+
code: DownloadByRangeErrorCode.DUPLICATE_COLUMN;
|
|
962
|
+
slot: Slot;
|
|
963
|
+
index: number;
|
|
964
|
+
}
|
|
965
|
+
| {
|
|
966
|
+
code: DownloadByRangeErrorCode.EXTRA_COLUMNS | DownloadByRangeErrorCode.NO_COLUMNS_FOR_BLOCK;
|
|
802
967
|
slot: Slot;
|
|
803
968
|
blockRoot: string;
|
|
804
969
|
invalidIndices: string;
|