@lodestar/beacon-node 1.36.0-dev.f259361847 → 1.36.0-dev.f3703b7882

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +41 -22
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/api/impl/lodestar/index.d.ts +5 -0
  5. package/lib/api/impl/lodestar/index.d.ts.map +1 -1
  6. package/lib/api/impl/lodestar/index.js +35 -10
  7. package/lib/api/impl/lodestar/index.js.map +1 -1
  8. package/lib/api/impl/node/utils.js +1 -1
  9. package/lib/api/impl/node/utils.js.map +1 -1
  10. package/lib/chain/chain.d.ts +5 -2
  11. package/lib/chain/chain.d.ts.map +1 -1
  12. package/lib/chain/chain.js +32 -16
  13. package/lib/chain/chain.js.map +1 -1
  14. package/lib/chain/errors/blobSidecarError.d.ts +5 -0
  15. package/lib/chain/errors/blobSidecarError.d.ts.map +1 -1
  16. package/lib/chain/errors/blobSidecarError.js.map +1 -1
  17. package/lib/chain/errors/dataColumnSidecarError.d.ts +21 -14
  18. package/lib/chain/errors/dataColumnSidecarError.d.ts.map +1 -1
  19. package/lib/chain/errors/dataColumnSidecarError.js +4 -0
  20. package/lib/chain/errors/dataColumnSidecarError.js.map +1 -1
  21. package/lib/chain/forkChoice/index.d.ts +9 -1
  22. package/lib/chain/forkChoice/index.d.ts.map +1 -1
  23. package/lib/chain/forkChoice/index.js +109 -4
  24. package/lib/chain/forkChoice/index.js.map +1 -1
  25. package/lib/chain/interface.d.ts +2 -0
  26. package/lib/chain/interface.d.ts.map +1 -1
  27. package/lib/chain/options.d.ts +0 -2
  28. package/lib/chain/options.d.ts.map +1 -1
  29. package/lib/chain/options.js +2 -2
  30. package/lib/chain/options.js.map +1 -1
  31. package/lib/chain/stateCache/datastore/db.d.ts +12 -0
  32. package/lib/chain/stateCache/datastore/db.d.ts.map +1 -1
  33. package/lib/chain/stateCache/datastore/db.js +70 -0
  34. package/lib/chain/stateCache/datastore/db.js.map +1 -1
  35. package/lib/chain/stateCache/datastore/file.d.ts +1 -0
  36. package/lib/chain/stateCache/datastore/file.d.ts.map +1 -1
  37. package/lib/chain/stateCache/datastore/file.js +7 -0
  38. package/lib/chain/stateCache/datastore/file.js.map +1 -1
  39. package/lib/chain/stateCache/datastore/types.d.ts +1 -0
  40. package/lib/chain/stateCache/datastore/types.d.ts.map +1 -1
  41. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts +16 -1
  42. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts.map +1 -1
  43. package/lib/chain/stateCache/persistentCheckpointsCache.js +31 -1
  44. package/lib/chain/stateCache/persistentCheckpointsCache.js.map +1 -1
  45. package/lib/chain/validation/blobSidecar.d.ts +4 -1
  46. package/lib/chain/validation/blobSidecar.d.ts.map +1 -1
  47. package/lib/chain/validation/blobSidecar.js +46 -11
  48. package/lib/chain/validation/blobSidecar.js.map +1 -1
  49. package/lib/chain/validation/dataColumnSidecar.d.ts +4 -1
  50. package/lib/chain/validation/dataColumnSidecar.d.ts.map +1 -1
  51. package/lib/chain/validation/dataColumnSidecar.js +64 -19
  52. package/lib/chain/validation/dataColumnSidecar.js.map +1 -1
  53. package/lib/index.d.ts +2 -0
  54. package/lib/index.d.ts.map +1 -1
  55. package/lib/index.js +2 -0
  56. package/lib/index.js.map +1 -1
  57. package/lib/metrics/metrics/lodestar.js +1 -1
  58. package/lib/metrics/metrics/lodestar.js.map +1 -1
  59. package/lib/network/core/networkCore.d.ts.map +1 -1
  60. package/lib/network/core/networkCore.js +5 -1
  61. package/lib/network/core/networkCore.js.map +1 -1
  62. package/lib/network/core/networkCoreWorker.js +8 -8
  63. package/lib/network/core/networkCoreWorker.js.map +1 -1
  64. package/lib/network/core/networkCoreWorkerHandler.js +1 -1
  65. package/lib/network/core/networkCoreWorkerHandler.js.map +1 -1
  66. package/lib/network/discv5/worker.js +2 -7
  67. package/lib/network/discv5/worker.js.map +1 -1
  68. package/lib/network/events.d.ts +1 -0
  69. package/lib/network/events.d.ts.map +1 -1
  70. package/lib/network/gossip/encoding.js +1 -1
  71. package/lib/network/gossip/encoding.js.map +1 -1
  72. package/lib/network/gossip/gossipsub.d.ts.map +1 -1
  73. package/lib/network/gossip/gossipsub.js +6 -1
  74. package/lib/network/gossip/gossipsub.js.map +1 -1
  75. package/lib/network/gossip/interface.d.ts +2 -0
  76. package/lib/network/gossip/interface.d.ts.map +1 -1
  77. package/lib/network/gossip/snappy_bun.d.ts +3 -0
  78. package/lib/network/gossip/snappy_bun.d.ts.map +1 -0
  79. package/lib/network/gossip/snappy_bun.js +3 -0
  80. package/lib/network/gossip/snappy_bun.js.map +1 -0
  81. package/lib/network/metadata.d.ts +1 -1
  82. package/lib/network/metadata.d.ts.map +1 -1
  83. package/lib/network/metadata.js +1 -0
  84. package/lib/network/metadata.js.map +1 -1
  85. package/lib/network/options.d.ts +0 -1
  86. package/lib/network/options.d.ts.map +1 -1
  87. package/lib/network/options.js.map +1 -1
  88. package/lib/network/peers/discover.js +2 -2
  89. package/lib/network/peers/discover.js.map +1 -1
  90. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  91. package/lib/network/processor/gossipHandlers.js +15 -1
  92. package/lib/network/processor/gossipHandlers.js.map +1 -1
  93. package/lib/network/processor/gossipValidatorFn.d.ts.map +1 -1
  94. package/lib/network/processor/gossipValidatorFn.js +3 -2
  95. package/lib/network/processor/gossipValidatorFn.js.map +1 -1
  96. package/lib/network/processor/types.d.ts +2 -0
  97. package/lib/network/processor/types.d.ts.map +1 -1
  98. package/lib/network/reqresp/ReqRespBeaconNode.d.ts.map +1 -1
  99. package/lib/network/reqresp/ReqRespBeaconNode.js +3 -1
  100. package/lib/network/reqresp/ReqRespBeaconNode.js.map +1 -1
  101. package/lib/network/reqresp/handlers/beaconBlocksByRange.d.ts +2 -1
  102. package/lib/network/reqresp/handlers/beaconBlocksByRange.d.ts.map +1 -1
  103. package/lib/network/reqresp/handlers/beaconBlocksByRange.js +14 -3
  104. package/lib/network/reqresp/handlers/beaconBlocksByRange.js.map +1 -1
  105. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.d.ts +2 -1
  106. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.d.ts.map +1 -1
  107. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.js +9 -1
  108. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.js.map +1 -1
  109. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.d.ts +2 -1
  110. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.d.ts.map +1 -1
  111. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js +9 -1
  112. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js.map +1 -1
  113. package/lib/network/reqresp/handlers/index.js +6 -6
  114. package/lib/network/reqresp/handlers/index.js.map +1 -1
  115. package/lib/network/reqresp/types.d.ts +1 -0
  116. package/lib/network/reqresp/types.d.ts.map +1 -1
  117. package/lib/node/nodejs.d.ts +2 -1
  118. package/lib/node/nodejs.d.ts.map +1 -1
  119. package/lib/node/nodejs.js +2 -1
  120. package/lib/node/nodejs.js.map +1 -1
  121. package/lib/sync/range/range.d.ts.map +1 -1
  122. package/lib/sync/range/range.js +2 -1
  123. package/lib/sync/range/range.js.map +1 -1
  124. package/lib/sync/unknownBlock.js +1 -1
  125. package/lib/sync/unknownBlock.js.map +1 -1
  126. package/lib/sync/utils/downloadByRange.d.ts +59 -15
  127. package/lib/sync/utils/downloadByRange.d.ts.map +1 -1
  128. package/lib/sync/utils/downloadByRange.js +204 -83
  129. package/lib/sync/utils/downloadByRange.js.map +1 -1
  130. package/lib/sync/utils/downloadByRoot.d.ts +8 -14
  131. package/lib/sync/utils/downloadByRoot.d.ts.map +1 -1
  132. package/lib/sync/utils/downloadByRoot.js +18 -33
  133. package/lib/sync/utils/downloadByRoot.js.map +1 -1
  134. package/lib/sync/utils/remoteSyncType.d.ts +2 -1
  135. package/lib/sync/utils/remoteSyncType.d.ts.map +1 -1
  136. package/lib/sync/utils/remoteSyncType.js +19 -4
  137. package/lib/sync/utils/remoteSyncType.js.map +1 -1
  138. package/lib/util/blobs.d.ts +1 -1
  139. package/lib/util/blobs.d.ts.map +1 -1
  140. package/lib/util/blobs.js +53 -20
  141. package/lib/util/blobs.js.map +1 -1
  142. package/lib/util/profile.d.ts +6 -4
  143. package/lib/util/profile.d.ts.map +1 -1
  144. package/lib/util/profile.js +40 -3
  145. package/lib/util/profile.js.map +1 -1
  146. package/lib/util/sszBytes.d.ts +2 -0
  147. package/lib/util/sszBytes.d.ts.map +1 -1
  148. package/lib/util/sszBytes.js +25 -0
  149. package/lib/util/sszBytes.js.map +1 -1
  150. package/package.json +32 -25
  151. package/src/api/impl/beacon/blocks/index.ts +47 -25
  152. package/src/api/impl/lodestar/index.ts +42 -10
  153. package/src/api/impl/node/utils.ts +1 -1
  154. package/src/chain/chain.ts +48 -23
  155. package/src/chain/errors/blobSidecarError.ts +12 -2
  156. package/src/chain/errors/dataColumnSidecarError.ts +31 -16
  157. package/src/chain/forkChoice/index.ts +178 -2
  158. package/src/chain/interface.ts +2 -0
  159. package/src/chain/options.ts +2 -3
  160. package/src/chain/stateCache/datastore/db.ts +89 -1
  161. package/src/chain/stateCache/datastore/file.ts +8 -0
  162. package/src/chain/stateCache/datastore/types.ts +1 -0
  163. package/src/chain/stateCache/persistentCheckpointsCache.ts +45 -2
  164. package/src/chain/validation/blobSidecar.ts +54 -10
  165. package/src/chain/validation/dataColumnSidecar.ts +76 -19
  166. package/src/index.ts +2 -0
  167. package/src/metrics/metrics/lodestar.ts +1 -1
  168. package/src/network/core/networkCore.ts +5 -1
  169. package/src/network/core/networkCoreWorker.ts +9 -9
  170. package/src/network/core/networkCoreWorkerHandler.ts +1 -1
  171. package/src/network/discv5/worker.ts +2 -7
  172. package/src/network/events.ts +1 -1
  173. package/src/network/gossip/encoding.ts +1 -1
  174. package/src/network/gossip/gossipsub.ts +7 -1
  175. package/src/network/gossip/interface.ts +2 -0
  176. package/src/network/gossip/snappy_bun.ts +2 -0
  177. package/src/network/metadata.ts +3 -1
  178. package/src/network/options.ts +0 -1
  179. package/src/network/peers/discover.ts +2 -2
  180. package/src/network/processor/gossipHandlers.ts +16 -1
  181. package/src/network/processor/gossipValidatorFn.ts +15 -2
  182. package/src/network/processor/types.ts +2 -0
  183. package/src/network/reqresp/ReqRespBeaconNode.ts +3 -1
  184. package/src/network/reqresp/handlers/beaconBlocksByRange.ts +18 -3
  185. package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts +13 -1
  186. package/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts +13 -1
  187. package/src/network/reqresp/handlers/index.ts +6 -6
  188. package/src/network/reqresp/types.ts +1 -0
  189. package/src/node/nodejs.ts +3 -0
  190. package/src/sync/range/range.ts +2 -1
  191. package/src/sync/unknownBlock.ts +1 -1
  192. package/src/sync/utils/downloadByRange.ts +273 -108
  193. package/src/sync/utils/downloadByRoot.ts +22 -56
  194. package/src/sync/utils/remoteSyncType.ts +23 -4
  195. package/src/util/blobs.ts +64 -20
  196. package/src/util/profile.ts +45 -3
  197. package/src/util/sszBytes.ts +30 -0
@@ -1,7 +1,14 @@
1
1
  import {ChainForkConfig} from "@lodestar/config";
2
- import {ForkPostDeneb, ForkPostFulu, ForkPreFulu, ForkPreGloas} from "@lodestar/params";
2
+ import {
3
+ ForkPostDeneb,
4
+ ForkPostFulu,
5
+ ForkPreFulu,
6
+ ForkPreGloas,
7
+ isForkPostFulu,
8
+ isForkPostGloas,
9
+ } from "@lodestar/params";
3
10
  import {SignedBeaconBlock, Slot, deneb, fulu, phase0} from "@lodestar/types";
4
- import {LodestarError, Logger, fromHex, prettyBytes, prettyPrintIndices, toRootHex} from "@lodestar/utils";
11
+ import {LodestarError, Logger, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
5
12
  import {
6
13
  BlockInputSource,
7
14
  DAType,
@@ -15,7 +22,6 @@ import {validateBlockDataColumnSidecars} from "../../chain/validation/dataColumn
15
22
  import {INetwork} from "../../network/index.js";
16
23
  import {PeerIdStr} from "../../util/peerId.js";
17
24
  import {WarnResult} from "../../util/wrapError.js";
18
- import {DownloadByRootErrorCode} from "./downloadByRoot.js";
19
25
 
20
26
  export type DownloadByRangeRequests = {
21
27
  blocksRequest?: phase0.BeaconBlocksByRangeRequest;
@@ -31,7 +37,6 @@ export type DownloadByRangeResponses = {
31
37
 
32
38
  export type DownloadAndCacheByRangeProps = DownloadByRangeRequests & {
33
39
  config: ChainForkConfig;
34
- cache: SeenBlockInput;
35
40
  network: INetwork;
36
41
  logger: Logger;
37
42
  peerIdStr: string;
@@ -111,7 +116,13 @@ export function cacheByRangeResponses({
111
116
  }
112
117
 
113
118
  for (const {blockRoot, blobSidecars} of responses.validatedBlobSidecars ?? []) {
114
- const existing = updatedBatchBlocks.get(blobSidecars[0].signedBlockHeader.message.slot);
119
+ const dataSlot = blobSidecars.at(0)?.signedBlockHeader.message.slot;
120
+ if (dataSlot === undefined) {
121
+ throw new Error(
122
+ `Coding Error: empty blobSidecars returned for blockRoot=${toRootHex(blockRoot)} from validation functions`
123
+ );
124
+ }
125
+ const existing = updatedBatchBlocks.get(dataSlot);
115
126
  const blockRootHex = toRootHex(blockRoot);
116
127
 
117
128
  if (!existing) {
@@ -122,7 +133,7 @@ export function cacheByRangeResponses({
122
133
  throw new DownloadByRangeError({
123
134
  code: DownloadByRangeErrorCode.MISMATCH_BLOCK_INPUT_TYPE,
124
135
  slot: existing.slot,
125
- blockRoot: prettyBytes(existing.blockRootHex),
136
+ blockRoot: existing.blockRootHex,
126
137
  expected: DAType.Blobs,
127
138
  actual: existing.type,
128
139
  });
@@ -143,18 +154,24 @@ export function cacheByRangeResponses({
143
154
  }
144
155
 
145
156
  for (const {blockRoot, columnSidecars} of responses.validatedColumnSidecars ?? []) {
146
- const existing = updatedBatchBlocks.get(columnSidecars[0].signedBlockHeader.message.slot);
157
+ const dataSlot = columnSidecars.at(0)?.signedBlockHeader.message.slot;
158
+ if (dataSlot === undefined) {
159
+ throw new Error(
160
+ `Coding Error: empty columnSidecars returned for blockRoot=${toRootHex(blockRoot)} from validation functions`
161
+ );
162
+ }
163
+ const existing = updatedBatchBlocks.get(dataSlot);
147
164
  const blockRootHex = toRootHex(blockRoot);
148
165
 
149
166
  if (!existing) {
150
- throw new Error("Coding error: blockInput must exist when adding blobs");
167
+ throw new Error("Coding error: blockInput must exist when adding columns");
151
168
  }
152
169
 
153
170
  if (!isBlockInputColumns(existing)) {
154
171
  throw new DownloadByRangeError({
155
172
  code: DownloadByRangeErrorCode.MISMATCH_BLOCK_INPUT_TYPE,
156
173
  slot: existing.slot,
157
- blockRoot: prettyBytes(existing.blockRootHex),
174
+ blockRoot: existing.blockRootHex,
158
175
  expected: DAType.Columns,
159
176
  actual: existing.type,
160
177
  });
@@ -185,7 +202,7 @@ export async function downloadByRange({
185
202
  blocksRequest,
186
203
  blobsRequest,
187
204
  columnsRequest,
188
- }: Omit<DownloadAndCacheByRangeProps, "cache">): Promise<WarnResult<ValidatedResponses, DownloadByRangeError>> {
205
+ }: DownloadAndCacheByRangeProps): Promise<WarnResult<ValidatedResponses, DownloadByRangeError>> {
189
206
  let response: DownloadByRangeResponses;
190
207
  try {
191
208
  response = await requestByRange({
@@ -290,7 +307,7 @@ export async function validateResponses({
290
307
  if ((blobsRequest || columnsRequest) && !(blocks || batchBlocks)) {
291
308
  throw new DownloadByRangeError(
292
309
  {
293
- code: DownloadByRangeErrorCode.MISSING_BLOCKS,
310
+ code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
294
311
  ...requestsLogMeta({blobsRequest, columnsRequest}),
295
312
  },
296
313
  "No blocks to validate data requests against"
@@ -301,24 +318,28 @@ export async function validateResponses({
301
318
  let warnings: DownloadByRangeError[] | null = null;
302
319
 
303
320
  if (blocksRequest) {
304
- validatedResponses.validatedBlocks = validateBlockByRangeResponse(config, blocksRequest, blocks ?? []);
321
+ const result = validateBlockByRangeResponse(config, blocksRequest, blocks ?? []);
322
+ if (result.warnings?.length) {
323
+ warnings = result.warnings;
324
+ }
325
+ validatedResponses.validatedBlocks = result.result;
305
326
  }
306
327
 
307
328
  const dataRequest = blobsRequest ?? columnsRequest;
308
329
  if (!dataRequest) {
309
- return {result: validatedResponses, warnings: null};
330
+ return {result: validatedResponses, warnings};
310
331
  }
311
332
 
312
- const dataRequestBlocks = getBlocksForDataValidation(
333
+ const blocksForDataValidation = getBlocksForDataValidation(
313
334
  dataRequest,
314
335
  batchBlocks,
315
- blocksRequest ? validatedResponses.validatedBlocks : undefined
336
+ validatedResponses.validatedBlocks?.length ? validatedResponses.validatedBlocks : undefined
316
337
  );
317
338
 
318
- if (!dataRequestBlocks.length) {
339
+ if (!blocksForDataValidation.length) {
319
340
  throw new DownloadByRangeError(
320
341
  {
321
- code: DownloadByRangeErrorCode.MISSING_BLOCKS,
342
+ code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
322
343
  ...requestsLogMeta({blobsRequest, columnsRequest}),
323
344
  },
324
345
  "No blocks in data request slot range to validate data response against"
@@ -336,7 +357,10 @@ export async function validateResponses({
336
357
  );
337
358
  }
338
359
 
339
- validatedResponses.validatedBlobSidecars = await validateBlobsByRangeResponse(dataRequestBlocks, blobSidecars);
360
+ validatedResponses.validatedBlobSidecars = await validateBlobsByRangeResponse(
361
+ blocksForDataValidation,
362
+ blobSidecars
363
+ );
340
364
  }
341
365
 
342
366
  if (columnsRequest) {
@@ -351,8 +375,9 @@ export async function validateResponses({
351
375
  }
352
376
 
353
377
  const validatedColumnSidecarsResult = await validateColumnsByRangeResponse(
378
+ config,
354
379
  columnsRequest,
355
- dataRequestBlocks,
380
+ blocksForDataValidation,
356
381
  columnSidecars
357
382
  );
358
383
  validatedResponses.validatedColumnSidecars = validatedColumnSidecarsResult.result;
@@ -375,20 +400,30 @@ export function validateBlockByRangeResponse(
375
400
  config: ChainForkConfig,
376
401
  blocksRequest: phase0.BeaconBlocksByRangeRequest,
377
402
  blocks: SignedBeaconBlock[]
378
- ): ValidatedBlock[] {
403
+ ): WarnResult<ValidatedBlock[], DownloadByRangeError> {
379
404
  const {startSlot, count} = blocksRequest;
380
405
 
381
- // TODO(fulu): This was added by @twoeths in #8150 but it breaks for epochs with 0 blocks during chain
382
- // liveness issues. See comment https://github.com/ChainSafe/lodestar/issues/8147#issuecomment-3246434697
383
- // if (!blocks.length) {
384
- // throw new DownloadByRangeError(
385
- // {
386
- // code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
387
- // expectedCount: blocksRequest.count,
388
- // },
389
- // "Zero blocks in response"
390
- // );
391
- // }
406
+ // An error was thrown here by @twoeths in #8150 but it breaks for epochs with 0 blocks during chain
407
+ // liveness issues. See comment https://github.com/ChainSafe/lodestar/issues/8147#issuecomment-3246434697
408
+ // There are instances where clients return no blocks though. Need to monitor this via the warns to see
409
+ // if what the correct behavior should be
410
+ if (!blocks.length) {
411
+ throw new DownloadByRangeError({
412
+ code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
413
+ ...requestsLogMeta({blocksRequest}),
414
+ });
415
+ // TODO: this was causing deadlock again. need to come back and fix this so that its possible to process through
416
+ // an empty epoch for periods with poor liveness
417
+ // return {
418
+ // result: [],
419
+ // warnings: [
420
+ // new DownloadByRangeError({
421
+ // code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
422
+ // ...requestsLogMeta({blocksRequest}),
423
+ // }),
424
+ // ],
425
+ // };
426
+ }
392
427
 
393
428
  if (blocks.length > count) {
394
429
  throw new DownloadByRangeError(
@@ -445,8 +480,8 @@ export function validateBlockByRangeResponse(
445
480
  {
446
481
  code: DownloadByRangeErrorCode.PARENT_ROOT_MISMATCH,
447
482
  slot: blocks[i].message.slot,
448
- expected: prettyBytes(blockRoot),
449
- actual: prettyBytes(parentRoot),
483
+ expected: toRootHex(blockRoot),
484
+ actual: toRootHex(parentRoot),
450
485
  },
451
486
  `Block parent root does not match the previous block's root in BeaconBlocksByRange response`
452
487
  );
@@ -454,7 +489,10 @@ export function validateBlockByRangeResponse(
454
489
  }
455
490
  }
456
491
 
457
- return response;
492
+ return {
493
+ result: response,
494
+ warnings: null,
495
+ };
458
496
  }
459
497
 
460
498
  /**
@@ -516,9 +554,13 @@ export async function validateBlobsByRangeResponse(
516
554
  }
517
555
 
518
556
  validateSidecarsPromises.push(
519
- validateBlockBlobSidecars(block.message.slot, blockRoot, blockKzgCommitments.length, blockBlobSidecars).then(
520
- () => ({blockRoot, blobSidecars: blockBlobSidecars})
521
- )
557
+ validateBlockBlobSidecars(
558
+ null, // do not pass chain here so we do not validate header signature
559
+ block.message.slot,
560
+ blockRoot,
561
+ blockKzgCommitments.length,
562
+ blockBlobSidecars
563
+ ).then(() => ({blockRoot, blobSidecars: blockBlobSidecars}))
522
564
  );
523
565
  }
524
566
 
@@ -528,76 +570,184 @@ export async function validateBlobsByRangeResponse(
528
570
 
529
571
  /**
530
572
  * Should not be called directly. Only exported for unit testing purposes
573
+ *
574
+ * Spec states:
575
+ * 1) must be within range [start_slot, start_slot + count]
576
+ * 2) should respond with all columns in the range or and 3:ResourceUnavailable (and potentially get down-scored)
577
+ * 3) must response with at least the sidecars of the first blob-carrying block that exists in the range
578
+ * 4) must include all sidecars from each block from which there are blobs
579
+ * 5) where they exists, sidecars must be sent in (slot, index) order
580
+ * 6) clients may limit the number of sidecars in a response
581
+ * 7) clients may stop responding mid-response if their view of fork-choice changes
582
+ *
583
+ * We will interpret the spec as follows
584
+ * - Errors when validating: 1, 3, 5
585
+ * - Warnings when validating: 2, 4, 6, 7
586
+ *
587
+ * For "warning" cases, where we get a partial response but sidecars are validated and correct with respect to the
588
+ * blocks, then they will be kept. This loosening of the spec is to help ensure sync goes smoothly and we can find
589
+ * the data needed in difficult network situations.
590
+ *
591
+ * Assume for the following two examples we request indices 5, 10, 15 for a range of slots 32-63
592
+ *
593
+ * For slots where we receive no sidecars, example slot 45, but blobs exist we will stop validating subsequent
594
+ * slots, 45-63. The next round of requests will get structured to pull the from the slot that had columns
595
+ * missing to the end of the range for all columns indices that were requested for the current partially failed
596
+ * request (slots 45-63 and indices 5, 10, 15).
597
+ *
598
+ * For slots where only some of the requested sidecars are received we will proceed with validation. For simplicity sake
599
+ * we will assume that if we only get some indices back for a (or several) slot(s) that the indices we get will be
600
+ * consistent. IE if a peer returns only index 5, they will most likely return that same index for subsequent slot
601
+ * (index 5 for slots 34, 35, 36, etc). They will not likely return 5 on slot 34, 10 on slot 35, 15 on slot 36, etc.
602
+ * This assumption makes the code simpler. For both cases the request for the next round will be structured correctly
603
+ * to pull any missing column indices for whatever range remains. The simplification just leads to re-verification
604
+ * of the columns but the number of columns downloaded will be the same regardless of if they are validated twice.
605
+ *
606
+ * validateColumnsByRangeResponse makes some assumptions about the data being passed in
607
+ * blocks are:
608
+ * - slotwise in order
609
+ * - form a chain
610
+ * - non-sparse response (any missing block is a skipped slot not a bad response)
611
+ * - last block is last slot received
531
612
  */
532
613
  export async function validateColumnsByRangeResponse(
614
+ config: ChainForkConfig,
533
615
  request: fulu.DataColumnSidecarsByRangeRequest,
534
- dataRequestBlocks: ValidatedBlock[],
616
+ blocks: ValidatedBlock[],
535
617
  columnSidecars: fulu.DataColumnSidecars
536
618
  ): Promise<WarnResult<ValidatedColumnSidecars[], DownloadByRangeError>> {
537
- // Expected column count considering currently-validated batch blocks
538
- // TODO GLOAS: Post-gloas's blobKzgCommitments is not in beacon block body. Need to source it from somewhere else.
539
- const expectedColumnCount = dataRequestBlocks.reduce((acc, {block}) => {
540
- return (block as SignedBeaconBlock<ForkPostDeneb & ForkPreGloas>).message.body.blobKzgCommitments.length > 0
541
- ? request.columns.length + acc
542
- : acc;
543
- }, 0);
544
- const nextSlot = dataRequestBlocks.length
545
- ? (dataRequestBlocks.at(-1) as ValidatedBlock).block.message.slot + 1
546
- : request.startSlot;
547
- const possiblyMissingBlocks = nextSlot - request.startSlot + request.count;
548
-
549
- // Allow for extra columns if some blocks are missing from the end of a batch
550
- // Eg: If we requested 10 blocks but only 8 were returned, allow for up to 2 * columns.length extra columns
551
- const maxColumnCount = expectedColumnCount + possiblyMissingBlocks * request.columns.length;
552
-
553
- if (columnSidecars.length > maxColumnCount) {
554
- // this never happens on devnet, so throw error for now
555
- throw new DownloadByRangeError(
556
- {
557
- code: DownloadByRangeErrorCode.OVER_COLUMNS,
558
- max: maxColumnCount,
559
- actual: columnSidecars.length,
560
- },
561
- "Extra data columns received in DataColumnSidecarsByRange response"
562
- );
563
- }
564
-
565
619
  const warnings: DownloadByRangeError[] = [];
566
- // no need to check for columnSidecars.length vs expectedColumnCount here, will be checked per-block below
567
- const requestedColumns = new Set(request.columns);
568
- const validateSidecarsPromises: Promise<ValidatedColumnSidecars>[] = [];
569
- for (let blockIndex = 0, columnSidecarIndex = 0; blockIndex < dataRequestBlocks.length; blockIndex++) {
570
- const {block, blockRoot} = dataRequestBlocks[blockIndex];
571
- const slot = block.message.slot;
572
- const blockRootHex = toRootHex(blockRoot);
573
- // TODO GLOAS: Post-gloas's blobKzgCommitments is not in beacon block body. Need to source it from somewhere else.
574
- const blockKzgCommitments = (block as SignedBeaconBlock<ForkPostFulu & ForkPreGloas>).message.body
575
- .blobKzgCommitments;
576
- const expectedColumns = blockKzgCommitments.length ? request.columns.length : 0;
577
620
 
578
- if (expectedColumns === 0) {
621
+ const seenColumns = new Map<Slot, Map<number, fulu.DataColumnSidecar>>();
622
+ let currentSlot = -1;
623
+ let currentIndex = -1;
624
+ // Check for duplicates and order
625
+ for (const columnSidecar of columnSidecars) {
626
+ const slot = columnSidecar.signedBlockHeader.message.slot;
627
+ let seenSlotColumns = seenColumns.get(slot);
628
+ if (!seenSlotColumns) {
629
+ seenSlotColumns = new Map();
630
+ seenColumns.set(slot, seenSlotColumns);
631
+ }
632
+
633
+ if (seenSlotColumns.has(columnSidecar.index)) {
634
+ warnings.push(
635
+ new DownloadByRangeError({
636
+ code: DownloadByRangeErrorCode.DUPLICATE_COLUMN,
637
+ slot,
638
+ index: columnSidecar.index,
639
+ })
640
+ );
641
+
579
642
  continue;
580
643
  }
581
- const blockColumnSidecars: fulu.DataColumnSidecar[] = [];
582
- while (columnSidecarIndex < columnSidecars.length) {
583
- const columnSidecar = columnSidecars[columnSidecarIndex];
584
- if (columnSidecar.signedBlockHeader.message.slot !== block.message.slot) {
585
- // We've reached columns for the next block
586
- break;
644
+
645
+ if (currentSlot > slot) {
646
+ warnings.push(
647
+ new DownloadByRangeError(
648
+ {
649
+ code: DownloadByRangeErrorCode.OUT_OF_ORDER_COLUMNS,
650
+ slot,
651
+ },
652
+ "ColumnSidecars received out of slot order"
653
+ )
654
+ );
655
+ }
656
+
657
+ if (currentSlot === slot && currentIndex > columnSidecar.index) {
658
+ warnings.push(
659
+ new DownloadByRangeError(
660
+ {
661
+ code: DownloadByRangeErrorCode.OUT_OF_ORDER_COLUMNS,
662
+ slot,
663
+ },
664
+ "Column indices out of order within a slot"
665
+ )
666
+ );
667
+ }
668
+
669
+ seenSlotColumns.set(columnSidecar.index, columnSidecar);
670
+ if (currentSlot !== slot) {
671
+ // a new slot has started, reset index
672
+ currentIndex = -1;
673
+ } else {
674
+ currentIndex = columnSidecar.index;
675
+ }
676
+ currentSlot = slot;
677
+ }
678
+
679
+ const validationPromises: Promise<ValidatedColumnSidecars>[] = [];
680
+
681
+ for (const {blockRoot, block} of blocks) {
682
+ const slot = block.message.slot;
683
+ const rootHex = toRootHex(blockRoot);
684
+ const forkName = config.getForkName(slot);
685
+ const columnSidecarsMap: Map<number, fulu.DataColumnSidecar> = seenColumns.get(slot) ?? new Map();
686
+ const columnSidecars = Array.from(columnSidecarsMap.values()).sort((a, b) => a.index - b.index);
687
+
688
+ let blobCount: number;
689
+ if (!isForkPostFulu(forkName)) {
690
+ const dataSlot = columnSidecars.at(0)?.signedBlockHeader.message.slot;
691
+ throw new DownloadByRangeError({
692
+ code: DownloadByRangeErrorCode.MISMATCH_BLOCK_FORK,
693
+ slot,
694
+ blockFork: forkName,
695
+ dataFork: dataSlot ? config.getForkName(dataSlot) : "unknown",
696
+ });
697
+ }
698
+ if (isForkPostGloas(forkName)) {
699
+ // TODO GLOAS: Post-gloas's blobKzgCommitments is not in beacon block body. Need to source it from somewhere else.
700
+ // if block without columns is passed default to zero and throw below
701
+ blobCount = 0;
702
+ } else {
703
+ blobCount = (block as SignedBeaconBlock<ForkPostFulu & ForkPreGloas>).message.body.blobKzgCommitments.length;
704
+ }
705
+
706
+ if (columnSidecars.length === 0) {
707
+ if (!blobCount) {
708
+ // no columns in the slot
709
+ continue;
587
710
  }
588
- blockColumnSidecars.push(columnSidecar);
589
- columnSidecarIndex++;
711
+
712
+ /**
713
+ * If no columns are found for a block and there are commitments on the block then stop checking and just
714
+ * return early. Even if there were columns returned for subsequent slots that doesn't matter because
715
+ * we will be re-requesting them again anyway. Leftovers just get ignored
716
+ */
717
+ warnings.push(
718
+ new DownloadByRangeError({
719
+ code: DownloadByRangeErrorCode.MISSING_COLUMNS,
720
+ slot,
721
+ blockRoot: rootHex,
722
+ missingIndices: prettyPrintIndices(request.columns),
723
+ })
724
+ );
725
+ break;
726
+ }
727
+
728
+ const returnedColumns = Array.from(columnSidecarsMap.keys()).sort();
729
+ if (!blobCount) {
730
+ // columns for a block that does not have blobs
731
+ // TODO(fulu): should this be a hard error with no data retained from peer or just a warning
732
+ throw new DownloadByRangeError(
733
+ {
734
+ code: DownloadByRangeErrorCode.NO_COLUMNS_FOR_BLOCK,
735
+ slot,
736
+ blockRoot: rootHex,
737
+ invalidIndices: prettyPrintIndices(returnedColumns),
738
+ },
739
+ "Block has no blob commitments but data column sidecars were provided"
740
+ );
590
741
  }
591
742
 
592
- const returnedColumns = new Set(blockColumnSidecars.map((c) => c.index));
593
- const missingIndices = request.columns.filter((i) => !returnedColumns.has(i));
743
+ const missingIndices = request.columns.filter((i) => !columnSidecarsMap.has(i));
594
744
  if (missingIndices.length > 0) {
595
745
  warnings.push(
596
746
  new DownloadByRangeError(
597
747
  {
598
748
  code: DownloadByRangeErrorCode.MISSING_COLUMNS,
599
749
  slot,
600
- blockRoot: blockRootHex,
750
+ blockRoot: rootHex,
601
751
  missingIndices: prettyPrintIndices(missingIndices),
602
752
  },
603
753
  "Missing data columns in DataColumnSidecarsByRange response"
@@ -605,14 +755,14 @@ export async function validateColumnsByRangeResponse(
605
755
  );
606
756
  }
607
757
 
608
- const extraIndices = [...returnedColumns].filter((i) => !requestedColumns.has(i));
758
+ const extraIndices = returnedColumns.filter((i) => !request.columns.includes(i));
609
759
  if (extraIndices.length > 0) {
610
760
  warnings.push(
611
761
  new DownloadByRangeError(
612
762
  {
613
763
  code: DownloadByRangeErrorCode.EXTRA_COLUMNS,
614
764
  slot,
615
- blockRoot: blockRootHex,
765
+ blockRoot: rootHex,
616
766
  invalidIndices: prettyPrintIndices(extraIndices),
617
767
  },
618
768
  "Data column in not in requested columns in DataColumnSidecarsByRange response"
@@ -620,17 +770,25 @@ export async function validateColumnsByRangeResponse(
620
770
  );
621
771
  }
622
772
 
623
- validateSidecarsPromises.push(
624
- validateBlockDataColumnSidecars(slot, blockRoot, blockKzgCommitments.length, blockColumnSidecars).then(() => ({
773
+ validationPromises.push(
774
+ validateBlockDataColumnSidecars(
775
+ null, // do not pass chain here so we do not validate header signature
776
+ slot,
625
777
  blockRoot,
626
- columnSidecars: blockColumnSidecars,
778
+ blobCount,
779
+ columnSidecars
780
+ ).then(() => ({
781
+ blockRoot,
782
+ columnSidecars,
627
783
  }))
628
784
  );
629
785
  }
630
786
 
631
- // Await all sidecar validations in parallel
632
- const result = await Promise.all(validateSidecarsPromises);
633
- return {result, warnings: warnings.length ? warnings : null};
787
+ const validatedColumns = await Promise.all(validationPromises);
788
+ return {
789
+ result: validatedColumns,
790
+ warnings: warnings.length ? warnings : null,
791
+ };
634
792
  }
635
793
 
636
794
  /**
@@ -697,7 +855,7 @@ function requestsLogMeta({blocksRequest, blobsRequest, columnsRequest}: Download
697
855
  }
698
856
 
699
857
  export enum DownloadByRangeErrorCode {
700
- MISSING_BLOCKS = "DOWNLOAD_BY_RANGE_ERROR_MISSING_BLOCKS",
858
+ MISSING_BLOCKS_RESPONSE = "DOWNLOAD_BY_RANGE_ERROR_MISSING_BLOCK_RESPONSE",
701
859
  MISSING_BLOBS_RESPONSE = "DOWNLOAD_BY_RANGE_ERROR_MISSING_BLOBS_RESPONSE",
702
860
  MISSING_COLUMNS_RESPONSE = "DOWNLOAD_BY_RANGE_ERROR_MISSING_COLUMNS_RESPONSE",
703
861
 
@@ -718,19 +876,19 @@ export enum DownloadByRangeErrorCode {
718
876
  MISSING_COLUMNS = "DOWNLOAD_BY_RANGE_ERROR_MISSING_COLUMNS",
719
877
  OVER_COLUMNS = "DOWNLOAD_BY_RANGE_ERROR_OVER_COLUMNS",
720
878
  EXTRA_COLUMNS = "DOWNLOAD_BY_RANGE_ERROR_EXTRA_COLUMNS",
879
+ NO_COLUMNS_FOR_BLOCK = "DOWNLOAD_BY_RANGE_ERROR_NO_COLUMNS_FOR_BLOCK",
880
+ DUPLICATE_COLUMN = "DOWNLOAD_BY_RANGE_ERROR_DUPLICATE_COLUMN",
881
+ OUT_OF_ORDER_COLUMNS = "DOWNLOAD_BY_RANGE_OUT_OF_ORDER_COLUMNS",
721
882
 
722
883
  /** Cached block input type mismatches new data */
884
+ MISMATCH_BLOCK_FORK = "DOWNLOAD_BY_RANGE_ERROR_MISMATCH_BLOCK_FORK",
723
885
  MISMATCH_BLOCK_INPUT_TYPE = "DOWNLOAD_BY_RANGE_ERROR_MISMATCH_BLOCK_INPUT_TYPE",
724
886
  }
725
887
 
726
888
  export type DownloadByRangeErrorType =
727
- | {
728
- code: DownloadByRootErrorCode.MISSING_BLOCK_RESPONSE;
729
- expectedCount: number;
730
- }
731
889
  | {
732
890
  code:
733
- | DownloadByRangeErrorCode.MISSING_BLOCKS
891
+ | DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE
734
892
  | DownloadByRangeErrorCode.MISSING_BLOBS_RESPONSE
735
893
  | DownloadByRangeErrorCode.MISSING_COLUMNS_RESPONSE;
736
894
  blockStartSlot?: number;
@@ -741,12 +899,14 @@ export type DownloadByRangeErrorType =
741
899
  columnCount?: number;
742
900
  }
743
901
  | {
744
- code: DownloadByRootErrorCode.MISSING_BLOCK_RESPONSE;
745
- expectedCount: number;
902
+ code: DownloadByRangeErrorCode.OUT_OF_RANGE_BLOCKS;
903
+ slot: number;
746
904
  }
747
905
  | {
748
- code: DownloadByRangeErrorCode.OUT_OF_RANGE_BLOCKS;
906
+ code: DownloadByRangeErrorCode.MISMATCH_BLOCK_FORK;
749
907
  slot: number;
908
+ dataFork: string;
909
+ blockFork: string;
750
910
  }
751
911
  | {
752
912
  code: DownloadByRangeErrorCode.OUT_OF_ORDER_BLOCKS;
@@ -778,7 +938,7 @@ export type DownloadByRangeErrorType =
778
938
  actual: number;
779
939
  }
780
940
  | {
781
- code: DownloadByRangeErrorCode.OUT_OF_ORDER_BLOBS;
941
+ code: DownloadByRangeErrorCode.OUT_OF_ORDER_BLOBS | DownloadByRangeErrorCode.OUT_OF_ORDER_COLUMNS;
782
942
  slot: number;
783
943
  }
784
944
  | {
@@ -798,7 +958,12 @@ export type DownloadByRangeErrorType =
798
958
  missingIndices: string;
799
959
  }
800
960
  | {
801
- code: DownloadByRangeErrorCode.EXTRA_COLUMNS;
961
+ code: DownloadByRangeErrorCode.DUPLICATE_COLUMN;
962
+ slot: Slot;
963
+ index: number;
964
+ }
965
+ | {
966
+ code: DownloadByRangeErrorCode.EXTRA_COLUMNS | DownloadByRangeErrorCode.NO_COLUMNS_FOR_BLOCK;
802
967
  slot: Slot;
803
968
  blockRoot: string;
804
969
  invalidIndices: string;