@lodestar/beacon-node 1.49.0-dev.a261342520 → 1.49.0-dev.a56f18f2ea

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/lib/api/impl/beacon/blocks/index.js +2 -2
  2. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  3. package/lib/api/impl/beacon/pool/index.d.ts.map +1 -1
  4. package/lib/api/impl/beacon/pool/index.js +4 -2
  5. package/lib/api/impl/beacon/pool/index.js.map +1 -1
  6. package/lib/api/impl/validator/index.d.ts.map +1 -1
  7. package/lib/api/impl/validator/index.js +9 -7
  8. package/lib/api/impl/validator/index.js.map +1 -1
  9. package/lib/chain/archiveStore/archiveStore.d.ts +1 -2
  10. package/lib/chain/archiveStore/archiveStore.d.ts.map +1 -1
  11. package/lib/chain/archiveStore/archiveStore.js +0 -5
  12. package/lib/chain/archiveStore/archiveStore.js.map +1 -1
  13. package/lib/chain/archiveStore/historicalState/metrics.d.ts.map +1 -1
  14. package/lib/chain/archiveStore/historicalState/metrics.js +4 -6
  15. package/lib/chain/archiveStore/historicalState/metrics.js.map +1 -1
  16. package/lib/chain/archiveStore/utils/pruneHistory.d.ts.map +1 -1
  17. package/lib/chain/archiveStore/utils/pruneHistory.js +14 -3
  18. package/lib/chain/archiveStore/utils/pruneHistory.js.map +1 -1
  19. package/lib/chain/blocks/importExecutionPayload.d.ts.map +1 -1
  20. package/lib/chain/blocks/importExecutionPayload.js +13 -0
  21. package/lib/chain/blocks/importExecutionPayload.js.map +1 -1
  22. package/lib/chain/blocks/index.js +2 -2
  23. package/lib/chain/blocks/index.js.map +1 -1
  24. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts +1 -1
  25. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts.map +1 -1
  26. package/lib/chain/emitter.d.ts +10 -0
  27. package/lib/chain/emitter.d.ts.map +1 -1
  28. package/lib/chain/emitter.js +4 -0
  29. package/lib/chain/emitter.js.map +1 -1
  30. package/lib/chain/errors/attesterSlashingError.d.ts +4 -1
  31. package/lib/chain/errors/attesterSlashingError.d.ts.map +1 -1
  32. package/lib/chain/errors/attesterSlashingError.js +1 -0
  33. package/lib/chain/errors/attesterSlashingError.js.map +1 -1
  34. package/lib/chain/errors/payloadAttestation.d.ts +4 -0
  35. package/lib/chain/errors/payloadAttestation.d.ts.map +1 -1
  36. package/lib/chain/errors/payloadAttestation.js +1 -0
  37. package/lib/chain/errors/payloadAttestation.js.map +1 -1
  38. package/lib/chain/errors/proposerSlashingError.d.ts +4 -1
  39. package/lib/chain/errors/proposerSlashingError.d.ts.map +1 -1
  40. package/lib/chain/errors/proposerSlashingError.js +1 -0
  41. package/lib/chain/errors/proposerSlashingError.js.map +1 -1
  42. package/lib/chain/options.d.ts +0 -4
  43. package/lib/chain/options.d.ts.map +1 -1
  44. package/lib/chain/options.js +0 -1
  45. package/lib/chain/options.js.map +1 -1
  46. package/lib/chain/produceBlock/computeNewStateRoot.d.ts.map +1 -1
  47. package/lib/chain/produceBlock/computeNewStateRoot.js +4 -2
  48. package/lib/chain/produceBlock/computeNewStateRoot.js.map +1 -1
  49. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  50. package/lib/chain/produceBlock/produceBlockBody.js +6 -6
  51. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  52. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts +2 -1
  53. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts.map +1 -1
  54. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js +3 -2
  55. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js.map +1 -1
  56. package/lib/chain/validation/attesterSlashing.d.ts.map +1 -1
  57. package/lib/chain/validation/attesterSlashing.js +1 -2
  58. package/lib/chain/validation/attesterSlashing.js.map +1 -1
  59. package/lib/chain/validation/payloadAttestationMessage.d.ts.map +1 -1
  60. package/lib/chain/validation/payloadAttestationMessage.js +8 -1
  61. package/lib/chain/validation/payloadAttestationMessage.js.map +1 -1
  62. package/lib/chain/validation/proposerSlashing.js +1 -2
  63. package/lib/chain/validation/proposerSlashing.js.map +1 -1
  64. package/lib/chain/validatorMonitor.d.ts.map +1 -1
  65. package/lib/chain/validatorMonitor.js +2 -2
  66. package/lib/chain/validatorMonitor.js.map +1 -1
  67. package/lib/db/beacon.d.ts +1 -2
  68. package/lib/db/beacon.d.ts.map +1 -1
  69. package/lib/db/beacon.js +1 -3
  70. package/lib/db/beacon.js.map +1 -1
  71. package/lib/db/buckets.d.ts +0 -1
  72. package/lib/db/buckets.d.ts.map +1 -1
  73. package/lib/db/buckets.js +1 -1
  74. package/lib/db/buckets.js.map +1 -1
  75. package/lib/db/interface.d.ts +1 -2
  76. package/lib/db/interface.d.ts.map +1 -1
  77. package/lib/db/repositories/blockArchive.d.ts +5 -0
  78. package/lib/db/repositories/blockArchive.d.ts.map +1 -1
  79. package/lib/db/repositories/blockArchive.js +36 -1
  80. package/lib/db/repositories/blockArchive.js.map +1 -1
  81. package/lib/db/repositories/blockArchiveIndex.d.ts +2 -2
  82. package/lib/db/repositories/blockArchiveIndex.d.ts.map +1 -1
  83. package/lib/db/repositories/index.d.ts +0 -1
  84. package/lib/db/repositories/index.d.ts.map +1 -1
  85. package/lib/db/repositories/index.js +0 -1
  86. package/lib/db/repositories/index.js.map +1 -1
  87. package/lib/metrics/metrics/lodestar.d.ts +12 -11
  88. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  89. package/lib/metrics/metrics/lodestar.js +25 -23
  90. package/lib/metrics/metrics/lodestar.js.map +1 -1
  91. package/lib/network/gossip/encoding.d.ts +1 -2
  92. package/lib/network/gossip/encoding.d.ts.map +1 -1
  93. package/lib/network/gossip/encoding.js +5 -10
  94. package/lib/network/gossip/encoding.js.map +1 -1
  95. package/lib/network/gossip/gossipsub.d.ts.map +1 -1
  96. package/lib/network/gossip/gossipsub.js +16 -1
  97. package/lib/network/gossip/gossipsub.js.map +1 -1
  98. package/lib/network/gossip/topic.d.ts +9 -6
  99. package/lib/network/gossip/topic.d.ts.map +1 -1
  100. package/lib/network/gossip/topic.js +23 -8
  101. package/lib/network/gossip/topic.js.map +1 -1
  102. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  103. package/lib/network/processor/gossipHandlers.js +16 -5
  104. package/lib/network/processor/gossipHandlers.js.map +1 -1
  105. package/lib/network/processor/gossipValidatorFn.d.ts +2 -0
  106. package/lib/network/processor/gossipValidatorFn.d.ts.map +1 -1
  107. package/lib/network/processor/gossipValidatorFn.js +88 -7
  108. package/lib/network/processor/gossipValidatorFn.js.map +1 -1
  109. package/lib/network/processor/index.d.ts +6 -2
  110. package/lib/network/processor/index.d.ts.map +1 -1
  111. package/lib/network/processor/index.js +62 -24
  112. package/lib/network/processor/index.js.map +1 -1
  113. package/lib/network/reqresp/protocols.d.ts.map +1 -1
  114. package/lib/network/reqresp/protocols.js +19 -5
  115. package/lib/network/reqresp/protocols.js.map +1 -1
  116. package/lib/node/nodejs.d.ts +2 -7
  117. package/lib/node/nodejs.d.ts.map +1 -1
  118. package/lib/node/nodejs.js +2 -22
  119. package/lib/node/nodejs.js.map +1 -1
  120. package/lib/sync/interface.d.ts +1 -2
  121. package/lib/sync/interface.d.ts.map +1 -1
  122. package/lib/sync/options.d.ts +0 -6
  123. package/lib/sync/options.d.ts.map +1 -1
  124. package/lib/sync/options.js +0 -2
  125. package/lib/sync/options.js.map +1 -1
  126. package/lib/sync/types.d.ts +21 -0
  127. package/lib/sync/types.d.ts.map +1 -1
  128. package/lib/sync/types.js +22 -0
  129. package/lib/sync/types.js.map +1 -1
  130. package/lib/sync/unknownBlock.d.ts +20 -0
  131. package/lib/sync/unknownBlock.d.ts.map +1 -1
  132. package/lib/sync/unknownBlock.js +158 -2
  133. package/lib/sync/unknownBlock.js.map +1 -1
  134. package/lib/util/shuffle.d.ts +0 -4
  135. package/lib/util/shuffle.d.ts.map +1 -1
  136. package/lib/util/shuffle.js +0 -8
  137. package/lib/util/shuffle.js.map +1 -1
  138. package/lib/util/sszBytes.d.ts +5 -0
  139. package/lib/util/sszBytes.d.ts.map +1 -1
  140. package/lib/util/sszBytes.js +13 -1
  141. package/lib/util/sszBytes.js.map +1 -1
  142. package/package.json +15 -15
  143. package/src/api/impl/beacon/blocks/index.ts +2 -2
  144. package/src/api/impl/beacon/pool/index.ts +6 -2
  145. package/src/api/impl/validator/index.ts +9 -7
  146. package/src/chain/archiveStore/archiveStore.ts +0 -6
  147. package/src/chain/archiveStore/historicalState/metrics.ts +4 -7
  148. package/src/chain/archiveStore/utils/pruneHistory.ts +14 -4
  149. package/src/chain/blocks/importExecutionPayload.ts +14 -0
  150. package/src/chain/blocks/index.ts +2 -2
  151. package/src/chain/blocks/payloadEnvelopeInput/types.ts +1 -1
  152. package/src/chain/emitter.ts +7 -2
  153. package/src/chain/errors/attesterSlashingError.ts +3 -1
  154. package/src/chain/errors/payloadAttestation.ts +2 -0
  155. package/src/chain/errors/proposerSlashingError.ts +3 -1
  156. package/src/chain/options.ts +0 -5
  157. package/src/chain/produceBlock/computeNewStateRoot.ts +9 -2
  158. package/src/chain/produceBlock/produceBlockBody.ts +17 -6
  159. package/src/chain/seenCache/seenPayloadEnvelopeInput.ts +3 -2
  160. package/src/chain/validation/attesterSlashing.ts +1 -2
  161. package/src/chain/validation/payloadAttestationMessage.ts +9 -0
  162. package/src/chain/validation/proposerSlashing.ts +1 -2
  163. package/src/chain/validatorMonitor.ts +12 -2
  164. package/src/db/beacon.ts +0 -5
  165. package/src/db/buckets.ts +1 -1
  166. package/src/db/interface.ts +0 -3
  167. package/src/db/repositories/blockArchive.ts +38 -1
  168. package/src/db/repositories/index.ts +0 -1
  169. package/src/metrics/metrics/lodestar.ts +34 -26
  170. package/src/network/gossip/encoding.ts +5 -8
  171. package/src/network/gossip/gossipsub.ts +19 -2
  172. package/src/network/gossip/topic.ts +25 -7
  173. package/src/network/processor/gossipHandlers.ts +18 -5
  174. package/src/network/processor/gossipValidatorFn.ts +111 -7
  175. package/src/network/processor/index.ts +81 -24
  176. package/src/network/reqresp/protocols.ts +31 -6
  177. package/src/node/nodejs.ts +0 -27
  178. package/src/sync/interface.ts +1 -2
  179. package/src/sync/options.ts +0 -8
  180. package/src/sync/types.ts +24 -3
  181. package/src/sync/unknownBlock.ts +166 -1
  182. package/src/util/shuffle.ts +0 -9
  183. package/src/util/sszBytes.ts +18 -0
  184. package/lib/chain/archiveStore/utils/updateBackfillRange.d.ts +0 -20
  185. package/lib/chain/archiveStore/utils/updateBackfillRange.d.ts.map +0 -1
  186. package/lib/chain/archiveStore/utils/updateBackfillRange.js +0 -40
  187. package/lib/chain/archiveStore/utils/updateBackfillRange.js.map +0 -1
  188. package/lib/db/repositories/backfilledRanges.d.ts +0 -18
  189. package/lib/db/repositories/backfilledRanges.d.ts.map +0 -1
  190. package/lib/db/repositories/backfilledRanges.js +0 -26
  191. package/lib/db/repositories/backfilledRanges.js.map +0 -1
  192. package/lib/sync/backfill/backfill.d.ts +0 -169
  193. package/lib/sync/backfill/backfill.d.ts.map +0 -1
  194. package/lib/sync/backfill/backfill.js +0 -686
  195. package/lib/sync/backfill/backfill.js.map +0 -1
  196. package/lib/sync/backfill/errors.d.ts +0 -30
  197. package/lib/sync/backfill/errors.d.ts.map +0 -1
  198. package/lib/sync/backfill/errors.js +0 -15
  199. package/lib/sync/backfill/errors.js.map +0 -1
  200. package/lib/sync/backfill/index.d.ts +0 -2
  201. package/lib/sync/backfill/index.d.ts.map +0 -1
  202. package/lib/sync/backfill/index.js +0 -2
  203. package/lib/sync/backfill/index.js.map +0 -1
  204. package/lib/sync/backfill/verify.d.ts +0 -18
  205. package/lib/sync/backfill/verify.d.ts.map +0 -1
  206. package/lib/sync/backfill/verify.js +0 -36
  207. package/lib/sync/backfill/verify.js.map +0 -1
  208. package/src/chain/archiveStore/utils/updateBackfillRange.ts +0 -50
  209. package/src/db/repositories/backfilledRanges.ts +0 -29
  210. package/src/sync/backfill/backfill.ts +0 -916
  211. package/src/sync/backfill/errors.ts +0 -23
  212. package/src/sync/backfill/index.ts +0 -1
  213. package/src/sync/backfill/verify.ts +0 -58
@@ -59,7 +59,18 @@ import {
59
59
  heze,
60
60
  ssz,
61
61
  } from "@lodestar/types";
62
- import {GWEI_TO_WEI, Logger, byteArrayEquals, fromHex, sleep, toHex, toPubkeyHex, toRootHex} from "@lodestar/utils";
62
+ import {
63
+ GWEI_TO_WEI,
64
+ Logger,
65
+ byteArrayEquals,
66
+ fromHex,
67
+ prettyGweiToEth,
68
+ prettyWeiToEth,
69
+ sleep,
70
+ toHex,
71
+ toPubkeyHex,
72
+ toRootHex,
73
+ } from "@lodestar/utils";
63
74
  import {ZERO_HASH_HEX} from "../../constants/index.js";
64
75
  import {numToQuantity} from "../../execution/engine/utils.js";
65
76
  import {IExecutionBuilder, IExecutionEngine, PayloadAttributes, PayloadId} from "../../execution/index.js";
@@ -259,7 +270,7 @@ export async function produceBlockBody<T extends BlockType>(
259
270
  this.logger.verbose("Produced block with builder bid", {
260
271
  slot: blockSlot,
261
272
  builderIndex: builderBid.message.builderIndex,
262
- bidValue: builderBid.message.value,
273
+ bidValue: prettyGweiToEth(builderBid.message.value),
263
274
  parentBlockHash: toRootHex(builderBid.message.parentBlockHash),
264
275
  parentBlockRoot: toRootHex(builderBid.message.parentBlockRoot),
265
276
  blockHash: toRootHex(builderBid.message.blockHash),
@@ -410,7 +421,7 @@ export async function produceBlockBody<T extends BlockType>(
410
421
  this.metrics?.blockPayload.payloadFetchedTime.observe({prepType}, fetchedTime);
411
422
  this.logger.verbose("Produced block with self-build bid", {
412
423
  slot: blockSlot,
413
- executionPayloadValue,
424
+ executionPayloadValue: prettyWeiToEth(executionPayloadValue),
414
425
  prepType,
415
426
  payloadId,
416
427
  fetchedTime,
@@ -490,7 +501,7 @@ export async function produceBlockBody<T extends BlockType>(
490
501
  this.metrics?.blockPayload.payloadFetchedTime.observe({prepType}, fetchedTime);
491
502
  this.logger.verbose("Fetched execution payload header from builder", {
492
503
  slot: blockSlot,
493
- executionPayloadValue,
504
+ executionPayloadValue: prettyWeiToEth(executionPayloadValue),
494
505
  prepType,
495
506
  fetchedTime,
496
507
  });
@@ -607,7 +618,7 @@ export async function produceBlockBody<T extends BlockType>(
607
618
  this.metrics?.blockPayload.payloadFetchedTime.observe({prepType}, fetchedTime);
608
619
  this.logger.verbose("Fetched execution payload from engine", {
609
620
  slot: blockSlot,
610
- executionPayloadValue,
621
+ executionPayloadValue: prettyWeiToEth(executionPayloadValue),
611
622
  prepType,
612
623
  payloadId,
613
624
  fetchedTime,
@@ -708,7 +719,7 @@ export async function produceBlockBody<T extends BlockType>(
708
719
  }
709
720
  }
710
721
 
711
- Object.assign(logMeta, {executionPayloadValue});
722
+ Object.assign(logMeta, {executionPayloadValue: prettyWeiToEth(executionPayloadValue)});
712
723
  this.logger.verbose("Produced beacon block body", logMeta);
713
724
 
714
725
  return {body: blockBody as AssembledBodyType<T>, produceResult, executionPayloadValue, shouldOverrideBuilder};
@@ -256,10 +256,11 @@ export class SeenPayloadEnvelopeInput {
256
256
  return this.payloadInputs.size;
257
257
  }
258
258
 
259
- prune(blockRootHex: RootHex): void {
259
+ /** Removes the single PayloadEnvelopeInput from the cache */
260
+ remove(blockRootHex: RootHex): void {
260
261
  const input = this.payloadInputs.get(blockRootHex);
261
262
  if (input) {
262
- this.evictPayloadInput(input, "prune");
263
+ this.evictPayloadInput(input, "remove");
263
264
  }
264
265
  }
265
266
 
@@ -63,8 +63,7 @@ export async function validateAttesterSlashing(
63
63
  const signatureSets = getAttesterSlashingSignatureSets(chain.config, state.slot, attesterSlashing);
64
64
  if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true, priority: prioritizeBls}))) {
65
65
  throw new AttesterSlashingError(GossipAction.REJECT, {
66
- code: AttesterSlashingErrorCode.INVALID,
67
- error: Error("Invalid signature"),
66
+ code: AttesterSlashingErrorCode.INVALID_SIGNATURE,
68
67
  });
69
68
  }
70
69
  }
@@ -1,4 +1,5 @@
1
1
  import {
2
+ computeEpochAtSlot,
2
3
  createIndexedSignatureSetFromComponents,
3
4
  getPayloadAttestationDataSigningRoot,
4
5
  isStatePostGloas,
@@ -35,6 +36,14 @@ async function validatePayloadAttestationMessage(
35
36
  ): Promise<PayloadAttestationValidationResult> {
36
37
  const {data, validatorIndex} = payloadAttestationMessage;
37
38
 
39
+ // [REJECT] The payload attestation slot is at or after the Gloas fork.
40
+ if (computeEpochAtSlot(data.slot) < chain.config.GLOAS_FORK_EPOCH) {
41
+ throw new PayloadAttestationError(GossipAction.REJECT, {
42
+ code: PayloadAttestationErrorCode.PRE_GLOAS_SLOT,
43
+ slot: data.slot,
44
+ });
45
+ }
46
+
38
47
  // [IGNORE] The message's slot is for the current slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance), i.e. `data.slot == current_slot`.
39
48
  if (!chain.clock.isCurrentSlotGivenGossipDisparity(data.slot)) {
40
49
  throw new PayloadAttestationError(GossipAction.IGNORE, {
@@ -48,8 +48,7 @@ async function validateProposerSlashing(
48
48
  const signatureSets = getProposerSlashingSignatureSets(chain.config, state.slot, proposerSlashing);
49
49
  if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true, priority: prioritizeBls}))) {
50
50
  throw new ProposerSlashingError(GossipAction.REJECT, {
51
- code: ProposerSlashingErrorCode.INVALID,
52
- error: Error("Invalid signature"),
51
+ code: ProposerSlashingErrorCode.INVALID_SIGNATURE,
53
52
  });
54
53
  }
55
54
  }
@@ -24,7 +24,17 @@ import {
24
24
  deneb,
25
25
  gloas,
26
26
  } from "@lodestar/types";
27
- import {LogData, LogHandler, LogLevel, Logger, MapDef, MapDefMax, prettyPrintIndices, toRootHex} from "@lodestar/utils";
27
+ import {
28
+ LogData,
29
+ LogHandler,
30
+ LogLevel,
31
+ Logger,
32
+ MapDef,
33
+ MapDefMax,
34
+ prettyGweiToEth,
35
+ prettyPrintIndices,
36
+ toRootHex,
37
+ } from "@lodestar/utils";
28
38
  import {GENESIS_SLOT} from "../constants/constants.js";
29
39
  import {RegistryMetricCreator} from "../metrics/index.js";
30
40
 
@@ -470,7 +480,7 @@ export function createValidatorMonitor(
470
480
  src,
471
481
  builderIndex: bid.builderIndex,
472
482
  gasLimit: Number(bid.gasLimit),
473
- value: bid.value.toString(),
483
+ value: prettyGweiToEth(bid.value),
474
484
  parentBlockRoot: toRootHex(bid.parentBlockRoot),
475
485
  parentBlockHash: toRootHex(bid.parentBlockHash),
476
486
  blockHash: toRootHex(bid.blockHash),
package/src/db/beacon.ts CHANGED
@@ -11,7 +11,6 @@ import {CheckpointStateRepository} from "./repositories/checkpointState.js";
11
11
  import {
12
12
  AttesterSlashingRepository,
13
13
  BLSToExecutionChangeRepository,
14
- BackfilledRanges,
15
14
  BestLightClientUpdateRepository,
16
15
  BlobSidecarsArchiveRepository,
17
16
  BlobSidecarsRepository,
@@ -61,8 +60,6 @@ export class BeaconDb implements IBeaconDb {
61
60
  syncCommittee: SyncCommitteeRepository;
62
61
  syncCommitteeWitness: SyncCommitteeWitnessRepository;
63
62
 
64
- backfilledRanges: BackfilledRanges;
65
-
66
63
  readonly dataColumns: IDataColumnStore;
67
64
  lastLegacyArchiveSlot: Slot | null = null;
68
65
  private readonly flatFileStore: FlatFileStore;
@@ -97,8 +94,6 @@ export class BeaconDb implements IBeaconDb {
97
94
  this.syncCommittee = new SyncCommitteeRepository(config, db);
98
95
  this.syncCommitteeWitness = new SyncCommitteeWitnessRepository(config, db);
99
96
 
100
- this.backfilledRanges = new BackfilledRanges(config, db);
101
-
102
97
  this.flatFileStore = new FlatFileStore(opts.dataColumnDir, config, opts.logger, opts.metrics);
103
98
  this.dataColumns = new LegacyDataColumnStore(
104
99
  this.flatFileStore,
package/src/db/buckets.ts CHANGED
@@ -56,7 +56,7 @@ export enum Bucket {
56
56
  // altair_lightClientSyncCommitteeProof = 35, // DEPRECATED on v0.32.0
57
57
  // index_lightClientInitProof = 36, // DEPRECATED on v0.32.0
58
58
 
59
- backfilled_ranges = 42, // Backfilled From to To, inclusive of both From, To
59
+ // backfilled_ranges = 42, // DEPRECATED on v1.48.0
60
60
 
61
61
  // Buckets to support LightClient server v2
62
62
  lightClient_syncCommitteeWitness = 51, // BlockRoot -> SyncCommitteeWitness
@@ -6,7 +6,6 @@ import {CheckpointStateRepository} from "./repositories/checkpointState.js";
6
6
  import {
7
7
  AttesterSlashingRepository,
8
8
  BLSToExecutionChangeRepository,
9
- BackfilledRanges,
10
9
  BestLightClientUpdateRepository,
11
10
  BlobSidecarsArchiveRepository,
12
11
  BlobSidecarsRepository,
@@ -60,8 +59,6 @@ export interface IBeaconDb {
60
59
  syncCommittee: SyncCommitteeRepository;
61
60
  syncCommitteeWitness: SyncCommitteeWitnessRepository;
62
61
 
63
- backfilledRanges: BackfilledRanges;
64
-
65
62
  dataColumns: IDataColumnStore;
66
63
  /** Highest legacy column slot at startup; later archival writes the block-root index first. */
67
64
  readonly lastLegacyArchiveSlot: Slot | null;
@@ -1,8 +1,9 @@
1
1
  import {ChainForkConfig} from "@lodestar/config";
2
2
  import {Db, DbBatch, FilterOptions, KeyValue, Repository} from "@lodestar/db";
3
3
  import {Root, SignedBeaconBlock, Slot, ssz} from "@lodestar/types";
4
- import {bytesToInt, intToBytes} from "@lodestar/utils";
4
+ import {bytesToInt, fromHex, intToBytes} from "@lodestar/utils";
5
5
  import {getSignedBlockTypeFromBytes} from "../../util/multifork.js";
6
+ import {getParentRootFromSignedBeaconBlockSerialized} from "../../util/sszBytes.js";
6
7
  import {Bucket, getBucketNameByValue} from "../buckets.js";
7
8
  import {
8
9
  deleteParentRootIndex,
@@ -24,6 +25,10 @@ export type BlockArchiveBatchPutBinaryItem = KeyValue<Slot, Uint8Array> & {
24
25
  parentRoot: Root;
25
26
  };
26
27
 
28
+ // The initial prune can span millions of slots. This bounds the slot index reads in flight and keeps each
29
+ // delete batch at a few thousand keys, both tiny next to the block value batches in archiveBlocks
30
+ const DELETE_RANGE_CHUNK_SIZE = 1000;
31
+
27
32
  /**
28
33
  * Stores finalized blocks. Block slot is identifier.
29
34
  */
@@ -103,6 +108,38 @@ export class BlockArchiveRepository extends Repository<Slot, SignedBeaconBlock>
103
108
  );
104
109
  }
105
110
 
111
+ /**
112
+ * Delete a contiguous range of archived blocks together with their index entries.
113
+ * `slots` must be every archived slot in the range, so the parent of each block is the entry before it.
114
+ */
115
+ async batchDeleteRange(slots: Slot[]): Promise<void> {
116
+ const sorted = [...slots].sort((a, b) => a - b);
117
+ let prevRoot: Root | null = null;
118
+
119
+ for (let i = 0; i < sorted.length; i += DELETE_RANGE_CHUNK_SIZE) {
120
+ const chunk = sorted.slice(i, i + DELETE_RANGE_CHUNK_SIZE);
121
+ const roots = await Promise.all(chunk.map((slot) => this.getRootBySlot(slot)));
122
+
123
+ const keys: Uint8Array[] = [];
124
+ for (let j = 0; j < chunk.length; j++) {
125
+ keys.push(this.encodeKey(chunk[j]), getSlotIndexKey(chunk[j]));
126
+ const root = roots[j];
127
+ if (root) keys.push(getRootIndexKey(root));
128
+ // The parent index entry pointing at this block is keyed by its parent root, which is the
129
+ // previous block's root unless that block is unindexed, then it is read from the block itself
130
+ let parentRoot = prevRoot;
131
+ if (parentRoot === null) {
132
+ const block = await this.getBinary(chunk[j]);
133
+ const parentRootHex = block ? getParentRootFromSignedBeaconBlockSerialized(block) : null;
134
+ parentRoot = parentRootHex ? fromHex(parentRootHex) : null;
135
+ }
136
+ if (parentRoot) keys.push(getParentRootIndexKey(parentRoot));
137
+ prevRoot = root;
138
+ }
139
+ await this.db.batchDelete(keys, this.dbReqOpts);
140
+ }
141
+ }
142
+
106
143
  async batch(operations: DbBatch<Slot, SignedBeaconBlock>): Promise<void> {
107
144
  await this.batchBinary(
108
145
  operations.map((operation) =>
@@ -1,5 +1,4 @@
1
1
  export {AttesterSlashingRepository} from "./attesterSlashing.js";
2
- export {BackfilledRanges} from "./backfilledRanges.js";
3
2
  export {BlobSidecarsRepository} from "./blobSidecars.js";
4
3
  export {BlobSidecarsArchiveRepository} from "./blobSidecarsArchive.js";
5
4
  export {BlockRepository} from "./block.js";
@@ -1,5 +1,6 @@
1
1
  /** biome-ignore-all lint/suspicious/noTemplateCurlyInString: The metric templates requires to have `${}` in a normal string */
2
2
  import {NotReorgedReason} from "@lodestar/fork-choice";
3
+ import {StateHashTreeRootSource} from "@lodestar/state-transition";
3
4
  import {ArchiveStoreTask} from "../../chain/archiveStore/archiveStore.js";
4
5
  import {FrequencyStateArchiveStep} from "../../chain/archiveStore/strategies/frequencyStateArchiveStrategy.js";
5
6
  import type {LateCanonicalBlockReason} from "../../chain/archiveStore/utils/archiveBlocks.js";
@@ -25,8 +26,13 @@ import type {FlatFileStoreOperation} from "../../db/flatFileStore/metrics.js";
25
26
  import {ExecutionPayloadStatus} from "../../execution/index.js";
26
27
  import {GossipType} from "../../network/index.js";
27
28
  import {CannotAcceptWorkReason, ReprocessRejectReason} from "../../network/processor/index.js";
28
- import {BackfillSyncMethod} from "../../sync/backfill/backfill.js";
29
- import {DownloadResult, DroppedItemReason, FetchResult, PendingBlockType} from "../../sync/types.js";
29
+ import {
30
+ DeferredPayloadResult,
31
+ DownloadResult,
32
+ DroppedItemReason,
33
+ FetchResult,
34
+ PendingBlockType,
35
+ } from "../../sync/types.js";
30
36
  import {PeerSyncType, RangeSyncType} from "../../sync/utils/remoteSyncType.js";
31
37
  import {AllocSource} from "../../util/bufferPool.js";
32
38
  import {DataColumnReconstructionCode} from "../../util/dataColumns.js";
@@ -403,6 +409,15 @@ export function createLodestarMetrics(
403
409
  labelNames: ["caller"],
404
410
  }),
405
411
 
412
+ // this metrics stay here instead of @lodestar/state-transition in order to prepare
413
+ // for the native zig state-transition in the future
414
+ stateHashTreeRootTime: register.histogram<{source: StateHashTreeRootSource}>({
415
+ name: "lodestar_stfn_hash_tree_root_seconds",
416
+ help: "Time to compute the hash tree root of a post state in seconds",
417
+ buckets: [0.01, 0.05, 0.1, 0.2, 0.3, 0.5],
418
+ labelNames: ["source"],
419
+ }),
420
+
406
421
  // BLS verifier thread pool and queue
407
422
 
408
423
  bls: {
@@ -655,6 +670,16 @@ export function createLodestarMetrics(
655
670
  help: "Count of payload (execution payload envelope) sync triggers, labeled by their source",
656
671
  labelNames: ["source"],
657
672
  }),
673
+ deferredPayloadResult: register.counter<{result: DeferredPayloadResult}>({
674
+ name: "lodestar_sync_deferred_payload_result_total",
675
+ help: "Outcome of a slot's deferred optimistic payload searches, by result",
676
+ labelNames: ["result"],
677
+ }),
678
+ deferredPayloadPolls: register.histogram({
679
+ name: "lodestar_sync_deferred_payload_polls_count",
680
+ help: "Poll ticks after PAYLOAD_DUE until a searched payload was imported",
681
+ buckets: [0, 1, 2, 3, 6],
682
+ }),
658
683
  pendingBlocks: register.gauge({
659
684
  name: "lodestar_sync_unknown_block_pending_blocks_size",
660
685
  help: "Current size of UnknownBlockSync pending blocks cache",
@@ -986,6 +1011,13 @@ export function createLodestarMetrics(
986
1011
  help: "Count of errors, by error type, while processing execution payload envelopes",
987
1012
  labelNames: ["error"],
988
1013
  }),
1014
+ executionPayload: {
1015
+ recvToValidation: register.histogram({
1016
+ name: "lodestar_gossip_execution_payload_envelope_received_to_execution_payload_verification_seconds",
1017
+ help: "Time elapsed between execution payload envelope received and execution payload verification",
1018
+ buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 3, 4, 6, 8],
1019
+ }),
1020
+ },
989
1021
  },
990
1022
  gossipExecutionPayloadBid: {
991
1023
  elapsedTimeTillReceived: register.histogram<{source: OpSource}>({
@@ -1120,30 +1152,6 @@ export function createLodestarMetrics(
1120
1152
  help: "The total result of calling notifyForkchoiceUpdate execution engine api",
1121
1153
  labelNames: ["result"],
1122
1154
  }),
1123
- backfillSync: {
1124
- backfilledTillSlot: register.gauge({
1125
- name: "lodestar_backfill_till_slot",
1126
- help: "Current lowest backfilled slot",
1127
- }),
1128
- prevFinOrWsSlot: register.gauge({
1129
- name: "lodestar_backfill_prev_fin_or_ws_slot",
1130
- help: "Slot of previous finalized or wsCheckpoint block to be validated",
1131
- }),
1132
- totalBlocks: register.gauge<{method: BackfillSyncMethod}>({
1133
- name: "lodestar_backfill_sync_blocks_total",
1134
- help: "Total amount of backfilled blocks",
1135
- labelNames: ["method"],
1136
- }),
1137
- errors: register.gauge({
1138
- name: "lodestar_backfill_sync_errors_total",
1139
- help: "Total number of errors while backfilling",
1140
- }),
1141
- status: register.gauge({
1142
- name: "lodestar_backfill_sync_status",
1143
- help: "Current backfill syncing status: [Aborted, Pending, Syncing, Completed]",
1144
- }),
1145
- },
1146
-
1147
1155
  opPool: {
1148
1156
  aggregatedAttestationPool: {
1149
1157
  size: register.gauge({
@@ -8,7 +8,7 @@ import {ForkName} from "@lodestar/params";
8
8
  import {intToBytes} from "@lodestar/utils";
9
9
  import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js";
10
10
  import {Eth2GossipsubMetrics} from "./metrics.js";
11
- import {GossipTopicCache, getGossipSSZMaxSize, getGossipSSZType} from "./topic.js";
11
+ import {GossipTopicCache} from "./topic.js";
12
12
 
13
13
  // Load WASM
14
14
  const xxhash = await xxhashFactory();
@@ -97,7 +97,6 @@ export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8
97
97
  export class DataTransformSnappy implements DataTransform {
98
98
  constructor(
99
99
  private readonly gossipTopicCache: GossipTopicCache,
100
- private readonly maxPayloadSize: number,
101
100
  private readonly metrics: Eth2GossipsubMetrics | null
102
101
  ) {}
103
102
 
@@ -112,12 +111,11 @@ export class DataTransformSnappy implements DataTransform {
112
111
  const uncompressedDataLength = snappyWasm.decompress_len(data);
113
112
 
114
113
  const topic = this.gossipTopicCache.getTopic(topicStr);
115
- const sszType = getGossipSSZType(topic);
116
- const maxSize = getGossipSSZMaxSize(topic, this.maxPayloadSize, sszType);
114
+ const {minSize, maxSize} = this.gossipTopicCache.getTypeSizes(topicStr);
117
115
  this.metrics?.dataTransform.inbound.inc({type: topic.type});
118
116
 
119
- if (uncompressedDataLength < sszType.minSize) {
120
- throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} < ${sszType.minSize}`);
117
+ if (uncompressedDataLength < minSize) {
118
+ throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} < ${minSize}`);
121
119
  }
122
120
  if (uncompressedDataLength > maxSize) {
123
121
  throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${maxSize}`);
@@ -136,8 +134,7 @@ export class DataTransformSnappy implements DataTransform {
136
134
  */
137
135
  outboundTransform(topicStr: string, data: Uint8Array): Uint8Array {
138
136
  const topic = this.gossipTopicCache.getTopic(topicStr);
139
- const sszType = getGossipSSZType(topic);
140
- const maxSize = getGossipSSZMaxSize(topic, this.maxPayloadSize, sszType);
137
+ const {maxSize} = this.gossipTopicCache.getTypeSizes(topicStr);
141
138
  this.metrics?.dataTransform.outbound.inc({type: topic.type});
142
139
  if (data.length > maxSize) {
143
140
  throw Error(`ssz_snappy encoded data length ${data.length} > ${maxSize}`);
@@ -14,7 +14,7 @@ import {peerIdFromString} from "@libp2p/peer-id";
14
14
  import {type Multiaddr, multiaddr} from "@multiformats/multiaddr";
15
15
  import {ENR} from "@chainsafe/enr";
16
16
  import {routes} from "@lodestar/api";
17
- import {BeaconConfig, ForkBoundary} from "@lodestar/config";
17
+ import {BeaconConfig, ChainConfig, ForkBoundary} from "@lodestar/config";
18
18
  import {
19
19
  ATTESTATION_SUBNET_COUNT,
20
20
  MAX_SIGNED_AGGREGATE_AND_PROOF_SIZE,
@@ -47,6 +47,22 @@ const GOSSIPSUB_HEARTBEAT_INTERVAL = 0.7 * 1000;
47
47
 
48
48
  const MAX_OUTBOUND_BUFFER_SIZE = 2 ** 24; // 16MB
49
49
 
50
+ /**
51
+ * Snappy worst-case compressed length for a payload of `n` bytes
52
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-beta.0/specs/phase0/p2p-interface.md#max_compressed_len
53
+ */
54
+ function maxCompressedLen(n: number): number {
55
+ return 32 + n + Math.floor(n / 6);
56
+ }
57
+
58
+ /**
59
+ * Max size of an inbound gossipsub RPC frame, including all bundled messages and control
60
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-beta.0/specs/phase0/p2p-interface.md#max_message_size
61
+ */
62
+ function getMaxInboundDataLength(config: Pick<ChainConfig, "MAX_PAYLOAD_SIZE">): number {
63
+ return Math.max(maxCompressedLen(config.MAX_PAYLOAD_SIZE) + 1024, 1024 * 1024);
64
+ }
65
+
50
66
  export type Eth2Context = {
51
67
  activeValidatorCount: number;
52
68
  currentSlot: number;
@@ -168,7 +184,7 @@ export class Eth2Gossipsub {
168
184
  fastMsgIdFn: fastMsgIdFn,
169
185
  msgIdFn: msgIdFn.bind(msgIdFn, gossipTopicCache),
170
186
  msgIdToStrFn: msgIdToStrFn,
171
- dataTransform: new DataTransformSnappy(gossipTopicCache, config.MAX_PAYLOAD_SIZE, metrics),
187
+ dataTransform: new DataTransformSnappy(gossipTopicCache, metrics),
172
188
  metricsRegister: metricsRegister as MetricsRegister | null,
173
189
  metricsTopicStrToLabel: metricsRegister
174
190
  ? getMetricsTopicStrToLabel(networkConfig, {disableLightClientServer: opts.disableLightClientServer ?? false})
@@ -176,6 +192,7 @@ export class Eth2Gossipsub {
176
192
  asyncValidation: true,
177
193
 
178
194
  maxOutboundBufferSize: MAX_OUTBOUND_BUFFER_SIZE,
195
+ maxInboundDataLength: getMaxInboundDataLength(config),
179
196
  // serialize message once and send to all peers when publishing
180
197
  batchPublish: true,
181
198
  // if this is false, only publish to mesh peers. If there is not enough GOSSIP_D mesh peers,
@@ -1,5 +1,5 @@
1
1
  import {type CompositeTypeAny} from "@chainsafe/ssz";
2
- import {ForkDigestContext} from "@lodestar/config";
2
+ import {BeaconConfig, ChainForkConfig, ForkDigestContext} from "@lodestar/config";
3
3
  import {
4
4
  ATTESTATION_SUBNET_COUNT,
5
5
  ForkName,
@@ -10,32 +10,47 @@ import {
10
10
  isForkPostFulu,
11
11
  isForkPostGloas,
12
12
  } from "@lodestar/params";
13
+ import {TypeSizes} from "@lodestar/reqresp";
13
14
  import {Attestation, SingleAttestation, ssz, sszTypesFor} from "@lodestar/types";
14
15
  import {GossipAction, GossipActionError, GossipErrorCode} from "../../chain/errors/gossipValidation.js";
16
+ import {computeMaxGloasDataColumnSidecarSize} from "../../util/sszBytes.js";
15
17
  import {NetworkConfig} from "../networkConfig.js";
16
18
  import {DEFAULT_ENCODING} from "./constants.js";
17
19
  import {GossipEncoding, GossipTopic, GossipTopicTypeMap, GossipType, SSZTypeOfGossipTopic} from "./interface.js";
18
20
 
19
21
  export interface IGossipTopicCache {
20
22
  getTopic(topicStr: string): GossipTopic;
23
+ getTypeSizes(topicStr: string): TypeSizes;
21
24
  }
22
25
 
23
26
  export class GossipTopicCache implements IGossipTopicCache {
24
27
  private topicsByTopicStr = new Map<string, Required<GossipTopic>>();
28
+ private typeSizesByTopicStr = new Map<string, TypeSizes>();
25
29
 
26
- constructor(private readonly forkDigestContext: ForkDigestContext) {}
30
+ constructor(private readonly config: BeaconConfig) {}
27
31
 
28
32
  /** Returns cached GossipTopic, otherwise attempts to parse it from the str */
29
33
  getTopic(topicStr: string): GossipTopic {
30
34
  let topic = this.topicsByTopicStr.get(topicStr);
31
35
  if (topic === undefined) {
32
- topic = parseGossipTopic(this.forkDigestContext, topicStr);
36
+ topic = parseGossipTopic(this.config, topicStr);
33
37
  // TODO: Consider just throwing here. We should only receive messages from known subscribed topics
34
38
  this.topicsByTopicStr.set(topicStr, topic);
35
39
  }
36
40
  return topic;
37
41
  }
38
42
 
43
+ getTypeSizes(topicStr: string): TypeSizes {
44
+ let typeSizes = this.typeSizesByTopicStr.get(topicStr);
45
+ if (typeSizes === undefined) {
46
+ const topic = this.getTopic(topicStr);
47
+ const sszType = getGossipSSZType(topic);
48
+ typeSizes = {minSize: sszType.minSize, maxSize: getGossipSSZMaxSize(topic, this.config, sszType)};
49
+ this.typeSizesByTopicStr.set(topicStr, typeSizes);
50
+ }
51
+ return typeSizes;
52
+ }
53
+
39
54
  /** Returns cached GossipTopic, otherwise returns undefined */
40
55
  getKnownTopic(topicStr: string): GossipTopic | undefined {
41
56
  return this.topicsByTopicStr.get(topicStr);
@@ -133,11 +148,14 @@ export function getGossipSSZType(topic: GossipTopic) {
133
148
  }
134
149
 
135
150
  /**
136
- * Return the maximum uncompressed SSZ byte length accepted for a gossip object, the SSZ type max size
137
- * or MAX_PAYLOAD_SIZE, whichever is smaller.
151
+ * Return the maximum uncompressed SSZ byte length allowed by the type and configured network bounds.
138
152
  */
139
- export function getGossipSSZMaxSize(topic: GossipTopic, maxPayloadSize: number, sszType?: CompositeTypeAny): number {
140
- return Math.min((sszType ?? getGossipSSZType(topic)).maxSize, maxPayloadSize);
153
+ export function getGossipSSZMaxSize(topic: GossipTopic, config: ChainForkConfig, sszType: CompositeTypeAny): number {
154
+ const maxSize = Math.min(sszType.maxSize, config.MAX_PAYLOAD_SIZE);
155
+ if (isForkPostGloas(topic.boundary.fork) && topic.type === GossipType.data_column_sidecar) {
156
+ return Math.min(maxSize, computeMaxGloasDataColumnSidecarSize(config));
157
+ }
158
+ return maxSize;
141
159
  }
142
160
 
143
161
  /**
@@ -244,8 +244,20 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
244
244
 
245
245
  // IGNORE means the block is acceptable (e.g. FUTURE_SLOT, ALREADY_KNOWN), just not propagated.
246
246
  // Keep the optimistically-added cache entries; they are pruned on finalization. Only REJECT
247
- // (provably invalid) and unexpected errors prune below.
247
+ // (provably invalid), unexpected errors and repeat proposals that are not imported prune.
248
248
  if (e.action === GossipAction.IGNORE) {
249
+ // Only a signature-verified sibling is imported by the beacon_block handler, any other repeat proposal
250
+ // is dropped from the caches and re-downloaded by sync if it ever becomes relevant. Its signature was not
251
+ // verified, so only its own entry is removed, never its claimed ancestors
252
+ if (
253
+ e.type.code === BlockErrorCode.REPEAT_PROPOSAL &&
254
+ !chain.seenBlockProposers.hasBlockRoot(slot, signedBlock.message.proposerIndex, blockRootHex)
255
+ ) {
256
+ chain.seenBlockInputCache.remove(blockRootHex);
257
+ if (isForkPostGloas(fork)) {
258
+ chain.seenPayloadEnvelopeInputCache.remove(blockRootHex);
259
+ }
260
+ }
249
261
  throw e;
250
262
  }
251
263
 
@@ -258,10 +270,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
258
270
  }
259
271
 
260
272
  // REJECT or unexpected (non-BlockGossipError) error: drop the optimistically-added entries from
261
- // both caches, keeping them consistent.
262
- chain.seenBlockInputCache.prune(blockRootHex);
273
+ // both caches, keeping them consistent. The block may carry any parent root, so only its own entry is removed
274
+ chain.seenBlockInputCache.remove(blockRootHex);
263
275
  if (isForkPostGloas(fork)) {
264
- chain.seenPayloadEnvelopeInputCache.prune(blockRootHex);
276
+ chain.seenPayloadEnvelopeInputCache.remove(blockRootHex);
265
277
  }
266
278
  throw e;
267
279
  } finally {
@@ -738,7 +750,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
738
750
  if (
739
751
  e instanceof BlockGossipError &&
740
752
  e.type.code === BlockErrorCode.REPEAT_PROPOSAL &&
741
- // this is make sure the block's proposer signature was verified, it should be true anyway
753
+ // Only a signature-verified sibling recorded in the seen cache is imported. The cache holds at most two
754
+ // roots per proposer and slot, which bounds full imports over gossip to one alternate
742
755
  chain.seenBlockProposers.hasBlockRoot(signedBlock.message.slot, e.type.proposerIndex, e.type.root)
743
756
  ) {
744
757
  // blockInput was optimistically seeded in validateBeaconBlock and retained on IGNORE