@lodestar/beacon-node 1.49.0-dev.c9061c6942 → 1.49.0-dev.fbb58f7280

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/lib/api/impl/beacon/blocks/index.js +2 -2
  2. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  3. package/lib/api/impl/validator/index.d.ts.map +1 -1
  4. package/lib/api/impl/validator/index.js +9 -7
  5. package/lib/api/impl/validator/index.js.map +1 -1
  6. package/lib/chain/archiveStore/utils/pruneHistory.d.ts.map +1 -1
  7. package/lib/chain/archiveStore/utils/pruneHistory.js +14 -3
  8. package/lib/chain/archiveStore/utils/pruneHistory.js.map +1 -1
  9. package/lib/chain/blocks/index.js +2 -2
  10. package/lib/chain/blocks/index.js.map +1 -1
  11. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts +1 -1
  12. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts.map +1 -1
  13. package/lib/chain/emitter.d.ts +10 -0
  14. package/lib/chain/emitter.d.ts.map +1 -1
  15. package/lib/chain/emitter.js +4 -0
  16. package/lib/chain/emitter.js.map +1 -1
  17. package/lib/chain/errors/attesterSlashingError.d.ts +4 -1
  18. package/lib/chain/errors/attesterSlashingError.d.ts.map +1 -1
  19. package/lib/chain/errors/attesterSlashingError.js +1 -0
  20. package/lib/chain/errors/attesterSlashingError.js.map +1 -1
  21. package/lib/chain/errors/proposerSlashingError.d.ts +4 -1
  22. package/lib/chain/errors/proposerSlashingError.d.ts.map +1 -1
  23. package/lib/chain/errors/proposerSlashingError.js +1 -0
  24. package/lib/chain/errors/proposerSlashingError.js.map +1 -1
  25. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  26. package/lib/chain/produceBlock/produceBlockBody.js +6 -6
  27. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  28. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts +2 -1
  29. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts.map +1 -1
  30. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js +3 -2
  31. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js.map +1 -1
  32. package/lib/chain/validation/attesterSlashing.d.ts.map +1 -1
  33. package/lib/chain/validation/attesterSlashing.js +1 -2
  34. package/lib/chain/validation/attesterSlashing.js.map +1 -1
  35. package/lib/chain/validation/proposerSlashing.js +1 -2
  36. package/lib/chain/validation/proposerSlashing.js.map +1 -1
  37. package/lib/chain/validatorMonitor.d.ts.map +1 -1
  38. package/lib/chain/validatorMonitor.js +2 -2
  39. package/lib/chain/validatorMonitor.js.map +1 -1
  40. package/lib/db/repositories/blockArchive.d.ts +5 -0
  41. package/lib/db/repositories/blockArchive.d.ts.map +1 -1
  42. package/lib/db/repositories/blockArchive.js +36 -1
  43. package/lib/db/repositories/blockArchive.js.map +1 -1
  44. package/lib/network/gossip/encoding.d.ts +1 -2
  45. package/lib/network/gossip/encoding.d.ts.map +1 -1
  46. package/lib/network/gossip/encoding.js +5 -10
  47. package/lib/network/gossip/encoding.js.map +1 -1
  48. package/lib/network/gossip/gossipsub.d.ts.map +1 -1
  49. package/lib/network/gossip/gossipsub.js +16 -1
  50. package/lib/network/gossip/gossipsub.js.map +1 -1
  51. package/lib/network/gossip/topic.d.ts +9 -6
  52. package/lib/network/gossip/topic.d.ts.map +1 -1
  53. package/lib/network/gossip/topic.js +23 -8
  54. package/lib/network/gossip/topic.js.map +1 -1
  55. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  56. package/lib/network/processor/gossipHandlers.js +16 -5
  57. package/lib/network/processor/gossipHandlers.js.map +1 -1
  58. package/lib/network/processor/gossipValidatorFn.d.ts +2 -0
  59. package/lib/network/processor/gossipValidatorFn.d.ts.map +1 -1
  60. package/lib/network/processor/gossipValidatorFn.js +88 -7
  61. package/lib/network/processor/gossipValidatorFn.js.map +1 -1
  62. package/lib/network/processor/index.d.ts +6 -2
  63. package/lib/network/processor/index.d.ts.map +1 -1
  64. package/lib/network/processor/index.js +62 -24
  65. package/lib/network/processor/index.js.map +1 -1
  66. package/lib/network/reqresp/protocols.d.ts.map +1 -1
  67. package/lib/network/reqresp/protocols.js +19 -5
  68. package/lib/network/reqresp/protocols.js.map +1 -1
  69. package/lib/sync/types.d.ts +4 -0
  70. package/lib/sync/types.d.ts.map +1 -1
  71. package/lib/sync/types.js +4 -0
  72. package/lib/sync/types.js.map +1 -1
  73. package/lib/sync/unknownBlock.d.ts +1 -0
  74. package/lib/sync/unknownBlock.d.ts.map +1 -1
  75. package/lib/sync/unknownBlock.js +15 -0
  76. package/lib/sync/unknownBlock.js.map +1 -1
  77. package/lib/util/sszBytes.d.ts +5 -0
  78. package/lib/util/sszBytes.d.ts.map +1 -1
  79. package/lib/util/sszBytes.js +13 -1
  80. package/lib/util/sszBytes.js.map +1 -1
  81. package/package.json +14 -14
  82. package/src/api/impl/beacon/blocks/index.ts +2 -2
  83. package/src/api/impl/validator/index.ts +9 -7
  84. package/src/chain/archiveStore/utils/pruneHistory.ts +14 -4
  85. package/src/chain/blocks/index.ts +2 -2
  86. package/src/chain/blocks/payloadEnvelopeInput/types.ts +1 -1
  87. package/src/chain/emitter.ts +7 -2
  88. package/src/chain/errors/attesterSlashingError.ts +3 -1
  89. package/src/chain/errors/proposerSlashingError.ts +3 -1
  90. package/src/chain/produceBlock/produceBlockBody.ts +17 -6
  91. package/src/chain/seenCache/seenPayloadEnvelopeInput.ts +3 -2
  92. package/src/chain/validation/attesterSlashing.ts +1 -2
  93. package/src/chain/validation/proposerSlashing.ts +1 -2
  94. package/src/chain/validatorMonitor.ts +12 -2
  95. package/src/db/repositories/blockArchive.ts +38 -1
  96. package/src/network/gossip/encoding.ts +5 -8
  97. package/src/network/gossip/gossipsub.ts +19 -2
  98. package/src/network/gossip/topic.ts +25 -7
  99. package/src/network/processor/gossipHandlers.ts +18 -5
  100. package/src/network/processor/gossipValidatorFn.ts +111 -7
  101. package/src/network/processor/index.ts +81 -24
  102. package/src/network/reqresp/protocols.ts +31 -6
  103. package/src/sync/types.ts +6 -3
  104. package/src/sync/unknownBlock.ts +15 -0
  105. package/src/util/sszBytes.ts +18 -0
@@ -104,7 +104,7 @@ export async function processBlocks(
104
104
  payloadEnvelopesToImport.delete(orphaned.slot);
105
105
  // never validated, drop it from the shared cache so it is not served to peers, by-root sync reloads the
106
106
  // entry from db if the payload is ever needed
107
- this.seenPayloadEnvelopeInputCache.prune(orphaned.payloadEnvelopeInput.blockRootHex);
107
+ this.seenPayloadEnvelopeInputCache.remove(orphaned.payloadEnvelopeInput.blockRootHex);
108
108
  }
109
109
  }
110
110
 
@@ -278,7 +278,7 @@ async function importPayloadEnvelopesOfKnownBlocks(
278
278
  this.forkChoice.isDescendant(blockRootHex, PayloadStatus.EMPTY, head.blockRoot, head.payloadStatus)
279
279
  ) {
280
280
  // never validated, see processBlocks()
281
- this.seenPayloadEnvelopeInputCache.prune(blockRootHex);
281
+ this.seenPayloadEnvelopeInputCache.remove(blockRootHex);
282
282
  orphaned.push({slot, payloadEnvelopeInput: payloadInput});
283
283
  continue;
284
284
  }
@@ -23,7 +23,7 @@ export enum PayloadEnvelopeInputSource {
23
23
  * - prune: explicit prune by root
24
24
  * - cap: insertion-order backstop cap (MAX_PAYLOAD_ENVELOPE_INPUT_CACHE_SIZE)
25
25
  */
26
- export type PayloadEnvelopeInputPruneReason = "belowParent" | "finalized" | "prune" | "cap";
26
+ export type PayloadEnvelopeInputPruneReason = "belowParent" | "finalized" | "remove" | "cap";
27
27
 
28
28
  export type SourceMeta = {
29
29
  source: PayloadEnvelopeInputSource;
@@ -72,6 +72,10 @@ export enum ChainEvent {
72
72
  * Trigger BlockInputSync to find a SignedExecutionPayloadEnvelope with specified block root.
73
73
  */
74
74
  unknownEnvelopeBlockRoot = "unknownEnvelopeBlockRoot",
75
+ /**
76
+ * Same as unknownEnvelopeBlockRoot, but with slot
77
+ */
78
+ unknownEnvelopeBlockRootSlot = "unknownEnvelopeBlockRootSlot",
75
79
  /**
76
80
  * Trigger BlockInputSync for blocks that are partially received via gossip but are not complete by time the
77
81
  * cut-off window passes for waiting on gossip
@@ -99,8 +103,8 @@ export type ChainEventData = {
99
103
  peer: PeerIdStr;
100
104
  source: BlockInputSource;
101
105
  };
102
- // slot is the message slot, not necessarily the envelope's slot, but useful as a logging/prune hint
103
- [ChainEvent.unknownEnvelopeBlockRoot]: {rootHex: RootHex; slot: Slot; peer?: PeerIdStr; source: BlockInputSource};
106
+ [ChainEvent.unknownEnvelopeBlockRoot]: {rootHex: RootHex; peer?: PeerIdStr; source: BlockInputSource};
107
+ [ChainEvent.unknownEnvelopeBlockRootSlot]: {rootHex: RootHex; slot: Slot; peer?: PeerIdStr; source: BlockInputSource};
104
108
  };
105
109
 
106
110
  export type IChainEvents = ApiEvents & {
@@ -126,6 +130,7 @@ export type IChainEvents = ApiEvents & {
126
130
  [ChainEvent.incompleteBlockInput]: (data: ChainEventData[ChainEvent.incompleteBlockInput]) => void;
127
131
  [ChainEvent.incompletePayloadEnvelope]: (data: ChainEventData[ChainEvent.incompletePayloadEnvelope]) => void;
128
132
  [ChainEvent.unknownEnvelopeBlockRoot]: (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRoot]) => void;
133
+ [ChainEvent.unknownEnvelopeBlockRootSlot]: (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRootSlot]) => void;
129
134
  };
130
135
 
131
136
  /**
@@ -3,9 +3,11 @@ import {GossipActionError} from "./gossipValidation.js";
3
3
  export enum AttesterSlashingErrorCode {
4
4
  ALREADY_EXISTS = "ATTESTATION_SLASHING_ERROR_ALREADY_EXISTS",
5
5
  INVALID = "ATTESTATION_SLASHING_ERROR_INVALID",
6
+ INVALID_SIGNATURE = "ATTESTATION_SLASHING_ERROR_INVALID_SIGNATURE",
6
7
  }
7
8
  export type AttesterSlashingErrorType =
8
9
  | {code: AttesterSlashingErrorCode.ALREADY_EXISTS}
9
- | {code: AttesterSlashingErrorCode.INVALID; error: Error};
10
+ | {code: AttesterSlashingErrorCode.INVALID; error: Error}
11
+ | {code: AttesterSlashingErrorCode.INVALID_SIGNATURE};
10
12
 
11
13
  export class AttesterSlashingError extends GossipActionError<AttesterSlashingErrorType> {}
@@ -3,9 +3,11 @@ import {GossipActionError} from "./gossipValidation.js";
3
3
  export enum ProposerSlashingErrorCode {
4
4
  ALREADY_EXISTS = "PROPOSER_SLASHING_ERROR_ALREADY_EXISTS",
5
5
  INVALID = "PROPOSER_SLASHING_ERROR_INVALID",
6
+ INVALID_SIGNATURE = "PROPOSER_SLASHING_ERROR_INVALID_SIGNATURE",
6
7
  }
7
8
  export type ProposerSlashingErrorType =
8
9
  | {code: ProposerSlashingErrorCode.ALREADY_EXISTS}
9
- | {code: ProposerSlashingErrorCode.INVALID; error: Error};
10
+ | {code: ProposerSlashingErrorCode.INVALID; error: Error}
11
+ | {code: ProposerSlashingErrorCode.INVALID_SIGNATURE};
10
12
 
11
13
  export class ProposerSlashingError extends GossipActionError<ProposerSlashingErrorType> {}
@@ -59,7 +59,18 @@ import {
59
59
  heze,
60
60
  ssz,
61
61
  } from "@lodestar/types";
62
- import {GWEI_TO_WEI, Logger, byteArrayEquals, fromHex, sleep, toHex, toPubkeyHex, toRootHex} from "@lodestar/utils";
62
+ import {
63
+ GWEI_TO_WEI,
64
+ Logger,
65
+ byteArrayEquals,
66
+ fromHex,
67
+ prettyGweiToEth,
68
+ prettyWeiToEth,
69
+ sleep,
70
+ toHex,
71
+ toPubkeyHex,
72
+ toRootHex,
73
+ } from "@lodestar/utils";
63
74
  import {ZERO_HASH_HEX} from "../../constants/index.js";
64
75
  import {numToQuantity} from "../../execution/engine/utils.js";
65
76
  import {IExecutionBuilder, IExecutionEngine, PayloadAttributes, PayloadId} from "../../execution/index.js";
@@ -259,7 +270,7 @@ export async function produceBlockBody<T extends BlockType>(
259
270
  this.logger.verbose("Produced block with builder bid", {
260
271
  slot: blockSlot,
261
272
  builderIndex: builderBid.message.builderIndex,
262
- bidValue: builderBid.message.value,
273
+ bidValue: prettyGweiToEth(builderBid.message.value),
263
274
  parentBlockHash: toRootHex(builderBid.message.parentBlockHash),
264
275
  parentBlockRoot: toRootHex(builderBid.message.parentBlockRoot),
265
276
  blockHash: toRootHex(builderBid.message.blockHash),
@@ -410,7 +421,7 @@ export async function produceBlockBody<T extends BlockType>(
410
421
  this.metrics?.blockPayload.payloadFetchedTime.observe({prepType}, fetchedTime);
411
422
  this.logger.verbose("Produced block with self-build bid", {
412
423
  slot: blockSlot,
413
- executionPayloadValue,
424
+ executionPayloadValue: prettyWeiToEth(executionPayloadValue),
414
425
  prepType,
415
426
  payloadId,
416
427
  fetchedTime,
@@ -490,7 +501,7 @@ export async function produceBlockBody<T extends BlockType>(
490
501
  this.metrics?.blockPayload.payloadFetchedTime.observe({prepType}, fetchedTime);
491
502
  this.logger.verbose("Fetched execution payload header from builder", {
492
503
  slot: blockSlot,
493
- executionPayloadValue,
504
+ executionPayloadValue: prettyWeiToEth(executionPayloadValue),
494
505
  prepType,
495
506
  fetchedTime,
496
507
  });
@@ -607,7 +618,7 @@ export async function produceBlockBody<T extends BlockType>(
607
618
  this.metrics?.blockPayload.payloadFetchedTime.observe({prepType}, fetchedTime);
608
619
  this.logger.verbose("Fetched execution payload from engine", {
609
620
  slot: blockSlot,
610
- executionPayloadValue,
621
+ executionPayloadValue: prettyWeiToEth(executionPayloadValue),
611
622
  prepType,
612
623
  payloadId,
613
624
  fetchedTime,
@@ -708,7 +719,7 @@ export async function produceBlockBody<T extends BlockType>(
708
719
  }
709
720
  }
710
721
 
711
- Object.assign(logMeta, {executionPayloadValue});
722
+ Object.assign(logMeta, {executionPayloadValue: prettyWeiToEth(executionPayloadValue)});
712
723
  this.logger.verbose("Produced beacon block body", logMeta);
713
724
 
714
725
  return {body: blockBody as AssembledBodyType<T>, produceResult, executionPayloadValue, shouldOverrideBuilder};
@@ -256,10 +256,11 @@ export class SeenPayloadEnvelopeInput {
256
256
  return this.payloadInputs.size;
257
257
  }
258
258
 
259
- prune(blockRootHex: RootHex): void {
259
+ /** Removes the single PayloadEnvelopeInput from the cache */
260
+ remove(blockRootHex: RootHex): void {
260
261
  const input = this.payloadInputs.get(blockRootHex);
261
262
  if (input) {
262
- this.evictPayloadInput(input, "prune");
263
+ this.evictPayloadInput(input, "remove");
263
264
  }
264
265
  }
265
266
 
@@ -63,8 +63,7 @@ export async function validateAttesterSlashing(
63
63
  const signatureSets = getAttesterSlashingSignatureSets(chain.config, state.slot, attesterSlashing);
64
64
  if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true, priority: prioritizeBls}))) {
65
65
  throw new AttesterSlashingError(GossipAction.REJECT, {
66
- code: AttesterSlashingErrorCode.INVALID,
67
- error: Error("Invalid signature"),
66
+ code: AttesterSlashingErrorCode.INVALID_SIGNATURE,
68
67
  });
69
68
  }
70
69
  }
@@ -48,8 +48,7 @@ async function validateProposerSlashing(
48
48
  const signatureSets = getProposerSlashingSignatureSets(chain.config, state.slot, proposerSlashing);
49
49
  if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true, priority: prioritizeBls}))) {
50
50
  throw new ProposerSlashingError(GossipAction.REJECT, {
51
- code: ProposerSlashingErrorCode.INVALID,
52
- error: Error("Invalid signature"),
51
+ code: ProposerSlashingErrorCode.INVALID_SIGNATURE,
53
52
  });
54
53
  }
55
54
  }
@@ -24,7 +24,17 @@ import {
24
24
  deneb,
25
25
  gloas,
26
26
  } from "@lodestar/types";
27
- import {LogData, LogHandler, LogLevel, Logger, MapDef, MapDefMax, prettyPrintIndices, toRootHex} from "@lodestar/utils";
27
+ import {
28
+ LogData,
29
+ LogHandler,
30
+ LogLevel,
31
+ Logger,
32
+ MapDef,
33
+ MapDefMax,
34
+ prettyGweiToEth,
35
+ prettyPrintIndices,
36
+ toRootHex,
37
+ } from "@lodestar/utils";
28
38
  import {GENESIS_SLOT} from "../constants/constants.js";
29
39
  import {RegistryMetricCreator} from "../metrics/index.js";
30
40
 
@@ -470,7 +480,7 @@ export function createValidatorMonitor(
470
480
  src,
471
481
  builderIndex: bid.builderIndex,
472
482
  gasLimit: Number(bid.gasLimit),
473
- value: bid.value.toString(),
483
+ value: prettyGweiToEth(bid.value),
474
484
  parentBlockRoot: toRootHex(bid.parentBlockRoot),
475
485
  parentBlockHash: toRootHex(bid.parentBlockHash),
476
486
  blockHash: toRootHex(bid.blockHash),
@@ -1,8 +1,9 @@
1
1
  import {ChainForkConfig} from "@lodestar/config";
2
2
  import {Db, DbBatch, FilterOptions, KeyValue, Repository} from "@lodestar/db";
3
3
  import {Root, SignedBeaconBlock, Slot, ssz} from "@lodestar/types";
4
- import {bytesToInt, intToBytes} from "@lodestar/utils";
4
+ import {bytesToInt, fromHex, intToBytes} from "@lodestar/utils";
5
5
  import {getSignedBlockTypeFromBytes} from "../../util/multifork.js";
6
+ import {getParentRootFromSignedBeaconBlockSerialized} from "../../util/sszBytes.js";
6
7
  import {Bucket, getBucketNameByValue} from "../buckets.js";
7
8
  import {
8
9
  deleteParentRootIndex,
@@ -24,6 +25,10 @@ export type BlockArchiveBatchPutBinaryItem = KeyValue<Slot, Uint8Array> & {
24
25
  parentRoot: Root;
25
26
  };
26
27
 
28
+ // The initial prune can span millions of slots. This bounds the slot index reads in flight and keeps each
29
+ // delete batch at a few thousand keys, both tiny next to the block value batches in archiveBlocks
30
+ const DELETE_RANGE_CHUNK_SIZE = 1000;
31
+
27
32
  /**
28
33
  * Stores finalized blocks. Block slot is identifier.
29
34
  */
@@ -103,6 +108,38 @@ export class BlockArchiveRepository extends Repository<Slot, SignedBeaconBlock>
103
108
  );
104
109
  }
105
110
 
111
+ /**
112
+ * Delete a contiguous range of archived blocks together with their index entries.
113
+ * `slots` must be every archived slot in the range, so the parent of each block is the entry before it.
114
+ */
115
+ async batchDeleteRange(slots: Slot[]): Promise<void> {
116
+ const sorted = [...slots].sort((a, b) => a - b);
117
+ let prevRoot: Root | null = null;
118
+
119
+ for (let i = 0; i < sorted.length; i += DELETE_RANGE_CHUNK_SIZE) {
120
+ const chunk = sorted.slice(i, i + DELETE_RANGE_CHUNK_SIZE);
121
+ const roots = await Promise.all(chunk.map((slot) => this.getRootBySlot(slot)));
122
+
123
+ const keys: Uint8Array[] = [];
124
+ for (let j = 0; j < chunk.length; j++) {
125
+ keys.push(this.encodeKey(chunk[j]), getSlotIndexKey(chunk[j]));
126
+ const root = roots[j];
127
+ if (root) keys.push(getRootIndexKey(root));
128
+ // The parent index entry pointing at this block is keyed by its parent root, which is the
129
+ // previous block's root unless that block is unindexed, then it is read from the block itself
130
+ let parentRoot = prevRoot;
131
+ if (parentRoot === null) {
132
+ const block = await this.getBinary(chunk[j]);
133
+ const parentRootHex = block ? getParentRootFromSignedBeaconBlockSerialized(block) : null;
134
+ parentRoot = parentRootHex ? fromHex(parentRootHex) : null;
135
+ }
136
+ if (parentRoot) keys.push(getParentRootIndexKey(parentRoot));
137
+ prevRoot = root;
138
+ }
139
+ await this.db.batchDelete(keys, this.dbReqOpts);
140
+ }
141
+ }
142
+
106
143
  async batch(operations: DbBatch<Slot, SignedBeaconBlock>): Promise<void> {
107
144
  await this.batchBinary(
108
145
  operations.map((operation) =>
@@ -8,7 +8,7 @@ import {ForkName} from "@lodestar/params";
8
8
  import {intToBytes} from "@lodestar/utils";
9
9
  import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js";
10
10
  import {Eth2GossipsubMetrics} from "./metrics.js";
11
- import {GossipTopicCache, getGossipSSZMaxSize, getGossipSSZType} from "./topic.js";
11
+ import {GossipTopicCache} from "./topic.js";
12
12
 
13
13
  // Load WASM
14
14
  const xxhash = await xxhashFactory();
@@ -97,7 +97,6 @@ export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8
97
97
  export class DataTransformSnappy implements DataTransform {
98
98
  constructor(
99
99
  private readonly gossipTopicCache: GossipTopicCache,
100
- private readonly maxPayloadSize: number,
101
100
  private readonly metrics: Eth2GossipsubMetrics | null
102
101
  ) {}
103
102
 
@@ -112,12 +111,11 @@ export class DataTransformSnappy implements DataTransform {
112
111
  const uncompressedDataLength = snappyWasm.decompress_len(data);
113
112
 
114
113
  const topic = this.gossipTopicCache.getTopic(topicStr);
115
- const sszType = getGossipSSZType(topic);
116
- const maxSize = getGossipSSZMaxSize(topic, this.maxPayloadSize, sszType);
114
+ const {minSize, maxSize} = this.gossipTopicCache.getTypeSizes(topicStr);
117
115
  this.metrics?.dataTransform.inbound.inc({type: topic.type});
118
116
 
119
- if (uncompressedDataLength < sszType.minSize) {
120
- throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} < ${sszType.minSize}`);
117
+ if (uncompressedDataLength < minSize) {
118
+ throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} < ${minSize}`);
121
119
  }
122
120
  if (uncompressedDataLength > maxSize) {
123
121
  throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${maxSize}`);
@@ -136,8 +134,7 @@ export class DataTransformSnappy implements DataTransform {
136
134
  */
137
135
  outboundTransform(topicStr: string, data: Uint8Array): Uint8Array {
138
136
  const topic = this.gossipTopicCache.getTopic(topicStr);
139
- const sszType = getGossipSSZType(topic);
140
- const maxSize = getGossipSSZMaxSize(topic, this.maxPayloadSize, sszType);
137
+ const {maxSize} = this.gossipTopicCache.getTypeSizes(topicStr);
141
138
  this.metrics?.dataTransform.outbound.inc({type: topic.type});
142
139
  if (data.length > maxSize) {
143
140
  throw Error(`ssz_snappy encoded data length ${data.length} > ${maxSize}`);
@@ -14,7 +14,7 @@ import {peerIdFromString} from "@libp2p/peer-id";
14
14
  import {type Multiaddr, multiaddr} from "@multiformats/multiaddr";
15
15
  import {ENR} from "@chainsafe/enr";
16
16
  import {routes} from "@lodestar/api";
17
- import {BeaconConfig, ForkBoundary} from "@lodestar/config";
17
+ import {BeaconConfig, ChainConfig, ForkBoundary} from "@lodestar/config";
18
18
  import {
19
19
  ATTESTATION_SUBNET_COUNT,
20
20
  MAX_SIGNED_AGGREGATE_AND_PROOF_SIZE,
@@ -47,6 +47,22 @@ const GOSSIPSUB_HEARTBEAT_INTERVAL = 0.7 * 1000;
47
47
 
48
48
  const MAX_OUTBOUND_BUFFER_SIZE = 2 ** 24; // 16MB
49
49
 
50
+ /**
51
+ * Snappy worst-case compressed length for a payload of `n` bytes
52
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-beta.0/specs/phase0/p2p-interface.md#max_compressed_len
53
+ */
54
+ function maxCompressedLen(n: number): number {
55
+ return 32 + n + Math.floor(n / 6);
56
+ }
57
+
58
+ /**
59
+ * Max size of an inbound gossipsub RPC frame, including all bundled messages and control
60
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-beta.0/specs/phase0/p2p-interface.md#max_message_size
61
+ */
62
+ function getMaxInboundDataLength(config: Pick<ChainConfig, "MAX_PAYLOAD_SIZE">): number {
63
+ return Math.max(maxCompressedLen(config.MAX_PAYLOAD_SIZE) + 1024, 1024 * 1024);
64
+ }
65
+
50
66
  export type Eth2Context = {
51
67
  activeValidatorCount: number;
52
68
  currentSlot: number;
@@ -168,7 +184,7 @@ export class Eth2Gossipsub {
168
184
  fastMsgIdFn: fastMsgIdFn,
169
185
  msgIdFn: msgIdFn.bind(msgIdFn, gossipTopicCache),
170
186
  msgIdToStrFn: msgIdToStrFn,
171
- dataTransform: new DataTransformSnappy(gossipTopicCache, config.MAX_PAYLOAD_SIZE, metrics),
187
+ dataTransform: new DataTransformSnappy(gossipTopicCache, metrics),
172
188
  metricsRegister: metricsRegister as MetricsRegister | null,
173
189
  metricsTopicStrToLabel: metricsRegister
174
190
  ? getMetricsTopicStrToLabel(networkConfig, {disableLightClientServer: opts.disableLightClientServer ?? false})
@@ -176,6 +192,7 @@ export class Eth2Gossipsub {
176
192
  asyncValidation: true,
177
193
 
178
194
  maxOutboundBufferSize: MAX_OUTBOUND_BUFFER_SIZE,
195
+ maxInboundDataLength: getMaxInboundDataLength(config),
179
196
  // serialize message once and send to all peers when publishing
180
197
  batchPublish: true,
181
198
  // if this is false, only publish to mesh peers. If there is not enough GOSSIP_D mesh peers,
@@ -1,5 +1,5 @@
1
1
  import {type CompositeTypeAny} from "@chainsafe/ssz";
2
- import {ForkDigestContext} from "@lodestar/config";
2
+ import {BeaconConfig, ChainForkConfig, ForkDigestContext} from "@lodestar/config";
3
3
  import {
4
4
  ATTESTATION_SUBNET_COUNT,
5
5
  ForkName,
@@ -10,32 +10,47 @@ import {
10
10
  isForkPostFulu,
11
11
  isForkPostGloas,
12
12
  } from "@lodestar/params";
13
+ import {TypeSizes} from "@lodestar/reqresp";
13
14
  import {Attestation, SingleAttestation, ssz, sszTypesFor} from "@lodestar/types";
14
15
  import {GossipAction, GossipActionError, GossipErrorCode} from "../../chain/errors/gossipValidation.js";
16
+ import {computeMaxGloasDataColumnSidecarSize} from "../../util/sszBytes.js";
15
17
  import {NetworkConfig} from "../networkConfig.js";
16
18
  import {DEFAULT_ENCODING} from "./constants.js";
17
19
  import {GossipEncoding, GossipTopic, GossipTopicTypeMap, GossipType, SSZTypeOfGossipTopic} from "./interface.js";
18
20
 
19
21
  export interface IGossipTopicCache {
20
22
  getTopic(topicStr: string): GossipTopic;
23
+ getTypeSizes(topicStr: string): TypeSizes;
21
24
  }
22
25
 
23
26
  export class GossipTopicCache implements IGossipTopicCache {
24
27
  private topicsByTopicStr = new Map<string, Required<GossipTopic>>();
28
+ private typeSizesByTopicStr = new Map<string, TypeSizes>();
25
29
 
26
- constructor(private readonly forkDigestContext: ForkDigestContext) {}
30
+ constructor(private readonly config: BeaconConfig) {}
27
31
 
28
32
  /** Returns cached GossipTopic, otherwise attempts to parse it from the str */
29
33
  getTopic(topicStr: string): GossipTopic {
30
34
  let topic = this.topicsByTopicStr.get(topicStr);
31
35
  if (topic === undefined) {
32
- topic = parseGossipTopic(this.forkDigestContext, topicStr);
36
+ topic = parseGossipTopic(this.config, topicStr);
33
37
  // TODO: Consider just throwing here. We should only receive messages from known subscribed topics
34
38
  this.topicsByTopicStr.set(topicStr, topic);
35
39
  }
36
40
  return topic;
37
41
  }
38
42
 
43
+ getTypeSizes(topicStr: string): TypeSizes {
44
+ let typeSizes = this.typeSizesByTopicStr.get(topicStr);
45
+ if (typeSizes === undefined) {
46
+ const topic = this.getTopic(topicStr);
47
+ const sszType = getGossipSSZType(topic);
48
+ typeSizes = {minSize: sszType.minSize, maxSize: getGossipSSZMaxSize(topic, this.config, sszType)};
49
+ this.typeSizesByTopicStr.set(topicStr, typeSizes);
50
+ }
51
+ return typeSizes;
52
+ }
53
+
39
54
  /** Returns cached GossipTopic, otherwise returns undefined */
40
55
  getKnownTopic(topicStr: string): GossipTopic | undefined {
41
56
  return this.topicsByTopicStr.get(topicStr);
@@ -133,11 +148,14 @@ export function getGossipSSZType(topic: GossipTopic) {
133
148
  }
134
149
 
135
150
  /**
136
- * Return the maximum uncompressed SSZ byte length accepted for a gossip object, the SSZ type max size
137
- * or MAX_PAYLOAD_SIZE, whichever is smaller.
151
+ * Return the maximum uncompressed SSZ byte length allowed by the type and configured network bounds.
138
152
  */
139
- export function getGossipSSZMaxSize(topic: GossipTopic, maxPayloadSize: number, sszType?: CompositeTypeAny): number {
140
- return Math.min((sszType ?? getGossipSSZType(topic)).maxSize, maxPayloadSize);
153
+ export function getGossipSSZMaxSize(topic: GossipTopic, config: ChainForkConfig, sszType: CompositeTypeAny): number {
154
+ const maxSize = Math.min(sszType.maxSize, config.MAX_PAYLOAD_SIZE);
155
+ if (isForkPostGloas(topic.boundary.fork) && topic.type === GossipType.data_column_sidecar) {
156
+ return Math.min(maxSize, computeMaxGloasDataColumnSidecarSize(config));
157
+ }
158
+ return maxSize;
141
159
  }
142
160
 
143
161
  /**
@@ -244,8 +244,20 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
244
244
 
245
245
  // IGNORE means the block is acceptable (e.g. FUTURE_SLOT, ALREADY_KNOWN), just not propagated.
246
246
  // Keep the optimistically-added cache entries; they are pruned on finalization. Only REJECT
247
- // (provably invalid) and unexpected errors prune below.
247
+ // (provably invalid), unexpected errors and repeat proposals that are not imported prune.
248
248
  if (e.action === GossipAction.IGNORE) {
249
+ // Only a signature-verified sibling is imported by the beacon_block handler, any other repeat proposal
250
+ // is dropped from the caches and re-downloaded by sync if it ever becomes relevant. Its signature was not
251
+ // verified, so only its own entry is removed, never its claimed ancestors
252
+ if (
253
+ e.type.code === BlockErrorCode.REPEAT_PROPOSAL &&
254
+ !chain.seenBlockProposers.hasBlockRoot(slot, signedBlock.message.proposerIndex, blockRootHex)
255
+ ) {
256
+ chain.seenBlockInputCache.remove(blockRootHex);
257
+ if (isForkPostGloas(fork)) {
258
+ chain.seenPayloadEnvelopeInputCache.remove(blockRootHex);
259
+ }
260
+ }
249
261
  throw e;
250
262
  }
251
263
 
@@ -258,10 +270,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
258
270
  }
259
271
 
260
272
  // REJECT or unexpected (non-BlockGossipError) error: drop the optimistically-added entries from
261
- // both caches, keeping them consistent.
262
- chain.seenBlockInputCache.prune(blockRootHex);
273
+ // both caches, keeping them consistent. The block may carry any parent root, so only its own entry is removed
274
+ chain.seenBlockInputCache.remove(blockRootHex);
263
275
  if (isForkPostGloas(fork)) {
264
- chain.seenPayloadEnvelopeInputCache.prune(blockRootHex);
276
+ chain.seenPayloadEnvelopeInputCache.remove(blockRootHex);
265
277
  }
266
278
  throw e;
267
279
  } finally {
@@ -738,7 +750,8 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
738
750
  if (
739
751
  e instanceof BlockGossipError &&
740
752
  e.type.code === BlockErrorCode.REPEAT_PROPOSAL &&
741
- // this is make sure the block's proposer signature was verified, it should be true anyway
753
+ // Only a signature-verified sibling recorded in the seen cache is imported. The cache holds at most two
754
+ // roots per proposer and slot, which bounds full imports over gossip to one alternate
742
755
  chain.seenBlockProposers.hasBlockRoot(signedBlock.message.slot, e.type.proposerIndex, e.type.root)
743
756
  ) {
744
757
  // blockInput was optimistically seeded in validateBeaconBlock and retained on IGNORE