@lodestar/beacon-node 1.36.0 → 1.37.0-dev.0c3b3f119c

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/lib/api/impl/validator/index.d.ts.map +1 -1
  2. package/lib/api/impl/validator/index.js +9 -3
  3. package/lib/api/impl/validator/index.js.map +1 -1
  4. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  5. package/lib/chain/blocks/importBlock.js +4 -2
  6. package/lib/chain/blocks/importBlock.js.map +1 -1
  7. package/lib/chain/blocks/writeBlockInputToDb.d.ts.map +1 -1
  8. package/lib/chain/blocks/writeBlockInputToDb.js +14 -1
  9. package/lib/chain/blocks/writeBlockInputToDb.js.map +1 -1
  10. package/lib/chain/prepareNextSlot.d.ts.map +1 -1
  11. package/lib/chain/prepareNextSlot.js +2 -1
  12. package/lib/chain/prepareNextSlot.js.map +1 -1
  13. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  14. package/lib/chain/produceBlock/produceBlockBody.js +2 -1
  15. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  16. package/lib/chain/seenCache/seenGossipBlockInput.d.ts +11 -1
  17. package/lib/chain/seenCache/seenGossipBlockInput.d.ts.map +1 -1
  18. package/lib/chain/seenCache/seenGossipBlockInput.js +30 -1
  19. package/lib/chain/seenCache/seenGossipBlockInput.js.map +1 -1
  20. package/lib/chain/validation/blobSidecar.d.ts.map +1 -1
  21. package/lib/chain/validation/blobSidecar.js +29 -22
  22. package/lib/chain/validation/blobSidecar.js.map +1 -1
  23. package/lib/chain/validation/block.d.ts.map +1 -1
  24. package/lib/chain/validation/block.js +10 -7
  25. package/lib/chain/validation/block.js.map +1 -1
  26. package/lib/chain/validation/dataColumnSidecar.d.ts.map +1 -1
  27. package/lib/chain/validation/dataColumnSidecar.js +33 -26
  28. package/lib/chain/validation/dataColumnSidecar.js.map +1 -1
  29. package/lib/network/gossip/encoding.d.ts +4 -2
  30. package/lib/network/gossip/encoding.d.ts.map +1 -1
  31. package/lib/network/gossip/encoding.js +9 -3
  32. package/lib/network/gossip/encoding.js.map +1 -1
  33. package/lib/network/gossip/gossipsub.d.ts.map +1 -1
  34. package/lib/network/gossip/gossipsub.js +6 -5
  35. package/lib/network/gossip/gossipsub.js.map +1 -1
  36. package/lib/network/gossip/metrics.d.ts +8 -0
  37. package/lib/network/gossip/metrics.d.ts.map +1 -1
  38. package/lib/network/gossip/metrics.js +12 -0
  39. package/lib/network/gossip/metrics.js.map +1 -1
  40. package/lib/network/interface.d.ts +3 -3
  41. package/lib/network/interface.d.ts.map +1 -1
  42. package/lib/network/network.d.ts +3 -3
  43. package/lib/network/network.d.ts.map +1 -1
  44. package/lib/network/network.js +3 -3
  45. package/lib/network/network.js.map +1 -1
  46. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  47. package/lib/network/processor/gossipHandlers.js +2 -0
  48. package/lib/network/processor/gossipHandlers.js.map +1 -1
  49. package/lib/network/reqresp/utils/collect.d.ts +3 -3
  50. package/lib/network/reqresp/utils/collect.d.ts.map +1 -1
  51. package/lib/network/reqresp/utils/collect.js +7 -3
  52. package/lib/network/reqresp/utils/collect.js.map +1 -1
  53. package/lib/network/reqresp/utils/collectSequentialBlocksInRange.d.ts +3 -2
  54. package/lib/network/reqresp/utils/collectSequentialBlocksInRange.d.ts.map +1 -1
  55. package/lib/network/reqresp/utils/collectSequentialBlocksInRange.js +5 -3
  56. package/lib/network/reqresp/utils/collectSequentialBlocksInRange.js.map +1 -1
  57. package/lib/sync/backfill/backfill.d.ts.map +1 -1
  58. package/lib/sync/backfill/backfill.js +37 -14
  59. package/lib/sync/backfill/backfill.js.map +1 -1
  60. package/lib/sync/backfill/verify.d.ts +4 -4
  61. package/lib/sync/backfill/verify.d.ts.map +1 -1
  62. package/lib/sync/backfill/verify.js +6 -6
  63. package/lib/sync/backfill/verify.js.map +1 -1
  64. package/lib/sync/utils/downloadByRange.js +1 -1
  65. package/lib/sync/utils/downloadByRange.js.map +1 -1
  66. package/lib/sync/utils/downloadByRoot.js +1 -1
  67. package/lib/sync/utils/downloadByRoot.js.map +1 -1
  68. package/package.json +14 -19
  69. package/src/api/impl/validator/index.ts +9 -2
  70. package/src/chain/blocks/importBlock.ts +5 -1
  71. package/src/chain/blocks/writeBlockInputToDb.ts +13 -1
  72. package/src/chain/prepareNextSlot.ts +2 -1
  73. package/src/chain/produceBlock/produceBlockBody.ts +2 -1
  74. package/src/chain/seenCache/seenGossipBlockInput.ts +33 -2
  75. package/src/chain/validation/blobSidecar.ts +33 -24
  76. package/src/chain/validation/block.ts +11 -7
  77. package/src/chain/validation/dataColumnSidecar.ts +43 -33
  78. package/src/network/gossip/encoding.ts +9 -3
  79. package/src/network/gossip/gossipsub.ts +8 -6
  80. package/src/network/gossip/metrics.ts +12 -0
  81. package/src/network/interface.ts +2 -9
  82. package/src/network/network.ts +8 -9
  83. package/src/network/processor/gossipHandlers.ts +2 -0
  84. package/src/network/reqresp/utils/collect.ts +12 -6
  85. package/src/network/reqresp/utils/collectSequentialBlocksInRange.ts +10 -6
  86. package/src/sync/backfill/backfill.ts +40 -16
  87. package/src/sync/backfill/verify.ts +10 -10
  88. package/src/sync/utils/downloadByRange.ts +1 -1
  89. package/src/sync/utils/downloadByRoot.ts +1 -1
  90. package/lib/network/gossip/snappy_bun.d.ts +0 -3
  91. package/lib/network/gossip/snappy_bun.d.ts.map +0 -1
  92. package/lib/network/gossip/snappy_bun.js +0 -3
  93. package/lib/network/gossip/snappy_bun.js.map +0 -1
  94. package/src/network/gossip/snappy_bun.ts +0 -2
@@ -60,7 +60,19 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInputs: IBloc
60
60
  );
61
61
  }
62
62
 
63
- fnPromises.push(this.db.dataColumnSidecar.putMany(blockRoot, dataColumnSidecars));
63
+ const binaryPuts = [];
64
+ const nonbinaryPuts = [];
65
+ for (const dataColumnSidecar of dataColumnSidecars) {
66
+ // skip reserializing column if we already have it
67
+ const serialized = this.serializedCache.get(dataColumnSidecar);
68
+ if (serialized) {
69
+ binaryPuts.push({key: dataColumnSidecar.index, value: serialized});
70
+ } else {
71
+ nonbinaryPuts.push(dataColumnSidecar);
72
+ }
73
+ }
74
+ fnPromises.push(this.db.dataColumnSidecar.putManyBinary(blockRoot, binaryPuts));
75
+ fnPromises.push(this.db.dataColumnSidecar.putMany(blockRoot, nonbinaryPuts));
64
76
  this.logger.debug("Persisted dataColumnSidecars to hot DB", {
65
77
  slot: block.message.slot,
66
78
  root: blockRootHex,
@@ -1,5 +1,6 @@
1
1
  import {routes} from "@lodestar/api";
2
2
  import {ChainForkConfig} from "@lodestar/config";
3
+ import {getSafeExecutionBlockHash} from "@lodestar/fork-choice";
3
4
  import {ForkPostBellatrix, ForkSeq, SLOTS_PER_EPOCH, isForkPostElectra} from "@lodestar/params";
4
5
  import {
5
6
  BeaconStateElectra,
@@ -166,7 +167,7 @@ export class PrepareNextSlotScheduler {
166
167
  computeTimeAtSlot(this.config, prepareSlot, this.chain.genesisTime) - Date.now() / 1000;
167
168
  this.metrics?.blockPayload.payloadAdvancePrepTime.observe(preparationTime);
168
169
 
169
- const safeBlockHash = this.chain.forkChoice.getJustifiedBlock().executionPayloadBlockHash ?? ZERO_HASH_HEX;
170
+ const safeBlockHash = getSafeExecutionBlockHash(this.chain.forkChoice);
170
171
  const finalizedBlockHash =
171
172
  this.chain.forkChoice.getFinalizedBlock().executionPayloadBlockHash ?? ZERO_HASH_HEX;
172
173
  // awaiting here instead of throwing an async call because there is no other task
@@ -1,4 +1,5 @@
1
1
  import {ChainForkConfig} from "@lodestar/config";
2
+ import {getSafeExecutionBlockHash} from "@lodestar/fork-choice";
2
3
  import {
3
4
  ForkName,
4
5
  ForkPostBellatrix,
@@ -191,7 +192,7 @@ export async function produceBlockBody<T extends BlockType>(
191
192
 
192
193
  // We don't deal with blinded blocks, execution engine, blobs and execution requests post-gloas
193
194
  } else if (isForkPostBellatrix(fork)) {
194
- const safeBlockHash = this.forkChoice.getJustifiedBlock().executionPayloadBlockHash ?? ZERO_HASH_HEX;
195
+ const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice);
195
196
  const finalizedBlockHash = this.forkChoice.getFinalizedBlock().executionPayloadBlockHash ?? ZERO_HASH_HEX;
196
197
  const feeRecipient = requestedFeeRecipient ?? this.beaconProposerCache.getOrDefault(proposerIndex);
197
198
  const feeRecipientType = requestedFeeRecipient
@@ -2,8 +2,8 @@ import {ChainForkConfig} from "@lodestar/config";
2
2
  import {CheckpointWithHex} from "@lodestar/fork-choice";
3
3
  import {ForkName, ForkPostFulu, ForkPreGloas, isForkPostDeneb, isForkPostFulu, isForkPostGloas} from "@lodestar/params";
4
4
  import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
5
- import {RootHex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types";
6
- import {LodestarError, Logger} from "@lodestar/utils";
5
+ import {BLSSignature, RootHex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types";
6
+ import {LodestarError, Logger, pruneSetToMax} from "@lodestar/utils";
7
7
  import {Metrics} from "../../metrics/metrics.js";
8
8
  import {IClock} from "../../util/clock.js";
9
9
  import {CustodyConfig} from "../../util/dataColumns.js";
@@ -83,6 +83,10 @@ export class SeenBlockInput {
83
83
  private readonly metrics: Metrics | null;
84
84
  private readonly logger?: Logger;
85
85
  private blockInputs = new Map<RootHex, IBlockInput>();
86
+ // using a Map of slot helps it more convenient to prune
87
+ // there should only 1 block root per slot but we need to always compare against rootHex
88
+ // and the signature to ensure we only skip verification if both match
89
+ private verifiedProposerSignatures = new Map<Slot, Map<RootHex, BLSSignature>>();
86
90
 
87
91
  constructor({config, custodyConfig, clock, chainEvents, signal, metrics, logger}: SeenBlockInputCacheModules) {
88
92
  this.config = config;
@@ -330,6 +334,32 @@ export class SeenBlockInput {
330
334
  return blockInput;
331
335
  }
332
336
 
337
+ /**
338
+ * Check if a proposer signature has already been verified for this slot and block root.
339
+ */
340
+ isVerifiedProposerSignature(slot: Slot, blockRootHex: RootHex, signature: BLSSignature): boolean {
341
+ const seenMap = this.verifiedProposerSignatures.get(slot);
342
+ const cachedSignature = seenMap?.get(blockRootHex);
343
+ if (!cachedSignature) {
344
+ return false;
345
+ }
346
+ // Only consider verified if the signature matches
347
+ return Buffer.compare(cachedSignature, signature) === 0;
348
+ }
349
+
350
+ /**
351
+ * Mark that the proposer signature for this slot and block root has been verified
352
+ * so that we only verify it once per slot
353
+ */
354
+ markVerifiedProposerSignature(slot: Slot, blockRootHex: RootHex, signature: BLSSignature): void {
355
+ let seenMap = this.verifiedProposerSignatures.get(slot);
356
+ if (!seenMap) {
357
+ seenMap = new Map<RootHex, BLSSignature>();
358
+ this.verifiedProposerSignatures.set(slot, seenMap);
359
+ }
360
+ seenMap.set(blockRootHex, signature);
361
+ }
362
+
333
363
  private buildCommonProps(slot: Slot): {
334
364
  daOutOfRange: boolean;
335
365
  forkName: ForkName;
@@ -356,6 +386,7 @@ export class SeenBlockInput {
356
386
  if (itemsToDelete <= 0) return;
357
387
  }
358
388
  }
389
+ pruneSetToMax(this.verifiedProposerSignatures, MAX_BLOCK_INPUT_CACHE_SIZE);
359
390
  }
360
391
  }
361
392
 
@@ -135,15 +135,20 @@ export async function validateGossipBlobSidecar(
135
135
  });
136
136
 
137
137
  // [REJECT] The proposer signature, signed_beacon_block.signature, is valid with respect to the proposer_index pubkey.
138
- const signatureSet = getBlockHeaderProposerSignatureSetByParentStateSlot(blockState, blobSidecar.signedBlockHeader);
139
- // Don't batch so verification is not delayed
140
- if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) {
141
- throw new BlobSidecarGossipError(GossipAction.REJECT, {
142
- code: BlobSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
143
- blockRoot: blockHex,
144
- index: blobSidecar.index,
145
- slot: blobSlot,
146
- });
138
+ const signature = blobSidecar.signedBlockHeader.signature;
139
+ if (!chain.seenBlockInputCache.isVerifiedProposerSignature(blobSlot, blockHex, signature)) {
140
+ const signatureSet = getBlockHeaderProposerSignatureSetByParentStateSlot(blockState, blobSidecar.signedBlockHeader);
141
+ // Don't batch so verification is not delayed
142
+ if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) {
143
+ throw new BlobSidecarGossipError(GossipAction.REJECT, {
144
+ code: BlobSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
145
+ blockRoot: blockHex,
146
+ index: blobSidecar.index,
147
+ slot: blobSlot,
148
+ });
149
+ }
150
+
151
+ chain.seenBlockInputCache.markVerifiedProposerSignature(blobSlot, blockHex, signature);
147
152
  }
148
153
 
149
154
  // verify if the blob inclusion proof is correct
@@ -231,22 +236,26 @@ export async function validateBlockBlobSidecars(
231
236
  }
232
237
 
233
238
  if (chain !== null) {
234
- const headState = await chain.getHeadState();
235
- const signatureSet = getBlockHeaderProposerSignatureSetByHeaderSlot(headState, firstSidecarSignedBlockHeader);
239
+ const blockRootHex = toRootHex(blockRoot);
240
+ const signature = firstSidecarSignedBlockHeader.signature;
241
+ if (!chain.seenBlockInputCache.isVerifiedProposerSignature(blockSlot, blockRootHex, signature)) {
242
+ const headState = await chain.getHeadState();
243
+ const signatureSet = getBlockHeaderProposerSignatureSetByHeaderSlot(headState, firstSidecarSignedBlockHeader);
236
244
 
237
- if (
238
- !(await chain.bls.verifySignatureSets([signatureSet], {
239
- batchable: true,
240
- priority: true,
241
- verifyOnMainThread: false,
242
- }))
243
- ) {
244
- throw new BlobSidecarValidationError({
245
- code: BlobSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
246
- blockRoot: toRootHex(blockRoot),
247
- slot: blockSlot,
248
- index: blobSidecars[0].index,
249
- });
245
+ if (
246
+ !(await chain.bls.verifySignatureSets([signatureSet], {
247
+ verifyOnMainThread: true,
248
+ }))
249
+ ) {
250
+ throw new BlobSidecarValidationError({
251
+ code: BlobSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
252
+ blockRoot: blockRootHex,
253
+ slot: blockSlot,
254
+ index: blobSidecars[0].index,
255
+ });
256
+ }
257
+
258
+ chain.seenBlockInputCache.markVerifiedProposerSignature(blockSlot, blockRootHex, signature);
250
259
  }
251
260
  }
252
261
 
@@ -153,13 +153,17 @@ export async function validateGossipBlock(
153
153
  }
154
154
 
155
155
  // [REJECT] The proposer signature, signed_beacon_block.signature, is valid with respect to the proposer_index pubkey.
156
- const signatureSet = getBlockProposerSignatureSet(blockState, signedBlock);
157
- // Don't batch so verification is not delayed
158
- if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) {
159
- throw new BlockGossipError(GossipAction.REJECT, {
160
- code: BlockErrorCode.PROPOSAL_SIGNATURE_INVALID,
161
- blockSlot,
162
- });
156
+ if (!chain.seenBlockInputCache.isVerifiedProposerSignature(blockSlot, blockRoot, signedBlock.signature)) {
157
+ const signatureSet = getBlockProposerSignatureSet(blockState, signedBlock);
158
+ // Don't batch so verification is not delayed
159
+ if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) {
160
+ throw new BlockGossipError(GossipAction.REJECT, {
161
+ code: BlockErrorCode.PROPOSAL_SIGNATURE_INVALID,
162
+ blockSlot,
163
+ });
164
+ }
165
+
166
+ chain.seenBlockInputCache.markVerifiedProposerSignature(blockSlot, blockRoot, signedBlock.signature);
163
167
  }
164
168
 
165
169
  // [REJECT] The block is proposed by the expected proposer_index for the block's slot in the context of the current
@@ -32,6 +32,7 @@ export async function validateGossipDataColumnSidecar(
32
32
  metrics: Metrics | null
33
33
  ): Promise<void> {
34
34
  const blockHeader = dataColumnSidecar.signedBlockHeader.message;
35
+ const blockRootHex = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader));
35
36
 
36
37
  // 1) [REJECT] The sidecar is valid as verified by verify_data_column_sidecar
37
38
  verifyDataColumnSidecar(chain.config, dataColumnSidecar);
@@ -131,24 +132,28 @@ export async function validateGossipDataColumnSidecar(
131
132
  }
132
133
 
133
134
  // 5) [REJECT] The proposer signature of sidecar.signed_block_header, is valid with respect to the block_header.proposer_index pubkey.
134
- const signatureSet = getBlockHeaderProposerSignatureSetByParentStateSlot(
135
- blockState,
136
- dataColumnSidecar.signedBlockHeader
137
- );
138
- // Don't batch so verification is not delayed
139
- if (
140
- !(await chain.bls.verifySignatureSets([signatureSet], {
141
- verifyOnMainThread: blockHeader.slot > chain.forkChoice.getHead().slot,
142
- }))
143
- ) {
144
- const blockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(dataColumnSidecar.signedBlockHeader.message);
145
- const blockRootHex = toRootHex(blockRoot);
146
- throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
147
- code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
148
- blockRoot: blockRootHex,
149
- index: dataColumnSidecar.index,
150
- slot: blockHeader.slot,
151
- });
135
+ const signature = dataColumnSidecar.signedBlockHeader.signature;
136
+ if (!chain.seenBlockInputCache.isVerifiedProposerSignature(blockHeader.slot, blockRootHex, signature)) {
137
+ const signatureSet = getBlockHeaderProposerSignatureSetByParentStateSlot(
138
+ blockState,
139
+ dataColumnSidecar.signedBlockHeader
140
+ );
141
+
142
+ if (
143
+ !(await chain.bls.verifySignatureSets([signatureSet], {
144
+ // verify on main thread so that we only need to verify block proposer signature once per block
145
+ verifyOnMainThread: true,
146
+ }))
147
+ ) {
148
+ throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
149
+ code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
150
+ blockRoot: blockRootHex,
151
+ index: dataColumnSidecar.index,
152
+ slot: blockHeader.slot,
153
+ });
154
+ }
155
+
156
+ chain.seenBlockInputCache.markVerifiedProposerSignature(blockHeader.slot, blockRootHex, signature);
152
157
  }
153
158
 
154
159
  // 9) [REJECT] The current finalized_checkpoint is an ancestor of the sidecar's block
@@ -326,22 +331,27 @@ export async function validateBlockDataColumnSidecars(
326
331
  }
327
332
 
328
333
  if (chain !== null) {
329
- const headState = await chain.getHeadState();
330
- const signatureSet = getBlockHeaderProposerSignatureSetByHeaderSlot(headState, firstSidecarSignedBlockHeader);
334
+ const rootHex = toRootHex(blockRoot);
335
+ const slot = firstSidecarSignedBlockHeader.message.slot;
336
+ const signature = firstSidecarSignedBlockHeader.signature;
337
+ if (!chain.seenBlockInputCache.isVerifiedProposerSignature(slot, rootHex, signature)) {
338
+ const headState = await chain.getHeadState();
339
+ const signatureSet = getBlockHeaderProposerSignatureSetByHeaderSlot(headState, firstSidecarSignedBlockHeader);
340
+
341
+ if (
342
+ !(await chain.bls.verifySignatureSets([signatureSet], {
343
+ verifyOnMainThread: true,
344
+ }))
345
+ ) {
346
+ throw new DataColumnSidecarValidationError({
347
+ code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
348
+ blockRoot: rootHex,
349
+ slot: blockSlot,
350
+ index: dataColumnSidecars[0].index,
351
+ });
352
+ }
331
353
 
332
- if (
333
- !(await chain.bls.verifySignatureSets([signatureSet], {
334
- batchable: true,
335
- priority: true,
336
- verifyOnMainThread: false,
337
- }))
338
- ) {
339
- throw new DataColumnSidecarValidationError({
340
- code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
341
- blockRoot: toRootHex(blockRoot),
342
- slot: blockSlot,
343
- index: dataColumnSidecars[0].index,
344
- });
354
+ chain.seenBlockInputCache.markVerifiedProposerSignature(slot, rootHex, signature);
345
355
  }
346
356
  }
347
357
 
@@ -1,12 +1,14 @@
1
1
  import {Message} from "@libp2p/interface";
2
+ // snappyjs is better for compression for smaller payloads
3
+ import {compress, uncompress} from "snappyjs";
2
4
  import xxhashFactory from "xxhash-wasm";
3
- import {compress, uncompress} from "#snappy";
4
5
  import {digest} from "@chainsafe/as-sha256";
5
6
  import {RPC} from "@chainsafe/libp2p-gossipsub/message";
6
7
  import {DataTransform} from "@chainsafe/libp2p-gossipsub/types";
7
8
  import {ForkName} from "@lodestar/params";
8
9
  import {intToBytes} from "@lodestar/utils";
9
10
  import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js";
11
+ import {Eth2GossipsubMetrics} from "./metrics.js";
10
12
  import {GossipTopicCache, getGossipSSZType} from "./topic.js";
11
13
 
12
14
  // Load WASM
@@ -69,7 +71,8 @@ export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8
69
71
  export class DataTransformSnappy implements DataTransform {
70
72
  constructor(
71
73
  private readonly gossipTopicCache: GossipTopicCache,
72
- private readonly maxSizePerMessage: number
74
+ private readonly maxSizePerMessage: number,
75
+ private readonly metrics: Eth2GossipsubMetrics | null
73
76
  ) {}
74
77
 
75
78
  /**
@@ -86,6 +89,7 @@ export class DataTransformSnappy implements DataTransform {
86
89
  const uncompressedDataLength = uncompressedData.length;
87
90
  const topic = this.gossipTopicCache.getTopic(topicStr);
88
91
  const sszType = getGossipSSZType(topic);
92
+ this.metrics?.dataTransform.inbound.inc({type: topic.type});
89
93
 
90
94
  if (uncompressedDataLength < sszType.minSize) {
91
95
  throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} < ${sszType.minSize}`);
@@ -101,7 +105,9 @@ export class DataTransformSnappy implements DataTransform {
101
105
  * Takes the data to be published (a topic and associated data) transforms the data. The
102
106
  * transformed data will then be used to create a `RawGossipsubMessage` to be sent to peers.
103
107
  */
104
- outboundTransform(_topicStr: string, data: Uint8Array): Uint8Array {
108
+ outboundTransform(topicStr: string, data: Uint8Array): Uint8Array {
109
+ const topic = this.gossipTopicCache.getTopic(topicStr);
110
+ this.metrics?.dataTransform.outbound.inc({type: topic.type});
105
111
  if (data.length > this.maxSizePerMessage) {
106
112
  throw Error(`ssz_snappy encoded data length ${data.length} > ${this.maxSizePerMessage}`);
107
113
  }
@@ -89,6 +89,13 @@ export class Eth2Gossipsub extends GossipSub {
89
89
  const gossipTopicCache = new GossipTopicCache(config);
90
90
 
91
91
  const scoreParams = computeGossipPeerScoreParams({config, eth2Context: modules.eth2Context});
92
+ let metrics: Eth2GossipsubMetrics | null = null;
93
+ if (metricsRegister) {
94
+ metrics = createEth2GossipsubMetrics(metricsRegister);
95
+ metrics.gossipMesh.peersByType.addCollect(() =>
96
+ this.onScrapeLodestarMetrics(metrics as Eth2GossipsubMetrics, networkConfig)
97
+ );
98
+ }
92
99
 
93
100
  // Gossipsub parameters defined here:
94
101
  // https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub
@@ -116,7 +123,7 @@ export class Eth2Gossipsub extends GossipSub {
116
123
  fastMsgIdFn: fastMsgIdFn,
117
124
  msgIdFn: msgIdFn.bind(msgIdFn, gossipTopicCache),
118
125
  msgIdToStrFn: msgIdToStrFn,
119
- dataTransform: new DataTransformSnappy(gossipTopicCache, config.MAX_PAYLOAD_SIZE),
126
+ dataTransform: new DataTransformSnappy(gossipTopicCache, config.MAX_PAYLOAD_SIZE, metrics),
120
127
  metricsRegister: metricsRegister as MetricsRegister | null,
121
128
  metricsTopicStrToLabel: metricsRegister
122
129
  ? getMetricsTopicStrToLabel(networkConfig, {disableLightClientServer: opts.disableLightClientServer ?? false})
@@ -141,11 +148,6 @@ export class Eth2Gossipsub extends GossipSub {
141
148
  this.events = events;
142
149
  this.gossipTopicCache = gossipTopicCache;
143
150
 
144
- if (metricsRegister) {
145
- const metrics = createEth2GossipsubMetrics(metricsRegister);
146
- metrics.gossipMesh.peersByType.addCollect(() => this.onScrapeLodestarMetrics(metrics, networkConfig));
147
- }
148
-
149
151
  this.addEventListener("gossipsub:message", this.onGossipsubMessage.bind(this));
150
152
  this.events.on(NetworkEvent.gossipMessageValidationResult, this.onValidationResult.bind(this));
151
153
 
@@ -67,5 +67,17 @@ export function createEth2GossipsubMetrics(register: RegistryMetricCreator) {
67
67
  labelNames: ["subnet", "boundary"],
68
68
  }),
69
69
  },
70
+ dataTransform: {
71
+ inbound: register.counter<{type: GossipType}>({
72
+ name: "lodestar_gossip_data_transform_inbound_total",
73
+ help: "Total number of inbound data transforms by gossip type",
74
+ labelNames: ["type"],
75
+ }),
76
+ outbound: register.counter<{type: GossipType}>({
77
+ name: "lodestar_gossip_data_transform_outbound_total",
78
+ help: "Total number of outbound data transforms by gossip type",
79
+ labelNames: ["type"],
80
+ }),
81
+ },
70
82
  };
71
83
  }
@@ -27,7 +27,6 @@ import {
27
27
  Slot,
28
28
  SlotRootHex,
29
29
  SubnetID,
30
- WithBytes,
31
30
  altair,
32
31
  capella,
33
32
  deneb,
@@ -69,14 +68,8 @@ export interface INetwork extends INetworkCorePublic {
69
68
  reStatusPeers(peers: PeerIdStr[]): Promise<void>;
70
69
  searchUnknownSlotRoot(slotRoot: SlotRootHex, source: BlockInputSource, peer?: PeerIdStr): void;
71
70
  // ReqResp
72
- sendBeaconBlocksByRange(
73
- peerId: PeerIdStr,
74
- request: phase0.BeaconBlocksByRangeRequest
75
- ): Promise<WithBytes<SignedBeaconBlock>[]>;
76
- sendBeaconBlocksByRoot(
77
- peerId: PeerIdStr,
78
- request: BeaconBlocksByRootRequest
79
- ): Promise<WithBytes<SignedBeaconBlock>[]>;
71
+ sendBeaconBlocksByRange(peerId: PeerIdStr, request: phase0.BeaconBlocksByRangeRequest): Promise<SignedBeaconBlock[]>;
72
+ sendBeaconBlocksByRoot(peerId: PeerIdStr, request: BeaconBlocksByRootRequest): Promise<SignedBeaconBlock[]>;
80
73
  sendBlobSidecarsByRange(peerId: PeerIdStr, request: deneb.BlobSidecarsByRangeRequest): Promise<deneb.BlobSidecar[]>;
81
74
  sendBlobSidecarsByRoot(peerId: PeerIdStr, request: BlobSidecarsByRootRequest): Promise<deneb.BlobSidecar[]>;
82
75
  sendDataColumnSidecarsByRange(
@@ -20,7 +20,6 @@ import {
20
20
  SingleAttestation,
21
21
  SlotRootHex,
22
22
  SubnetID,
23
- WithBytes,
24
23
  altair,
25
24
  capella,
26
25
  deneb,
@@ -513,7 +512,7 @@ export class Network implements INetwork {
513
512
  async sendBeaconBlocksByRange(
514
513
  peerId: PeerIdStr,
515
514
  request: phase0.BeaconBlocksByRangeRequest
516
- ): Promise<WithBytes<SignedBeaconBlock>[]> {
515
+ ): Promise<SignedBeaconBlock[]> {
517
516
  return collectSequentialBlocksInRange(
518
517
  this.sendReqRespRequest(
519
518
  peerId,
@@ -526,10 +525,7 @@ export class Network implements INetwork {
526
525
  );
527
526
  }
528
527
 
529
- async sendBeaconBlocksByRoot(
530
- peerId: PeerIdStr,
531
- request: BeaconBlocksByRootRequest
532
- ): Promise<WithBytes<SignedBeaconBlock>[]> {
528
+ async sendBeaconBlocksByRoot(peerId: PeerIdStr, request: BeaconBlocksByRootRequest): Promise<SignedBeaconBlock[]> {
533
529
  return collectMaxResponseTypedWithBytes(
534
530
  this.sendReqRespRequest(
535
531
  peerId,
@@ -539,7 +535,8 @@ export class Network implements INetwork {
539
535
  request
540
536
  ),
541
537
  request.length,
542
- responseSszTypeByMethod[ReqRespMethod.BeaconBlocksByRoot]
538
+ responseSszTypeByMethod[ReqRespMethod.BeaconBlocksByRoot],
539
+ this.chain.serializedCache
543
540
  );
544
541
  }
545
542
 
@@ -592,7 +589,8 @@ export class Network implements INetwork {
592
589
  return collectMaxResponseTyped(
593
590
  this.sendReqRespRequest(peerId, ReqRespMethod.BlobSidecarsByRoot, [Version.V1], request),
594
591
  request.length,
595
- responseSszTypeByMethod[ReqRespMethod.BlobSidecarsByRoot]
592
+ responseSszTypeByMethod[ReqRespMethod.BlobSidecarsByRoot],
593
+ this.chain.serializedCache
596
594
  );
597
595
  }
598
596
 
@@ -614,7 +612,8 @@ export class Network implements INetwork {
614
612
  return collectMaxResponseTyped(
615
613
  this.sendReqRespRequest(peerId, ReqRespMethod.DataColumnSidecarsByRoot, [Version.V1], request),
616
614
  request.reduce((total, {columns}) => total + columns.length, 0),
617
- responseSszTypeByMethod[ReqRespMethod.DataColumnSidecarsByRoot]
615
+ responseSszTypeByMethod[ReqRespMethod.DataColumnSidecarsByRoot],
616
+ this.chain.serializedCache
618
617
  );
619
618
  }
620
619
 
@@ -516,6 +516,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
516
516
  throw new GossipActionError(GossipAction.REJECT, {code: "PRE_DENEB_BLOCK"});
517
517
  }
518
518
  const blockInput = await validateBeaconBlob(blobSidecar, topic.subnet, peerIdStr, seenTimestampSec);
519
+ chain.serializedCache.set(blobSidecar, serializedData);
519
520
  if (!blockInput.hasBlockAndAllData()) {
520
521
  const cutoffTimeMs = getCutoffTimeMs(chain, blobSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
521
522
  chain.logger.debug("Received gossip blob, waiting for full data availability", {
@@ -562,6 +563,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
562
563
  peerIdStr,
563
564
  seenTimestampSec
564
565
  );
566
+ chain.serializedCache.set(dataColumnSidecar, serializedData);
565
567
  const blockInputMeta = blockInput.getLogMeta();
566
568
  const {receivedColumns} = blockInputMeta;
567
569
  // it's not helpful to track every single column received
@@ -1,6 +1,6 @@
1
1
  import {Type} from "@chainsafe/ssz";
2
2
  import {RequestError, RequestErrorCode, ResponseIncoming} from "@lodestar/reqresp";
3
- import {WithBytes} from "@lodestar/types";
3
+ import {SerializedCache} from "../../../util/serializedCache.ts";
4
4
  import {ResponseTypeGetter} from "../types.js";
5
5
 
6
6
  /**
@@ -32,13 +32,16 @@ export async function collectExactOneTyped<T>(
32
32
  export async function collectMaxResponseTyped<T>(
33
33
  source: AsyncIterable<ResponseIncoming>,
34
34
  maxResponses: number,
35
- typeFn: ResponseTypeGetter<T>
35
+ typeFn: ResponseTypeGetter<T>,
36
+ serializedCache?: SerializedCache
36
37
  ): Promise<T[]> {
37
38
  // else: zero or more responses
38
39
  const responses: T[] = [];
39
40
  for await (const chunk of source) {
40
41
  const type = typeFn(chunk.fork, chunk.protocolVersion);
41
42
  const response = sszDeserializeResponse(type, chunk.data);
43
+ // optionally cache the serialized response if the cache is available
44
+ serializedCache?.set(response as object, chunk.data);
42
45
  responses.push(response);
43
46
 
44
47
  if (maxResponses !== undefined && responses.length >= maxResponses) {
@@ -58,14 +61,17 @@ export async function collectMaxResponseTyped<T>(
58
61
  export async function collectMaxResponseTypedWithBytes<T>(
59
62
  source: AsyncIterable<ResponseIncoming>,
60
63
  maxResponses: number,
61
- typeFn: ResponseTypeGetter<T>
62
- ): Promise<WithBytes<T>[]> {
64
+ typeFn: ResponseTypeGetter<T>,
65
+ serializedCache?: SerializedCache
66
+ ): Promise<T[]> {
63
67
  // else: zero or more responses
64
- const responses: WithBytes<T>[] = [];
68
+ const responses: T[] = [];
65
69
  for await (const chunk of source) {
66
70
  const type = typeFn(chunk.fork, chunk.protocolVersion);
67
71
  const data = sszDeserializeResponse(type, chunk.data);
68
- responses.push({data, bytes: chunk.data});
72
+ responses.push(data);
73
+ // optionally cache the serialized response if the cache is available
74
+ serializedCache?.set(data as object, chunk.data);
69
75
 
70
76
  if (maxResponses !== undefined && responses.length >= maxResponses) {
71
77
  break;
@@ -1,6 +1,7 @@
1
1
  import {ResponseIncoming} from "@lodestar/reqresp";
2
- import {SignedBeaconBlock, WithBytes, phase0} from "@lodestar/types";
2
+ import {SignedBeaconBlock, phase0} from "@lodestar/types";
3
3
  import {LodestarError} from "@lodestar/utils";
4
+ import {SerializedCache} from "../../../util/serializedCache.ts";
4
5
  import {ReqRespMethod, responseSszTypeByMethod} from "../types.js";
5
6
  import {sszDeserializeResponse} from "./collect.js";
6
7
 
@@ -10,9 +11,10 @@ import {sszDeserializeResponse} from "./collect.js";
10
11
  */
11
12
  export async function collectSequentialBlocksInRange(
12
13
  blockStream: AsyncIterable<ResponseIncoming>,
13
- {count, startSlot}: Pick<phase0.BeaconBlocksByRangeRequest, "count" | "startSlot">
14
- ): Promise<WithBytes<SignedBeaconBlock>[]> {
15
- const blocks: WithBytes<SignedBeaconBlock>[] = [];
14
+ {count, startSlot}: Pick<phase0.BeaconBlocksByRangeRequest, "count" | "startSlot">,
15
+ serializedCache?: SerializedCache
16
+ ): Promise<SignedBeaconBlock[]> {
17
+ const blocks: SignedBeaconBlock[] = [];
16
18
 
17
19
  for await (const chunk of blockStream) {
18
20
  const blockType = responseSszTypeByMethod[ReqRespMethod.BeaconBlocksByRange](chunk.fork, chunk.protocolVersion);
@@ -30,11 +32,13 @@ export async function collectSequentialBlocksInRange(
30
32
  }
31
33
 
32
34
  const prevBlock = blocks.at(-1);
33
- if (prevBlock && prevBlock.data.message.slot >= blockSlot) {
35
+ if (prevBlock && prevBlock.message.slot >= blockSlot) {
34
36
  throw new BlocksByRangeError({code: BlocksByRangeErrorCode.BAD_SEQUENCE});
35
37
  }
36
38
 
37
- blocks.push({data: block, bytes: chunk.data});
39
+ blocks.push(block);
40
+ // optionally cache the serialized response if the cache is available
41
+ serializedCache?.set(block, chunk.data);
38
42
  if (blocks.length >= count) {
39
43
  break; // Done, collected all blocks
40
44
  }