@lodestar/beacon-node 1.50.0-rc.0 → 1.51.0-dev.13ce72c336

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +7 -2
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/api/impl/validator/index.d.ts.map +1 -1
  5. package/lib/api/impl/validator/index.js +31 -17
  6. package/lib/api/impl/validator/index.js.map +1 -1
  7. package/lib/chain/ColumnReconstructionTracker.d.ts +11 -13
  8. package/lib/chain/ColumnReconstructionTracker.d.ts.map +1 -1
  9. package/lib/chain/ColumnReconstructionTracker.js +49 -44
  10. package/lib/chain/ColumnReconstructionTracker.js.map +1 -1
  11. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map +1 -1
  12. package/lib/chain/archiveStore/utils/archiveBlocks.js +4 -5
  13. package/lib/chain/archiveStore/utils/archiveBlocks.js.map +1 -1
  14. package/lib/chain/blocks/blockInput/blockInput.d.ts +1 -1
  15. package/lib/chain/blocks/blockInput/blockInput.d.ts.map +1 -1
  16. package/lib/chain/blocks/blockInput/blockInput.js.map +1 -1
  17. package/lib/chain/blocks/blockInput/utils.js +2 -2
  18. package/lib/chain/blocks/blockInput/utils.js.map +1 -1
  19. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  20. package/lib/chain/blocks/importBlock.js +7 -4
  21. package/lib/chain/blocks/importBlock.js.map +1 -1
  22. package/lib/chain/chain.d.ts.map +1 -1
  23. package/lib/chain/chain.js +4 -2
  24. package/lib/chain/chain.js.map +1 -1
  25. package/lib/chain/interface.d.ts +1 -1
  26. package/lib/chain/interface.d.ts.map +1 -1
  27. package/lib/chain/prepareNextSlot.d.ts.map +1 -1
  28. package/lib/chain/prepareNextSlot.js +4 -3
  29. package/lib/chain/prepareNextSlot.js.map +1 -1
  30. package/lib/chain/produceBlock/computeNewStateRoot.d.ts.map +1 -1
  31. package/lib/chain/produceBlock/computeNewStateRoot.js.map +1 -1
  32. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  33. package/lib/chain/produceBlock/produceBlockBody.js +8 -6
  34. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  35. package/lib/chain/reprocess.d.ts +2 -1
  36. package/lib/chain/reprocess.d.ts.map +1 -1
  37. package/lib/chain/reprocess.js +2 -1
  38. package/lib/chain/reprocess.js.map +1 -1
  39. package/lib/chain/seenCache/seenGossipBlockInput.d.ts.map +1 -1
  40. package/lib/chain/seenCache/seenGossipBlockInput.js +1 -1
  41. package/lib/chain/seenCache/seenGossipBlockInput.js.map +1 -1
  42. package/lib/chain/stateCache/persistentCheckpointsCache.d.ts.map +1 -1
  43. package/lib/chain/stateCache/persistentCheckpointsCache.js +1 -1
  44. package/lib/chain/stateCache/persistentCheckpointsCache.js.map +1 -1
  45. package/lib/chain/validatorMonitor.d.ts.map +1 -1
  46. package/lib/chain/validatorMonitor.js +6 -4
  47. package/lib/chain/validatorMonitor.js.map +1 -1
  48. package/lib/constants/constants.d.ts +1 -0
  49. package/lib/constants/constants.d.ts.map +1 -1
  50. package/lib/constants/constants.js +1 -0
  51. package/lib/constants/constants.js.map +1 -1
  52. package/lib/db/repositories/blockArchive.js +2 -2
  53. package/lib/db/repositories/blockArchive.js.map +1 -1
  54. package/lib/db/repositories/blockArchiveIndex.d.ts +1 -1
  55. package/lib/db/repositories/blockArchiveIndex.d.ts.map +1 -1
  56. package/lib/db/repositories/blockArchiveIndex.js +1 -2
  57. package/lib/db/repositories/blockArchiveIndex.js.map +1 -1
  58. package/lib/execution/engine/index.d.ts.map +1 -1
  59. package/lib/execution/engine/index.js +1 -1
  60. package/lib/execution/engine/index.js.map +1 -1
  61. package/lib/execution/engine/jsonRpcHttpClient.d.ts.map +1 -1
  62. package/lib/execution/engine/jsonRpcHttpClient.js +14 -4
  63. package/lib/execution/engine/jsonRpcHttpClient.js.map +1 -1
  64. package/lib/execution/engine/restTransport.d.ts +3 -1
  65. package/lib/execution/engine/restTransport.d.ts.map +1 -1
  66. package/lib/execution/engine/restTransport.js +33 -21
  67. package/lib/execution/engine/restTransport.js.map +1 -1
  68. package/lib/execution/engine/transportMetrics.d.ts +5 -1
  69. package/lib/execution/engine/transportMetrics.d.ts.map +1 -1
  70. package/lib/execution/engine/types.d.ts.map +1 -1
  71. package/lib/execution/engine/types.js +12 -11
  72. package/lib/execution/engine/types.js.map +1 -1
  73. package/lib/metrics/metrics/lodestar.d.ts +11 -0
  74. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  75. package/lib/metrics/metrics/lodestar.js +31 -1
  76. package/lib/metrics/metrics/lodestar.js.map +1 -1
  77. package/lib/network/gossip/gossipsub.d.ts.map +1 -1
  78. package/lib/network/gossip/gossipsub.js +3 -1
  79. package/lib/network/gossip/gossipsub.js.map +1 -1
  80. package/lib/network/gossip/topic.d.ts +0 -14
  81. package/lib/network/gossip/topic.d.ts.map +1 -1
  82. package/lib/network/peers/utils/enrSubnetsDeserialize.d.ts.map +1 -1
  83. package/lib/network/peers/utils/enrSubnetsDeserialize.js +5 -2
  84. package/lib/network/peers/utils/enrSubnetsDeserialize.js.map +1 -1
  85. package/lib/network/processor/index.js +3 -3
  86. package/lib/network/processor/index.js.map +1 -1
  87. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.d.ts.map +1 -1
  88. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.js +4 -4
  89. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.js.map +1 -1
  90. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js +3 -3
  91. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js.map +1 -1
  92. package/lib/network/reqresp/rateLimit.js +2 -2
  93. package/lib/network/reqresp/rateLimit.js.map +1 -1
  94. package/lib/network/subnets/attnetsService.d.ts.map +1 -1
  95. package/lib/network/subnets/attnetsService.js +3 -1
  96. package/lib/network/subnets/attnetsService.js.map +1 -1
  97. package/lib/node/notifier.d.ts.map +1 -1
  98. package/lib/node/notifier.js +12 -13
  99. package/lib/node/notifier.js.map +1 -1
  100. package/lib/sync/unknownBlock.d.ts.map +1 -1
  101. package/lib/sync/unknownBlock.js +3 -2
  102. package/lib/sync/unknownBlock.js.map +1 -1
  103. package/lib/util/clock.d.ts.map +1 -1
  104. package/lib/util/clock.js +3 -4
  105. package/lib/util/clock.js.map +1 -1
  106. package/lib/util/sszBytes.d.ts +2 -2
  107. package/lib/util/sszBytes.d.ts.map +1 -1
  108. package/lib/util/sszBytes.js +22 -14
  109. package/lib/util/sszBytes.js.map +1 -1
  110. package/package.json +15 -15
  111. package/src/api/impl/beacon/blocks/index.ts +10 -3
  112. package/src/api/impl/validator/index.ts +41 -19
  113. package/src/chain/ColumnReconstructionTracker.ts +63 -46
  114. package/src/chain/archiveStore/utils/archiveBlocks.ts +6 -7
  115. package/src/chain/blocks/blockInput/blockInput.ts +1 -1
  116. package/src/chain/blocks/blockInput/utils.ts +2 -2
  117. package/src/chain/blocks/importBlock.ts +8 -4
  118. package/src/chain/chain.ts +4 -2
  119. package/src/chain/interface.ts +2 -1
  120. package/src/chain/prepareNextSlot.ts +7 -3
  121. package/src/chain/produceBlock/computeNewStateRoot.ts +9 -2
  122. package/src/chain/produceBlock/produceBlockBody.ts +10 -5
  123. package/src/chain/reprocess.ts +2 -1
  124. package/src/chain/seenCache/seenGossipBlockInput.ts +2 -1
  125. package/src/chain/stateCache/persistentCheckpointsCache.ts +4 -1
  126. package/src/chain/validatorMonitor.ts +7 -4
  127. package/src/constants/constants.ts +1 -0
  128. package/src/db/repositories/blockArchive.ts +2 -2
  129. package/src/db/repositories/blockArchiveIndex.ts +2 -3
  130. package/src/execution/engine/index.ts +2 -1
  131. package/src/execution/engine/jsonRpcHttpClient.ts +12 -5
  132. package/src/execution/engine/restTransport.ts +53 -22
  133. package/src/execution/engine/transportMetrics.ts +2 -1
  134. package/src/execution/engine/types.ts +13 -11
  135. package/src/metrics/metrics/lodestar.ts +32 -1
  136. package/src/network/gossip/gossipsub.ts +3 -1
  137. package/src/network/peers/utils/enrSubnetsDeserialize.ts +5 -2
  138. package/src/network/processor/index.ts +3 -3
  139. package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts +4 -10
  140. package/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts +3 -3
  141. package/src/network/reqresp/rateLimit.ts +2 -2
  142. package/src/network/subnets/attnetsService.ts +8 -3
  143. package/src/node/notifier.ts +14 -13
  144. package/src/sync/unknownBlock.ts +3 -2
  145. package/src/util/clock.ts +3 -4
  146. package/src/util/sszBytes.ts +25 -12
@@ -20,6 +20,7 @@ import {
20
20
  isForkPostAltair,
21
21
  isForkPostBellatrix,
22
22
  isForkPostGloas,
23
+ isForkPostHeze,
23
24
  } from "@lodestar/params";
24
25
  import {
25
26
  G2_POINT_AT_INFINITY,
@@ -684,12 +685,11 @@ export async function produceBlockBody<T extends BlockType>(
684
685
  executionPayloadValue = BigInt(0);
685
686
  }
686
687
 
687
- const {graffiti, attestations, deposits, voluntaryExits, attesterSlashings, proposerSlashings} = blockBody;
688
+ const {graffiti, attestations, voluntaryExits, attesterSlashings, proposerSlashings} = blockBody;
688
689
 
689
690
  Object.assign(logMeta, {
690
691
  graffiti: fromGraffitiBytes(graffiti),
691
692
  attestations: attestations.length,
692
- deposits: deposits.length,
693
693
  voluntaryExits: voluntaryExits.length,
694
694
  attesterSlashings: attesterSlashings.length,
695
695
  proposerSlashings: proposerSlashings.length,
@@ -1106,16 +1106,21 @@ export async function produceCommonBlockBody<T extends BlockType>(
1106
1106
  });
1107
1107
 
1108
1108
  // Live proposal production is supported from Fulu onward.
1109
- return {
1109
+ const commonBlockBody: CommonBlockBody = {
1110
1110
  randaoReveal,
1111
1111
  graffiti,
1112
- eth1Data: currentState.eth1Data,
1113
1112
  proposerSlashings: this.opts.disableProposerSlashings === true ? [] : proposerSlashings,
1114
1113
  attesterSlashings,
1115
1114
  attestations,
1116
- deposits: [],
1117
1115
  voluntaryExits,
1118
1116
  blsToExecutionChanges,
1119
1117
  syncAggregate,
1120
1118
  };
1119
+
1120
+ if (!isForkPostHeze(fork)) {
1121
+ commonBlockBody.eth1Data = currentState.eth1Data;
1122
+ commonBlockBody.deposits = [];
1123
+ }
1124
+
1125
+ return commonBlockBody;
1121
1126
  }
@@ -5,8 +5,9 @@ import {Metrics} from "../metrics/index.js";
5
5
  /**
6
6
  * To prevent our node from having to reprocess while struggling to sync,
7
7
  * we only want to reprocess attestations if block reaches our node before this time.
8
+ * Expressed in basis points of slot duration (1667 BPS = 2s for 12s slots).
8
9
  */
9
- export const REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC = 2;
10
+ export const REPROCESS_MIN_TIME_TO_NEXT_SLOT_BPS = 1667;
10
11
 
11
12
  /**
12
13
  * Reprocess status for metrics
@@ -4,6 +4,7 @@ import {
4
4
  ForkName,
5
5
  ForkPostFulu,
6
6
  ForkPostGloas,
7
+ ForkPreDeneb,
7
8
  ForkPreGloas,
8
9
  SLOTS_PER_EPOCH,
9
10
  isForkPostDeneb,
@@ -240,7 +241,7 @@ export class SeenBlockInput {
240
241
  } else if (!isForkPostDeneb(forkName)) {
241
242
  // Pre-deneb
242
243
  blockInput = BlockInputPreData.createFromBlock({
243
- block,
244
+ block: block as SignedBeaconBlock<ForkPreDeneb>,
244
245
  blockRootHex,
245
246
  daOutOfRange,
246
247
  forkName,
@@ -592,7 +592,10 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
592
592
  }
593
593
 
594
594
  const blockSlot = state.slot;
595
- const processCPStatesTimeMs = this.config.getSlotComponentDurationMs(PROCESS_CHECKPOINT_STATES_BPS);
595
+ const processCPStatesTimeMs = this.config.getSlotComponentDurationMs(
596
+ this.config.getForkName(blockSlot),
597
+ PROCESS_CHECKPOINT_STATES_BPS
598
+ );
596
599
  // we always have clock in production, fallback value is only for test
597
600
  const msFromSlot = this.clock?.msFromSlot(blockSlot) ?? processCPStatesTimeMs;
598
601
  const msToProcessCPStates = processCPStatesTimeMs - msFromSlot;
@@ -1,5 +1,5 @@
1
1
  import {ChainForkConfig} from "@lodestar/config";
2
- import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
2
+ import {ForkName, MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
3
3
  import {
4
4
  IBeaconStateView,
5
5
  ParticipationFlags,
@@ -776,13 +776,14 @@ export function createValidatorMonitor(
776
776
 
777
777
  if (isStatePostAltair(headState)) {
778
778
  const prevEpochStartSlot = computeStartSlotAtEpoch(prevEpoch);
779
+ const prevEpochFork = config.getForkName(prevEpochStartSlot);
779
780
  const prevEpochTargetRoot = toRootHex(headState.getBlockRootAtSlot(prevEpochStartSlot));
780
781
 
781
782
  // Check attestation performance
782
783
  for (const [index, validator] of validators.entries()) {
783
784
  const flags = parseParticipationFlags(headState.getPreviousEpochParticipation(index));
784
785
  const attestationSummary = validator.attestations.get(prevEpoch)?.get(prevEpochTargetRoot);
785
- const summary = renderAttestationSummary(config, rootCache, attestationSummary, flags);
786
+ const summary = renderAttestationSummary(config, prevEpochFork, rootCache, attestationSummary, flags);
786
787
  validatorMonitorMetrics?.prevEpochAttestationSummary.inc({summary});
787
788
  log("Previous epoch attestation", {
788
789
  validator: index,
@@ -941,6 +942,7 @@ export function createValidatorMonitor(
941
942
  */
942
943
  function renderAttestationSummary(
943
944
  config: ChainForkConfig,
945
+ fork: ForkName,
944
946
  rootCache: RootHexCache,
945
947
  summary: AttestationSummary | undefined,
946
948
  flags: ParticipationFlags
@@ -1043,7 +1045,7 @@ function renderAttestationSummary(
1043
1045
  }
1044
1046
 
1045
1047
  const submittedLate =
1046
- summary.poolSubmitDelayMinSec > config.getSlotComponentDurationMs(LATE_ATTESTATION_SUBMISSION_BPS) / 1000;
1048
+ summary.poolSubmitDelayMinSec > config.getSlotComponentDurationMs(fork, LATE_ATTESTATION_SUBMISSION_BPS) / 1000;
1047
1049
 
1048
1050
  const aggregateInclusion = summary.aggregateInclusionDelaysSec.length > 0;
1049
1051
 
@@ -1178,7 +1180,8 @@ function renderBlockProposalSummary(
1178
1180
 
1179
1181
  if (
1180
1182
  proposal.poolSubmitDelaySec !== null &&
1181
- proposal.poolSubmitDelaySec > config.getSlotComponentDurationMs(LATE_BLOCK_SUBMISSION_BPS) / 1000
1183
+ proposal.poolSubmitDelaySec >
1184
+ config.getSlotComponentDurationMs(config.getForkName(proposalSlot), LATE_BLOCK_SUBMISSION_BPS) / 1000
1182
1185
  ) {
1183
1186
  out += "_late";
1184
1187
  }
@@ -7,3 +7,4 @@ export const ZERO_HASH = Buffer.alloc(32, 0);
7
7
  export const ZERO_HASH_HEX = "0x" + "00".repeat(32);
8
8
  export const EMPTY_SIGNATURE = Buffer.alloc(96, 0);
9
9
  export const GRAFFITI_SIZE = 32;
10
+ export const HALF_SLOT_BPS = 5000;
@@ -184,7 +184,7 @@ export class BlockArchiveRepository extends Repository<Slot, SignedBeaconBlock>
184
184
  async remove(value: SignedBeaconBlock): Promise<void> {
185
185
  await Promise.all([
186
186
  super.remove(value),
187
- deleteRootIndex(this.db, this.config.getForkTypes(value.message.slot).SignedBeaconBlock, value),
187
+ deleteRootIndex(this.db, this.config.getForkTypes(value.message.slot).BeaconBlock, value),
188
188
  deleteParentRootIndex(this.db, value),
189
189
  ]);
190
190
  }
@@ -193,7 +193,7 @@ export class BlockArchiveRepository extends Repository<Slot, SignedBeaconBlock>
193
193
  await Promise.all([
194
194
  super.batchRemove(values),
195
195
  ...values.map((value) =>
196
- deleteRootIndex(this.db, this.config.getForkTypes(value.message.slot).SignedBeaconBlock, value)
196
+ deleteRootIndex(this.db, this.config.getForkTypes(value.message.slot).BeaconBlock, value)
197
197
  ),
198
198
  ...values.map((value) => deleteParentRootIndex(this.db, value)),
199
199
  ]);
@@ -1,6 +1,6 @@
1
1
  import {Db, encodeKey} from "@lodestar/db";
2
2
  import {ForkAll} from "@lodestar/params";
3
- import {Root, SSZTypesFor, SignedBeaconBlock, Slot, ssz} from "@lodestar/types";
3
+ import {Root, SSZTypesFor, SignedBeaconBlock, Slot} from "@lodestar/types";
4
4
  import {Bucket, getBucketNameByValue} from "../buckets.js";
5
5
 
6
6
  export const rootIndexBucketId = getBucketNameByValue(Bucket.index_blockArchiveRootIndex);
@@ -16,10 +16,9 @@ export async function getParentRootIndex(db: Db, parentRoot: Root): Promise<Uint
16
16
 
17
17
  export async function deleteRootIndex(
18
18
  db: Db,
19
- signedBeaconBlockType: SSZTypesFor<ForkAll, "SignedBeaconBlock">,
19
+ beaconBlockType: SSZTypesFor<ForkAll, "BeaconBlock">,
20
20
  block: SignedBeaconBlock
21
21
  ): Promise<void> {
22
- const beaconBlockType = (signedBeaconBlockType as typeof ssz.phase0.SignedBeaconBlock).fields.message;
23
22
  return db.delete(getRootIndexKey(beaconBlockType.hashTreeRoot(block.message)), {bucketId: rootIndexBucketId});
24
23
  }
25
24
 
@@ -70,7 +70,8 @@ export function getExecutionEngineHttp(
70
70
  jwtId: opts.jwtId,
71
71
  clientVersion: formatClientVersionHeader(getLodestarClientVersion(opts)),
72
72
  metrics,
73
- })
73
+ }),
74
+ metrics
74
75
  );
75
76
 
76
77
  modules.logger.info("Execution client", {urls: opts.urls.map(toPrintableUrl).toString(), engineApi});
@@ -284,7 +284,12 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient {
284
284
  });
285
285
 
286
286
  const streamTimer = this.metrics?.streamTime.startTimer({routeId});
287
- const bodyText = await res.text();
287
+ let bodyText: string;
288
+ try {
289
+ bodyText = await res.text();
290
+ } finally {
291
+ streamTimer?.();
292
+ }
288
293
  this.metrics?.responseBytes.inc({routeId}, bodyText.length);
289
294
  if (!res.ok) {
290
295
  // Infura errors:
@@ -292,10 +297,12 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient {
292
297
  throw new HttpRpcError(res.status, `${res.statusText}: ${bodyText.slice(0, maxStringLengthToPrint)}`);
293
298
  }
294
299
 
295
- const bodyJson = parseJson<R>(bodyText);
296
- streamTimer?.();
297
-
298
- return bodyJson;
300
+ const parseTimer = this.metrics?.responseParseTime.startTimer({routeId, encoding: "json"});
301
+ try {
302
+ return parseJson<R>(bodyText);
303
+ } finally {
304
+ parseTimer?.();
305
+ }
299
306
  } catch (e) {
300
307
  this.metrics?.requestErrors.inc({routeId});
301
308
  if (controller.signal.aborted) {
@@ -3,7 +3,7 @@ import {CELLS_PER_EXT_BLOB, ForkName, ForkSeq, isForkPostBellatrix} from "@lodes
3
3
  import {ExecutionPayload, ExecutionRequests, Root, RootHex, capella, deneb, electra, gloas} from "@lodestar/types";
4
4
  import {BlobAndProof} from "@lodestar/types/deneb";
5
5
  import {BlobAndProofV2} from "@lodestar/types/fulu";
6
- import {LodestarError, fromHex, strip0xPrefix, toHex, toRootHex} from "@lodestar/utils";
6
+ import {EndTimer, LodestarError, fromHex, strip0xPrefix, toHex, toRootHex} from "@lodestar/utils";
7
7
  import {isValidBlobVersionedHashes} from "./blobVersionedHashes.js";
8
8
  import {
9
9
  ClientCode,
@@ -48,6 +48,7 @@ import {
48
48
  executionForkName,
49
49
  } from "./sszTypes.js";
50
50
  import {ForkchoiceUpdatedResult, GetPayloadResult, IEngineTransport, PayloadStatusResult} from "./transport.js";
51
+ import {EngineTransportMetrics} from "./transportMetrics.js";
51
52
  import {
52
53
  ExecutionPayloadBodyV2,
53
54
  assertReqSizeLimit,
@@ -102,7 +103,10 @@ export function formatClientVersionHeader(clientVersion: ClientVersion): string
102
103
  export class RestEngineTransport implements IEngineTransport {
103
104
  private limits = DEFAULT_LIMITS;
104
105
 
105
- constructor(private readonly client: EngineRestHttpClient) {}
106
+ constructor(
107
+ private readonly client: EngineRestHttpClient,
108
+ private readonly metrics?: EngineTransportMetrics | null
109
+ ) {}
106
110
 
107
111
  get emitter(): JsonRpcHttpClientEventEmitter {
108
112
  return this.client.emitter;
@@ -114,7 +118,7 @@ export class RestEngineTransport implements IEngineTransport {
114
118
  {method: "GET", path: "/capabilities", responseType: "json"},
115
119
  getCapabilitiesOpts
116
120
  );
117
- const capabilities = parseJson(body, "getCapabilities");
121
+ const capabilities = parseJson(body, "getCapabilities", this.metrics);
118
122
  if (
119
123
  !isRecord(capabilities) ||
120
124
  !Array.isArray(capabilities.supported_forks) ||
@@ -220,7 +224,7 @@ export class RestEngineTransport implements IEngineTransport {
220
224
  {method: "POST", path: "/payloads", executionFork: toExecutionForkName(fork), body, responseType: "ssz"},
221
225
  notifyNewPayloadOpts
222
226
  );
223
- return toPayloadStatusResult(deserializeResponse(PayloadStatus, res.body, "notifyNewPayload"));
227
+ return toPayloadStatusResult(deserializeResponse(PayloadStatus, res.body, "notifyNewPayload", this.metrics));
224
228
  }
225
229
 
226
230
  async forkchoiceUpdated(
@@ -268,7 +272,12 @@ export class RestEngineTransport implements IEngineTransport {
268
272
  {method: "POST", path: "/forkchoice", executionFork: toExecutionForkName(fork), body, responseType: "ssz"},
269
273
  fcUReqOpts
270
274
  );
271
- const {payloadStatus, payloadId} = deserializeResponse(ForkchoiceUpdateResponse, res.body, "forkchoiceUpdated");
275
+ const {payloadStatus, payloadId} = deserializeResponse(
276
+ ForkchoiceUpdateResponse,
277
+ res.body,
278
+ "forkchoiceUpdated",
279
+ this.metrics
280
+ );
272
281
  return {
273
282
  payloadStatus: toPayloadStatusResult(payloadStatus),
274
283
  payloadId: payloadId.length > 0 ? toHex(payloadId[0]) : null,
@@ -289,7 +298,8 @@ export class RestEngineTransport implements IEngineTransport {
289
298
  const {payload, blockValue, blobsBundle, executionRequests, shouldOverrideBuilder} = deserializeResponse(
290
299
  BuiltPayloadGloas,
291
300
  res.body,
292
- "getPayload"
301
+ "getPayload",
302
+ this.metrics
293
303
  );
294
304
  return {
295
305
  executionPayload: payload,
@@ -303,7 +313,8 @@ export class RestEngineTransport implements IEngineTransport {
303
313
  const {payload, blockValue, blobsBundle, executionRequests, shouldOverrideBuilder} = deserializeResponse(
304
314
  BuiltPayloadFulu,
305
315
  res.body,
306
- "getPayload"
316
+ "getPayload",
317
+ this.metrics
307
318
  );
308
319
  return {
309
320
  executionPayload: payload,
@@ -317,7 +328,8 @@ export class RestEngineTransport implements IEngineTransport {
317
328
  const {payload, blockValue, blobsBundle, executionRequests, shouldOverrideBuilder} = deserializeResponse(
318
329
  BuiltPayloadElectra,
319
330
  res.body,
320
- "getPayload"
331
+ "getPayload",
332
+ this.metrics
321
333
  );
322
334
  return {
323
335
  executionPayload: payload,
@@ -331,15 +343,16 @@ export class RestEngineTransport implements IEngineTransport {
331
343
  const {payload, blockValue, blobsBundle, shouldOverrideBuilder} = deserializeResponse(
332
344
  BuiltPayloadDeneb,
333
345
  res.body,
334
- "getPayload"
346
+ "getPayload",
347
+ this.metrics
335
348
  );
336
349
  return {executionPayload: payload, executionPayloadValue: blockValue, blobsBundle, shouldOverrideBuilder};
337
350
  }
338
351
  if (ForkSeq[fork] >= ForkSeq.capella) {
339
- const {payload, blockValue} = deserializeResponse(BuiltPayloadCapella, res.body, "getPayload");
352
+ const {payload, blockValue} = deserializeResponse(BuiltPayloadCapella, res.body, "getPayload", this.metrics);
340
353
  return {executionPayload: payload, executionPayloadValue: blockValue, shouldOverrideBuilder: false};
341
354
  }
342
- const {payload, blockValue} = deserializeResponse(BuiltPayloadBellatrix, res.body, "getPayload");
355
+ const {payload, blockValue} = deserializeResponse(BuiltPayloadBellatrix, res.body, "getPayload", this.metrics);
343
356
  return {executionPayload: payload, executionPayloadValue: blockValue, shouldOverrideBuilder: false};
344
357
  }
345
358
 
@@ -356,7 +369,7 @@ export class RestEngineTransport implements IEngineTransport {
356
369
  },
357
370
  getPayloadBodiesByHashOpts
358
371
  );
359
- const bodies = deserializeBodiesResponse(ForkName.gloas, res.body);
372
+ const bodies = deserializeBodiesResponse(ForkName.gloas, res.body, this.metrics);
360
373
  if (bodies.length !== blockHashes.length) {
361
374
  throw Error(`Invalid bodies response length=${bodies.length} blockHashes=${blockHashes.length}`);
362
375
  }
@@ -374,7 +387,7 @@ export class RestEngineTransport implements IEngineTransport {
374
387
  return versionedHashes.map(() => null);
375
388
  }
376
389
 
377
- const {entries} = deserializeResponse(BlobsV1Response, res.body, "getBlobsV1");
390
+ const {entries} = deserializeResponse(BlobsV1Response, res.body, "getBlobsV1", this.metrics);
378
391
  if (entries.length !== versionedHashes.length) {
379
392
  throw Error(`Invalid blobs/v1 response length=${entries.length} versionedHashes=${versionedHashes.length}`);
380
393
  }
@@ -397,7 +410,7 @@ export class RestEngineTransport implements IEngineTransport {
397
410
  return null;
398
411
  }
399
412
 
400
- const {entries} = deserializeResponse(BlobsV2Response, res.body, "getBlobsV2", {reuseBytes: true});
413
+ const {entries} = deserializeResponse(BlobsV2Response, res.body, "getBlobsV2", this.metrics, {reuseBytes: true});
401
414
  if (entries.length !== versionedHashes.length) {
402
415
  throw Error(`Invalid blobs/v2 response length=${entries.length} versionedHashes=${versionedHashes.length}`);
403
416
  }
@@ -418,7 +431,7 @@ export class RestEngineTransport implements IEngineTransport {
418
431
  {method: "GET", path: "/identity", responseType: "json"},
419
432
  getClientVersionOpts
420
433
  );
421
- const versions = parseJson(body, "getClientVersion");
434
+ const versions = parseJson(body, "getClientVersion", this.metrics);
422
435
  if (!Array.isArray(versions)) {
423
436
  throw new EngineRestResponseError("getClientVersion", "Expected an array of client versions");
424
437
  }
@@ -502,36 +515,54 @@ function toPayloadStatusResult(payloadStatus: PayloadStatusSsz): PayloadStatusRe
502
515
  }
503
516
  }
504
517
 
505
- function deserializeBodiesResponse(fork: ForkName, data: Uint8Array): (ExecutionPayloadBodyV2 | null)[] {
518
+ function deserializeBodiesResponse(
519
+ fork: ForkName,
520
+ data: Uint8Array,
521
+ metrics?: EngineTransportMetrics | null
522
+ ): (ExecutionPayloadBodyV2 | null)[] {
506
523
  if (ForkSeq[fork] >= ForkSeq.gloas) {
507
- return deserializeResponse(BodiesResponseGloas, data, "getPayloadBodiesByHash").entries.map((entry) =>
524
+ return deserializeResponse(BodiesResponseGloas, data, "getPayloadBodiesByHash", metrics).entries.map((entry) =>
508
525
  entry.available ? entry.body : null
509
526
  );
510
527
  }
511
528
  if (ForkSeq[fork] >= ForkSeq.capella) {
512
- return deserializeResponse(BodiesResponseCapella, data, "getPayloadBodiesByHash").entries.map((entry) =>
529
+ return deserializeResponse(BodiesResponseCapella, data, "getPayloadBodiesByHash", metrics).entries.map((entry) =>
513
530
  entry.available ? {...entry.body, blockAccessList: null} : null
514
531
  );
515
532
  }
516
- return deserializeResponse(BodiesResponseBellatrix, data, "getPayloadBodiesByHash").entries.map((entry) =>
533
+ return deserializeResponse(BodiesResponseBellatrix, data, "getPayloadBodiesByHash", metrics).entries.map((entry) =>
517
534
  entry.available ? {transactions: entry.body.transactions, withdrawals: null, blockAccessList: null} : null
518
535
  );
519
536
  }
520
537
 
521
538
  /** A 2xx response that does not decode is an execution client error, not an unreachable client */
522
- function deserializeResponse<T>(type: Type<T>, body: Uint8Array, routeId: string, opts?: {reuseBytes?: boolean}): T {
539
+ function deserializeResponse<T>(
540
+ type: Type<T>,
541
+ body: Uint8Array,
542
+ routeId: string,
543
+ metrics?: EngineTransportMetrics | null,
544
+ opts?: {reuseBytes?: boolean}
545
+ ): T {
546
+ const timer = metrics?.responseParseTime.startTimer({routeId, encoding: "ssz"});
523
547
  try {
524
548
  return type.deserialize(body, opts);
525
549
  } catch (e) {
526
550
  throw new EngineRestResponseError(routeId, `Invalid SSZ: ${(e as Error).message}`);
551
+ } finally {
552
+ timer?.();
527
553
  }
528
554
  }
529
555
 
530
- function parseJson(body: Uint8Array, routeId: string): unknown {
556
+ function parseJson(body: Uint8Array, routeId: string, metrics?: EngineTransportMetrics | null): unknown {
557
+ let timer: EndTimer | undefined;
531
558
  try {
532
- return JSON.parse(textDecoder.decode(body));
559
+ const text = textDecoder.decode(body);
560
+ timer = metrics?.responseParseTime.startTimer({routeId, encoding: "json"});
561
+ return JSON.parse(text);
533
562
  } catch (e) {
534
563
  throw new EngineRestResponseError(routeId, `Invalid JSON: ${(e as Error).message}`);
564
+ } finally {
565
+ timer?.();
535
566
  }
536
567
  }
537
568
 
@@ -1,4 +1,4 @@
1
- import {Counter} from "@lodestar/utils";
1
+ import {Counter, Histogram} from "@lodestar/utils";
2
2
 
3
3
  /**
4
4
  * Common metrics type for both json rpc and ssz transport.
@@ -6,4 +6,5 @@ import {Counter} from "@lodestar/utils";
6
6
  export type EngineTransportMetrics = {
7
7
  requestBytes: Counter<{routeId: string}>;
8
8
  responseBytes: Counter<{routeId: string}>;
9
+ responseParseTime: Histogram<{routeId: string; encoding: "json" | "ssz"}>;
9
10
  };
@@ -1,3 +1,4 @@
1
+ import {Type} from "@chainsafe/ssz";
1
2
  import {
2
3
  BUILDER_DEPOSIT_REQUEST_TYPE,
3
4
  BUILDER_EXIT_REQUEST_TYPE,
@@ -500,14 +501,15 @@ export function deserializeWithdrawal(serialized: WithdrawalRpc): capella.Withdr
500
501
  }
501
502
 
502
503
  /**
503
- * Prepend a single-byte requestType to requestsBytes
504
+ * Serialize requests prefixed with a single-byte requestType in one allocation
504
505
  */
505
- function prefixRequests(requestsBytes: Uint8Array, requestType: ExecutionRequestType): Uint8Array {
506
- const prefixedRequests = new Uint8Array(1 + requestsBytes.length);
507
- prefixedRequests[0] = requestType;
508
- prefixedRequests.set(requestsBytes, 1);
506
+ function serializePrefixedRequests<T>(type: Type<T>, requests: T, requestType: ExecutionRequestType): Uint8Array {
507
+ const uint8Array = new Uint8Array(1 + type.value_serializedSize(requests));
508
+ const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength);
509
+ uint8Array[0] = requestType;
510
+ type.value_serializeToBytes({uint8Array, dataView}, 1, requests);
509
511
 
510
- return prefixedRequests;
512
+ return uint8Array;
511
513
  }
512
514
 
513
515
  /**
@@ -522,16 +524,16 @@ export function serializeExecutionRequestsToBytes(fork: ForkName, executionReque
522
524
  const result: Uint8Array[] = [];
523
525
 
524
526
  if (deposits.length !== 0) {
525
- result.push(prefixRequests(ssz.electra.DepositRequests.serialize(deposits), DEPOSIT_REQUEST_TYPE));
527
+ result.push(serializePrefixedRequests(ssz.electra.DepositRequests, deposits, DEPOSIT_REQUEST_TYPE));
526
528
  }
527
529
 
528
530
  if (withdrawals.length !== 0) {
529
- result.push(prefixRequests(ssz.electra.WithdrawalRequests.serialize(withdrawals), WITHDRAWAL_REQUEST_TYPE));
531
+ result.push(serializePrefixedRequests(ssz.electra.WithdrawalRequests, withdrawals, WITHDRAWAL_REQUEST_TYPE));
530
532
  }
531
533
 
532
534
  if (consolidations.length !== 0) {
533
535
  result.push(
534
- prefixRequests(ssz.electra.ConsolidationRequests.serialize(consolidations), CONSOLIDATION_REQUEST_TYPE)
536
+ serializePrefixedRequests(ssz.electra.ConsolidationRequests, consolidations, CONSOLIDATION_REQUEST_TYPE)
535
537
  );
536
538
  }
537
539
 
@@ -540,12 +542,12 @@ export function serializeExecutionRequestsToBytes(fork: ForkName, executionReque
540
542
 
541
543
  if (builderDeposits.length !== 0) {
542
544
  result.push(
543
- prefixRequests(ssz.gloas.BuilderDepositRequests.serialize(builderDeposits), BUILDER_DEPOSIT_REQUEST_TYPE)
545
+ serializePrefixedRequests(ssz.gloas.BuilderDepositRequests, builderDeposits, BUILDER_DEPOSIT_REQUEST_TYPE)
544
546
  );
545
547
  }
546
548
 
547
549
  if (builderExits.length !== 0) {
548
- result.push(prefixRequests(ssz.gloas.BuilderExitRequests.serialize(builderExits), BUILDER_EXIT_REQUEST_TYPE));
550
+ result.push(serializePrefixedRequests(ssz.gloas.BuilderExitRequests, builderExits, BUILDER_EXIT_REQUEST_TYPE));
549
551
  }
550
552
  }
551
553
 
@@ -330,6 +330,31 @@ export function createLodestarMetrics(
330
330
  }),
331
331
  },
332
332
 
333
+ columnReconstructionQueue: {
334
+ length: register.gauge({
335
+ name: "lodestar_column_reconstruction_queue_length",
336
+ help: "Count of total column reconstruction queue length",
337
+ }),
338
+ droppedJobs: register.gauge({
339
+ name: "lodestar_column_reconstruction_queue_dropped_jobs_total",
340
+ help: "Count of total column reconstruction queue dropped jobs",
341
+ }),
342
+ jobTime: register.histogram({
343
+ name: "lodestar_column_reconstruction_queue_job_time_seconds",
344
+ help: "Time to process column reconstruction queue job in seconds",
345
+ buckets: [0.01, 0.1, 1, 4, 12],
346
+ }),
347
+ jobWaitTime: register.histogram({
348
+ name: "lodestar_column_reconstruction_queue_job_wait_time_seconds",
349
+ help: "Time from job added to the column reconstruction queue to starting in seconds",
350
+ buckets: [0.01, 0.1, 1, 4, 12],
351
+ }),
352
+ concurrency: register.gauge({
353
+ name: "lodestar_column_reconstruction_queue_concurrency",
354
+ help: "Current concurrency of column reconstruction queue",
355
+ }),
356
+ },
357
+
333
358
  engineHttpProcessorQueue: {
334
359
  length: register.gauge({
335
360
  name: "lodestar_engine_http_processor_queue_length",
@@ -2061,11 +2086,17 @@ export function createLodestarMetrics(
2061
2086
  }),
2062
2087
  streamTime: register.histogram<{routeId: string}>({
2063
2088
  name: "lodestar_execution_engine_http_client_stream_time_seconds",
2064
- help: "ExecutionEngineHttp client - streaming time by routeId",
2089
+ help: "ExecutionEngineHttp client - response body consumption time, including failures and excluding parsing",
2065
2090
  labelNames: ["routeId"],
2066
2091
  // Provide max resolution on problematic values around 1 second
2067
2092
  buckets: [0.1, 0.5, 1, 2, 5, 15],
2068
2093
  }),
2094
+ responseParseTime: register.histogram<{routeId: string; encoding: "json" | "ssz"}>({
2095
+ name: "lodestar_execution_engine_http_client_response_parse_time_seconds",
2096
+ help: "ExecutionEngineHttp client - JSON.parse or SSZ deserialize time, including failures, excluding body reads and subsequent value conversion",
2097
+ labelNames: ["routeId", "encoding"],
2098
+ buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1],
2099
+ }),
2069
2100
  requestErrors: register.gauge<{routeId: string}>({
2070
2101
  name: "lodestar_execution_engine_http_client_request_errors_total",
2071
2102
  help: "ExecutionEngineHttp client - total count of request errors",
@@ -21,6 +21,7 @@ import {
21
21
  SLOTS_PER_EPOCH,
22
22
  SYNC_COMMITTEE_SUBNET_COUNT,
23
23
  } from "@lodestar/params";
24
+ import {getSlotDurations} from "@lodestar/state-transition";
24
25
  import {SubnetID} from "@lodestar/types";
25
26
  import {Logger, Map2d, Map2dArr} from "@lodestar/utils";
26
27
  import {RegistryMetricCreator} from "../../metrics/index.js";
@@ -172,7 +173,8 @@ export class Eth2Gossipsub {
172
173
  mcacheLength: 6,
173
174
  mcacheGossip: 3,
174
175
  // this should be in ms
175
- seenTTL: config.SLOT_DURATION_MS * SLOTS_PER_EPOCH * 2,
176
+ // use the largest configured slot duration so the TTL stays conservative across duration changes
177
+ seenTTL: Math.max(...getSlotDurations(config).map(([, slotDurationMs]) => slotDurationMs)) * SLOTS_PER_EPOCH * 2,
176
178
  scoreParams,
177
179
  scoreThresholds: gossipScoreThresholds,
178
180
  // For a single stream, await processing each RPC before processing the next
@@ -17,10 +17,13 @@ export function deserializeEnrSubnets(bytes: Uint8Array, subnetCount: number): b
17
17
  return getUint8ByteToBitBooleanArray(bytes[0] ?? 0);
18
18
  }
19
19
 
20
- let boolsArr: boolean[] = [];
21
20
  const byteCount = Math.ceil(subnetCount / 8);
21
+ const boolsArr = new Array<boolean>(byteCount * 8);
22
22
  for (let i = 0; i < byteCount; i++) {
23
- boolsArr = boolsArr.concat(getUint8ByteToBitBooleanArray(bytes[i] ?? 0));
23
+ const bits = getUint8ByteToBitBooleanArray(bytes[i] ?? 0);
24
+ for (let j = 0; j < 8; j++) {
25
+ boolsArr[i * 8 + j] = bits[j];
26
+ }
24
27
  }
25
28
 
26
29
  return boolsArr;
@@ -1,6 +1,6 @@
1
1
  import {TopicValidatorResult} from "@libp2p/gossipsub";
2
2
  import {routes} from "@lodestar/api";
3
- import {ForkSeq, NUMBER_OF_COLUMNS} from "@lodestar/params";
3
+ import {ForkSeq, NUMBER_OF_COLUMNS, isForkPostGloas} from "@lodestar/params";
4
4
  import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
5
5
  import {RootHex, Slot, SlotRootHex} from "@lodestar/types";
6
6
  import {Logger, MapDef, mapValues, sleep} from "@lodestar/utils";
@@ -528,9 +528,9 @@ export class NetworkProcessor {
528
528
  const parentRoot = getParentRootFromSignedBeaconBlockSerialized(message.msg.data);
529
529
  if (parentRoot) {
530
530
  let search: SearchUnknownRootTarget | null = null;
531
- if (ForkSeq[fork] >= ForkSeq.gloas) {
531
+ if (isForkPostGloas(fork)) {
532
532
  // GLOAS: also check parent envelope, same logic as execution_payload_bid
533
- const parentBlockHash = getParentBlockHashFromGloasSignedBeaconBlockSerialized(message.msg.data);
533
+ const parentBlockHash = getParentBlockHashFromGloasSignedBeaconBlockSerialized(message.msg.data, fork);
534
534
  if (parentBlockHash && !this.chain.forkChoice.getBlockHexAndBlockHash(parentRoot, parentBlockHash)) {
535
535
  const protoBlock = this.chain.forkChoice.getBlockHexDefaultStatus(parentRoot);
536
536
  if (protoBlock === null) {
@@ -3,7 +3,7 @@ import {ChainConfig} from "@lodestar/config";
3
3
  import {PayloadStatus} from "@lodestar/fork-choice";
4
4
  import {ForkSeq, GENESIS_SLOT, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
5
5
  import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
6
- import {computeEpochAtSlot} from "@lodestar/state-transition";
6
+ import {computeEpochAtSlot, getBlobDataRetentionStartEpoch} from "@lodestar/state-transition";
7
7
  import {ColumnIndex, Epoch, RootHex, fulu} from "@lodestar/types";
8
8
  import {fromHex, toRootHex} from "@lodestar/utils";
9
9
  import {IBeaconChain} from "../../../chain/index.js";
@@ -216,16 +216,10 @@ export function validateDataColumnSidecarsByRangeRequest(
216
216
  throw new ResponseError(RespStatus.INVALID_REQUEST, "startSlot < genesis");
217
217
  }
218
218
 
219
- // Spec: [max(current_epoch - MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS, FULU_FORK_EPOCH), current_epoch]
220
- const minimumRequestEpoch = Math.max(
221
- currentEpoch - config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS,
222
- config.FULU_FORK_EPOCH
223
- );
219
+ // Spec: [max(compute_blob_data_retention_start_epoch(current_epoch), FULU_FORK_EPOCH), current_epoch]
220
+ const minimumRequestEpoch = Math.max(getBlobDataRetentionStartEpoch(config, currentEpoch), config.FULU_FORK_EPOCH);
224
221
  if (computeEpochAtSlot(startSlot) < minimumRequestEpoch) {
225
- throw new ResponseError(
226
- RespStatus.RESOURCE_UNAVAILABLE,
227
- "startSlot is before MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS"
228
- );
222
+ throw new ResponseError(RespStatus.RESOURCE_UNAVAILABLE, "startSlot is before the blob data retention window");
229
223
  }
230
224
 
231
225
  if (count > config.MAX_REQUEST_BLOCKS_DENEB) {