@lodestar/beacon-node 1.49.0-dev.ee3c432d85 → 1.49.0-dev.f1b14bce8f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +13 -4
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/chain/archiveStore/archiveStore.d.ts.map +1 -1
  5. package/lib/chain/archiveStore/archiveStore.js +5 -3
  6. package/lib/chain/archiveStore/archiveStore.js.map +1 -1
  7. package/lib/chain/archiveStore/historicalState/metrics.d.ts.map +1 -1
  8. package/lib/chain/archiveStore/historicalState/metrics.js +2 -1
  9. package/lib/chain/archiveStore/historicalState/metrics.js.map +1 -1
  10. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts +11 -2
  11. package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map +1 -1
  12. package/lib/chain/archiveStore/utils/archiveBlocks.js +46 -28
  13. package/lib/chain/archiveStore/utils/archiveBlocks.js.map +1 -1
  14. package/lib/chain/archiveStore/utils/pruneHistory.d.ts +2 -2
  15. package/lib/chain/archiveStore/utils/pruneHistory.d.ts.map +1 -1
  16. package/lib/chain/archiveStore/utils/pruneHistory.js +1 -0
  17. package/lib/chain/archiveStore/utils/pruneHistory.js.map +1 -1
  18. package/lib/chain/chain.d.ts +10 -0
  19. package/lib/chain/chain.d.ts.map +1 -1
  20. package/lib/chain/chain.js +85 -16
  21. package/lib/chain/chain.js.map +1 -1
  22. package/lib/chain/errors/envelopeReconstructionError.d.ts +23 -0
  23. package/lib/chain/errors/envelopeReconstructionError.d.ts.map +1 -0
  24. package/lib/chain/errors/envelopeReconstructionError.js +13 -0
  25. package/lib/chain/errors/envelopeReconstructionError.js.map +1 -0
  26. package/lib/chain/errors/index.d.ts +1 -0
  27. package/lib/chain/errors/index.d.ts.map +1 -1
  28. package/lib/chain/errors/index.js +1 -0
  29. package/lib/chain/errors/index.js.map +1 -1
  30. package/lib/chain/interface.d.ts +6 -1
  31. package/lib/chain/interface.d.ts.map +1 -1
  32. package/lib/chain/interface.js.map +1 -1
  33. package/lib/chain/options.d.ts +2 -0
  34. package/lib/chain/options.d.ts.map +1 -1
  35. package/lib/chain/options.js +1 -0
  36. package/lib/chain/options.js.map +1 -1
  37. package/lib/db/repositories/executionPayloadEnvelopeArchive.d.ts +21 -9
  38. package/lib/db/repositories/executionPayloadEnvelopeArchive.d.ts.map +1 -1
  39. package/lib/db/repositories/executionPayloadEnvelopeArchive.js +32 -12
  40. package/lib/db/repositories/executionPayloadEnvelopeArchive.js.map +1 -1
  41. package/lib/db/repositories/index.d.ts +1 -1
  42. package/lib/db/repositories/index.d.ts.map +1 -1
  43. package/lib/db/repositories/index.js +1 -1
  44. package/lib/db/repositories/index.js.map +1 -1
  45. package/lib/execution/engine/disabled.d.ts +1 -0
  46. package/lib/execution/engine/disabled.d.ts.map +1 -1
  47. package/lib/execution/engine/disabled.js +3 -0
  48. package/lib/execution/engine/disabled.js.map +1 -1
  49. package/lib/execution/engine/http.d.ts +2 -1
  50. package/lib/execution/engine/http.d.ts.map +1 -1
  51. package/lib/execution/engine/http.js +7 -1
  52. package/lib/execution/engine/http.js.map +1 -1
  53. package/lib/execution/engine/interface.d.ts +3 -1
  54. package/lib/execution/engine/interface.d.ts.map +1 -1
  55. package/lib/execution/engine/mock.d.ts +1 -0
  56. package/lib/execution/engine/mock.d.ts.map +1 -1
  57. package/lib/execution/engine/mock.js +14 -2
  58. package/lib/execution/engine/mock.js.map +1 -1
  59. package/lib/execution/engine/types.d.ts +11 -0
  60. package/lib/execution/engine/types.d.ts.map +1 -1
  61. package/lib/execution/engine/types.js +10 -0
  62. package/lib/execution/engine/types.js.map +1 -1
  63. package/lib/metrics/metrics/beacon.d.ts +9 -0
  64. package/lib/metrics/metrics/beacon.d.ts.map +1 -1
  65. package/lib/metrics/metrics/beacon.js +16 -0
  66. package/lib/metrics/metrics/beacon.js.map +1 -1
  67. package/lib/metrics/metrics/lodestar.d.ts +9 -0
  68. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  69. package/lib/metrics/metrics/lodestar.js +23 -0
  70. package/lib/metrics/metrics/lodestar.js.map +1 -1
  71. package/lib/metrics/metrics.d.ts +3 -1
  72. package/lib/metrics/metrics.d.ts.map +1 -1
  73. package/lib/metrics/metrics.js +2 -2
  74. package/lib/metrics/metrics.js.map +1 -1
  75. package/lib/network/reqresp/handlers/executionPayloadEnvelopesByRange.d.ts.map +1 -1
  76. package/lib/network/reqresp/handlers/executionPayloadEnvelopesByRange.js +47 -34
  77. package/lib/network/reqresp/handlers/executionPayloadEnvelopesByRange.js.map +1 -1
  78. package/lib/network/reqresp/handlers/executionPayloadEnvelopesByRoot.d.ts.map +1 -1
  79. package/lib/network/reqresp/handlers/executionPayloadEnvelopesByRoot.js +23 -4
  80. package/lib/network/reqresp/handlers/executionPayloadEnvelopesByRoot.js.map +1 -1
  81. package/lib/network/reqresp/handlers/index.d.ts.map +1 -1
  82. package/lib/network/reqresp/handlers/index.js +20 -4
  83. package/lib/network/reqresp/handlers/index.js.map +1 -1
  84. package/lib/sync/types.d.ts +0 -2
  85. package/lib/sync/types.d.ts.map +1 -1
  86. package/lib/sync/types.js +0 -2
  87. package/lib/sync/types.js.map +1 -1
  88. package/lib/sync/unknownBlock.d.ts.map +1 -1
  89. package/lib/sync/unknownBlock.js +41 -13
  90. package/lib/sync/unknownBlock.js.map +1 -1
  91. package/lib/util/execution.d.ts +42 -0
  92. package/lib/util/execution.d.ts.map +1 -1
  93. package/lib/util/execution.js +123 -2
  94. package/lib/util/execution.js.map +1 -1
  95. package/lib/util/headerEnvelope.d.ts +17 -0
  96. package/lib/util/headerEnvelope.d.ts.map +1 -0
  97. package/lib/util/headerEnvelope.js +43 -0
  98. package/lib/util/headerEnvelope.js.map +1 -0
  99. package/package.json +15 -14
  100. package/src/api/impl/beacon/blocks/index.ts +16 -3
  101. package/src/chain/archiveStore/archiveStore.ts +6 -3
  102. package/src/chain/archiveStore/historicalState/metrics.ts +3 -2
  103. package/src/chain/archiveStore/utils/archiveBlocks.ts +54 -32
  104. package/src/chain/archiveStore/utils/pruneHistory.ts +4 -2
  105. package/src/chain/chain.ts +96 -20
  106. package/src/chain/errors/envelopeReconstructionError.ts +22 -0
  107. package/src/chain/errors/index.ts +1 -0
  108. package/src/chain/interface.ts +6 -1
  109. package/src/chain/options.ts +3 -0
  110. package/src/db/buckets.ts +1 -1
  111. package/src/db/repositories/executionPayloadEnvelopeArchive.ts +43 -13
  112. package/src/db/repositories/index.ts +6 -1
  113. package/src/execution/engine/disabled.ts +4 -0
  114. package/src/execution/engine/http.ts +12 -0
  115. package/src/execution/engine/interface.ts +4 -1
  116. package/src/execution/engine/mock.ts +22 -2
  117. package/src/execution/engine/types.ts +25 -0
  118. package/src/metrics/metrics/beacon.ts +16 -0
  119. package/src/metrics/metrics/lodestar.ts +24 -0
  120. package/src/metrics/metrics.ts +7 -2
  121. package/src/network/reqresp/handlers/executionPayloadEnvelopesByRange.ts +57 -39
  122. package/src/network/reqresp/handlers/executionPayloadEnvelopesByRoot.ts +27 -5
  123. package/src/network/reqresp/handlers/index.ts +49 -5
  124. package/src/sync/types.ts +0 -2
  125. package/src/sync/unknownBlock.ts +41 -13
  126. package/src/util/execution.ts +175 -2
  127. package/src/util/headerEnvelope.ts +76 -0
@@ -61,6 +61,7 @@ import {ProcessShutdownCallback} from "@lodestar/validator";
61
61
  import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js";
62
62
  import {IBeaconDb} from "../db/index.js";
63
63
  import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../db/repositories/blobSidecars.js";
64
+ import {decodeArchivedEnvelope} from "../db/repositories/index.js";
64
65
  import {BuilderApiClient, BuilderApiClientOpts} from "../execution/builder/apiClient.js";
65
66
  import {BuilderStatus} from "../execution/builder/http.js";
66
67
  import {IExecutionBuilder, IExecutionEngine} from "../execution/index.js";
@@ -70,6 +71,7 @@ import {BufferPool} from "../util/bufferPool.js";
70
71
  import {Clock, ClockEvent, IClock} from "../util/clock.js";
71
72
  import {CustodyConfig, getValidatorsCustodyRequirement} from "../util/dataColumns.js";
72
73
  import {callInNextEventLoop} from "../util/eventLoop.js";
74
+ import {ReconstructMismatchPolicy, isRebuildMiss, reconstructExecutionPayloadEnvelopes} from "../util/execution.js";
73
75
  import {ensureDir, writeIfNotExist} from "../util/file.js";
74
76
  import {isOptimisticBlock} from "../util/forkChoice.js";
75
77
  import {JobItemQueue} from "../util/queue/itemQueue.js";
@@ -926,21 +928,71 @@ export class BeaconChain implements IBeaconChain {
926
928
  }
927
929
 
928
930
  async getSerializedExecutionPayloadEnvelope(blockSlot: Slot, blockRootHex: string): Promise<Uint8Array | null> {
929
- const payloadInput = this.seenPayloadEnvelopeInputCache.get(blockRootHex);
930
- if (payloadInput?.hasPayloadEnvelope()) {
931
- const envelope = payloadInput.getPayloadEnvelope();
932
- const serialized = this.serializedCache.get(envelope);
933
- if (serialized) {
934
- return serialized;
931
+ const [bytes] = await this.getSerializedExecutionPayloadEnvelopes([{blockSlot, blockRootHex}]);
932
+ return bytes;
933
+ }
934
+
935
+ /**
936
+ * Batch variant: archived header envelopes are rebuilt 32 per EL round-trip. Aligned with `requests`.
937
+ * A body root mismatch is a local inconsistency: `"throw"` surfaces it, `"omit"` logs it and
938
+ * returns null for that entry, for peer-facing paths where the spec allows omission.
939
+ */
940
+ async getSerializedExecutionPayloadEnvelopes(
941
+ requests: {blockSlot: Slot; blockRootHex: RootHex}[],
942
+ onMismatch: ReconstructMismatchPolicy = "throw"
943
+ ): Promise<(Uint8Array | null)[]> {
944
+ const out: (Uint8Array | null)[] = new Array(requests.length).fill(null);
945
+ const headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[] = [];
946
+ const headerEnvelopeIdxs: number[] = [];
947
+
948
+ for (let i = 0; i < requests.length; i++) {
949
+ const {blockSlot, blockRootHex} = requests[i];
950
+
951
+ const payloadInput = this.seenPayloadEnvelopeInputCache.get(blockRootHex);
952
+ if (payloadInput?.hasPayloadEnvelope()) {
953
+ const envelope = payloadInput.getPayloadEnvelope();
954
+ out[i] = this.serializedCache.get(envelope) ?? ssz.gloas.SignedExecutionPayloadEnvelope.serialize(envelope);
955
+ continue;
956
+ }
957
+
958
+ const hot = await this.db.executionPayloadEnvelope.getBinary(fromHex(blockRootHex));
959
+ if (hot !== null) {
960
+ out[i] = hot;
961
+ continue;
962
+ }
963
+
964
+ const archivedBytes = await this.db.executionPayloadEnvelopeArchive.getBinary(blockSlot);
965
+ if (archivedBytes === null) continue;
966
+
967
+ const archived = decodeArchivedEnvelope(archivedBytes);
968
+ if (archived.envelopeBytes !== undefined) {
969
+ out[i] = archived.envelopeBytes;
970
+ continue;
971
+ }
972
+ headerEnvelopes.push(archived.headerEnvelope);
973
+ headerEnvelopeIdxs.push(i);
974
+ }
975
+
976
+ if (headerEnvelopes.length > 0) {
977
+ const rebuilt = await reconstructExecutionPayloadEnvelopes(this.executionEngine, this.metrics, headerEnvelopes);
978
+ for (let j = 0; j < rebuilt.length; j++) {
979
+ const result = rebuilt[j];
980
+ if (isRebuildMiss(result)) {
981
+ if (result.reason === "mismatch") {
982
+ if (onMismatch === "throw") throw result.error;
983
+ this.logger.debug(
984
+ "Archived envelope failed body root check against EL bodies",
985
+ {slot: result.slot},
986
+ result.error
987
+ );
988
+ }
989
+ continue;
990
+ }
991
+ out[headerEnvelopeIdxs[j]] = ssz.gloas.SignedExecutionPayloadEnvelope.serialize(result);
935
992
  }
936
- return ssz.gloas.SignedExecutionPayloadEnvelope.serialize(envelope);
937
993
  }
938
994
 
939
- return (
940
- (await this.db.executionPayloadEnvelope.getBinary(fromHex(blockRootHex))) ??
941
- (await this.db.executionPayloadEnvelopeArchive.getBinary(blockSlot)) ??
942
- null
943
- );
995
+ return out;
944
996
  }
945
997
 
946
998
  async getExecutionPayloadEnvelope(
@@ -952,11 +1004,23 @@ export class BeaconChain implements IBeaconChain {
952
1004
  return payloadInput.getPayloadEnvelope();
953
1005
  }
954
1006
 
955
- return (
956
- (await this.db.executionPayloadEnvelope.get(fromHex(blockRootHex))) ??
957
- (await this.db.executionPayloadEnvelopeArchive.get(blockSlot)) ??
958
- null
959
- );
1007
+ const hot = await this.db.executionPayloadEnvelope.get(fromHex(blockRootHex));
1008
+ if (hot !== null) return hot;
1009
+
1010
+ const archivedBytes = await this.db.executionPayloadEnvelopeArchive.getBinary(blockSlot);
1011
+ if (archivedBytes === null) return null;
1012
+ const archived = decodeArchivedEnvelope(archivedBytes);
1013
+ if (archived.envelopeBytes !== undefined) {
1014
+ return ssz.gloas.SignedExecutionPayloadEnvelope.deserialize(archived.envelopeBytes);
1015
+ }
1016
+ const [result] = await reconstructExecutionPayloadEnvelopes(this.executionEngine, this.metrics, [
1017
+ archived.headerEnvelope,
1018
+ ]);
1019
+ if (isRebuildMiss(result)) {
1020
+ if (result.reason === "mismatch") throw result.error;
1021
+ return null;
1022
+ }
1023
+ return result;
960
1024
  }
961
1025
 
962
1026
  async getParentExecutionRequests(
@@ -967,11 +1031,23 @@ export class BeaconChain implements IBeaconChain {
967
1031
  if (!isForkPostGloas(this.config.getForkName(parentBlockSlot))) {
968
1032
  return ssz.gloas.ExecutionRequests.defaultValue();
969
1033
  }
970
- const envelope = await this.getExecutionPayloadEnvelope(parentBlockSlot, parentBlockRootHex);
971
- if (envelope === null) {
1034
+ // executionRequests is kept in the header envelope, so read it without reconstructing
1035
+ const payloadInput = this.seenPayloadEnvelopeInputCache.get(parentBlockRootHex);
1036
+ if (payloadInput?.hasPayloadEnvelope()) {
1037
+ return payloadInput.getPayloadEnvelope().message.executionRequests;
1038
+ }
1039
+
1040
+ const hot = await this.db.executionPayloadEnvelope.get(fromHex(parentBlockRootHex));
1041
+ if (hot !== null) return hot.message.executionRequests;
1042
+
1043
+ const archivedBytes = await this.db.executionPayloadEnvelopeArchive.getBinary(parentBlockSlot);
1044
+ if (archivedBytes === null) {
972
1045
  throw Error(`Parent execution payload envelope not found slot=${parentBlockSlot}, root=${parentBlockRootHex}`);
973
1046
  }
974
- return envelope.message.executionRequests;
1047
+ const archived = decodeArchivedEnvelope(archivedBytes);
1048
+ return archived.envelopeBytes !== undefined
1049
+ ? ssz.gloas.SignedExecutionPayloadEnvelope.deserialize(archived.envelopeBytes).message.executionRequests
1050
+ : archived.headerEnvelope.message.executionRequests;
975
1051
  }
976
1052
 
977
1053
  async getDataColumnSidecars(blockSlot: Slot, blockRootHex: string): Promise<DataColumnSidecar[]> {
@@ -0,0 +1,22 @@
1
+ import {Slot} from "@lodestar/types";
2
+ import {LodestarError} from "@lodestar/utils";
3
+
4
+ export enum EnvelopeReconstructionErrorCode {
5
+ /** The EL failed to answer engine_getPayloadBodiesByHashV2 (transport / EL down). Transient. */
6
+ ENGINE_UNAVAILABLE = "ENVELOPE_RECONSTRUCTION_ERROR_ENGINE_UNAVAILABLE",
7
+ /** A body returned by the EL does not hash to the root stored in the header envelope. Local DB/EL inconsistency. */
8
+ BODY_ROOT_MISMATCH = "ENVELOPE_RECONSTRUCTION_ERROR_BODY_ROOT_MISMATCH",
9
+ /** A by-range stream stopped short at a slot that cannot be served; the response so far is still valid */
10
+ RANGE_UNSERVABLE = "ENVELOPE_RECONSTRUCTION_ERROR_RANGE_UNSERVABLE",
11
+ }
12
+
13
+ export type EnvelopeReconstructionErrorType =
14
+ | {code: EnvelopeReconstructionErrorCode.ENGINE_UNAVAILABLE}
15
+ | {
16
+ code: EnvelopeReconstructionErrorCode.BODY_ROOT_MISMATCH;
17
+ slot: Slot;
18
+ field: "transactions" | "withdrawals" | "blockAccessList";
19
+ }
20
+ | {code: EnvelopeReconstructionErrorCode.RANGE_UNSERVABLE; slot: Slot};
21
+
22
+ export class EnvelopeReconstructionError extends LodestarError<EnvelopeReconstructionErrorType> {}
@@ -4,6 +4,7 @@ export * from "./blobSidecarError.js";
4
4
  export * from "./blockError.js";
5
5
  export * from "./blsToExecutionChangeError.js";
6
6
  export * from "./dataColumnSidecarError.js";
7
+ export * from "./envelopeReconstructionError.js";
7
8
  export * from "./executionPayloadBid.js";
8
9
  export * from "./executionPayloadEnvelope.js";
9
10
  export * from "./gossipValidation.js";
@@ -30,6 +30,7 @@ import {Metrics} from "../metrics/metrics.js";
30
30
  import {BufferPool} from "../util/bufferPool.js";
31
31
  import {IClock} from "../util/clock.js";
32
32
  import {CustodyConfig} from "../util/dataColumns.js";
33
+ import {ReconstructMismatchPolicy} from "../util/execution.js";
33
34
  import {SerializedCache} from "../util/serializedCache.js";
34
35
  import {IArchiveStore} from "./archiveStore/interface.js";
35
36
  import {CheckpointBalancesCache} from "./balancesCache.js";
@@ -96,7 +97,7 @@ export enum FindHeadFnName {
96
97
  export interface IBeaconChain {
97
98
  readonly genesisTime: UintNum64;
98
99
  readonly genesisValidatorsRoot: Root;
99
- readonly earliestAvailableSlot: Slot;
100
+ earliestAvailableSlot: Slot;
100
101
  readonly executionEngine: IExecutionEngine;
101
102
  readonly executionBuilder?: IExecutionBuilder;
102
103
  readonly builderCircuitBreaker: BuilderCircuitBreaker;
@@ -236,6 +237,10 @@ export interface IBeaconChain {
236
237
  indices: number[]
237
238
  ): Promise<(Uint8Array | undefined)[]>;
238
239
  getSerializedExecutionPayloadEnvelope(blockSlot: Slot, blockRootHex: string): Promise<Uint8Array | null>;
240
+ getSerializedExecutionPayloadEnvelopes(
241
+ requests: {blockSlot: Slot; blockRootHex: RootHex}[],
242
+ onMismatch?: ReconstructMismatchPolicy
243
+ ): Promise<(Uint8Array | null)[]>;
239
244
  getExecutionPayloadEnvelope(
240
245
  blockSlot: Slot,
241
246
  blockRootHex: string
@@ -29,6 +29,8 @@ export type IChainOptions = BlockProcessOpts &
29
29
  blacklistedBlocks?: string[];
30
30
  // TODO GLOAS: add similar option for execution payload envelopes?
31
31
  persistProducedBlocks?: boolean;
32
+ /** Archive finalized Gloas payload envelopes as headers (bodies reconstructed from the EL on read) */
33
+ dedupePayloads?: boolean;
32
34
  persistInvalidSszObjects?: boolean;
33
35
  persistInvalidSszObjectsDir?: string;
34
36
  persistOrphanedBlocks?: boolean;
@@ -105,6 +107,7 @@ export const defaultChainOptions: IChainOptions = {
105
107
  disableBlsBatchVerify: false,
106
108
  proposerBoost: true,
107
109
  proposerBoostReorg: true,
110
+ dedupePayloads: true,
108
111
  computeUnrealized: true,
109
112
  fastConfirmation: false,
110
113
  suggestedFeeRecipient: defaultValidatorOptions.suggestedFeeRecipient,
package/src/db/buckets.ts CHANGED
@@ -71,7 +71,7 @@ export enum Bucket {
71
71
  allForks_dataColumnSidecarsArchive = 58, // BeaconBlockSlot -> DataColumnSidecars
72
72
 
73
73
  gloas_executionPayloadEnvelope = 59, // GLOAS BeaconBlockRoot -> SignedExecutionPayloadEnvelope
74
- gloas_executionPayloadEnvelopeArchive = 60, // GLOAS Slot -> SignedExecutionPayloadEnvelope
74
+ gloas_executionPayloadEnvelopeArchive = 60, // GLOAS Slot -> SignedExecutionPayloadEnvelope, or 0x00 + SignedExecutionPayloadHeaderEnvelope
75
75
  }
76
76
 
77
77
  export function getBucketNameByValue<T extends Bucket>(enumValue: T): keyof typeof Bucket {
@@ -1,25 +1,44 @@
1
1
  import {ChainForkConfig} from "@lodestar/config";
2
- import {BUCKET_LENGTH, Db, Repository, encodeKey as encodeDbKey} from "@lodestar/db";
2
+ import {BUCKET_LENGTH, BinaryRepository, Db, DbBatch, encodeKey as encodeDbKey} from "@lodestar/db";
3
3
  import {Slot, gloas, ssz} from "@lodestar/types";
4
4
  import {bytesToInt} from "@lodestar/utils";
5
5
  import {Bucket, getBucketNameByValue} from "../buckets.js";
6
6
 
7
7
  /**
8
- * Used to store finalized `SignedExecutionPayloadEnvelope`
9
- *
10
- * Indexed by slot for chronological archival
8
+ * Prefix of a header entry. A full `SignedExecutionPayloadEnvelope` starts with the offset of its
9
+ * variable-size `message` field (4 + 96 = 0x64), so a leading 0x00 byte is unambiguous and full
10
+ * entries can be stored as the hot db bytes, untouched.
11
11
  */
12
- export class ExecutionPayloadEnvelopeArchiveRepository extends Repository<Slot, gloas.SignedExecutionPayloadEnvelope> {
12
+ const HEADER_ENVELOPE_PREFIX = 0x00;
13
+
14
+ /** An archive entry as the serving paths need it: full stays bytes, header is deserialized for reconstruction */
15
+ export type ArchivedEnvelope =
16
+ | {headerEnvelope: gloas.SignedExecutionPayloadHeaderEnvelope; envelopeBytes?: undefined}
17
+ | {headerEnvelope?: undefined; envelopeBytes: Uint8Array};
18
+
19
+ export function decodeArchivedEnvelope(bytes: Uint8Array): ArchivedEnvelope {
20
+ return bytes[0] === HEADER_ENVELOPE_PREFIX
21
+ ? {headerEnvelope: ssz.gloas.SignedExecutionPayloadHeaderEnvelope.deserialize(bytes.subarray(1))}
22
+ : {envelopeBytes: bytes};
23
+ }
24
+
25
+ export function encodeArchivedHeaderEnvelope(headerEnvelope: gloas.SignedExecutionPayloadHeaderEnvelope): Uint8Array {
26
+ const value = ssz.gloas.SignedExecutionPayloadHeaderEnvelope.serialize(headerEnvelope);
27
+ const out = new Uint8Array(1 + value.length);
28
+ out[0] = HEADER_ENVELOPE_PREFIX;
29
+ out.set(value, 1);
30
+ return out;
31
+ }
32
+
33
+ /**
34
+ * Finalized envelopes indexed by slot: `SignedExecutionPayloadEnvelope` bytes as-is
35
+ * (`--chain.dedupePayloads=false`, or the block was still optimistic when archived), or a
36
+ * `SignedExecutionPayloadHeaderEnvelope` behind a 0x00 prefix. See {@link decodeArchivedEnvelope}.
37
+ */
38
+ export class ExecutionPayloadEnvelopeArchiveRepository extends BinaryRepository<Slot> {
13
39
  constructor(config: ChainForkConfig, db: Db) {
14
40
  const bucket = Bucket.gloas_executionPayloadEnvelopeArchive;
15
- super(config, db, bucket, ssz.gloas.SignedExecutionPayloadEnvelope, getBucketNameByValue(bucket));
16
- }
17
-
18
- /**
19
- * Id is the slot from the envelope
20
- */
21
- getId(value: gloas.SignedExecutionPayloadEnvelope): Slot {
22
- return value.message.payload.slotNumber;
41
+ super(config, db, bucket, getBucketNameByValue(bucket));
23
42
  }
24
43
 
25
44
  encodeKey(id: Slot): Uint8Array {
@@ -29,4 +48,15 @@ export class ExecutionPayloadEnvelopeArchiveRepository extends Repository<Slot,
29
48
  decodeKey(data: Uint8Array): number {
30
49
  return bytesToInt(data.subarray(BUCKET_LENGTH), "be");
31
50
  }
51
+
52
+ /** Archive entries and delete their hot counterparts (`hotKey` pre-encoded) in one atomic batch */
53
+ async batchArchiveAndDeleteHot(
54
+ entries: {slot: Slot; archivedBytes: Uint8Array; hotKey: Uint8Array}[]
55
+ ): Promise<void> {
56
+ const operations: DbBatch<Uint8Array, Uint8Array> = [];
57
+ for (const {slot, archivedBytes, hotKey} of entries) {
58
+ operations.push({type: "put", key: this.encodeKey(slot), value: archivedBytes}, {type: "del", key: hotKey});
59
+ }
60
+ await this.db.batch(operations, this.dbReqOpts);
61
+ }
32
62
  }
@@ -8,7 +8,12 @@ export {BLSToExecutionChangeRepository} from "./blsToExecutionChange.js";
8
8
  export {DataColumnSidecarRepository} from "./dataColumnSidecar.js";
9
9
  export {DataColumnSidecarArchiveRepository} from "./dataColumnSidecarArchive.js";
10
10
  export {ExecutionPayloadEnvelopeRepository} from "./executionPayloadEnvelope.js";
11
- export {ExecutionPayloadEnvelopeArchiveRepository} from "./executionPayloadEnvelopeArchive.js";
11
+ export {
12
+ type ArchivedEnvelope,
13
+ ExecutionPayloadEnvelopeArchiveRepository,
14
+ decodeArchivedEnvelope,
15
+ encodeArchivedHeaderEnvelope,
16
+ } from "./executionPayloadEnvelopeArchive.js";
12
17
  export {BestLightClientUpdateRepository} from "./lightclientBestUpdate.js";
13
18
  export {CheckpointHeaderRepository} from "./lightclientCheckpointHeader.js";
14
19
  export {SyncCommitteeRepository} from "./lightclientSyncCommittee.js";
@@ -25,6 +25,10 @@ export class ExecutionEngineDisabled implements IExecutionEngine {
25
25
  throw Error("Execution engine disabled");
26
26
  }
27
27
 
28
+ getPayloadBodiesByHashV2(): Promise<never> {
29
+ throw Error("Execution engine disabled");
30
+ }
31
+
28
32
  getPayloadBodiesByRange(): Promise<never> {
29
33
  throw Error("Execution engine disabled");
30
34
  }
@@ -32,11 +32,13 @@ import {
32
32
  EngineApiRpcParamTypes,
33
33
  EngineApiRpcReturnTypes,
34
34
  ExecutionPayloadBody,
35
+ ExecutionPayloadBodyV2,
35
36
  assertReqSizeLimit,
36
37
  deserializeBlobAndProofs,
37
38
  deserializeBlobAndProofsV2,
38
39
  deserializeBlobAndProofsV2IntoBytes,
39
40
  deserializeExecutionPayloadBody,
41
+ deserializeExecutionPayloadBodyV2,
40
42
  parseExecutionPayload,
41
43
  serializeBeaconBlockRoot,
42
44
  serializeExecutionPayload,
@@ -476,6 +478,16 @@ export class ExecutionEngineHttp implements IExecutionEngine {
476
478
  return response.map(deserializeExecutionPayloadBody);
477
479
  }
478
480
 
481
+ async getPayloadBodiesByHashV2(blockHashes: RootHex[]): Promise<(ExecutionPayloadBodyV2 | null)[]> {
482
+ const method = "engine_getPayloadBodiesByHashV2";
483
+ assertReqSizeLimit(blockHashes.length, 32);
484
+ const response = await this.rpc.fetchWithRetries<
485
+ EngineApiRpcReturnTypes[typeof method],
486
+ EngineApiRpcParamTypes[typeof method]
487
+ >({method, params: [blockHashes]}, getPayloadBodiesByHashOpts);
488
+ return response.map(deserializeExecutionPayloadBodyV2);
489
+ }
490
+
479
491
  async getPayloadBodiesByRange(
480
492
  _fork: ForkName,
481
493
  startBlockNumber: number,
@@ -13,7 +13,7 @@ import {BlobsBundle, ExecutionPayload, ExecutionRequests, Root, RootHex, Wei, ca
13
13
  import {BlobAndProof} from "@lodestar/types/deneb";
14
14
  import {BlobAndProofV2} from "@lodestar/types/fulu";
15
15
  import {PayloadId, PayloadIdCache, WithdrawalV1} from "./payloadIdCache.js";
16
- import {ExecutionPayloadBody} from "./types.js";
16
+ import {ExecutionPayloadBody, ExecutionPayloadBodyV2} from "./types.js";
17
17
  import {DATA} from "./utils.js";
18
18
 
19
19
  export {PayloadIdCache, type PayloadId, type WithdrawalV1};
@@ -172,6 +172,9 @@ export interface IExecutionEngine {
172
172
 
173
173
  getPayloadBodiesByHash(fork: ForkName, blockHash: DATA[]): Promise<(ExecutionPayloadBody | null)[]>;
174
174
 
175
+ /** Amsterdam: like V1 but also returns the block access list (null if the EL no longer has it) */
176
+ getPayloadBodiesByHashV2(blockHash: DATA[]): Promise<(ExecutionPayloadBodyV2 | null)[]>;
177
+
175
178
  getPayloadBodiesByRange(fork: ForkName, start: number, count: number): Promise<(ExecutionPayloadBody | null)[]>;
176
179
 
177
180
  getBlobs(
@@ -23,6 +23,7 @@ import {
23
23
  EngineApiRpcParamTypes,
24
24
  EngineApiRpcReturnTypes,
25
25
  ExecutionPayloadBodyRpc,
26
+ ExecutionPayloadBodyV2Rpc,
26
27
  ExecutionPayloadRpc,
27
28
  ExecutionRequestsRpc,
28
29
  PayloadStatus,
@@ -49,6 +50,8 @@ type ExecutionBlock = {
49
50
  blockHash: RootHex;
50
51
  timestamp: number;
51
52
  blockNumber: number;
53
+ /** Bodies as received via newPayload, served back by engine_getPayloadBodiesByHash{V1,V2} */
54
+ body: ExecutionPayloadBodyV2Rpc;
52
55
  };
53
56
 
54
57
  const TX_TYPE_EIP1559 = 2;
@@ -93,6 +96,7 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
93
96
  blockHash: ZERO_HASH_HEX,
94
97
  timestamp: 0,
95
98
  blockNumber: 0,
99
+ body: {transactions: [], withdrawals: null, blockAccessList: null},
96
100
  });
97
101
 
98
102
  const eth1BlockHash = opts.eth1BlockHash ?? toRootHex(INTEROP_BLOCK_HASH);
@@ -102,6 +106,7 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
102
106
  blockHash: eth1BlockHash,
103
107
  timestamp: 0,
104
108
  blockNumber: 1,
109
+ body: {transactions: [], withdrawals: null, blockAccessList: null},
105
110
  });
106
111
 
107
112
  const {config} = opts;
@@ -144,6 +149,7 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
144
149
  engine_getPayloadV5: this.getPayloadV5.bind(this),
145
150
  engine_getPayloadV6: this.getPayloadV5.bind(this),
146
151
  engine_getPayloadBodiesByHashV1: this.getPayloadBodiesByHash.bind(this),
152
+ engine_getPayloadBodiesByHashV2: this.getPayloadBodiesByHashV2.bind(this),
147
153
  engine_getPayloadBodiesByRangeV1: this.getPayloadBodiesByRange.bind(this),
148
154
  engine_getClientVersionV1: this.getClientVersionV1.bind(this),
149
155
  engine_getBlobsV1: this.getBlobs.bind(this),
@@ -152,9 +158,18 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
152
158
  }
153
159
 
154
160
  private getPayloadBodiesByHash(
155
- _blockHex: EngineApiRpcParamTypes["engine_getPayloadBodiesByHashV1"][0]
161
+ blockHashes: EngineApiRpcParamTypes["engine_getPayloadBodiesByHashV1"][0]
156
162
  ): EngineApiRpcReturnTypes["engine_getPayloadBodiesByHashV1"] {
157
- return [] as ExecutionPayloadBodyRpc[];
163
+ return this.getPayloadBodiesByHashV2(blockHashes).map((body) =>
164
+ body ? {transactions: body.transactions, withdrawals: body.withdrawals} : null
165
+ );
166
+ }
167
+
168
+ private getPayloadBodiesByHashV2(
169
+ blockHashes: EngineApiRpcParamTypes["engine_getPayloadBodiesByHashV2"][0]
170
+ ): EngineApiRpcReturnTypes["engine_getPayloadBodiesByHashV2"] {
171
+ // null for unknown blocks, as a real EL does; the genesis/eth1 seed blocks carry no body
172
+ return blockHashes.map((hash) => this.validBlocks.get(hash)?.body ?? null);
158
173
  }
159
174
 
160
175
  private getPayloadBodiesByRange(
@@ -227,6 +242,11 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
227
242
  blockHash,
228
243
  timestamp: quantityToNum(executionPayloadRpc.timestamp),
229
244
  blockNumber: quantityToNum(executionPayloadRpc.blockNumber),
245
+ body: {
246
+ transactions: executionPayloadRpc.transactions,
247
+ withdrawals: executionPayloadRpc.withdrawals ?? null,
248
+ blockAccessList: executionPayloadRpc.blockAccessList ?? null,
249
+ },
230
250
  });
231
251
 
232
252
  // IF the payload has been fully validated while processing the call
@@ -89,6 +89,7 @@ export type EngineApiRpcParamTypes = {
89
89
  * 1. Array of DATA - Array of block_hash field values of the ExecutionPayload structure
90
90
  * */
91
91
  engine_getPayloadBodiesByHashV1: DATA[][];
92
+ engine_getPayloadBodiesByHashV2: DATA[][];
92
93
 
93
94
  /**
94
95
  * 1. start: QUANTITY, 64 bits - Starting block number
@@ -148,6 +149,7 @@ export type EngineApiRpcReturnTypes = {
148
149
  engine_getPayloadV6: ExecutionPayloadResponse;
149
150
 
150
151
  engine_getPayloadBodiesByHashV1: (ExecutionPayloadBodyRpc | null)[];
152
+ engine_getPayloadBodiesByHashV2: (ExecutionPayloadBodyV2Rpc | null)[];
151
153
 
152
154
  engine_getPayloadBodiesByRangeV1: (ExecutionPayloadBodyRpc | null)[];
153
155
 
@@ -177,6 +179,15 @@ export type ExecutionPayloadBody = {
177
179
  withdrawals: capella.Withdrawals | null;
178
180
  };
179
181
 
182
+ /** engine_getPayloadBodiesByHashV2 (Amsterdam): adds the RLP-encoded block access list, or null if the EL has pruned it */
183
+ export type ExecutionPayloadBodyV2Rpc = ExecutionPayloadBodyRpc & {
184
+ blockAccessList?: DATA | null;
185
+ };
186
+
187
+ export type ExecutionPayloadBodyV2 = ExecutionPayloadBody & {
188
+ blockAccessList: Uint8Array | null;
189
+ };
190
+
180
191
  export type ExecutionPayloadRpc = {
181
192
  parentHash: DATA; // 32 bytes
182
193
  feeRecipient: DATA; // 20 bytes
@@ -678,6 +689,20 @@ export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc |
678
689
  : null;
679
690
  }
680
691
 
692
+ export function deserializeExecutionPayloadBodyV2(
693
+ data: ExecutionPayloadBodyV2Rpc | null
694
+ ): ExecutionPayloadBodyV2 | null {
695
+ const body = deserializeExecutionPayloadBody(data);
696
+ return body
697
+ ? {...body, blockAccessList: data?.blockAccessList ? dataToBytes(data.blockAccessList, null) : null}
698
+ : null;
699
+ }
700
+
701
+ export function serializeExecutionPayloadBodyV2(data: ExecutionPayloadBodyV2 | null): ExecutionPayloadBodyV2Rpc | null {
702
+ const body = serializeExecutionPayloadBody(data);
703
+ return body ? {...body, blockAccessList: data?.blockAccessList ? bytesToData(data.blockAccessList) : null} : null;
704
+ }
705
+
681
706
  export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null): ExecutionPayloadBodyRpc | null {
682
707
  return data
683
708
  ? {
@@ -215,6 +215,22 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
215
215
  help: "Number of getBlobsV1 hits where a versioned hash returns blob and the blob is needed so call is useful",
216
216
  }),
217
217
  },
218
+ payloadEnvelopeReconstruction: {
219
+ envelopes: register.counter<{result: "ok" | "unavailable" | "mismatch"}>({
220
+ name: "beacon_payload_envelope_reconstruction_envelopes_total",
221
+ help: "Archived header envelopes rebuilt from EL bodies, by outcome",
222
+ labelNames: ["result"],
223
+ }),
224
+ mismatchByField: register.counter<{field: "transactions" | "withdrawals" | "blockAccessList"}>({
225
+ name: "beacon_payload_envelope_reconstruction_mismatch_total",
226
+ help: "Body root mismatches, by which body differed from the stored root",
227
+ labelNames: ["field"],
228
+ }),
229
+ engineErrors: register.counter({
230
+ name: "beacon_payload_envelope_reconstruction_engine_errors_total",
231
+ help: "engine_getPayloadBodiesByHashV2 round-trips that failed while rebuilding envelopes",
232
+ }),
233
+ },
218
234
 
219
235
  blockInputFetchStats: {
220
236
  // of already available blocks which didn't have to go through blobs pull
@@ -180,6 +180,30 @@ export function createLodestarMetrics(
180
180
  buckets: [0.001, 0.003, 0.01, 0.03, 0.1],
181
181
  }),
182
182
 
183
+ reqRespByRange: {
184
+ requestedSlots: register.histogram<{method: string}>({
185
+ name: "lodestar_reqresp_by_range_requested_slots",
186
+ help: "Histogram of slot count requested per served by_range request",
187
+ labelNames: ["method"],
188
+ buckets: [1, 16, 32, 64, 128],
189
+ }),
190
+ // data_column_sidecars_by_range is the only by_range request carrying columns
191
+ requestedColumns: register.histogram({
192
+ name: "lodestar_reqresp_by_range_requested_columns",
193
+ help: "Histogram of column count requested per served data_column_sidecars_by_range request",
194
+ // 128 is NUMBER_OF_COLUMNS.
195
+ buckets: [1, 4, 8, 32, 128],
196
+ }),
197
+ servedBytes: register.histogram<{method: string}>({
198
+ name: "lodestar_reqresp_by_range_served_bytes",
199
+ help: "Histogram of total serialized bytes served per by_range request",
200
+ labelNames: ["method"],
201
+ // min bucket is roughly a typical mainnet block is 100MB,
202
+ // max bucket is roughly 100 * MAX_PAYLOAD_SIZE
203
+ buckets: [100e3, 1e6, 10e6, 100e6, 1000e6],
204
+ }),
205
+ },
206
+
183
207
  regenQueue: {
184
208
  length: register.gauge({
185
209
  name: "lodestar_regen_queue_length",
@@ -12,7 +12,12 @@ export type Metrics = BeaconMetrics &
12
12
  BeaconStateTransitionMetrics &
13
13
  LodestarMetrics & {register: RegistryMetricCreator; close: () => void};
14
14
 
15
- export function createMetrics(opts: MetricsOptions, genesisTime: number, externalRegistries: Registry[] = []): Metrics {
15
+ export function createMetrics(
16
+ opts: MetricsOptions,
17
+ genesisTime: number,
18
+ externalRegistries: Registry[] = [],
19
+ {collectNodeMetrics = true}: {collectNodeMetrics?: boolean} = {}
20
+ ): Metrics {
16
21
  const register = new RegistryMetricCreator();
17
22
  const beacon = createBeaconMetrics(register);
18
23
  const forkChoice = getForkChoiceMetrics(register);
@@ -24,7 +29,7 @@ export function createMetrics(opts: MetricsOptions, genesisTime: number, externa
24
29
  };
25
30
  process.on("unhandledRejection", onUnhandledRejection);
26
31
 
27
- const nodeJsMetricsClose = collectNodeJSMetrics(register);
32
+ const nodeJsMetricsClose = collectNodeMetrics ? collectNodeJSMetrics(register) : (): void => {};
28
33
  const close = (): void => {
29
34
  process.removeListener("unhandledRejection", onUnhandledRejection);
30
35
  nodeJsMetricsClose();