@lodestar/beacon-node 1.42.0-dev.5f2fffc2ce → 1.42.0-dev.70938e1eec

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +35 -16
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/chain/blocks/blockInput/types.d.ts +3 -3
  5. package/lib/chain/blocks/blockInput/types.d.ts.map +1 -1
  6. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  7. package/lib/chain/blocks/importBlock.js +18 -2
  8. package/lib/chain/blocks/importBlock.js.map +1 -1
  9. package/lib/chain/blocks/importExecutionPayload.d.ts +48 -0
  10. package/lib/chain/blocks/importExecutionPayload.d.ts.map +1 -0
  11. package/lib/chain/blocks/importExecutionPayload.js +159 -0
  12. package/lib/chain/blocks/importExecutionPayload.js.map +1 -0
  13. package/lib/chain/blocks/payloadEnvelopeInput/index.d.ts +3 -0
  14. package/lib/chain/blocks/payloadEnvelopeInput/index.d.ts.map +1 -0
  15. package/lib/chain/blocks/payloadEnvelopeInput/index.js +3 -0
  16. package/lib/chain/blocks/payloadEnvelopeInput/index.js.map +1 -0
  17. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts +80 -0
  18. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts.map +1 -0
  19. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js +248 -0
  20. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js.map +1 -0
  21. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts +29 -0
  22. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts.map +1 -0
  23. package/lib/chain/blocks/payloadEnvelopeInput/types.js +11 -0
  24. package/lib/chain/blocks/payloadEnvelopeInput/types.js.map +1 -0
  25. package/lib/chain/blocks/payloadEnvelopeProcessor.d.ts +15 -0
  26. package/lib/chain/blocks/payloadEnvelopeProcessor.d.ts.map +1 -0
  27. package/lib/chain/blocks/payloadEnvelopeProcessor.js +46 -0
  28. package/lib/chain/blocks/payloadEnvelopeProcessor.js.map +1 -0
  29. package/lib/chain/blocks/types.d.ts +7 -0
  30. package/lib/chain/blocks/types.d.ts.map +1 -1
  31. package/lib/chain/blocks/writePayloadEnvelopeInputToDb.d.ts +12 -0
  32. package/lib/chain/blocks/writePayloadEnvelopeInputToDb.d.ts.map +1 -0
  33. package/lib/chain/blocks/writePayloadEnvelopeInputToDb.js +40 -0
  34. package/lib/chain/blocks/writePayloadEnvelopeInputToDb.js.map +1 -0
  35. package/lib/chain/chain.d.ts +7 -2
  36. package/lib/chain/chain.d.ts.map +1 -1
  37. package/lib/chain/chain.js +28 -3
  38. package/lib/chain/chain.js.map +1 -1
  39. package/lib/chain/errors/executionPayloadEnvelope.d.ts +12 -2
  40. package/lib/chain/errors/executionPayloadEnvelope.d.ts.map +1 -1
  41. package/lib/chain/errors/executionPayloadEnvelope.js +3 -1
  42. package/lib/chain/errors/executionPayloadEnvelope.js.map +1 -1
  43. package/lib/chain/forkChoice/index.d.ts.map +1 -1
  44. package/lib/chain/forkChoice/index.js +0 -10
  45. package/lib/chain/forkChoice/index.js.map +1 -1
  46. package/lib/chain/interface.d.ts +6 -3
  47. package/lib/chain/interface.d.ts.map +1 -1
  48. package/lib/chain/opPools/utils.js +1 -1
  49. package/lib/chain/opPools/utils.js.map +1 -1
  50. package/lib/chain/produceBlock/computeNewStateRoot.d.ts.map +1 -1
  51. package/lib/chain/produceBlock/computeNewStateRoot.js +6 -1
  52. package/lib/chain/produceBlock/computeNewStateRoot.js.map +1 -1
  53. package/lib/chain/produceBlock/produceBlockBody.js +1 -1
  54. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  55. package/lib/chain/regen/interface.d.ts +2 -0
  56. package/lib/chain/regen/interface.d.ts.map +1 -1
  57. package/lib/chain/regen/interface.js +2 -0
  58. package/lib/chain/regen/interface.js.map +1 -1
  59. package/lib/chain/seenCache/index.d.ts +1 -1
  60. package/lib/chain/seenCache/index.d.ts.map +1 -1
  61. package/lib/chain/seenCache/index.js +1 -1
  62. package/lib/chain/seenCache/index.js.map +1 -1
  63. package/lib/chain/seenCache/seenGossipBlockInput.js +2 -2
  64. package/lib/chain/seenCache/seenGossipBlockInput.js.map +1 -1
  65. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts +38 -0
  66. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts.map +1 -0
  67. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js +76 -0
  68. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js.map +1 -0
  69. package/lib/chain/validation/executionPayloadEnvelope.d.ts.map +1 -1
  70. package/lib/chain/validation/executionPayloadEnvelope.js +30 -19
  71. package/lib/chain/validation/executionPayloadEnvelope.js.map +1 -1
  72. package/lib/chain/validation/lightClientFinalityUpdate.js +1 -1
  73. package/lib/chain/validation/lightClientFinalityUpdate.js.map +1 -1
  74. package/lib/chain/validation/lightClientOptimisticUpdate.js +1 -1
  75. package/lib/chain/validation/lightClientOptimisticUpdate.js.map +1 -1
  76. package/lib/chain/validatorMonitor.d.ts +2 -1
  77. package/lib/chain/validatorMonitor.d.ts.map +1 -1
  78. package/lib/chain/validatorMonitor.js +4 -1
  79. package/lib/chain/validatorMonitor.js.map +1 -1
  80. package/lib/execution/engine/interface.d.ts +2 -2
  81. package/lib/metrics/metrics/lodestar.d.ts +28 -0
  82. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  83. package/lib/metrics/metrics/lodestar.js +74 -0
  84. package/lib/metrics/metrics/lodestar.js.map +1 -1
  85. package/lib/network/gossip/topic.d.ts +727 -0
  86. package/lib/network/gossip/topic.d.ts.map +1 -1
  87. package/lib/network/network.js +2 -2
  88. package/lib/network/network.js.map +1 -1
  89. package/lib/network/processor/extractSlotRootFns.d.ts.map +1 -1
  90. package/lib/network/processor/extractSlotRootFns.js +14 -4
  91. package/lib/network/processor/extractSlotRootFns.js.map +1 -1
  92. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  93. package/lib/network/processor/gossipHandlers.js +31 -3
  94. package/lib/network/processor/gossipHandlers.js.map +1 -1
  95. package/lib/network/reqresp/ReqRespBeaconNode.d.ts +1 -1
  96. package/lib/network/reqresp/ReqRespBeaconNode.js +1 -1
  97. package/lib/sync/backfill/backfill.d.ts +1 -1
  98. package/lib/sync/backfill/backfill.js +1 -1
  99. package/lib/sync/constants.d.ts +1 -1
  100. package/lib/sync/constants.js +1 -1
  101. package/lib/util/sszBytes.d.ts +4 -1
  102. package/lib/util/sszBytes.d.ts.map +1 -1
  103. package/lib/util/sszBytes.js +69 -12
  104. package/lib/util/sszBytes.js.map +1 -1
  105. package/package.json +15 -15
  106. package/src/api/impl/beacon/blocks/index.ts +36 -17
  107. package/src/chain/blocks/blockInput/types.ts +3 -3
  108. package/src/chain/blocks/importBlock.ts +36 -2
  109. package/src/chain/blocks/importExecutionPayload.ts +241 -0
  110. package/src/chain/blocks/payloadEnvelopeInput/index.ts +2 -0
  111. package/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts +336 -0
  112. package/src/chain/blocks/payloadEnvelopeInput/types.ts +33 -0
  113. package/src/chain/blocks/payloadEnvelopeProcessor.ts +61 -0
  114. package/src/chain/blocks/types.ts +8 -0
  115. package/src/chain/blocks/writePayloadEnvelopeInputToDb.ts +55 -0
  116. package/src/chain/chain.ts +37 -3
  117. package/src/chain/errors/executionPayloadEnvelope.ts +6 -2
  118. package/src/chain/forkChoice/index.ts +0 -10
  119. package/src/chain/interface.ts +6 -3
  120. package/src/chain/opPools/utils.ts +1 -1
  121. package/src/chain/produceBlock/computeNewStateRoot.ts +6 -1
  122. package/src/chain/produceBlock/produceBlockBody.ts +1 -1
  123. package/src/chain/regen/interface.ts +2 -0
  124. package/src/chain/seenCache/index.ts +1 -1
  125. package/src/chain/seenCache/seenGossipBlockInput.ts +2 -2
  126. package/src/chain/seenCache/seenPayloadEnvelopeInput.ts +106 -0
  127. package/src/chain/validation/executionPayloadEnvelope.ts +38 -25
  128. package/src/chain/validation/lightClientFinalityUpdate.ts +1 -1
  129. package/src/chain/validation/lightClientOptimisticUpdate.ts +1 -1
  130. package/src/chain/validatorMonitor.ts +11 -1
  131. package/src/execution/engine/interface.ts +2 -2
  132. package/src/metrics/metrics/lodestar.ts +77 -0
  133. package/src/network/network.ts +2 -2
  134. package/src/network/processor/extractSlotRootFns.ts +18 -5
  135. package/src/network/processor/gossipHandlers.ts +37 -1
  136. package/src/network/reqresp/ReqRespBeaconNode.ts +1 -1
  137. package/src/sync/backfill/backfill.ts +1 -1
  138. package/src/sync/constants.ts +1 -1
  139. package/src/util/sszBytes.ts +90 -10
  140. package/lib/chain/seenCache/seenExecutionPayloadEnvelope.d.ts +0 -15
  141. package/lib/chain/seenCache/seenExecutionPayloadEnvelope.d.ts.map +0 -1
  142. package/lib/chain/seenCache/seenExecutionPayloadEnvelope.js +0 -28
  143. package/lib/chain/seenCache/seenExecutionPayloadEnvelope.js.map +0 -1
  144. package/src/chain/seenCache/seenExecutionPayloadEnvelope.ts +0 -34
@@ -0,0 +1,336 @@
1
+ import {NUMBER_OF_COLUMNS} from "@lodestar/params";
2
+ import {ColumnIndex, DataColumnSidecars, RootHex, Slot, ValidatorIndex, deneb, gloas} from "@lodestar/types";
3
+ import {toRootHex, withTimeout} from "@lodestar/utils";
4
+ import {VersionedHashes} from "../../../execution/index.js";
5
+ import {kzgCommitmentToVersionedHash} from "../../../util/blobs.js";
6
+ import {AddPayloadEnvelopeProps, ColumnWithSource, CreateFromBlockProps, SourceMeta} from "./types.js";
7
+
8
+ export type PayloadEnvelopeInputState =
9
+ | {
10
+ hasPayload: false;
11
+ hasAllData: false;
12
+ hasComputedAllData: false;
13
+ }
14
+ | {
15
+ hasPayload: false;
16
+ hasAllData: true;
17
+ hasComputedAllData: boolean;
18
+ }
19
+ | {
20
+ hasPayload: true;
21
+ hasAllData: false;
22
+ hasComputedAllData: false;
23
+ payloadEnvelope: gloas.SignedExecutionPayloadEnvelope;
24
+ payloadEnvelopeSource: SourceMeta;
25
+ }
26
+ | {
27
+ hasPayload: true;
28
+ hasAllData: true;
29
+ hasComputedAllData: boolean;
30
+ payloadEnvelope: gloas.SignedExecutionPayloadEnvelope;
31
+ payloadEnvelopeSource: SourceMeta;
32
+ timeCompleteSec: number;
33
+ };
34
+
35
+ type PromiseParts<T> = {
36
+ promise: Promise<T>;
37
+ resolve: (value: T) => void;
38
+ reject: (e: Error) => void;
39
+ };
40
+
41
+ function createPromise<T>(): PromiseParts<T> {
42
+ let resolve!: (value: T) => void;
43
+ let reject!: (e: Error) => void;
44
+ const promise = new Promise<T>((_resolve, _reject) => {
45
+ resolve = _resolve;
46
+ reject = _reject;
47
+ });
48
+ return {promise, resolve, reject};
49
+ }
50
+
51
+ /**
52
+ * Tracks bid + payload envelope + data columns for a Gloas block.
53
+ *
54
+ * Created during block import from signedExecutionPayloadBid in block body.
55
+ * Always has bid (required for creation).
56
+ *
57
+ * Completion requires: payload envelope + all sampled columns
58
+ */
59
+ export class PayloadEnvelopeInput {
60
+ readonly blockRootHex: RootHex;
61
+ readonly slot: Slot;
62
+ readonly proposerIndex: ValidatorIndex;
63
+ readonly bid: gloas.ExecutionPayloadBid;
64
+ readonly versionedHashes: VersionedHashes;
65
+
66
+ private columnsCache = new Map<ColumnIndex, ColumnWithSource>();
67
+
68
+ private readonly sampledColumns: ColumnIndex[];
69
+ private readonly custodyColumns: ColumnIndex[];
70
+
71
+ private timeCreatedSec: number;
72
+
73
+ private readonly payloadEnvelopeDataPromise: PromiseParts<gloas.SignedExecutionPayloadEnvelope>;
74
+ private readonly columnsDataPromise: PromiseParts<DataColumnSidecars>;
75
+
76
+ state: PayloadEnvelopeInputState;
77
+
78
+ private constructor(props: {
79
+ blockRootHex: RootHex;
80
+ slot: Slot;
81
+ proposerIndex: ValidatorIndex;
82
+ bid: gloas.ExecutionPayloadBid;
83
+ sampledColumns: ColumnIndex[];
84
+ custodyColumns: ColumnIndex[];
85
+ timeCreatedSec: number;
86
+ }) {
87
+ this.blockRootHex = props.blockRootHex;
88
+ this.slot = props.slot;
89
+ this.proposerIndex = props.proposerIndex;
90
+ this.bid = props.bid;
91
+ this.versionedHashes = props.bid.blobKzgCommitments.map(kzgCommitmentToVersionedHash);
92
+ this.sampledColumns = props.sampledColumns;
93
+ this.custodyColumns = props.custodyColumns;
94
+ this.timeCreatedSec = props.timeCreatedSec;
95
+ this.payloadEnvelopeDataPromise = createPromise();
96
+ this.columnsDataPromise = createPromise();
97
+
98
+ const noBlobs = props.bid.blobKzgCommitments.length === 0;
99
+ const noSampledColumns = props.sampledColumns.length === 0;
100
+ const hasAllData = noBlobs || noSampledColumns;
101
+
102
+ if (hasAllData) {
103
+ this.state = {hasPayload: false, hasAllData: true, hasComputedAllData: true};
104
+ this.columnsDataPromise.resolve(this.getSampledColumns());
105
+ } else {
106
+ this.state = {hasPayload: false, hasAllData: false, hasComputedAllData: false};
107
+ }
108
+ }
109
+
110
+ static createFromBlock(props: CreateFromBlockProps): PayloadEnvelopeInput {
111
+ const bid = (props.block.message.body as gloas.BeaconBlockBody).signedExecutionPayloadBid.message;
112
+ return new PayloadEnvelopeInput({
113
+ blockRootHex: props.blockRootHex,
114
+ slot: props.block.message.slot,
115
+ proposerIndex: props.block.message.proposerIndex,
116
+ bid,
117
+ sampledColumns: props.sampledColumns,
118
+ custodyColumns: props.custodyColumns,
119
+ timeCreatedSec: props.timeCreatedSec,
120
+ });
121
+ }
122
+
123
+ getBid(): gloas.ExecutionPayloadBid {
124
+ return this.bid;
125
+ }
126
+
127
+ getBuilderIndex(): ValidatorIndex {
128
+ return this.bid.builderIndex;
129
+ }
130
+
131
+ getBlockHashHex(): RootHex {
132
+ return toRootHex(this.bid.blockHash);
133
+ }
134
+
135
+ getBlobKzgCommitments(): deneb.BlobKzgCommitments {
136
+ return this.bid.blobKzgCommitments;
137
+ }
138
+
139
+ addPayloadEnvelope(props: AddPayloadEnvelopeProps): void {
140
+ if (this.state.hasPayload) {
141
+ throw new Error(`Payload envelope already set for block ${this.blockRootHex}`);
142
+ }
143
+ if (toRootHex(props.envelope.message.beaconBlockRoot) !== this.blockRootHex) {
144
+ throw new Error("Payload envelope beacon_block_root mismatch");
145
+ }
146
+
147
+ const source: SourceMeta = {
148
+ source: props.source,
149
+ seenTimestampSec: props.seenTimestampSec,
150
+ peerIdStr: props.peerIdStr,
151
+ };
152
+
153
+ if (this.state.hasAllData) {
154
+ // Complete state
155
+ this.state = {
156
+ hasPayload: true,
157
+ hasAllData: true,
158
+ hasComputedAllData: this.state.hasComputedAllData,
159
+ payloadEnvelope: props.envelope,
160
+ payloadEnvelopeSource: source,
161
+ timeCompleteSec: props.seenTimestampSec,
162
+ };
163
+ this.payloadEnvelopeDataPromise.resolve(props.envelope);
164
+ } else {
165
+ // Has payload, waiting for columns
166
+ this.state = {
167
+ hasPayload: true,
168
+ hasAllData: false,
169
+ hasComputedAllData: false,
170
+ payloadEnvelope: props.envelope,
171
+ payloadEnvelopeSource: source,
172
+ };
173
+ }
174
+ }
175
+
176
+ addColumn(columnWithSource: ColumnWithSource): void {
177
+ const {columnSidecar, seenTimestampSec} = columnWithSource;
178
+ this.columnsCache.set(columnSidecar.index, columnWithSource);
179
+
180
+ const sampledColumns = this.getSampledColumns();
181
+ const hasAllData =
182
+ // already hasAllData
183
+ this.state.hasAllData ||
184
+ // has all sampled columns
185
+ sampledColumns.length === this.sampledColumns.length ||
186
+ // has enough columns to reconstruct the rest
187
+ this.columnsCache.size >= NUMBER_OF_COLUMNS / 2;
188
+
189
+ const hasComputedAllData =
190
+ // has all sampled columns
191
+ sampledColumns.length === this.sampledColumns.length;
192
+
193
+ if (!hasAllData) {
194
+ return;
195
+ }
196
+
197
+ if (hasComputedAllData) {
198
+ this.columnsDataPromise.resolve(sampledColumns);
199
+ }
200
+
201
+ if (this.state.hasPayload) {
202
+ // Complete state
203
+ this.state = {
204
+ hasPayload: true,
205
+ hasAllData: true,
206
+ hasComputedAllData: hasComputedAllData || this.state.hasComputedAllData,
207
+ payloadEnvelope: this.state.payloadEnvelope,
208
+ payloadEnvelopeSource: this.state.payloadEnvelopeSource,
209
+ timeCompleteSec: seenTimestampSec,
210
+ };
211
+ this.payloadEnvelopeDataPromise.resolve(this.state.payloadEnvelope);
212
+ } else {
213
+ // No payload yet, all data ready
214
+ this.state = {
215
+ hasPayload: false,
216
+ hasAllData: true,
217
+ hasComputedAllData: hasComputedAllData || this.state.hasComputedAllData,
218
+ };
219
+ }
220
+ }
221
+
222
+ getVersionedHashes(): VersionedHashes {
223
+ return this.versionedHashes;
224
+ }
225
+
226
+ hasPayloadEnvelope(): boolean {
227
+ return this.state.hasPayload;
228
+ }
229
+
230
+ getPayloadEnvelope(): gloas.SignedExecutionPayloadEnvelope {
231
+ if (!this.state.hasPayload) throw new Error("Payload envelope not set");
232
+ return this.state.payloadEnvelope;
233
+ }
234
+
235
+ getPayloadEnvelopeSource(): SourceMeta {
236
+ if (!this.state.hasPayload) throw new Error("Payload envelope source not set");
237
+ return this.state.payloadEnvelopeSource;
238
+ }
239
+
240
+ getSampledColumns(): gloas.DataColumnSidecars {
241
+ const columns: gloas.DataColumnSidecars = [];
242
+ for (const index of this.sampledColumns) {
243
+ const column = this.columnsCache.get(index);
244
+ if (column) {
245
+ columns.push(column.columnSidecar);
246
+ }
247
+ }
248
+ return columns;
249
+ }
250
+
251
+ getSampledColumnsWithSource(): ColumnWithSource[] {
252
+ const columns: ColumnWithSource[] = [];
253
+ for (const index of this.sampledColumns) {
254
+ const column = this.columnsCache.get(index);
255
+ if (column) {
256
+ columns.push(column);
257
+ }
258
+ }
259
+ return columns;
260
+ }
261
+
262
+ getCustodyColumns(): gloas.DataColumnSidecars {
263
+ const columns: gloas.DataColumnSidecars = [];
264
+ for (const index of this.custodyColumns) {
265
+ const column = this.columnsCache.get(index);
266
+ if (column) {
267
+ columns.push(column.columnSidecar);
268
+ }
269
+ }
270
+ return columns;
271
+ }
272
+
273
+ hasComputedAllData(): boolean {
274
+ return this.state.hasComputedAllData;
275
+ }
276
+
277
+ waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<DataColumnSidecars> {
278
+ if (this.state.hasComputedAllData) {
279
+ return Promise.resolve(this.getSampledColumns());
280
+ }
281
+ return withTimeout(() => this.columnsDataPromise.promise, timeout, signal);
282
+ }
283
+
284
+ getTimeCreated(): number {
285
+ return this.timeCreatedSec;
286
+ }
287
+
288
+ getTimeComplete(): number {
289
+ if (!this.state.hasPayload || !this.state.hasAllData) throw new Error("Not yet complete");
290
+ return this.state.timeCompleteSec;
291
+ }
292
+
293
+ isComplete(): boolean {
294
+ return this.state.hasPayload && this.state.hasAllData;
295
+ }
296
+
297
+ async waitForData(): Promise<gloas.SignedExecutionPayloadEnvelope> {
298
+ return this.payloadEnvelopeDataPromise.promise;
299
+ }
300
+
301
+ getSerializedCacheKeys(): object[] {
302
+ const objects: object[] = [];
303
+
304
+ if (this.state.hasPayload) {
305
+ objects.push(this.state.payloadEnvelope);
306
+ }
307
+
308
+ for (const {columnSidecar} of this.columnsCache.values()) {
309
+ objects.push(columnSidecar);
310
+ }
311
+
312
+ return objects;
313
+ }
314
+
315
+ getLogMeta(): {
316
+ slot: number;
317
+ blockRoot: string;
318
+ hasPayload: boolean;
319
+ hasAllData: boolean;
320
+ hasComputedAllData: boolean;
321
+ isComplete: boolean;
322
+ columnsCount: number;
323
+ sampledColumnsCount: number;
324
+ } {
325
+ return {
326
+ slot: this.slot,
327
+ blockRoot: this.blockRootHex,
328
+ hasPayload: this.state.hasPayload,
329
+ hasAllData: this.state.hasAllData,
330
+ hasComputedAllData: this.state.hasComputedAllData,
331
+ isComplete: this.isComplete(),
332
+ columnsCount: this.columnsCache.size,
333
+ sampledColumnsCount: this.sampledColumns.length,
334
+ };
335
+ }
336
+ }
@@ -0,0 +1,33 @@
1
+ import {ForkPostGloas} from "@lodestar/params";
2
+ import {ColumnIndex, RootHex, SignedBeaconBlock, gloas} from "@lodestar/types";
3
+
4
+ export enum PayloadEnvelopeInputSource {
5
+ gossip = "gossip",
6
+ api = "api",
7
+ engine = "engine",
8
+ byRange = "req_resp_by_range",
9
+ byRoot = "req_resp_by_root",
10
+ recovery = "recovery",
11
+ }
12
+
13
+ export type SourceMeta = {
14
+ source: PayloadEnvelopeInputSource;
15
+ seenTimestampSec: number;
16
+ peerIdStr?: string;
17
+ };
18
+
19
+ export type ColumnWithSource = SourceMeta & {
20
+ columnSidecar: gloas.DataColumnSidecar;
21
+ };
22
+
23
+ export type CreateFromBlockProps = {
24
+ blockRootHex: RootHex;
25
+ block: SignedBeaconBlock<ForkPostGloas>;
26
+ sampledColumns: ColumnIndex[];
27
+ custodyColumns: ColumnIndex[];
28
+ timeCreatedSec: number;
29
+ };
30
+
31
+ export type AddPayloadEnvelopeProps = SourceMeta & {
32
+ envelope: gloas.SignedExecutionPayloadEnvelope;
33
+ };
@@ -0,0 +1,61 @@
1
+ import {Metrics} from "../../metrics/metrics.js";
2
+ import {JobItemQueue} from "../../util/queue/index.js";
3
+ import type {BeaconChain} from "../chain.js";
4
+ import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js";
5
+ import {importExecutionPayload} from "./importExecutionPayload.js";
6
+ import {ImportPayloadOpts} from "./types.js";
7
+
8
+ // TODO GLOAS: Set to be equal to DEFAULT_MAX_PENDING_UNFINALIZED_PAYLOAD_ENVELOPE_WRITES for now
9
+ const QUEUE_MAX_LENGTH = 16;
10
+
11
+ enum PayloadEnvelopeImportStatus {
12
+ queued = "queued",
13
+ importing = "importing",
14
+ imported = "imported",
15
+ }
16
+
17
+ /**
18
+ * PayloadEnvelopeProcessor processes payload envelope jobs in a queued fashion, one after the other.
19
+ */
20
+ export class PayloadEnvelopeProcessor {
21
+ readonly jobQueue: JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>;
22
+ private readonly importStatus = new WeakMap<PayloadEnvelopeInput, PayloadEnvelopeImportStatus>();
23
+
24
+ constructor(chain: BeaconChain, metrics: Metrics | null, signal: AbortSignal) {
25
+ this.jobQueue = new JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>(
26
+ (payloadInput, opts) => {
27
+ this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.importing);
28
+ return importExecutionPayload.call(chain, payloadInput, opts);
29
+ },
30
+ {maxLength: QUEUE_MAX_LENGTH, noYieldIfOneItem: true, signal},
31
+ metrics?.payloadEnvelopeProcessorQueue ?? undefined
32
+ );
33
+ }
34
+
35
+ async processPayloadEnvelopeJob(payloadInput: PayloadEnvelopeInput, opts: ImportPayloadOpts = {}): Promise<void> {
36
+ if (!payloadInput.isComplete()) {
37
+ return;
38
+ }
39
+
40
+ if (this.importStatus.get(payloadInput) !== undefined) {
41
+ return;
42
+ }
43
+
44
+ await this.jobQueue.waitForSpace();
45
+
46
+ // Re-check after await, as another call may have queued this payload.
47
+ if (this.importStatus.get(payloadInput) !== undefined) {
48
+ return;
49
+ }
50
+
51
+ this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.queued);
52
+
53
+ try {
54
+ await this.jobQueue.push(payloadInput, opts);
55
+ this.importStatus.set(payloadInput, PayloadEnvelopeImportStatus.imported);
56
+ } catch (e) {
57
+ this.importStatus.delete(payloadInput);
58
+ throw e;
59
+ }
60
+ }
61
+ }
@@ -41,6 +41,14 @@ export enum BlobSidecarValidation {
41
41
  Full,
42
42
  }
43
43
 
44
+ export type ImportPayloadOpts = {
45
+ /**
46
+ * Set to true if envelope signature was already verified (e.g., during gossip/API validation).
47
+ * When false/undefined, signature will be verified during import.
48
+ */
49
+ validSignature?: boolean;
50
+ };
51
+
44
52
  export type ImportBlockOpts = {
45
53
  /**
46
54
  * TEMP: Review if this is safe, Lighthouse always imports attestations even in finalized sync.
@@ -0,0 +1,55 @@
1
+ import {BeaconChain} from "../chain.js";
2
+ import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js";
3
+ import {writeDataColumnsToDb} from "./writeBlockInputToDb.js";
4
+
5
+ /**
6
+ * Persists payload envelope data to DB. This operation must be eventually completed if a payload is imported.
7
+ *
8
+ * TODO GLOAS: Persist envelope metadata (stateRoot, executionRequests, builderIndex, etc.) without the full
9
+ * execution payload body — only keep the blockHash reference. The EL already stores the payload.
10
+ * See https://github.com/ChainSafe/lodestar/issues/5671
11
+ */
12
+ export async function writePayloadEnvelopeInputToDb(
13
+ this: BeaconChain,
14
+ payloadInput: PayloadEnvelopeInput
15
+ ): Promise<void> {
16
+ const envelope = payloadInput.getPayloadEnvelope();
17
+ const blockRootHex = payloadInput.blockRootHex;
18
+
19
+ const envelopeBytes = this.serializedCache.get(envelope);
20
+ const envelopePromise = envelopeBytes
21
+ ? this.db.executionPayloadEnvelope.putBinary(this.db.executionPayloadEnvelope.getId(envelope), envelopeBytes)
22
+ : this.db.executionPayloadEnvelope.add(envelope);
23
+
24
+ // Write envelope and data columns in parallel (reuses shared column writing logic)
25
+ await Promise.all([envelopePromise, writeDataColumnsToDb.call(this, payloadInput)]);
26
+ this.logger.debug("Persisted payload envelope to db", {
27
+ slot: payloadInput.slot,
28
+ root: blockRootHex,
29
+ });
30
+ }
31
+
32
+ export async function persistPayloadEnvelopeInput(
33
+ this: BeaconChain,
34
+ payloadInput: PayloadEnvelopeInput
35
+ ): Promise<void> {
36
+ await writePayloadEnvelopeInputToDb
37
+ .call(this, payloadInput)
38
+ .catch((e) => {
39
+ this.logger.error(
40
+ "Error persisting payload envelope in hot db",
41
+ {
42
+ slot: payloadInput.slot,
43
+ root: payloadInput.blockRootHex,
44
+ },
45
+ e
46
+ );
47
+ })
48
+ .finally(() => {
49
+ this.seenPayloadEnvelopeInputCache.prune(payloadInput.blockRootHex);
50
+ this.logger.debug("Pruned payload envelope input", {
51
+ slot: payloadInput.slot,
52
+ root: payloadInput.blockRootHex,
53
+ });
54
+ });
55
+ }
@@ -84,7 +84,10 @@ import {CheckpointBalancesCache} from "./balancesCache.js";
84
84
  import {BeaconProposerCache} from "./beaconProposerCache.js";
85
85
  import {IBlockInput, isBlockInputBlobs, isBlockInputColumns} from "./blocks/blockInput/index.js";
86
86
  import {BlockProcessor, ImportBlockOpts} from "./blocks/index.js";
87
+ import {PayloadEnvelopeProcessor} from "./blocks/payloadEnvelopeProcessor.js";
88
+ import {ImportPayloadOpts} from "./blocks/types.js";
87
89
  import {persistBlockInput} from "./blocks/writeBlockInputToDb.js";
90
+ import {persistPayloadEnvelopeInput} from "./blocks/writePayloadEnvelopeInputToDb.js";
88
91
  import {BlsMultiThreadWorkerPool, BlsSingleThreadVerifier, IBlsVerifier} from "./bls/index.js";
89
92
  import {ColumnReconstructionTracker} from "./ColumnReconstructionTracker.js";
90
93
  import {ChainEvent, ChainEventEmitter} from "./emitter.js";
@@ -109,13 +112,14 @@ import {BlockAttributes, produceBlockBody, produceCommonBlockBody} from "./produ
109
112
  import {QueuedStateRegenerator, RegenCaller} from "./regen/index.js";
110
113
  import {ReprocessController} from "./reprocess.js";
111
114
  import {
115
+ PayloadEnvelopeInput,
112
116
  SeenAggregators,
113
117
  SeenAttesters,
114
118
  SeenBlockProposers,
115
119
  SeenContributionAndProof,
116
120
  SeenExecutionPayloadBids,
117
- SeenExecutionPayloadEnvelopes,
118
121
  SeenPayloadAttesters,
122
+ SeenPayloadEnvelopeInput,
119
123
  SeenSyncCommitteeMessages,
120
124
  } from "./seenCache/index.js";
121
125
  import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js";
@@ -148,6 +152,13 @@ const DEFAULT_MAX_CACHED_PRODUCED_RESULTS = 4;
148
152
  */
149
153
  const DEFAULT_MAX_PENDING_UNFINALIZED_BLOCK_WRITES = 16;
150
154
 
155
+ /**
156
+ * The maximum number of pending unfinalized payload envelope writes to the database before backpressure is applied.
157
+ * Payload envelope write queue entries hold references to payload inputs (including columns),
158
+ * keeping them in memory. Keep moderate to avoid OOM during sync.
159
+ */
160
+ const DEFAULT_MAX_PENDING_UNFINALIZED_PAYLOAD_ENVELOPE_WRITES = 16;
161
+
151
162
  export class BeaconChain implements IBeaconChain {
152
163
  readonly genesisTime: UintNum64;
153
164
  readonly genesisValidatorsRoot: Root;
@@ -172,6 +183,7 @@ export class BeaconChain implements IBeaconChain {
172
183
  readonly reprocessController: ReprocessController;
173
184
  readonly archiveStore: ArchiveStore;
174
185
  readonly unfinalizedBlockWrites: JobItemQueue<[IBlockInput], void>;
186
+ readonly unfinalizedPayloadEnvelopeWrites: JobItemQueue<[PayloadEnvelopeInput], void>;
175
187
 
176
188
  // Ops pool
177
189
  readonly attestationPool: AttestationPool;
@@ -187,13 +199,13 @@ export class BeaconChain implements IBeaconChain {
187
199
  readonly seenAggregators = new SeenAggregators();
188
200
  readonly seenPayloadAttesters = new SeenPayloadAttesters();
189
201
  readonly seenAggregatedAttestations: SeenAggregatedAttestations;
190
- readonly seenExecutionPayloadEnvelopes = new SeenExecutionPayloadEnvelopes();
191
202
  readonly seenExecutionPayloadBids = new SeenExecutionPayloadBids();
192
203
  readonly seenBlockProposers = new SeenBlockProposers();
193
204
  readonly seenSyncCommitteeMessages = new SeenSyncCommitteeMessages();
194
205
  readonly seenContributionAndProof: SeenContributionAndProof;
195
206
  readonly seenAttestationDatas: SeenAttestationDatas;
196
207
  readonly seenBlockInputCache: SeenBlockInput;
208
+ readonly seenPayloadEnvelopeInputCache: SeenPayloadEnvelopeInput;
197
209
  // Seen cache for liveness checks
198
210
  readonly seenBlockAttesters = new SeenBlockAttesters();
199
211
 
@@ -221,6 +233,7 @@ export class BeaconChain implements IBeaconChain {
221
233
  readonly opts: IChainOptions;
222
234
 
223
235
  protected readonly blockProcessor: BlockProcessor;
236
+ protected readonly payloadEnvelopeProcessor: PayloadEnvelopeProcessor;
224
237
  protected readonly db: IBeaconDb;
225
238
  // this is only available if nHistoricalStates is enabled
226
239
  private readonly cpStateDatastore?: CPStateDatastore;
@@ -334,6 +347,13 @@ export class BeaconChain implements IBeaconChain {
334
347
  metrics,
335
348
  logger,
336
349
  });
350
+ this.seenPayloadEnvelopeInputCache = new SeenPayloadEnvelopeInput({
351
+ chainEvents: emitter,
352
+ signal,
353
+ serializedCache: this.serializedCache,
354
+ metrics,
355
+ logger,
356
+ });
337
357
 
338
358
  this._earliestAvailableSlot = anchorState.slot;
339
359
 
@@ -411,6 +431,7 @@ export class BeaconChain implements IBeaconChain {
411
431
  this.reprocessController = new ReprocessController(this.metrics);
412
432
 
413
433
  this.blockProcessor = new BlockProcessor(this, metrics, opts, signal);
434
+ this.payloadEnvelopeProcessor = new PayloadEnvelopeProcessor(this, metrics, signal);
414
435
 
415
436
  this.forkChoice = forkChoice;
416
437
  this.clock = clock;
@@ -447,6 +468,15 @@ export class BeaconChain implements IBeaconChain {
447
468
  metrics?.unfinalizedBlockWritesQueue
448
469
  );
449
470
 
471
+ this.unfinalizedPayloadEnvelopeWrites = new JobItemQueue(
472
+ persistPayloadEnvelopeInput.bind(this),
473
+ {
474
+ maxLength: DEFAULT_MAX_PENDING_UNFINALIZED_PAYLOAD_ENVELOPE_WRITES,
475
+ signal,
476
+ },
477
+ metrics?.unfinalizedPayloadEnvelopeWritesQueue
478
+ );
479
+
450
480
  // always run PrepareNextSlotScheduler except for fork_choice spec tests
451
481
  if (!opts?.disablePrepareNextSlot) {
452
482
  new PrepareNextSlotScheduler(this, this.config, metrics, this.logger, signal);
@@ -477,6 +507,7 @@ export class BeaconChain implements IBeaconChain {
477
507
  // we can abort any ongoing unfinalized block writes.
478
508
  // TODO: persist fork choice to disk and allow unfinalized block writes to complete.
479
509
  this.unfinalizedBlockWrites.dropAllJobs();
510
+ this.unfinalizedPayloadEnvelopeWrites.dropAllJobs();
480
511
 
481
512
  this.abortController.abort();
482
513
  }
@@ -1010,6 +1041,10 @@ export class BeaconChain implements IBeaconChain {
1010
1041
  return this.blockProcessor.processBlocksJob(blocks, opts);
1011
1042
  }
1012
1043
 
1044
+ async processExecutionPayload(payloadInput: PayloadEnvelopeInput, opts?: ImportPayloadOpts): Promise<void> {
1045
+ return this.payloadEnvelopeProcessor.processPayloadEnvelopeJob(payloadInput, opts);
1046
+ }
1047
+
1013
1048
  getStatus(): Status {
1014
1049
  const head = this.forkChoice.getHead();
1015
1050
  const finalizedCheckpoint = this.forkChoice.getFinalizedCheckpoint();
@@ -1402,7 +1437,6 @@ export class BeaconChain implements IBeaconChain {
1402
1437
  this.logger.verbose("Fork choice finalized", {epoch: cp.epoch, root: cp.rootHex});
1403
1438
  const finalizedSlot = computeStartSlotAtEpoch(cp.epoch);
1404
1439
  this.seenBlockProposers.prune(finalizedSlot);
1405
- this.seenExecutionPayloadEnvelopes.prune(finalizedSlot);
1406
1440
 
1407
1441
  // Update validator custody to account for effective balance changes
1408
1442
  await this.updateValidatorsCustodyRequirement(cp);
@@ -4,17 +4,21 @@ import {GossipActionError} from "./gossipValidation.js";
4
4
  export enum ExecutionPayloadEnvelopeErrorCode {
5
5
  BELONG_TO_FINALIZED_BLOCK = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BELONG_TO_FINALIZED_BLOCK",
6
6
  BLOCK_ROOT_UNKNOWN = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BLOCK_ROOT_UNKNOWN",
7
+ PARENT_UNKNOWN = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_PARENT_UNKNOWN",
8
+ UNKNOWN_BLOCK_STATE = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_UNKNOWN_BLOCK_STATE",
7
9
  ENVELOPE_ALREADY_KNOWN = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_ALREADY_KNOWN",
8
10
  INVALID_BLOCK = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_INVALID_BLOCK",
9
11
  SLOT_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_SLOT_MISMATCH",
10
12
  BUILDER_INDEX_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BUILDER_INDEX_MISMATCH",
11
13
  BLOCK_HASH_MISMATCH = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_BLOCK_HASH_MISMATCH",
12
14
  INVALID_SIGNATURE = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_INVALID_SIGNATURE",
13
- CACHE_FAIL = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_CACHE_FAIL",
15
+ PAYLOAD_ENVELOPE_INPUT_MISSING = "EXECUTION_PAYLOAD_ENVELOPE_ERROR_PAYLOAD_ENVELOPE_INPUT_MISSING",
14
16
  }
15
17
  export type ExecutionPayloadEnvelopeErrorType =
16
18
  | {code: ExecutionPayloadEnvelopeErrorCode.BELONG_TO_FINALIZED_BLOCK; envelopeSlot: Slot; finalizedSlot: Slot}
17
19
  | {code: ExecutionPayloadEnvelopeErrorCode.BLOCK_ROOT_UNKNOWN; blockRoot: RootHex}
20
+ | {code: ExecutionPayloadEnvelopeErrorCode.PARENT_UNKNOWN; parentRoot: RootHex; slot: Slot}
21
+ | {code: ExecutionPayloadEnvelopeErrorCode.UNKNOWN_BLOCK_STATE; blockRoot: RootHex; slot: Slot}
18
22
  | {
19
23
  code: ExecutionPayloadEnvelopeErrorCode.ENVELOPE_ALREADY_KNOWN;
20
24
  blockRoot: RootHex;
@@ -33,6 +37,6 @@ export type ExecutionPayloadEnvelopeErrorType =
33
37
  bidBlockHash: RootHex | null;
34
38
  }
35
39
  | {code: ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE}
36
- | {code: ExecutionPayloadEnvelopeErrorCode.CACHE_FAIL; blockRoot: RootHex};
40
+ | {code: ExecutionPayloadEnvelopeErrorCode.PAYLOAD_ENVELOPE_INPUT_MISSING; blockRoot: RootHex};
37
41
 
38
42
  export class ExecutionPayloadEnvelopeError extends GossipActionError<ExecutionPayloadEnvelopeErrorType> {}
@@ -158,10 +158,6 @@ export function initializeForkChoiceFromFinalizedState(
158
158
 
159
159
  dataAvailabilityStatus: DataAvailabilityStatus.PreData,
160
160
  payloadStatus: isForkPostGloas ? PayloadStatus.PENDING : PayloadStatus.FULL, // TODO GLOAS: Post-gloas how do we know if the checkpoint payload is FULL or EMPTY?
161
- builderIndex: isForkPostGloas ? (state as CachedBeaconStateGloas).latestExecutionPayloadBid.builderIndex : null,
162
- blockHashFromBid: isForkPostGloas
163
- ? toRootHex((state as CachedBeaconStateGloas).latestExecutionPayloadBid.blockHash)
164
- : null,
165
161
  parentBlockHash: isForkPostGloas ? toRootHex((state as CachedBeaconStateGloas).latestBlockHash) : null,
166
162
  },
167
163
  currentSlot
@@ -255,12 +251,6 @@ export function initializeForkChoiceFromUnfinalizedState(
255
251
 
256
252
  dataAvailabilityStatus: DataAvailabilityStatus.PreData,
257
253
  payloadStatus: isForkPostGloas ? PayloadStatus.PENDING : PayloadStatus.FULL, // TODO GLOAS: Post-gloas how do we know if the checkpoint payload is FULL or EMPTY?
258
- builderIndex: isForkPostGloas
259
- ? (unfinalizedState as CachedBeaconStateGloas).latestExecutionPayloadBid.builderIndex
260
- : null,
261
- blockHashFromBid: isForkPostGloas
262
- ? toRootHex((unfinalizedState as CachedBeaconStateGloas).latestExecutionPayloadBid.blockHash)
263
- : null,
264
254
  parentBlockHash: isForkPostGloas ? toRootHex((unfinalizedState as CachedBeaconStateGloas).latestBlockHash) : null,
265
255
  };
266
256