@lodestar/beacon-node 1.42.0-dev.84f3b9f030 → 1.42.0-dev.89ddaaeed2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +11 -11
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/api/impl/debug/index.js.map +1 -1
  5. package/lib/chain/ColumnReconstructionTracker.d.ts +2 -1
  6. package/lib/chain/ColumnReconstructionTracker.d.ts.map +1 -1
  7. package/lib/chain/ColumnReconstructionTracker.js +5 -5
  8. package/lib/chain/ColumnReconstructionTracker.js.map +1 -1
  9. package/lib/chain/GetBlobsTracker.d.ts +2 -1
  10. package/lib/chain/GetBlobsTracker.d.ts.map +1 -1
  11. package/lib/chain/GetBlobsTracker.js +14 -12
  12. package/lib/chain/GetBlobsTracker.js.map +1 -1
  13. package/lib/chain/blocks/blockInput/blockInput.d.ts +5 -5
  14. package/lib/chain/blocks/blockInput/blockInput.d.ts.map +1 -1
  15. package/lib/chain/blocks/blockInput/blockInput.js.map +1 -1
  16. package/lib/chain/blocks/blockInput/types.d.ts +4 -4
  17. package/lib/chain/blocks/blockInput/types.d.ts.map +1 -1
  18. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  19. package/lib/chain/blocks/importBlock.js +13 -1
  20. package/lib/chain/blocks/importBlock.js.map +1 -1
  21. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts +14 -6
  22. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts.map +1 -1
  23. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js +33 -2
  24. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js.map +1 -1
  25. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts +2 -1
  26. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts.map +1 -1
  27. package/lib/chain/chain.d.ts +3 -2
  28. package/lib/chain/chain.d.ts.map +1 -1
  29. package/lib/chain/chain.js +55 -20
  30. package/lib/chain/chain.js.map +1 -1
  31. package/lib/chain/emitter.d.ts +29 -7
  32. package/lib/chain/emitter.d.ts.map +1 -1
  33. package/lib/chain/emitter.js +12 -3
  34. package/lib/chain/emitter.js.map +1 -1
  35. package/lib/chain/errors/dataColumnSidecarError.d.ts +31 -1
  36. package/lib/chain/errors/dataColumnSidecarError.d.ts.map +1 -1
  37. package/lib/chain/errors/dataColumnSidecarError.js +7 -0
  38. package/lib/chain/errors/dataColumnSidecarError.js.map +1 -1
  39. package/lib/chain/interface.d.ts +4 -2
  40. package/lib/chain/interface.d.ts.map +1 -1
  41. package/lib/chain/seenCache/seenGossipBlockInput.d.ts +1 -1
  42. package/lib/chain/seenCache/seenGossipBlockInput.d.ts.map +1 -1
  43. package/lib/chain/seenCache/seenGossipBlockInput.js +2 -2
  44. package/lib/chain/seenCache/seenGossipBlockInput.js.map +1 -1
  45. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts +1 -1
  46. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts.map +1 -1
  47. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js +2 -2
  48. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js.map +1 -1
  49. package/lib/chain/validation/dataColumnSidecar.d.ts +11 -4
  50. package/lib/chain/validation/dataColumnSidecar.d.ts.map +1 -1
  51. package/lib/chain/validation/dataColumnSidecar.js +184 -5
  52. package/lib/chain/validation/dataColumnSidecar.js.map +1 -1
  53. package/lib/db/buckets.d.ts +2 -2
  54. package/lib/db/buckets.d.ts.map +1 -1
  55. package/lib/db/buckets.js +2 -2
  56. package/lib/db/buckets.js.map +1 -1
  57. package/lib/db/repositories/blockArchiveIndex.d.ts +2 -2
  58. package/lib/db/repositories/blockArchiveIndex.d.ts.map +1 -1
  59. package/lib/db/repositories/dataColumnSidecar.d.ts.map +1 -1
  60. package/lib/db/repositories/dataColumnSidecar.js +4 -2
  61. package/lib/db/repositories/dataColumnSidecar.js.map +1 -1
  62. package/lib/db/repositories/dataColumnSidecarArchive.d.ts.map +1 -1
  63. package/lib/db/repositories/dataColumnSidecarArchive.js +4 -2
  64. package/lib/db/repositories/dataColumnSidecarArchive.js.map +1 -1
  65. package/lib/metrics/metrics/lodestar.d.ts +20 -0
  66. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  67. package/lib/metrics/metrics/lodestar.js +33 -0
  68. package/lib/metrics/metrics/lodestar.js.map +1 -1
  69. package/lib/network/interface.d.ts +3 -2
  70. package/lib/network/interface.d.ts.map +1 -1
  71. package/lib/network/libp2p/index.d.ts.map +1 -1
  72. package/lib/network/libp2p/index.js +19 -13
  73. package/lib/network/libp2p/index.js.map +1 -1
  74. package/lib/network/network.d.ts +3 -2
  75. package/lib/network/network.d.ts.map +1 -1
  76. package/lib/network/network.js +3 -0
  77. package/lib/network/network.js.map +1 -1
  78. package/lib/network/options.d.ts.map +1 -1
  79. package/lib/network/options.js +7 -2
  80. package/lib/network/options.js.map +1 -1
  81. package/lib/network/processor/extractSlotRootFns.d.ts +1 -1
  82. package/lib/network/processor/extractSlotRootFns.d.ts.map +1 -1
  83. package/lib/network/processor/extractSlotRootFns.js +25 -5
  84. package/lib/network/processor/extractSlotRootFns.js.map +1 -1
  85. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  86. package/lib/network/processor/gossipHandlers.js +242 -66
  87. package/lib/network/processor/gossipHandlers.js.map +1 -1
  88. package/lib/network/processor/index.d.ts +11 -1
  89. package/lib/network/processor/index.d.ts.map +1 -1
  90. package/lib/network/processor/index.js +232 -22
  91. package/lib/network/processor/index.js.map +1 -1
  92. package/lib/network/reqresp/types.d.ts +3 -3
  93. package/lib/network/reqresp/types.d.ts.map +1 -1
  94. package/lib/network/reqresp/types.js +9 -3
  95. package/lib/network/reqresp/types.js.map +1 -1
  96. package/lib/sync/unknownBlock.js +2 -2
  97. package/lib/sync/unknownBlock.js.map +1 -1
  98. package/lib/sync/utils/downloadByRange.d.ts +3 -3
  99. package/lib/sync/utils/downloadByRange.d.ts.map +1 -1
  100. package/lib/sync/utils/downloadByRange.js +4 -2
  101. package/lib/sync/utils/downloadByRange.js.map +1 -1
  102. package/lib/sync/utils/downloadByRoot.d.ts +3 -3
  103. package/lib/sync/utils/downloadByRoot.d.ts.map +1 -1
  104. package/lib/sync/utils/downloadByRoot.js +10 -5
  105. package/lib/sync/utils/downloadByRoot.js.map +1 -1
  106. package/lib/util/blobs.d.ts +3 -3
  107. package/lib/util/blobs.d.ts.map +1 -1
  108. package/lib/util/blobs.js +21 -10
  109. package/lib/util/blobs.js.map +1 -1
  110. package/lib/util/dataColumns.d.ts +18 -11
  111. package/lib/util/dataColumns.d.ts.map +1 -1
  112. package/lib/util/dataColumns.js +51 -17
  113. package/lib/util/dataColumns.js.map +1 -1
  114. package/lib/util/execution.d.ts +6 -2
  115. package/lib/util/execution.d.ts.map +1 -1
  116. package/lib/util/execution.js +49 -25
  117. package/lib/util/execution.js.map +1 -1
  118. package/lib/util/sszBytes.d.ts +25 -1
  119. package/lib/util/sszBytes.d.ts.map +1 -1
  120. package/lib/util/sszBytes.js +189 -2
  121. package/lib/util/sszBytes.js.map +1 -1
  122. package/package.json +15 -15
  123. package/src/api/impl/beacon/blocks/index.ts +17 -14
  124. package/src/api/impl/debug/index.ts +2 -2
  125. package/src/chain/ColumnReconstructionTracker.ts +6 -5
  126. package/src/chain/GetBlobsTracker.ts +14 -12
  127. package/src/chain/blocks/blockInput/blockInput.ts +8 -8
  128. package/src/chain/blocks/blockInput/types.ts +4 -4
  129. package/src/chain/blocks/importBlock.ts +18 -1
  130. package/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts +53 -12
  131. package/src/chain/blocks/payloadEnvelopeInput/types.ts +2 -1
  132. package/src/chain/chain.ts +63 -24
  133. package/src/chain/emitter.ts +25 -7
  134. package/src/chain/errors/dataColumnSidecarError.ts +32 -1
  135. package/src/chain/interface.ts +4 -2
  136. package/src/chain/seenCache/seenGossipBlockInput.ts +2 -2
  137. package/src/chain/seenCache/seenPayloadEnvelopeInput.ts +2 -2
  138. package/src/chain/validation/dataColumnSidecar.ts +230 -7
  139. package/src/db/buckets.ts +2 -2
  140. package/src/db/repositories/dataColumnSidecar.ts +4 -2
  141. package/src/db/repositories/dataColumnSidecarArchive.ts +4 -2
  142. package/src/metrics/metrics/lodestar.ts +34 -0
  143. package/src/network/interface.ts +3 -2
  144. package/src/network/libp2p/index.ts +21 -15
  145. package/src/network/network.ts +7 -4
  146. package/src/network/options.ts +7 -2
  147. package/src/network/processor/extractSlotRootFns.ts +32 -6
  148. package/src/network/processor/gossipHandlers.ts +305 -79
  149. package/src/network/processor/index.ts +302 -22
  150. package/src/network/reqresp/types.ts +13 -5
  151. package/src/sync/unknownBlock.ts +3 -3
  152. package/src/sync/utils/downloadByRange.ts +9 -7
  153. package/src/sync/utils/downloadByRoot.ts +16 -12
  154. package/src/util/blobs.ts +35 -15
  155. package/src/util/dataColumns.ts +69 -25
  156. package/src/util/execution.ts +49 -30
  157. package/src/util/sszBytes.ts +245 -3
@@ -617,7 +617,7 @@ type BlockInputColumnsState =
617
617
  * - The block is not yet seen and all required sampled columns are seen
618
618
  * - The block is not yet seen and all required sampled columns are not yet seen
619
619
  */
620
- export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.DataColumnSidecars> {
620
+ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.DataColumnSidecar[]> {
621
621
  type = DAType.Columns as const;
622
622
 
623
623
  state: BlockInputColumnsState;
@@ -630,7 +630,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
630
630
  *
631
631
  * This is different from `dataPromise` which resolves when all data is available or could become available (e.g. through reconstruction)
632
632
  */
633
- protected computedDataPromise = createPromise<fulu.DataColumnSidecars>();
633
+ protected computedDataPromise = createPromise<fulu.DataColumnSidecar[]>();
634
634
 
635
635
  private constructor(
636
636
  init: BlockInputInit,
@@ -854,8 +854,8 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
854
854
  return this.state.versionedHashes;
855
855
  }
856
856
 
857
- getCustodyColumns(): fulu.DataColumnSidecars {
858
- const columns: fulu.DataColumnSidecars = [];
857
+ getCustodyColumns(): fulu.DataColumnSidecar[] {
858
+ const columns: fulu.DataColumnSidecar[] = [];
859
859
  for (const index of this.custodyColumns) {
860
860
  const column = this.columnsCache.get(index);
861
861
  if (column) {
@@ -876,8 +876,8 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
876
876
  return columns;
877
877
  }
878
878
 
879
- getSampledColumns(): fulu.DataColumnSidecars {
880
- const columns: fulu.DataColumnSidecars = [];
879
+ getSampledColumns(): fulu.DataColumnSidecar[] {
880
+ const columns: fulu.DataColumnSidecar[] = [];
881
881
  for (const index of this.sampledColumns) {
882
882
  const column = this.columnsCache.get(index);
883
883
  if (column) {
@@ -891,7 +891,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
891
891
  return [...this.columnsCache.values()];
892
892
  }
893
893
 
894
- getAllColumns(): fulu.DataColumnSidecars {
894
+ getAllColumns(): fulu.DataColumnSidecar[] {
895
895
  return this.getAllColumnsWithSource().map(({columnSidecar}) => columnSidecar);
896
896
  }
897
897
 
@@ -919,7 +919,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
919
919
  return this.state.hasComputedAllData;
920
920
  }
921
921
 
922
- waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<fulu.DataColumnSidecars> {
922
+ waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<fulu.DataColumnSidecar[]> {
923
923
  if (!this.state.hasComputedAllData) {
924
924
  return withTimeout(() => this.computedDataPromise.promise, timeout, signal);
925
925
  }
@@ -1,5 +1,5 @@
1
1
  import {ForkName} from "@lodestar/params";
2
- import {ColumnIndex, DataColumnSidecars, RootHex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types";
2
+ import {ColumnIndex, DataColumnSidecar, RootHex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types";
3
3
  import {VersionedHashes} from "../../../execution/index.js";
4
4
 
5
5
  export enum DAType {
@@ -9,7 +9,7 @@ export enum DAType {
9
9
  NoData = "no-data",
10
10
  }
11
11
 
12
- export type DAData = null | deneb.BlobSidecars | fulu.DataColumnSidecars;
12
+ export type DAData = null | deneb.BlobSidecars | fulu.DataColumnSidecar[];
13
13
 
14
14
  /**
15
15
  * Represents were input originated. Blocks and Data can come from different
@@ -108,9 +108,9 @@ export type MissingColumnMeta = {
108
108
  export interface IDataColumnsInput {
109
109
  readonly slot: Slot;
110
110
  readonly blockRootHex: string;
111
- getCustodyColumns(): DataColumnSidecars;
111
+ getCustodyColumns(): DataColumnSidecar[];
112
112
  hasComputedAllData(): boolean;
113
- waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<DataColumnSidecars>;
113
+ waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<DataColumnSidecar[]>;
114
114
  }
115
115
 
116
116
  /**
@@ -140,9 +140,10 @@ export async function importBlock(
140
140
 
141
141
  // For Gloas blocks, create PayloadEnvelopeInput so it's available for later payload import
142
142
  if (fork >= ForkSeq.gloas) {
143
- this.seenPayloadEnvelopeInputCache.add({
143
+ const payloadInput = this.seenPayloadEnvelopeInputCache.add({
144
144
  blockRootHex,
145
145
  block: block as SignedBeaconBlock<ForkPostGloas>,
146
+ forkName: blockInput.forkName,
146
147
  sampledColumns: this.custodyConfig.sampledColumns,
147
148
  custodyColumns: this.custodyConfig.custodyColumns,
148
149
  timeCreatedSec: fullyVerifiedBlock.seenTimestampSec,
@@ -153,6 +154,22 @@ export async function importBlock(
153
154
  source: source.source,
154
155
  ...(opts.seenTimestampSec !== undefined ? {recvToImport: Date.now() / 1000 - opts.seenTimestampSec} : {}),
155
156
  });
157
+
158
+ // Immediately attempt fetch of data columns from execution engine as the bid contains kzg commitments
159
+ // which is all the information we need so there is no reason to delay until execution payload arrives
160
+ // TODO GLOAS: If we want EL retries after this initial attempt, add an explicit retry policy here
161
+ // (for example later in the slot). Do not couple retries to incoming gossip columns.
162
+ this.getBlobsTracker.triggerGetBlobs(payloadInput, () => {
163
+ // TODO GLOAS: come up with a better mechanism to trigger processExecutionPayload after data becomes available,
164
+ // similar to how pre-gloas uses waitForBlockAndAllData with a cutoff timeout and incompleteBlockInput event
165
+ this.processExecutionPayload(payloadInput, {validSignature: true}).catch((e) => {
166
+ this.logger.debug(
167
+ "Error processing execution payload after getBlobs",
168
+ {slot: blockSlot, root: blockRootHex},
169
+ e as Error
170
+ );
171
+ });
172
+ });
156
173
  }
157
174
 
158
175
  this.metrics?.importBlock.bySource.inc({source: source.source});
@@ -1,8 +1,9 @@
1
- import {NUMBER_OF_COLUMNS} from "@lodestar/params";
2
- import {ColumnIndex, DataColumnSidecars, RootHex, Slot, ValidatorIndex, deneb, gloas} from "@lodestar/types";
1
+ import {ForkName, NUMBER_OF_COLUMNS} from "@lodestar/params";
2
+ import {ColumnIndex, RootHex, Slot, ValidatorIndex, deneb, gloas} from "@lodestar/types";
3
3
  import {toRootHex, withTimeout} from "@lodestar/utils";
4
4
  import {VersionedHashes} from "../../../execution/index.js";
5
5
  import {kzgCommitmentToVersionedHash} from "../../../util/blobs.js";
6
+ import {MissingColumnMeta} from "../blockInput/types.js";
6
7
  import {AddPayloadEnvelopeProps, ColumnWithSource, CreateFromBlockProps, SourceMeta} from "./types.js";
7
8
 
8
9
  export type PayloadEnvelopeInputState =
@@ -59,6 +60,7 @@ function createPromise<T>(): PromiseParts<T> {
59
60
  export class PayloadEnvelopeInput {
60
61
  readonly blockRootHex: RootHex;
61
62
  readonly slot: Slot;
63
+ readonly forkName: ForkName;
62
64
  readonly proposerIndex: ValidatorIndex;
63
65
  readonly bid: gloas.ExecutionPayloadBid;
64
66
  readonly versionedHashes: VersionedHashes;
@@ -71,13 +73,14 @@ export class PayloadEnvelopeInput {
71
73
  private timeCreatedSec: number;
72
74
 
73
75
  private readonly payloadEnvelopeDataPromise: PromiseParts<gloas.SignedExecutionPayloadEnvelope>;
74
- private readonly columnsDataPromise: PromiseParts<DataColumnSidecars>;
76
+ private readonly columnsDataPromise: PromiseParts<gloas.DataColumnSidecar[]>;
75
77
 
76
78
  state: PayloadEnvelopeInputState;
77
79
 
78
80
  private constructor(props: {
79
81
  blockRootHex: RootHex;
80
82
  slot: Slot;
83
+ forkName: ForkName;
81
84
  proposerIndex: ValidatorIndex;
82
85
  bid: gloas.ExecutionPayloadBid;
83
86
  sampledColumns: ColumnIndex[];
@@ -86,6 +89,7 @@ export class PayloadEnvelopeInput {
86
89
  }) {
87
90
  this.blockRootHex = props.blockRootHex;
88
91
  this.slot = props.slot;
92
+ this.forkName = props.forkName;
89
93
  this.proposerIndex = props.proposerIndex;
90
94
  this.bid = props.bid;
91
95
  this.versionedHashes = props.bid.blobKzgCommitments.map(kzgCommitmentToVersionedHash);
@@ -112,6 +116,7 @@ export class PayloadEnvelopeInput {
112
116
  return new PayloadEnvelopeInput({
113
117
  blockRootHex: props.blockRootHex,
114
118
  slot: props.block.message.slot,
119
+ forkName: props.forkName,
115
120
  proposerIndex: props.block.message.proposerIndex,
116
121
  bid,
117
122
  sampledColumns: props.sampledColumns,
@@ -173,8 +178,12 @@ export class PayloadEnvelopeInput {
173
178
  }
174
179
  }
175
180
 
176
- addColumn(columnWithSource: ColumnWithSource): void {
181
+ addColumn(columnWithSource: ColumnWithSource): boolean {
177
182
  const {columnSidecar, seenTimestampSec} = columnWithSource;
183
+ if (this.columnsCache.has(columnSidecar.index)) {
184
+ return false;
185
+ }
186
+
178
187
  this.columnsCache.set(columnSidecar.index, columnWithSource);
179
188
 
180
189
  const sampledColumns = this.getSampledColumns();
@@ -191,7 +200,7 @@ export class PayloadEnvelopeInput {
191
200
  sampledColumns.length === this.sampledColumns.length;
192
201
 
193
202
  if (!hasAllData) {
194
- return;
203
+ return true;
195
204
  }
196
205
 
197
206
  if (hasComputedAllData) {
@@ -217,6 +226,20 @@ export class PayloadEnvelopeInput {
217
226
  hasComputedAllData: hasComputedAllData || this.state.hasComputedAllData,
218
227
  };
219
228
  }
229
+
230
+ return true;
231
+ }
232
+
233
+ hasColumn(index: ColumnIndex): boolean {
234
+ return this.columnsCache.has(index);
235
+ }
236
+
237
+ getColumn(index: ColumnIndex): gloas.DataColumnSidecar | undefined {
238
+ return this.columnsCache.get(index)?.columnSidecar;
239
+ }
240
+
241
+ getAllColumns(): gloas.DataColumnSidecar[] {
242
+ return [...this.columnsCache.values()].map(({columnSidecar}) => columnSidecar);
220
243
  }
221
244
 
222
245
  getVersionedHashes(): VersionedHashes {
@@ -237,8 +260,8 @@ export class PayloadEnvelopeInput {
237
260
  return this.state.payloadEnvelopeSource;
238
261
  }
239
262
 
240
- getSampledColumns(): gloas.DataColumnSidecars {
241
- const columns: gloas.DataColumnSidecars = [];
263
+ getSampledColumns(): gloas.DataColumnSidecar[] {
264
+ const columns: gloas.DataColumnSidecar[] = [];
242
265
  for (const index of this.sampledColumns) {
243
266
  const column = this.columnsCache.get(index);
244
267
  if (column) {
@@ -259,8 +282,8 @@ export class PayloadEnvelopeInput {
259
282
  return columns;
260
283
  }
261
284
 
262
- getCustodyColumns(): gloas.DataColumnSidecars {
263
- const columns: gloas.DataColumnSidecars = [];
285
+ getCustodyColumns(): gloas.DataColumnSidecar[] {
286
+ const columns: gloas.DataColumnSidecar[] = [];
264
287
  for (const index of this.custodyColumns) {
265
288
  const column = this.columnsCache.get(index);
266
289
  if (column) {
@@ -270,11 +293,29 @@ export class PayloadEnvelopeInput {
270
293
  return columns;
271
294
  }
272
295
 
296
+ hasAllData(): boolean {
297
+ return this.state.hasAllData;
298
+ }
299
+
300
+ getMissingSampledColumnMeta(): MissingColumnMeta {
301
+ if (this.state.hasAllData) {
302
+ return {missing: [], versionedHashes: this.versionedHashes};
303
+ }
304
+
305
+ const missing: ColumnIndex[] = [];
306
+ for (const index of this.sampledColumns) {
307
+ if (!this.columnsCache.has(index)) {
308
+ missing.push(index);
309
+ }
310
+ }
311
+ return {missing, versionedHashes: this.versionedHashes};
312
+ }
313
+
273
314
  hasComputedAllData(): boolean {
274
315
  return this.state.hasComputedAllData;
275
316
  }
276
317
 
277
- waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<DataColumnSidecars> {
318
+ waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<gloas.DataColumnSidecar[]> {
278
319
  if (this.state.hasComputedAllData) {
279
320
  return Promise.resolve(this.getSampledColumns());
280
321
  }
@@ -319,7 +360,7 @@ export class PayloadEnvelopeInput {
319
360
  hasAllData: boolean;
320
361
  hasComputedAllData: boolean;
321
362
  isComplete: boolean;
322
- columnsCount: number;
363
+ receivedColumns: number;
323
364
  sampledColumnsCount: number;
324
365
  } {
325
366
  return {
@@ -329,7 +370,7 @@ export class PayloadEnvelopeInput {
329
370
  hasAllData: this.state.hasAllData,
330
371
  hasComputedAllData: this.state.hasComputedAllData,
331
372
  isComplete: this.isComplete(),
332
- columnsCount: this.columnsCache.size,
373
+ receivedColumns: this.columnsCache.size,
333
374
  sampledColumnsCount: this.sampledColumns.length,
334
375
  };
335
376
  }
@@ -1,4 +1,4 @@
1
- import {ForkPostGloas} from "@lodestar/params";
1
+ import {ForkName, ForkPostGloas} from "@lodestar/params";
2
2
  import {ColumnIndex, RootHex, SignedBeaconBlock, gloas} from "@lodestar/types";
3
3
 
4
4
  export enum PayloadEnvelopeInputSource {
@@ -23,6 +23,7 @@ export type ColumnWithSource = SourceMeta & {
23
23
  export type CreateFromBlockProps = {
24
24
  blockRootHex: RootHex;
25
25
  block: SignedBeaconBlock<ForkPostGloas>;
26
+ forkName: ForkName;
26
27
  sampledColumns: ColumnIndex[];
27
28
  custodyColumns: ColumnIndex[];
28
29
  timeCreatedSec: number;
@@ -15,6 +15,7 @@ import {
15
15
  BUILDER_INDEX_SELF_BUILD,
16
16
  EFFECTIVE_BALANCE_INCREMENT,
17
17
  type ForkPostFulu,
18
+ type ForkPostGloas,
18
19
  GENESIS_SLOT,
19
20
  SLOTS_PER_EPOCH,
20
21
  isForkPostElectra,
@@ -34,7 +35,7 @@ import {
34
35
  BeaconBlock,
35
36
  BlindedBeaconBlock,
36
37
  BlindedBeaconBlockBody,
37
- DataColumnSidecars,
38
+ DataColumnSidecar,
38
39
  Epoch,
39
40
  Root,
40
41
  RootHex,
@@ -504,7 +505,11 @@ export class BeaconChain implements IBeaconChain {
504
505
  }
505
506
 
506
507
  seenBlock(blockRoot: RootHex): boolean {
507
- return this.seenBlockInputCache.has(blockRoot) || this.forkChoice.hasBlockHex(blockRoot);
508
+ return this.seenBlockInputCache.hasBlock(blockRoot) || this.forkChoice.hasBlockHexUnsafe(blockRoot);
509
+ }
510
+
511
+ seenPayloadEnvelope(blockRoot: RootHex): boolean {
512
+ return this.seenPayloadEnvelopeInputCache.hasPayload(blockRoot) || this.forkChoice.hasPayloadHexUnsafe(blockRoot);
508
513
  }
509
514
 
510
515
  regenCanAcceptWork(): boolean {
@@ -873,20 +878,32 @@ export class BeaconChain implements IBeaconChain {
873
878
  );
874
879
  }
875
880
 
876
- async getDataColumnSidecars(blockSlot: Slot, blockRootHex: string): Promise<DataColumnSidecars> {
877
- const blockInput = this.seenBlockInputCache.get(blockRootHex);
878
- if (blockInput) {
879
- if (!isBlockInputColumns(blockInput)) {
880
- throw new Error(`Expected block input to have columns: slot=${blockSlot} root=${blockRootHex}`);
881
+ async getDataColumnSidecars(blockSlot: Slot, blockRootHex: string): Promise<DataColumnSidecar[]> {
882
+ const fork = this.config.getForkName(blockSlot);
883
+
884
+ if (isForkPostGloas(fork)) {
885
+ // After gloas, columns are tracked in PayloadEnvelopeInput
886
+ const payloadInput = this.seenPayloadEnvelopeInputCache.get(blockRootHex);
887
+ if (payloadInput) {
888
+ return payloadInput.getAllColumns();
889
+ }
890
+ } else {
891
+ // Before gloas, columns are tracked in BlockInput
892
+ const blockInput = this.seenBlockInputCache.get(blockRootHex);
893
+ if (blockInput) {
894
+ if (!isBlockInputColumns(blockInput)) {
895
+ throw new Error(`Expected block input to have columns: slot=${blockSlot} root=${blockRootHex}`);
896
+ }
897
+ return blockInput.getAllColumns();
881
898
  }
882
- return blockInput.getAllColumns();
883
899
  }
900
+
884
901
  const sidecarsUnfinalized = await this.db.dataColumnSidecar.values(fromHex(blockRootHex));
885
902
  if (sidecarsUnfinalized.length > 0) {
886
- return sidecarsUnfinalized as DataColumnSidecars;
903
+ return sidecarsUnfinalized;
887
904
  }
888
905
  const sidecarsFinalized = await this.db.dataColumnSidecarArchive.values(blockSlot);
889
- return sidecarsFinalized as DataColumnSidecars;
906
+ return sidecarsFinalized;
890
907
  }
891
908
 
892
909
  async getSerializedDataColumnSidecars(
@@ -894,23 +911,45 @@ export class BeaconChain implements IBeaconChain {
894
911
  blockRootHex: string,
895
912
  indices: number[]
896
913
  ): Promise<(Uint8Array | undefined)[]> {
897
- const blockInput = this.seenBlockInputCache.get(blockRootHex);
898
- if (blockInput) {
899
- if (!isBlockInputColumns(blockInput)) {
900
- throw new Error(`Expected block input to have columns: slot=${blockSlot} root=${blockRootHex}`);
914
+ const fork = this.config.getForkName(blockSlot);
915
+
916
+ if (isForkPostGloas(fork)) {
917
+ // After gloas, columns are tracked in PayloadEnvelopeInput
918
+ const payloadInput = this.seenPayloadEnvelopeInputCache.get(blockRootHex);
919
+ if (payloadInput) {
920
+ return indices.map((index) => {
921
+ const sidecar = payloadInput.getColumn(index);
922
+ if (!sidecar) {
923
+ return undefined;
924
+ }
925
+ const serialized = this.serializedCache.get(sidecar);
926
+ if (serialized) {
927
+ return serialized;
928
+ }
929
+ return sszTypesFor(fork as ForkPostGloas).DataColumnSidecar.serialize(sidecar);
930
+ });
901
931
  }
902
- return indices.map((index) => {
903
- const sidecar = blockInput.getColumn(index);
904
- if (!sidecar) {
905
- return undefined;
906
- }
907
- const serialized = this.serializedCache.get(sidecar);
908
- if (serialized) {
909
- return serialized;
932
+ } else {
933
+ // Before gloas, columns are tracked in BlockInput
934
+ const blockInput = this.seenBlockInputCache.get(blockRootHex);
935
+ if (blockInput) {
936
+ if (!isBlockInputColumns(blockInput)) {
937
+ throw new Error(`Expected block input to have columns: slot=${blockSlot} root=${blockRootHex}`);
910
938
  }
911
- return sszTypesFor(blockInput.forkName as ForkPostFulu).DataColumnSidecar.serialize(sidecar);
912
- });
939
+ return indices.map((index) => {
940
+ const sidecar = blockInput.getColumn(index);
941
+ if (!sidecar) {
942
+ return undefined;
943
+ }
944
+ const serialized = this.serializedCache.get(sidecar);
945
+ if (serialized) {
946
+ return serialized;
947
+ }
948
+ return sszTypesFor(blockInput.forkName as ForkPostFulu).DataColumnSidecar.serialize(sidecar);
949
+ });
950
+ }
913
951
  }
952
+
914
953
  const sidecarsUnfinalized = await this.db.dataColumnSidecar.getManyBinary(fromHex(blockRootHex), indices);
915
954
  if (sidecarsUnfinalized.some((sidecar) => sidecar != null)) {
916
955
  return sidecarsUnfinalized;
@@ -3,7 +3,8 @@ import {StrictEventEmitter} from "strict-event-emitter-types";
3
3
  import {routes} from "@lodestar/api";
4
4
  import {CheckpointWithPayloadStatus} from "@lodestar/fork-choice";
5
5
  import {IBeaconStateView} from "@lodestar/state-transition";
6
- import {DataColumnSidecars, RootHex, deneb, phase0} from "@lodestar/types";
6
+ import {DataColumnSidecar, RootHex, deneb, phase0} from "@lodestar/types";
7
+ import {SignedExecutionPayloadEnvelope} from "@lodestar/types/gloas";
7
8
  import {PeerIdStr} from "../util/peerId.js";
8
9
  import {BlockInputSource, IBlockInput} from "./blocks/blockInput/types.js";
9
10
 
@@ -54,13 +55,22 @@ export enum ChainEvent {
54
55
  */
55
56
  updateStatus = "updateStatus",
56
57
  /**
57
- * Trigger a BlockInputSync for blocks where the parentRoot is not known to fork choice
58
+ * Trigger BlockInputSync to find parent of a SignedBeaconBlock received
59
+ * Post-gloas, missing parent could be a SignedBeaconBlock and/or a SignedExecutionPayloadEnvelope
58
60
  */
59
- unknownParent = "unknownParent",
61
+ blockUnknownParent = "blockUnknownParent",
60
62
  /**
61
- * Trigger BlockInputSync for objects that correspond to a block that is not known to fork choice
63
+ * Trigger BlockInputSync to find a SignedBeaconBlock given a SignedExecutionPayloadEnvelop received
64
+ */
65
+ envelopeUnknownBlock = "envelopeUnknownBlock",
66
+ /**
67
+ * Trigger BlockInputSync to find a SignedBeaconBlock with specified block root.
62
68
  */
63
69
  unknownBlockRoot = "unknownBlockRoot",
70
+ /**
71
+ * Trigger BlockInputSync to find a SignedExecutionPayloadEnvelope with specified block root.
72
+ */
73
+ unknownEnvelopeBlockRoot = "unknownEnvelopeBlockRoot",
64
74
  /**
65
75
  * Trigger BlockInputSync for blocks that are partially received via gossip but are not complete by time the
66
76
  * cut-off window passes for waiting on gossip
@@ -75,9 +85,15 @@ export type ReorgEventData = routes.events.EventData[routes.events.EventType.cha
75
85
  type ApiEvents = {[K in routes.events.EventType]: (data: routes.events.EventData[K]) => void};
76
86
 
77
87
  export type ChainEventData = {
78
- [ChainEvent.unknownParent]: {blockInput: IBlockInput; peer: PeerIdStr; source: BlockInputSource};
88
+ [ChainEvent.blockUnknownParent]: {blockInput: IBlockInput; peer: PeerIdStr; source: BlockInputSource};
89
+ [ChainEvent.envelopeUnknownBlock]: {
90
+ envelope: SignedExecutionPayloadEnvelope;
91
+ peer?: PeerIdStr;
92
+ source: BlockInputSource;
93
+ };
79
94
  [ChainEvent.unknownBlockRoot]: {rootHex: RootHex; peer?: PeerIdStr; source: BlockInputSource};
80
95
  [ChainEvent.incompleteBlockInput]: {blockInput: IBlockInput; peer: PeerIdStr; source: BlockInputSource};
96
+ [ChainEvent.unknownEnvelopeBlockRoot]: {rootHex: RootHex; peer?: PeerIdStr; source: BlockInputSource};
81
97
  };
82
98
 
83
99
  export type IChainEvents = ApiEvents & {
@@ -88,7 +104,7 @@ export type IChainEvents = ApiEvents & {
88
104
 
89
105
  [ChainEvent.updateTargetCustodyGroupCount]: (targetGroupCount: number) => void;
90
106
 
91
- [ChainEvent.publishDataColumns]: (sidecars: DataColumnSidecars) => void;
107
+ [ChainEvent.publishDataColumns]: (sidecars: DataColumnSidecar[]) => void;
92
108
 
93
109
  [ChainEvent.publishBlobSidecars]: (sidecars: deneb.BlobSidecar[]) => void;
94
110
 
@@ -96,9 +112,11 @@ export type IChainEvents = ApiEvents & {
96
112
 
97
113
  // Sync events that are chain->chain. Initiated from network requests but do not cross the network
98
114
  // barrier so are considered ChainEvent(s).
99
- [ChainEvent.unknownParent]: (data: ChainEventData[ChainEvent.unknownParent]) => void;
115
+ [ChainEvent.blockUnknownParent]: (data: ChainEventData[ChainEvent.blockUnknownParent]) => void;
116
+ [ChainEvent.envelopeUnknownBlock]: (data: ChainEventData[ChainEvent.envelopeUnknownBlock]) => void;
100
117
  [ChainEvent.unknownBlockRoot]: (data: ChainEventData[ChainEvent.unknownBlockRoot]) => void;
101
118
  [ChainEvent.incompleteBlockInput]: (data: ChainEventData[ChainEvent.incompleteBlockInput]) => void;
119
+ [ChainEvent.unknownEnvelopeBlockRoot]: (data: ChainEventData[ChainEvent.unknownEnvelopeBlockRoot]) => void;
102
120
  };
103
121
 
104
122
  /**
@@ -1,3 +1,4 @@
1
+ import {ForkName} from "@lodestar/params";
1
2
  import {RootHex, Slot, SubnetID} from "@lodestar/types";
2
3
  import {LodestarError} from "@lodestar/utils";
3
4
  import {GossipActionError} from "./gossipValidation.js";
@@ -6,6 +7,7 @@ export enum DataColumnSidecarErrorCode {
6
7
  INVALID_INDEX = "DATA_COLUMN_SIDECAR_ERROR_INVALID_INDEX",
7
8
  NO_COMMITMENTS = "DATA_COLUMN_SIDECAR_ERROR_NO_COMMITMENTS",
8
9
  MISMATCHED_LENGTHS = "DATA_COLUMN_SIDECAR_ERROR_MISMATCHED_LENGTHS",
10
+ INCORRECT_TYPE = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_TYPE",
9
11
  INVALID_SUBNET = "DATA_COLUMN_SIDECAR_ERROR_INVALID_SUBNET",
10
12
  INVALID_KZG_PROOF = "DATA_COLUMN_SIDECAR_ERROR_INVALID_KZG_PROOF",
11
13
  TOO_MANY_KZG_COMMITMENTS = "DATA_COLUMN_SIDECAR_ERROR_TOO_MANY_KZG_COMMITMENTS",
@@ -18,6 +20,10 @@ export enum DataColumnSidecarErrorCode {
18
20
  INCORRECT_SIDECAR_COUNT = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_SIDECAR_COUNT",
19
21
  /** Sidecar doesn't match block */
20
22
  INCORRECT_BLOCK = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_BLOCK",
23
+ /** Sidecar slot doesn't match block slot */
24
+ INCORRECT_SIDECAR_SLOT = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_SIDECAR_SLOT",
25
+ /** Sidecar referenced block is not the expected block type */
26
+ INCORRECT_BLOCK_TYPE = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_BLOCK_TYPE",
21
27
  /** Sidecar cell count not as expected */
22
28
  INCORRECT_CELL_COUNT = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_CELL_COUNT",
23
29
  /** Sidecar kzg proof count not as expected */
@@ -32,10 +38,12 @@ export enum DataColumnSidecarErrorCode {
32
38
  FUTURE_SLOT = "DATA_COLUMN_SIDECAR_ERROR_FUTURE_SLOT",
33
39
  WOULD_REVERT_FINALIZED_SLOT = "DATA_COLUMN_SIDECAR_ERROR_WOULD_REVERT_FINALIZED_SLOT",
34
40
  PARENT_UNKNOWN = "DATA_COLUMN_SIDECAR_ERROR_PARENT_UNKNOWN",
41
+ BLOCK_UNKNOWN = "DATA_COLUMN_SIDECAR_ERROR_BLOCK_UNKNOWN",
35
42
  NOT_LATER_THAN_PARENT = "DATA_COLUMN_SIDECAR_ERROR_NOT_LATER_THAN_PARENT",
36
43
  PROPOSAL_SIGNATURE_INVALID = "DATA_COLUMN_SIDECAR_ERROR_PROPOSAL_SIGNATURE_INVALID",
37
44
  INCLUSION_PROOF_INVALID = "DATA_COLUMN_SIDECAR_ERROR_INCLUSION_PROOF_INVALID",
38
45
  INCORRECT_PROPOSER = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_PROPOSER",
46
+ PAYLOAD_ENVELOPE_INPUT_MISSING = "DATA_COLUMN_SIDECAR_ERROR_PAYLOAD_ENVELOPE_INPUT_MISSING",
39
47
  }
40
48
 
41
49
  export type DataColumnSidecarErrorType =
@@ -47,6 +55,12 @@ export type DataColumnSidecarErrorType =
47
55
  commitmentsLength: number;
48
56
  proofsLength: number;
49
57
  }
58
+ | {
59
+ code: DataColumnSidecarErrorCode.INCORRECT_TYPE;
60
+ slot: Slot;
61
+ columnIndex: number;
62
+ fork: ForkName;
63
+ }
50
64
  | {code: DataColumnSidecarErrorCode.INVALID_SUBNET; columnIndex: number; gossipSubnet: SubnetID}
51
65
  | {
52
66
  code: DataColumnSidecarErrorCode.TOO_MANY_KZG_COMMITMENTS;
@@ -63,6 +77,11 @@ export type DataColumnSidecarErrorType =
63
77
  parentRoot: RootHex;
64
78
  slot: Slot;
65
79
  }
80
+ | {
81
+ code: DataColumnSidecarErrorCode.BLOCK_UNKNOWN;
82
+ blockRoot: RootHex;
83
+ slot: Slot;
84
+ }
66
85
  | {
67
86
  code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID;
68
87
  slot: Slot;
@@ -80,6 +99,17 @@ export type DataColumnSidecarErrorType =
80
99
  expected: string;
81
100
  actual: string;
82
101
  }
102
+ | {
103
+ code: DataColumnSidecarErrorCode.INCORRECT_BLOCK_TYPE;
104
+ slot: Slot;
105
+ columnIndex: number;
106
+ }
107
+ | {
108
+ code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_SLOT;
109
+ columnIndex: number;
110
+ expected: Slot;
111
+ actual: Slot;
112
+ }
83
113
  | {
84
114
  code: DataColumnSidecarErrorCode.INCORRECT_HEADER_ROOT;
85
115
  slot: number;
@@ -97,7 +127,8 @@ export type DataColumnSidecarErrorType =
97
127
  actual: number;
98
128
  }
99
129
  | {code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF_BATCH; slot: number; reason: string}
100
- | {code: DataColumnSidecarErrorCode.INCORRECT_PROPOSER; actualProposerIndex: number; expectedProposerIndex: number};
130
+ | {code: DataColumnSidecarErrorCode.INCORRECT_PROPOSER; actualProposerIndex: number; expectedProposerIndex: number}
131
+ | {code: DataColumnSidecarErrorCode.PAYLOAD_ENVELOPE_INPUT_MISSING; slot: Slot; blockRoot: RootHex};
101
132
 
102
133
  export class DataColumnSidecarGossipError extends GossipActionError<DataColumnSidecarErrorType> {}
103
134
  export class DataColumnSidecarValidationError extends LodestarError<DataColumnSidecarErrorType> {}
@@ -5,7 +5,7 @@ import {EpochShuffling, IBeaconStateView, PubkeyCache} from "@lodestar/state-tra
5
5
  import {
6
6
  BeaconBlock,
7
7
  BlindedBeaconBlock,
8
- DataColumnSidecars,
8
+ DataColumnSidecar,
9
9
  Epoch,
10
10
  Root,
11
11
  RootHex,
@@ -159,6 +159,8 @@ export interface IBeaconChain {
159
159
  close(): Promise<void>;
160
160
  /** Chain has seen the specified block root or not. The block may not be processed yet, use forkchoice.hasBlock to check it */
161
161
  seenBlock(blockRoot: RootHex): boolean;
162
+ /** Chain has seen a SignedExecutionPayloadEnvelope for this block root (via seenCache or fork choice FULL variant) */
163
+ seenPayloadEnvelope(blockRoot: RootHex): boolean;
162
164
  /** Populate in-memory caches with persisted data. Call at least once on startup */
163
165
  loadFromDisk(): Promise<void>;
164
166
  /** Persist in-memory data to the DB. Call at least once before stopping the process */
@@ -217,7 +219,7 @@ export interface IBeaconChain {
217
219
  ): Promise<{block: SignedBeaconBlock; executionOptimistic: boolean; finalized: boolean} | null>;
218
220
  getBlobSidecars(blockSlot: Slot, blockRootHex: string): Promise<deneb.BlobSidecars | null>;
219
221
  getSerializedBlobSidecars(blockSlot: Slot, blockRootHex: string): Promise<Uint8Array | null>;
220
- getDataColumnSidecars(blockSlot: Slot, blockRootHex: string): Promise<DataColumnSidecars>;
222
+ getDataColumnSidecars(blockSlot: Slot, blockRootHex: string): Promise<DataColumnSidecar[]>;
221
223
  getSerializedDataColumnSidecars(
222
224
  blockSlot: Slot,
223
225
  blockRootHex: string,
@@ -149,8 +149,8 @@ export class SeenBlockInput {
149
149
  });
150
150
  }
151
151
 
152
- has(rootHex: RootHex): boolean {
153
- return this.blockInputs.has(rootHex);
152
+ hasBlock(rootHex: RootHex): boolean {
153
+ return this.blockInputs.get(rootHex)?.hasBlock() ?? false;
154
154
  }
155
155
 
156
156
  get(rootHex: RootHex): IBlockInput | undefined {
@@ -84,8 +84,8 @@ export class SeenPayloadEnvelopeInput {
84
84
  return this.payloadInputs.get(blockRootHex);
85
85
  }
86
86
 
87
- has(blockRootHex: RootHex): boolean {
88
- return this.payloadInputs.has(blockRootHex);
87
+ hasPayload(blockRootHex: RootHex): boolean {
88
+ return this.payloadInputs.get(blockRootHex)?.hasPayloadEnvelope() ?? false;
89
89
  }
90
90
 
91
91
  prune(blockRootHex: RootHex): void {