@lodestar/beacon-node 1.51.0-dev.aa1ecb7c18 → 1.51.0-dev.ab8f3f6381

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/lib/api/impl/beacon/pool/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/pool/index.js +17 -0
  3. package/lib/api/impl/beacon/pool/index.js.map +1 -1
  4. package/lib/chain/GetBlobsTracker.d.ts +2 -2
  5. package/lib/chain/GetBlobsTracker.d.ts.map +1 -1
  6. package/lib/chain/GetBlobsTracker.js.map +1 -1
  7. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  8. package/lib/chain/blocks/importBlock.js +35 -47
  9. package/lib/chain/blocks/importBlock.js.map +1 -1
  10. package/lib/chain/blocks/importExecutionPayload.d.ts +4 -0
  11. package/lib/chain/blocks/importExecutionPayload.d.ts.map +1 -1
  12. package/lib/chain/blocks/importExecutionPayload.js +24 -3
  13. package/lib/chain/blocks/importExecutionPayload.js.map +1 -1
  14. package/lib/chain/blocks/index.d.ts.map +1 -1
  15. package/lib/chain/blocks/index.js +2 -2
  16. package/lib/chain/blocks/index.js.map +1 -1
  17. package/lib/chain/blocks/verifyBlocksExecutionPayloads.d.ts +1 -1
  18. package/lib/chain/blocks/verifyBlocksExecutionPayloads.d.ts.map +1 -1
  19. package/lib/chain/blocks/verifyBlocksSanityChecks.d.ts.map +1 -1
  20. package/lib/chain/blocks/verifyBlocksSanityChecks.js +4 -0
  21. package/lib/chain/blocks/verifyBlocksSanityChecks.js.map +1 -1
  22. package/lib/chain/chain.d.ts +24 -2
  23. package/lib/chain/chain.d.ts.map +1 -1
  24. package/lib/chain/chain.js +119 -6
  25. package/lib/chain/chain.js.map +1 -1
  26. package/lib/chain/emitter.d.ts +6 -0
  27. package/lib/chain/emitter.d.ts.map +1 -1
  28. package/lib/chain/emitter.js +5 -0
  29. package/lib/chain/emitter.js.map +1 -1
  30. package/lib/chain/forkChoice/index.d.ts +11 -1
  31. package/lib/chain/forkChoice/index.d.ts.map +1 -1
  32. package/lib/chain/forkChoice/index.js +11 -0
  33. package/lib/chain/forkChoice/index.js.map +1 -1
  34. package/lib/chain/forkChoiceExecutionEngine.d.ts +53 -0
  35. package/lib/chain/forkChoiceExecutionEngine.d.ts.map +1 -0
  36. package/lib/chain/forkChoiceExecutionEngine.js +121 -0
  37. package/lib/chain/forkChoiceExecutionEngine.js.map +1 -0
  38. package/lib/chain/interface.d.ts +7 -3
  39. package/lib/chain/interface.d.ts.map +1 -1
  40. package/lib/chain/interface.js.map +1 -1
  41. package/lib/chain/opPools/aggregatedAttestationPool.d.ts +17 -4
  42. package/lib/chain/opPools/aggregatedAttestationPool.d.ts.map +1 -1
  43. package/lib/chain/opPools/aggregatedAttestationPool.js +177 -37
  44. package/lib/chain/opPools/aggregatedAttestationPool.js.map +1 -1
  45. package/lib/chain/opPools/index.d.ts +1 -0
  46. package/lib/chain/opPools/index.d.ts.map +1 -1
  47. package/lib/chain/opPools/index.js +1 -0
  48. package/lib/chain/opPools/index.js.map +1 -1
  49. package/lib/chain/opPools/singleAttestationPool.d.ts +77 -0
  50. package/lib/chain/opPools/singleAttestationPool.d.ts.map +1 -0
  51. package/lib/chain/opPools/singleAttestationPool.js +209 -0
  52. package/lib/chain/opPools/singleAttestationPool.js.map +1 -0
  53. package/lib/chain/options.d.ts +5 -0
  54. package/lib/chain/options.d.ts.map +1 -1
  55. package/lib/chain/options.js.map +1 -1
  56. package/lib/chain/produceBlock/produceBlockBody.d.ts +3 -2
  57. package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
  58. package/lib/chain/produceBlock/produceBlockBody.js +3 -2
  59. package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
  60. package/lib/chain/proposerTracker.d.ts +31 -0
  61. package/lib/chain/proposerTracker.d.ts.map +1 -0
  62. package/lib/chain/proposerTracker.js +75 -0
  63. package/lib/chain/proposerTracker.js.map +1 -0
  64. package/lib/chain/regen/interface.d.ts +1 -1
  65. package/lib/chain/regen/interface.d.ts.map +1 -1
  66. package/lib/chain/regen/queued.d.ts +1 -1
  67. package/lib/chain/regen/queued.d.ts.map +1 -1
  68. package/lib/chain/regen/queued.js +5 -3
  69. package/lib/chain/regen/queued.js.map +1 -1
  70. package/lib/execution/engine/http.d.ts +2 -2
  71. package/lib/execution/engine/http.d.ts.map +1 -1
  72. package/lib/execution/engine/http.js +5 -4
  73. package/lib/execution/engine/http.js.map +1 -1
  74. package/lib/execution/engine/interface.d.ts +16 -2
  75. package/lib/execution/engine/interface.d.ts.map +1 -1
  76. package/lib/execution/engine/jsonRpcTransport.d.ts +2 -2
  77. package/lib/execution/engine/jsonRpcTransport.d.ts.map +1 -1
  78. package/lib/execution/engine/jsonRpcTransport.js +1 -1
  79. package/lib/execution/engine/jsonRpcTransport.js.map +1 -1
  80. package/lib/execution/engine/restTransport.d.ts +2 -2
  81. package/lib/execution/engine/restTransport.d.ts.map +1 -1
  82. package/lib/execution/engine/restTransport.js.map +1 -1
  83. package/lib/execution/engine/transport.d.ts +1 -12
  84. package/lib/execution/engine/transport.d.ts.map +1 -1
  85. package/lib/index.d.ts +1 -1
  86. package/lib/index.d.ts.map +1 -1
  87. package/lib/index.js +1 -1
  88. package/lib/index.js.map +1 -1
  89. package/lib/metrics/metrics/lodestar.d.ts +26 -0
  90. package/lib/metrics/metrics/lodestar.d.ts.map +1 -1
  91. package/lib/metrics/metrics/lodestar.js +69 -0
  92. package/lib/metrics/metrics/lodestar.js.map +1 -1
  93. package/lib/network/core/networkCore.d.ts +2 -1
  94. package/lib/network/core/networkCore.d.ts.map +1 -1
  95. package/lib/network/core/networkCore.js +4 -0
  96. package/lib/network/core/networkCore.js.map +1 -1
  97. package/lib/network/core/networkCoreWorker.js +1 -0
  98. package/lib/network/core/networkCoreWorker.js.map +1 -1
  99. package/lib/network/core/networkCoreWorkerHandler.d.ts +2 -1
  100. package/lib/network/core/networkCoreWorkerHandler.d.ts.map +1 -1
  101. package/lib/network/core/networkCoreWorkerHandler.js +3 -0
  102. package/lib/network/core/networkCoreWorkerHandler.js.map +1 -1
  103. package/lib/network/core/types.d.ts +4 -1
  104. package/lib/network/core/types.d.ts.map +1 -1
  105. package/lib/network/network.d.ts +2 -0
  106. package/lib/network/network.d.ts.map +1 -1
  107. package/lib/network/network.js +8 -0
  108. package/lib/network/network.js.map +1 -1
  109. package/lib/network/options.d.ts.map +1 -1
  110. package/lib/network/options.js +2 -0
  111. package/lib/network/options.js.map +1 -1
  112. package/lib/network/peers/utils/subnetMap.d.ts +5 -2
  113. package/lib/network/peers/utils/subnetMap.d.ts.map +1 -1
  114. package/lib/network/peers/utils/subnetMap.js +6 -3
  115. package/lib/network/peers/utils/subnetMap.js.map +1 -1
  116. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  117. package/lib/network/processor/gossipHandlers.js +20 -0
  118. package/lib/network/processor/gossipHandlers.js.map +1 -1
  119. package/lib/network/subnets/attnetsService.d.ts +35 -5
  120. package/lib/network/subnets/attnetsService.d.ts.map +1 -1
  121. package/lib/network/subnets/attnetsService.js +84 -6
  122. package/lib/network/subnets/attnetsService.js.map +1 -1
  123. package/lib/network/subnets/interface.d.ts +7 -0
  124. package/lib/network/subnets/interface.d.ts.map +1 -1
  125. package/lib/network/subnets/interface.js.map +1 -1
  126. package/lib/util/execution.d.ts +4 -4
  127. package/lib/util/execution.d.ts.map +1 -1
  128. package/lib/util/execution.js.map +1 -1
  129. package/package.json +16 -16
  130. package/src/api/impl/beacon/pool/index.ts +26 -0
  131. package/src/chain/GetBlobsTracker.ts +2 -2
  132. package/src/chain/blocks/importBlock.ts +55 -67
  133. package/src/chain/blocks/importExecutionPayload.ts +42 -13
  134. package/src/chain/blocks/index.ts +5 -2
  135. package/src/chain/blocks/verifyBlocksExecutionPayloads.ts +1 -1
  136. package/src/chain/blocks/verifyBlocksSanityChecks.ts +4 -1
  137. package/src/chain/chain.ts +134 -3
  138. package/src/chain/emitter.ts +7 -0
  139. package/src/chain/forkChoice/index.ts +11 -0
  140. package/src/chain/forkChoiceExecutionEngine.ts +197 -0
  141. package/src/chain/interface.ts +7 -2
  142. package/src/chain/opPools/aggregatedAttestationPool.ts +222 -35
  143. package/src/chain/opPools/index.ts +1 -0
  144. package/src/chain/opPools/singleAttestationPool.ts +287 -0
  145. package/src/chain/options.ts +5 -0
  146. package/src/chain/produceBlock/produceBlockBody.ts +7 -3
  147. package/src/chain/proposerTracker.ts +94 -0
  148. package/src/chain/regen/interface.ts +1 -1
  149. package/src/chain/regen/queued.ts +6 -4
  150. package/src/execution/engine/http.ts +12 -13
  151. package/src/execution/engine/interface.ts +18 -2
  152. package/src/execution/engine/jsonRpcTransport.ts +10 -2
  153. package/src/execution/engine/restTransport.ts +3 -1
  154. package/src/execution/engine/transport.ts +8 -14
  155. package/src/index.ts +1 -1
  156. package/src/metrics/metrics/lodestar.ts +70 -0
  157. package/src/network/core/networkCore.ts +6 -1
  158. package/src/network/core/networkCoreWorker.ts +1 -0
  159. package/src/network/core/networkCoreWorkerHandler.ts +4 -1
  160. package/src/network/core/types.ts +4 -1
  161. package/src/network/network.ts +10 -0
  162. package/src/network/options.ts +2 -0
  163. package/src/network/peers/utils/subnetMap.ts +6 -3
  164. package/src/network/processor/gossipHandlers.ts +29 -0
  165. package/src/network/subnets/attnetsService.ts +94 -6
  166. package/src/network/subnets/interface.ts +7 -0
  167. package/src/util/execution.ts +6 -6
@@ -0,0 +1,287 @@
1
+ import {ATTESTATION_SUBNET_COUNT, ForkPostElectra} from "@lodestar/params";
2
+ import {EffectiveBalanceIncrements} from "@lodestar/state-transition";
3
+ import {BLSSignature, CommitteeIndex, RootHex, SingleAttestation, Slot, ValidatorIndex, phase0} from "@lodestar/types";
4
+ import {Metrics} from "../../metrics/metrics.js";
5
+ import {ProposerTracker} from "../proposerTracker.js";
6
+ import {InsertOutcome} from "./types.js";
7
+
8
+ type DataRootHex = string;
9
+
10
+ /** Position of a validator in its committee, i.e. its bit in `aggregationBits` */
11
+ type ValidatorCommitteeIndex = number;
12
+
13
+ /** `data` and `committeeIndex` are shared per data root and committee, only keep what packing reads */
14
+ type StoredSingle = {attesterIndex: ValidatorIndex; signature: BLSSignature};
15
+
16
+ type CommitteeSingles = {
17
+ /** Bit length of this committee's `aggregationBits` */
18
+ committeeSize: number;
19
+ singles: Map<ValidatorCommitteeIndex, StoredSingle>;
20
+ };
21
+
22
+ type DataRootSingles = {
23
+ /** Same for every single attestation under this data root */
24
+ data: phase0.AttestationData;
25
+ byCommittee: Map<CommitteeIndex, CommitteeSingles>;
26
+ };
27
+
28
+ /** Single attestations of not yet included committee members, signatures are not aggregated yet */
29
+ export type SinglesForCommittee = {
30
+ committeeSize: number;
31
+ singles: {validatorCommitteeIndex: ValidatorCommitteeIndex; single: StoredSingle}[];
32
+ newSeenEffectiveBalance: number;
33
+ newSeenAttesters: number;
34
+ };
35
+
36
+ /** Attesters of an attestation included in a block, `attestingIndices` is not sorted */
37
+ export type IncludedAttesters = {slot: Slot; attDataRootHex: DataRootHex; attestingIndices: ValidatorIndex[]};
38
+
39
+ /**
40
+ * Backstop only, single attestations included in a canonical block are deleted (see `onHead`) so the real retention is
41
+ * far lower. Sized for all subnets during the default 8 slots before a proposal (`slotsToSubscribeBeforeProposalDuty`)
42
+ * with no block at all and up to 1024 attesters per committee, i.e. ~2M active validators. Earlier slots of the
43
+ * collected window only add the few subnets the node subscribes to anyway. Beyond it, newer singles are dropped
44
+ */
45
+ const MAX_SINGLE_ATTESTATIONS_RETAINED = 8 * ATTESTATION_SUBNET_COUNT * 1024;
46
+
47
+ /**
48
+ * Single attestations to be packed into a block produced by this node, for votes that no aggregate covers.
49
+ * Callers only add to it for vote slots of an upcoming local proposal, see `ProposerTracker.shouldCollect()`.
50
+ * They are kept even if an aggregate covers them since proposers may not include that aggregate, until a canonical
51
+ * block includes them.
52
+ */
53
+ export class SingleAttestationPool {
54
+ private readonly singlesByCommitteeByDataRootBySlot = new Map<Slot, Map<DataRootHex, DataRootSingles>>();
55
+ private size = 0;
56
+ /** Attesters included by the head block, deleted once a child of it becomes head */
57
+ private pendingPrune: {blockRoot: RootHex; included: IncludedAttesters[]} | null = null;
58
+
59
+ constructor(metrics: Metrics | null = null) {
60
+ metrics?.opPool.singleAttestationPool.size.addCollect(() =>
61
+ metrics.opPool.singleAttestationPool.size.set(this.size)
62
+ );
63
+ }
64
+
65
+ /**
66
+ * @param validatorCommitteeIndex position of the attester in its committee, computed by gossip validation
67
+ */
68
+ add(
69
+ attestation: SingleAttestation<ForkPostElectra>,
70
+ attDataRootHex: DataRootHex,
71
+ validatorCommitteeIndex: ValidatorCommitteeIndex,
72
+ committeeSize: number
73
+ ): InsertOutcome {
74
+ if (this.size >= MAX_SINGLE_ATTESTATIONS_RETAINED) {
75
+ return InsertOutcome.ReachLimit;
76
+ }
77
+
78
+ const {data, committeeIndex, attesterIndex, signature} = attestation;
79
+ let singlesByCommitteeByDataRoot = this.singlesByCommitteeByDataRootBySlot.get(data.slot);
80
+ if (singlesByCommitteeByDataRoot === undefined) {
81
+ singlesByCommitteeByDataRoot = new Map();
82
+ this.singlesByCommitteeByDataRootBySlot.set(data.slot, singlesByCommitteeByDataRoot);
83
+ }
84
+
85
+ let dataRootSingles = singlesByCommitteeByDataRoot.get(attDataRootHex);
86
+ if (dataRootSingles === undefined) {
87
+ dataRootSingles = {data, byCommittee: new Map()};
88
+ singlesByCommitteeByDataRoot.set(attDataRootHex, dataRootSingles);
89
+ }
90
+
91
+ let committee = dataRootSingles.byCommittee.get(committeeIndex);
92
+ if (committee === undefined) {
93
+ committee = {committeeSize, singles: new Map()};
94
+ dataRootSingles.byCommittee.set(committeeIndex, committee);
95
+ }
96
+
97
+ if (committee.singles.has(validatorCommitteeIndex)) {
98
+ return InsertOutcome.AlreadyKnown;
99
+ }
100
+
101
+ committee.singles.set(validatorCommitteeIndex, {attesterIndex, signature});
102
+ this.size++;
103
+ return InsertOutcome.NewData;
104
+ }
105
+
106
+ /**
107
+ * Delete single attestations included by the parent of the new head. Not the ones of the head itself: a proposer
108
+ * boost reorg of a weak head must keep them, once a child of it is the head that reorg can no longer happen.
109
+ * @param included attesters of the attestations included in the new head block
110
+ * @returns number of deleted single attestations
111
+ */
112
+ onHead(blockRoot: RootHex, parentRoot: RootHex, included: IncludedAttesters[]): number {
113
+ if (this.singlesByCommitteeByDataRootBySlot.size === 0) {
114
+ this.pendingPrune = null;
115
+ return 0;
116
+ }
117
+
118
+ // a reorg or a parent imported while the pool was empty drops the pending block without deleting
119
+ const deleted = this.pendingPrune?.blockRoot === parentRoot ? this.deleteIncluded(this.pendingPrune.included) : 0;
120
+ this.pendingPrune = {blockRoot, included};
121
+ return deleted;
122
+ }
123
+
124
+ /** Drop vote slots that no upcoming local proposal can include */
125
+ prune(proposerTracker: ProposerTracker): void {
126
+ for (const [slot, singlesByCommitteeByDataRoot] of this.singlesByCommitteeByDataRootBySlot) {
127
+ if (proposerTracker.shouldCollect(slot)) {
128
+ continue;
129
+ }
130
+
131
+ for (const dataRootSingles of singlesByCommitteeByDataRoot.values()) {
132
+ for (const committee of dataRootSingles.byCommittee.values()) {
133
+ this.size -= committee.singles.size;
134
+ }
135
+ }
136
+ this.singlesByCommitteeByDataRootBySlot.delete(slot);
137
+ }
138
+
139
+ if (this.singlesByCommitteeByDataRootBySlot.size === 0) {
140
+ this.pendingPrune = null;
141
+ }
142
+ }
143
+
144
+ isEmpty(): boolean {
145
+ return this.singlesByCommitteeByDataRootBySlot.size === 0;
146
+ }
147
+
148
+ getStoredSlots(): Slot[] {
149
+ return Array.from(this.singlesByCommitteeByDataRootBySlot.keys());
150
+ }
151
+
152
+ getDataRoots(slot: Slot): Map<DataRootHex, phase0.AttestationData> {
153
+ const dataRoots = new Map<DataRootHex, phase0.AttestationData>();
154
+ for (const [attDataRootHex, dataRootSingles] of this.singlesByCommitteeByDataRootBySlot.get(slot) ?? []) {
155
+ dataRoots.set(attDataRootHex, dataRootSingles.data);
156
+ }
157
+ return dataRoots;
158
+ }
159
+
160
+ getCommitteeIndices(slot: Slot, attDataRootHex: DataRootHex): CommitteeIndex[] {
161
+ return Array.from(this.singlesByCommitteeByDataRootBySlot.get(slot)?.get(attDataRootHex)?.byCommittee.keys() ?? []);
162
+ }
163
+
164
+ /**
165
+ * Single attestations of a committee whose attesters are in `notSeenCommitteeMembers`, i.e. not on chain and not
166
+ * covered by the aggregates picked for the block. `notSeenCommitteeMembers` is only read.
167
+ * Returns at most `maxCount` single attestations with the highest effective balances, without removing them from the
168
+ * pool.
169
+ */
170
+ getUncovered(
171
+ slot: Slot,
172
+ attDataRootHex: DataRootHex,
173
+ committeeIndex: CommitteeIndex,
174
+ notSeenCommitteeMembers: ReadonlySet<ValidatorCommitteeIndex>,
175
+ effectiveBalanceIncrements: EffectiveBalanceIncrements,
176
+ maxCount: number
177
+ ): SinglesForCommittee | null {
178
+ const committee = this.singlesByCommitteeByDataRootBySlot
179
+ .get(slot)
180
+ ?.get(attDataRootHex)
181
+ ?.byCommittee.get(committeeIndex);
182
+ if (committee === undefined) {
183
+ return null;
184
+ }
185
+
186
+ const singles: SinglesForCommittee["singles"] = [];
187
+ // walk the smaller side, notSeenCommitteeMembers is usually tiny but close to the committee size if no aggregate
188
+ if (notSeenCommitteeMembers.size <= committee.singles.size) {
189
+ for (const validatorCommitteeIndex of notSeenCommitteeMembers) {
190
+ const single = committee.singles.get(validatorCommitteeIndex);
191
+ if (single !== undefined) {
192
+ singles.push({validatorCommitteeIndex, single});
193
+ }
194
+ }
195
+ } else {
196
+ for (const [validatorCommitteeIndex, single] of committee.singles) {
197
+ if (notSeenCommitteeMembers.has(validatorCommitteeIndex)) {
198
+ singles.push({validatorCommitteeIndex, single});
199
+ }
200
+ }
201
+ }
202
+ if (singles.length > maxCount) {
203
+ singles.sort(
204
+ (a, b) =>
205
+ effectiveBalanceIncrements[b.single.attesterIndex] - effectiveBalanceIncrements[a.single.attesterIndex]
206
+ );
207
+ singles.length = maxCount;
208
+ }
209
+ if (singles.length === 0) {
210
+ return null;
211
+ }
212
+
213
+ let newSeenEffectiveBalance = 0;
214
+ for (const {single} of singles) {
215
+ newSeenEffectiveBalance += effectiveBalanceIncrements[single.attesterIndex];
216
+ }
217
+ return {committeeSize: committee.committeeSize, singles, newSeenEffectiveBalance, newSeenAttesters: singles.length};
218
+ }
219
+
220
+ /**
221
+ * Delete single attestations whose votes are already on chain, they add nothing to a block built on top of it.
222
+ * @param includedAttestations one entry per attestation in the body of a canonical block (the parent of the new
223
+ * head): the attestation slot and data root, and the validators it covers. A single attestation is included if its
224
+ * attester is in `attestingIndices` of an entry with the same slot and data root.
225
+ * @returns number of deleted single attestations
226
+ */
227
+ private deleteIncluded(includedAttestations: IncludedAttesters[]): number {
228
+ // a block may include several attestations of the same data, merge their attesters per data root
229
+ const includedAttestersByDataRootBySlot = new Map<Slot, Map<DataRootHex, Set<ValidatorIndex>>>();
230
+ for (const {slot, attDataRootHex, attestingIndices} of includedAttestations) {
231
+ if (this.singlesByCommitteeByDataRootBySlot.get(slot)?.has(attDataRootHex) !== true) {
232
+ continue;
233
+ }
234
+ let includedAttestersByDataRoot = includedAttestersByDataRootBySlot.get(slot);
235
+ if (includedAttestersByDataRoot === undefined) {
236
+ includedAttestersByDataRoot = new Map();
237
+ includedAttestersByDataRootBySlot.set(slot, includedAttestersByDataRoot);
238
+ }
239
+ let includedAttesters = includedAttestersByDataRoot.get(attDataRootHex);
240
+ if (includedAttesters === undefined) {
241
+ includedAttesters = new Set();
242
+ includedAttestersByDataRoot.set(attDataRootHex, includedAttesters);
243
+ }
244
+ for (const attesterIndex of attestingIndices) {
245
+ includedAttesters.add(attesterIndex);
246
+ }
247
+ }
248
+
249
+ let deleted = 0;
250
+ for (const [slot, includedAttestersByDataRoot] of includedAttestersByDataRootBySlot) {
251
+ const singlesByCommitteeByDataRoot = this.singlesByCommitteeByDataRootBySlot.get(slot);
252
+ if (singlesByCommitteeByDataRoot === undefined) {
253
+ continue;
254
+ }
255
+
256
+ for (const [attDataRootHex, includedAttesters] of includedAttestersByDataRoot) {
257
+ const dataRootSingles = singlesByCommitteeByDataRoot.get(attDataRootHex);
258
+ if (dataRootSingles === undefined) {
259
+ continue;
260
+ }
261
+
262
+ for (const [committeeIndex, committee] of dataRootSingles.byCommittee) {
263
+ for (const [validatorCommitteeIndex, {attesterIndex}] of committee.singles) {
264
+ if (includedAttesters.has(attesterIndex)) {
265
+ committee.singles.delete(validatorCommitteeIndex);
266
+ deleted++;
267
+ }
268
+ }
269
+ if (committee.singles.size === 0) {
270
+ dataRootSingles.byCommittee.delete(committeeIndex);
271
+ }
272
+ }
273
+
274
+ if (dataRootSingles.byCommittee.size === 0) {
275
+ singlesByCommitteeByDataRoot.delete(attDataRootHex);
276
+ }
277
+ }
278
+
279
+ if (singlesByCommitteeByDataRoot.size === 0) {
280
+ this.singlesByCommitteeByDataRootBySlot.delete(slot);
281
+ }
282
+ }
283
+
284
+ this.size -= deleted;
285
+ return deleted;
286
+ }
287
+ }
@@ -90,6 +90,11 @@ export type PoolOpts = {
90
90
  * Only preaggregate attestation/sync committee message since clockSlot - preaggregateSlotDistance
91
91
  */
92
92
  preaggregateSlotDistance?: number;
93
+ /**
94
+ * Dev only: collect single attestations of every vote slot a block at the clock slot can include regardless of
95
+ * proposal duties, so that any produced block can pack them. Combine with `subscribeAllSubnets`
96
+ */
97
+ singleAttestationIgnoreProposalDuty?: boolean;
93
98
  };
94
99
 
95
100
  export type SeenCacheOpts = {
@@ -72,12 +72,13 @@ import {
72
72
  } from "@lodestar/utils";
73
73
  import {ZERO_HASH_HEX} from "../../constants/index.js";
74
74
  import {numToQuantity} from "../../execution/engine/utils.js";
75
- import {IExecutionBuilder, IExecutionEngine, PayloadAttributes, PayloadId} from "../../execution/index.js";
75
+ import {IExecutionBuilder, PayloadAttributes, PayloadId} from "../../execution/index.js";
76
76
  import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
77
77
  import {fromGraffitiBytes} from "../../util/graffiti.js";
78
78
  import {kzg} from "../../util/kzg.js";
79
79
  import {BeaconProposerCache} from "../beaconProposerCache.js";
80
80
  import type {BeaconChain} from "../chain.js";
81
+ import {ForkChoiceExecutionEngine} from "../forkChoiceExecutionEngine.js";
81
82
  import {CommonBlockBody} from "../interface.js";
82
83
  import {InclusionListStore, ProposerPreferencesPool} from "../opPools/index.js";
83
84
  import {validateBlobsAndKzgCommitments, validateCellsAndKzgCommitments} from "./validateBlobsAndKzgCommitments.js";
@@ -742,7 +743,7 @@ export async function produceBlockBody<T extends BlockType>(
742
743
  */
743
744
  export async function prepareExecutionPayload(
744
745
  chain: {
745
- executionEngine: IExecutionEngine;
746
+ executionEngine: ForkChoiceExecutionEngine;
746
747
  config: ChainForkConfig;
747
748
  forkChoice: IForkChoice;
748
749
  proposerPreferencesPool: ProposerPreferencesPool;
@@ -807,6 +808,7 @@ export async function prepareExecutionPayload(
807
808
 
808
809
  payloadId = await chain.executionEngine.notifyForkchoiceUpdate(
809
810
  fork,
811
+ toRootHex(parentBlockRoot),
810
812
  toRootHex(parentBlockHash),
811
813
  safeBlockHash,
812
814
  finalizedBlockHash,
@@ -1125,11 +1127,13 @@ export async function produceCommonBlockBody<T extends BlockType>(
1125
1127
  this.opPool.getSlashingsAndExits(currentState, blockType, this.metrics);
1126
1128
 
1127
1129
  const endAttestations = stepsMetrics?.startTimer();
1130
+ // TODO: extract to a separate BeaconChain method to coordinate the 2 op-pools
1128
1131
  const attestations = this.aggregatedAttestationPool.getAttestationsForBlock(
1129
1132
  fork,
1130
1133
  this.forkChoice,
1131
1134
  this.shufflingCache,
1132
- currentState
1135
+ currentState,
1136
+ this.singleAttestationPool
1133
1137
  );
1134
1138
  endAttestations?.({
1135
1139
  step: BlockProductionStep.attestations,
@@ -0,0 +1,94 @@
1
+ import {MIN_ATTESTATION_INCLUSION_DELAY} from "@lodestar/params";
2
+ import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
3
+ import {Slot} from "@lodestar/types";
4
+ import {Metrics} from "../metrics/metrics.js";
5
+
6
+ /**
7
+ * Track upcoming block proposals of validators connected to this node, and the vote slots those blocks can include,
8
+ * to only then insert single attestations into the SingleAttestationPool
9
+ */
10
+ export class ProposerTracker {
11
+ /** Local proposal slots that have not passed yet */
12
+ private readonly proposalSlots = new Set<Slot>();
13
+ /** Vote slots of tracked proposals, see `getFirstIncludableVoteSlot` */
14
+ private readonly collectableSlots = new Set<Slot>();
15
+ private clockSlot = 0;
16
+
17
+ /**
18
+ * @param ignoreProposalDuty dev only, collect every vote slot a block at the clock slot can include regardless of
19
+ * local proposals
20
+ */
21
+ constructor(
22
+ private readonly ignoreProposalDuty = false,
23
+ metrics: Metrics | null = null
24
+ ) {
25
+ metrics?.proposerTracker.proposalSlots.addCollect(() =>
26
+ metrics.proposerTracker.proposalSlots.set(this.proposalSlots.size)
27
+ );
28
+ metrics?.proposerTracker.collectableSlot.addCollect(() =>
29
+ metrics.proposerTracker.collectableSlot.set(this.shouldCollect(this.clockSlot) ? 1 : 0)
30
+ );
31
+ }
32
+
33
+ /**
34
+ * Idempotent, a known proposal is ignored
35
+ * @returns true if the proposal was not tracked before
36
+ */
37
+ addProposal(proposalSlot: Slot): boolean {
38
+ if (this.proposalSlots.has(proposalSlot)) {
39
+ return false;
40
+ }
41
+
42
+ this.proposalSlots.add(proposalSlot);
43
+ for (
44
+ let voteSlot = getFirstIncludableVoteSlot(proposalSlot);
45
+ voteSlot <= proposalSlot - MIN_ATTESTATION_INCLUSION_DELAY;
46
+ voteSlot++
47
+ ) {
48
+ this.collectableSlots.add(voteSlot);
49
+ }
50
+ return true;
51
+ }
52
+
53
+ shouldCollect(attestationSlot: Slot): boolean {
54
+ if (this.ignoreProposalDuty) {
55
+ // to test block production without proposal duties
56
+ return attestationSlot >= getFirstIncludableVoteSlot(this.clockSlot);
57
+ }
58
+ return this.collectableSlots.has(attestationSlot);
59
+ }
60
+
61
+ /**
62
+ * Drop proposals that have passed. A proposal is kept during its own slot so that its single attestations stay
63
+ * available until the block is produced. Vote slots are dropped once no remaining proposal can include them.
64
+ */
65
+ prune(clockSlot: Slot): void {
66
+ this.clockSlot = clockSlot;
67
+ for (const proposalSlot of this.proposalSlots) {
68
+ if (proposalSlot < clockSlot) {
69
+ this.proposalSlots.delete(proposalSlot);
70
+ }
71
+ }
72
+
73
+ if (this.proposalSlots.size === 0) {
74
+ this.collectableSlots.clear();
75
+ return;
76
+ }
77
+
78
+ // Each remaining proposal covers its own vote slots, so a vote slot below the lowest one is covered by none
79
+ const lowestCollectableVoteSlot = getFirstIncludableVoteSlot(Math.min(...this.proposalSlots));
80
+ for (const voteSlot of this.collectableSlots) {
81
+ if (voteSlot < lowestCollectableVoteSlot) {
82
+ this.collectableSlots.delete(voteSlot);
83
+ }
84
+ }
85
+ }
86
+ }
87
+
88
+ /**
89
+ * Since deneb (EIP-7045) a block includes attestations of its current and previous epoch, so the first vote slot it can
90
+ * include is the start of the previous epoch, 32 to 63 slots before it on mainnet
91
+ */
92
+ function getFirstIncludableVoteSlot(blockSlot: Slot): Slot {
93
+ return computeStartSlotAtEpoch(Math.max(computeEpochAtSlot(blockSlot) - 1, 0));
94
+ }
@@ -52,7 +52,7 @@ export interface IStateRegenerator extends IStateRegeneratorInternal {
52
52
  onCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void;
53
53
  processState(blockRootHex: RootHex, postState: IBeaconStateView): void;
54
54
  addCheckpointState(cp: phase0.Checkpoint, item: IBeaconStateView): void;
55
- updateHeadState(newHead: ProtoBlock, maybeHeadState: IBeaconStateView): void;
55
+ updateHeadState(newHead: ProtoBlock, maybeHeadState?: IBeaconStateView): void;
56
56
  updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null;
57
57
  }
58
58
 
@@ -154,18 +154,20 @@ export class QueuedStateRegenerator implements IStateRegenerator {
154
154
  this.checkpointStateCache.add(cp, item);
155
155
  }
156
156
 
157
- updateHeadState(newHead: ProtoBlock, maybeHeadState: IBeaconStateView): void {
157
+ updateHeadState(newHead: ProtoBlock, maybeHeadState?: IBeaconStateView): void {
158
158
  const {stateRoot: newHeadStateRoot, blockRoot: newHeadBlockRoot, slot: newHeadSlot} = newHead;
159
- const maybeHeadStateRoot = toRootHex(maybeHeadState.hashTreeRoot());
159
+ const maybeHeadStateRoot = maybeHeadState ? toRootHex(maybeHeadState.hashTreeRoot()) : undefined;
160
160
  const logCtx = {
161
161
  newHeadSlot,
162
162
  newHeadBlockRoot,
163
163
  newHeadStateRoot,
164
- maybeHeadSlot: maybeHeadState.slot,
164
+ maybeHeadSlot: maybeHeadState?.slot,
165
165
  maybeHeadStateRoot,
166
166
  };
167
167
  const headState =
168
- newHeadStateRoot === maybeHeadStateRoot ? maybeHeadState : this.blockStateCache.get(newHeadStateRoot);
168
+ maybeHeadState && newHeadStateRoot === maybeHeadStateRoot
169
+ ? maybeHeadState
170
+ : this.blockStateCache.get(newHeadStateRoot);
169
171
 
170
172
  if (headState) {
171
173
  this.blockStateCache.setHeadState(headState);
@@ -22,9 +22,11 @@ import {
22
22
  ExecutePayloadResponse,
23
23
  ExecutionEngineState,
24
24
  ExecutionPayloadStatus,
25
+ ForkchoiceUpdatedResult,
25
26
  IExecutionEngine,
26
27
  PayloadAttributes,
27
28
  PayloadId,
29
+ PayloadStatusResult,
28
30
  VersionedHashes,
29
31
  } from "./interface.js";
30
32
  import {ErrorJsonRpcResponse, HttpRpcError, JsonRpcHttpClientEvent} from "./jsonRpcHttpClient.js";
@@ -32,7 +34,7 @@ import {PayloadIdCache} from "./payloadIdCache.js";
32
34
  import {EngineRestError, EngineRestResponseError, isRetryableEngineRestError} from "./restHttpClient.js";
33
35
  import {EngineCapabilities, RestEngineTransport} from "./restTransport.js";
34
36
  import {executionForkName} from "./sszTypes.js";
35
- import {IEngineTransport, PayloadStatusResult} from "./transport.js";
37
+ import {IEngineTransport} from "./transport.js";
36
38
  import {ExecutionPayloadBodyV2, serializePayloadAttributes} from "./types.js";
37
39
  import {getExecutionEngineState} from "./utils.js";
38
40
 
@@ -342,15 +344,16 @@ export class ExecutionEngineHttp implements IExecutionEngine {
342
344
  safeBlockHash: RootHex,
343
345
  finalizedBlockHash: RootHex,
344
346
  payloadAttributes?: PayloadAttributes
345
- ): Promise<PayloadId | null> {
346
- const {
347
- payloadStatus: {status, validationError},
348
- payloadId,
349
- } = await this.enqueue(() =>
347
+ ): Promise<ForkchoiceUpdatedResult> {
348
+ const result = await this.enqueue(() =>
350
349
  this.withTransport(fork, undefined, (transport) =>
351
350
  transport.forkchoiceUpdated(fork, headBlockHash, safeBlockHash, finalizedBlockHash, payloadAttributes)
352
351
  )
353
352
  );
353
+ const {
354
+ payloadStatus: {status},
355
+ payloadId,
356
+ } = result;
354
357
 
355
358
  this.updateEngineState(getExecutionEngineState({payloadStatus: status, oldState: this.state}));
356
359
  this.metrics?.engineNotifyForkchoiceUpdateResult.inc({result: status});
@@ -374,21 +377,17 @@ export class ExecutionEngineHttp implements IExecutionEngine {
374
377
  );
375
378
  void this.prunePayloadIdCache();
376
379
  }
377
- return payloadId;
380
+ return result;
378
381
 
379
382
  case ExecutionPayloadStatus.SYNCING:
380
383
  // Throw error on syncing if requested to produce a block, else silently ignore
381
384
  if (payloadAttributes) {
382
385
  throw Error("Execution Layer Syncing");
383
386
  }
384
- return null;
387
+ return result;
385
388
 
386
389
  case ExecutionPayloadStatus.INVALID:
387
- throw Error(
388
- `Invalid ${payloadAttributes ? "prepare payload" : "forkchoice request"}, validationError=${
389
- validationError ?? ""
390
- }`
391
- );
390
+ return result;
392
391
 
393
392
  default:
394
393
  throw Error(`Unknown status ${status}`);
@@ -99,6 +99,19 @@ export type ExecutePayloadResponse =
99
99
  validationError: string;
100
100
  };
101
101
 
102
+ export type PayloadStatusResult = {
103
+ status: ExecutionPayloadStatus;
104
+ latestValidHash: RootHex | null;
105
+ validationError: string | null;
106
+ /** PayloadStatusV2: whether a VALID payload satisfied the inclusion list constraints, null otherwise */
107
+ inclusionListSatisfied?: boolean | null;
108
+ };
109
+
110
+ export type ForkchoiceUpdatedResult = {
111
+ payloadStatus: PayloadStatusResult;
112
+ payloadId: PayloadId | null;
113
+ };
114
+
102
115
  export type ForkChoiceUpdateStatus =
103
116
  | ExecutionPayloadStatus.VALID
104
117
  | ExecutionPayloadStatus.INVALID
@@ -166,7 +179,10 @@ export interface IExecutionEngine {
166
179
  * The call of the notify_forkchoice_updated function maps on the POS_FORKCHOICE_UPDATED event defined in the EIP-3675.
167
180
  * https://github.com/ethereum/consensus-specs/blob/v1.1.7/specs/merge/fork-choice.md#notify_forkchoice_updated
168
181
  *
169
- * Should be called in response to fork-choice head and finalized events
182
+ * Should be called in response to fork-choice head and finalized events.
183
+ *
184
+ * An INVALID verdict is returned, not thrown: the caller feeds it back into fork choice together with the
185
+ * `latestValidHash`. Only SYNCING while building a payload is an error.
170
186
  */
171
187
  notifyForkchoiceUpdate(
172
188
  fork: ForkName,
@@ -174,7 +190,7 @@ export interface IExecutionEngine {
174
190
  safeBlockHash: RootHex,
175
191
  finalizedBlockHash: RootHex,
176
192
  payloadAttributes?: PayloadAttributes
177
- ): Promise<PayloadId | null>;
193
+ ): Promise<ForkchoiceUpdatedResult>;
178
194
 
179
195
  /**
180
196
  * Given the payload_id, get_payload returns the most recent version of the execution payload that has been built
@@ -3,9 +3,17 @@ import {ExecutionPayload, ExecutionRequests, Root, RootHex, bellatrix} from "@lo
3
3
  import {BlobAndProof} from "@lodestar/types/deneb";
4
4
  import {BlobAndProofV2} from "@lodestar/types/fulu";
5
5
  import {strip0xPrefix} from "@lodestar/utils";
6
- import {ClientCode, ClientVersion, PayloadAttributes, PayloadId, VersionedHashes} from "./interface.js";
6
+ import {
7
+ ClientCode,
8
+ ClientVersion,
9
+ ForkchoiceUpdatedResult,
10
+ PayloadAttributes,
11
+ PayloadId,
12
+ PayloadStatusResult,
13
+ VersionedHashes,
14
+ } from "./interface.js";
7
15
  import {IJsonRpcHttpClient, JsonRpcHttpClientEventEmitter, ReqOpts} from "./jsonRpcHttpClient.js";
8
- import {ForkchoiceUpdatedResult, GetPayloadResult, IEngineTransport, PayloadStatusResult} from "./transport.js";
16
+ import {GetPayloadResult, IEngineTransport} from "./transport.js";
9
17
  import {
10
18
  BLOB_AND_PROOF_V2_RPC_BYTES,
11
19
  EngineApiRpcParamTypes,
@@ -19,8 +19,10 @@ import {
19
19
  ClientCode,
20
20
  ClientVersion,
21
21
  ExecutionPayloadStatus,
22
+ ForkchoiceUpdatedResult,
22
23
  PayloadAttributes,
23
24
  PayloadId,
25
+ PayloadStatusResult,
24
26
  VersionedHashes,
25
27
  } from "./interface.js";
26
28
  import {JsonRpcHttpClientEventEmitter, ReqOpts} from "./jsonRpcHttpClient.js";
@@ -57,7 +59,7 @@ import {
57
59
  PayloadStatusSsz,
58
60
  executionForkName,
59
61
  } from "./sszTypes.js";
60
- import {ForkchoiceUpdatedResult, GetPayloadResult, IEngineTransport, PayloadStatusResult} from "./transport.js";
62
+ import {GetPayloadResult, IEngineTransport} from "./transport.js";
61
63
  import {EngineTransportMetrics} from "./transportMetrics.js";
62
64
  import {
63
65
  ExecutionPayloadBodyV2,
@@ -2,23 +2,17 @@ import {ForkName} from "@lodestar/params";
2
2
  import {BlobsBundle, ExecutionPayload, ExecutionRequests, Root, RootHex, Wei, bellatrix} from "@lodestar/types";
3
3
  import {BlobAndProof} from "@lodestar/types/deneb";
4
4
  import {BlobAndProofV2} from "@lodestar/types/fulu";
5
- import {ClientVersion, ExecutionPayloadStatus, PayloadAttributes, PayloadId, VersionedHashes} from "./interface.js";
5
+ import {
6
+ ClientVersion,
7
+ ForkchoiceUpdatedResult,
8
+ PayloadAttributes,
9
+ PayloadId,
10
+ PayloadStatusResult,
11
+ VersionedHashes,
12
+ } from "./interface.js";
6
13
  import {JsonRpcHttpClientEventEmitter} from "./jsonRpcHttpClient.js";
7
14
  import {ExecutionPayloadBodyV2} from "./types.js";
8
15
 
9
- export type PayloadStatusResult = {
10
- status: ExecutionPayloadStatus;
11
- latestValidHash: RootHex | null;
12
- validationError: string | null;
13
- /** PayloadStatusV2: whether a VALID payload satisfied the inclusion list constraints, null otherwise */
14
- inclusionListSatisfied?: boolean | null;
15
- };
16
-
17
- export type ForkchoiceUpdatedResult = {
18
- payloadStatus: PayloadStatusResult;
19
- payloadId: PayloadId | null;
20
- };
21
-
22
16
  export type GetPayloadResult = {
23
17
  executionPayload: ExecutionPayload;
24
18
  executionPayloadValue: Wei;
package/src/index.ts CHANGED
@@ -7,7 +7,7 @@ export {DbCPStateDatastore} from "./chain/stateCache/datastore/db.js";
7
7
  export {FileCPStateDatastore} from "./chain/stateCache/datastore/file.js";
8
8
  export {BeaconDb, type IBeaconDb} from "./db/index.js";
9
9
  // Export execution engine to make it usable by builder
10
- export {initializeExecutionEngine} from "./execution/index.js";
10
+ export {ExecutionPayloadStatus, initializeExecutionEngine} from "./execution/index.js";
11
11
  // Export metrics utilities to de-duplicate validator metrics
12
12
  export {
13
13
  type HttpMetricsServer,