@lodestar/beacon-node 1.36.0-dev.fe5f423da3 → 1.36.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
  2. package/lib/api/impl/beacon/blocks/index.js +41 -22
  3. package/lib/api/impl/beacon/blocks/index.js.map +1 -1
  4. package/lib/api/impl/lodestar/index.d.ts +5 -0
  5. package/lib/api/impl/lodestar/index.d.ts.map +1 -1
  6. package/lib/api/impl/lodestar/index.js +35 -10
  7. package/lib/api/impl/lodestar/index.js.map +1 -1
  8. package/lib/api/impl/node/utils.js +1 -1
  9. package/lib/api/impl/node/utils.js.map +1 -1
  10. package/lib/chain/chain.d.ts +5 -2
  11. package/lib/chain/chain.d.ts.map +1 -1
  12. package/lib/chain/chain.js +32 -16
  13. package/lib/chain/chain.js.map +1 -1
  14. package/lib/chain/errors/dataColumnSidecarError.d.ts +17 -14
  15. package/lib/chain/errors/dataColumnSidecarError.d.ts.map +1 -1
  16. package/lib/chain/errors/dataColumnSidecarError.js +4 -0
  17. package/lib/chain/errors/dataColumnSidecarError.js.map +1 -1
  18. package/lib/chain/forkChoice/index.d.ts +9 -1
  19. package/lib/chain/forkChoice/index.d.ts.map +1 -1
  20. package/lib/chain/forkChoice/index.js +109 -4
  21. package/lib/chain/forkChoice/index.js.map +1 -1
  22. package/lib/chain/interface.d.ts +2 -0
  23. package/lib/chain/interface.d.ts.map +1 -1
  24. package/lib/chain/options.d.ts +0 -2
  25. package/lib/chain/options.d.ts.map +1 -1
  26. package/lib/chain/options.js +0 -1
  27. package/lib/chain/options.js.map +1 -1
  28. package/lib/chain/stateCache/datastore/db.d.ts +12 -0
  29. package/lib/chain/stateCache/datastore/db.d.ts.map +1 -1
  30. package/lib/chain/stateCache/datastore/db.js +70 -0
  31. package/lib/chain/stateCache/datastore/db.js.map +1 -1
  32. package/lib/chain/stateCache/datastore/file.d.ts +1 -0
  33. package/lib/chain/stateCache/datastore/file.d.ts.map +1 -1
  34. package/lib/chain/stateCache/datastore/file.js +7 -0
  35. package/lib/chain/stateCache/datastore/file.js.map +1 -1
  36. package/lib/chain/stateCache/datastore/types.d.ts +1 -0
  37. package/lib/chain/stateCache/datastore/types.d.ts.map +1 -1
  38. package/lib/chain/validation/dataColumnSidecar.d.ts.map +1 -1
  39. package/lib/chain/validation/dataColumnSidecar.js +32 -15
  40. package/lib/chain/validation/dataColumnSidecar.js.map +1 -1
  41. package/lib/index.d.ts +2 -0
  42. package/lib/index.d.ts.map +1 -1
  43. package/lib/index.js +2 -0
  44. package/lib/index.js.map +1 -1
  45. package/lib/metrics/metrics/lodestar.js +1 -1
  46. package/lib/metrics/metrics/lodestar.js.map +1 -1
  47. package/lib/network/core/networkCore.d.ts.map +1 -1
  48. package/lib/network/core/networkCore.js +5 -1
  49. package/lib/network/core/networkCore.js.map +1 -1
  50. package/lib/network/core/networkCoreWorker.js +8 -8
  51. package/lib/network/core/networkCoreWorker.js.map +1 -1
  52. package/lib/network/core/networkCoreWorkerHandler.js +1 -1
  53. package/lib/network/core/networkCoreWorkerHandler.js.map +1 -1
  54. package/lib/network/discv5/worker.js +2 -7
  55. package/lib/network/discv5/worker.js.map +1 -1
  56. package/lib/network/events.d.ts +1 -0
  57. package/lib/network/events.d.ts.map +1 -1
  58. package/lib/network/gossip/encoding.js +1 -1
  59. package/lib/network/gossip/encoding.js.map +1 -1
  60. package/lib/network/gossip/snappy_bun.d.ts +3 -0
  61. package/lib/network/gossip/snappy_bun.d.ts.map +1 -0
  62. package/lib/network/gossip/snappy_bun.js +3 -0
  63. package/lib/network/gossip/snappy_bun.js.map +1 -0
  64. package/lib/network/metadata.d.ts +1 -1
  65. package/lib/network/metadata.d.ts.map +1 -1
  66. package/lib/network/metadata.js +1 -0
  67. package/lib/network/metadata.js.map +1 -1
  68. package/lib/network/options.d.ts +0 -1
  69. package/lib/network/options.d.ts.map +1 -1
  70. package/lib/network/options.js.map +1 -1
  71. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  72. package/lib/network/processor/gossipHandlers.js +6 -2
  73. package/lib/network/processor/gossipHandlers.js.map +1 -1
  74. package/lib/network/reqresp/ReqRespBeaconNode.d.ts.map +1 -1
  75. package/lib/network/reqresp/ReqRespBeaconNode.js +3 -1
  76. package/lib/network/reqresp/ReqRespBeaconNode.js.map +1 -1
  77. package/lib/network/reqresp/handlers/beaconBlocksByRange.d.ts +2 -1
  78. package/lib/network/reqresp/handlers/beaconBlocksByRange.d.ts.map +1 -1
  79. package/lib/network/reqresp/handlers/beaconBlocksByRange.js +14 -3
  80. package/lib/network/reqresp/handlers/beaconBlocksByRange.js.map +1 -1
  81. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.d.ts +2 -1
  82. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.d.ts.map +1 -1
  83. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.js +9 -1
  84. package/lib/network/reqresp/handlers/dataColumnSidecarsByRange.js.map +1 -1
  85. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.d.ts +2 -1
  86. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.d.ts.map +1 -1
  87. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js +9 -1
  88. package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js.map +1 -1
  89. package/lib/network/reqresp/handlers/index.js +6 -6
  90. package/lib/network/reqresp/handlers/index.js.map +1 -1
  91. package/lib/network/reqresp/types.d.ts +1 -0
  92. package/lib/network/reqresp/types.d.ts.map +1 -1
  93. package/lib/node/nodejs.d.ts +2 -1
  94. package/lib/node/nodejs.d.ts.map +1 -1
  95. package/lib/node/nodejs.js +2 -1
  96. package/lib/node/nodejs.js.map +1 -1
  97. package/lib/sync/range/range.d.ts.map +1 -1
  98. package/lib/sync/range/range.js +2 -1
  99. package/lib/sync/range/range.js.map +1 -1
  100. package/lib/sync/utils/downloadByRange.d.ts +58 -13
  101. package/lib/sync/utils/downloadByRange.d.ts.map +1 -1
  102. package/lib/sync/utils/downloadByRange.js +201 -82
  103. package/lib/sync/utils/downloadByRange.js.map +1 -1
  104. package/lib/sync/utils/remoteSyncType.d.ts +2 -1
  105. package/lib/sync/utils/remoteSyncType.d.ts.map +1 -1
  106. package/lib/sync/utils/remoteSyncType.js +19 -4
  107. package/lib/sync/utils/remoteSyncType.js.map +1 -1
  108. package/lib/util/blobs.d.ts +1 -1
  109. package/lib/util/blobs.d.ts.map +1 -1
  110. package/lib/util/blobs.js +53 -20
  111. package/lib/util/blobs.js.map +1 -1
  112. package/lib/util/profile.d.ts +6 -4
  113. package/lib/util/profile.d.ts.map +1 -1
  114. package/lib/util/profile.js +40 -3
  115. package/lib/util/profile.js.map +1 -1
  116. package/lib/util/sszBytes.d.ts +2 -0
  117. package/lib/util/sszBytes.d.ts.map +1 -1
  118. package/lib/util/sszBytes.js +25 -0
  119. package/lib/util/sszBytes.js.map +1 -1
  120. package/package.json +31 -24
  121. package/src/api/impl/beacon/blocks/index.ts +47 -25
  122. package/src/api/impl/lodestar/index.ts +42 -10
  123. package/src/api/impl/node/utils.ts +1 -1
  124. package/src/chain/chain.ts +48 -23
  125. package/src/chain/errors/dataColumnSidecarError.ts +20 -14
  126. package/src/chain/forkChoice/index.ts +178 -2
  127. package/src/chain/interface.ts +2 -0
  128. package/src/chain/options.ts +0 -3
  129. package/src/chain/stateCache/datastore/db.ts +89 -1
  130. package/src/chain/stateCache/datastore/file.ts +8 -0
  131. package/src/chain/stateCache/datastore/types.ts +1 -0
  132. package/src/chain/validation/dataColumnSidecar.ts +34 -16
  133. package/src/index.ts +2 -0
  134. package/src/metrics/metrics/lodestar.ts +1 -1
  135. package/src/network/core/networkCore.ts +5 -1
  136. package/src/network/core/networkCoreWorker.ts +9 -9
  137. package/src/network/core/networkCoreWorkerHandler.ts +1 -1
  138. package/src/network/discv5/worker.ts +2 -7
  139. package/src/network/events.ts +1 -1
  140. package/src/network/gossip/encoding.ts +1 -1
  141. package/src/network/gossip/snappy_bun.ts +2 -0
  142. package/src/network/metadata.ts +3 -1
  143. package/src/network/options.ts +0 -1
  144. package/src/network/processor/gossipHandlers.ts +6 -1
  145. package/src/network/reqresp/ReqRespBeaconNode.ts +3 -1
  146. package/src/network/reqresp/handlers/beaconBlocksByRange.ts +18 -3
  147. package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts +13 -1
  148. package/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts +13 -1
  149. package/src/network/reqresp/handlers/index.ts +6 -6
  150. package/src/network/reqresp/types.ts +1 -0
  151. package/src/node/nodejs.ts +3 -0
  152. package/src/sync/range/range.ts +2 -1
  153. package/src/sync/utils/downloadByRange.ts +259 -103
  154. package/src/sync/utils/remoteSyncType.ts +23 -4
  155. package/src/util/blobs.ts +64 -20
  156. package/src/util/profile.ts +45 -3
  157. package/src/util/sszBytes.ts +30 -0
@@ -1,5 +1,3 @@
1
- import fs from "node:fs";
2
- import path from "node:path";
3
1
  import {Tree} from "@chainsafe/persistent-merkle-tree";
4
2
  import {routes} from "@lodestar/api";
5
3
  import {ApplicationMethods} from "@lodestar/api/server";
@@ -8,13 +6,16 @@ import {Repository} from "@lodestar/db";
8
6
  import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
9
7
  import {BeaconStateCapella, getLatestWeakSubjectivityCheckpointEpoch, loadState} from "@lodestar/state-transition";
10
8
  import {ssz} from "@lodestar/types";
11
- import {toHex, toRootHex} from "@lodestar/utils";
9
+ import {Checkpoint} from "@lodestar/types/phase0";
10
+ import {fromHex, toHex, toRootHex} from "@lodestar/utils";
12
11
  import {BeaconChain} from "../../../chain/index.js";
13
12
  import {QueuedStateRegenerator, RegenRequest} from "../../../chain/regen/index.js";
14
13
  import {IBeaconDb} from "../../../db/interface.js";
15
14
  import {GossipType} from "../../../network/index.js";
16
- import {profileNodeJS, writeHeapSnapshot} from "../../../util/profile.js";
15
+ import {getStateSlotFromBytes} from "../../../util/multifork.js";
16
+ import {ProfileThread, profileThread, writeHeapSnapshot} from "../../../util/profile.js";
17
17
  import {getStateResponseWithRegen} from "../beacon/state/utils.js";
18
+ import {ApiError} from "../errors.js";
18
19
  import {ApiModules} from "../types.js";
19
20
 
20
21
  export function getLodestarApi({
@@ -26,7 +27,9 @@ export function getLodestarApi({
26
27
  }: Pick<ApiModules, "chain" | "config" | "db" | "network" | "sync">): ApplicationMethods<routes.lodestar.Endpoints> {
27
28
  let writingHeapdump = false;
28
29
  let writingProfile = false;
29
- const defaultProfileMs = SLOTS_PER_EPOCH * config.SLOT_DURATION_MS;
30
+ // for NodeJS, profile the whole epoch
31
+ // for Bun, profile 1 slot. Otherwise it will either crash the app, and/or inspector cannot render the profile
32
+ const defaultProfileMs = globalThis.Bun ? config.SLOT_DURATION_MS : SLOTS_PER_EPOCH * config.SLOT_DURATION_MS;
30
33
 
31
34
  return {
32
35
  async writeHeapdump({thread = "main", dirpath = "."}) {
@@ -63,7 +66,6 @@ export function getLodestarApi({
63
66
 
64
67
  try {
65
68
  let filepath: string;
66
- let profile: string;
67
69
  switch (thread) {
68
70
  case "network":
69
71
  filepath = await network.writeNetworkThreadProfile(duration, dirpath);
@@ -73,12 +75,10 @@ export function getLodestarApi({
73
75
  break;
74
76
  default:
75
77
  // main thread
76
- profile = await profileNodeJS(duration);
77
- filepath = path.join(dirpath, `main_thread_${new Date().toISOString()}.cpuprofile`);
78
- fs.writeFileSync(filepath, profile);
78
+ filepath = await profileThread(ProfileThread.MAIN, duration, dirpath);
79
79
  break;
80
80
  }
81
- return {data: {filepath}};
81
+ return {data: {result: filepath}};
82
82
  } finally {
83
83
  writingProfile = false;
84
84
  }
@@ -218,6 +218,26 @@ export function getLodestarApi({
218
218
  meta: {executionOptimistic, finalized, version: fork},
219
219
  };
220
220
  },
221
+
222
+ // the optional checkpoint is in root:epoch format
223
+ async getPersistedCheckpointState({checkpointId}) {
224
+ const checkpoint = checkpointId ? getCheckpointFromArg(checkpointId) : undefined;
225
+ const stateBytes = await chain.getPersistedCheckpointState(checkpoint);
226
+ if (stateBytes === null) {
227
+ throw new ApiError(
228
+ 404,
229
+ checkpointId ? `Checkpoint state not found for id ${checkpointId}` : "Latest safe checkpoint state not found"
230
+ );
231
+ }
232
+
233
+ const slot = getStateSlotFromBytes(stateBytes);
234
+ return {
235
+ data: stateBytes,
236
+ meta: {
237
+ version: config.getForkName(slot),
238
+ },
239
+ };
240
+ },
221
241
  };
222
242
  }
223
243
 
@@ -247,6 +267,18 @@ function regenRequestToJson(config: ChainForkConfig, regenRequest: RegenRequest)
247
267
  }
248
268
  }
249
269
 
270
+ const CHECKPOINT_REGEX = /^(?:0x)?([0-9a-f]{64}):([0-9]+)$/;
271
+ /**
272
+ * Extract a checkpoint from a string in the format `rootHex:epoch`.
273
+ */
274
+ export function getCheckpointFromArg(checkpointStr: string): Checkpoint {
275
+ const match = CHECKPOINT_REGEX.exec(checkpointStr.toLowerCase());
276
+ if (!match) {
277
+ throw new ApiError(400, `Could not parse checkpoint string: ${checkpointStr}`);
278
+ }
279
+ return {root: fromHex(match[1]), epoch: parseInt(match[2])};
280
+ }
281
+
250
282
  function stringifyKeys(keys: (Uint8Array | number | string)[]): string[] {
251
283
  return keys.map((key) => {
252
284
  if (key instanceof Uint8Array) {
@@ -10,7 +10,7 @@ export function formatNodePeer(peerIdStr: string, connections: Connection[]): ro
10
10
  return {
11
11
  peerId: conn ? conn.remotePeer.toString() : peerIdStr,
12
12
  // TODO: figure out how to get enr of peer
13
- enr: "",
13
+ enr: null,
14
14
  lastSeenP2pAddress: conn ? conn.remoteAddr.toString() : "",
15
15
  direction: conn ? (conn.direction as routes.node.PeerDirection) : null,
16
16
  state: conn ? getPeerState(conn.status) : "disconnected",
@@ -37,6 +37,7 @@ import {
37
37
  ValidatorIndex,
38
38
  Wei,
39
39
  isBlindedBeaconBlock,
40
+ phase0,
40
41
  } from "@lodestar/types";
41
42
  import {Logger, fromHex, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
42
43
  import {ProcessShutdownCallback} from "@lodestar/validator";
@@ -95,11 +96,13 @@ import {SeenBlockAttesters} from "./seenCache/seenBlockAttesters.js";
95
96
  import {SeenBlockInput} from "./seenCache/seenGossipBlockInput.js";
96
97
  import {ShufflingCache} from "./shufflingCache.js";
97
98
  import {BlockStateCacheImpl} from "./stateCache/blockStateCacheImpl.js";
98
- import {DbCPStateDatastore} from "./stateCache/datastore/db.js";
99
+ import {DbCPStateDatastore, checkpointToDatastoreKey} from "./stateCache/datastore/db.js";
99
100
  import {FileCPStateDatastore} from "./stateCache/datastore/file.js";
101
+ import {CPStateDatastore} from "./stateCache/datastore/types.js";
100
102
  import {FIFOBlockStateCache} from "./stateCache/fifoBlockStateCache.js";
101
103
  import {InMemoryCheckpointStateCache} from "./stateCache/inMemoryCheckpointsCache.js";
102
104
  import {PersistentCheckpointStateCache} from "./stateCache/persistentCheckpointsCache.js";
105
+ import {CheckpointStateCache} from "./stateCache/types.js";
103
106
  import {ValidatorMonitor} from "./validatorMonitor.js";
104
107
 
105
108
  /**
@@ -181,6 +184,8 @@ export class BeaconChain implements IBeaconChain {
181
184
 
182
185
  protected readonly blockProcessor: BlockProcessor;
183
186
  protected readonly db: IBeaconDb;
187
+ // this is only available if nHistoricalStates is enabled
188
+ private readonly cpStateDatastore?: CPStateDatastore;
184
189
  private abortController = new AbortController();
185
190
  private processShutdownCallback: ProcessShutdownCallback;
186
191
  private _earliestAvailableSlot: Slot;
@@ -210,6 +215,7 @@ export class BeaconChain implements IBeaconChain {
210
215
  metrics,
211
216
  validatorMonitor,
212
217
  anchorState,
218
+ isAnchorStateFinalized,
213
219
  eth1,
214
220
  executionEngine,
215
221
  executionBuilder,
@@ -226,6 +232,7 @@ export class BeaconChain implements IBeaconChain {
226
232
  metrics: Metrics | null;
227
233
  validatorMonitor: ValidatorMonitor | null;
228
234
  anchorState: BeaconStateAllForks;
235
+ isAnchorStateFinalized: boolean;
229
236
  eth1: IEth1ForBlockProduction;
230
237
  executionEngine: IExecutionEngine;
231
238
  executionBuilder?: IExecutionBuilder;
@@ -264,8 +271,7 @@ export class BeaconChain implements IBeaconChain {
264
271
  this.seenAttestationDatas = new SeenAttestationDatas(metrics, this.opts?.attDataCacheSlotDistance);
265
272
 
266
273
  const nodeId = computeNodeIdFromPrivateKey(privateKey);
267
- const initialCustodyGroupCount =
268
- opts.initialCustodyGroupCount ?? (opts.supernode ? config.NUMBER_OF_CUSTODY_GROUPS : config.CUSTODY_REQUIREMENT);
274
+ const initialCustodyGroupCount = opts.initialCustodyGroupCount ?? config.CUSTODY_REQUIREMENT;
269
275
  this.metrics?.peerDas.targetCustodyGroupCount.set(initialCustodyGroupCount);
270
276
  this.custodyConfig = new CustodyConfig({
271
277
  nodeId,
@@ -326,23 +332,26 @@ export class BeaconChain implements IBeaconChain {
326
332
  this.bufferPool = this.opts.nHistoricalStates
327
333
  ? new BufferPool(anchorState.type.tree_serializedSize(anchorState.node), metrics)
328
334
  : null;
329
- const checkpointStateCache = this.opts.nHistoricalStates
330
- ? new PersistentCheckpointStateCache(
331
- {
332
- metrics,
333
- logger,
334
- clock,
335
- blockStateCache,
336
- bufferPool: this.bufferPool,
337
- datastore: fileDataStore
338
- ? // debug option if we want to investigate any issues with the DB
339
- new FileCPStateDatastore(dataDir)
340
- : // production option
341
- new DbCPStateDatastore(this.db),
342
- },
343
- this.opts
344
- )
345
- : new InMemoryCheckpointStateCache({metrics});
335
+
336
+ let checkpointStateCache: CheckpointStateCache;
337
+ this.cpStateDatastore = undefined;
338
+ if (this.opts.nHistoricalStates) {
339
+ this.cpStateDatastore = fileDataStore ? new FileCPStateDatastore(dataDir) : new DbCPStateDatastore(this.db);
340
+ checkpointStateCache = new PersistentCheckpointStateCache(
341
+ {
342
+ metrics,
343
+ logger,
344
+ clock,
345
+ blockStateCache,
346
+ bufferPool: this.bufferPool,
347
+ datastore: this.cpStateDatastore,
348
+ },
349
+ this.opts
350
+ );
351
+ } else {
352
+ checkpointStateCache = new InMemoryCheckpointStateCache({metrics});
353
+ }
354
+
346
355
  const {checkpoint} = computeAnchorCheckpoint(config, anchorState);
347
356
  blockStateCache.add(cachedState);
348
357
  blockStateCache.setHeadState(cachedState);
@@ -353,6 +362,7 @@ export class BeaconChain implements IBeaconChain {
353
362
  emitter,
354
363
  clock.currentSlot,
355
364
  cachedState,
365
+ isAnchorStateFinalized,
356
366
  opts,
357
367
  this.justifiedBalancesGetter.bind(this),
358
368
  metrics,
@@ -604,6 +614,20 @@ export class BeaconChain implements IBeaconChain {
604
614
  return data && {state: data, executionOptimistic: false, finalized: true};
605
615
  }
606
616
 
617
+ async getPersistedCheckpointState(checkpoint?: phase0.Checkpoint): Promise<Uint8Array | null> {
618
+ if (!this.cpStateDatastore) {
619
+ throw new Error("n-historical-state flag is not enabled");
620
+ }
621
+
622
+ if (checkpoint == null) {
623
+ // return the last safe checkpoint state by default
624
+ return this.cpStateDatastore.readLatestSafe();
625
+ }
626
+
627
+ const persistedKey = checkpointToDatastoreKey(checkpoint);
628
+ return this.cpStateDatastore.read(persistedKey);
629
+ }
630
+
607
631
  getStateByCheckpoint(
608
632
  checkpoint: CheckpointWithHex
609
633
  ): {state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null {
@@ -1216,9 +1240,10 @@ export class BeaconChain implements IBeaconChain {
1216
1240
  }
1217
1241
 
1218
1242
  private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithHex): Promise<void> {
1219
- if (this.opts.supernode) {
1220
- // Disable dynamic custody updates for supernodes since they must maintain custody
1221
- // of all custody groups regardless of validator effective balances
1243
+ if (this.custodyConfig.targetCustodyGroupCount === this.config.NUMBER_OF_CUSTODY_GROUPS) {
1244
+ // Custody requirements can only be increased, we can disable dynamic custody updates
1245
+ // if the node already maintains custody of all custody groups in case it is configured
1246
+ // as a supernode or has validators attached with a total effective balance of at least 4096 ETH.
1222
1247
  return;
1223
1248
  }
1224
1249
 
@@ -12,10 +12,14 @@ export enum DataColumnSidecarErrorCode {
12
12
 
13
13
  // Validation errors when validating against an existing block
14
14
 
15
+ /** Block and sidecars header root mismatch */
16
+ INCORRECT_HEADER_ROOT = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_HEADER_ROOT",
15
17
  /** Block and sidecars data column count mismatch */
16
18
  INCORRECT_SIDECAR_COUNT = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_SIDECAR_COUNT",
17
19
  /** Sidecar doesn't match block */
18
20
  INCORRECT_BLOCK = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_BLOCK",
21
+ /** Sidecar cell count not as expected */
22
+ INCORRECT_CELL_COUNT = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_CELL_COUNT",
19
23
  /** Sidecar kzg proof count not as expected */
20
24
  INCORRECT_KZG_COMMITMENTS_COUNT = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_KZG_COMMITMENTS_COUNT",
21
25
  /** Sidecar kzg proof count not as expected */
@@ -35,49 +39,51 @@ export enum DataColumnSidecarErrorCode {
35
39
  }
36
40
 
37
41
  export type DataColumnSidecarErrorType =
38
- | {code: DataColumnSidecarErrorCode.INVALID_INDEX; slot: Slot; columnIdx: number}
39
- | {code: DataColumnSidecarErrorCode.NO_COMMITMENTS; slot: Slot; columnIdx: number}
42
+ | {code: DataColumnSidecarErrorCode.INVALID_INDEX; slot: Slot; columnIndex: number}
43
+ | {code: DataColumnSidecarErrorCode.NO_COMMITMENTS; slot: Slot; columnIndex: number}
40
44
  | {
41
45
  code: DataColumnSidecarErrorCode.MISMATCHED_LENGTHS;
42
46
  columnLength: number;
43
47
  commitmentsLength: number;
44
48
  proofsLength: number;
45
49
  }
46
- | {code: DataColumnSidecarErrorCode.INVALID_SUBNET; columnIdx: number; gossipSubnet: SubnetID}
50
+ | {code: DataColumnSidecarErrorCode.INVALID_SUBNET; columnIndex: number; gossipSubnet: SubnetID}
47
51
  | {
48
52
  code: DataColumnSidecarErrorCode.TOO_MANY_KZG_COMMITMENTS;
49
53
  slot: number;
50
- columnIdx: number;
54
+ columnIndex: number;
51
55
  count: number;
52
56
  limit: number;
53
57
  }
54
- | {code: DataColumnSidecarErrorCode.ALREADY_KNOWN; columnIdx: number; slot: Slot}
58
+ | {code: DataColumnSidecarErrorCode.ALREADY_KNOWN; columnIndex: number; slot: Slot}
55
59
  | {code: DataColumnSidecarErrorCode.FUTURE_SLOT; blockSlot: Slot; currentSlot: Slot}
56
60
  | {code: DataColumnSidecarErrorCode.WOULD_REVERT_FINALIZED_SLOT; blockSlot: Slot; finalizedSlot: Slot}
57
61
  | {code: DataColumnSidecarErrorCode.PARENT_UNKNOWN; parentRoot: RootHex}
58
62
  | {code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID}
59
63
  | {code: DataColumnSidecarErrorCode.NOT_LATER_THAN_PARENT; parentSlot: Slot; slot: Slot}
60
- | {code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID; slot: Slot; columnIdx: number}
61
- | {code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF; slot: Slot; columnIdx: number}
64
+ | {code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID; slot: Slot; columnIndex: number}
65
+ | {code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF; slot: Slot; columnIndex: number}
62
66
  | {code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_COUNT; slot: number; expected: number; actual: number}
63
67
  | {
64
68
  code: DataColumnSidecarErrorCode.INCORRECT_BLOCK;
65
69
  slot: number;
66
- columnIdx: number;
70
+ columnIndex: number;
67
71
  expected: string;
68
72
  actual: string;
69
73
  }
70
74
  | {
71
- code: DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT;
75
+ code: DataColumnSidecarErrorCode.INCORRECT_HEADER_ROOT;
72
76
  slot: number;
73
- columnIdx: number;
74
- expected: number;
75
- actual: number;
77
+ expected: string;
78
+ actual: string;
76
79
  }
77
80
  | {
78
- code: DataColumnSidecarErrorCode.INCORRECT_KZG_PROOF_COUNT;
81
+ code:
82
+ | DataColumnSidecarErrorCode.INCORRECT_CELL_COUNT
83
+ | DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT
84
+ | DataColumnSidecarErrorCode.INCORRECT_KZG_PROOF_COUNT;
79
85
  slot: number;
80
- columnIdx: number;
86
+ columnIndex: number;
81
87
  expected: number;
82
88
  actual: number;
83
89
  }
@@ -5,17 +5,22 @@ import {
5
5
  ForkChoiceStore,
6
6
  JustifiedBalancesGetter,
7
7
  ProtoArray,
8
+ ProtoBlock,
8
9
  ForkChoiceOpts as RawForkChoiceOpts,
9
10
  } from "@lodestar/fork-choice";
11
+ import {ZERO_HASH_HEX} from "@lodestar/params";
10
12
  import {
11
13
  CachedBeaconStateAllForks,
12
14
  DataAvailabilityStatus,
13
15
  computeAnchorCheckpoint,
16
+ computeEpochAtSlot,
17
+ computeStartSlotAtEpoch,
18
+ getBlockRootAtSlot,
14
19
  getEffectiveBalanceIncrementsZeroInactive,
15
20
  isExecutionStateType,
16
21
  isMergeTransitionComplete,
17
22
  } from "@lodestar/state-transition";
18
- import {Slot} from "@lodestar/types";
23
+ import {Slot, ssz} from "@lodestar/types";
19
24
  import {Logger, toRootHex} from "@lodestar/utils";
20
25
  import {GENESIS_SLOT} from "../../constants/index.js";
21
26
  import {Metrics} from "../../metrics/index.js";
@@ -35,6 +40,43 @@ export enum ForkchoiceCaller {
35
40
  * Fork Choice extended with a ChainEventEmitter
36
41
  */
37
42
  export function initializeForkChoice(
43
+ config: ChainForkConfig,
44
+ emitter: ChainEventEmitter,
45
+ currentSlot: Slot,
46
+ state: CachedBeaconStateAllForks,
47
+ isFinalizedState: boolean,
48
+ opts: ForkChoiceOpts,
49
+ justifiedBalancesGetter: JustifiedBalancesGetter,
50
+ metrics: Metrics | null,
51
+ logger?: Logger
52
+ ): ForkChoice {
53
+ return isFinalizedState
54
+ ? initializeForkChoiceFromFinalizedState(
55
+ config,
56
+ emitter,
57
+ currentSlot,
58
+ state,
59
+ opts,
60
+ justifiedBalancesGetter,
61
+ metrics,
62
+ logger
63
+ )
64
+ : initializeForkChoiceFromUnfinalizedState(
65
+ config,
66
+ emitter,
67
+ currentSlot,
68
+ state,
69
+ opts,
70
+ justifiedBalancesGetter,
71
+ metrics,
72
+ logger
73
+ );
74
+ }
75
+
76
+ /**
77
+ * Initialize forkchoice from a finalized state.
78
+ */
79
+ export function initializeForkChoiceFromFinalizedState(
38
80
  config: ChainForkConfig,
39
81
  emitter: ChainEventEmitter,
40
82
  currentSlot: Slot,
@@ -82,7 +124,7 @@ export function initializeForkChoice(
82
124
  parentRoot: toRootHex(blockHeader.parentRoot),
83
125
  stateRoot: toRootHex(blockHeader.stateRoot),
84
126
  blockRoot: toRootHex(checkpoint.root),
85
- timeliness: true, // Optimisitcally assume is timely
127
+ timeliness: true, // Optimistically assume is timely
86
128
 
87
129
  justifiedEpoch: justifiedCheckpoint.epoch,
88
130
  justifiedRoot: toRootHex(justifiedCheckpoint.root),
@@ -105,6 +147,140 @@ export function initializeForkChoice(
105
147
  },
106
148
  currentSlot
107
149
  ),
150
+ state.validators.length,
151
+ metrics,
152
+ opts,
153
+ logger
154
+ );
155
+ }
156
+
157
+ /**
158
+ * Initialize forkchoice from an unfinalized state.
159
+ */
160
+ export function initializeForkChoiceFromUnfinalizedState(
161
+ config: ChainForkConfig,
162
+ emitter: ChainEventEmitter,
163
+ currentSlot: Slot,
164
+ unfinalizedState: CachedBeaconStateAllForks,
165
+ opts: ForkChoiceOpts,
166
+ justifiedBalancesGetter: JustifiedBalancesGetter,
167
+ metrics: Metrics | null,
168
+ logger?: Logger
169
+ ): ForkChoice {
170
+ const {blockHeader} = computeAnchorCheckpoint(config, unfinalizedState);
171
+ const finalizedCheckpoint = unfinalizedState.finalizedCheckpoint.toValue();
172
+ const justifiedCheckpoint = unfinalizedState.currentJustifiedCheckpoint.toValue();
173
+ const headRoot = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader));
174
+
175
+ const logCtx = {
176
+ currentSlot: currentSlot,
177
+ stateSlot: unfinalizedState.slot,
178
+ headSlot: blockHeader.slot,
179
+ headRoot: headRoot,
180
+ finalizedEpoch: finalizedCheckpoint.epoch,
181
+ finalizedRoot: toRootHex(finalizedCheckpoint.root),
182
+ justifiedEpoch: justifiedCheckpoint.epoch,
183
+ justifiedRoot: toRootHex(justifiedCheckpoint.root),
184
+ };
185
+ logger?.warn("Initializing fork choice from unfinalized state", logCtx);
186
+
187
+ // this is not the justified state, but there is no other ways to get justified balances
188
+ const justifiedBalances = getEffectiveBalanceIncrementsZeroInactive(unfinalizedState);
189
+ const store = new ForkChoiceStore(
190
+ currentSlot,
191
+ justifiedCheckpoint,
192
+ finalizedCheckpoint,
193
+ justifiedBalances,
194
+ justifiedBalancesGetter,
195
+ {
196
+ onJustified: (cp) => emitter.emit(ChainEvent.forkChoiceJustified, cp),
197
+ onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp),
198
+ }
199
+ );
200
+
201
+ // this is the same to the finalized state
202
+ const headBlock: ProtoBlock = {
203
+ slot: blockHeader.slot,
204
+ parentRoot: toRootHex(blockHeader.parentRoot),
205
+ stateRoot: toRootHex(blockHeader.stateRoot),
206
+ blockRoot: headRoot,
207
+ targetRoot: headRoot,
208
+ timeliness: true, // Optimistically assume is timely
209
+
210
+ justifiedEpoch: justifiedCheckpoint.epoch,
211
+ justifiedRoot: toRootHex(justifiedCheckpoint.root),
212
+ finalizedEpoch: finalizedCheckpoint.epoch,
213
+ finalizedRoot: toRootHex(finalizedCheckpoint.root),
214
+ unrealizedJustifiedEpoch: justifiedCheckpoint.epoch,
215
+ unrealizedJustifiedRoot: toRootHex(justifiedCheckpoint.root),
216
+ unrealizedFinalizedEpoch: finalizedCheckpoint.epoch,
217
+ unrealizedFinalizedRoot: toRootHex(finalizedCheckpoint.root),
218
+
219
+ ...(isExecutionStateType(unfinalizedState) && isMergeTransitionComplete(unfinalizedState)
220
+ ? {
221
+ executionPayloadBlockHash: toRootHex(unfinalizedState.latestExecutionPayloadHeader.blockHash),
222
+ executionPayloadNumber: unfinalizedState.latestExecutionPayloadHeader.blockNumber,
223
+ executionStatus: blockHeader.slot === GENESIS_SLOT ? ExecutionStatus.Valid : ExecutionStatus.Syncing,
224
+ }
225
+ : {executionPayloadBlockHash: null, executionStatus: ExecutionStatus.PreMerge}),
226
+
227
+ dataAvailabilityStatus: DataAvailabilityStatus.PreData,
228
+ };
229
+
230
+ const parentSlot = blockHeader.slot - 1;
231
+ const parentEpoch = computeEpochAtSlot(parentSlot);
232
+ // parent of head block
233
+ const parentBlock: ProtoBlock = {
234
+ ...headBlock,
235
+ slot: parentSlot,
236
+ // link this to the dummy justified block
237
+ parentRoot: toRootHex(justifiedCheckpoint.root),
238
+ // dummy data, we're not able to regen state before headBlock
239
+ stateRoot: ZERO_HASH_HEX,
240
+ blockRoot: headBlock.parentRoot,
241
+ targetRoot: toRootHex(getBlockRootAtSlot(unfinalizedState, computeStartSlotAtEpoch(parentEpoch))),
242
+ };
243
+
244
+ const justifiedBlock: ProtoBlock = {
245
+ ...headBlock,
246
+ slot: computeStartSlotAtEpoch(justifiedCheckpoint.epoch),
247
+ // link this to the finalized root so that getAncestors can find the finalized block
248
+ parentRoot: toRootHex(finalizedCheckpoint.root),
249
+ // dummy data, we're not able to regen state before headBlock
250
+ stateRoot: ZERO_HASH_HEX,
251
+ blockRoot: toRootHex(justifiedCheckpoint.root),
252
+ // same to blockRoot
253
+ targetRoot: toRootHex(justifiedCheckpoint.root),
254
+ };
255
+
256
+ const finalizedBlock: ProtoBlock = {
257
+ ...headBlock,
258
+ slot: computeStartSlotAtEpoch(finalizedCheckpoint.epoch),
259
+ // we don't care parent of finalized block
260
+ parentRoot: ZERO_HASH_HEX,
261
+ // dummy data, we're not able to regen state before headBlock
262
+ stateRoot: ZERO_HASH_HEX,
263
+ blockRoot: toRootHex(finalizedCheckpoint.root),
264
+ // same to blockRoot
265
+ targetRoot: toRootHex(finalizedCheckpoint.root),
266
+ };
267
+
268
+ const protoArray = ProtoArray.initialize(finalizedBlock, currentSlot);
269
+ protoArray.onBlock(justifiedBlock, currentSlot);
270
+ protoArray.onBlock(parentBlock, currentSlot);
271
+ protoArray.onBlock(headBlock, currentSlot);
272
+
273
+ logger?.verbose("Initialized protoArray successfully", {...logCtx, length: protoArray.length()});
274
+
275
+ // forkchoiceConstructor is only used for some test cases
276
+ // production code use ForkChoice constructor directly
277
+ const forkchoiceConstructor = opts.forkchoiceConstructor ?? ForkChoice;
278
+
279
+ return new forkchoiceConstructor(
280
+ config,
281
+ store,
282
+ protoArray,
283
+ unfinalizedState.validators.length,
108
284
  metrics,
109
285
  opts,
110
286
  logger
@@ -178,6 +178,8 @@ export interface IBeaconChain {
178
178
  stateRoot: RootHex,
179
179
  opts?: StateGetOpts
180
180
  ): Promise<{state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null>;
181
+ /** Return serialized bytes of a persisted checkpoint state */
182
+ getPersistedCheckpointState(checkpoint?: phase0.Checkpoint): Promise<Uint8Array | null>;
181
183
  /** Returns a cached state by checkpoint */
182
184
  getStateByCheckpoint(
183
185
  checkpoint: CheckpointWithHex
@@ -41,8 +41,6 @@ export type IChainOptions = BlockProcessOpts &
41
41
  maxCachedBlobSidecars?: number;
42
42
  /** Max number of produced block roots (blinded or full) cached for broadcast validations */
43
43
  maxCachedProducedRoots?: number;
44
- /** Subscribe to and custody all data column sidecar subnets */
45
- supernode?: boolean;
46
44
  initialCustodyGroupCount?: number;
47
45
  broadcastValidationStrictness?: string;
48
46
  minSameMessageSignatureSetsToBatch: number;
@@ -118,7 +116,6 @@ export const defaultChainOptions: IChainOptions = {
118
116
  archiveMode: DEFAULT_ARCHIVE_MODE,
119
117
  pruneHistory: false,
120
118
  emitPayloadAttributes: false,
121
- supernode: false,
122
119
  // for gossip block validation, it's unlikely we see a reorg with 32 slots
123
120
  // for attestation validation, having this value ensures we don't have to regen states most of the time
124
121
  maxSkipSlots: 32,