@lodestar/beacon-node 1.40.0-dev.2ae7375100 → 1.40.0-dev.2b1dac30dd
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
- package/lib/api/impl/beacon/blocks/index.js +8 -2
- package/lib/api/impl/beacon/blocks/index.js.map +1 -1
- package/lib/api/impl/lodestar/index.d.ts.map +1 -1
- package/lib/api/impl/lodestar/index.js +14 -0
- package/lib/api/impl/lodestar/index.js.map +1 -1
- package/lib/api/impl/validator/index.d.ts.map +1 -1
- package/lib/api/impl/validator/index.js.map +1 -1
- package/lib/api/rest/base.d.ts.map +1 -1
- package/lib/api/rest/base.js +2 -2
- package/lib/api/rest/base.js.map +1 -1
- package/lib/chain/blocks/blockInput/blockInput.d.ts +28 -0
- package/lib/chain/blocks/blockInput/blockInput.d.ts.map +1 -1
- package/lib/chain/blocks/blockInput/blockInput.js +36 -1
- package/lib/chain/blocks/blockInput/blockInput.js.map +1 -1
- package/lib/chain/blocks/importBlock.js +1 -1
- package/lib/chain/blocks/importBlock.js.map +1 -1
- package/lib/chain/blocks/writeBlockInputToDb.d.ts.map +1 -1
- package/lib/chain/blocks/writeBlockInputToDb.js +8 -0
- package/lib/chain/blocks/writeBlockInputToDb.js.map +1 -1
- package/lib/chain/chain.d.ts +1 -1
- package/lib/chain/chain.d.ts.map +1 -1
- package/lib/chain/chain.js +12 -25
- package/lib/chain/chain.js.map +1 -1
- package/lib/chain/options.d.ts +0 -1
- package/lib/chain/options.d.ts.map +1 -1
- package/lib/chain/options.js +0 -1
- package/lib/chain/options.js.map +1 -1
- package/lib/chain/regen/interface.d.ts +1 -1
- package/lib/chain/regen/queued.d.ts +1 -1
- package/lib/chain/regen/queued.d.ts.map +1 -1
- package/lib/chain/regen/queued.js.map +1 -1
- package/lib/chain/stateCache/index.d.ts +0 -2
- package/lib/chain/stateCache/index.d.ts.map +1 -1
- package/lib/chain/stateCache/index.js +0 -2
- package/lib/chain/stateCache/index.js.map +1 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.d.ts +2 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.d.ts.map +1 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.js +3 -0
- package/lib/chain/stateCache/persistentCheckpointsCache.js.map +1 -1
- package/lib/chain/validation/block.d.ts.map +1 -1
- package/lib/chain/validation/block.js +1 -2
- package/lib/chain/validation/block.js.map +1 -1
- package/lib/network/core/networkCore.d.ts +3 -0
- package/lib/network/core/networkCore.d.ts.map +1 -1
- package/lib/network/core/networkCore.js +9 -0
- package/lib/network/core/networkCore.js.map +1 -1
- package/lib/network/core/networkCoreWorker.js +3 -0
- package/lib/network/core/networkCoreWorker.js.map +1 -1
- package/lib/network/core/networkCoreWorkerHandler.d.ts +3 -0
- package/lib/network/core/networkCoreWorkerHandler.d.ts.map +1 -1
- package/lib/network/core/networkCoreWorkerHandler.js +9 -0
- package/lib/network/core/networkCoreWorkerHandler.js.map +1 -1
- package/lib/network/core/types.d.ts +3 -0
- package/lib/network/core/types.d.ts.map +1 -1
- package/lib/network/gossip/gossipsub.d.ts +34 -0
- package/lib/network/gossip/gossipsub.d.ts.map +1 -1
- package/lib/network/gossip/gossipsub.js +123 -0
- package/lib/network/gossip/gossipsub.js.map +1 -1
- package/lib/network/network.d.ts +3 -0
- package/lib/network/network.d.ts.map +1 -1
- package/lib/network/network.js +9 -0
- package/lib/network/network.js.map +1 -1
- package/lib/network/options.d.ts +6 -0
- package/lib/network/options.d.ts.map +1 -1
- package/lib/network/options.js.map +1 -1
- package/lib/network/peers/peerManager.d.ts.map +1 -1
- package/lib/network/peers/peerManager.js +9 -0
- package/lib/network/peers/peerManager.js.map +1 -1
- package/lib/network/processor/gossipHandlers.js +1 -1
- package/lib/network/processor/gossipHandlers.js.map +1 -1
- package/package.json +16 -16
- package/src/api/impl/beacon/blocks/index.ts +22 -12
- package/src/api/impl/lodestar/index.ts +17 -0
- package/src/api/impl/validator/index.ts +2 -1
- package/src/api/rest/base.ts +4 -4
- package/src/chain/blocks/blockInput/blockInput.ts +45 -2
- package/src/chain/blocks/importBlock.ts +1 -1
- package/src/chain/blocks/writeBlockInputToDb.ts +9 -0
- package/src/chain/chain.ts +17 -29
- package/src/chain/options.ts +0 -2
- package/src/chain/regen/interface.ts +1 -1
- package/src/chain/regen/queued.ts +1 -2
- package/src/chain/stateCache/index.ts +0 -2
- package/src/chain/stateCache/persistentCheckpointsCache.ts +6 -2
- package/src/chain/validation/block.ts +1 -2
- package/src/network/core/networkCore.ts +12 -0
- package/src/network/core/networkCoreWorker.ts +3 -0
- package/src/network/core/networkCoreWorkerHandler.ts +9 -0
- package/src/network/core/types.ts +6 -0
- package/src/network/gossip/gossipsub.ts +147 -1
- package/src/network/network.ts +12 -0
- package/src/network/options.ts +6 -0
- package/src/network/peers/peerManager.ts +11 -0
- package/src/network/processor/gossipHandlers.ts +1 -1
- package/lib/chain/stateCache/blockStateCacheImpl.d.ts +0 -54
- package/lib/chain/stateCache/blockStateCacheImpl.d.ts.map +0 -1
- package/lib/chain/stateCache/blockStateCacheImpl.js +0 -130
- package/lib/chain/stateCache/blockStateCacheImpl.js.map +0 -1
- package/lib/chain/stateCache/inMemoryCheckpointsCache.d.ts +0 -60
- package/lib/chain/stateCache/inMemoryCheckpointsCache.d.ts.map +0 -1
- package/lib/chain/stateCache/inMemoryCheckpointsCache.js +0 -156
- package/lib/chain/stateCache/inMemoryCheckpointsCache.js.map +0 -1
- package/src/chain/stateCache/blockStateCacheImpl.ts +0 -149
- package/src/chain/stateCache/inMemoryCheckpointsCache.ts +0 -192
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {ForkName, ForkPostFulu, ForkPreDeneb, ForkPreGloas} from "@lodestar/params";
|
|
1
|
+
import {ForkName, ForkPostFulu, ForkPreDeneb, ForkPreGloas, NUMBER_OF_COLUMNS} from "@lodestar/params";
|
|
2
2
|
import {BeaconBlockBody, BlobIndex, ColumnIndex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types";
|
|
3
3
|
import {fromHex, prettyBytes, toRootHex, withTimeout} from "@lodestar/utils";
|
|
4
4
|
import {VersionedHashes} from "../../../execution/index.js";
|
|
@@ -561,6 +561,7 @@ type BlockInputColumnsState =
|
|
|
561
561
|
| {
|
|
562
562
|
hasBlock: true;
|
|
563
563
|
hasAllData: true;
|
|
564
|
+
hasComputedAllData: boolean;
|
|
564
565
|
versionedHashes: VersionedHashes;
|
|
565
566
|
block: SignedBeaconBlock<ForkColumnsDA>;
|
|
566
567
|
source: SourceMeta;
|
|
@@ -569,6 +570,7 @@ type BlockInputColumnsState =
|
|
|
569
570
|
| {
|
|
570
571
|
hasBlock: true;
|
|
571
572
|
hasAllData: false;
|
|
573
|
+
hasComputedAllData: false;
|
|
572
574
|
versionedHashes: VersionedHashes;
|
|
573
575
|
block: SignedBeaconBlock<ForkColumnsDA>;
|
|
574
576
|
source: SourceMeta;
|
|
@@ -576,11 +578,13 @@ type BlockInputColumnsState =
|
|
|
576
578
|
| {
|
|
577
579
|
hasBlock: false;
|
|
578
580
|
hasAllData: true;
|
|
581
|
+
hasComputedAllData: boolean;
|
|
579
582
|
versionedHashes: VersionedHashes;
|
|
580
583
|
}
|
|
581
584
|
| {
|
|
582
585
|
hasBlock: false;
|
|
583
586
|
hasAllData: false;
|
|
587
|
+
hasComputedAllData: false;
|
|
584
588
|
versionedHashes: VersionedHashes;
|
|
585
589
|
};
|
|
586
590
|
/**
|
|
@@ -598,6 +602,12 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
|
|
|
598
602
|
private columnsCache = new Map<ColumnIndex, ColumnWithSource>();
|
|
599
603
|
private readonly sampledColumns: ColumnIndex[];
|
|
600
604
|
private readonly custodyColumns: ColumnIndex[];
|
|
605
|
+
/**
|
|
606
|
+
* This promise resolves when all sampled columns are available
|
|
607
|
+
*
|
|
608
|
+
* This is different from `dataPromise` which resolves when all data is available or could become available (e.g. through reconstruction)
|
|
609
|
+
*/
|
|
610
|
+
protected computedDataPromise = createPromise<fulu.DataColumnSidecars>();
|
|
601
611
|
|
|
602
612
|
private constructor(
|
|
603
613
|
init: BlockInputInit,
|
|
@@ -626,6 +636,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
|
|
|
626
636
|
const state = {
|
|
627
637
|
hasBlock: true,
|
|
628
638
|
hasAllData,
|
|
639
|
+
hasComputedAllData: hasAllData,
|
|
629
640
|
versionedHashes: props.block.message.body.blobKzgCommitments.map(kzgCommitmentToVersionedHash),
|
|
630
641
|
block: props.block,
|
|
631
642
|
source: {
|
|
@@ -649,6 +660,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
|
|
|
649
660
|
blockInput.blockPromise.resolve(props.block);
|
|
650
661
|
if (hasAllData) {
|
|
651
662
|
blockInput.dataPromise.resolve([]);
|
|
663
|
+
blockInput.computedDataPromise.resolve([]);
|
|
652
664
|
}
|
|
653
665
|
return blockInput;
|
|
654
666
|
}
|
|
@@ -661,6 +673,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
|
|
|
661
673
|
const state: BlockInputColumnsState = {
|
|
662
674
|
hasBlock: false,
|
|
663
675
|
hasAllData,
|
|
676
|
+
hasComputedAllData: hasAllData as false,
|
|
664
677
|
versionedHashes: props.columnSidecar.kzgCommitments.map(kzgCommitmentToVersionedHash),
|
|
665
678
|
};
|
|
666
679
|
const init: BlockInputInit = {
|
|
@@ -674,6 +687,7 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
|
|
|
674
687
|
const blockInput = new BlockInputColumns(init, state, props.sampledColumns, props.custodyColumns);
|
|
675
688
|
if (hasAllData) {
|
|
676
689
|
blockInput.dataPromise.resolve([]);
|
|
690
|
+
blockInput.computedDataPromise.resolve([]);
|
|
677
691
|
}
|
|
678
692
|
return blockInput;
|
|
679
693
|
}
|
|
@@ -722,11 +736,14 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
|
|
|
722
736
|
const hasAllData =
|
|
723
737
|
(props.block.message.body as BeaconBlockBody<ForkPostFulu & ForkPreGloas>).blobKzgCommitments.length === 0 ||
|
|
724
738
|
this.state.hasAllData;
|
|
739
|
+
const hasComputedAllData =
|
|
740
|
+
props.block.message.body.blobKzgCommitments.length === 0 || this.state.hasComputedAllData;
|
|
725
741
|
|
|
726
742
|
this.state = {
|
|
727
743
|
...this.state,
|
|
728
744
|
hasBlock: true,
|
|
729
745
|
hasAllData,
|
|
746
|
+
hasComputedAllData,
|
|
730
747
|
block: props.block,
|
|
731
748
|
source: {
|
|
732
749
|
source: props.source,
|
|
@@ -774,17 +791,32 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
|
|
|
774
791
|
this.columnsCache.set(columnSidecar.index, {columnSidecar, source, seenTimestampSec, peerIdStr});
|
|
775
792
|
|
|
776
793
|
const sampledColumns = this.getSampledColumns();
|
|
777
|
-
const hasAllData =
|
|
794
|
+
const hasAllData =
|
|
795
|
+
// already hasAllData
|
|
796
|
+
this.state.hasAllData ||
|
|
797
|
+
// has all sampled columns
|
|
798
|
+
sampledColumns.length === this.sampledColumns.length ||
|
|
799
|
+
// has enough columns to reconstruct the rest
|
|
800
|
+
this.columnsCache.size >= NUMBER_OF_COLUMNS / 2;
|
|
801
|
+
|
|
802
|
+
const hasComputedAllData =
|
|
803
|
+
// has all sampled columns
|
|
804
|
+
sampledColumns.length === this.sampledColumns.length;
|
|
778
805
|
|
|
779
806
|
this.state = {
|
|
780
807
|
...this.state,
|
|
781
808
|
hasAllData: hasAllData || this.state.hasAllData,
|
|
809
|
+
hasComputedAllData: hasComputedAllData || this.state.hasComputedAllData,
|
|
782
810
|
timeCompleteSec: hasAllData ? seenTimestampSec : undefined,
|
|
783
811
|
} as BlockInputColumnsState;
|
|
784
812
|
|
|
785
813
|
if (hasAllData && sampledColumns !== null) {
|
|
786
814
|
this.dataPromise.resolve(sampledColumns);
|
|
787
815
|
}
|
|
816
|
+
|
|
817
|
+
if (hasComputedAllData && sampledColumns !== null) {
|
|
818
|
+
this.computedDataPromise.resolve(sampledColumns);
|
|
819
|
+
}
|
|
788
820
|
}
|
|
789
821
|
|
|
790
822
|
hasColumn(columnIndex: number): boolean {
|
|
@@ -859,4 +891,15 @@ export class BlockInputColumns extends AbstractBlockInput<ForkColumnsDA, fulu.Da
|
|
|
859
891
|
versionedHashes: this.state.versionedHashes,
|
|
860
892
|
};
|
|
861
893
|
}
|
|
894
|
+
|
|
895
|
+
hasComputedAllData(): boolean {
|
|
896
|
+
return this.state.hasComputedAllData;
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
waitForComputedAllData(timeout: number, signal?: AbortSignal): Promise<fulu.DataColumnSidecars> {
|
|
900
|
+
if (!this.state.hasComputedAllData) {
|
|
901
|
+
return withTimeout(() => this.computedDataPromise.promise, timeout, signal);
|
|
902
|
+
}
|
|
903
|
+
return Promise.resolve(this.getSampledColumns());
|
|
904
|
+
}
|
|
862
905
|
}
|
|
@@ -30,7 +30,7 @@ import type {BeaconChain} from "../chain.js";
|
|
|
30
30
|
import {ChainEvent, ReorgEventData} from "../emitter.js";
|
|
31
31
|
import {ForkchoiceCaller} from "../forkChoice/index.js";
|
|
32
32
|
import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js";
|
|
33
|
-
import {toCheckpointHex} from "../stateCache/
|
|
33
|
+
import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js";
|
|
34
34
|
import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
|
|
35
35
|
import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
|
|
36
36
|
import {getCheckpointFromState} from "./utils/checkpoint.js";
|
|
@@ -44,6 +44,15 @@ export async function writeBlockInputToDb(this: BeaconChain, blocksInputs: IBloc
|
|
|
44
44
|
|
|
45
45
|
// NOTE: Old data is pruned on archive
|
|
46
46
|
if (isBlockInputColumns(blockInput)) {
|
|
47
|
+
if (!blockInput.hasComputedAllData()) {
|
|
48
|
+
// Supernodes may only have a subset of the data columns by the time the block begins to be imported
|
|
49
|
+
// because full data availability can be assumed after NUMBER_OF_COLUMNS / 2 columns are available.
|
|
50
|
+
// Here, however, all data columns must be fully available/reconstructed before persisting to the DB.
|
|
51
|
+
await blockInput.waitForComputedAllData(BLOB_AVAILABILITY_TIMEOUT).catch(() => {
|
|
52
|
+
this.logger.debug("Failed to wait for computed all data", {slot, blockRoot: blockRootHex});
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
|
|
47
56
|
const {custodyColumns} = this.custodyConfig;
|
|
48
57
|
const blobsLen = (block.message as fulu.BeaconBlock).body.blobKzgCommitments.length;
|
|
49
58
|
let dataColumnsLen: number;
|
package/src/chain/chain.ts
CHANGED
|
@@ -107,12 +107,10 @@ import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js";
|
|
|
107
107
|
import {SeenBlockAttesters} from "./seenCache/seenBlockAttesters.js";
|
|
108
108
|
import {SeenBlockInput} from "./seenCache/seenGossipBlockInput.js";
|
|
109
109
|
import {ShufflingCache} from "./shufflingCache.js";
|
|
110
|
-
import {BlockStateCacheImpl} from "./stateCache/blockStateCacheImpl.js";
|
|
111
110
|
import {DbCPStateDatastore, checkpointToDatastoreKey} from "./stateCache/datastore/db.js";
|
|
112
111
|
import {FileCPStateDatastore} from "./stateCache/datastore/file.js";
|
|
113
112
|
import {CPStateDatastore} from "./stateCache/datastore/types.js";
|
|
114
113
|
import {FIFOBlockStateCache} from "./stateCache/fifoBlockStateCache.js";
|
|
115
|
-
import {InMemoryCheckpointStateCache} from "./stateCache/inMemoryCheckpointsCache.js";
|
|
116
114
|
import {PersistentCheckpointStateCache} from "./stateCache/persistentCheckpointsCache.js";
|
|
117
115
|
import {CheckpointStateCache} from "./stateCache/types.js";
|
|
118
116
|
import {ValidatorMonitor} from "./validatorMonitor.js";
|
|
@@ -142,7 +140,7 @@ export class BeaconChain implements IBeaconChain {
|
|
|
142
140
|
readonly logger: Logger;
|
|
143
141
|
readonly metrics: Metrics | null;
|
|
144
142
|
readonly validatorMonitor: ValidatorMonitor | null;
|
|
145
|
-
readonly bufferPool: BufferPool
|
|
143
|
+
readonly bufferPool: BufferPool;
|
|
146
144
|
|
|
147
145
|
readonly anchorStateLatestBlockSlot: Slot;
|
|
148
146
|
|
|
@@ -339,32 +337,22 @@ export class BeaconChain implements IBeaconChain {
|
|
|
339
337
|
this.index2pubkey = index2pubkey;
|
|
340
338
|
|
|
341
339
|
const fileDataStore = opts.nHistoricalStatesFileDataStore ?? true;
|
|
342
|
-
const blockStateCache = this.opts
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
this.
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
clock,
|
|
359
|
-
blockStateCache,
|
|
360
|
-
bufferPool: this.bufferPool,
|
|
361
|
-
datastore: this.cpStateDatastore,
|
|
362
|
-
},
|
|
363
|
-
this.opts
|
|
364
|
-
);
|
|
365
|
-
} else {
|
|
366
|
-
checkpointStateCache = new InMemoryCheckpointStateCache({metrics});
|
|
367
|
-
}
|
|
340
|
+
const blockStateCache = new FIFOBlockStateCache(this.opts, {metrics});
|
|
341
|
+
this.bufferPool = new BufferPool(anchorState.type.tree_serializedSize(anchorState.node), metrics);
|
|
342
|
+
|
|
343
|
+
this.cpStateDatastore = fileDataStore ? new FileCPStateDatastore(dataDir) : new DbCPStateDatastore(this.db);
|
|
344
|
+
const checkpointStateCache: CheckpointStateCache = new PersistentCheckpointStateCache(
|
|
345
|
+
{
|
|
346
|
+
config,
|
|
347
|
+
metrics,
|
|
348
|
+
logger,
|
|
349
|
+
clock,
|
|
350
|
+
blockStateCache,
|
|
351
|
+
bufferPool: this.bufferPool,
|
|
352
|
+
datastore: this.cpStateDatastore,
|
|
353
|
+
},
|
|
354
|
+
this.opts
|
|
355
|
+
);
|
|
368
356
|
|
|
369
357
|
const {checkpoint} = computeAnchorCheckpoint(config, anchorState);
|
|
370
358
|
blockStateCache.add(anchorState);
|
package/src/chain/options.ts
CHANGED
|
@@ -45,7 +45,6 @@ export type IChainOptions = BlockProcessOpts &
|
|
|
45
45
|
broadcastValidationStrictness?: string;
|
|
46
46
|
minSameMessageSignatureSetsToBatch: number;
|
|
47
47
|
archiveDateEpochs?: number;
|
|
48
|
-
nHistoricalStates?: boolean;
|
|
49
48
|
nHistoricalStatesFileDataStore?: boolean;
|
|
50
49
|
};
|
|
51
50
|
|
|
@@ -119,7 +118,6 @@ export const defaultChainOptions: IChainOptions = {
|
|
|
119
118
|
// batching too much may block the I/O thread so if useWorker=false, suggest this value to be 32
|
|
120
119
|
// since this batch attestation work is designed to work with useWorker=true, make this the lowest value
|
|
121
120
|
minSameMessageSignatureSetsToBatch: 2,
|
|
122
|
-
nHistoricalStates: true,
|
|
123
121
|
// as of Feb 2025, this option turned out to be very useful:
|
|
124
122
|
// - it allows to share a persisted checkpoint state to other nodes
|
|
125
123
|
// - users can prune the persisted checkpoint state files manually to save disc space
|
|
@@ -2,7 +2,7 @@ import {routes} from "@lodestar/api";
|
|
|
2
2
|
import {ProtoBlock} from "@lodestar/fork-choice";
|
|
3
3
|
import {CachedBeaconStateAllForks} from "@lodestar/state-transition";
|
|
4
4
|
import {BeaconBlock, Epoch, RootHex, Slot, phase0} from "@lodestar/types";
|
|
5
|
-
import {CheckpointHex} from "../stateCache/
|
|
5
|
+
import {CheckpointHex} from "../stateCache/types.js";
|
|
6
6
|
|
|
7
7
|
export enum RegenCaller {
|
|
8
8
|
getDuties = "getDuties",
|
|
@@ -5,8 +5,7 @@ import {BeaconBlock, Epoch, RootHex, Slot, phase0} from "@lodestar/types";
|
|
|
5
5
|
import {Logger, toRootHex} from "@lodestar/utils";
|
|
6
6
|
import {Metrics} from "../../metrics/index.js";
|
|
7
7
|
import {JobItemQueue} from "../../util/queue/index.js";
|
|
8
|
-
import {CheckpointHex} from "../stateCache/
|
|
9
|
-
import {BlockStateCache, CheckpointStateCache} from "../stateCache/types.js";
|
|
8
|
+
import {BlockStateCache, CheckpointHex, CheckpointStateCache} from "../stateCache/types.js";
|
|
10
9
|
import {RegenError, RegenErrorCode} from "./errors.js";
|
|
11
10
|
import {
|
|
12
11
|
IStateRegenerator,
|
|
@@ -31,7 +31,7 @@ type PersistentCheckpointStateCacheModules = {
|
|
|
31
31
|
signal?: AbortSignal;
|
|
32
32
|
datastore: CPStateDatastore;
|
|
33
33
|
blockStateCache: BlockStateCache;
|
|
34
|
-
bufferPool?: BufferPool
|
|
34
|
+
bufferPool?: BufferPool;
|
|
35
35
|
};
|
|
36
36
|
|
|
37
37
|
/** checkpoint serialized as a string */
|
|
@@ -119,7 +119,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache {
|
|
|
119
119
|
private readonly maxEpochsOnDisk: number;
|
|
120
120
|
private readonly datastore: CPStateDatastore;
|
|
121
121
|
private readonly blockStateCache: BlockStateCache;
|
|
122
|
-
private readonly bufferPool?: BufferPool
|
|
122
|
+
private readonly bufferPool?: BufferPool;
|
|
123
123
|
|
|
124
124
|
constructor(
|
|
125
125
|
{
|
|
@@ -851,6 +851,10 @@ export function toCheckpointHex(checkpoint: phase0.Checkpoint): CheckpointHex {
|
|
|
851
851
|
};
|
|
852
852
|
}
|
|
853
853
|
|
|
854
|
+
export function toCheckpointKey(cp: CheckpointHex): string {
|
|
855
|
+
return `${cp.rootHex}:${cp.epoch}`;
|
|
856
|
+
}
|
|
857
|
+
|
|
854
858
|
function toCacheKey(cp: CheckpointHex | phase0.Checkpoint): CacheKey {
|
|
855
859
|
if (isCheckpointHex(cp)) {
|
|
856
860
|
return `${cp.rootHex}_${cp.epoch}`;
|
|
@@ -138,11 +138,10 @@ export async function validateGossipBlock(
|
|
|
138
138
|
// in forky condition, make sure to populate ShufflingCache with regened state
|
|
139
139
|
chain.shufflingCache.processState(blockState);
|
|
140
140
|
|
|
141
|
-
// Extra conditions for merge fork blocks
|
|
142
141
|
// [REJECT] The block's execution payload timestamp is correct with respect to the slot
|
|
143
142
|
// -- i.e. execution_payload.timestamp == compute_timestamp_at_slot(state, block.slot).
|
|
144
143
|
if (isForkPostBellatrix(fork) && !isForkPostGloas(fork)) {
|
|
145
|
-
if (!isExecutionBlockBodyType(block.body)) throw Error("Not
|
|
144
|
+
if (!isExecutionBlockBodyType(block.body)) throw Error("Not execution block body type");
|
|
146
145
|
const executionPayload = block.body.executionPayload;
|
|
147
146
|
if (isExecutionStateType(blockState) && isExecutionEnabled(blockState, block)) {
|
|
148
147
|
const expectedTimestamp = computeTimeAtSlot(config, blockSlot, chain.genesisTime);
|
|
@@ -454,6 +454,18 @@ export class NetworkCore implements INetworkCore {
|
|
|
454
454
|
await this.libp2p.hangUp(peerIdFromString(peerIdStr));
|
|
455
455
|
}
|
|
456
456
|
|
|
457
|
+
async addDirectPeer(peer: routes.lodestar.DirectPeer): Promise<string | null> {
|
|
458
|
+
return this.gossip.addDirectPeer(peer);
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
async removeDirectPeer(peerIdStr: PeerIdStr): Promise<boolean> {
|
|
462
|
+
return this.gossip.removeDirectPeer(peerIdStr);
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
async getDirectPeers(): Promise<string[]> {
|
|
466
|
+
return this.gossip.getDirectPeers();
|
|
467
|
+
}
|
|
468
|
+
|
|
457
469
|
private _dumpPeer(peerIdStr: string, connections: Connection[]): routes.lodestar.LodestarNodePeer {
|
|
458
470
|
const peerData = this.peersData.connectedPeers.get(peerIdStr);
|
|
459
471
|
const fork = this.config.getForkName(this.clock.currentSlot);
|
|
@@ -153,6 +153,9 @@ const libp2pWorkerApi: NetworkWorkerApi = {
|
|
|
153
153
|
getConnectedPeerCount: () => core.getConnectedPeerCount(),
|
|
154
154
|
connectToPeer: (peer, multiaddr) => core.connectToPeer(peer, multiaddr),
|
|
155
155
|
disconnectPeer: (peer) => core.disconnectPeer(peer),
|
|
156
|
+
addDirectPeer: (peer) => core.addDirectPeer(peer),
|
|
157
|
+
removeDirectPeer: (peerId) => core.removeDirectPeer(peerId),
|
|
158
|
+
getDirectPeers: () => core.getDirectPeers(),
|
|
156
159
|
dumpPeers: () => core.dumpPeers(),
|
|
157
160
|
dumpPeer: (peerIdStr) => core.dumpPeer(peerIdStr),
|
|
158
161
|
dumpPeerScoreStats: () => core.dumpPeerScoreStats(),
|
|
@@ -247,6 +247,15 @@ export class WorkerNetworkCore implements INetworkCore {
|
|
|
247
247
|
disconnectPeer(peer: PeerIdStr): Promise<void> {
|
|
248
248
|
return this.getApi().disconnectPeer(peer);
|
|
249
249
|
}
|
|
250
|
+
addDirectPeer(peer: routes.lodestar.DirectPeer): Promise<string | null> {
|
|
251
|
+
return this.getApi().addDirectPeer(peer);
|
|
252
|
+
}
|
|
253
|
+
removeDirectPeer(peerId: PeerIdStr): Promise<boolean> {
|
|
254
|
+
return this.getApi().removeDirectPeer(peerId);
|
|
255
|
+
}
|
|
256
|
+
getDirectPeers(): Promise<string[]> {
|
|
257
|
+
return this.getApi().getDirectPeers();
|
|
258
|
+
}
|
|
250
259
|
dumpPeers(): Promise<routes.lodestar.LodestarNodePeer[]> {
|
|
251
260
|
return this.getApi().dumpPeers();
|
|
252
261
|
}
|
|
@@ -30,6 +30,12 @@ export interface INetworkCorePublic {
|
|
|
30
30
|
// Debug
|
|
31
31
|
connectToPeer(peer: PeerIdStr, multiaddr: MultiaddrStr[]): Promise<void>;
|
|
32
32
|
disconnectPeer(peer: PeerIdStr): Promise<void>;
|
|
33
|
+
|
|
34
|
+
// Direct peers management
|
|
35
|
+
addDirectPeer(peer: routes.lodestar.DirectPeer): Promise<string | null>;
|
|
36
|
+
removeDirectPeer(peerId: PeerIdStr): Promise<boolean>;
|
|
37
|
+
getDirectPeers(): Promise<string[]>;
|
|
38
|
+
|
|
33
39
|
dumpPeers(): Promise<routes.lodestar.LodestarNodePeer[]>;
|
|
34
40
|
dumpPeer(peerIdStr: PeerIdStr): Promise<routes.lodestar.LodestarNodePeer | undefined>;
|
|
35
41
|
dumpPeerScoreStats(): Promise<PeerScoreStats>;
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
+
import {peerIdFromString} from "@libp2p/peer-id";
|
|
2
|
+
import {multiaddr} from "@multiformats/multiaddr";
|
|
3
|
+
import {ENR} from "@chainsafe/enr";
|
|
1
4
|
import {GossipSub, GossipsubEvents} from "@chainsafe/libp2p-gossipsub";
|
|
2
5
|
import {MetricsRegister, TopicLabel, TopicStrToLabel} from "@chainsafe/libp2p-gossipsub/metrics";
|
|
3
6
|
import {PeerScoreParams} from "@chainsafe/libp2p-gossipsub/score";
|
|
4
|
-
import {SignaturePolicy, TopicStr} from "@chainsafe/libp2p-gossipsub/types";
|
|
7
|
+
import {AddrInfo, SignaturePolicy, TopicStr} from "@chainsafe/libp2p-gossipsub/types";
|
|
8
|
+
import {routes} from "@lodestar/api";
|
|
5
9
|
import {BeaconConfig, ForkBoundary} from "@lodestar/config";
|
|
6
10
|
import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params";
|
|
7
11
|
import {SubnetID} from "@lodestar/types";
|
|
@@ -55,6 +59,12 @@ export type Eth2GossipsubOpts = {
|
|
|
55
59
|
disableFloodPublish?: boolean;
|
|
56
60
|
skipParamsLog?: boolean;
|
|
57
61
|
disableLightClientServer?: boolean;
|
|
62
|
+
/**
|
|
63
|
+
* Direct peers for GossipSub - these peers maintain permanent mesh connections without GRAFT/PRUNE.
|
|
64
|
+
* Supports multiaddr strings with peer ID (e.g., "/ip4/192.168.1.1/tcp/9000/p2p/16Uiu2HAmKLhW7...")
|
|
65
|
+
* or ENR strings (e.g., "enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOo...")
|
|
66
|
+
*/
|
|
67
|
+
directPeers?: string[];
|
|
58
68
|
};
|
|
59
69
|
|
|
60
70
|
export type ForkBoundaryLabel = string;
|
|
@@ -78,6 +88,7 @@ export class Eth2Gossipsub extends GossipSub {
|
|
|
78
88
|
private readonly logger: Logger;
|
|
79
89
|
private readonly peersData: PeersData;
|
|
80
90
|
private readonly events: NetworkEventBus;
|
|
91
|
+
private readonly libp2p: Libp2p;
|
|
81
92
|
|
|
82
93
|
// Internal caches
|
|
83
94
|
private readonly gossipTopicCache: GossipTopicCache;
|
|
@@ -97,6 +108,9 @@ export class Eth2Gossipsub extends GossipSub {
|
|
|
97
108
|
);
|
|
98
109
|
}
|
|
99
110
|
|
|
111
|
+
// Parse direct peers from multiaddr strings to AddrInfo objects
|
|
112
|
+
const directPeers = parseDirectPeers(opts.directPeers ?? [], logger);
|
|
113
|
+
|
|
100
114
|
// Gossipsub parameters defined here:
|
|
101
115
|
// https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub
|
|
102
116
|
super(modules.libp2p.services.components, {
|
|
@@ -106,6 +120,7 @@ export class Eth2Gossipsub extends GossipSub {
|
|
|
106
120
|
Dlo: gossipsubDLow ?? GOSSIP_D_LOW,
|
|
107
121
|
Dhi: gossipsubDHigh ?? GOSSIP_D_HIGH,
|
|
108
122
|
Dlazy: 6,
|
|
123
|
+
directPeers,
|
|
109
124
|
heartbeatInterval: GOSSIPSUB_HEARTBEAT_INTERVAL,
|
|
110
125
|
fanoutTTL: 60 * 1000,
|
|
111
126
|
mcacheLength: 6,
|
|
@@ -146,6 +161,7 @@ export class Eth2Gossipsub extends GossipSub {
|
|
|
146
161
|
this.logger = logger;
|
|
147
162
|
this.peersData = peersData;
|
|
148
163
|
this.events = events;
|
|
164
|
+
this.libp2p = modules.libp2p;
|
|
149
165
|
this.gossipTopicCache = gossipTopicCache;
|
|
150
166
|
|
|
151
167
|
this.addEventListener("gossipsub:message", this.onGossipsubMessage.bind(this));
|
|
@@ -328,6 +344,64 @@ export class Eth2Gossipsub extends GossipSub {
|
|
|
328
344
|
this.reportMessageValidationResult(data.msgId, data.propagationSource, data.acceptance);
|
|
329
345
|
});
|
|
330
346
|
}
|
|
347
|
+
|
|
348
|
+
/**
|
|
349
|
+
* Add a peer as a direct peer at runtime. Accepts multiaddr with peer ID or ENR string.
|
|
350
|
+
* Direct peers maintain permanent mesh connections without GRAFT/PRUNE negotiation.
|
|
351
|
+
*/
|
|
352
|
+
async addDirectPeer(peerStr: routes.lodestar.DirectPeer): Promise<string | null> {
|
|
353
|
+
const parsed = parseDirectPeers([peerStr], this.logger);
|
|
354
|
+
if (parsed.length === 0) {
|
|
355
|
+
return null;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const {id: peerId, addrs} = parsed[0];
|
|
359
|
+
const peerIdStr = peerId.toString();
|
|
360
|
+
|
|
361
|
+
// Prevent adding self as a direct peer
|
|
362
|
+
if (peerId.equals(this.libp2p.peerId)) {
|
|
363
|
+
this.logger.warn("Cannot add self as a direct peer", {peerId: peerIdStr});
|
|
364
|
+
return null;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// Direct peers need addresses to connect - reject if none provided
|
|
368
|
+
if (addrs.length === 0) {
|
|
369
|
+
this.logger.warn("Cannot add direct peer without addresses", {peerId: peerIdStr});
|
|
370
|
+
return null;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// Add addresses to peer store first so we can connect
|
|
374
|
+
try {
|
|
375
|
+
await this.libp2p.peerStore.merge(peerId, {multiaddrs: addrs});
|
|
376
|
+
} catch (e) {
|
|
377
|
+
this.logger.warn("Failed to add direct peer addresses to peer store", {peerId: peerIdStr}, e as Error);
|
|
378
|
+
return null;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
// Add to direct peers set only after addresses are stored
|
|
382
|
+
this.direct.add(peerIdStr);
|
|
383
|
+
|
|
384
|
+
this.logger.info("Added direct peer via API", {peerId: peerIdStr});
|
|
385
|
+
return peerIdStr;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* Remove a peer from direct peers.
|
|
390
|
+
*/
|
|
391
|
+
removeDirectPeer(peerIdStr: string): boolean {
|
|
392
|
+
const removed = this.direct.delete(peerIdStr);
|
|
393
|
+
if (removed) {
|
|
394
|
+
this.logger.info("Removed direct peer via API", {peerId: peerIdStr});
|
|
395
|
+
}
|
|
396
|
+
return removed;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Get list of current direct peer IDs.
|
|
401
|
+
*/
|
|
402
|
+
getDirectPeers(): string[] {
|
|
403
|
+
return Array.from(this.direct);
|
|
404
|
+
}
|
|
331
405
|
}
|
|
332
406
|
|
|
333
407
|
/**
|
|
@@ -381,3 +455,75 @@ function getForkBoundaryLabel(boundary: ForkBoundary): ForkBoundaryLabel {
|
|
|
381
455
|
|
|
382
456
|
return label;
|
|
383
457
|
}
|
|
458
|
+
|
|
459
|
+
/**
|
|
460
|
+
* Parse direct peer strings into AddrInfo objects for GossipSub.
|
|
461
|
+
* Direct peers maintain permanent mesh connections without GRAFT/PRUNE negotiation.
|
|
462
|
+
*
|
|
463
|
+
* Supported formats:
|
|
464
|
+
* - Multiaddr with peer ID: `/ip4/192.168.1.1/tcp/9000/p2p/16Uiu2HAmKLhW7...`
|
|
465
|
+
* - ENR: `enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOo...`
|
|
466
|
+
*
|
|
467
|
+
* For multiaddrs, the string must contain a /p2p/ component with the peer ID.
|
|
468
|
+
* For ENRs, the TCP multiaddr and peer ID are extracted from the encoded record.
|
|
469
|
+
*/
|
|
470
|
+
export function parseDirectPeers(directPeerStrs: routes.lodestar.DirectPeer[], logger: Logger): AddrInfo[] {
|
|
471
|
+
const directPeers: AddrInfo[] = [];
|
|
472
|
+
|
|
473
|
+
for (const peerStr of directPeerStrs) {
|
|
474
|
+
// Check if this is an ENR (starts with "enr:")
|
|
475
|
+
if (peerStr.startsWith("enr:")) {
|
|
476
|
+
try {
|
|
477
|
+
const enr = ENR.decodeTxt(peerStr);
|
|
478
|
+
const peerId = enr.peerId;
|
|
479
|
+
|
|
480
|
+
// Get TCP multiaddr from ENR
|
|
481
|
+
const multiaddrTCP = enr.getLocationMultiaddr("tcp");
|
|
482
|
+
if (!multiaddrTCP) {
|
|
483
|
+
logger.warn("ENR does not contain TCP multiaddr", {enr: peerStr});
|
|
484
|
+
continue;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
directPeers.push({
|
|
488
|
+
id: peerId,
|
|
489
|
+
addrs: [multiaddrTCP],
|
|
490
|
+
});
|
|
491
|
+
|
|
492
|
+
logger.info("Added direct peer from ENR", {peerId: peerId.toString(), addr: multiaddrTCP.toString()});
|
|
493
|
+
} catch (e) {
|
|
494
|
+
logger.warn("Failed to parse direct peer ENR", {enr: peerStr}, e as Error);
|
|
495
|
+
}
|
|
496
|
+
} else {
|
|
497
|
+
// Parse as multiaddr
|
|
498
|
+
try {
|
|
499
|
+
const ma = multiaddr(peerStr);
|
|
500
|
+
|
|
501
|
+
const peerIdStr = ma.getPeerId();
|
|
502
|
+
if (!peerIdStr) {
|
|
503
|
+
logger.warn("Direct peer multiaddr must contain /p2p/ component with peer ID", {multiaddr: peerStr});
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
try {
|
|
508
|
+
const peerId = peerIdFromString(peerIdStr);
|
|
509
|
+
|
|
510
|
+
// Get the address without the /p2p/ component
|
|
511
|
+
const addr = ma.decapsulate("/p2p/" + peerIdStr);
|
|
512
|
+
|
|
513
|
+
directPeers.push({
|
|
514
|
+
id: peerId,
|
|
515
|
+
addrs: [addr],
|
|
516
|
+
});
|
|
517
|
+
|
|
518
|
+
logger.info("Added direct peer", {peerId: peerIdStr, addr: addr.toString()});
|
|
519
|
+
} catch (e) {
|
|
520
|
+
logger.warn("Invalid peer ID in direct peer multiaddr", {multiaddr: peerStr, peerId: peerIdStr}, e as Error);
|
|
521
|
+
}
|
|
522
|
+
} catch (e) {
|
|
523
|
+
logger.warn("Failed to parse direct peer multiaddr", {multiaddr: peerStr}, e as Error);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
return directPeers;
|
|
529
|
+
}
|
package/src/network/network.ts
CHANGED
|
@@ -641,6 +641,18 @@ export class Network implements INetwork {
|
|
|
641
641
|
return this.core.disconnectPeer(peer);
|
|
642
642
|
}
|
|
643
643
|
|
|
644
|
+
addDirectPeer(peer: routes.lodestar.DirectPeer): Promise<string | null> {
|
|
645
|
+
return this.core.addDirectPeer(peer);
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
removeDirectPeer(peerId: string): Promise<boolean> {
|
|
649
|
+
return this.core.removeDirectPeer(peerId);
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
getDirectPeers(): Promise<string[]> {
|
|
653
|
+
return this.core.getDirectPeers();
|
|
654
|
+
}
|
|
655
|
+
|
|
644
656
|
dumpPeer(peerIdStr: string): Promise<routes.lodestar.LodestarNodePeer | undefined> {
|
|
645
657
|
return this.core.dumpPeer(peerIdStr);
|
|
646
658
|
}
|
package/src/network/options.ts
CHANGED
|
@@ -15,6 +15,12 @@ export interface NetworkOptions
|
|
|
15
15
|
Omit<Eth2GossipsubOpts, "disableLightClientServer"> {
|
|
16
16
|
localMultiaddrs: string[];
|
|
17
17
|
bootMultiaddrs?: string[];
|
|
18
|
+
/**
|
|
19
|
+
* Direct peers for GossipSub - these peers maintain permanent mesh connections without GRAFT/PRUNE.
|
|
20
|
+
* Format: multiaddr strings with peer ID, e.g., "/ip4/192.168.1.1/tcp/9000/p2p/16Uiu2HAmKLhW7..."
|
|
21
|
+
* Both peers must configure each other as direct peers for the feature to work properly.
|
|
22
|
+
*/
|
|
23
|
+
directPeers?: string[];
|
|
18
24
|
subscribeAllSubnets?: boolean;
|
|
19
25
|
mdns?: boolean;
|
|
20
26
|
connectToDiscv5Bootnodes?: boolean;
|
|
@@ -721,6 +721,17 @@ export class PeerManager {
|
|
|
721
721
|
// NOTE: libp2p may emit two "peer:connect" events: One for inbound, one for outbound
|
|
722
722
|
// If that happens, it's okay. Only the "outbound" connection triggers immediate action
|
|
723
723
|
const now = Date.now();
|
|
724
|
+
|
|
725
|
+
// Ethereum uses secp256k1 for node IDs, reject peers with other key types
|
|
726
|
+
if (remotePeer.type !== "secp256k1") {
|
|
727
|
+
this.logger.debug("Peer does not have secp256k1 key, disconnecting", {
|
|
728
|
+
peer: remotePeerPrettyStr,
|
|
729
|
+
type: remotePeer.type,
|
|
730
|
+
});
|
|
731
|
+
void this.goodbyeAndDisconnect(remotePeer, GoodByeReasonCode.IRRELEVANT_NETWORK);
|
|
732
|
+
return;
|
|
733
|
+
}
|
|
734
|
+
|
|
724
735
|
const nodeId = computeNodeId(remotePeer);
|
|
725
736
|
const peerData: PeerData = {
|
|
726
737
|
lastReceivedMsgUnixTsMs: direction === "outbound" ? 0 : now,
|
|
@@ -579,7 +579,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
|
|
|
579
579
|
break;
|
|
580
580
|
}
|
|
581
581
|
|
|
582
|
-
if (!blockInput.
|
|
582
|
+
if (!blockInput.hasComputedAllData()) {
|
|
583
583
|
// immediately attempt fetch of data columns from execution engine
|
|
584
584
|
chain.getBlobsTracker.triggerGetBlobs(blockInput);
|
|
585
585
|
// if we've received at least half of the columns, trigger reconstruction of the rest
|