@lodestar/beacon-node 1.43.0-dev.aef3645690 → 1.43.0-dev.b7d69444c9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/api/impl/beacon/blocks/index.d.ts.map +1 -1
- package/lib/api/impl/beacon/blocks/index.js +1 -4
- package/lib/api/impl/beacon/blocks/index.js.map +1 -1
- package/lib/api/impl/beacon/state/utils.d.ts +2 -2
- package/lib/api/impl/beacon/state/utils.d.ts.map +1 -1
- package/lib/api/impl/beacon/state/utils.js.map +1 -1
- package/lib/api/impl/validator/index.d.ts.map +1 -1
- package/lib/api/impl/validator/index.js +1 -4
- package/lib/api/impl/validator/index.js.map +1 -1
- package/lib/chain/GetBlobsTracker.d.ts +1 -1
- package/lib/chain/GetBlobsTracker.d.ts.map +1 -1
- package/lib/chain/GetBlobsTracker.js +1 -2
- package/lib/chain/GetBlobsTracker.js.map +1 -1
- package/lib/chain/archiveStore/archiveStore.d.ts.map +1 -1
- package/lib/chain/archiveStore/archiveStore.js.map +1 -1
- package/lib/chain/archiveStore/interface.d.ts +4 -4
- package/lib/chain/archiveStore/interface.d.ts.map +1 -1
- package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.d.ts +4 -4
- package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.d.ts.map +1 -1
- package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.js +2 -4
- package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.js.map +1 -1
- package/lib/chain/archiveStore/utils/archiveBlocks.d.ts +2 -2
- package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map +1 -1
- package/lib/chain/archiveStore/utils/archiveBlocks.js +110 -58
- package/lib/chain/archiveStore/utils/archiveBlocks.js.map +1 -1
- package/lib/chain/blocks/importBlock.d.ts.map +1 -1
- package/lib/chain/blocks/importBlock.js +27 -35
- package/lib/chain/blocks/importBlock.js.map +1 -1
- package/lib/chain/blocks/importExecutionPayload.d.ts +15 -14
- package/lib/chain/blocks/importExecutionPayload.d.ts.map +1 -1
- package/lib/chain/blocks/importExecutionPayload.js +63 -85
- package/lib/chain/blocks/importExecutionPayload.js.map +1 -1
- package/lib/chain/blocks/index.d.ts.map +1 -1
- package/lib/chain/blocks/index.js +1 -2
- package/lib/chain/blocks/index.js.map +1 -1
- package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts +3 -0
- package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts.map +1 -1
- package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js +20 -0
- package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js.map +1 -1
- package/lib/chain/blocks/payloadEnvelopeProcessor.d.ts +5 -0
- package/lib/chain/blocks/payloadEnvelopeProcessor.d.ts.map +1 -1
- package/lib/chain/blocks/payloadEnvelopeProcessor.js +6 -4
- package/lib/chain/blocks/payloadEnvelopeProcessor.js.map +1 -1
- package/lib/chain/blocks/types.d.ts +15 -21
- package/lib/chain/blocks/types.d.ts.map +1 -1
- package/lib/chain/blocks/verifyExecutionPayloadEnvelope.d.ts +24 -0
- package/lib/chain/blocks/verifyExecutionPayloadEnvelope.d.ts.map +1 -0
- package/lib/chain/blocks/verifyExecutionPayloadEnvelope.js +76 -0
- package/lib/chain/blocks/verifyExecutionPayloadEnvelope.js.map +1 -0
- package/lib/chain/blocks/verifyPayloadsDataAvailability.d.ts +14 -0
- package/lib/chain/blocks/verifyPayloadsDataAvailability.d.ts.map +1 -0
- package/lib/chain/blocks/verifyPayloadsDataAvailability.js +25 -0
- package/lib/chain/blocks/verifyPayloadsDataAvailability.js.map +1 -0
- package/lib/chain/blocks/writePayloadEnvelopeInputToDb.d.ts +1 -1
- package/lib/chain/blocks/writePayloadEnvelopeInputToDb.d.ts.map +1 -1
- package/lib/chain/blocks/writePayloadEnvelopeInputToDb.js +2 -11
- package/lib/chain/blocks/writePayloadEnvelopeInputToDb.js.map +1 -1
- package/lib/chain/chain.d.ts +5 -4
- package/lib/chain/chain.d.ts.map +1 -1
- package/lib/chain/chain.js +26 -39
- package/lib/chain/chain.js.map +1 -1
- package/lib/chain/emitter.d.ts +16 -4
- package/lib/chain/emitter.d.ts.map +1 -1
- package/lib/chain/emitter.js +5 -0
- package/lib/chain/emitter.js.map +1 -1
- package/lib/chain/errors/attestationError.d.ts +8 -1
- package/lib/chain/errors/attestationError.d.ts.map +1 -1
- package/lib/chain/errors/attestationError.js +4 -0
- package/lib/chain/errors/attestationError.js.map +1 -1
- package/lib/chain/errors/executionPayloadBid.d.ts +5 -0
- package/lib/chain/errors/executionPayloadBid.d.ts.map +1 -1
- package/lib/chain/errors/executionPayloadBid.js +1 -0
- package/lib/chain/errors/executionPayloadBid.js.map +1 -1
- package/lib/chain/errors/executionPayloadEnvelope.d.ts +5 -0
- package/lib/chain/errors/executionPayloadEnvelope.d.ts.map +1 -1
- package/lib/chain/errors/executionPayloadEnvelope.js +1 -0
- package/lib/chain/errors/executionPayloadEnvelope.js.map +1 -1
- package/lib/chain/forkChoice/index.d.ts.map +1 -1
- package/lib/chain/forkChoice/index.js +11 -15
- package/lib/chain/forkChoice/index.js.map +1 -1
- package/lib/chain/interface.d.ts +4 -3
- package/lib/chain/interface.d.ts.map +1 -1
- package/lib/chain/interface.js.map +1 -1
- package/lib/chain/prepareNextSlot.d.ts.map +1 -1
- package/lib/chain/prepareNextSlot.js +48 -22
- package/lib/chain/prepareNextSlot.js.map +1 -1
- package/lib/chain/produceBlock/computeNewStateRoot.d.ts +3 -9
- package/lib/chain/produceBlock/computeNewStateRoot.d.ts.map +1 -1
- package/lib/chain/produceBlock/computeNewStateRoot.js +5 -32
- package/lib/chain/produceBlock/computeNewStateRoot.js.map +1 -1
- package/lib/chain/produceBlock/produceBlockBody.d.ts +4 -8
- package/lib/chain/produceBlock/produceBlockBody.d.ts.map +1 -1
- package/lib/chain/produceBlock/produceBlockBody.js +48 -23
- package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
- package/lib/chain/regen/errors.d.ts +1 -11
- package/lib/chain/regen/errors.d.ts.map +1 -1
- package/lib/chain/regen/errors.js +0 -2
- package/lib/chain/regen/errors.js.map +1 -1
- package/lib/chain/regen/interface.d.ts +6 -12
- package/lib/chain/regen/interface.d.ts.map +1 -1
- package/lib/chain/regen/queued.d.ts +6 -11
- package/lib/chain/regen/queued.d.ts.map +1 -1
- package/lib/chain/regen/queued.js +8 -40
- package/lib/chain/regen/queued.js.map +1 -1
- package/lib/chain/regen/regen.d.ts +0 -5
- package/lib/chain/regen/regen.d.ts.map +1 -1
- package/lib/chain/regen/regen.js +7 -34
- package/lib/chain/regen/regen.js.map +1 -1
- package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts +11 -4
- package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts.map +1 -1
- package/lib/chain/seenCache/seenPayloadEnvelopeInput.js +20 -18
- package/lib/chain/seenCache/seenPayloadEnvelopeInput.js.map +1 -1
- package/lib/chain/stateCache/datastore/db.d.ts +5 -4
- package/lib/chain/stateCache/datastore/db.d.ts.map +1 -1
- package/lib/chain/stateCache/datastore/db.js +10 -32
- package/lib/chain/stateCache/datastore/db.js.map +1 -1
- package/lib/chain/stateCache/datastore/file.d.ts +1 -1
- package/lib/chain/stateCache/datastore/file.d.ts.map +1 -1
- package/lib/chain/stateCache/datastore/file.js +5 -5
- package/lib/chain/stateCache/datastore/file.js.map +1 -1
- package/lib/chain/stateCache/datastore/types.d.ts +1 -1
- package/lib/chain/stateCache/datastore/types.d.ts.map +1 -1
- package/lib/chain/stateCache/fifoBlockStateCache.d.ts +1 -7
- package/lib/chain/stateCache/fifoBlockStateCache.d.ts.map +1 -1
- package/lib/chain/stateCache/fifoBlockStateCache.js +0 -8
- package/lib/chain/stateCache/fifoBlockStateCache.js.map +1 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.d.ts +13 -30
- package/lib/chain/stateCache/persistentCheckpointsCache.d.ts.map +1 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.js +120 -216
- package/lib/chain/stateCache/persistentCheckpointsCache.js.map +1 -1
- package/lib/chain/stateCache/types.d.ts +8 -15
- package/lib/chain/stateCache/types.d.ts.map +1 -1
- package/lib/chain/stateCache/types.js.map +1 -1
- package/lib/chain/validation/aggregateAndProof.js +12 -0
- package/lib/chain/validation/aggregateAndProof.js.map +1 -1
- package/lib/chain/validation/attestation.d.ts.map +1 -1
- package/lib/chain/validation/attestation.js +12 -0
- package/lib/chain/validation/attestation.js.map +1 -1
- package/lib/chain/validation/executionPayloadBid.d.ts.map +1 -1
- package/lib/chain/validation/executionPayloadBid.js +13 -1
- package/lib/chain/validation/executionPayloadBid.js.map +1 -1
- package/lib/chain/validation/executionPayloadEnvelope.d.ts.map +1 -1
- package/lib/chain/validation/executionPayloadEnvelope.js +21 -11
- package/lib/chain/validation/executionPayloadEnvelope.js.map +1 -1
- package/lib/chain/validation/payloadAttestationMessage.d.ts.map +1 -1
- package/lib/chain/validation/payloadAttestationMessage.js +4 -3
- package/lib/chain/validation/payloadAttestationMessage.js.map +1 -1
- package/lib/db/repositories/executionPayloadEnvelopeArchive.js +1 -1
- package/lib/db/repositories/executionPayloadEnvelopeArchive.js.map +1 -1
- package/lib/execution/engine/http.d.ts.map +1 -1
- package/lib/execution/engine/http.js +21 -14
- package/lib/execution/engine/http.js.map +1 -1
- package/lib/execution/engine/interface.d.ts +1 -0
- package/lib/execution/engine/interface.d.ts.map +1 -1
- package/lib/execution/engine/mock.d.ts.map +1 -1
- package/lib/execution/engine/mock.js +6 -0
- package/lib/execution/engine/mock.js.map +1 -1
- package/lib/execution/engine/types.d.ts +20 -0
- package/lib/execution/engine/types.d.ts.map +1 -1
- package/lib/execution/engine/types.js +18 -0
- package/lib/execution/engine/types.js.map +1 -1
- package/lib/network/gossip/topic.d.ts +23 -2
- package/lib/network/gossip/topic.d.ts.map +1 -1
- package/lib/network/network.js +1 -1
- package/lib/network/network.js.map +1 -1
- package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
- package/lib/network/processor/gossipHandlers.js +22 -6
- package/lib/network/processor/gossipHandlers.js.map +1 -1
- package/lib/node/nodejs.d.ts.map +1 -1
- package/lib/node/nodejs.js +4 -2
- package/lib/node/nodejs.js.map +1 -1
- package/lib/util/sszBytes.d.ts.map +1 -1
- package/lib/util/sszBytes.js +16 -3
- package/lib/util/sszBytes.js.map +1 -1
- package/package.json +16 -16
- package/src/api/impl/beacon/blocks/index.ts +1 -4
- package/src/api/impl/beacon/state/utils.ts +2 -2
- package/src/api/impl/validator/index.ts +3 -6
- package/src/chain/GetBlobsTracker.ts +1 -2
- package/src/chain/archiveStore/archiveStore.ts +5 -5
- package/src/chain/archiveStore/interface.ts +4 -4
- package/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts +6 -8
- package/src/chain/archiveStore/utils/archiveBlocks.ts +153 -94
- package/src/chain/blocks/importBlock.ts +26 -39
- package/src/chain/blocks/importExecutionPayload.ts +77 -103
- package/src/chain/blocks/index.ts +1 -2
- package/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts +27 -0
- package/src/chain/blocks/payloadEnvelopeProcessor.ts +6 -5
- package/src/chain/blocks/types.ts +15 -26
- package/src/chain/blocks/verifyExecutionPayloadEnvelope.ts +129 -0
- package/src/chain/blocks/verifyPayloadsDataAvailability.ts +38 -0
- package/src/chain/blocks/writePayloadEnvelopeInputToDb.ts +9 -18
- package/src/chain/chain.ts +37 -62
- package/src/chain/emitter.ts +15 -3
- package/src/chain/errors/attestationError.ts +6 -1
- package/src/chain/errors/executionPayloadBid.ts +6 -0
- package/src/chain/errors/executionPayloadEnvelope.ts +6 -0
- package/src/chain/forkChoice/index.ts +8 -20
- package/src/chain/interface.ts +4 -2
- package/src/chain/prepareNextSlot.ts +62 -23
- package/src/chain/produceBlock/computeNewStateRoot.ts +6 -43
- package/src/chain/produceBlock/produceBlockBody.ts +66 -26
- package/src/chain/regen/errors.ts +1 -6
- package/src/chain/regen/interface.ts +6 -12
- package/src/chain/regen/queued.ts +12 -48
- package/src/chain/regen/regen.ts +8 -36
- package/src/chain/seenCache/seenPayloadEnvelopeInput.ts +22 -20
- package/src/chain/stateCache/datastore/db.ts +10 -33
- package/src/chain/stateCache/datastore/file.ts +5 -6
- package/src/chain/stateCache/datastore/types.ts +2 -3
- package/src/chain/stateCache/fifoBlockStateCache.ts +1 -10
- package/src/chain/stateCache/persistentCheckpointsCache.ts +139 -247
- package/src/chain/stateCache/types.ts +8 -14
- package/src/chain/validation/aggregateAndProof.ts +13 -0
- package/src/chain/validation/attestation.ts +13 -0
- package/src/chain/validation/executionPayloadBid.ts +14 -0
- package/src/chain/validation/executionPayloadEnvelope.ts +22 -12
- package/src/chain/validation/payloadAttestationMessage.ts +5 -3
- package/src/db/repositories/executionPayloadEnvelopeArchive.ts +1 -1
- package/src/execution/engine/http.ts +21 -14
- package/src/execution/engine/interface.ts +1 -0
- package/src/execution/engine/mock.ts +8 -1
- package/src/execution/engine/types.ts +41 -0
- package/src/network/network.ts +1 -1
- package/src/network/processor/gossipHandlers.ts +26 -10
- package/src/node/nodejs.ts +4 -2
- package/src/util/sszBytes.ts +21 -3
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { routes } from "@lodestar/api";
|
|
2
2
|
import { BeaconConfig } from "@lodestar/config";
|
|
3
|
-
import { CheckpointWithPayloadStatus } from "@lodestar/fork-choice";
|
|
4
3
|
import { IBeaconStateView } from "@lodestar/state-transition";
|
|
5
4
|
import { Epoch, RootHex, phase0 } from "@lodestar/types";
|
|
6
5
|
import { Logger } from "@lodestar/utils";
|
|
@@ -8,7 +7,7 @@ import { Metrics } from "../../metrics/index.js";
|
|
|
8
7
|
import { BufferPool } from "../../util/bufferPool.js";
|
|
9
8
|
import { IClock } from "../../util/clock.js";
|
|
10
9
|
import { CPStateDatastore, DatastoreKey } from "./datastore/index.js";
|
|
11
|
-
import { BlockStateCache,
|
|
10
|
+
import { BlockStateCache, CheckpointHex, CheckpointStateCache } from "./types.js";
|
|
12
11
|
export type PersistentCheckpointStateCacheOpts = {
|
|
13
12
|
/** Keep max n state epochs in memory, persist the rest to disk */
|
|
14
13
|
maxCPStateEpochsInMemory?: number;
|
|
@@ -78,7 +77,7 @@ export declare const DEFAULT_MAX_CP_STATE_ON_DISK: number;
|
|
|
78
77
|
*/
|
|
79
78
|
export declare class PersistentCheckpointStateCache implements CheckpointStateCache {
|
|
80
79
|
private readonly cache;
|
|
81
|
-
/** Epoch ->
|
|
80
|
+
/** Epoch -> Set<blockRoot> */
|
|
82
81
|
private readonly epochIndex;
|
|
83
82
|
private readonly config;
|
|
84
83
|
private readonly metrics;
|
|
@@ -104,29 +103,27 @@ export declare class PersistentCheckpointStateCache implements CheckpointStateCa
|
|
|
104
103
|
* - Get block for processing
|
|
105
104
|
* - Regen head state
|
|
106
105
|
*/
|
|
107
|
-
getOrReload(cp:
|
|
106
|
+
getOrReload(cp: CheckpointHex): Promise<IBeaconStateView | null>;
|
|
108
107
|
/**
|
|
109
108
|
* Return either state or state bytes loaded from db.
|
|
110
109
|
*/
|
|
111
|
-
getStateOrBytes(cp:
|
|
110
|
+
getStateOrBytes(cp: CheckpointHex): Promise<IBeaconStateView | Uint8Array | null>;
|
|
112
111
|
/**
|
|
113
112
|
* Return either state or state bytes with persisted key loaded from db.
|
|
114
113
|
*/
|
|
115
|
-
getStateOrLoadDb(cp:
|
|
114
|
+
getStateOrLoadDb(cp: CheckpointHex): Promise<IBeaconStateView | LoadedStateBytesData | null>;
|
|
116
115
|
/**
|
|
117
116
|
* Similar to get() api without reloading from disk
|
|
118
117
|
*/
|
|
119
|
-
get(cpOrKey:
|
|
118
|
+
get(cpOrKey: CheckpointHex | CacheKey): IBeaconStateView | null;
|
|
120
119
|
/**
|
|
121
120
|
* Add a state of a checkpoint to this cache, prune from memory if necessary.
|
|
122
|
-
* @param payloadPresent - For Gloas: true if this is payload state, false if block state.
|
|
123
|
-
* Always true for pre-Gloas.
|
|
124
121
|
*/
|
|
125
|
-
add(cp: phase0.Checkpoint, state: IBeaconStateView
|
|
122
|
+
add(cp: phase0.Checkpoint, state: IBeaconStateView): void;
|
|
126
123
|
/**
|
|
127
124
|
* Searches in-memory state for the latest cached state with a `root` without reload, starting with `epoch` and descending
|
|
128
125
|
*/
|
|
129
|
-
getLatest(rootHex: RootHex, maxEpoch: Epoch
|
|
126
|
+
getLatest(rootHex: RootHex, maxEpoch: Epoch): IBeaconStateView | null;
|
|
130
127
|
/**
|
|
131
128
|
* Searches state for the latest cached state with a `root`, reload if needed, starting with `epoch` and descending
|
|
132
129
|
* This is expensive api, should only be called in some important flows:
|
|
@@ -134,14 +131,12 @@ export declare class PersistentCheckpointStateCache implements CheckpointStateCa
|
|
|
134
131
|
* - Get block for processing
|
|
135
132
|
* - Regen head state
|
|
136
133
|
*/
|
|
137
|
-
getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch
|
|
134
|
+
getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise<IBeaconStateView | null>;
|
|
138
135
|
/**
|
|
139
136
|
* Update the precomputed checkpoint and return the number of hits for the
|
|
140
137
|
* previous one (if any).
|
|
141
|
-
* @param payloadPresent - For Gloas: true if head block has FULL payload, false if EMPTY.
|
|
142
|
-
* Always true for pre-Gloas.
|
|
143
138
|
*/
|
|
144
|
-
updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch
|
|
139
|
+
updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null;
|
|
145
140
|
/**
|
|
146
141
|
* This is just to conform to the old implementation
|
|
147
142
|
*/
|
|
@@ -199,9 +194,6 @@ export declare class PersistentCheckpointStateCache implements CheckpointStateCa
|
|
|
199
194
|
* - 2 then we'll persist {root: b2, epoch n-2} checkpoint state to disk, there are also 2 checkpoint states in memory at epoch n, same to the above (maxEpochsInMemory=1)
|
|
200
195
|
*
|
|
201
196
|
* As of Mar 2024, it takes <=350ms to persist a holesky state on fast server
|
|
202
|
-
*
|
|
203
|
-
* For Gloas: Processes both block state and payload state variants together. The decision of which roots to persist/prune
|
|
204
|
-
* is based on root canonicality (from state's view), not payload presence. Both variants are managed as a unit.
|
|
205
197
|
*/
|
|
206
198
|
processState(blockRootHex: RootHex, state: IBeaconStateView): Promise<number>;
|
|
207
199
|
/**
|
|
@@ -221,11 +213,8 @@ export declare class PersistentCheckpointStateCache implements CheckpointStateCa
|
|
|
221
213
|
*
|
|
222
214
|
* Use seed state from the block cache if cannot find any seed states within this cache.
|
|
223
215
|
*/
|
|
224
|
-
findSeedStateToReload(reloadedCp:
|
|
216
|
+
findSeedStateToReload(reloadedCp: CheckpointHex): IBeaconStateView;
|
|
225
217
|
clear(): void;
|
|
226
|
-
private addToEpochIndex;
|
|
227
|
-
private removeFromEpochIndex;
|
|
228
|
-
private hasPayloadVariant;
|
|
229
218
|
/** ONLY FOR DEBUGGING PURPOSES. For lodestar debug API */
|
|
230
219
|
dumpSummary(): routes.lodestar.StateCacheItem[];
|
|
231
220
|
getStates(): IterableIterator<IBeaconStateView>;
|
|
@@ -246,13 +235,7 @@ export declare class PersistentCheckpointStateCache implements CheckpointStateCa
|
|
|
246
235
|
*/
|
|
247
236
|
private serializeStateValidators;
|
|
248
237
|
}
|
|
249
|
-
export declare function
|
|
250
|
-
|
|
251
|
-
* Convert fork-choice CheckpointWithPayloadStatus to beacon-node CheckpointHexPayload.
|
|
252
|
-
* Maps PayloadStatus enum to boolean payloadPresent.
|
|
253
|
-
* @throws Error if checkpoint has PENDING payload status (ambiguous which variant to use)
|
|
254
|
-
*/
|
|
255
|
-
export declare function fcCheckpointToHexPayload(checkpoint: CheckpointWithPayloadStatus): CheckpointHexPayload;
|
|
256
|
-
export declare function toCheckpointKey(cp: CheckpointHexPayload): string;
|
|
238
|
+
export declare function toCheckpointHex(checkpoint: phase0.Checkpoint): CheckpointHex;
|
|
239
|
+
export declare function toCheckpointKey(cp: CheckpointHex): string;
|
|
257
240
|
export {};
|
|
258
241
|
//# sourceMappingURL=persistentCheckpointsCache.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"persistentCheckpointsCache.d.ts","sourceRoot":"","sources":["../../../src/chain/stateCache/persistentCheckpointsCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,eAAe,CAAC;AACrC,OAAO,EAAC,YAAY,EAAC,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAC,
|
|
1
|
+
{"version":3,"file":"persistentCheckpointsCache.d.ts","sourceRoot":"","sources":["../../../src/chain/stateCache/persistentCheckpointsCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,eAAe,CAAC;AACrC,OAAO,EAAC,YAAY,EAAC,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAC,gBAAgB,EAA0B,MAAM,4BAA4B,CAAC;AACrF,OAAO,EAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAC,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAC,MAAM,EAA2C,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAC,OAAO,EAAC,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAc,UAAU,EAAgB,MAAM,0BAA0B,CAAC;AAChF,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAE3C,OAAO,EAAC,gBAAgB,EAAE,YAAY,EAAC,MAAM,sBAAsB,CAAC;AAEpE,OAAO,EAAC,eAAe,EAAiB,aAAa,EAAE,oBAAoB,EAAC,MAAM,YAAY,CAAC;AAE/F,MAAM,MAAM,kCAAkC,GAAG;IAC/C,kEAAkE;IAClE,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,sCAAsC;IACtC,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC,CAAC;AAEF,KAAK,qCAAqC,GAAG;IAC3C,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,EAAE,gBAAgB,CAAC;IAC5B,eAAe,EAAE,eAAe,CAAC;IACjC,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB,CAAC;AAEF,wCAAwC;AACxC,KAAK,QAAQ,GAAG,MAAM,CAAC;AAiBvB,KAAK,oBAAoB,GAAG;IAAC,YAAY,EAAE,YAAY,CAAC;IAAC,UAAU,EAAE,UAAU,CAAA;CAAC,CAAC;AAEjF;;;;GAIG;AACH,eAAO,MAAM,qCAAqC,IAAI,CAAC;AAEvD;;;;;GAKG;AACH,eAAO,MAAM,4BAA4B,QAAW,CAAC;AAKrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,8BAA+B,YAAW,oBAAoB;IACzE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkC;IACxD,8BAA8B;IAC9B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA4D;IACvF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA6B;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA4B;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;IACjD,OAAO,CAAC,qBAAqB,CAAuB;IACpD,OAAO,CAAC,yBAAyB,CAAuB;IACxD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAa;IAEzC,YACE,EACE,MAAM,EACN,OAAO,EACP,MAAM,EACN,KAAK,EACL,MAAM,EACN,SAAS,EACT,eAAe,EACf,UAAU,EACX,EAAE,qCAAqC,EACxC,IAAI,EAAE,kCAAkC,EA4CzC;IAED;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAa1B;IAED;;;;;;OAMG;IACG,WAAW,CAAC,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAsDrE;IAED;;OAEG;IACG,eAAe,CAAC,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,gBAAgB,GAAG,UAAU,GAAG,IAAI,CAAC,CAMtF;IAED;;OAEG;IACG,gBAAgB,CAAC,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,gBAAgB,GAAG,oBAAoB,GAAG,IAAI,CAAC,CA0BjG;IAED;;OAEG;IACH,GAAG,CAAC,OAAO,EAAE,aAAa,GAAG,QAAQ,GAAG,gBAAgB,GAAG,IAAI,CAsB9D;IAED;;OAEG;IACH,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,gBAAgB,GAAG,IAAI,CAoBxD;IAED;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,GAAG,gBAAgB,GAAG,IAAI,CAcpE;IAED;;;;;;OAMG;IACG,iBAAiB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAkB3F;IAED;;;OAGG;IACH,2BAA2B,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,MAAM,GAAG,IAAI,CAKzE;IAED;;OAEG;IACH,KAAK,IAAI,IAAI,CAEZ;IAED;;OAEG;IACH,cAAc,CAAC,cAAc,EAAE,KAAK,GAAG,IAAI,CAQ1C;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiDG;IACG,YAAY,CAAC,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAkDlF;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,qBAAqB,CAAC,UAAU,EAAE,aAAa,GAAG,gBAAgB,CA+CjE;IAED,KAAK,IAAI,IAAI,CAGZ;IAED,0DAA0D;IAC1D,WAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,cAAc,EAAE,CAY9C;IAED,SAAS,IAAI,gBAAgB,CAAC,gBAAgB,CAAC,CAM9C;IAED,2DAA2D;IAC3D,kBAAkB,IAAI,MAAM,EAAE,CAE7B;YAuCa,gBAAgB;YAwGhB,mBAAmB;IAyBjC;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAqB5B;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;CAcjC;AAED,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,GAAG,aAAa,CAK5E;AAED,wBAAgB,eAAe,CAAC,EAAE,EAAE,aAAa,GAAG,MAAM,CAEzD"}
|
|
@@ -56,19 +56,6 @@ import { AllocSource } from "../../util/bufferPool.js";
|
|
|
56
56
|
import { serializeState } from "../serializeState.js";
|
|
57
57
|
import { MapTracker } from "./mapMetrics.js";
|
|
58
58
|
import { CacheItemType } from "./types.js";
|
|
59
|
-
/** Bitmask for tracking which payload variants exist per root in the epochIndex */
|
|
60
|
-
var PayloadAvailability;
|
|
61
|
-
(function (PayloadAvailability) {
|
|
62
|
-
PayloadAvailability[PayloadAvailability["NOT_PRESENT"] = 1] = "NOT_PRESENT";
|
|
63
|
-
PayloadAvailability[PayloadAvailability["PRESENT"] = 2] = "PRESENT";
|
|
64
|
-
})(PayloadAvailability || (PayloadAvailability = {}));
|
|
65
|
-
const PAYLOAD_AVAILABILITY_ALL = [PayloadAvailability.NOT_PRESENT, PayloadAvailability.PRESENT];
|
|
66
|
-
function toPayloadAvailability(payloadPresent) {
|
|
67
|
-
return payloadPresent ? PayloadAvailability.PRESENT : PayloadAvailability.NOT_PRESENT;
|
|
68
|
-
}
|
|
69
|
-
function fromPayloadAvailability(flag) {
|
|
70
|
-
return flag === PayloadAvailability.PRESENT;
|
|
71
|
-
}
|
|
72
59
|
/**
|
|
73
60
|
* Before n-historical states, lodestar keeps all checkpoint states since finalized
|
|
74
61
|
* Since Sep 2024, lodestar stores 3 most recent checkpoint states in memory and the rest on disk. The finalized state
|
|
@@ -118,8 +105,8 @@ const PROCESS_CHECKPOINT_STATES_BPS = 6667;
|
|
|
118
105
|
*/
|
|
119
106
|
export class PersistentCheckpointStateCache {
|
|
120
107
|
cache;
|
|
121
|
-
/** Epoch ->
|
|
122
|
-
epochIndex = new MapDef(() => new
|
|
108
|
+
/** Epoch -> Set<blockRoot> */
|
|
109
|
+
epochIndex = new MapDef(() => new Set());
|
|
123
110
|
config;
|
|
124
111
|
metrics;
|
|
125
112
|
logger;
|
|
@@ -205,12 +192,7 @@ export class PersistentCheckpointStateCache {
|
|
|
205
192
|
return stateOrStateBytesData ?? null;
|
|
206
193
|
}
|
|
207
194
|
const { persistedKey, stateBytes } = stateOrStateBytesData;
|
|
208
|
-
const logMeta = {
|
|
209
|
-
epoch: cp.epoch,
|
|
210
|
-
rootHex: cp.rootHex,
|
|
211
|
-
payloadPresent: cp.payloadPresent,
|
|
212
|
-
persistedKey: toHex(persistedKey),
|
|
213
|
-
};
|
|
195
|
+
const logMeta = { persistedKey: toHex(persistedKey) };
|
|
214
196
|
this.logger.debug("Reload: read state successful", logMeta);
|
|
215
197
|
this.metrics?.cpStateCache.stateReloadSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0);
|
|
216
198
|
const seedState = this.findSeedStateToReload(cp);
|
|
@@ -231,7 +213,10 @@ export class PersistentCheckpointStateCache {
|
|
|
231
213
|
}
|
|
232
214
|
sszTimer?.();
|
|
233
215
|
const timer = this.metrics?.cpStateCache.stateReloadDuration.startTimer();
|
|
234
|
-
|
|
216
|
+
// preload validators and balances for faster state transition
|
|
217
|
+
const newCachedState = seedState.loadOtherState(stateBytes, validatorsBytes, {
|
|
218
|
+
preloadValidatorsAndBalances: true,
|
|
219
|
+
});
|
|
235
220
|
// hashTreeRoot() calls the commit() inside
|
|
236
221
|
// there is no modification inside the state, it's just that we want to compute and cache all roots
|
|
237
222
|
const stateRoot = toRootHex(newCachedState.hashTreeRoot());
|
|
@@ -245,7 +230,7 @@ export class PersistentCheckpointStateCache {
|
|
|
245
230
|
// only remove persisted state once we reload successfully
|
|
246
231
|
const cpKey = toCacheKey(cp);
|
|
247
232
|
this.cache.set(cpKey, { type: CacheItemType.inMemory, state: newCachedState, persistedKey });
|
|
248
|
-
this.
|
|
233
|
+
this.epochIndex.getOrDefault(cp.epoch).add(cp.rootHex);
|
|
249
234
|
// don't prune from memory here, call it at the last 1/3 of slot 0 of an epoch
|
|
250
235
|
return newCachedState;
|
|
251
236
|
}
|
|
@@ -321,11 +306,9 @@ export class PersistentCheckpointStateCache {
|
|
|
321
306
|
}
|
|
322
307
|
/**
|
|
323
308
|
* Add a state of a checkpoint to this cache, prune from memory if necessary.
|
|
324
|
-
* @param payloadPresent - For Gloas: true if this is payload state, false if block state.
|
|
325
|
-
* Always true for pre-Gloas.
|
|
326
309
|
*/
|
|
327
|
-
add(cp, state
|
|
328
|
-
const cpHex =
|
|
310
|
+
add(cp, state) {
|
|
311
|
+
const cpHex = toCheckpointHex(cp);
|
|
329
312
|
const key = toCacheKey(cpHex);
|
|
330
313
|
const cacheItem = this.cache.get(key);
|
|
331
314
|
this.metrics?.cpStateCache.adds.inc();
|
|
@@ -336,32 +319,27 @@ export class PersistentCheckpointStateCache {
|
|
|
336
319
|
this.logger.verbose("Added checkpoint state to memory but a persisted key existed", {
|
|
337
320
|
epoch: cp.epoch,
|
|
338
321
|
rootHex: cpHex.rootHex,
|
|
339
|
-
payloadPresent,
|
|
340
322
|
persistedKey: toHex(persistedKey),
|
|
341
323
|
});
|
|
342
324
|
}
|
|
343
325
|
else {
|
|
344
326
|
this.cache.set(key, { type: CacheItemType.inMemory, state });
|
|
345
|
-
this.logger.verbose("Added checkpoint state to memory", {
|
|
346
|
-
epoch: cp.epoch,
|
|
347
|
-
rootHex: cpHex.rootHex,
|
|
348
|
-
payloadPresent,
|
|
349
|
-
});
|
|
327
|
+
this.logger.verbose("Added checkpoint state to memory", { epoch: cp.epoch, rootHex: cpHex.rootHex });
|
|
350
328
|
}
|
|
351
|
-
this.
|
|
329
|
+
this.epochIndex.getOrDefault(cp.epoch).add(cpHex.rootHex);
|
|
352
330
|
this.prunePersistedStates();
|
|
353
331
|
}
|
|
354
332
|
/**
|
|
355
333
|
* Searches in-memory state for the latest cached state with a `root` without reload, starting with `epoch` and descending
|
|
356
334
|
*/
|
|
357
|
-
getLatest(rootHex, maxEpoch
|
|
335
|
+
getLatest(rootHex, maxEpoch) {
|
|
358
336
|
// sort epochs in descending order, only consider epochs lte `epoch`
|
|
359
337
|
const epochs = Array.from(this.epochIndex.keys())
|
|
360
338
|
.sort((a, b) => b - a)
|
|
361
339
|
.filter((e) => e <= maxEpoch);
|
|
362
340
|
for (const epoch of epochs) {
|
|
363
|
-
if (this.
|
|
364
|
-
const inMemoryClonedState = this.get({ rootHex, epoch
|
|
341
|
+
if (this.epochIndex.get(epoch)?.has(rootHex)) {
|
|
342
|
+
const inMemoryClonedState = this.get({ rootHex, epoch });
|
|
365
343
|
if (inMemoryClonedState) {
|
|
366
344
|
return inMemoryClonedState;
|
|
367
345
|
}
|
|
@@ -376,21 +354,21 @@ export class PersistentCheckpointStateCache {
|
|
|
376
354
|
* - Get block for processing
|
|
377
355
|
* - Regen head state
|
|
378
356
|
*/
|
|
379
|
-
async getOrReloadLatest(rootHex, maxEpoch
|
|
357
|
+
async getOrReloadLatest(rootHex, maxEpoch) {
|
|
380
358
|
// sort epochs in descending order, only consider epochs lte `epoch`
|
|
381
359
|
const epochs = Array.from(this.epochIndex.keys())
|
|
382
360
|
.sort((a, b) => b - a)
|
|
383
361
|
.filter((e) => e <= maxEpoch);
|
|
384
362
|
for (const epoch of epochs) {
|
|
385
|
-
if (this.
|
|
363
|
+
if (this.epochIndex.get(epoch)?.has(rootHex)) {
|
|
386
364
|
try {
|
|
387
|
-
const state = await this.getOrReload({ rootHex, epoch
|
|
365
|
+
const state = await this.getOrReload({ rootHex, epoch });
|
|
388
366
|
if (state) {
|
|
389
367
|
return state;
|
|
390
368
|
}
|
|
391
369
|
}
|
|
392
370
|
catch (e) {
|
|
393
|
-
this.logger.debug("Error get or reload state", { epoch, rootHex
|
|
371
|
+
this.logger.debug("Error get or reload state", { epoch, rootHex }, e);
|
|
394
372
|
}
|
|
395
373
|
}
|
|
396
374
|
}
|
|
@@ -399,12 +377,10 @@ export class PersistentCheckpointStateCache {
|
|
|
399
377
|
/**
|
|
400
378
|
* Update the precomputed checkpoint and return the number of hits for the
|
|
401
379
|
* previous one (if any).
|
|
402
|
-
* @param payloadPresent - For Gloas: true if head block has FULL payload, false if EMPTY.
|
|
403
|
-
* Always true for pre-Gloas.
|
|
404
380
|
*/
|
|
405
|
-
updatePreComputedCheckpoint(rootHex, epoch
|
|
381
|
+
updatePreComputedCheckpoint(rootHex, epoch) {
|
|
406
382
|
const previousHits = this.preComputedCheckpointHits;
|
|
407
|
-
this.preComputedCheckpoint = toCacheKey({ rootHex, epoch
|
|
383
|
+
this.preComputedCheckpoint = toCacheKey({ rootHex, epoch });
|
|
408
384
|
this.preComputedCheckpointHits = 0;
|
|
409
385
|
return previousHits;
|
|
410
386
|
}
|
|
@@ -473,9 +449,6 @@ export class PersistentCheckpointStateCache {
|
|
|
473
449
|
* - 2 then we'll persist {root: b2, epoch n-2} checkpoint state to disk, there are also 2 checkpoint states in memory at epoch n, same to the above (maxEpochsInMemory=1)
|
|
474
450
|
*
|
|
475
451
|
* As of Mar 2024, it takes <=350ms to persist a holesky state on fast server
|
|
476
|
-
*
|
|
477
|
-
* For Gloas: Processes both block state and payload state variants together. The decision of which roots to persist/prune
|
|
478
|
-
* is based on root canonicality (from state's view), not payload presence. Both variants are managed as a unit.
|
|
479
452
|
*/
|
|
480
453
|
async processState(blockRootHex, state) {
|
|
481
454
|
let persistCount = 0;
|
|
@@ -550,35 +523,30 @@ export class PersistentCheckpointStateCache {
|
|
|
550
523
|
if (firstState !== null) {
|
|
551
524
|
return firstState;
|
|
552
525
|
}
|
|
553
|
-
for (const
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
526
|
+
for (const rootHex of this.epochIndex.get(epoch) || []) {
|
|
527
|
+
const cpKey = toCacheKey({ rootHex, epoch });
|
|
528
|
+
const cacheItem = this.cache.get(cpKey);
|
|
529
|
+
if (cacheItem === undefined) {
|
|
530
|
+
continue;
|
|
531
|
+
}
|
|
532
|
+
if (isInMemoryCacheItem(cacheItem)) {
|
|
533
|
+
const { state } = cacheItem;
|
|
534
|
+
if (firstState === null) {
|
|
535
|
+
firstState = state;
|
|
562
536
|
}
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
// amongst states of the same epoch, choose the one with the same view of reloadedCp
|
|
571
|
-
if (reloadedCpSlot < state.slot &&
|
|
572
|
-
toRootHex(state.getBlockRootAtSlot(reloadedCpSlot)) === reloadedCp.rootHex) {
|
|
573
|
-
this.logger.verbose("Reload: use checkpoint state as seed state", { ...cpLog, ...logCtx });
|
|
574
|
-
return state;
|
|
575
|
-
}
|
|
576
|
-
}
|
|
577
|
-
catch (e) {
|
|
578
|
-
// getBlockRootAtSlot may throw error
|
|
579
|
-
this.logger.debug("Error finding checkpoint state to reload", { ...cpLog, ...logCtx }, e);
|
|
537
|
+
const cpLog = { cpEpoch: epoch, cpRoot: rootHex };
|
|
538
|
+
try {
|
|
539
|
+
// amongst states of the same epoch, choose the one with the same view of reloadedCp
|
|
540
|
+
if (reloadedCpSlot < state.slot &&
|
|
541
|
+
toRootHex(state.getBlockRootAtSlot(reloadedCpSlot)) === reloadedCp.rootHex) {
|
|
542
|
+
this.logger.verbose("Reload: use checkpoint state as seed state", { ...cpLog, ...logCtx });
|
|
543
|
+
return state;
|
|
580
544
|
}
|
|
581
545
|
}
|
|
546
|
+
catch (e) {
|
|
547
|
+
// getBlockRootAtSlot may throw error
|
|
548
|
+
this.logger.debug("Error finding checkpoint state to reload", { ...cpLog, ...logCtx }, e);
|
|
549
|
+
}
|
|
582
550
|
}
|
|
583
551
|
}
|
|
584
552
|
}
|
|
@@ -591,31 +559,6 @@ export class PersistentCheckpointStateCache {
|
|
|
591
559
|
this.cache.clear();
|
|
592
560
|
this.epochIndex.clear();
|
|
593
561
|
}
|
|
594
|
-
addToEpochIndex(epoch, rootHex, payloadPresent) {
|
|
595
|
-
const rootMap = this.epochIndex.getOrDefault(epoch);
|
|
596
|
-
rootMap.set(rootHex, (rootMap.get(rootHex) ?? 0) | toPayloadAvailability(payloadPresent));
|
|
597
|
-
}
|
|
598
|
-
removeFromEpochIndex(epoch, rootHex, payloadPresent) {
|
|
599
|
-
const rootMap = this.epochIndex.get(epoch);
|
|
600
|
-
if (rootMap === undefined)
|
|
601
|
-
return;
|
|
602
|
-
const existing = rootMap.get(rootHex);
|
|
603
|
-
if (existing === undefined)
|
|
604
|
-
return;
|
|
605
|
-
const updated = existing & ~toPayloadAvailability(payloadPresent);
|
|
606
|
-
if (updated === 0) {
|
|
607
|
-
rootMap.delete(rootHex);
|
|
608
|
-
if (rootMap.size === 0) {
|
|
609
|
-
this.epochIndex.delete(epoch);
|
|
610
|
-
}
|
|
611
|
-
}
|
|
612
|
-
else {
|
|
613
|
-
rootMap.set(rootHex, updated);
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
hasPayloadVariant(epoch, rootHex, payloadPresent) {
|
|
617
|
-
return Boolean((this.epochIndex.get(epoch)?.get(rootHex) ?? 0) & toPayloadAvailability(payloadPresent));
|
|
618
|
-
}
|
|
619
562
|
/** ONLY FOR DEBUGGING PURPOSES. For lodestar debug API */
|
|
620
563
|
dumpSummary() {
|
|
621
564
|
return Array.from(this.cache.keys()).map((key) => {
|
|
@@ -684,81 +627,81 @@ export class PersistentCheckpointStateCache {
|
|
|
684
627
|
const epochBoundaryHex = toRootHex(epochBoundaryRoot);
|
|
685
628
|
const prevEpochRoot = toRootHex(state.getBlockRootAtSlot(epochBoundarySlot - 1));
|
|
686
629
|
// for each epoch, usually there are 2 rootHexes respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State
|
|
687
|
-
const
|
|
630
|
+
const cpRootHexes = this.epochIndex.get(epoch) ?? [];
|
|
688
631
|
const persistedRootHexes = new Set();
|
|
689
632
|
// 1) if there is no CRCS, persist PRCS (block 0 of epoch is skipped). In this case prevEpochRoot === epochBoundaryHex
|
|
690
633
|
// 2) if there are PRCS and CRCS, persist CRCS => persist CRCS
|
|
691
634
|
// => this is simplified to always persist epochBoundaryHex
|
|
692
635
|
persistedRootHexes.add(epochBoundaryHex);
|
|
693
636
|
// 3) persist any states with unknown roots to this state
|
|
694
|
-
for (const rootHex of
|
|
637
|
+
for (const rootHex of cpRootHexes) {
|
|
695
638
|
if (rootHex !== epochBoundaryHex && rootHex !== prevEpochRoot) {
|
|
696
639
|
persistedRootHexes.add(rootHex);
|
|
697
640
|
}
|
|
698
641
|
}
|
|
699
|
-
for (const
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
const
|
|
705
|
-
const
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
}
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
//
|
|
642
|
+
for (const rootHex of cpRootHexes) {
|
|
643
|
+
const cpKey = toCacheKey({ epoch: epoch, rootHex });
|
|
644
|
+
const cacheItem = this.cache.get(cpKey);
|
|
645
|
+
if (cacheItem !== undefined && isInMemoryCacheItem(cacheItem)) {
|
|
646
|
+
let { persistedKey } = cacheItem;
|
|
647
|
+
const { state } = cacheItem;
|
|
648
|
+
const logMeta = {
|
|
649
|
+
stateSlot: state.slot,
|
|
650
|
+
rootHex,
|
|
651
|
+
epochBoundaryHex,
|
|
652
|
+
persistedKey: persistedKey ? toHex(persistedKey) : "",
|
|
653
|
+
};
|
|
654
|
+
if (persistedRootHexes.has(rootHex)) {
|
|
655
|
+
if (persistedKey) {
|
|
656
|
+
// we don't care if the checkpoint state is already persisted
|
|
657
|
+
this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta);
|
|
658
|
+
}
|
|
659
|
+
else {
|
|
660
|
+
// persist and do not update epochIndex
|
|
661
|
+
this.metrics?.cpStateCache.statePersistSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0);
|
|
662
|
+
const cpPersist = { epoch: epoch, root: fromHex(rootHex) };
|
|
663
|
+
// It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory.
|
|
664
|
+
// As monitored on holesky as of Jan 2024:
|
|
665
|
+
// - This does not increase heap allocation while gc time is the same
|
|
666
|
+
// - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s)
|
|
667
|
+
// - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s)
|
|
668
|
+
// - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization
|
|
669
|
+
const timer = this.metrics?.stateSerializeDuration.startTimer({
|
|
670
|
+
source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE,
|
|
671
|
+
});
|
|
672
|
+
persistedKey = await serializeState(state, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, (stateBytes) => {
|
|
673
|
+
timer?.();
|
|
674
|
+
return this.datastore.write(cpPersist, stateBytes);
|
|
675
|
+
}, this.bufferPool);
|
|
676
|
+
persistCount++;
|
|
677
|
+
this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", {
|
|
678
|
+
...logMeta,
|
|
679
|
+
persistedKey: toHex(persistedKey),
|
|
680
|
+
});
|
|
681
|
+
}
|
|
682
|
+
// overwrite cpKey, this means the state is deleted from memory
|
|
683
|
+
this.cache.set(cpKey, { type: CacheItemType.persisted, value: persistedKey });
|
|
684
|
+
}
|
|
685
|
+
else {
|
|
686
|
+
if (persistedKey) {
|
|
687
|
+
// persisted file will be eventually deleted by the archive task
|
|
688
|
+
// this also means the state is deleted from memory
|
|
745
689
|
this.cache.set(cpKey, { type: CacheItemType.persisted, value: persistedKey });
|
|
690
|
+
// do not update epochIndex
|
|
746
691
|
}
|
|
747
692
|
else {
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
this.cache.delete(cpKey);
|
|
757
|
-
this.removeFromEpochIndex(epoch, rootHex, payloadPresent);
|
|
693
|
+
// delete the state from memory
|
|
694
|
+
this.cache.delete(cpKey);
|
|
695
|
+
const rootSet = this.epochIndex.get(epoch);
|
|
696
|
+
if (rootSet) {
|
|
697
|
+
rootSet.delete(rootHex);
|
|
698
|
+
if (rootSet.size === 0) {
|
|
699
|
+
this.epochIndex.delete(epoch);
|
|
700
|
+
}
|
|
758
701
|
}
|
|
759
|
-
this.metrics?.cpStateCache.statePruneFromMemoryCount.inc();
|
|
760
|
-
this.logger.verbose("Pruned checkpoint state from memory", logMeta);
|
|
761
702
|
}
|
|
703
|
+
this.metrics?.cpStateCache.statePruneFromMemoryCount.inc();
|
|
704
|
+
this.logger.verbose("Pruned checkpoint state from memory", logMeta);
|
|
762
705
|
}
|
|
763
706
|
}
|
|
764
707
|
}
|
|
@@ -769,38 +712,25 @@ export class PersistentCheckpointStateCache {
|
|
|
769
712
|
*/
|
|
770
713
|
async deleteAllEpochItems(epoch) {
|
|
771
714
|
let persistCount = 0;
|
|
772
|
-
const
|
|
773
|
-
for (const
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
const
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
if (persistedKey) {
|
|
783
|
-
await this.datastore.remove(persistedKey);
|
|
784
|
-
persistCount++;
|
|
785
|
-
this.metrics?.cpStateCache.persistedStateRemoveCount.inc();
|
|
786
|
-
}
|
|
715
|
+
const rootHexes = this.epochIndex.get(epoch) || [];
|
|
716
|
+
for (const rootHex of rootHexes) {
|
|
717
|
+
const key = toCacheKey({ rootHex, epoch });
|
|
718
|
+
const cacheItem = this.cache.get(key);
|
|
719
|
+
if (cacheItem) {
|
|
720
|
+
const persistedKey = isPersistedCacheItem(cacheItem) ? cacheItem.value : cacheItem.persistedKey;
|
|
721
|
+
if (persistedKey) {
|
|
722
|
+
await this.datastore.remove(persistedKey);
|
|
723
|
+
persistCount++;
|
|
724
|
+
this.metrics?.cpStateCache.persistedStateRemoveCount.inc();
|
|
787
725
|
}
|
|
788
|
-
this.cache.delete(key);
|
|
789
|
-
this.logger.verbose("Pruned checkpoint state", {
|
|
790
|
-
epoch,
|
|
791
|
-
rootHex,
|
|
792
|
-
payloadPresent,
|
|
793
|
-
type: cacheItem ? (isPersistedCacheItem(cacheItem) ? "persisted" : "in-memory") : "missing",
|
|
794
|
-
});
|
|
795
726
|
}
|
|
727
|
+
this.cache.delete(key);
|
|
796
728
|
}
|
|
797
729
|
this.epochIndex.delete(epoch);
|
|
798
|
-
this.logger.verbose("Pruned
|
|
730
|
+
this.logger.verbose("Pruned checkpoint states for epoch", {
|
|
799
731
|
epoch,
|
|
800
732
|
persistCount,
|
|
801
|
-
|
|
802
|
-
.flatMap(([rootHex, bitmask]) => PAYLOAD_AVAILABILITY_ALL.filter((f) => bitmask & f).map((f) => `${rootHex}:${fromPayloadAvailability(f)}`))
|
|
803
|
-
.join(","),
|
|
733
|
+
rootHexes: Array.from(rootHexes).join(","),
|
|
804
734
|
});
|
|
805
735
|
}
|
|
806
736
|
/**
|
|
@@ -840,49 +770,23 @@ export class PersistentCheckpointStateCache {
|
|
|
840
770
|
return null;
|
|
841
771
|
}
|
|
842
772
|
}
|
|
843
|
-
export function
|
|
773
|
+
export function toCheckpointHex(checkpoint) {
|
|
844
774
|
return {
|
|
845
775
|
epoch: checkpoint.epoch,
|
|
846
776
|
rootHex: toRootHex(checkpoint.root),
|
|
847
|
-
payloadPresent,
|
|
848
|
-
};
|
|
849
|
-
}
|
|
850
|
-
/**
|
|
851
|
-
* Convert fork-choice CheckpointWithPayloadStatus to beacon-node CheckpointHexPayload.
|
|
852
|
-
* Maps PayloadStatus enum to boolean payloadPresent.
|
|
853
|
-
* @throws Error if checkpoint has PENDING payload status (ambiguous which variant to use)
|
|
854
|
-
*/
|
|
855
|
-
export function fcCheckpointToHexPayload(checkpoint) {
|
|
856
|
-
const PayloadStatus = { PENDING: 0, EMPTY: 1, FULL: 2 };
|
|
857
|
-
if (checkpoint.payloadStatus === PayloadStatus.PENDING) {
|
|
858
|
-
throw Error(`Cannot convert checkpoint with PENDING payload status at epoch ${checkpoint.epoch} root ${checkpoint.rootHex}`);
|
|
859
|
-
}
|
|
860
|
-
return {
|
|
861
|
-
epoch: checkpoint.epoch,
|
|
862
|
-
rootHex: checkpoint.rootHex,
|
|
863
|
-
payloadPresent: checkpoint.payloadStatus === PayloadStatus.FULL,
|
|
864
777
|
};
|
|
865
778
|
}
|
|
866
779
|
export function toCheckpointKey(cp) {
|
|
867
|
-
return `${cp.rootHex}:${cp.epoch}
|
|
780
|
+
return `${cp.rootHex}:${cp.epoch}`;
|
|
868
781
|
}
|
|
869
|
-
/**
|
|
870
|
-
* Convert checkpoint to cache key string.
|
|
871
|
-
* Format: `{rootHex}_{epoch}_{payloadPresent}`
|
|
872
|
-
*/
|
|
873
782
|
function toCacheKey(cp) {
|
|
874
|
-
return `${cp.rootHex}_${cp.epoch}
|
|
783
|
+
return `${cp.rootHex}_${cp.epoch}`;
|
|
875
784
|
}
|
|
876
785
|
function fromCacheKey(key) {
|
|
877
|
-
const
|
|
878
|
-
const rootHex = parts[0];
|
|
879
|
-
const epoch = Number(parts[1]);
|
|
880
|
-
// For backward compatibility with old format (rootHex_epoch), default to true
|
|
881
|
-
const payloadPresent = parts.length > 2 ? parts[2] === "true" : true;
|
|
786
|
+
const [rootHex, epoch] = key.split("_");
|
|
882
787
|
return {
|
|
883
788
|
rootHex,
|
|
884
|
-
epoch,
|
|
885
|
-
payloadPresent,
|
|
789
|
+
epoch: Number(epoch),
|
|
886
790
|
};
|
|
887
791
|
}
|
|
888
792
|
function isBeaconStateView(stateOrBytes) {
|