@aztec/node-lib 5.3.0-nightly.20261016.a → 5.3.0-nightly.20261017
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -7,16 +7,18 @@ import type { ChainConfig } from '@aztec/stdlib/config';
|
|
|
7
7
|
import { type ReadOnlyFileStore } from '@aztec/stdlib/file-store';
|
|
8
8
|
import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
|
|
9
9
|
import { type SnapshotMetadata } from '@aztec/stdlib/snapshots';
|
|
10
|
+
import type { GenesisData } from '@aztec/stdlib/world-state';
|
|
11
|
+
import { type WorldStateConfig } from '@aztec/world-state';
|
|
10
12
|
import type { SharedNodeConfig } from '../config/index.js';
|
|
11
|
-
type SnapshotSyncConfig = Pick<SharedNodeConfig, 'syncMode'> & Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'> & Pick<L1ContractsConfig, 'aztecEpochDuration'> & Pick<ArchiverConfig, 'archiverStoreMapSizeKb'> & DataStoreConfig & Required<Pick<DataStoreConfig, 'rollupAddress'>> & EthereumClientConfig & {
|
|
13
|
+
type SnapshotSyncConfig = Pick<SharedNodeConfig, 'syncMode'> & Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'> & Pick<L1ContractsConfig, 'aztecEpochDuration'> & Pick<ArchiverConfig, 'archiverStoreMapSizeKb'> & WorldStateConfig & DataStoreConfig & Required<Pick<DataStoreConfig, 'rollupAddress'>> & EthereumClientConfig & {
|
|
12
14
|
snapshotsUrls?: string[];
|
|
13
15
|
minL1BlocksToTriggerReplace?: number;
|
|
14
16
|
};
|
|
15
17
|
/**
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
+
* Validates local history and restores the newest compatible snapshot when bootstrap is needed.
|
|
19
|
+
* Fixed history boundaries are resolved against archived block and checkpoint metadata.
|
|
18
20
|
*/
|
|
19
|
-
export declare function trySnapshotSync(config: SnapshotSyncConfig, log: Logger): Promise<boolean>;
|
|
21
|
+
export declare function trySnapshotSync(config: SnapshotSyncConfig, log: Logger, genesis?: GenesisData): Promise<boolean>;
|
|
20
22
|
/**
|
|
21
23
|
* Downloads the given snapshot replacing any local data stores.
|
|
22
24
|
*/
|
|
@@ -24,6 +26,9 @@ export declare function snapshotSync(snapshot: Pick<SnapshotMetadata, 'dataUrls'
|
|
|
24
26
|
dataDirectory: string;
|
|
25
27
|
rollupAddress: EthAddress;
|
|
26
28
|
fileStore: ReadOnlyFileStore;
|
|
29
|
+
worldStateDataDirectory?: string;
|
|
30
|
+
validateArchiver?: (stagingDirectory: string) => Promise<void>;
|
|
31
|
+
validate?: (stagingDirectory: string) => Promise<void>;
|
|
27
32
|
}): Promise<void>;
|
|
28
33
|
export {};
|
|
29
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
34
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic25hcHNob3Qtc3luYy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2FjdGlvbnMvc25hcHNob3Qtc3luYy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBR0wsS0FBSyxjQUFjLEVBR3BCLE1BQU0saUJBQWlCLENBQUM7QUFFekIsT0FBTyxFQUFFLEtBQUssb0JBQW9CLEVBQW1CLE1BQU0sd0JBQXdCLENBQUM7QUFDcEYsT0FBTyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSx3QkFBd0IsQ0FBQztBQUVoRSxPQUFPLEtBQUssRUFBRSxVQUFVLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUVoRSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUdwRCxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUV4RCxPQUFPLEVBQUUsS0FBSyxpQkFBaUIsRUFBMkIsTUFBTSwwQkFBMEIsQ0FBQztBQUMzRixPQUFPLEtBQUssRUFBRSxlQUFlLEVBQUUsTUFBTSx3QkFBd0IsQ0FBQztBQUM5RCxPQUFPLEVBRUwsS0FBSyxnQkFBZ0IsRUFLdEIsTUFBTSx5QkFBeUIsQ0FBQztBQUNqQyxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsTUFBTSwyQkFBMkIsQ0FBQztBQUM3RCxPQUFPLEVBSUwsS0FBSyxnQkFBZ0IsRUFJdEIsTUFBTSxvQkFBb0IsQ0FBQztBQUs1QixPQUFPLEtBQUssRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLG9CQUFvQixDQUFDO0FBSzNELEtBQUssa0JBQWtCLEdBQUcsSUFBSSxDQUFDLGdCQUFnQixFQUFFLFVBQVUsQ0FBQyxHQUMxRCxJQUFJLENBQUMsV0FBVyxFQUFFLFdBQVcsR0FBRyxlQUFlLENBQUMsR0FDaEQsSUFBSSxDQUFDLGlCQUFpQixFQUFFLG9CQUFvQixDQUFDLEdBQzdDLElBQUksQ0FBQyxjQUFjLEVBQUUsd0JBQXdCLENBQUMsR0FDOUMsZ0JBQWdCLEdBQ2hCLGVBQWUsR0FDZixRQUFRLENBQUMsSUFBSSxDQUFDLGVBQWUsRUFBRSxlQUFlLENBQUMsQ0FBQyxHQUNoRCxvQkFBb0IsR0FBRztJQUNyQixhQUFhLENBQUMsRUFBRSxNQUFNLEVBQUUsQ0FBQztJQUN6QiwyQkFBMkIsQ0FBQyxFQUFFLE1BQU0sQ0FBQztDQUN0QyxDQUFDO0FBRUo7OztHQUdHO0FBQ0gsd0JBQXNCLGVBQWUsQ0FBQyxNQUFNLEVBQUUsa0JBQWtCLEVBQUUsR0FBRyxFQUFFLE1BQU0sRUFBRSxPQUFPLENBQUMsRUFBRSxXQUFXLG9CQXNQbkc7QUFFRDs7R0FFRztBQUNILHdCQUFzQixZQUFZLENBQ2hDLFFBQVEsRUFBRSxJQUFJLENBQUMsZ0JBQWdCLEVBQUUsVUFBVSxDQUFDLEVBQzVDLEdBQUcsRUFBRSxNQUFNLEVBQ1gsTUFBTSxFQUFFO0lBQ04sYUFBYSxFQUFFLE1BQU0sQ0FBQztJQUN0QixhQUFhLEVBQUUsVUFBVSxDQUFDO0lBQzFCLFNBQVMsRUFBRSxpQkFBaUIsQ0FBQztJQUM3Qix1QkFBdUIsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNqQyxnQkFBZ0IsQ0FBQyxFQUFFLENBQUMsZ0JBQWdCLEVBQUUsTUFBTSxLQUFLLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUMvRCxRQUFRLENBQUMsRUFBRSxDQUFDLGdCQUFnQixFQUFFLE1BQU0sS0FBSyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7Q0FDeEQsaUJBcUVGIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot-sync.d.ts","sourceRoot":"","sources":["../../src/actions/snapshot-sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,cAAc,EAGpB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,KAAK,oBAAoB,EAAmB,MAAM,wBAAwB,CAAC;AACpF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"snapshot-sync.d.ts","sourceRoot":"","sources":["../../src/actions/snapshot-sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,cAAc,EAGpB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,KAAK,oBAAoB,EAAmB,MAAM,wBAAwB,CAAC;AACpF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEhE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAEhE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAGpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAExD,OAAO,EAAE,KAAK,iBAAiB,EAA2B,MAAM,0BAA0B,CAAC;AAC3F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAEL,KAAK,gBAAgB,EAKtB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAIL,KAAK,gBAAgB,EAItB,MAAM,oBAAoB,CAAC;AAK5B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAK3D,KAAK,kBAAkB,GAAG,IAAI,CAAC,gBAAgB,EAAE,UAAU,CAAC,GAC1D,IAAI,CAAC,WAAW,EAAE,WAAW,GAAG,eAAe,CAAC,GAChD,IAAI,CAAC,iBAAiB,EAAE,oBAAoB,CAAC,GAC7C,IAAI,CAAC,cAAc,EAAE,wBAAwB,CAAC,GAC9C,gBAAgB,GAChB,eAAe,GACf,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC,GAChD,oBAAoB,GAAG;IACrB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACtC,CAAC;AAEJ;;;GAGG;AACH,wBAAsB,eAAe,CAAC,MAAM,EAAE,kBAAkB,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,oBAsPnG;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,EAAE,UAAU,CAAC,EAC5C,GAAG,EAAE,MAAM,EACX,MAAM,EAAE;IACN,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,UAAU,CAAC;IAC1B,SAAS,EAAE,iBAAiB,CAAC;IAC7B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,gBAAgB,CAAC,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/D,QAAQ,CAAC,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACxD,iBAqEF"}
|
|
@@ -1,22 +1,33 @@
|
|
|
1
1
|
import { ARCHIVER_DB_VERSION, ARCHIVER_STORE_NAME, createArchiverStore, getArchiverSynchPoint } from '@aztec/archiver';
|
|
2
2
|
import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
|
|
3
3
|
import { getPublicClient } from '@aztec/ethereum/client';
|
|
4
|
+
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
4
5
|
import { tryRmDir } from '@aztec/foundation/fs';
|
|
5
6
|
import { P2P_STORE_NAME } from '@aztec/p2p';
|
|
6
7
|
import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/stdlib/block';
|
|
7
8
|
import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
|
|
8
9
|
import { createReadOnlyFileStore } from '@aztec/stdlib/file-store';
|
|
9
|
-
import { downloadSnapshot,
|
|
10
|
-
import { NATIVE_WORLD_STATE_DBS, WORLD_STATE_DB_VERSION, WORLD_STATE_DIR } from '@aztec/world-state';
|
|
11
|
-
import { mkdir, mkdtemp, rename } from 'fs/promises';
|
|
10
|
+
import { SnapshotDataKeys, downloadSnapshot, getSnapshotIndex, makeSnapshotPaths } from '@aztec/stdlib/snapshots';
|
|
11
|
+
import { NATIVE_WORLD_STATE_DBS, WORLD_STATE_DB_VERSION, WORLD_STATE_DIR, createWorldState, getWorldStateHistoryStartCheckpoint, validateWorldStateHistory } from '@aztec/world-state';
|
|
12
|
+
import { copyFile, mkdir, mkdtemp, rename, stat, unlink } from 'fs/promises';
|
|
12
13
|
import { join } from 'path';
|
|
13
14
|
// Half day worth of L1 blocks
|
|
14
15
|
const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
15
16
|
/**
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*/ export async function trySnapshotSync(config, log) {
|
|
17
|
+
* Validates local history and restores the newest compatible snapshot when bootstrap is needed.
|
|
18
|
+
* Fixed history boundaries are resolved against archived block and checkpoint metadata.
|
|
19
|
+
*/ export async function trySnapshotSync(config, log, genesis) {
|
|
19
20
|
const { syncMode, snapshotsUrls, dataDirectory, l1ChainId, rollupVersion, rollupAddress } = config;
|
|
21
|
+
const historyStartCheckpoint = config.worldStatePruneBeforeCheckpoint;
|
|
22
|
+
let maxSnapshotBlock = historyStartCheckpoint === 0 ? BlockNumber.ZERO : undefined;
|
|
23
|
+
const worldStateDirectory = config.worldStateDataDirectory ?? dataDirectory;
|
|
24
|
+
if (historyStartCheckpoint !== undefined && worldStateDirectory && await hasWorldStateDatabase(worldStateDirectory)) {
|
|
25
|
+
await validateLocalWorldStateHistory(config, log, genesis);
|
|
26
|
+
log.info('Using existing world-state database with compatible retained history', {
|
|
27
|
+
historyStartCheckpoint
|
|
28
|
+
});
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
20
31
|
if (syncMode === 'full') {
|
|
21
32
|
log.debug('Snapshot sync is disabled. Running full sync.', {
|
|
22
33
|
syncMode: syncMode
|
|
@@ -51,12 +62,12 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
51
62
|
await archiverStore.db.close();
|
|
52
63
|
}
|
|
53
64
|
const minL1BlocksToTriggerReplace = config.minL1BlocksToTriggerReplace ?? MIN_L1_BLOCKS_TO_TRIGGER_REPLACE;
|
|
54
|
-
if (syncMode === 'snapshot' && archiverL2BlockNumber !== undefined && archiverL2BlockNumber >= INITIAL_L2_BLOCK_NUM) {
|
|
65
|
+
if (historyStartCheckpoint === undefined && syncMode === 'snapshot' && archiverL2BlockNumber !== undefined && archiverL2BlockNumber >= INITIAL_L2_BLOCK_NUM) {
|
|
55
66
|
log.verbose(`Skipping non-forced snapshot sync as archiver is already synced to L2 block ${archiverL2BlockNumber}.`);
|
|
56
67
|
return false;
|
|
57
68
|
}
|
|
58
69
|
const currentL1BlockNumber = await getPublicClient(config).getBlockNumber();
|
|
59
|
-
if (archiverL1BlockNumber && currentL1BlockNumber >= archiverL1BlockNumber && currentL1BlockNumber - archiverL1BlockNumber < minL1BlocksToTriggerReplace) {
|
|
70
|
+
if (historyStartCheckpoint === undefined && archiverL1BlockNumber && currentL1BlockNumber >= archiverL1BlockNumber && currentL1BlockNumber - archiverL1BlockNumber < minL1BlocksToTriggerReplace) {
|
|
60
71
|
log.verbose(`Skipping snapshot sync as archiver is less than ${currentL1BlockNumber - archiverL1BlockNumber} L1 blocks behind.`, {
|
|
61
72
|
archiverL1BlockNumber,
|
|
62
73
|
currentL1BlockNumber,
|
|
@@ -78,52 +89,54 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
78
89
|
log.error(`Invalid config for downloading snapshots from ${snapshotsUrl}`, err);
|
|
79
90
|
continue;
|
|
80
91
|
}
|
|
81
|
-
let
|
|
92
|
+
let snapshots;
|
|
82
93
|
try {
|
|
83
|
-
|
|
94
|
+
snapshots = (await getSnapshotIndex(indexMetadata, fileStore))?.snapshots ?? [];
|
|
84
95
|
} catch (err) {
|
|
85
|
-
log.error(`Failed to
|
|
96
|
+
log.error(`Failed to read snapshot index from ${snapshotsUrl}. Skipping this URL.`, err, {
|
|
86
97
|
...indexMetadata,
|
|
87
98
|
snapshotsUrl
|
|
88
99
|
});
|
|
89
100
|
continue;
|
|
90
101
|
}
|
|
91
|
-
if (
|
|
102
|
+
if (snapshots.length === 0) {
|
|
92
103
|
log.verbose(`No snapshot found at ${snapshotsUrl}. Skipping this URL.`, {
|
|
93
104
|
...indexMetadata,
|
|
94
105
|
snapshotsUrl
|
|
95
106
|
});
|
|
96
107
|
continue;
|
|
97
108
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
109
|
+
for (const snapshot of snapshots){
|
|
110
|
+
if (snapshot.schemaVersions.archiver !== ARCHIVER_DB_VERSION) {
|
|
111
|
+
log.warn(`Skipping snapshot from ${snapshotsUrl} as it has schema version ${snapshot.schemaVersions.archiver} but expected ${ARCHIVER_DB_VERSION}.`, snapshot);
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
if (snapshot.schemaVersions.worldState !== WORLD_STATE_DB_VERSION) {
|
|
115
|
+
log.warn(`Skipping snapshot from ${snapshotsUrl} as it has world state schema version ${snapshot.schemaVersions.worldState} but we expected ${WORLD_STATE_DB_VERSION}.`, snapshot);
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
if (historyStartCheckpoint === undefined && archiverL1BlockNumber && snapshot.l1BlockNumber < archiverL1BlockNumber) {
|
|
119
|
+
log.verbose(`Skipping snapshot from ${snapshotsUrl} since local archiver is at L1 block ${archiverL1BlockNumber} which is further than snapshot at ${snapshot.l1BlockNumber}`, {
|
|
120
|
+
snapshot,
|
|
121
|
+
archiverL1BlockNumber,
|
|
122
|
+
snapshotsUrl
|
|
123
|
+
});
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
if (historyStartCheckpoint === undefined && archiverL1BlockNumber && snapshot.l1BlockNumber - Number(archiverL1BlockNumber) < minL1BlocksToTriggerReplace) {
|
|
127
|
+
log.verbose(`Skipping snapshot from ${snapshotsUrl} as archiver is less than ${snapshot.l1BlockNumber - Number(archiverL1BlockNumber)} L1 blocks behind this snapshot.`, {
|
|
128
|
+
snapshot,
|
|
129
|
+
archiverL1BlockNumber,
|
|
130
|
+
snapshotsUrl
|
|
131
|
+
});
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
snapshotCandidates.push({
|
|
108
135
|
snapshot,
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
});
|
|
112
|
-
continue;
|
|
113
|
-
}
|
|
114
|
-
if (archiverL1BlockNumber && snapshot.l1BlockNumber - Number(archiverL1BlockNumber) < minL1BlocksToTriggerReplace) {
|
|
115
|
-
log.verbose(`Skipping snapshot from ${snapshotsUrl} as archiver is less than ${snapshot.l1BlockNumber - Number(archiverL1BlockNumber)} L1 blocks behind this snapshot.`, {
|
|
116
|
-
snapshot,
|
|
117
|
-
archiverL1BlockNumber,
|
|
118
|
-
snapshotsUrl
|
|
136
|
+
url: snapshotsUrl,
|
|
137
|
+
fileStore
|
|
119
138
|
});
|
|
120
|
-
continue;
|
|
121
139
|
}
|
|
122
|
-
snapshotCandidates.push({
|
|
123
|
-
snapshot,
|
|
124
|
-
url: snapshotsUrl,
|
|
125
|
-
fileStore
|
|
126
|
-
});
|
|
127
140
|
}
|
|
128
141
|
if (snapshotCandidates.length === 0) {
|
|
129
142
|
log.verbose(`No valid snapshots found from any URL, skipping snapshot sync`, {
|
|
@@ -132,20 +145,64 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
132
145
|
});
|
|
133
146
|
return false;
|
|
134
147
|
}
|
|
135
|
-
|
|
136
|
-
snapshotCandidates.sort((a, b)=>b.snapshot.l1BlockNumber - a.snapshot.l1BlockNumber);
|
|
148
|
+
snapshotCandidates.sort((a, b)=>historyStartCheckpoint === undefined ? b.snapshot.l1BlockNumber - a.snapshot.l1BlockNumber : b.snapshot.l2BlockNumber - a.snapshot.l2BlockNumber || b.snapshot.l1BlockNumber - a.snapshot.l1BlockNumber);
|
|
137
149
|
// Try each candidate in order until one succeeds
|
|
138
150
|
for (const { snapshot, url, fileStore } of snapshotCandidates){
|
|
151
|
+
if (maxSnapshotBlock !== undefined && snapshot.l2BlockNumber > maxSnapshotBlock) {
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
139
154
|
const { l1BlockNumber, l2BlockNumber } = snapshot;
|
|
140
155
|
log.info(`Attempting to sync from snapshot at L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, {
|
|
141
156
|
snapshot,
|
|
142
157
|
snapshotsUrl: url
|
|
143
158
|
});
|
|
144
159
|
try {
|
|
160
|
+
const available = await Promise.all(Object.values(snapshot.dataUrls).map((path)=>fileStore.exists(path)));
|
|
161
|
+
if (!available.every(Boolean)) {
|
|
162
|
+
log.warn('Skipping snapshot with missing database files', {
|
|
163
|
+
l1BlockNumber,
|
|
164
|
+
l2BlockNumber,
|
|
165
|
+
snapshotsUrl: url
|
|
166
|
+
});
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
145
169
|
await snapshotSync(snapshot, log, {
|
|
146
170
|
dataDirectory: config.dataDirectory,
|
|
147
171
|
rollupAddress: config.rollupAddress,
|
|
148
|
-
fileStore
|
|
172
|
+
fileStore,
|
|
173
|
+
worldStateDataDirectory: config.worldStateDataDirectory,
|
|
174
|
+
validateArchiver: historyStartCheckpoint === undefined || historyStartCheckpoint === 0 ? undefined : async (stagingDirectory)=>{
|
|
175
|
+
const stores = await createArchiverStore({
|
|
176
|
+
...config,
|
|
177
|
+
dataDirectory: stagingDirectory
|
|
178
|
+
}, GENESIS_BLOCK_HEADER_HASH);
|
|
179
|
+
try {
|
|
180
|
+
const latestBlockNumber = await stores.blocks.getLatestL2BlockNumber();
|
|
181
|
+
if (latestBlockNumber === undefined) {
|
|
182
|
+
throw new Error('Snapshot archiver has no blocks');
|
|
183
|
+
}
|
|
184
|
+
const latestBlock = await stores.blocks.getBlockData({
|
|
185
|
+
number: latestBlockNumber
|
|
186
|
+
});
|
|
187
|
+
if (!latestBlock) {
|
|
188
|
+
throw new Error(`Snapshot archiver is missing block ${latestBlockNumber}`);
|
|
189
|
+
}
|
|
190
|
+
if (latestBlock.checkpointNumber > historyStartCheckpoint) {
|
|
191
|
+
const checkpoint = await stores.blocks.getCheckpointData(historyStartCheckpoint);
|
|
192
|
+
if (checkpoint) {
|
|
193
|
+
maxSnapshotBlock = BlockNumber(checkpoint.startBlock + checkpoint.blockCount - 1);
|
|
194
|
+
}
|
|
195
|
+
throw new Error(`Snapshot is at checkpoint ${latestBlock.checkpointNumber}, after requested checkpoint ${historyStartCheckpoint}`);
|
|
196
|
+
}
|
|
197
|
+
} finally{
|
|
198
|
+
await stores.db.close();
|
|
199
|
+
}
|
|
200
|
+
},
|
|
201
|
+
validate: historyStartCheckpoint === undefined ? undefined : (stagingDirectory)=>validateLocalWorldStateHistory({
|
|
202
|
+
...config,
|
|
203
|
+
dataDirectory: stagingDirectory,
|
|
204
|
+
worldStateDataDirectory: stagingDirectory
|
|
205
|
+
}, log, genesis)
|
|
149
206
|
});
|
|
150
207
|
log.info(`Snapshot synced to L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, {
|
|
151
208
|
snapshot,
|
|
@@ -180,11 +237,26 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
180
237
|
});
|
|
181
238
|
downloadDir = await mkdtemp(join(dataDirectory, 'download-'));
|
|
182
239
|
const downloadPaths = makeSnapshotPaths(downloadDir);
|
|
240
|
+
await prepareTarget(join(downloadDir, ARCHIVER_STORE_NAME), ARCHIVER_DB_VERSION, rollupAddress);
|
|
241
|
+
await prepareTarget(join(downloadDir, WORLD_STATE_DIR), WORLD_STATE_DB_VERSION, rollupAddress);
|
|
242
|
+
downloadPaths.archiver = join(downloadDir, ARCHIVER_STORE_NAME, 'data.mdb');
|
|
243
|
+
for (const [name, dir] of NATIVE_WORLD_STATE_DBS){
|
|
244
|
+
downloadPaths[name] = join(downloadDir, WORLD_STATE_DIR, dir, 'data.mdb');
|
|
245
|
+
}
|
|
183
246
|
log.info(`Downloading snapshot to ${downloadDir}`, {
|
|
184
247
|
snapshot,
|
|
185
248
|
downloadPaths
|
|
186
249
|
});
|
|
187
|
-
|
|
250
|
+
if (config.validateArchiver) {
|
|
251
|
+
await downloadSnapshot(snapshot, downloadPaths, fileStore, [
|
|
252
|
+
'archiver'
|
|
253
|
+
]);
|
|
254
|
+
await config.validateArchiver(downloadDir);
|
|
255
|
+
await downloadSnapshot(snapshot, downloadPaths, fileStore, SnapshotDataKeys.filter((key)=>key !== 'archiver'));
|
|
256
|
+
} else {
|
|
257
|
+
await downloadSnapshot(snapshot, downloadPaths, fileStore);
|
|
258
|
+
}
|
|
259
|
+
await config.validate?.(downloadDir);
|
|
188
260
|
log.info(`Snapshot downloaded at ${downloadDir}`, {
|
|
189
261
|
snapshot,
|
|
190
262
|
downloadPaths
|
|
@@ -199,14 +271,14 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
199
271
|
rollupAddress
|
|
200
272
|
});
|
|
201
273
|
// Same for the world state dbs, only that we do not close them, since we assume they are not yet in use
|
|
202
|
-
const worldStateBasePath = join(dataDirectory, WORLD_STATE_DIR);
|
|
274
|
+
const worldStateBasePath = join(config.worldStateDataDirectory ?? dataDirectory, WORLD_STATE_DIR);
|
|
203
275
|
await prepareTarget(worldStateBasePath, WORLD_STATE_DB_VERSION, rollupAddress);
|
|
204
276
|
for (const [name, dir] of NATIVE_WORLD_STATE_DBS){
|
|
205
277
|
const path = join(worldStateBasePath, dir);
|
|
206
278
|
await mkdir(path, {
|
|
207
279
|
recursive: true
|
|
208
280
|
});
|
|
209
|
-
await
|
|
281
|
+
await moveSnapshotFile(downloadPaths[name], join(path, 'data.mdb'));
|
|
210
282
|
log.info(`World state database ${name} set up from snapshot`, {
|
|
211
283
|
path,
|
|
212
284
|
dbVersion: WORLD_STATE_DB_VERSION,
|
|
@@ -225,6 +297,54 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
225
297
|
}
|
|
226
298
|
}
|
|
227
299
|
}
|
|
300
|
+
async function moveSnapshotFile(source, destination) {
|
|
301
|
+
try {
|
|
302
|
+
await rename(source, destination);
|
|
303
|
+
} catch (err) {
|
|
304
|
+
if (!err || typeof err !== 'object' || !('code' in err) || err.code !== 'EXDEV') {
|
|
305
|
+
throw err;
|
|
306
|
+
}
|
|
307
|
+
await copyFile(source, destination);
|
|
308
|
+
await unlink(source);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
async function hasWorldStateDatabase(dataDirectory) {
|
|
312
|
+
const sizes = await Promise.all(NATIVE_WORLD_STATE_DBS.map(async ([, directory])=>{
|
|
313
|
+
try {
|
|
314
|
+
return (await stat(join(dataDirectory, WORLD_STATE_DIR, directory, 'data.mdb'))).size;
|
|
315
|
+
} catch (err) {
|
|
316
|
+
if (err && typeof err === 'object' && 'code' in err && err.code === 'ENOENT') {
|
|
317
|
+
return 0;
|
|
318
|
+
}
|
|
319
|
+
throw err;
|
|
320
|
+
}
|
|
321
|
+
}));
|
|
322
|
+
return sizes.some((size)=>size > 0);
|
|
323
|
+
}
|
|
324
|
+
async function validateLocalWorldStateHistory(config, log, genesis) {
|
|
325
|
+
const worldState = await createWorldState(config, genesis);
|
|
326
|
+
try {
|
|
327
|
+
const summary = await worldState.getStatusSummary();
|
|
328
|
+
if (getWorldStateHistoryStartCheckpoint(config) !== 0 && summary.oldestHistoricalBlock > 1) {
|
|
329
|
+
const archiverStore = await createArchiverStore(config, GENESIS_BLOCK_HEADER_HASH);
|
|
330
|
+
try {
|
|
331
|
+
await validateWorldStateHistory(config, summary, archiverStore.blocks);
|
|
332
|
+
} finally{
|
|
333
|
+
await archiverStore.db.close();
|
|
334
|
+
}
|
|
335
|
+
} else {
|
|
336
|
+
await validateWorldStateHistory(config, summary);
|
|
337
|
+
}
|
|
338
|
+
log.info('Validated retained world-state history', {
|
|
339
|
+
oldestHistoricBlock: summary.oldestHistoricalBlock,
|
|
340
|
+
latestBlock: summary.unfinalizedBlockNumber,
|
|
341
|
+
historyStartCheckpoint: getWorldStateHistoryStartCheckpoint(config),
|
|
342
|
+
checkpointHistory: config.worldStateCheckpointHistory
|
|
343
|
+
});
|
|
344
|
+
} finally{
|
|
345
|
+
await worldState.close();
|
|
346
|
+
}
|
|
347
|
+
}
|
|
228
348
|
/** Deletes target dir and writes the new version file. */ async function prepareTarget(target, schemaVersion, rollupAddress) {
|
|
229
349
|
const noOpen = ()=>Promise.resolve(undefined);
|
|
230
350
|
const versionManager = new DatabaseVersionManager({
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/node-lib",
|
|
3
|
-
"version": "5.3.0-nightly.
|
|
3
|
+
"version": "5.3.0-nightly.20261017",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"typedocOptions": {
|
|
6
6
|
"entryPoints": [
|
|
@@ -66,28 +66,28 @@
|
|
|
66
66
|
]
|
|
67
67
|
},
|
|
68
68
|
"dependencies": {
|
|
69
|
-
"@aztec/archiver": "5.3.0-nightly.
|
|
70
|
-
"@aztec/bb-prover": "5.3.0-nightly.
|
|
71
|
-
"@aztec/blob-client": "5.3.0-nightly.
|
|
72
|
-
"@aztec/constants": "5.3.0-nightly.
|
|
73
|
-
"@aztec/epoch-cache": "5.3.0-nightly.
|
|
74
|
-
"@aztec/ethereum": "5.3.0-nightly.
|
|
75
|
-
"@aztec/foundation": "5.3.0-nightly.
|
|
76
|
-
"@aztec/kv-store": "5.3.0-nightly.
|
|
77
|
-
"@aztec/p2p": "5.3.0-nightly.
|
|
78
|
-
"@aztec/protocol-contracts": "5.3.0-nightly.
|
|
79
|
-
"@aztec/prover-client": "5.3.0-nightly.
|
|
80
|
-
"@aztec/sequencer-client": "5.3.0-nightly.
|
|
81
|
-
"@aztec/simulator": "5.3.0-nightly.
|
|
82
|
-
"@aztec/stdlib": "5.3.0-nightly.
|
|
83
|
-
"@aztec/telemetry-client": "5.3.0-nightly.
|
|
84
|
-
"@aztec/validator-client": "5.3.0-nightly.
|
|
85
|
-
"@aztec/world-state": "5.3.0-nightly.
|
|
69
|
+
"@aztec/archiver": "5.3.0-nightly.20261017",
|
|
70
|
+
"@aztec/bb-prover": "5.3.0-nightly.20261017",
|
|
71
|
+
"@aztec/blob-client": "5.3.0-nightly.20261017",
|
|
72
|
+
"@aztec/constants": "5.3.0-nightly.20261017",
|
|
73
|
+
"@aztec/epoch-cache": "5.3.0-nightly.20261017",
|
|
74
|
+
"@aztec/ethereum": "5.3.0-nightly.20261017",
|
|
75
|
+
"@aztec/foundation": "5.3.0-nightly.20261017",
|
|
76
|
+
"@aztec/kv-store": "5.3.0-nightly.20261017",
|
|
77
|
+
"@aztec/p2p": "5.3.0-nightly.20261017",
|
|
78
|
+
"@aztec/protocol-contracts": "5.3.0-nightly.20261017",
|
|
79
|
+
"@aztec/prover-client": "5.3.0-nightly.20261017",
|
|
80
|
+
"@aztec/sequencer-client": "5.3.0-nightly.20261017",
|
|
81
|
+
"@aztec/simulator": "5.3.0-nightly.20261017",
|
|
82
|
+
"@aztec/stdlib": "5.3.0-nightly.20261017",
|
|
83
|
+
"@aztec/telemetry-client": "5.3.0-nightly.20261017",
|
|
84
|
+
"@aztec/validator-client": "5.3.0-nightly.20261017",
|
|
85
|
+
"@aztec/world-state": "5.3.0-nightly.20261017",
|
|
86
86
|
"tslib": "^2.4.0"
|
|
87
87
|
},
|
|
88
88
|
"devDependencies": {
|
|
89
|
-
"@aztec/blob-lib": "5.3.0-nightly.
|
|
90
|
-
"@aztec/node-keystore": "5.3.0-nightly.
|
|
89
|
+
"@aztec/blob-lib": "5.3.0-nightly.20261017",
|
|
90
|
+
"@aztec/node-keystore": "5.3.0-nightly.20261017",
|
|
91
91
|
"@jest/globals": "^30.0.0",
|
|
92
92
|
"@types/jest": "^30.0.0",
|
|
93
93
|
"@types/node": "^22.15.17",
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
|
|
9
9
|
import { type EthereumClientConfig, getPublicClient } from '@aztec/ethereum/client';
|
|
10
10
|
import type { L1ContractsConfig } from '@aztec/ethereum/config';
|
|
11
|
+
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
11
12
|
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
12
13
|
import { tryRmDir } from '@aztec/foundation/fs';
|
|
13
14
|
import type { Logger } from '@aztec/foundation/log';
|
|
@@ -18,15 +19,25 @@ import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
|
|
|
18
19
|
import { type ReadOnlyFileStore, createReadOnlyFileStore } from '@aztec/stdlib/file-store';
|
|
19
20
|
import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
|
|
20
21
|
import {
|
|
22
|
+
SnapshotDataKeys,
|
|
21
23
|
type SnapshotMetadata,
|
|
22
24
|
type SnapshotsIndexMetadata,
|
|
23
25
|
downloadSnapshot,
|
|
24
|
-
|
|
26
|
+
getSnapshotIndex,
|
|
25
27
|
makeSnapshotPaths,
|
|
26
28
|
} from '@aztec/stdlib/snapshots';
|
|
27
|
-
import {
|
|
28
|
-
|
|
29
|
-
|
|
29
|
+
import type { GenesisData } from '@aztec/stdlib/world-state';
|
|
30
|
+
import {
|
|
31
|
+
NATIVE_WORLD_STATE_DBS,
|
|
32
|
+
WORLD_STATE_DB_VERSION,
|
|
33
|
+
WORLD_STATE_DIR,
|
|
34
|
+
type WorldStateConfig,
|
|
35
|
+
createWorldState,
|
|
36
|
+
getWorldStateHistoryStartCheckpoint,
|
|
37
|
+
validateWorldStateHistory,
|
|
38
|
+
} from '@aztec/world-state';
|
|
39
|
+
|
|
40
|
+
import { copyFile, mkdir, mkdtemp, rename, stat, unlink } from 'fs/promises';
|
|
30
41
|
import { join } from 'path';
|
|
31
42
|
|
|
32
43
|
import type { SharedNodeConfig } from '../config/index.js';
|
|
@@ -38,6 +49,7 @@ type SnapshotSyncConfig = Pick<SharedNodeConfig, 'syncMode'> &
|
|
|
38
49
|
Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'> &
|
|
39
50
|
Pick<L1ContractsConfig, 'aztecEpochDuration'> &
|
|
40
51
|
Pick<ArchiverConfig, 'archiverStoreMapSizeKb'> &
|
|
52
|
+
WorldStateConfig &
|
|
41
53
|
DataStoreConfig &
|
|
42
54
|
Required<Pick<DataStoreConfig, 'rollupAddress'>> &
|
|
43
55
|
EthereumClientConfig & {
|
|
@@ -46,11 +58,23 @@ type SnapshotSyncConfig = Pick<SharedNodeConfig, 'syncMode'> &
|
|
|
46
58
|
};
|
|
47
59
|
|
|
48
60
|
/**
|
|
49
|
-
*
|
|
50
|
-
*
|
|
61
|
+
* Validates local history and restores the newest compatible snapshot when bootstrap is needed.
|
|
62
|
+
* Fixed history boundaries are resolved against archived block and checkpoint metadata.
|
|
51
63
|
*/
|
|
52
|
-
export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
64
|
+
export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger, genesis?: GenesisData) {
|
|
53
65
|
const { syncMode, snapshotsUrls, dataDirectory, l1ChainId, rollupVersion, rollupAddress } = config;
|
|
66
|
+
const historyStartCheckpoint = config.worldStatePruneBeforeCheckpoint;
|
|
67
|
+
let maxSnapshotBlock = historyStartCheckpoint === 0 ? BlockNumber.ZERO : undefined;
|
|
68
|
+
const worldStateDirectory = config.worldStateDataDirectory ?? dataDirectory;
|
|
69
|
+
if (
|
|
70
|
+
historyStartCheckpoint !== undefined &&
|
|
71
|
+
worldStateDirectory &&
|
|
72
|
+
(await hasWorldStateDatabase(worldStateDirectory))
|
|
73
|
+
) {
|
|
74
|
+
await validateLocalWorldStateHistory(config, log, genesis);
|
|
75
|
+
log.info('Using existing world-state database with compatible retained history', { historyStartCheckpoint });
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
54
78
|
if (syncMode === 'full') {
|
|
55
79
|
log.debug('Snapshot sync is disabled. Running full sync.', { syncMode: syncMode });
|
|
56
80
|
return false;
|
|
@@ -84,7 +108,12 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
84
108
|
}
|
|
85
109
|
|
|
86
110
|
const minL1BlocksToTriggerReplace = config.minL1BlocksToTriggerReplace ?? MIN_L1_BLOCKS_TO_TRIGGER_REPLACE;
|
|
87
|
-
if (
|
|
111
|
+
if (
|
|
112
|
+
historyStartCheckpoint === undefined &&
|
|
113
|
+
syncMode === 'snapshot' &&
|
|
114
|
+
archiverL2BlockNumber !== undefined &&
|
|
115
|
+
archiverL2BlockNumber >= INITIAL_L2_BLOCK_NUM
|
|
116
|
+
) {
|
|
88
117
|
log.verbose(
|
|
89
118
|
`Skipping non-forced snapshot sync as archiver is already synced to L2 block ${archiverL2BlockNumber}.`,
|
|
90
119
|
);
|
|
@@ -93,6 +122,7 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
93
122
|
|
|
94
123
|
const currentL1BlockNumber = await getPublicClient(config).getBlockNumber();
|
|
95
124
|
if (
|
|
125
|
+
historyStartCheckpoint === undefined &&
|
|
96
126
|
archiverL1BlockNumber &&
|
|
97
127
|
currentL1BlockNumber >= archiverL1BlockNumber &&
|
|
98
128
|
currentL1BlockNumber - archiverL1BlockNumber < minL1BlocksToTriggerReplace
|
|
@@ -112,7 +142,7 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
112
142
|
rollupAddress,
|
|
113
143
|
};
|
|
114
144
|
|
|
115
|
-
//
|
|
145
|
+
// Index entries may outlive their database files, so retain older candidates for download fallback.
|
|
116
146
|
type SnapshotCandidate = { snapshot: SnapshotMetadata; url: string; fileStore: ReadOnlyFileStore };
|
|
117
147
|
const snapshotCandidates: SnapshotCandidate[] = [];
|
|
118
148
|
|
|
@@ -125,57 +155,67 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
125
155
|
continue;
|
|
126
156
|
}
|
|
127
157
|
|
|
128
|
-
let
|
|
158
|
+
let snapshots: SnapshotMetadata[];
|
|
129
159
|
try {
|
|
130
|
-
|
|
160
|
+
snapshots = (await getSnapshotIndex(indexMetadata, fileStore))?.snapshots ?? [];
|
|
131
161
|
} catch (err) {
|
|
132
|
-
log.error(`Failed to
|
|
162
|
+
log.error(`Failed to read snapshot index from ${snapshotsUrl}. Skipping this URL.`, err, {
|
|
133
163
|
...indexMetadata,
|
|
134
164
|
snapshotsUrl,
|
|
135
165
|
});
|
|
136
166
|
continue;
|
|
137
167
|
}
|
|
138
168
|
|
|
139
|
-
if (
|
|
169
|
+
if (snapshots.length === 0) {
|
|
140
170
|
log.verbose(`No snapshot found at ${snapshotsUrl}. Skipping this URL.`, { ...indexMetadata, snapshotsUrl });
|
|
141
171
|
continue;
|
|
142
172
|
}
|
|
143
173
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
174
|
+
for (const snapshot of snapshots) {
|
|
175
|
+
if (snapshot.schemaVersions.archiver !== ARCHIVER_DB_VERSION) {
|
|
176
|
+
log.warn(
|
|
177
|
+
`Skipping snapshot from ${snapshotsUrl} as it has schema version ${snapshot.schemaVersions.archiver} but expected ${ARCHIVER_DB_VERSION}.`,
|
|
178
|
+
snapshot,
|
|
179
|
+
);
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
if (snapshot.schemaVersions.worldState !== WORLD_STATE_DB_VERSION) {
|
|
184
|
+
log.warn(
|
|
185
|
+
`Skipping snapshot from ${snapshotsUrl} as it has world state schema version ${snapshot.schemaVersions.worldState} but we expected ${WORLD_STATE_DB_VERSION}.`,
|
|
186
|
+
snapshot,
|
|
187
|
+
);
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
if (
|
|
192
|
+
historyStartCheckpoint === undefined &&
|
|
193
|
+
archiverL1BlockNumber &&
|
|
194
|
+
snapshot.l1BlockNumber < archiverL1BlockNumber
|
|
195
|
+
) {
|
|
196
|
+
log.verbose(
|
|
197
|
+
`Skipping snapshot from ${snapshotsUrl} since local archiver is at L1 block ${archiverL1BlockNumber} which is further than snapshot at ${snapshot.l1BlockNumber}`,
|
|
198
|
+
{ snapshot, archiverL1BlockNumber, snapshotsUrl },
|
|
199
|
+
);
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
if (
|
|
204
|
+
historyStartCheckpoint === undefined &&
|
|
205
|
+
archiverL1BlockNumber &&
|
|
206
|
+
snapshot.l1BlockNumber - Number(archiverL1BlockNumber) < minL1BlocksToTriggerReplace
|
|
207
|
+
) {
|
|
208
|
+
log.verbose(
|
|
209
|
+
`Skipping snapshot from ${snapshotsUrl} as archiver is less than ${
|
|
210
|
+
snapshot.l1BlockNumber - Number(archiverL1BlockNumber)
|
|
211
|
+
} L1 blocks behind this snapshot.`,
|
|
212
|
+
{ snapshot, archiverL1BlockNumber, snapshotsUrl },
|
|
213
|
+
);
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
snapshotCandidates.push({ snapshot, url: snapshotsUrl, fileStore });
|
|
176
218
|
}
|
|
177
|
-
|
|
178
|
-
snapshotCandidates.push({ snapshot, url: snapshotsUrl, fileStore });
|
|
179
219
|
}
|
|
180
220
|
|
|
181
221
|
if (snapshotCandidates.length === 0) {
|
|
@@ -183,11 +223,17 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
183
223
|
return false;
|
|
184
224
|
}
|
|
185
225
|
|
|
186
|
-
|
|
187
|
-
|
|
226
|
+
snapshotCandidates.sort((a, b) =>
|
|
227
|
+
historyStartCheckpoint === undefined
|
|
228
|
+
? b.snapshot.l1BlockNumber - a.snapshot.l1BlockNumber
|
|
229
|
+
: b.snapshot.l2BlockNumber - a.snapshot.l2BlockNumber || b.snapshot.l1BlockNumber - a.snapshot.l1BlockNumber,
|
|
230
|
+
);
|
|
188
231
|
|
|
189
232
|
// Try each candidate in order until one succeeds
|
|
190
233
|
for (const { snapshot, url, fileStore } of snapshotCandidates) {
|
|
234
|
+
if (maxSnapshotBlock !== undefined && snapshot.l2BlockNumber > maxSnapshotBlock) {
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
191
237
|
const { l1BlockNumber, l2BlockNumber } = snapshot;
|
|
192
238
|
log.info(`Attempting to sync from snapshot at L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, {
|
|
193
239
|
snapshot,
|
|
@@ -195,10 +241,55 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
195
241
|
});
|
|
196
242
|
|
|
197
243
|
try {
|
|
244
|
+
const available = await Promise.all(Object.values(snapshot.dataUrls).map(path => fileStore.exists(path)));
|
|
245
|
+
if (!available.every(Boolean)) {
|
|
246
|
+
log.warn('Skipping snapshot with missing database files', { l1BlockNumber, l2BlockNumber, snapshotsUrl: url });
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
198
249
|
await snapshotSync(snapshot, log, {
|
|
199
250
|
dataDirectory: config.dataDirectory!,
|
|
200
251
|
rollupAddress: config.rollupAddress,
|
|
201
252
|
fileStore,
|
|
253
|
+
worldStateDataDirectory: config.worldStateDataDirectory,
|
|
254
|
+
validateArchiver:
|
|
255
|
+
historyStartCheckpoint === undefined || historyStartCheckpoint === 0
|
|
256
|
+
? undefined
|
|
257
|
+
: async stagingDirectory => {
|
|
258
|
+
const stores = await createArchiverStore(
|
|
259
|
+
{ ...config, dataDirectory: stagingDirectory },
|
|
260
|
+
GENESIS_BLOCK_HEADER_HASH,
|
|
261
|
+
);
|
|
262
|
+
try {
|
|
263
|
+
const latestBlockNumber = await stores.blocks.getLatestL2BlockNumber();
|
|
264
|
+
if (latestBlockNumber === undefined) {
|
|
265
|
+
throw new Error('Snapshot archiver has no blocks');
|
|
266
|
+
}
|
|
267
|
+
const latestBlock = await stores.blocks.getBlockData({ number: latestBlockNumber });
|
|
268
|
+
if (!latestBlock) {
|
|
269
|
+
throw new Error(`Snapshot archiver is missing block ${latestBlockNumber}`);
|
|
270
|
+
}
|
|
271
|
+
if (latestBlock.checkpointNumber > historyStartCheckpoint) {
|
|
272
|
+
const checkpoint = await stores.blocks.getCheckpointData(historyStartCheckpoint);
|
|
273
|
+
if (checkpoint) {
|
|
274
|
+
maxSnapshotBlock = BlockNumber(checkpoint.startBlock + checkpoint.blockCount - 1);
|
|
275
|
+
}
|
|
276
|
+
throw new Error(
|
|
277
|
+
`Snapshot is at checkpoint ${latestBlock.checkpointNumber}, after requested checkpoint ${historyStartCheckpoint}`,
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
} finally {
|
|
281
|
+
await stores.db.close();
|
|
282
|
+
}
|
|
283
|
+
},
|
|
284
|
+
validate:
|
|
285
|
+
historyStartCheckpoint === undefined
|
|
286
|
+
? undefined
|
|
287
|
+
: stagingDirectory =>
|
|
288
|
+
validateLocalWorldStateHistory(
|
|
289
|
+
{ ...config, dataDirectory: stagingDirectory, worldStateDataDirectory: stagingDirectory },
|
|
290
|
+
log,
|
|
291
|
+
genesis,
|
|
292
|
+
),
|
|
202
293
|
});
|
|
203
294
|
log.info(`Snapshot synced to L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, {
|
|
204
295
|
snapshot,
|
|
@@ -224,7 +315,14 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
224
315
|
export async function snapshotSync(
|
|
225
316
|
snapshot: Pick<SnapshotMetadata, 'dataUrls'>,
|
|
226
317
|
log: Logger,
|
|
227
|
-
config: {
|
|
318
|
+
config: {
|
|
319
|
+
dataDirectory: string;
|
|
320
|
+
rollupAddress: EthAddress;
|
|
321
|
+
fileStore: ReadOnlyFileStore;
|
|
322
|
+
worldStateDataDirectory?: string;
|
|
323
|
+
validateArchiver?: (stagingDirectory: string) => Promise<void>;
|
|
324
|
+
validate?: (stagingDirectory: string) => Promise<void>;
|
|
325
|
+
},
|
|
228
326
|
) {
|
|
229
327
|
const { dataDirectory, rollupAddress, fileStore } = config;
|
|
230
328
|
if (!dataDirectory) {
|
|
@@ -238,8 +336,26 @@ export async function snapshotSync(
|
|
|
238
336
|
await mkdir(dataDirectory, { recursive: true });
|
|
239
337
|
downloadDir = await mkdtemp(join(dataDirectory, 'download-'));
|
|
240
338
|
const downloadPaths = makeSnapshotPaths(downloadDir);
|
|
339
|
+
await prepareTarget(join(downloadDir, ARCHIVER_STORE_NAME), ARCHIVER_DB_VERSION, rollupAddress);
|
|
340
|
+
await prepareTarget(join(downloadDir, WORLD_STATE_DIR), WORLD_STATE_DB_VERSION, rollupAddress);
|
|
341
|
+
downloadPaths.archiver = join(downloadDir, ARCHIVER_STORE_NAME, 'data.mdb');
|
|
342
|
+
for (const [name, dir] of NATIVE_WORLD_STATE_DBS) {
|
|
343
|
+
downloadPaths[name] = join(downloadDir, WORLD_STATE_DIR, dir, 'data.mdb');
|
|
344
|
+
}
|
|
241
345
|
log.info(`Downloading snapshot to ${downloadDir}`, { snapshot, downloadPaths });
|
|
242
|
-
|
|
346
|
+
if (config.validateArchiver) {
|
|
347
|
+
await downloadSnapshot(snapshot, downloadPaths, fileStore, ['archiver']);
|
|
348
|
+
await config.validateArchiver(downloadDir);
|
|
349
|
+
await downloadSnapshot(
|
|
350
|
+
snapshot,
|
|
351
|
+
downloadPaths,
|
|
352
|
+
fileStore,
|
|
353
|
+
SnapshotDataKeys.filter(key => key !== 'archiver'),
|
|
354
|
+
);
|
|
355
|
+
} else {
|
|
356
|
+
await downloadSnapshot(snapshot, downloadPaths, fileStore);
|
|
357
|
+
}
|
|
358
|
+
await config.validate?.(downloadDir);
|
|
243
359
|
log.info(`Snapshot downloaded at ${downloadDir}`, { snapshot, downloadPaths });
|
|
244
360
|
|
|
245
361
|
// If download was successful, clear lock and version, and move download there
|
|
@@ -253,12 +369,12 @@ export async function snapshotSync(
|
|
|
253
369
|
});
|
|
254
370
|
|
|
255
371
|
// Same for the world state dbs, only that we do not close them, since we assume they are not yet in use
|
|
256
|
-
const worldStateBasePath = join(dataDirectory, WORLD_STATE_DIR);
|
|
372
|
+
const worldStateBasePath = join(config.worldStateDataDirectory ?? dataDirectory, WORLD_STATE_DIR);
|
|
257
373
|
await prepareTarget(worldStateBasePath, WORLD_STATE_DB_VERSION, rollupAddress);
|
|
258
374
|
for (const [name, dir] of NATIVE_WORLD_STATE_DBS) {
|
|
259
375
|
const path = join(worldStateBasePath, dir);
|
|
260
376
|
await mkdir(path, { recursive: true });
|
|
261
|
-
await
|
|
377
|
+
await moveSnapshotFile(downloadPaths[name], join(path, 'data.mdb'));
|
|
262
378
|
log.info(`World state database ${name} set up from snapshot`, {
|
|
263
379
|
path,
|
|
264
380
|
dbVersion: WORLD_STATE_DB_VERSION,
|
|
@@ -277,6 +393,59 @@ export async function snapshotSync(
|
|
|
277
393
|
}
|
|
278
394
|
}
|
|
279
395
|
|
|
396
|
+
async function moveSnapshotFile(source: string, destination: string) {
|
|
397
|
+
try {
|
|
398
|
+
await rename(source, destination);
|
|
399
|
+
} catch (err) {
|
|
400
|
+
if (!err || typeof err !== 'object' || !('code' in err) || err.code !== 'EXDEV') {
|
|
401
|
+
throw err;
|
|
402
|
+
}
|
|
403
|
+
await copyFile(source, destination);
|
|
404
|
+
await unlink(source);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
async function hasWorldStateDatabase(dataDirectory: string): Promise<boolean> {
|
|
409
|
+
const sizes = await Promise.all(
|
|
410
|
+
NATIVE_WORLD_STATE_DBS.map(async ([, directory]) => {
|
|
411
|
+
try {
|
|
412
|
+
return (await stat(join(dataDirectory, WORLD_STATE_DIR, directory, 'data.mdb'))).size;
|
|
413
|
+
} catch (err) {
|
|
414
|
+
if (err && typeof err === 'object' && 'code' in err && err.code === 'ENOENT') {
|
|
415
|
+
return 0;
|
|
416
|
+
}
|
|
417
|
+
throw err;
|
|
418
|
+
}
|
|
419
|
+
}),
|
|
420
|
+
);
|
|
421
|
+
return sizes.some(size => size > 0);
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
async function validateLocalWorldStateHistory(config: SnapshotSyncConfig, log: Logger, genesis?: GenesisData) {
|
|
425
|
+
const worldState = await createWorldState(config, genesis);
|
|
426
|
+
try {
|
|
427
|
+
const summary = await worldState.getStatusSummary();
|
|
428
|
+
if (getWorldStateHistoryStartCheckpoint(config) !== 0 && summary.oldestHistoricalBlock > 1) {
|
|
429
|
+
const archiverStore = await createArchiverStore(config, GENESIS_BLOCK_HEADER_HASH);
|
|
430
|
+
try {
|
|
431
|
+
await validateWorldStateHistory(config, summary, archiverStore.blocks);
|
|
432
|
+
} finally {
|
|
433
|
+
await archiverStore.db.close();
|
|
434
|
+
}
|
|
435
|
+
} else {
|
|
436
|
+
await validateWorldStateHistory(config, summary);
|
|
437
|
+
}
|
|
438
|
+
log.info('Validated retained world-state history', {
|
|
439
|
+
oldestHistoricBlock: summary.oldestHistoricalBlock,
|
|
440
|
+
latestBlock: summary.unfinalizedBlockNumber,
|
|
441
|
+
historyStartCheckpoint: getWorldStateHistoryStartCheckpoint(config),
|
|
442
|
+
checkpointHistory: config.worldStateCheckpointHistory,
|
|
443
|
+
});
|
|
444
|
+
} finally {
|
|
445
|
+
await worldState.close();
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
280
449
|
/** Deletes target dir and writes the new version file. */
|
|
281
450
|
async function prepareTarget(target: string, schemaVersion: number, rollupAddress: EthAddress) {
|
|
282
451
|
const noOpen = () => Promise.resolve(undefined);
|