@aztec-labs/node-lib 6.0.0-nightly.20261020 → 6.0.0-nightly.20261022
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -7,16 +7,18 @@ import type { ChainConfig } from '@aztec-labs/stdlib/config';
|
|
|
7
7
|
import { type ReadOnlyFileStore } from '@aztec-labs/stdlib/file-store';
|
|
8
8
|
import type { DataStoreConfig } from '@aztec-labs/stdlib/kv-store';
|
|
9
9
|
import { type SnapshotMetadata } from '@aztec-labs/stdlib/snapshots';
|
|
10
|
+
import type { GenesisData } from '@aztec-labs/stdlib/world-state';
|
|
11
|
+
import { type WorldStateConfig } from '@aztec-labs/world-state';
|
|
10
12
|
import type { SharedNodeConfig } from '../config/index.js';
|
|
11
|
-
type SnapshotSyncConfig = Pick<SharedNodeConfig, 'syncMode'> & Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'> & Pick<L1ContractsConfig, 'aztecEpochDuration'> & Pick<ArchiverConfig, 'archiverStoreMapSizeKb'> & DataStoreConfig & Required<Pick<DataStoreConfig, 'rollupAddress'>> & EthereumClientConfig & {
|
|
13
|
+
type SnapshotSyncConfig = Pick<SharedNodeConfig, 'syncMode'> & Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'> & Pick<L1ContractsConfig, 'aztecEpochDuration'> & Pick<ArchiverConfig, 'archiverStoreMapSizeKb'> & WorldStateConfig & DataStoreConfig & Required<Pick<DataStoreConfig, 'rollupAddress'>> & EthereumClientConfig & {
|
|
12
14
|
snapshotsUrls?: string[];
|
|
13
15
|
minL1BlocksToTriggerReplace?: number;
|
|
14
16
|
};
|
|
15
17
|
/**
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
+
* Validates local history and restores the newest compatible snapshot when bootstrap is needed.
|
|
19
|
+
* Fixed history boundaries are resolved against archived block and checkpoint metadata.
|
|
18
20
|
*/
|
|
19
|
-
export declare function trySnapshotSync(config: SnapshotSyncConfig, log: Logger): Promise<boolean>;
|
|
21
|
+
export declare function trySnapshotSync(config: SnapshotSyncConfig, log: Logger, genesis?: GenesisData): Promise<boolean>;
|
|
20
22
|
/**
|
|
21
23
|
* Downloads the given snapshot replacing any local data stores.
|
|
22
24
|
*/
|
|
@@ -24,6 +26,9 @@ export declare function snapshotSync(snapshot: Pick<SnapshotMetadata, 'dataUrls'
|
|
|
24
26
|
dataDirectory: string;
|
|
25
27
|
rollupAddress: EthAddress;
|
|
26
28
|
fileStore: ReadOnlyFileStore;
|
|
29
|
+
worldStateDataDirectory?: string;
|
|
30
|
+
validateArchiver?: (stagingDirectory: string) => Promise<void>;
|
|
31
|
+
validate?: (stagingDirectory: string) => Promise<void>;
|
|
27
32
|
}): Promise<void>;
|
|
28
33
|
export {};
|
|
29
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
34
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic25hcHNob3Qtc3luYy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2FjdGlvbnMvc25hcHNob3Qtc3luYy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBR0wsS0FBSyxjQUFjLEVBR3BCLE1BQU0sc0JBQXNCLENBQUM7QUFFOUIsT0FBTyxFQUFFLEtBQUssb0JBQW9CLEVBQW1CLE1BQU0sNkJBQTZCLENBQUM7QUFDekYsT0FBTyxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUVyRSxPQUFPLEtBQUssRUFBRSxVQUFVLEVBQUUsTUFBTSxvQ0FBb0MsQ0FBQztBQUVyRSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSw0QkFBNEIsQ0FBQztBQUd6RCxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsTUFBTSwyQkFBMkIsQ0FBQztBQUc3RCxPQUFPLEVBQUUsS0FBSyxpQkFBaUIsRUFBMkIsTUFBTSwrQkFBK0IsQ0FBQztBQUNoRyxPQUFPLEtBQUssRUFBRSxlQUFlLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUNuRSxPQUFPLEVBRUwsS0FBSyxnQkFBZ0IsRUFLdEIsTUFBTSw4QkFBOEIsQ0FBQztBQUN0QyxPQUFPLEtBQUssRUFBRSxXQUFXLEVBQUUsTUFBTSxnQ0FBZ0MsQ0FBQztBQUNsRSxPQUFPLEVBSUwsS0FBSyxnQkFBZ0IsRUFJdEIsTUFBTSx5QkFBeUIsQ0FBQztBQUlqQyxPQUFPLEtBQUssRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLG9CQUFvQixDQUFDO0FBSzNELEtBQUssa0JBQWtCLEdBQUcsSUFBSSxDQUFDLGdCQUFnQixFQUFFLFVBQVUsQ0FBQyxHQUMxRCxJQUFJLENBQUMsV0FBVyxFQUFFLFdBQVcsR0FBRyxlQUFlLENBQUMsR0FDaEQsSUFBSSxDQUFDLGlCQUFpQixFQUFFLG9CQUFvQixDQUFDLEdBQzdDLElBQUksQ0FBQyxjQUFjLEVBQUUsd0JBQXdCLENBQUMsR0FDOUMsZ0JBQWdCLEdBQ2hCLGVBQWUsR0FDZixRQUFRLENBQUMsSUFBSSxDQUFDLGVBQWUsRUFBRSxlQUFlLENBQUMsQ0FBQyxHQUNoRCxvQkFBb0IsR0FBRztJQUNyQixhQUFhLENBQUMsRUFBRSxNQUFNLEVBQUUsQ0FBQztJQUN6QiwyQkFBMkIsQ0FBQyxFQUFFLE1BQU0sQ0FBQztDQUN0QyxDQUFDO0FBRUo7OztHQUdHO0FBQ0gsd0JBQXNCLGVBQWUsQ0FBQyxNQUFNLEVBQUUsa0JBQWtCLEVBQUUsR0FBRyxFQUFFLE1BQU0sRUFBRSxPQUFPLENBQUMsRUFBRSxXQUFXLG9CQWlQbkc7QUFFRDs7R0FFRztBQUNILHdCQUFzQixZQUFZLENBQ2hDLFFBQVEsRUFBRSxJQUFJLENBQUMsZ0JBQWdCLEVBQUUsVUFBVSxDQUFDLEVBQzVDLEdBQUcsRUFBRSxNQUFNLEVBQ1gsTUFBTSxFQUFFO0lBQ04sYUFBYSxFQUFFLE1BQU0sQ0FBQztJQUN0QixhQUFhLEVBQUUsVUFBVSxDQUFDO0lBQzFCLFNBQVMsRUFBRSxpQkFBaUIsQ0FBQztJQUM3Qix1QkFBdUIsQ0FBQyxFQUFFLE1BQU0sQ0FBQztJQUNqQyxnQkFBZ0IsQ0FBQyxFQUFFLENBQUMsZ0JBQWdCLEVBQUUsTUFBTSxLQUFLLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUMvRCxRQUFRLENBQUMsRUFBRSxDQUFDLGdCQUFnQixFQUFFLE1BQU0sS0FBSyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7Q0FDeEQsaUJBcUVGIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot-sync.d.ts","sourceRoot":"","sources":["../../src/actions/snapshot-sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,cAAc,EAGpB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,KAAK,oBAAoB,EAAmB,MAAM,6BAA6B,CAAC;AACzF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"snapshot-sync.d.ts","sourceRoot":"","sources":["../../src/actions/snapshot-sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,cAAc,EAGpB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,KAAK,oBAAoB,EAAmB,MAAM,6BAA6B,CAAC;AACzF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAErE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oCAAoC,CAAC;AAErE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAGzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAG7D,OAAO,EAAE,KAAK,iBAAiB,EAA2B,MAAM,+BAA+B,CAAC;AAChG,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAEL,KAAK,gBAAgB,EAKtB,MAAM,8BAA8B,CAAC;AACtC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAIL,KAAK,gBAAgB,EAItB,MAAM,yBAAyB,CAAC;AAIjC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAK3D,KAAK,kBAAkB,GAAG,IAAI,CAAC,gBAAgB,EAAE,UAAU,CAAC,GAC1D,IAAI,CAAC,WAAW,EAAE,WAAW,GAAG,eAAe,CAAC,GAChD,IAAI,CAAC,iBAAiB,EAAE,oBAAoB,CAAC,GAC7C,IAAI,CAAC,cAAc,EAAE,wBAAwB,CAAC,GAC9C,gBAAgB,GAChB,eAAe,GACf,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC,GAChD,oBAAoB,GAAG;IACrB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACtC,CAAC;AAEJ;;;GAGG;AACH,wBAAsB,eAAe,CAAC,MAAM,EAAE,kBAAkB,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,oBAiPnG;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,EAAE,UAAU,CAAC,EAC5C,GAAG,EAAE,MAAM,EACX,MAAM,EAAE;IACN,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,UAAU,CAAC;IAC1B,SAAS,EAAE,iBAAiB,CAAC;IAC7B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,gBAAgB,CAAC,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/D,QAAQ,CAAC,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACxD,iBAqEF"}
|
|
@@ -1,22 +1,34 @@
|
|
|
1
1
|
import { ARCHIVER_DB_VERSION, ARCHIVER_STORE_NAME, createArchiverStore, getArchiverSynchPoint } from '@aztec-labs/archiver';
|
|
2
2
|
import { INITIAL_L2_BLOCK_NUM } from '@aztec-labs/constants';
|
|
3
3
|
import { getPublicClient } from '@aztec-labs/ethereum/client';
|
|
4
|
+
import { BlockNumber } from '@aztec-labs/foundation/branded-types';
|
|
4
5
|
import { tryRmDir } from '@aztec-labs/foundation/fs';
|
|
5
6
|
import { P2P_STORE_NAME } from '@aztec-labs/p2p';
|
|
6
7
|
import { GENESIS_BLOCK_HEADER_HASH } from '@aztec-labs/stdlib/block';
|
|
7
8
|
import { DatabaseVersionManager } from '@aztec-labs/stdlib/database-version/manager';
|
|
9
|
+
import { DatabaseVersion } from '@aztec-labs/stdlib/database-version/version';
|
|
8
10
|
import { createReadOnlyFileStore } from '@aztec-labs/stdlib/file-store';
|
|
9
|
-
import { downloadSnapshot,
|
|
10
|
-
import { NATIVE_WORLD_STATE_DBS, WORLD_STATE_DB_VERSION, WORLD_STATE_DIR } from '@aztec-labs/world-state';
|
|
11
|
-
import { mkdir, mkdtemp, rename } from 'fs/promises';
|
|
11
|
+
import { SnapshotDataKeys, downloadSnapshot, getSnapshotIndex, makeSnapshotPaths } from '@aztec-labs/stdlib/snapshots';
|
|
12
|
+
import { NATIVE_WORLD_STATE_DBS, WORLD_STATE_DB_VERSION, WORLD_STATE_DIR, createWorldState, getWorldStateHistoryStartCheckpoint, validateWorldStateHistory } from '@aztec-labs/world-state';
|
|
13
|
+
import { copyFile, mkdir, mkdtemp, readFile, rename, stat, unlink } from 'fs/promises';
|
|
12
14
|
import { join } from 'path';
|
|
13
15
|
// Half day worth of L1 blocks
|
|
14
16
|
const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
15
17
|
/**
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*/ export async function trySnapshotSync(config, log) {
|
|
18
|
+
* Validates local history and restores the newest compatible snapshot when bootstrap is needed.
|
|
19
|
+
* Fixed history boundaries are resolved against archived block and checkpoint metadata.
|
|
20
|
+
*/ export async function trySnapshotSync(config, log, genesis) {
|
|
19
21
|
const { syncMode, snapshotsUrls, dataDirectory, l1ChainId, rollupVersion, rollupAddress } = config;
|
|
22
|
+
const historyStartCheckpoint = getWorldStateHistoryStartCheckpoint(config);
|
|
23
|
+
let maxSnapshotBlock;
|
|
24
|
+
const worldStateDirectory = config.worldStateDataDirectory ?? dataDirectory;
|
|
25
|
+
if (historyStartCheckpoint !== undefined && worldStateDirectory && await hasWorldStateDatabase(worldStateDirectory, rollupAddress)) {
|
|
26
|
+
await validateLocalWorldStateHistory(config, log, genesis);
|
|
27
|
+
log.info('Using existing world-state database with compatible retained history', {
|
|
28
|
+
historyStartCheckpoint
|
|
29
|
+
});
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
20
32
|
if (syncMode === 'full') {
|
|
21
33
|
log.debug('Snapshot sync is disabled. Running full sync.', {
|
|
22
34
|
syncMode: syncMode
|
|
@@ -51,12 +63,12 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
51
63
|
await archiverStore.db.close();
|
|
52
64
|
}
|
|
53
65
|
const minL1BlocksToTriggerReplace = config.minL1BlocksToTriggerReplace ?? MIN_L1_BLOCKS_TO_TRIGGER_REPLACE;
|
|
54
|
-
if (syncMode === 'snapshot' && archiverL2BlockNumber !== undefined && archiverL2BlockNumber >= INITIAL_L2_BLOCK_NUM) {
|
|
66
|
+
if (historyStartCheckpoint === undefined && syncMode === 'snapshot' && archiverL2BlockNumber !== undefined && archiverL2BlockNumber >= INITIAL_L2_BLOCK_NUM) {
|
|
55
67
|
log.verbose(`Skipping non-forced snapshot sync as archiver is already synced to L2 block ${archiverL2BlockNumber}.`);
|
|
56
68
|
return false;
|
|
57
69
|
}
|
|
58
70
|
const currentL1BlockNumber = await getPublicClient(config).getBlockNumber();
|
|
59
|
-
if (archiverL1BlockNumber && currentL1BlockNumber >= archiverL1BlockNumber && currentL1BlockNumber - archiverL1BlockNumber < minL1BlocksToTriggerReplace) {
|
|
71
|
+
if (historyStartCheckpoint === undefined && archiverL1BlockNumber && currentL1BlockNumber >= archiverL1BlockNumber && currentL1BlockNumber - archiverL1BlockNumber < minL1BlocksToTriggerReplace) {
|
|
60
72
|
log.verbose(`Skipping snapshot sync as archiver is less than ${currentL1BlockNumber - archiverL1BlockNumber} L1 blocks behind.`, {
|
|
61
73
|
archiverL1BlockNumber,
|
|
62
74
|
currentL1BlockNumber,
|
|
@@ -78,52 +90,54 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
78
90
|
log.error(`Invalid config for downloading snapshots from ${snapshotsUrl}`, err);
|
|
79
91
|
continue;
|
|
80
92
|
}
|
|
81
|
-
let
|
|
93
|
+
let snapshots;
|
|
82
94
|
try {
|
|
83
|
-
|
|
95
|
+
snapshots = (await getSnapshotIndex(indexMetadata, fileStore))?.snapshots ?? [];
|
|
84
96
|
} catch (err) {
|
|
85
|
-
log.error(`Failed to
|
|
97
|
+
log.error(`Failed to read snapshot index from ${snapshotsUrl}. Skipping this URL.`, err, {
|
|
86
98
|
...indexMetadata,
|
|
87
99
|
snapshotsUrl
|
|
88
100
|
});
|
|
89
101
|
continue;
|
|
90
102
|
}
|
|
91
|
-
if (
|
|
103
|
+
if (snapshots.length === 0) {
|
|
92
104
|
log.verbose(`No snapshot found at ${snapshotsUrl}. Skipping this URL.`, {
|
|
93
105
|
...indexMetadata,
|
|
94
106
|
snapshotsUrl
|
|
95
107
|
});
|
|
96
108
|
continue;
|
|
97
109
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
110
|
+
for (const snapshot of snapshots){
|
|
111
|
+
if (snapshot.schemaVersions.archiver !== ARCHIVER_DB_VERSION) {
|
|
112
|
+
log.warn(`Skipping snapshot from ${snapshotsUrl} as it has schema version ${snapshot.schemaVersions.archiver} but expected ${ARCHIVER_DB_VERSION}.`, snapshot);
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
if (snapshot.schemaVersions.worldState !== WORLD_STATE_DB_VERSION) {
|
|
116
|
+
log.warn(`Skipping snapshot from ${snapshotsUrl} as it has world state schema version ${snapshot.schemaVersions.worldState} but we expected ${WORLD_STATE_DB_VERSION}.`, snapshot);
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
if (historyStartCheckpoint === undefined && archiverL1BlockNumber && snapshot.l1BlockNumber < archiverL1BlockNumber) {
|
|
120
|
+
log.verbose(`Skipping snapshot from ${snapshotsUrl} since local archiver is at L1 block ${archiverL1BlockNumber} which is further than snapshot at ${snapshot.l1BlockNumber}`, {
|
|
121
|
+
snapshot,
|
|
122
|
+
archiverL1BlockNumber,
|
|
123
|
+
snapshotsUrl
|
|
124
|
+
});
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
if (historyStartCheckpoint === undefined && archiverL1BlockNumber && snapshot.l1BlockNumber - Number(archiverL1BlockNumber) < minL1BlocksToTriggerReplace) {
|
|
128
|
+
log.verbose(`Skipping snapshot from ${snapshotsUrl} as archiver is less than ${snapshot.l1BlockNumber - Number(archiverL1BlockNumber)} L1 blocks behind this snapshot.`, {
|
|
129
|
+
snapshot,
|
|
130
|
+
archiverL1BlockNumber,
|
|
131
|
+
snapshotsUrl
|
|
132
|
+
});
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
snapshotCandidates.push({
|
|
108
136
|
snapshot,
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
});
|
|
112
|
-
continue;
|
|
113
|
-
}
|
|
114
|
-
if (archiverL1BlockNumber && snapshot.l1BlockNumber - Number(archiverL1BlockNumber) < minL1BlocksToTriggerReplace) {
|
|
115
|
-
log.verbose(`Skipping snapshot from ${snapshotsUrl} as archiver is less than ${snapshot.l1BlockNumber - Number(archiverL1BlockNumber)} L1 blocks behind this snapshot.`, {
|
|
116
|
-
snapshot,
|
|
117
|
-
archiverL1BlockNumber,
|
|
118
|
-
snapshotsUrl
|
|
137
|
+
url: snapshotsUrl,
|
|
138
|
+
fileStore
|
|
119
139
|
});
|
|
120
|
-
continue;
|
|
121
140
|
}
|
|
122
|
-
snapshotCandidates.push({
|
|
123
|
-
snapshot,
|
|
124
|
-
url: snapshotsUrl,
|
|
125
|
-
fileStore
|
|
126
|
-
});
|
|
127
141
|
}
|
|
128
142
|
if (snapshotCandidates.length === 0) {
|
|
129
143
|
log.verbose(`No valid snapshots found from any URL, skipping snapshot sync`, {
|
|
@@ -132,10 +146,12 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
132
146
|
});
|
|
133
147
|
return false;
|
|
134
148
|
}
|
|
135
|
-
|
|
136
|
-
snapshotCandidates.sort((a, b)=>b.snapshot.l1BlockNumber - a.snapshot.l1BlockNumber);
|
|
149
|
+
snapshotCandidates.sort((a, b)=>historyStartCheckpoint === undefined ? b.snapshot.l1BlockNumber - a.snapshot.l1BlockNumber : b.snapshot.l2BlockNumber - a.snapshot.l2BlockNumber || b.snapshot.l1BlockNumber - a.snapshot.l1BlockNumber);
|
|
137
150
|
// Try each candidate in order until one succeeds
|
|
138
151
|
for (const { snapshot, url, fileStore } of snapshotCandidates){
|
|
152
|
+
if (maxSnapshotBlock !== undefined && snapshot.l2BlockNumber > maxSnapshotBlock) {
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
139
155
|
const { l1BlockNumber, l2BlockNumber } = snapshot;
|
|
140
156
|
log.info(`Attempting to sync from snapshot at L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, {
|
|
141
157
|
snapshot,
|
|
@@ -145,7 +161,40 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
145
161
|
await snapshotSync(snapshot, log, {
|
|
146
162
|
dataDirectory: config.dataDirectory,
|
|
147
163
|
rollupAddress: config.rollupAddress,
|
|
148
|
-
fileStore
|
|
164
|
+
fileStore,
|
|
165
|
+
worldStateDataDirectory: config.worldStateDataDirectory,
|
|
166
|
+
validateArchiver: historyStartCheckpoint === undefined || historyStartCheckpoint === 0 ? undefined : async (stagingDirectory)=>{
|
|
167
|
+
const stores = await createArchiverStore({
|
|
168
|
+
...config,
|
|
169
|
+
dataDirectory: stagingDirectory
|
|
170
|
+
}, GENESIS_BLOCK_HEADER_HASH);
|
|
171
|
+
try {
|
|
172
|
+
const latestBlockNumber = await stores.blocks.getLatestL2BlockNumber();
|
|
173
|
+
if (latestBlockNumber === undefined) {
|
|
174
|
+
throw new Error('Snapshot archiver has no blocks');
|
|
175
|
+
}
|
|
176
|
+
const latestBlock = await stores.blocks.getBlockData({
|
|
177
|
+
number: latestBlockNumber
|
|
178
|
+
});
|
|
179
|
+
if (!latestBlock) {
|
|
180
|
+
throw new Error(`Snapshot archiver is missing block ${latestBlockNumber}`);
|
|
181
|
+
}
|
|
182
|
+
if (latestBlock.checkpointNumber > historyStartCheckpoint) {
|
|
183
|
+
const checkpoint = await stores.blocks.getCheckpointData(historyStartCheckpoint);
|
|
184
|
+
if (checkpoint) {
|
|
185
|
+
maxSnapshotBlock = BlockNumber(checkpoint.startBlock + checkpoint.blockCount - 1);
|
|
186
|
+
}
|
|
187
|
+
throw new Error(`Snapshot is at checkpoint ${latestBlock.checkpointNumber}, after requested checkpoint ${historyStartCheckpoint}`);
|
|
188
|
+
}
|
|
189
|
+
} finally{
|
|
190
|
+
await stores.db.close();
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
validate: historyStartCheckpoint === undefined ? undefined : (stagingDirectory)=>validateLocalWorldStateHistory({
|
|
194
|
+
...config,
|
|
195
|
+
dataDirectory: stagingDirectory,
|
|
196
|
+
worldStateDataDirectory: stagingDirectory
|
|
197
|
+
}, log, genesis)
|
|
149
198
|
});
|
|
150
199
|
log.info(`Snapshot synced to L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, {
|
|
151
200
|
snapshot,
|
|
@@ -180,11 +229,26 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
180
229
|
});
|
|
181
230
|
downloadDir = await mkdtemp(join(dataDirectory, 'download-'));
|
|
182
231
|
const downloadPaths = makeSnapshotPaths(downloadDir);
|
|
232
|
+
await prepareTarget(join(downloadDir, ARCHIVER_STORE_NAME), ARCHIVER_DB_VERSION, rollupAddress);
|
|
233
|
+
await prepareTarget(join(downloadDir, WORLD_STATE_DIR), WORLD_STATE_DB_VERSION, rollupAddress);
|
|
234
|
+
downloadPaths.archiver = join(downloadDir, ARCHIVER_STORE_NAME, 'data.mdb');
|
|
235
|
+
for (const [name, dir] of NATIVE_WORLD_STATE_DBS){
|
|
236
|
+
downloadPaths[name] = join(downloadDir, WORLD_STATE_DIR, dir, 'data.mdb');
|
|
237
|
+
}
|
|
183
238
|
log.info(`Downloading snapshot to ${downloadDir}`, {
|
|
184
239
|
snapshot,
|
|
185
240
|
downloadPaths
|
|
186
241
|
});
|
|
187
|
-
|
|
242
|
+
if (config.validateArchiver) {
|
|
243
|
+
await downloadSnapshot(snapshot, downloadPaths, fileStore, [
|
|
244
|
+
'archiver'
|
|
245
|
+
]);
|
|
246
|
+
await config.validateArchiver(downloadDir);
|
|
247
|
+
await downloadSnapshot(snapshot, downloadPaths, fileStore, SnapshotDataKeys.filter((key)=>key !== 'archiver'));
|
|
248
|
+
} else {
|
|
249
|
+
await downloadSnapshot(snapshot, downloadPaths, fileStore);
|
|
250
|
+
}
|
|
251
|
+
await config.validate?.(downloadDir);
|
|
188
252
|
log.info(`Snapshot downloaded at ${downloadDir}`, {
|
|
189
253
|
snapshot,
|
|
190
254
|
downloadPaths
|
|
@@ -199,14 +263,14 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
199
263
|
rollupAddress
|
|
200
264
|
});
|
|
201
265
|
// Same for the world state dbs, only that we do not close them, since we assume they are not yet in use
|
|
202
|
-
const worldStateBasePath = join(dataDirectory, WORLD_STATE_DIR);
|
|
266
|
+
const worldStateBasePath = join(config.worldStateDataDirectory ?? dataDirectory, WORLD_STATE_DIR);
|
|
203
267
|
await prepareTarget(worldStateBasePath, WORLD_STATE_DB_VERSION, rollupAddress);
|
|
204
268
|
for (const [name, dir] of NATIVE_WORLD_STATE_DBS){
|
|
205
269
|
const path = join(worldStateBasePath, dir);
|
|
206
270
|
await mkdir(path, {
|
|
207
271
|
recursive: true
|
|
208
272
|
});
|
|
209
|
-
await
|
|
273
|
+
await moveSnapshotFile(downloadPaths[name], join(path, 'data.mdb'));
|
|
210
274
|
log.info(`World state database ${name} set up from snapshot`, {
|
|
211
275
|
path,
|
|
212
276
|
dbVersion: WORLD_STATE_DB_VERSION,
|
|
@@ -225,6 +289,64 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12;
|
|
|
225
289
|
}
|
|
226
290
|
}
|
|
227
291
|
}
|
|
292
|
+
async function moveSnapshotFile(source, destination) {
|
|
293
|
+
try {
|
|
294
|
+
await rename(source, destination);
|
|
295
|
+
} catch (err) {
|
|
296
|
+
if (!err || typeof err !== 'object' || !('code' in err) || err.code !== 'EXDEV') {
|
|
297
|
+
throw err;
|
|
298
|
+
}
|
|
299
|
+
await copyFile(source, destination);
|
|
300
|
+
await unlink(source);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
async function hasWorldStateDatabase(dataDirectory, rollupAddress) {
|
|
304
|
+
const versionPath = join(dataDirectory, WORLD_STATE_DIR, DatabaseVersionManager.VERSION_FILE);
|
|
305
|
+
let storedVersion;
|
|
306
|
+
try {
|
|
307
|
+
storedVersion = DatabaseVersion.fromBuffer(await readFile(versionPath));
|
|
308
|
+
} catch {
|
|
309
|
+
return false;
|
|
310
|
+
}
|
|
311
|
+
if (!storedVersion.equals(new DatabaseVersion(WORLD_STATE_DB_VERSION, rollupAddress))) {
|
|
312
|
+
return false;
|
|
313
|
+
}
|
|
314
|
+
const sizes = await Promise.all(NATIVE_WORLD_STATE_DBS.map(async ([, directory])=>{
|
|
315
|
+
try {
|
|
316
|
+
return (await stat(join(dataDirectory, WORLD_STATE_DIR, directory, 'data.mdb'))).size;
|
|
317
|
+
} catch (err) {
|
|
318
|
+
if (err && typeof err === 'object' && 'code' in err && err.code === 'ENOENT') {
|
|
319
|
+
return 0;
|
|
320
|
+
}
|
|
321
|
+
throw err;
|
|
322
|
+
}
|
|
323
|
+
}));
|
|
324
|
+
return sizes.some((size)=>size > 0);
|
|
325
|
+
}
|
|
326
|
+
async function validateLocalWorldStateHistory(config, log, genesis) {
|
|
327
|
+
const worldState = await createWorldState(config, genesis);
|
|
328
|
+
try {
|
|
329
|
+
const summary = await worldState.getStatusSummary();
|
|
330
|
+
if (getWorldStateHistoryStartCheckpoint(config) !== 0 && summary.oldestHistoricalBlock > 1) {
|
|
331
|
+
const archiverStore = await createArchiverStore(config, GENESIS_BLOCK_HEADER_HASH);
|
|
332
|
+
try {
|
|
333
|
+
await validateWorldStateHistory(config, summary, archiverStore.blocks);
|
|
334
|
+
} finally{
|
|
335
|
+
await archiverStore.db.close();
|
|
336
|
+
}
|
|
337
|
+
} else {
|
|
338
|
+
await validateWorldStateHistory(config, summary);
|
|
339
|
+
}
|
|
340
|
+
log.info('Validated retained world-state history', {
|
|
341
|
+
oldestHistoricBlock: summary.oldestHistoricalBlock,
|
|
342
|
+
latestBlock: summary.unfinalizedBlockNumber,
|
|
343
|
+
historyStartCheckpoint: getWorldStateHistoryStartCheckpoint(config),
|
|
344
|
+
historyRetention: config.worldStateHistoryRetention
|
|
345
|
+
});
|
|
346
|
+
} finally{
|
|
347
|
+
await worldState.close();
|
|
348
|
+
}
|
|
349
|
+
}
|
|
228
350
|
/** Deletes target dir and writes the new version file. */ async function prepareTarget(target, schemaVersion, rollupAddress) {
|
|
229
351
|
const noOpen = ()=>Promise.resolve(undefined);
|
|
230
352
|
const versionManager = new DatabaseVersionManager({
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec-labs/node-lib",
|
|
3
|
-
"version": "6.0.0-nightly.
|
|
3
|
+
"version": "6.0.0-nightly.20261022",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"typedocOptions": {
|
|
6
6
|
"entryPoints": [
|
|
@@ -65,20 +65,20 @@
|
|
|
65
65
|
]
|
|
66
66
|
},
|
|
67
67
|
"dependencies": {
|
|
68
|
-
"@aztec-labs/archiver": "6.0.0-nightly.
|
|
69
|
-
"@aztec-labs/blob-lib": "6.0.0-nightly.
|
|
70
|
-
"@aztec-labs/constants": "6.0.0-nightly.
|
|
71
|
-
"@aztec-labs/ethereum": "6.0.0-nightly.
|
|
72
|
-
"@aztec-labs/foundation": "6.0.0-nightly.
|
|
73
|
-
"@aztec-labs/kv-store": "6.0.0-nightly.
|
|
74
|
-
"@aztec-labs/p2p": "6.0.0-nightly.
|
|
75
|
-
"@aztec-labs/stdlib": "6.0.0-nightly.
|
|
76
|
-
"@aztec-labs/telemetry-client": "6.0.0-nightly.
|
|
77
|
-
"@aztec-labs/world-state": "6.0.0-nightly.
|
|
68
|
+
"@aztec-labs/archiver": "6.0.0-nightly.20261022",
|
|
69
|
+
"@aztec-labs/blob-lib": "6.0.0-nightly.20261022",
|
|
70
|
+
"@aztec-labs/constants": "6.0.0-nightly.20261022",
|
|
71
|
+
"@aztec-labs/ethereum": "6.0.0-nightly.20261022",
|
|
72
|
+
"@aztec-labs/foundation": "6.0.0-nightly.20261022",
|
|
73
|
+
"@aztec-labs/kv-store": "6.0.0-nightly.20261022",
|
|
74
|
+
"@aztec-labs/p2p": "6.0.0-nightly.20261022",
|
|
75
|
+
"@aztec-labs/stdlib": "6.0.0-nightly.20261022",
|
|
76
|
+
"@aztec-labs/telemetry-client": "6.0.0-nightly.20261022",
|
|
77
|
+
"@aztec-labs/world-state": "6.0.0-nightly.20261022",
|
|
78
78
|
"tslib": "^2.4.0"
|
|
79
79
|
},
|
|
80
80
|
"devDependencies": {
|
|
81
|
-
"@aztec-labs/node-keystore": "6.0.0-nightly.
|
|
81
|
+
"@aztec-labs/node-keystore": "6.0.0-nightly.20261022",
|
|
82
82
|
"@jest/globals": "^30.0.0",
|
|
83
83
|
"@types/jest": "^30.0.0",
|
|
84
84
|
"@types/node": "^22.15.17",
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
import { INITIAL_L2_BLOCK_NUM } from '@aztec-labs/constants';
|
|
9
9
|
import { type EthereumClientConfig, getPublicClient } from '@aztec-labs/ethereum/client';
|
|
10
10
|
import type { L1ContractsConfig } from '@aztec-labs/ethereum/config';
|
|
11
|
+
import { BlockNumber } from '@aztec-labs/foundation/branded-types';
|
|
11
12
|
import type { EthAddress } from '@aztec-labs/foundation/eth-address';
|
|
12
13
|
import { tryRmDir } from '@aztec-labs/foundation/fs';
|
|
13
14
|
import type { Logger } from '@aztec-labs/foundation/log';
|
|
@@ -15,17 +16,28 @@ import { P2P_STORE_NAME } from '@aztec-labs/p2p';
|
|
|
15
16
|
import { GENESIS_BLOCK_HEADER_HASH } from '@aztec-labs/stdlib/block';
|
|
16
17
|
import type { ChainConfig } from '@aztec-labs/stdlib/config';
|
|
17
18
|
import { DatabaseVersionManager } from '@aztec-labs/stdlib/database-version/manager';
|
|
19
|
+
import { DatabaseVersion } from '@aztec-labs/stdlib/database-version/version';
|
|
18
20
|
import { type ReadOnlyFileStore, createReadOnlyFileStore } from '@aztec-labs/stdlib/file-store';
|
|
19
21
|
import type { DataStoreConfig } from '@aztec-labs/stdlib/kv-store';
|
|
20
22
|
import {
|
|
23
|
+
SnapshotDataKeys,
|
|
21
24
|
type SnapshotMetadata,
|
|
22
25
|
type SnapshotsIndexMetadata,
|
|
23
26
|
downloadSnapshot,
|
|
24
|
-
|
|
27
|
+
getSnapshotIndex,
|
|
25
28
|
makeSnapshotPaths,
|
|
26
29
|
} from '@aztec-labs/stdlib/snapshots';
|
|
27
|
-
import {
|
|
28
|
-
import {
|
|
30
|
+
import type { GenesisData } from '@aztec-labs/stdlib/world-state';
|
|
31
|
+
import {
|
|
32
|
+
NATIVE_WORLD_STATE_DBS,
|
|
33
|
+
WORLD_STATE_DB_VERSION,
|
|
34
|
+
WORLD_STATE_DIR,
|
|
35
|
+
type WorldStateConfig,
|
|
36
|
+
createWorldState,
|
|
37
|
+
getWorldStateHistoryStartCheckpoint,
|
|
38
|
+
validateWorldStateHistory,
|
|
39
|
+
} from '@aztec-labs/world-state';
|
|
40
|
+
import { copyFile, mkdir, mkdtemp, readFile, rename, stat, unlink } from 'fs/promises';
|
|
29
41
|
import { join } from 'path';
|
|
30
42
|
|
|
31
43
|
import type { SharedNodeConfig } from '../config/index.js';
|
|
@@ -37,6 +49,7 @@ type SnapshotSyncConfig = Pick<SharedNodeConfig, 'syncMode'> &
|
|
|
37
49
|
Pick<ChainConfig, 'l1ChainId' | 'rollupVersion'> &
|
|
38
50
|
Pick<L1ContractsConfig, 'aztecEpochDuration'> &
|
|
39
51
|
Pick<ArchiverConfig, 'archiverStoreMapSizeKb'> &
|
|
52
|
+
WorldStateConfig &
|
|
40
53
|
DataStoreConfig &
|
|
41
54
|
Required<Pick<DataStoreConfig, 'rollupAddress'>> &
|
|
42
55
|
EthereumClientConfig & {
|
|
@@ -45,11 +58,23 @@ type SnapshotSyncConfig = Pick<SharedNodeConfig, 'syncMode'> &
|
|
|
45
58
|
};
|
|
46
59
|
|
|
47
60
|
/**
|
|
48
|
-
*
|
|
49
|
-
*
|
|
61
|
+
* Validates local history and restores the newest compatible snapshot when bootstrap is needed.
|
|
62
|
+
* Fixed history boundaries are resolved against archived block and checkpoint metadata.
|
|
50
63
|
*/
|
|
51
|
-
export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
64
|
+
export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger, genesis?: GenesisData) {
|
|
52
65
|
const { syncMode, snapshotsUrls, dataDirectory, l1ChainId, rollupVersion, rollupAddress } = config;
|
|
66
|
+
const historyStartCheckpoint = getWorldStateHistoryStartCheckpoint(config);
|
|
67
|
+
let maxSnapshotBlock: BlockNumber | undefined;
|
|
68
|
+
const worldStateDirectory = config.worldStateDataDirectory ?? dataDirectory;
|
|
69
|
+
if (
|
|
70
|
+
historyStartCheckpoint !== undefined &&
|
|
71
|
+
worldStateDirectory &&
|
|
72
|
+
(await hasWorldStateDatabase(worldStateDirectory, rollupAddress))
|
|
73
|
+
) {
|
|
74
|
+
await validateLocalWorldStateHistory(config, log, genesis);
|
|
75
|
+
log.info('Using existing world-state database with compatible retained history', { historyStartCheckpoint });
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
53
78
|
if (syncMode === 'full') {
|
|
54
79
|
log.debug('Snapshot sync is disabled. Running full sync.', { syncMode: syncMode });
|
|
55
80
|
return false;
|
|
@@ -83,7 +108,12 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
83
108
|
}
|
|
84
109
|
|
|
85
110
|
const minL1BlocksToTriggerReplace = config.minL1BlocksToTriggerReplace ?? MIN_L1_BLOCKS_TO_TRIGGER_REPLACE;
|
|
86
|
-
if (
|
|
111
|
+
if (
|
|
112
|
+
historyStartCheckpoint === undefined &&
|
|
113
|
+
syncMode === 'snapshot' &&
|
|
114
|
+
archiverL2BlockNumber !== undefined &&
|
|
115
|
+
archiverL2BlockNumber >= INITIAL_L2_BLOCK_NUM
|
|
116
|
+
) {
|
|
87
117
|
log.verbose(
|
|
88
118
|
`Skipping non-forced snapshot sync as archiver is already synced to L2 block ${archiverL2BlockNumber}.`,
|
|
89
119
|
);
|
|
@@ -92,6 +122,7 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
92
122
|
|
|
93
123
|
const currentL1BlockNumber = await getPublicClient(config).getBlockNumber();
|
|
94
124
|
if (
|
|
125
|
+
historyStartCheckpoint === undefined &&
|
|
95
126
|
archiverL1BlockNumber &&
|
|
96
127
|
currentL1BlockNumber >= archiverL1BlockNumber &&
|
|
97
128
|
currentL1BlockNumber - archiverL1BlockNumber < minL1BlocksToTriggerReplace
|
|
@@ -111,7 +142,7 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
111
142
|
rollupAddress,
|
|
112
143
|
};
|
|
113
144
|
|
|
114
|
-
//
|
|
145
|
+
// Index entries may outlive their database files, so retain older candidates for download fallback.
|
|
115
146
|
type SnapshotCandidate = { snapshot: SnapshotMetadata; url: string; fileStore: ReadOnlyFileStore };
|
|
116
147
|
const snapshotCandidates: SnapshotCandidate[] = [];
|
|
117
148
|
|
|
@@ -124,57 +155,67 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
124
155
|
continue;
|
|
125
156
|
}
|
|
126
157
|
|
|
127
|
-
let
|
|
158
|
+
let snapshots: SnapshotMetadata[];
|
|
128
159
|
try {
|
|
129
|
-
|
|
160
|
+
snapshots = (await getSnapshotIndex(indexMetadata, fileStore))?.snapshots ?? [];
|
|
130
161
|
} catch (err) {
|
|
131
|
-
log.error(`Failed to
|
|
162
|
+
log.error(`Failed to read snapshot index from ${snapshotsUrl}. Skipping this URL.`, err, {
|
|
132
163
|
...indexMetadata,
|
|
133
164
|
snapshotsUrl,
|
|
134
165
|
});
|
|
135
166
|
continue;
|
|
136
167
|
}
|
|
137
168
|
|
|
138
|
-
if (
|
|
169
|
+
if (snapshots.length === 0) {
|
|
139
170
|
log.verbose(`No snapshot found at ${snapshotsUrl}. Skipping this URL.`, { ...indexMetadata, snapshotsUrl });
|
|
140
171
|
continue;
|
|
141
172
|
}
|
|
142
173
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
174
|
+
for (const snapshot of snapshots) {
|
|
175
|
+
if (snapshot.schemaVersions.archiver !== ARCHIVER_DB_VERSION) {
|
|
176
|
+
log.warn(
|
|
177
|
+
`Skipping snapshot from ${snapshotsUrl} as it has schema version ${snapshot.schemaVersions.archiver} but expected ${ARCHIVER_DB_VERSION}.`,
|
|
178
|
+
snapshot,
|
|
179
|
+
);
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
150
182
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
183
|
+
if (snapshot.schemaVersions.worldState !== WORLD_STATE_DB_VERSION) {
|
|
184
|
+
log.warn(
|
|
185
|
+
`Skipping snapshot from ${snapshotsUrl} as it has world state schema version ${snapshot.schemaVersions.worldState} but we expected ${WORLD_STATE_DB_VERSION}.`,
|
|
186
|
+
snapshot,
|
|
187
|
+
);
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
158
190
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
)
|
|
164
|
-
|
|
165
|
-
|
|
191
|
+
if (
|
|
192
|
+
historyStartCheckpoint === undefined &&
|
|
193
|
+
archiverL1BlockNumber &&
|
|
194
|
+
snapshot.l1BlockNumber < archiverL1BlockNumber
|
|
195
|
+
) {
|
|
196
|
+
log.verbose(
|
|
197
|
+
`Skipping snapshot from ${snapshotsUrl} since local archiver is at L1 block ${archiverL1BlockNumber} which is further than snapshot at ${snapshot.l1BlockNumber}`,
|
|
198
|
+
{ snapshot, archiverL1BlockNumber, snapshotsUrl },
|
|
199
|
+
);
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
166
202
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
203
|
+
if (
|
|
204
|
+
historyStartCheckpoint === undefined &&
|
|
205
|
+
archiverL1BlockNumber &&
|
|
206
|
+
snapshot.l1BlockNumber - Number(archiverL1BlockNumber) < minL1BlocksToTriggerReplace
|
|
207
|
+
) {
|
|
208
|
+
log.verbose(
|
|
209
|
+
`Skipping snapshot from ${snapshotsUrl} as archiver is less than ${
|
|
210
|
+
snapshot.l1BlockNumber - Number(archiverL1BlockNumber)
|
|
211
|
+
} L1 blocks behind this snapshot.`,
|
|
212
|
+
{ snapshot, archiverL1BlockNumber, snapshotsUrl },
|
|
213
|
+
);
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
176
216
|
|
|
177
|
-
|
|
217
|
+
snapshotCandidates.push({ snapshot, url: snapshotsUrl, fileStore });
|
|
218
|
+
}
|
|
178
219
|
}
|
|
179
220
|
|
|
180
221
|
if (snapshotCandidates.length === 0) {
|
|
@@ -182,11 +223,17 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
182
223
|
return false;
|
|
183
224
|
}
|
|
184
225
|
|
|
185
|
-
|
|
186
|
-
|
|
226
|
+
snapshotCandidates.sort((a, b) =>
|
|
227
|
+
historyStartCheckpoint === undefined
|
|
228
|
+
? b.snapshot.l1BlockNumber - a.snapshot.l1BlockNumber
|
|
229
|
+
: b.snapshot.l2BlockNumber - a.snapshot.l2BlockNumber || b.snapshot.l1BlockNumber - a.snapshot.l1BlockNumber,
|
|
230
|
+
);
|
|
187
231
|
|
|
188
232
|
// Try each candidate in order until one succeeds
|
|
189
233
|
for (const { snapshot, url, fileStore } of snapshotCandidates) {
|
|
234
|
+
if (maxSnapshotBlock !== undefined && snapshot.l2BlockNumber > maxSnapshotBlock) {
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
190
237
|
const { l1BlockNumber, l2BlockNumber } = snapshot;
|
|
191
238
|
log.info(`Attempting to sync from snapshot at L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, {
|
|
192
239
|
snapshot,
|
|
@@ -198,6 +245,46 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
198
245
|
dataDirectory: config.dataDirectory!,
|
|
199
246
|
rollupAddress: config.rollupAddress,
|
|
200
247
|
fileStore,
|
|
248
|
+
worldStateDataDirectory: config.worldStateDataDirectory,
|
|
249
|
+
validateArchiver:
|
|
250
|
+
historyStartCheckpoint === undefined || historyStartCheckpoint === 0
|
|
251
|
+
? undefined
|
|
252
|
+
: async stagingDirectory => {
|
|
253
|
+
const stores = await createArchiverStore(
|
|
254
|
+
{ ...config, dataDirectory: stagingDirectory },
|
|
255
|
+
GENESIS_BLOCK_HEADER_HASH,
|
|
256
|
+
);
|
|
257
|
+
try {
|
|
258
|
+
const latestBlockNumber = await stores.blocks.getLatestL2BlockNumber();
|
|
259
|
+
if (latestBlockNumber === undefined) {
|
|
260
|
+
throw new Error('Snapshot archiver has no blocks');
|
|
261
|
+
}
|
|
262
|
+
const latestBlock = await stores.blocks.getBlockData({ number: latestBlockNumber });
|
|
263
|
+
if (!latestBlock) {
|
|
264
|
+
throw new Error(`Snapshot archiver is missing block ${latestBlockNumber}`);
|
|
265
|
+
}
|
|
266
|
+
if (latestBlock.checkpointNumber > historyStartCheckpoint) {
|
|
267
|
+
const checkpoint = await stores.blocks.getCheckpointData(historyStartCheckpoint);
|
|
268
|
+
if (checkpoint) {
|
|
269
|
+
maxSnapshotBlock = BlockNumber(checkpoint.startBlock + checkpoint.blockCount - 1);
|
|
270
|
+
}
|
|
271
|
+
throw new Error(
|
|
272
|
+
`Snapshot is at checkpoint ${latestBlock.checkpointNumber}, after requested checkpoint ${historyStartCheckpoint}`,
|
|
273
|
+
);
|
|
274
|
+
}
|
|
275
|
+
} finally {
|
|
276
|
+
await stores.db.close();
|
|
277
|
+
}
|
|
278
|
+
},
|
|
279
|
+
validate:
|
|
280
|
+
historyStartCheckpoint === undefined
|
|
281
|
+
? undefined
|
|
282
|
+
: stagingDirectory =>
|
|
283
|
+
validateLocalWorldStateHistory(
|
|
284
|
+
{ ...config, dataDirectory: stagingDirectory, worldStateDataDirectory: stagingDirectory },
|
|
285
|
+
log,
|
|
286
|
+
genesis,
|
|
287
|
+
),
|
|
201
288
|
});
|
|
202
289
|
log.info(`Snapshot synced to L1 block ${l1BlockNumber} L2 block ${l2BlockNumber}`, {
|
|
203
290
|
snapshot,
|
|
@@ -223,7 +310,14 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) {
|
|
|
223
310
|
export async function snapshotSync(
|
|
224
311
|
snapshot: Pick<SnapshotMetadata, 'dataUrls'>,
|
|
225
312
|
log: Logger,
|
|
226
|
-
config: {
|
|
313
|
+
config: {
|
|
314
|
+
dataDirectory: string;
|
|
315
|
+
rollupAddress: EthAddress;
|
|
316
|
+
fileStore: ReadOnlyFileStore;
|
|
317
|
+
worldStateDataDirectory?: string;
|
|
318
|
+
validateArchiver?: (stagingDirectory: string) => Promise<void>;
|
|
319
|
+
validate?: (stagingDirectory: string) => Promise<void>;
|
|
320
|
+
},
|
|
227
321
|
) {
|
|
228
322
|
const { dataDirectory, rollupAddress, fileStore } = config;
|
|
229
323
|
if (!dataDirectory) {
|
|
@@ -237,8 +331,26 @@ export async function snapshotSync(
|
|
|
237
331
|
await mkdir(dataDirectory, { recursive: true });
|
|
238
332
|
downloadDir = await mkdtemp(join(dataDirectory, 'download-'));
|
|
239
333
|
const downloadPaths = makeSnapshotPaths(downloadDir);
|
|
334
|
+
await prepareTarget(join(downloadDir, ARCHIVER_STORE_NAME), ARCHIVER_DB_VERSION, rollupAddress);
|
|
335
|
+
await prepareTarget(join(downloadDir, WORLD_STATE_DIR), WORLD_STATE_DB_VERSION, rollupAddress);
|
|
336
|
+
downloadPaths.archiver = join(downloadDir, ARCHIVER_STORE_NAME, 'data.mdb');
|
|
337
|
+
for (const [name, dir] of NATIVE_WORLD_STATE_DBS) {
|
|
338
|
+
downloadPaths[name] = join(downloadDir, WORLD_STATE_DIR, dir, 'data.mdb');
|
|
339
|
+
}
|
|
240
340
|
log.info(`Downloading snapshot to ${downloadDir}`, { snapshot, downloadPaths });
|
|
241
|
-
|
|
341
|
+
if (config.validateArchiver) {
|
|
342
|
+
await downloadSnapshot(snapshot, downloadPaths, fileStore, ['archiver']);
|
|
343
|
+
await config.validateArchiver(downloadDir);
|
|
344
|
+
await downloadSnapshot(
|
|
345
|
+
snapshot,
|
|
346
|
+
downloadPaths,
|
|
347
|
+
fileStore,
|
|
348
|
+
SnapshotDataKeys.filter(key => key !== 'archiver'),
|
|
349
|
+
);
|
|
350
|
+
} else {
|
|
351
|
+
await downloadSnapshot(snapshot, downloadPaths, fileStore);
|
|
352
|
+
}
|
|
353
|
+
await config.validate?.(downloadDir);
|
|
242
354
|
log.info(`Snapshot downloaded at ${downloadDir}`, { snapshot, downloadPaths });
|
|
243
355
|
|
|
244
356
|
// If download was successful, clear lock and version, and move download there
|
|
@@ -252,12 +364,12 @@ export async function snapshotSync(
|
|
|
252
364
|
});
|
|
253
365
|
|
|
254
366
|
// Same for the world state dbs, only that we do not close them, since we assume they are not yet in use
|
|
255
|
-
const worldStateBasePath = join(dataDirectory, WORLD_STATE_DIR);
|
|
367
|
+
const worldStateBasePath = join(config.worldStateDataDirectory ?? dataDirectory, WORLD_STATE_DIR);
|
|
256
368
|
await prepareTarget(worldStateBasePath, WORLD_STATE_DB_VERSION, rollupAddress);
|
|
257
369
|
for (const [name, dir] of NATIVE_WORLD_STATE_DBS) {
|
|
258
370
|
const path = join(worldStateBasePath, dir);
|
|
259
371
|
await mkdir(path, { recursive: true });
|
|
260
|
-
await
|
|
372
|
+
await moveSnapshotFile(downloadPaths[name], join(path, 'data.mdb'));
|
|
261
373
|
log.info(`World state database ${name} set up from snapshot`, {
|
|
262
374
|
path,
|
|
263
375
|
dbVersion: WORLD_STATE_DB_VERSION,
|
|
@@ -276,6 +388,70 @@ export async function snapshotSync(
|
|
|
276
388
|
}
|
|
277
389
|
}
|
|
278
390
|
|
|
391
|
+
async function moveSnapshotFile(source: string, destination: string) {
|
|
392
|
+
try {
|
|
393
|
+
await rename(source, destination);
|
|
394
|
+
} catch (err) {
|
|
395
|
+
if (!err || typeof err !== 'object' || !('code' in err) || err.code !== 'EXDEV') {
|
|
396
|
+
throw err;
|
|
397
|
+
}
|
|
398
|
+
await copyFile(source, destination);
|
|
399
|
+
await unlink(source);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
async function hasWorldStateDatabase(dataDirectory: string, rollupAddress: EthAddress): Promise<boolean> {
|
|
404
|
+
const versionPath = join(dataDirectory, WORLD_STATE_DIR, DatabaseVersionManager.VERSION_FILE);
|
|
405
|
+
let storedVersion: DatabaseVersion;
|
|
406
|
+
try {
|
|
407
|
+
storedVersion = DatabaseVersion.fromBuffer(await readFile(versionPath));
|
|
408
|
+
} catch {
|
|
409
|
+
return false;
|
|
410
|
+
}
|
|
411
|
+
if (!storedVersion.equals(new DatabaseVersion(WORLD_STATE_DB_VERSION, rollupAddress))) {
|
|
412
|
+
return false;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
const sizes = await Promise.all(
|
|
416
|
+
NATIVE_WORLD_STATE_DBS.map(async ([, directory]) => {
|
|
417
|
+
try {
|
|
418
|
+
return (await stat(join(dataDirectory, WORLD_STATE_DIR, directory, 'data.mdb'))).size;
|
|
419
|
+
} catch (err) {
|
|
420
|
+
if (err && typeof err === 'object' && 'code' in err && err.code === 'ENOENT') {
|
|
421
|
+
return 0;
|
|
422
|
+
}
|
|
423
|
+
throw err;
|
|
424
|
+
}
|
|
425
|
+
}),
|
|
426
|
+
);
|
|
427
|
+
return sizes.some(size => size > 0);
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
async function validateLocalWorldStateHistory(config: SnapshotSyncConfig, log: Logger, genesis?: GenesisData) {
|
|
431
|
+
const worldState = await createWorldState(config, genesis);
|
|
432
|
+
try {
|
|
433
|
+
const summary = await worldState.getStatusSummary();
|
|
434
|
+
if (getWorldStateHistoryStartCheckpoint(config) !== 0 && summary.oldestHistoricalBlock > 1) {
|
|
435
|
+
const archiverStore = await createArchiverStore(config, GENESIS_BLOCK_HEADER_HASH);
|
|
436
|
+
try {
|
|
437
|
+
await validateWorldStateHistory(config, summary, archiverStore.blocks);
|
|
438
|
+
} finally {
|
|
439
|
+
await archiverStore.db.close();
|
|
440
|
+
}
|
|
441
|
+
} else {
|
|
442
|
+
await validateWorldStateHistory(config, summary);
|
|
443
|
+
}
|
|
444
|
+
log.info('Validated retained world-state history', {
|
|
445
|
+
oldestHistoricBlock: summary.oldestHistoricalBlock,
|
|
446
|
+
latestBlock: summary.unfinalizedBlockNumber,
|
|
447
|
+
historyStartCheckpoint: getWorldStateHistoryStartCheckpoint(config),
|
|
448
|
+
historyRetention: config.worldStateHistoryRetention,
|
|
449
|
+
});
|
|
450
|
+
} finally {
|
|
451
|
+
await worldState.close();
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
279
455
|
/** Deletes target dir and writes the new version file. */
|
|
280
456
|
async function prepareTarget(target: string, schemaVersion: number, rollupAddress: EthAddress) {
|
|
281
457
|
const noOpen = () => Promise.resolve(undefined);
|