@aztec/world-state 0.0.0-test.0 → 0.0.1-commit.0208eb9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/index.d.ts +1 -1
- package/dest/instrumentation/instrumentation.d.ts +6 -4
- package/dest/instrumentation/instrumentation.d.ts.map +1 -1
- package/dest/instrumentation/instrumentation.js +25 -41
- package/dest/native/bench_metrics.d.ts +23 -0
- package/dest/native/bench_metrics.d.ts.map +1 -0
- package/dest/native/bench_metrics.js +81 -0
- package/dest/native/fork_checkpoint.d.ts +1 -1
- package/dest/native/fork_checkpoint.d.ts.map +1 -1
- package/dest/native/index.d.ts +1 -1
- package/dest/native/merkle_trees_facade.d.ts +19 -7
- package/dest/native/merkle_trees_facade.d.ts.map +1 -1
- package/dest/native/merkle_trees_facade.js +76 -14
- package/dest/native/message.d.ts +74 -52
- package/dest/native/message.d.ts.map +1 -1
- package/dest/native/message.js +61 -61
- package/dest/native/native_world_state.d.ts +27 -19
- package/dest/native/native_world_state.d.ts.map +1 -1
- package/dest/native/native_world_state.js +103 -41
- package/dest/native/native_world_state_instance.d.ts +20 -4
- package/dest/native/native_world_state_instance.d.ts.map +1 -1
- package/dest/native/native_world_state_instance.js +42 -3
- package/dest/native/world_state_ops_queue.d.ts +1 -1
- package/dest/native/world_state_ops_queue.d.ts.map +1 -1
- package/dest/native/world_state_ops_queue.js +1 -1
- package/dest/synchronizer/config.d.ts +12 -4
- package/dest/synchronizer/config.d.ts.map +1 -1
- package/dest/synchronizer/config.js +27 -7
- package/dest/synchronizer/errors.d.ts +4 -0
- package/dest/synchronizer/errors.d.ts.map +1 -0
- package/dest/synchronizer/errors.js +5 -0
- package/dest/synchronizer/factory.d.ts +11 -3
- package/dest/synchronizer/factory.d.ts.map +1 -1
- package/dest/synchronizer/factory.js +13 -8
- package/dest/synchronizer/index.d.ts +1 -1
- package/dest/synchronizer/server_world_state_synchronizer.d.ts +21 -31
- package/dest/synchronizer/server_world_state_synchronizer.d.ts.map +1 -1
- package/dest/synchronizer/server_world_state_synchronizer.js +144 -92
- package/dest/test/index.d.ts +1 -1
- package/dest/test/utils.d.ts +12 -5
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +54 -47
- package/dest/testing.d.ts +3 -3
- package/dest/testing.d.ts.map +1 -1
- package/dest/testing.js +7 -11
- package/dest/world-state-db/index.d.ts +1 -1
- package/dest/world-state-db/merkle_tree_db.d.ts +12 -9
- package/dest/world-state-db/merkle_tree_db.d.ts.map +1 -1
- package/package.json +24 -24
- package/src/instrumentation/instrumentation.ts +31 -43
- package/src/native/bench_metrics.ts +91 -0
- package/src/native/merkle_trees_facade.ts +83 -18
- package/src/native/message.ts +94 -74
- package/src/native/native_world_state.ts +132 -52
- package/src/native/native_world_state_instance.ts +62 -9
- package/src/native/world_state_ops_queue.ts +1 -1
- package/src/synchronizer/config.ts +48 -16
- package/src/synchronizer/errors.ts +5 -0
- package/src/synchronizer/factory.ts +38 -9
- package/src/synchronizer/server_world_state_synchronizer.ts +170 -117
- package/src/test/utils.ts +92 -82
- package/src/testing.ts +4 -8
- package/src/world-state-db/merkle_tree_db.ts +16 -8
|
@@ -1,17 +1,21 @@
|
|
|
1
1
|
import { MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIERS_PER_TX, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
|
|
2
|
-
import {
|
|
2
|
+
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
3
|
+
import { fromEntries, padArrayEnd } from '@aztec/foundation/collection';
|
|
4
|
+
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
3
5
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
4
|
-
import {
|
|
5
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
6
|
+
import { tryRmDir } from '@aztec/foundation/fs';
|
|
7
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
6
8
|
import type { L2Block } from '@aztec/stdlib/block';
|
|
7
|
-
import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
|
|
9
|
+
import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
|
|
8
10
|
import type {
|
|
9
11
|
IndexedTreeId,
|
|
10
12
|
MerkleTreeReadOperations,
|
|
11
13
|
MerkleTreeWriteOperations,
|
|
12
14
|
} from '@aztec/stdlib/interfaces/server';
|
|
15
|
+
import type { SnapshotDataKeys } from '@aztec/stdlib/snapshots';
|
|
13
16
|
import { MerkleTreeId, NullifierLeaf, type NullifierLeafPreimage, PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
14
17
|
import { BlockHeader, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
|
|
18
|
+
import { WorldStateRevision } from '@aztec/stdlib/world-state';
|
|
15
19
|
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
16
20
|
|
|
17
21
|
import assert from 'assert/strict';
|
|
@@ -20,6 +24,7 @@ import { tmpdir } from 'os';
|
|
|
20
24
|
import { join } from 'path';
|
|
21
25
|
|
|
22
26
|
import { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
|
|
27
|
+
import type { WorldStateTreeMapSizes } from '../synchronizer/factory.js';
|
|
23
28
|
import type { MerkleTreeAdminDatabase as MerkleTreeDatabase } from '../world-state-db/merkle_tree_db.js';
|
|
24
29
|
import { MerkleTreesFacade, MerkleTreesForkFacade, serializeLeaf } from './merkle_trees_facade.js';
|
|
25
30
|
import {
|
|
@@ -27,16 +32,17 @@ import {
|
|
|
27
32
|
type WorldStateStatusFull,
|
|
28
33
|
type WorldStateStatusSummary,
|
|
29
34
|
blockStateReference,
|
|
30
|
-
|
|
31
|
-
|
|
35
|
+
sanitizeFullStatus,
|
|
36
|
+
sanitizeSummary,
|
|
32
37
|
treeStateReferenceToSnapshot,
|
|
33
|
-
worldStateRevision,
|
|
34
38
|
} from './message.js';
|
|
35
39
|
import { NativeWorldState } from './native_world_state_instance.js';
|
|
36
40
|
|
|
37
41
|
// The current version of the world state database schema
|
|
38
42
|
// Increment this when making incompatible changes to the database schema
|
|
39
|
-
export const WORLD_STATE_DB_VERSION =
|
|
43
|
+
export const WORLD_STATE_DB_VERSION = 2; // The initial version
|
|
44
|
+
|
|
45
|
+
export const WORLD_STATE_DIR = 'world_state';
|
|
40
46
|
|
|
41
47
|
export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
42
48
|
protected initialHeader: BlockHeader | undefined;
|
|
@@ -44,38 +50,41 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
44
50
|
private cachedStatusSummary: WorldStateStatusSummary | undefined;
|
|
45
51
|
|
|
46
52
|
protected constructor(
|
|
47
|
-
protected
|
|
53
|
+
protected instance: NativeWorldState,
|
|
48
54
|
protected readonly worldStateInstrumentation: WorldStateInstrumentation,
|
|
49
|
-
protected readonly log
|
|
55
|
+
protected readonly log: Logger,
|
|
50
56
|
private readonly cleanup = () => Promise.resolve(),
|
|
51
57
|
) {}
|
|
52
58
|
|
|
53
59
|
static async new(
|
|
54
60
|
rollupAddress: EthAddress,
|
|
55
61
|
dataDir: string,
|
|
56
|
-
|
|
62
|
+
wsTreeMapSizes: WorldStateTreeMapSizes,
|
|
57
63
|
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
58
64
|
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
59
|
-
|
|
65
|
+
bindings?: LoggerBindings,
|
|
60
66
|
cleanup = () => Promise.resolve(),
|
|
61
67
|
): Promise<NativeWorldStateService> {
|
|
62
|
-
const
|
|
68
|
+
const log = createLogger('world-state:database', bindings);
|
|
69
|
+
const worldStateDirectory = join(dataDir, WORLD_STATE_DIR);
|
|
63
70
|
// Create a version manager to handle versioning
|
|
64
|
-
const versionManager = new DatabaseVersionManager(
|
|
65
|
-
WORLD_STATE_DB_VERSION,
|
|
71
|
+
const versionManager = new DatabaseVersionManager({
|
|
72
|
+
schemaVersion: WORLD_STATE_DB_VERSION,
|
|
66
73
|
rollupAddress,
|
|
67
|
-
worldStateDirectory,
|
|
68
|
-
(dir: string) => {
|
|
69
|
-
return Promise.resolve(
|
|
74
|
+
dataDirectory: worldStateDirectory,
|
|
75
|
+
onOpen: (dir: string) => {
|
|
76
|
+
return Promise.resolve(
|
|
77
|
+
new NativeWorldState(dir, wsTreeMapSizes, prefilledPublicData, instrumentation, bindings),
|
|
78
|
+
);
|
|
70
79
|
},
|
|
71
|
-
);
|
|
80
|
+
});
|
|
72
81
|
|
|
73
82
|
const [instance] = await versionManager.open();
|
|
74
83
|
const worldState = new this(instance, instrumentation, log, cleanup);
|
|
75
84
|
try {
|
|
76
85
|
await worldState.init();
|
|
77
86
|
} catch (e) {
|
|
78
|
-
log.error(`Error
|
|
87
|
+
log.error(`Error initializing world state: ${e}`);
|
|
79
88
|
throw e;
|
|
80
89
|
}
|
|
81
90
|
|
|
@@ -87,11 +96,19 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
87
96
|
cleanupTmpDir = true,
|
|
88
97
|
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
89
98
|
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
99
|
+
bindings?: LoggerBindings,
|
|
90
100
|
): Promise<NativeWorldStateService> {
|
|
91
|
-
const log = createLogger('world-state:database');
|
|
101
|
+
const log = createLogger('world-state:database', bindings);
|
|
92
102
|
const dataDir = await mkdtemp(join(tmpdir(), 'aztec-world-state-'));
|
|
93
103
|
const dbMapSizeKb = 10 * 1024 * 1024;
|
|
94
|
-
|
|
104
|
+
const worldStateTreeMapSizes: WorldStateTreeMapSizes = {
|
|
105
|
+
archiveTreeMapSizeKb: dbMapSizeKb,
|
|
106
|
+
nullifierTreeMapSizeKb: dbMapSizeKb,
|
|
107
|
+
noteHashTreeMapSizeKb: dbMapSizeKb,
|
|
108
|
+
messageTreeMapSizeKb: dbMapSizeKb,
|
|
109
|
+
publicDataTreeMapSizeKb: dbMapSizeKb,
|
|
110
|
+
};
|
|
111
|
+
log.debug(`Created temporary world state database at: ${dataDir} with tree map size: ${dbMapSizeKb}`);
|
|
95
112
|
|
|
96
113
|
// pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
|
|
97
114
|
const cleanup = async () => {
|
|
@@ -103,7 +120,15 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
103
120
|
}
|
|
104
121
|
};
|
|
105
122
|
|
|
106
|
-
return this.new(
|
|
123
|
+
return this.new(
|
|
124
|
+
rollupAddress,
|
|
125
|
+
dataDir,
|
|
126
|
+
worldStateTreeMapSizes,
|
|
127
|
+
prefilledPublicData,
|
|
128
|
+
instrumentation,
|
|
129
|
+
bindings,
|
|
130
|
+
cleanup,
|
|
131
|
+
);
|
|
107
132
|
}
|
|
108
133
|
|
|
109
134
|
protected async init() {
|
|
@@ -122,26 +147,49 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
122
147
|
|
|
123
148
|
// the initial header _must_ be the first element in the archive tree
|
|
124
149
|
// if this assertion fails, check that the hashing done in Header in yarn-project matches the initial header hash done in world_state.cpp
|
|
125
|
-
const indices = await committed.findLeafIndices(MerkleTreeId.ARCHIVE, [await this.initialHeader.hash()]);
|
|
150
|
+
const indices = await committed.findLeafIndices(MerkleTreeId.ARCHIVE, [(await this.initialHeader.hash()).toFr()]);
|
|
126
151
|
const initialHeaderIndex = indices[0];
|
|
127
152
|
assert.strictEqual(initialHeaderIndex, 0n, 'Invalid initial archive state');
|
|
128
153
|
}
|
|
129
154
|
|
|
155
|
+
public async clear() {
|
|
156
|
+
await this.instance.close();
|
|
157
|
+
this.cachedStatusSummary = undefined;
|
|
158
|
+
await tryRmDir(this.instance.getDataDir(), this.log);
|
|
159
|
+
this.instance = this.instance.clone();
|
|
160
|
+
}
|
|
161
|
+
|
|
130
162
|
public getCommitted(): MerkleTreeReadOperations {
|
|
131
|
-
return new MerkleTreesFacade(this.instance, this.initialHeader!,
|
|
163
|
+
return new MerkleTreesFacade(this.instance, this.initialHeader!, WorldStateRevision.empty());
|
|
132
164
|
}
|
|
133
165
|
|
|
134
|
-
public getSnapshot(blockNumber:
|
|
135
|
-
return new MerkleTreesFacade(
|
|
166
|
+
public getSnapshot(blockNumber: BlockNumber): MerkleTreeReadOperations {
|
|
167
|
+
return new MerkleTreesFacade(
|
|
168
|
+
this.instance,
|
|
169
|
+
this.initialHeader!,
|
|
170
|
+
new WorldStateRevision(/*forkId=*/ 0, /* blockNumber=*/ blockNumber, /* includeUncommitted=*/ false),
|
|
171
|
+
);
|
|
136
172
|
}
|
|
137
173
|
|
|
138
|
-
public async fork(
|
|
174
|
+
public async fork(
|
|
175
|
+
blockNumber?: BlockNumber,
|
|
176
|
+
opts: { closeDelayMs?: number } = {},
|
|
177
|
+
): Promise<MerkleTreeWriteOperations> {
|
|
139
178
|
const resp = await this.instance.call(WorldStateMessageType.CREATE_FORK, {
|
|
140
179
|
latest: blockNumber === undefined,
|
|
141
|
-
blockNumber: blockNumber ??
|
|
180
|
+
blockNumber: blockNumber ?? BlockNumber.ZERO,
|
|
142
181
|
canonical: true,
|
|
143
182
|
});
|
|
144
|
-
return new MerkleTreesForkFacade(
|
|
183
|
+
return new MerkleTreesForkFacade(
|
|
184
|
+
this.instance,
|
|
185
|
+
this.initialHeader!,
|
|
186
|
+
new WorldStateRevision(
|
|
187
|
+
/*forkId=*/ resp.forkId,
|
|
188
|
+
/* blockNumber=*/ BlockNumber.ZERO,
|
|
189
|
+
/* includeUncommitted=*/ true,
|
|
190
|
+
),
|
|
191
|
+
opts,
|
|
192
|
+
);
|
|
145
193
|
}
|
|
146
194
|
|
|
147
195
|
public getInitialHeader(): BlockHeader {
|
|
@@ -149,11 +197,22 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
149
197
|
}
|
|
150
198
|
|
|
151
199
|
public async handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise<WorldStateStatusFull> {
|
|
152
|
-
|
|
200
|
+
const isFirstBlock = l2Block.indexWithinCheckpoint === 0;
|
|
201
|
+
if (!isFirstBlock && l1ToL2Messages.length > 0) {
|
|
202
|
+
throw new Error(
|
|
203
|
+
`L1 to L2 messages must be empty for non-first blocks, but got ${l1ToL2Messages.length} messages for block ${l2Block.number}.`,
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// We have to pad the given l1 to l2 messages, and the note hashes and nullifiers within tx effects, because that's
|
|
208
|
+
// how the trees are built by circuits.
|
|
209
|
+
const paddedL1ToL2Messages = isFirstBlock
|
|
210
|
+
? padArrayEnd<Fr, number>(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP)
|
|
211
|
+
: [];
|
|
212
|
+
|
|
153
213
|
const paddedNoteHashes = l2Block.body.txEffects.flatMap(txEffect =>
|
|
154
214
|
padArrayEnd(txEffect.noteHashes, Fr.ZERO, MAX_NOTE_HASHES_PER_TX),
|
|
155
215
|
);
|
|
156
|
-
const paddedL1ToL2Messages = padArrayEnd(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP);
|
|
157
216
|
|
|
158
217
|
const paddedNullifiers = l2Block.body.txEffects
|
|
159
218
|
.flatMap(txEffect => padArrayEnd(txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX))
|
|
@@ -173,7 +232,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
173
232
|
WorldStateMessageType.SYNC_BLOCK,
|
|
174
233
|
{
|
|
175
234
|
blockNumber: l2Block.number,
|
|
176
|
-
blockHeaderHash: await l2Block.
|
|
235
|
+
blockHeaderHash: await l2Block.hash(),
|
|
177
236
|
paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf),
|
|
178
237
|
paddedNoteHashes: paddedNoteHashes.map(serializeLeaf),
|
|
179
238
|
paddedNullifiers: paddedNullifiers.map(serializeLeaf),
|
|
@@ -181,7 +240,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
181
240
|
blockStateRef: blockStateReference(l2Block.header.state),
|
|
182
241
|
canonical: true,
|
|
183
242
|
},
|
|
184
|
-
this.
|
|
243
|
+
this.sanitizeAndCacheSummaryFromFull.bind(this),
|
|
185
244
|
this.deleteCachedSummary.bind(this),
|
|
186
245
|
);
|
|
187
246
|
} catch (err) {
|
|
@@ -200,16 +259,16 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
200
259
|
return BlockHeader.empty({ state });
|
|
201
260
|
}
|
|
202
261
|
|
|
203
|
-
private
|
|
204
|
-
const
|
|
205
|
-
this.cachedStatusSummary = { ...
|
|
206
|
-
return
|
|
262
|
+
private sanitizeAndCacheSummaryFromFull(response: WorldStateStatusFull) {
|
|
263
|
+
const sanitized = sanitizeFullStatus(response);
|
|
264
|
+
this.cachedStatusSummary = { ...sanitized.summary };
|
|
265
|
+
return sanitized;
|
|
207
266
|
}
|
|
208
267
|
|
|
209
|
-
private
|
|
210
|
-
const
|
|
211
|
-
this.cachedStatusSummary = { ...
|
|
212
|
-
return
|
|
268
|
+
private sanitizeAndCacheSummary(response: WorldStateStatusSummary) {
|
|
269
|
+
const sanitized = sanitizeSummary(response);
|
|
270
|
+
this.cachedStatusSummary = { ...sanitized };
|
|
271
|
+
return sanitized;
|
|
213
272
|
}
|
|
214
273
|
|
|
215
274
|
private deleteCachedSummary(_: string) {
|
|
@@ -217,19 +276,19 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
217
276
|
}
|
|
218
277
|
|
|
219
278
|
/**
|
|
220
|
-
* Advances the
|
|
221
|
-
* @param toBlockNumber The block number that is now the tip of the
|
|
279
|
+
* Advances the finalized block number to be the number provided
|
|
280
|
+
* @param toBlockNumber The block number that is now the tip of the finalized chain
|
|
222
281
|
* @returns The new WorldStateStatus
|
|
223
282
|
*/
|
|
224
|
-
public async
|
|
283
|
+
public async setFinalized(toBlockNumber: BlockNumber) {
|
|
225
284
|
try {
|
|
226
285
|
await this.instance.call(
|
|
227
|
-
WorldStateMessageType.
|
|
286
|
+
WorldStateMessageType.FINALIZE_BLOCKS,
|
|
228
287
|
{
|
|
229
288
|
toBlockNumber,
|
|
230
289
|
canonical: true,
|
|
231
290
|
},
|
|
232
|
-
this.
|
|
291
|
+
this.sanitizeAndCacheSummary.bind(this),
|
|
233
292
|
this.deleteCachedSummary.bind(this),
|
|
234
293
|
);
|
|
235
294
|
} catch (err) {
|
|
@@ -244,7 +303,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
244
303
|
* @param toBlockNumber The block number of the new oldest historical block
|
|
245
304
|
* @returns The new WorldStateStatus
|
|
246
305
|
*/
|
|
247
|
-
public async removeHistoricalBlocks(toBlockNumber:
|
|
306
|
+
public async removeHistoricalBlocks(toBlockNumber: BlockNumber) {
|
|
248
307
|
try {
|
|
249
308
|
return await this.instance.call(
|
|
250
309
|
WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS,
|
|
@@ -252,7 +311,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
252
311
|
toBlockNumber,
|
|
253
312
|
canonical: true,
|
|
254
313
|
},
|
|
255
|
-
this.
|
|
314
|
+
this.sanitizeAndCacheSummaryFromFull.bind(this),
|
|
256
315
|
this.deleteCachedSummary.bind(this),
|
|
257
316
|
);
|
|
258
317
|
} catch (err) {
|
|
@@ -266,7 +325,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
266
325
|
* @param toBlockNumber The block number of the new tip of the pending chain,
|
|
267
326
|
* @returns The new WorldStateStatus
|
|
268
327
|
*/
|
|
269
|
-
public async unwindBlocks(toBlockNumber:
|
|
328
|
+
public async unwindBlocks(toBlockNumber: BlockNumber) {
|
|
270
329
|
try {
|
|
271
330
|
return await this.instance.call(
|
|
272
331
|
WorldStateMessageType.UNWIND_BLOCKS,
|
|
@@ -274,7 +333,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
274
333
|
toBlockNumber,
|
|
275
334
|
canonical: true,
|
|
276
335
|
},
|
|
277
|
-
this.
|
|
336
|
+
this.sanitizeAndCacheSummaryFromFull.bind(this),
|
|
278
337
|
this.deleteCachedSummary.bind(this),
|
|
279
338
|
);
|
|
280
339
|
} catch (err) {
|
|
@@ -290,7 +349,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
290
349
|
return await this.instance.call(
|
|
291
350
|
WorldStateMessageType.GET_STATUS,
|
|
292
351
|
{ canonical: true },
|
|
293
|
-
this.
|
|
352
|
+
this.sanitizeAndCacheSummary.bind(this),
|
|
294
353
|
);
|
|
295
354
|
}
|
|
296
355
|
|
|
@@ -314,4 +373,25 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
314
373
|
),
|
|
315
374
|
);
|
|
316
375
|
}
|
|
376
|
+
|
|
377
|
+
public async backupTo(
|
|
378
|
+
dstPath: string,
|
|
379
|
+
compact: boolean = true,
|
|
380
|
+
): Promise<Record<Exclude<SnapshotDataKeys, 'archiver'>, string>> {
|
|
381
|
+
await this.instance.call(WorldStateMessageType.COPY_STORES, {
|
|
382
|
+
dstPath,
|
|
383
|
+
compact,
|
|
384
|
+
canonical: true,
|
|
385
|
+
});
|
|
386
|
+
return fromEntries(NATIVE_WORLD_STATE_DBS.map(([name, dir]) => [name, join(dstPath, dir, 'data.mdb')] as const));
|
|
387
|
+
}
|
|
317
388
|
}
|
|
389
|
+
|
|
390
|
+
// The following paths are defined in cpp-land
|
|
391
|
+
export const NATIVE_WORLD_STATE_DBS = [
|
|
392
|
+
['l1-to-l2-message-tree', 'L1ToL2MessageTree'],
|
|
393
|
+
['archive-tree', 'ArchiveTree'],
|
|
394
|
+
['public-data-tree', 'PublicDataTree'],
|
|
395
|
+
['note-hash-tree', 'NoteHashTree'],
|
|
396
|
+
['nullifier-tree', 'NullifierTree'],
|
|
397
|
+
] as const;
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
NULLIFIER_TREE_HEIGHT,
|
|
9
9
|
PUBLIC_DATA_TREE_HEIGHT,
|
|
10
10
|
} from '@aztec/constants';
|
|
11
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
11
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
12
12
|
import { NativeWorldState as BaseNativeWorldState, MsgpackChannel } from '@aztec/native';
|
|
13
13
|
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
14
14
|
import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
@@ -17,6 +17,7 @@ import assert from 'assert';
|
|
|
17
17
|
import { cpus } from 'os';
|
|
18
18
|
|
|
19
19
|
import type { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
|
|
20
|
+
import type { WorldStateTreeMapSizes } from '../synchronizer/factory.js';
|
|
20
21
|
import {
|
|
21
22
|
WorldStateMessageType,
|
|
22
23
|
type WorldStateRequest,
|
|
@@ -35,6 +36,8 @@ export interface NativeWorldStateInstance {
|
|
|
35
36
|
messageType: T,
|
|
36
37
|
body: WorldStateRequest[T] & WorldStateRequestCategories,
|
|
37
38
|
): Promise<WorldStateResponse[T]>;
|
|
39
|
+
// TODO(dbanks12): this returns any type, but we should strongly type it
|
|
40
|
+
getHandle(): any;
|
|
38
41
|
}
|
|
39
42
|
|
|
40
43
|
/**
|
|
@@ -50,15 +53,18 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
50
53
|
|
|
51
54
|
/** Creates a new native WorldState instance */
|
|
52
55
|
constructor(
|
|
53
|
-
dataDir: string,
|
|
54
|
-
|
|
55
|
-
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
56
|
-
private instrumentation: WorldStateInstrumentation,
|
|
57
|
-
|
|
56
|
+
private readonly dataDir: string,
|
|
57
|
+
private readonly wsTreeMapSizes: WorldStateTreeMapSizes,
|
|
58
|
+
private readonly prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
59
|
+
private readonly instrumentation: WorldStateInstrumentation,
|
|
60
|
+
bindings?: LoggerBindings,
|
|
61
|
+
private readonly log: Logger = createLogger('world-state:database', bindings),
|
|
58
62
|
) {
|
|
59
63
|
const threads = Math.min(cpus().length, MAX_WORLD_STATE_THREADS);
|
|
60
64
|
log.info(
|
|
61
|
-
`Creating world state data store at directory ${dataDir} with map
|
|
65
|
+
`Creating world state data store at directory ${dataDir} with map sizes ${JSON.stringify(
|
|
66
|
+
wsTreeMapSizes,
|
|
67
|
+
)} and ${threads} threads.`,
|
|
62
68
|
);
|
|
63
69
|
const prefilledPublicDataBufferArray = prefilledPublicData.map(d => [d.slot.toBuffer(), d.value.toBuffer()]);
|
|
64
70
|
const ws = new BaseNativeWorldState(
|
|
@@ -75,8 +81,14 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
75
81
|
[MerkleTreeId.PUBLIC_DATA_TREE]: 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
|
|
76
82
|
},
|
|
77
83
|
prefilledPublicDataBufferArray,
|
|
78
|
-
GeneratorIndex.
|
|
79
|
-
|
|
84
|
+
GeneratorIndex.BLOCK_HEADER_HASH,
|
|
85
|
+
{
|
|
86
|
+
[MerkleTreeId.NULLIFIER_TREE]: wsTreeMapSizes.nullifierTreeMapSizeKb,
|
|
87
|
+
[MerkleTreeId.NOTE_HASH_TREE]: wsTreeMapSizes.noteHashTreeMapSizeKb,
|
|
88
|
+
[MerkleTreeId.PUBLIC_DATA_TREE]: wsTreeMapSizes.publicDataTreeMapSizeKb,
|
|
89
|
+
[MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: wsTreeMapSizes.messageTreeMapSizeKb,
|
|
90
|
+
[MerkleTreeId.ARCHIVE]: wsTreeMapSizes.archiveTreeMapSizeKb,
|
|
91
|
+
},
|
|
80
92
|
threads,
|
|
81
93
|
);
|
|
82
94
|
this.instance = new MsgpackChannel(ws);
|
|
@@ -84,6 +96,47 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
84
96
|
this.queues.set(0, new WorldStateOpsQueue());
|
|
85
97
|
}
|
|
86
98
|
|
|
99
|
+
public getDataDir() {
|
|
100
|
+
return this.dataDir;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
public clone() {
|
|
104
|
+
return new NativeWorldState(
|
|
105
|
+
this.dataDir,
|
|
106
|
+
this.wsTreeMapSizes,
|
|
107
|
+
this.prefilledPublicData,
|
|
108
|
+
this.instrumentation,
|
|
109
|
+
this.log.getBindings(),
|
|
110
|
+
this.log,
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Gets the native WorldState handle from the underlying native instance.
|
|
116
|
+
* We call the getHandle() method on the native WorldState to get a NAPI External
|
|
117
|
+
* that wraps the underlying C++ WorldState pointer.
|
|
118
|
+
* @returns The NAPI External handle to the native WorldState instance,since
|
|
119
|
+
* the NAPI external type is opaque, we return any (we could also use an opaque symbol type)
|
|
120
|
+
*/
|
|
121
|
+
public getHandle(): any {
|
|
122
|
+
const worldStateWrapper = (this.instance as any).dest;
|
|
123
|
+
|
|
124
|
+
if (!worldStateWrapper) {
|
|
125
|
+
throw new Error('No WorldStateWrapper found');
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (typeof worldStateWrapper.getHandle !== 'function') {
|
|
129
|
+
throw new Error('WorldStateWrapper does not have getHandle method');
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Call getHandle() to get the NAPI External
|
|
133
|
+
try {
|
|
134
|
+
return worldStateWrapper.getHandle();
|
|
135
|
+
} catch (error) {
|
|
136
|
+
this.log.error('Failed to get native WorldState handle', error);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
87
140
|
/**
|
|
88
141
|
* Sends a message to the native instance and returns the response.
|
|
89
142
|
* @param messageType - The type of message to send
|
|
@@ -35,7 +35,7 @@ export const MUTATING_MSG_TYPES = new Set([
|
|
|
35
35
|
WorldStateMessageType.SYNC_BLOCK,
|
|
36
36
|
WorldStateMessageType.CREATE_FORK,
|
|
37
37
|
WorldStateMessageType.DELETE_FORK,
|
|
38
|
-
WorldStateMessageType.
|
|
38
|
+
WorldStateMessageType.FINALIZE_BLOCKS,
|
|
39
39
|
WorldStateMessageType.UNWIND_BLOCKS,
|
|
40
40
|
WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS,
|
|
41
41
|
WorldStateMessageType.CREATE_CHECKPOINT,
|
|
@@ -1,24 +1,31 @@
|
|
|
1
|
-
import {
|
|
2
|
-
type ConfigMappingsType,
|
|
3
|
-
booleanConfigHelper,
|
|
4
|
-
getConfigFromMappings,
|
|
5
|
-
numberConfigHelper,
|
|
6
|
-
} from '@aztec/foundation/config';
|
|
1
|
+
import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
|
|
7
2
|
|
|
8
3
|
/** World State synchronizer configuration values. */
|
|
9
4
|
export interface WorldStateConfig {
|
|
10
5
|
/** The frequency in which to check. */
|
|
11
6
|
worldStateBlockCheckIntervalMS: number;
|
|
12
7
|
|
|
13
|
-
/** Whether to follow only the proven chain. */
|
|
14
|
-
worldStateProvenBlocksOnly: boolean;
|
|
15
|
-
|
|
16
8
|
/** Size of the batch for each get-blocks request from the synchronizer to the archiver. */
|
|
17
9
|
worldStateBlockRequestBatchSize?: number;
|
|
18
10
|
|
|
19
|
-
/** The map size to be provided to LMDB for each world state tree DB, optional, will inherit from the general
|
|
11
|
+
/** The map size to be provided to LMDB for each world state tree DB, optional, will inherit from the general dataStoreMapSizeKb if not specified*/
|
|
20
12
|
worldStateDbMapSizeKb?: number;
|
|
21
13
|
|
|
14
|
+
/** The map size to be provided to LMDB for each world state archive tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
|
|
15
|
+
archiveTreeMapSizeKb?: number;
|
|
16
|
+
|
|
17
|
+
/** The map size to be provided to LMDB for each world state nullifier tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
|
|
18
|
+
nullifierTreeMapSizeKb?: number;
|
|
19
|
+
|
|
20
|
+
/** The map size to be provided to LMDB for each world state note hash tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
|
|
21
|
+
noteHashTreeMapSizeKb?: number;
|
|
22
|
+
|
|
23
|
+
/** The map size to be provided to LMDB for each world state message tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
|
|
24
|
+
messageTreeMapSizeKb?: number;
|
|
25
|
+
|
|
26
|
+
/** The map size to be provided to LMDB for each world state public data tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
|
|
27
|
+
publicDataTreeMapSizeKb?: number;
|
|
28
|
+
|
|
22
29
|
/** Optional directory for the world state DB, if unspecified will default to the general data directory */
|
|
23
30
|
worldStateDataDirectory?: string;
|
|
24
31
|
|
|
@@ -33,11 +40,6 @@ export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
|
|
|
33
40
|
defaultValue: 100,
|
|
34
41
|
description: 'The frequency in which to check.',
|
|
35
42
|
},
|
|
36
|
-
worldStateProvenBlocksOnly: {
|
|
37
|
-
env: 'WS_PROVEN_BLOCKS_ONLY',
|
|
38
|
-
description: 'Whether to follow only the proven chain.',
|
|
39
|
-
...booleanConfigHelper(),
|
|
40
|
-
},
|
|
41
43
|
worldStateBlockRequestBatchSize: {
|
|
42
44
|
env: 'WS_BLOCK_REQUEST_BATCH_SIZE',
|
|
43
45
|
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
@@ -46,7 +48,37 @@ export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
|
|
|
46
48
|
worldStateDbMapSizeKb: {
|
|
47
49
|
env: 'WS_DB_MAP_SIZE_KB',
|
|
48
50
|
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
49
|
-
description: 'The maximum possible size of the world state DB',
|
|
51
|
+
description: 'The maximum possible size of the world state DB in KB. Overwrites the general dataStoreMapSizeKb.',
|
|
52
|
+
},
|
|
53
|
+
archiveTreeMapSizeKb: {
|
|
54
|
+
env: 'ARCHIVE_TREE_MAP_SIZE_KB',
|
|
55
|
+
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
56
|
+
description:
|
|
57
|
+
'The maximum possible size of the world state archive tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
58
|
+
},
|
|
59
|
+
nullifierTreeMapSizeKb: {
|
|
60
|
+
env: 'NULLIFIER_TREE_MAP_SIZE_KB',
|
|
61
|
+
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
62
|
+
description:
|
|
63
|
+
'The maximum possible size of the world state nullifier tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
64
|
+
},
|
|
65
|
+
noteHashTreeMapSizeKb: {
|
|
66
|
+
env: 'NOTE_HASH_TREE_MAP_SIZE_KB',
|
|
67
|
+
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
68
|
+
description:
|
|
69
|
+
'The maximum possible size of the world state note hash tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
70
|
+
},
|
|
71
|
+
messageTreeMapSizeKb: {
|
|
72
|
+
env: 'MESSAGE_TREE_MAP_SIZE_KB',
|
|
73
|
+
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
74
|
+
description:
|
|
75
|
+
'The maximum possible size of the world state message tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
76
|
+
},
|
|
77
|
+
publicDataTreeMapSizeKb: {
|
|
78
|
+
env: 'PUBLIC_DATA_TREE_MAP_SIZE_KB',
|
|
79
|
+
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
80
|
+
description:
|
|
81
|
+
'The maximum possible size of the world state public data tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
50
82
|
},
|
|
51
83
|
worldStateDataDirectory: {
|
|
52
84
|
env: 'WS_DATA_DIRECTORY',
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { LoggerBindings } from '@aztec/foundation/log';
|
|
1
2
|
import type { DataStoreConfig } from '@aztec/kv-store/config';
|
|
2
3
|
import type { L2BlockSource } from '@aztec/stdlib/block';
|
|
3
4
|
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
@@ -9,44 +10,72 @@ import { NativeWorldStateService } from '../native/native_world_state.js';
|
|
|
9
10
|
import type { WorldStateConfig } from './config.js';
|
|
10
11
|
import { ServerWorldStateSynchronizer } from './server_world_state_synchronizer.js';
|
|
11
12
|
|
|
13
|
+
export interface WorldStateTreeMapSizes {
|
|
14
|
+
archiveTreeMapSizeKb: number;
|
|
15
|
+
nullifierTreeMapSizeKb: number;
|
|
16
|
+
noteHashTreeMapSizeKb: number;
|
|
17
|
+
messageTreeMapSizeKb: number;
|
|
18
|
+
publicDataTreeMapSizeKb: number;
|
|
19
|
+
}
|
|
20
|
+
|
|
12
21
|
export async function createWorldStateSynchronizer(
|
|
13
22
|
config: WorldStateConfig & DataStoreConfig,
|
|
14
23
|
l2BlockSource: L2BlockSource & L1ToL2MessageSource,
|
|
15
24
|
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
16
25
|
client: TelemetryClient = getTelemetryClient(),
|
|
26
|
+
bindings?: LoggerBindings,
|
|
17
27
|
) {
|
|
18
28
|
const instrumentation = new WorldStateInstrumentation(client);
|
|
19
|
-
const merkleTrees = await createWorldState(config, prefilledPublicData, instrumentation);
|
|
29
|
+
const merkleTrees = await createWorldState(config, prefilledPublicData, instrumentation, bindings);
|
|
20
30
|
return new ServerWorldStateSynchronizer(merkleTrees, l2BlockSource, config, instrumentation);
|
|
21
31
|
}
|
|
22
32
|
|
|
23
33
|
export async function createWorldState(
|
|
24
|
-
config:
|
|
34
|
+
config: Pick<
|
|
35
|
+
WorldStateConfig,
|
|
36
|
+
| 'worldStateDataDirectory'
|
|
37
|
+
| 'worldStateDbMapSizeKb'
|
|
38
|
+
| 'archiveTreeMapSizeKb'
|
|
39
|
+
| 'nullifierTreeMapSizeKb'
|
|
40
|
+
| 'noteHashTreeMapSizeKb'
|
|
41
|
+
| 'messageTreeMapSizeKb'
|
|
42
|
+
| 'publicDataTreeMapSizeKb'
|
|
43
|
+
> &
|
|
44
|
+
Pick<DataStoreConfig, 'dataDirectory' | 'dataStoreMapSizeKb' | 'l1Contracts'>,
|
|
25
45
|
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
26
46
|
instrumentation: WorldStateInstrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
47
|
+
bindings?: LoggerBindings,
|
|
27
48
|
) {
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
49
|
+
const dataDirectory = config.worldStateDataDirectory ?? config.dataDirectory;
|
|
50
|
+
const dataStoreMapSizeKb = config.worldStateDbMapSizeKb ?? config.dataStoreMapSizeKb;
|
|
51
|
+
const wsTreeMapSizes: WorldStateTreeMapSizes = {
|
|
52
|
+
archiveTreeMapSizeKb: config.archiveTreeMapSizeKb ?? dataStoreMapSizeKb,
|
|
53
|
+
nullifierTreeMapSizeKb: config.nullifierTreeMapSizeKb ?? dataStoreMapSizeKb,
|
|
54
|
+
noteHashTreeMapSizeKb: config.noteHashTreeMapSizeKb ?? dataStoreMapSizeKb,
|
|
55
|
+
messageTreeMapSizeKb: config.messageTreeMapSizeKb ?? dataStoreMapSizeKb,
|
|
56
|
+
publicDataTreeMapSizeKb: config.publicDataTreeMapSizeKb ?? dataStoreMapSizeKb,
|
|
57
|
+
};
|
|
32
58
|
|
|
33
59
|
if (!config.l1Contracts?.rollupAddress) {
|
|
34
60
|
throw new Error('Rollup address is required to create a world state synchronizer.');
|
|
35
61
|
}
|
|
36
62
|
|
|
37
63
|
// If a data directory is provided in config, then create a persistent store.
|
|
38
|
-
const merkleTrees =
|
|
64
|
+
const merkleTrees = dataDirectory
|
|
39
65
|
? await NativeWorldStateService.new(
|
|
40
66
|
config.l1Contracts.rollupAddress,
|
|
41
|
-
|
|
42
|
-
|
|
67
|
+
dataDirectory,
|
|
68
|
+
wsTreeMapSizes,
|
|
43
69
|
prefilledPublicData,
|
|
44
70
|
instrumentation,
|
|
71
|
+
bindings,
|
|
45
72
|
)
|
|
46
73
|
: await NativeWorldStateService.tmp(
|
|
47
74
|
config.l1Contracts.rollupAddress,
|
|
48
75
|
!['true', '1'].includes(process.env.DEBUG_WORLD_STATE!),
|
|
49
76
|
prefilledPublicData,
|
|
77
|
+
instrumentation,
|
|
78
|
+
bindings,
|
|
50
79
|
);
|
|
51
80
|
|
|
52
81
|
return merkleTrees;
|