@aztec/world-state 0.0.0-test.0 → 0.0.1-commit.24de95ac
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/instrumentation/instrumentation.d.ts +5 -3
- package/dest/instrumentation/instrumentation.d.ts.map +1 -1
- package/dest/instrumentation/instrumentation.js +16 -8
- package/dest/native/bench_metrics.d.ts +23 -0
- package/dest/native/bench_metrics.d.ts.map +1 -0
- package/dest/native/bench_metrics.js +81 -0
- package/dest/native/merkle_trees_facade.d.ts +9 -3
- package/dest/native/merkle_trees_facade.d.ts.map +1 -1
- package/dest/native/merkle_trees_facade.js +38 -7
- package/dest/native/message.d.ts +64 -44
- package/dest/native/message.d.ts.map +1 -1
- package/dest/native/message.js +55 -56
- package/dest/native/native_world_state.d.ts +18 -13
- package/dest/native/native_world_state.d.ts.map +1 -1
- package/dest/native/native_world_state.js +90 -33
- package/dest/native/native_world_state_instance.d.ts +10 -3
- package/dest/native/native_world_state_instance.d.ts.map +1 -1
- package/dest/native/native_world_state_instance.js +21 -3
- package/dest/native/world_state_ops_queue.js +1 -1
- package/dest/synchronizer/config.d.ts +11 -1
- package/dest/synchronizer/config.d.ts.map +1 -1
- package/dest/synchronizer/config.js +26 -1
- package/dest/synchronizer/errors.d.ts +4 -0
- package/dest/synchronizer/errors.d.ts.map +1 -0
- package/dest/synchronizer/errors.js +5 -0
- package/dest/synchronizer/factory.d.ts +8 -1
- package/dest/synchronizer/factory.d.ts.map +1 -1
- package/dest/synchronizer/factory.js +9 -4
- package/dest/synchronizer/server_world_state_synchronizer.d.ts +13 -6
- package/dest/synchronizer/server_world_state_synchronizer.d.ts.map +1 -1
- package/dest/synchronizer/server_world_state_synchronizer.js +78 -38
- package/dest/test/utils.d.ts +1 -1
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +21 -18
- package/dest/testing.d.ts +1 -1
- package/dest/testing.d.ts.map +1 -1
- package/dest/testing.js +6 -10
- package/dest/world-state-db/merkle_tree_db.d.ts +8 -4
- package/dest/world-state-db/merkle_tree_db.d.ts.map +1 -1
- package/package.json +21 -22
- package/src/instrumentation/instrumentation.ts +22 -10
- package/src/native/bench_metrics.ts +91 -0
- package/src/native/merkle_trees_facade.ts +44 -13
- package/src/native/message.ts +81 -63
- package/src/native/native_world_state.ts +98 -42
- package/src/native/native_world_state_instance.ts +31 -8
- package/src/native/world_state_ops_queue.ts +1 -1
- package/src/synchronizer/config.ts +47 -2
- package/src/synchronizer/errors.ts +5 -0
- package/src/synchronizer/factory.ts +31 -8
- package/src/synchronizer/server_world_state_synchronizer.ts +93 -40
- package/src/test/utils.ts +46 -31
- package/src/testing.ts +3 -7
- package/src/world-state-db/merkle_tree_db.ts +9 -4
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIERS_PER_TX, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
|
|
2
|
-
import { padArrayEnd } from '@aztec/foundation/collection';
|
|
2
|
+
import { fromEntries, padArrayEnd } from '@aztec/foundation/collection';
|
|
3
3
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
4
4
|
import { Fr } from '@aztec/foundation/fields';
|
|
5
|
-
import {
|
|
5
|
+
import { tryRmDir } from '@aztec/foundation/fs';
|
|
6
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
6
7
|
import type { L2Block } from '@aztec/stdlib/block';
|
|
7
8
|
import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
|
|
8
9
|
import type {
|
|
@@ -10,8 +11,10 @@ import type {
|
|
|
10
11
|
MerkleTreeReadOperations,
|
|
11
12
|
MerkleTreeWriteOperations,
|
|
12
13
|
} from '@aztec/stdlib/interfaces/server';
|
|
14
|
+
import type { SnapshotDataKeys } from '@aztec/stdlib/snapshots';
|
|
13
15
|
import { MerkleTreeId, NullifierLeaf, type NullifierLeafPreimage, PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
14
16
|
import { BlockHeader, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
|
|
17
|
+
import { WorldStateRevision } from '@aztec/stdlib/world-state';
|
|
15
18
|
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
16
19
|
|
|
17
20
|
import assert from 'assert/strict';
|
|
@@ -20,6 +23,7 @@ import { tmpdir } from 'os';
|
|
|
20
23
|
import { join } from 'path';
|
|
21
24
|
|
|
22
25
|
import { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
|
|
26
|
+
import type { WorldStateTreeMapSizes } from '../synchronizer/factory.js';
|
|
23
27
|
import type { MerkleTreeAdminDatabase as MerkleTreeDatabase } from '../world-state-db/merkle_tree_db.js';
|
|
24
28
|
import { MerkleTreesFacade, MerkleTreesForkFacade, serializeLeaf } from './merkle_trees_facade.js';
|
|
25
29
|
import {
|
|
@@ -27,16 +31,17 @@ import {
|
|
|
27
31
|
type WorldStateStatusFull,
|
|
28
32
|
type WorldStateStatusSummary,
|
|
29
33
|
blockStateReference,
|
|
30
|
-
|
|
31
|
-
|
|
34
|
+
sanitizeFullStatus,
|
|
35
|
+
sanitizeSummary,
|
|
32
36
|
treeStateReferenceToSnapshot,
|
|
33
|
-
worldStateRevision,
|
|
34
37
|
} from './message.js';
|
|
35
38
|
import { NativeWorldState } from './native_world_state_instance.js';
|
|
36
39
|
|
|
37
40
|
// The current version of the world state database schema
|
|
38
41
|
// Increment this when making incompatible changes to the database schema
|
|
39
|
-
export const WORLD_STATE_DB_VERSION =
|
|
42
|
+
export const WORLD_STATE_DB_VERSION = 2; // The initial version
|
|
43
|
+
|
|
44
|
+
export const WORLD_STATE_DIR = 'world_state';
|
|
40
45
|
|
|
41
46
|
export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
42
47
|
protected initialHeader: BlockHeader | undefined;
|
|
@@ -44,38 +49,38 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
44
49
|
private cachedStatusSummary: WorldStateStatusSummary | undefined;
|
|
45
50
|
|
|
46
51
|
protected constructor(
|
|
47
|
-
protected
|
|
52
|
+
protected instance: NativeWorldState,
|
|
48
53
|
protected readonly worldStateInstrumentation: WorldStateInstrumentation,
|
|
49
|
-
protected readonly log = createLogger('world-state:database'),
|
|
54
|
+
protected readonly log: Logger = createLogger('world-state:database'),
|
|
50
55
|
private readonly cleanup = () => Promise.resolve(),
|
|
51
56
|
) {}
|
|
52
57
|
|
|
53
58
|
static async new(
|
|
54
59
|
rollupAddress: EthAddress,
|
|
55
60
|
dataDir: string,
|
|
56
|
-
|
|
61
|
+
wsTreeMapSizes: WorldStateTreeMapSizes,
|
|
57
62
|
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
58
63
|
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
59
64
|
log = createLogger('world-state:database'),
|
|
60
65
|
cleanup = () => Promise.resolve(),
|
|
61
66
|
): Promise<NativeWorldStateService> {
|
|
62
|
-
const worldStateDirectory = join(dataDir,
|
|
67
|
+
const worldStateDirectory = join(dataDir, WORLD_STATE_DIR);
|
|
63
68
|
// Create a version manager to handle versioning
|
|
64
|
-
const versionManager = new DatabaseVersionManager(
|
|
65
|
-
WORLD_STATE_DB_VERSION,
|
|
69
|
+
const versionManager = new DatabaseVersionManager({
|
|
70
|
+
schemaVersion: WORLD_STATE_DB_VERSION,
|
|
66
71
|
rollupAddress,
|
|
67
|
-
worldStateDirectory,
|
|
68
|
-
(dir: string) => {
|
|
69
|
-
return Promise.resolve(new NativeWorldState(dir,
|
|
72
|
+
dataDirectory: worldStateDirectory,
|
|
73
|
+
onOpen: (dir: string) => {
|
|
74
|
+
return Promise.resolve(new NativeWorldState(dir, wsTreeMapSizes, prefilledPublicData, instrumentation));
|
|
70
75
|
},
|
|
71
|
-
);
|
|
76
|
+
});
|
|
72
77
|
|
|
73
78
|
const [instance] = await versionManager.open();
|
|
74
79
|
const worldState = new this(instance, instrumentation, log, cleanup);
|
|
75
80
|
try {
|
|
76
81
|
await worldState.init();
|
|
77
82
|
} catch (e) {
|
|
78
|
-
log.error(`Error
|
|
83
|
+
log.error(`Error initializing world state: ${e}`);
|
|
79
84
|
throw e;
|
|
80
85
|
}
|
|
81
86
|
|
|
@@ -91,7 +96,14 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
91
96
|
const log = createLogger('world-state:database');
|
|
92
97
|
const dataDir = await mkdtemp(join(tmpdir(), 'aztec-world-state-'));
|
|
93
98
|
const dbMapSizeKb = 10 * 1024 * 1024;
|
|
94
|
-
|
|
99
|
+
const worldStateTreeMapSizes: WorldStateTreeMapSizes = {
|
|
100
|
+
archiveTreeMapSizeKb: dbMapSizeKb,
|
|
101
|
+
nullifierTreeMapSizeKb: dbMapSizeKb,
|
|
102
|
+
noteHashTreeMapSizeKb: dbMapSizeKb,
|
|
103
|
+
messageTreeMapSizeKb: dbMapSizeKb,
|
|
104
|
+
publicDataTreeMapSizeKb: dbMapSizeKb,
|
|
105
|
+
};
|
|
106
|
+
log.debug(`Created temporary world state database at: ${dataDir} with tree map size: ${dbMapSizeKb}`);
|
|
95
107
|
|
|
96
108
|
// pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
|
|
97
109
|
const cleanup = async () => {
|
|
@@ -103,7 +115,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
103
115
|
}
|
|
104
116
|
};
|
|
105
117
|
|
|
106
|
-
return this.new(rollupAddress, dataDir,
|
|
118
|
+
return this.new(rollupAddress, dataDir, worldStateTreeMapSizes, prefilledPublicData, instrumentation, log, cleanup);
|
|
107
119
|
}
|
|
108
120
|
|
|
109
121
|
protected async init() {
|
|
@@ -127,12 +139,23 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
127
139
|
assert.strictEqual(initialHeaderIndex, 0n, 'Invalid initial archive state');
|
|
128
140
|
}
|
|
129
141
|
|
|
142
|
+
public async clear() {
|
|
143
|
+
await this.instance.close();
|
|
144
|
+
this.cachedStatusSummary = undefined;
|
|
145
|
+
await tryRmDir(this.instance.getDataDir(), this.log);
|
|
146
|
+
this.instance = this.instance.clone();
|
|
147
|
+
}
|
|
148
|
+
|
|
130
149
|
public getCommitted(): MerkleTreeReadOperations {
|
|
131
|
-
return new MerkleTreesFacade(this.instance, this.initialHeader!,
|
|
150
|
+
return new MerkleTreesFacade(this.instance, this.initialHeader!, WorldStateRevision.empty());
|
|
132
151
|
}
|
|
133
152
|
|
|
134
153
|
public getSnapshot(blockNumber: number): MerkleTreeReadOperations {
|
|
135
|
-
return new MerkleTreesFacade(
|
|
154
|
+
return new MerkleTreesFacade(
|
|
155
|
+
this.instance,
|
|
156
|
+
this.initialHeader!,
|
|
157
|
+
new WorldStateRevision(/*forkId=*/ 0, /* blockNumber=*/ blockNumber, /* includeUncommitted=*/ false),
|
|
158
|
+
);
|
|
136
159
|
}
|
|
137
160
|
|
|
138
161
|
public async fork(blockNumber?: number): Promise<MerkleTreeWriteOperations> {
|
|
@@ -141,19 +164,31 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
141
164
|
blockNumber: blockNumber ?? 0,
|
|
142
165
|
canonical: true,
|
|
143
166
|
});
|
|
144
|
-
return new MerkleTreesForkFacade(
|
|
167
|
+
return new MerkleTreesForkFacade(
|
|
168
|
+
this.instance,
|
|
169
|
+
this.initialHeader!,
|
|
170
|
+
new WorldStateRevision(/*forkId=*/ resp.forkId, /* blockNumber=*/ 0, /* includeUncommitted=*/ true),
|
|
171
|
+
);
|
|
145
172
|
}
|
|
146
173
|
|
|
147
174
|
public getInitialHeader(): BlockHeader {
|
|
148
175
|
return this.initialHeader!;
|
|
149
176
|
}
|
|
150
177
|
|
|
151
|
-
public async handleL2BlockAndMessages(
|
|
178
|
+
public async handleL2BlockAndMessages(
|
|
179
|
+
l2Block: L2Block,
|
|
180
|
+
l1ToL2Messages: Fr[],
|
|
181
|
+
// TODO(#17027)
|
|
182
|
+
// Temporary hack to only insert l1 to l2 messages for the first block in a checkpoint.
|
|
183
|
+
isFirstBlock = true,
|
|
184
|
+
): Promise<WorldStateStatusFull> {
|
|
152
185
|
// We have to pad both the values within tx effects because that's how the trees are built by circuits.
|
|
153
186
|
const paddedNoteHashes = l2Block.body.txEffects.flatMap(txEffect =>
|
|
154
187
|
padArrayEnd(txEffect.noteHashes, Fr.ZERO, MAX_NOTE_HASHES_PER_TX),
|
|
155
188
|
);
|
|
156
|
-
const paddedL1ToL2Messages =
|
|
189
|
+
const paddedL1ToL2Messages = isFirstBlock
|
|
190
|
+
? padArrayEnd<Fr, number>(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP)
|
|
191
|
+
: [];
|
|
157
192
|
|
|
158
193
|
const paddedNullifiers = l2Block.body.txEffects
|
|
159
194
|
.flatMap(txEffect => padArrayEnd(txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX))
|
|
@@ -173,7 +208,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
173
208
|
WorldStateMessageType.SYNC_BLOCK,
|
|
174
209
|
{
|
|
175
210
|
blockNumber: l2Block.number,
|
|
176
|
-
blockHeaderHash: await l2Block.
|
|
211
|
+
blockHeaderHash: await l2Block.getBlockHeader().hash(),
|
|
177
212
|
paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf),
|
|
178
213
|
paddedNoteHashes: paddedNoteHashes.map(serializeLeaf),
|
|
179
214
|
paddedNullifiers: paddedNullifiers.map(serializeLeaf),
|
|
@@ -181,7 +216,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
181
216
|
blockStateRef: blockStateReference(l2Block.header.state),
|
|
182
217
|
canonical: true,
|
|
183
218
|
},
|
|
184
|
-
this.
|
|
219
|
+
this.sanitizeAndCacheSummaryFromFull.bind(this),
|
|
185
220
|
this.deleteCachedSummary.bind(this),
|
|
186
221
|
);
|
|
187
222
|
} catch (err) {
|
|
@@ -200,16 +235,16 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
200
235
|
return BlockHeader.empty({ state });
|
|
201
236
|
}
|
|
202
237
|
|
|
203
|
-
private
|
|
204
|
-
const
|
|
205
|
-
this.cachedStatusSummary = { ...
|
|
206
|
-
return
|
|
238
|
+
private sanitizeAndCacheSummaryFromFull(response: WorldStateStatusFull) {
|
|
239
|
+
const sanitized = sanitizeFullStatus(response);
|
|
240
|
+
this.cachedStatusSummary = { ...sanitized.summary };
|
|
241
|
+
return sanitized;
|
|
207
242
|
}
|
|
208
243
|
|
|
209
|
-
private
|
|
210
|
-
const
|
|
211
|
-
this.cachedStatusSummary = { ...
|
|
212
|
-
return
|
|
244
|
+
private sanitizeAndCacheSummary(response: WorldStateStatusSummary) {
|
|
245
|
+
const sanitized = sanitizeSummary(response);
|
|
246
|
+
this.cachedStatusSummary = { ...sanitized };
|
|
247
|
+
return sanitized;
|
|
213
248
|
}
|
|
214
249
|
|
|
215
250
|
private deleteCachedSummary(_: string) {
|
|
@@ -217,19 +252,19 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
217
252
|
}
|
|
218
253
|
|
|
219
254
|
/**
|
|
220
|
-
* Advances the
|
|
221
|
-
* @param toBlockNumber The block number that is now the tip of the
|
|
255
|
+
* Advances the finalized block number to be the number provided
|
|
256
|
+
* @param toBlockNumber The block number that is now the tip of the finalized chain
|
|
222
257
|
* @returns The new WorldStateStatus
|
|
223
258
|
*/
|
|
224
|
-
public async
|
|
259
|
+
public async setFinalized(toBlockNumber: bigint) {
|
|
225
260
|
try {
|
|
226
261
|
await this.instance.call(
|
|
227
|
-
WorldStateMessageType.
|
|
262
|
+
WorldStateMessageType.FINALIZE_BLOCKS,
|
|
228
263
|
{
|
|
229
264
|
toBlockNumber,
|
|
230
265
|
canonical: true,
|
|
231
266
|
},
|
|
232
|
-
this.
|
|
267
|
+
this.sanitizeAndCacheSummary.bind(this),
|
|
233
268
|
this.deleteCachedSummary.bind(this),
|
|
234
269
|
);
|
|
235
270
|
} catch (err) {
|
|
@@ -252,7 +287,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
252
287
|
toBlockNumber,
|
|
253
288
|
canonical: true,
|
|
254
289
|
},
|
|
255
|
-
this.
|
|
290
|
+
this.sanitizeAndCacheSummaryFromFull.bind(this),
|
|
256
291
|
this.deleteCachedSummary.bind(this),
|
|
257
292
|
);
|
|
258
293
|
} catch (err) {
|
|
@@ -274,7 +309,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
274
309
|
toBlockNumber,
|
|
275
310
|
canonical: true,
|
|
276
311
|
},
|
|
277
|
-
this.
|
|
312
|
+
this.sanitizeAndCacheSummaryFromFull.bind(this),
|
|
278
313
|
this.deleteCachedSummary.bind(this),
|
|
279
314
|
);
|
|
280
315
|
} catch (err) {
|
|
@@ -290,7 +325,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
290
325
|
return await this.instance.call(
|
|
291
326
|
WorldStateMessageType.GET_STATUS,
|
|
292
327
|
{ canonical: true },
|
|
293
|
-
this.
|
|
328
|
+
this.sanitizeAndCacheSummary.bind(this),
|
|
294
329
|
);
|
|
295
330
|
}
|
|
296
331
|
|
|
@@ -314,4 +349,25 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
314
349
|
),
|
|
315
350
|
);
|
|
316
351
|
}
|
|
352
|
+
|
|
353
|
+
public async backupTo(
|
|
354
|
+
dstPath: string,
|
|
355
|
+
compact: boolean = true,
|
|
356
|
+
): Promise<Record<Exclude<SnapshotDataKeys, 'archiver'>, string>> {
|
|
357
|
+
await this.instance.call(WorldStateMessageType.COPY_STORES, {
|
|
358
|
+
dstPath,
|
|
359
|
+
compact,
|
|
360
|
+
canonical: true,
|
|
361
|
+
});
|
|
362
|
+
return fromEntries(NATIVE_WORLD_STATE_DBS.map(([name, dir]) => [name, join(dstPath, dir, 'data.mdb')] as const));
|
|
363
|
+
}
|
|
317
364
|
}
|
|
365
|
+
|
|
366
|
+
// The following paths are defined in cpp-land
|
|
367
|
+
export const NATIVE_WORLD_STATE_DBS = [
|
|
368
|
+
['l1-to-l2-message-tree', 'L1ToL2MessageTree'],
|
|
369
|
+
['archive-tree', 'ArchiveTree'],
|
|
370
|
+
['public-data-tree', 'PublicDataTree'],
|
|
371
|
+
['note-hash-tree', 'NoteHashTree'],
|
|
372
|
+
['nullifier-tree', 'NullifierTree'],
|
|
373
|
+
] as const;
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
NULLIFIER_TREE_HEIGHT,
|
|
9
9
|
PUBLIC_DATA_TREE_HEIGHT,
|
|
10
10
|
} from '@aztec/constants';
|
|
11
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
11
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
12
12
|
import { NativeWorldState as BaseNativeWorldState, MsgpackChannel } from '@aztec/native';
|
|
13
13
|
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
14
14
|
import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
@@ -17,6 +17,7 @@ import assert from 'assert';
|
|
|
17
17
|
import { cpus } from 'os';
|
|
18
18
|
|
|
19
19
|
import type { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
|
|
20
|
+
import type { WorldStateTreeMapSizes } from '../synchronizer/factory.js';
|
|
20
21
|
import {
|
|
21
22
|
WorldStateMessageType,
|
|
22
23
|
type WorldStateRequest,
|
|
@@ -50,15 +51,17 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
50
51
|
|
|
51
52
|
/** Creates a new native WorldState instance */
|
|
52
53
|
constructor(
|
|
53
|
-
dataDir: string,
|
|
54
|
-
|
|
55
|
-
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
56
|
-
private instrumentation: WorldStateInstrumentation,
|
|
57
|
-
private log = createLogger('world-state:database'),
|
|
54
|
+
private readonly dataDir: string,
|
|
55
|
+
private readonly wsTreeMapSizes: WorldStateTreeMapSizes,
|
|
56
|
+
private readonly prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
57
|
+
private readonly instrumentation: WorldStateInstrumentation,
|
|
58
|
+
private readonly log: Logger = createLogger('world-state:database'),
|
|
58
59
|
) {
|
|
59
60
|
const threads = Math.min(cpus().length, MAX_WORLD_STATE_THREADS);
|
|
60
61
|
log.info(
|
|
61
|
-
`Creating world state data store at directory ${dataDir} with map
|
|
62
|
+
`Creating world state data store at directory ${dataDir} with map sizes ${JSON.stringify(
|
|
63
|
+
wsTreeMapSizes,
|
|
64
|
+
)} and ${threads} threads.`,
|
|
62
65
|
);
|
|
63
66
|
const prefilledPublicDataBufferArray = prefilledPublicData.map(d => [d.slot.toBuffer(), d.value.toBuffer()]);
|
|
64
67
|
const ws = new BaseNativeWorldState(
|
|
@@ -76,7 +79,13 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
76
79
|
},
|
|
77
80
|
prefilledPublicDataBufferArray,
|
|
78
81
|
GeneratorIndex.BLOCK_HASH,
|
|
79
|
-
|
|
82
|
+
{
|
|
83
|
+
[MerkleTreeId.NULLIFIER_TREE]: wsTreeMapSizes.nullifierTreeMapSizeKb,
|
|
84
|
+
[MerkleTreeId.NOTE_HASH_TREE]: wsTreeMapSizes.noteHashTreeMapSizeKb,
|
|
85
|
+
[MerkleTreeId.PUBLIC_DATA_TREE]: wsTreeMapSizes.publicDataTreeMapSizeKb,
|
|
86
|
+
[MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: wsTreeMapSizes.messageTreeMapSizeKb,
|
|
87
|
+
[MerkleTreeId.ARCHIVE]: wsTreeMapSizes.archiveTreeMapSizeKb,
|
|
88
|
+
},
|
|
80
89
|
threads,
|
|
81
90
|
);
|
|
82
91
|
this.instance = new MsgpackChannel(ws);
|
|
@@ -84,6 +93,20 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
84
93
|
this.queues.set(0, new WorldStateOpsQueue());
|
|
85
94
|
}
|
|
86
95
|
|
|
96
|
+
public getDataDir() {
|
|
97
|
+
return this.dataDir;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
public clone() {
|
|
101
|
+
return new NativeWorldState(
|
|
102
|
+
this.dataDir,
|
|
103
|
+
this.wsTreeMapSizes,
|
|
104
|
+
this.prefilledPublicData,
|
|
105
|
+
this.instrumentation,
|
|
106
|
+
this.log,
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
|
|
87
110
|
/**
|
|
88
111
|
* Sends a message to the native instance and returns the response.
|
|
89
112
|
* @param messageType - The type of message to send
|
|
@@ -35,7 +35,7 @@ export const MUTATING_MSG_TYPES = new Set([
|
|
|
35
35
|
WorldStateMessageType.SYNC_BLOCK,
|
|
36
36
|
WorldStateMessageType.CREATE_FORK,
|
|
37
37
|
WorldStateMessageType.DELETE_FORK,
|
|
38
|
-
WorldStateMessageType.
|
|
38
|
+
WorldStateMessageType.FINALIZE_BLOCKS,
|
|
39
39
|
WorldStateMessageType.UNWIND_BLOCKS,
|
|
40
40
|
WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS,
|
|
41
41
|
WorldStateMessageType.CREATE_CHECKPOINT,
|
|
@@ -16,9 +16,24 @@ export interface WorldStateConfig {
|
|
|
16
16
|
/** Size of the batch for each get-blocks request from the synchronizer to the archiver. */
|
|
17
17
|
worldStateBlockRequestBatchSize?: number;
|
|
18
18
|
|
|
19
|
-
/** The map size to be provided to LMDB for each world state tree DB, optional, will inherit from the general
|
|
19
|
+
/** The map size to be provided to LMDB for each world state tree DB, optional, will inherit from the general dataStoreMapSizeKb if not specified*/
|
|
20
20
|
worldStateDbMapSizeKb?: number;
|
|
21
21
|
|
|
22
|
+
/** The map size to be provided to LMDB for each world state archive tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
|
|
23
|
+
archiveTreeMapSizeKb?: number;
|
|
24
|
+
|
|
25
|
+
/** The map size to be provided to LMDB for each world state nullifier tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
|
|
26
|
+
nullifierTreeMapSizeKb?: number;
|
|
27
|
+
|
|
28
|
+
/** The map size to be provided to LMDB for each world state note hash tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
|
|
29
|
+
noteHashTreeMapSizeKb?: number;
|
|
30
|
+
|
|
31
|
+
/** The map size to be provided to LMDB for each world state message tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
|
|
32
|
+
messageTreeMapSizeKb?: number;
|
|
33
|
+
|
|
34
|
+
/** The map size to be provided to LMDB for each world state public data tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
|
|
35
|
+
publicDataTreeMapSizeKb?: number;
|
|
36
|
+
|
|
22
37
|
/** Optional directory for the world state DB, if unspecified will default to the general data directory */
|
|
23
38
|
worldStateDataDirectory?: string;
|
|
24
39
|
|
|
@@ -46,7 +61,37 @@ export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
|
|
|
46
61
|
worldStateDbMapSizeKb: {
|
|
47
62
|
env: 'WS_DB_MAP_SIZE_KB',
|
|
48
63
|
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
49
|
-
description: 'The maximum possible size of the world state DB',
|
|
64
|
+
description: 'The maximum possible size of the world state DB in KB. Overwrites the general dataStoreMapSizeKb.',
|
|
65
|
+
},
|
|
66
|
+
archiveTreeMapSizeKb: {
|
|
67
|
+
env: 'ARCHIVE_TREE_MAP_SIZE_KB',
|
|
68
|
+
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
69
|
+
description:
|
|
70
|
+
'The maximum possible size of the world state archive tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
71
|
+
},
|
|
72
|
+
nullifierTreeMapSizeKb: {
|
|
73
|
+
env: 'NULLIFIER_TREE_MAP_SIZE_KB',
|
|
74
|
+
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
75
|
+
description:
|
|
76
|
+
'The maximum possible size of the world state nullifier tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
77
|
+
},
|
|
78
|
+
noteHashTreeMapSizeKb: {
|
|
79
|
+
env: 'NOTE_HASH_TREE_MAP_SIZE_KB',
|
|
80
|
+
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
81
|
+
description:
|
|
82
|
+
'The maximum possible size of the world state note hash tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
83
|
+
},
|
|
84
|
+
messageTreeMapSizeKb: {
|
|
85
|
+
env: 'MESSAGE_TREE_MAP_SIZE_KB',
|
|
86
|
+
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
87
|
+
description:
|
|
88
|
+
'The maximum possible size of the world state message tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
89
|
+
},
|
|
90
|
+
publicDataTreeMapSizeKb: {
|
|
91
|
+
env: 'PUBLIC_DATA_TREE_MAP_SIZE_KB',
|
|
92
|
+
parseEnv: (val: string | undefined) => (val ? +val : undefined),
|
|
93
|
+
description:
|
|
94
|
+
'The maximum possible size of the world state public data tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
50
95
|
},
|
|
51
96
|
worldStateDataDirectory: {
|
|
52
97
|
env: 'WS_DATA_DIRECTORY',
|
|
@@ -9,6 +9,14 @@ import { NativeWorldStateService } from '../native/native_world_state.js';
|
|
|
9
9
|
import type { WorldStateConfig } from './config.js';
|
|
10
10
|
import { ServerWorldStateSynchronizer } from './server_world_state_synchronizer.js';
|
|
11
11
|
|
|
12
|
+
export interface WorldStateTreeMapSizes {
|
|
13
|
+
archiveTreeMapSizeKb: number;
|
|
14
|
+
nullifierTreeMapSizeKb: number;
|
|
15
|
+
noteHashTreeMapSizeKb: number;
|
|
16
|
+
messageTreeMapSizeKb: number;
|
|
17
|
+
publicDataTreeMapSizeKb: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
12
20
|
export async function createWorldStateSynchronizer(
|
|
13
21
|
config: WorldStateConfig & DataStoreConfig,
|
|
14
22
|
l2BlockSource: L2BlockSource & L1ToL2MessageSource,
|
|
@@ -21,25 +29,40 @@ export async function createWorldStateSynchronizer(
|
|
|
21
29
|
}
|
|
22
30
|
|
|
23
31
|
export async function createWorldState(
|
|
24
|
-
config:
|
|
32
|
+
config: Pick<
|
|
33
|
+
WorldStateConfig,
|
|
34
|
+
| 'worldStateDataDirectory'
|
|
35
|
+
| 'worldStateDbMapSizeKb'
|
|
36
|
+
| 'archiveTreeMapSizeKb'
|
|
37
|
+
| 'nullifierTreeMapSizeKb'
|
|
38
|
+
| 'noteHashTreeMapSizeKb'
|
|
39
|
+
| 'messageTreeMapSizeKb'
|
|
40
|
+
| 'publicDataTreeMapSizeKb'
|
|
41
|
+
> &
|
|
42
|
+
Pick<DataStoreConfig, 'dataDirectory' | 'dataStoreMapSizeKb' | 'l1Contracts'>,
|
|
25
43
|
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
26
44
|
instrumentation: WorldStateInstrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
27
45
|
) {
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
46
|
+
const dataDirectory = config.worldStateDataDirectory ?? config.dataDirectory;
|
|
47
|
+
const dataStoreMapSizeKb = config.worldStateDbMapSizeKb ?? config.dataStoreMapSizeKb;
|
|
48
|
+
const wsTreeMapSizes: WorldStateTreeMapSizes = {
|
|
49
|
+
archiveTreeMapSizeKb: config.archiveTreeMapSizeKb ?? dataStoreMapSizeKb,
|
|
50
|
+
nullifierTreeMapSizeKb: config.nullifierTreeMapSizeKb ?? dataStoreMapSizeKb,
|
|
51
|
+
noteHashTreeMapSizeKb: config.noteHashTreeMapSizeKb ?? dataStoreMapSizeKb,
|
|
52
|
+
messageTreeMapSizeKb: config.messageTreeMapSizeKb ?? dataStoreMapSizeKb,
|
|
53
|
+
publicDataTreeMapSizeKb: config.publicDataTreeMapSizeKb ?? dataStoreMapSizeKb,
|
|
54
|
+
};
|
|
32
55
|
|
|
33
56
|
if (!config.l1Contracts?.rollupAddress) {
|
|
34
57
|
throw new Error('Rollup address is required to create a world state synchronizer.');
|
|
35
58
|
}
|
|
36
59
|
|
|
37
60
|
// If a data directory is provided in config, then create a persistent store.
|
|
38
|
-
const merkleTrees =
|
|
61
|
+
const merkleTrees = dataDirectory
|
|
39
62
|
? await NativeWorldStateService.new(
|
|
40
63
|
config.l1Contracts.rollupAddress,
|
|
41
|
-
|
|
42
|
-
|
|
64
|
+
dataDirectory,
|
|
65
|
+
wsTreeMapSizes,
|
|
43
66
|
prefilledPublicData,
|
|
44
67
|
instrumentation,
|
|
45
68
|
)
|