@aztec/world-state 0.0.0-test.0 → 0.0.1-commit.21caa21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/index.d.ts +1 -1
- package/dest/instrumentation/instrumentation.d.ts +6 -4
- package/dest/instrumentation/instrumentation.d.ts.map +1 -1
- package/dest/instrumentation/instrumentation.js +16 -8
- package/dest/native/bench_metrics.d.ts +23 -0
- package/dest/native/bench_metrics.d.ts.map +1 -0
- package/dest/native/bench_metrics.js +81 -0
- package/dest/native/fork_checkpoint.d.ts +1 -1
- package/dest/native/fork_checkpoint.d.ts.map +1 -1
- package/dest/native/index.d.ts +1 -1
- package/dest/native/merkle_trees_facade.d.ts +10 -4
- package/dest/native/merkle_trees_facade.d.ts.map +1 -1
- package/dest/native/merkle_trees_facade.js +39 -7
- package/dest/native/message.d.ts +65 -45
- package/dest/native/message.d.ts.map +1 -1
- package/dest/native/message.js +55 -56
- package/dest/native/native_world_state.d.ts +20 -15
- package/dest/native/native_world_state.d.ts.map +1 -1
- package/dest/native/native_world_state.js +90 -33
- package/dest/native/native_world_state_instance.d.ts +20 -4
- package/dest/native/native_world_state_instance.d.ts.map +1 -1
- package/dest/native/native_world_state_instance.js +42 -3
- package/dest/native/world_state_ops_queue.d.ts +1 -1
- package/dest/native/world_state_ops_queue.d.ts.map +1 -1
- package/dest/native/world_state_ops_queue.js +1 -1
- package/dest/synchronizer/config.d.ts +12 -2
- package/dest/synchronizer/config.d.ts.map +1 -1
- package/dest/synchronizer/config.js +26 -1
- package/dest/synchronizer/errors.d.ts +4 -0
- package/dest/synchronizer/errors.d.ts.map +1 -0
- package/dest/synchronizer/errors.js +5 -0
- package/dest/synchronizer/factory.d.ts +9 -2
- package/dest/synchronizer/factory.d.ts.map +1 -1
- package/dest/synchronizer/factory.js +9 -4
- package/dest/synchronizer/index.d.ts +1 -1
- package/dest/synchronizer/server_world_state_synchronizer.d.ts +14 -18
- package/dest/synchronizer/server_world_state_synchronizer.d.ts.map +1 -1
- package/dest/synchronizer/server_world_state_synchronizer.js +78 -38
- package/dest/test/index.d.ts +1 -1
- package/dest/test/utils.d.ts +2 -2
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +21 -18
- package/dest/testing.d.ts +2 -2
- package/dest/testing.d.ts.map +1 -1
- package/dest/testing.js +6 -10
- package/dest/world-state-db/index.d.ts +1 -1
- package/dest/world-state-db/merkle_tree_db.d.ts +10 -6
- package/dest/world-state-db/merkle_tree_db.d.ts.map +1 -1
- package/package.json +24 -24
- package/src/instrumentation/instrumentation.ts +22 -10
- package/src/native/bench_metrics.ts +91 -0
- package/src/native/merkle_trees_facade.ts +44 -13
- package/src/native/message.ts +81 -63
- package/src/native/native_world_state.ts +99 -43
- package/src/native/native_world_state_instance.ts +59 -8
- package/src/native/world_state_ops_queue.ts +1 -1
- package/src/synchronizer/config.ts +47 -2
- package/src/synchronizer/errors.ts +5 -0
- package/src/synchronizer/factory.ts +31 -8
- package/src/synchronizer/server_world_state_synchronizer.ts +93 -40
- package/src/test/utils.ts +46 -31
- package/src/testing.ts +3 -7
- package/src/world-state-db/merkle_tree_db.ts +14 -5
package/src/native/message.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type { Tuple } from '@aztec/foundation/serialize';
|
|
|
3
3
|
import { AppendOnlyTreeSnapshot, MerkleTreeId } from '@aztec/stdlib/trees';
|
|
4
4
|
import type { StateReference } from '@aztec/stdlib/tx';
|
|
5
5
|
import type { UInt32 } from '@aztec/stdlib/types';
|
|
6
|
+
import type { WorldStateRevision } from '@aztec/stdlib/world-state';
|
|
6
7
|
|
|
7
8
|
export enum WorldStateMessageType {
|
|
8
9
|
GET_TREE_INFO = 100,
|
|
@@ -16,6 +17,7 @@ export enum WorldStateMessageType {
|
|
|
16
17
|
|
|
17
18
|
FIND_LEAF_INDICES,
|
|
18
19
|
FIND_LOW_LEAF,
|
|
20
|
+
FIND_SIBLING_PATHS,
|
|
19
21
|
|
|
20
22
|
APPEND_LEAVES,
|
|
21
23
|
BATCH_INSERT,
|
|
@@ -31,7 +33,7 @@ export enum WorldStateMessageType {
|
|
|
31
33
|
CREATE_FORK,
|
|
32
34
|
DELETE_FORK,
|
|
33
35
|
|
|
34
|
-
|
|
36
|
+
FINALIZE_BLOCKS,
|
|
35
37
|
UNWIND_BLOCKS,
|
|
36
38
|
REMOVE_HISTORICAL_BLOCKS,
|
|
37
39
|
|
|
@@ -40,6 +42,10 @@ export enum WorldStateMessageType {
|
|
|
40
42
|
CREATE_CHECKPOINT,
|
|
41
43
|
COMMIT_CHECKPOINT,
|
|
42
44
|
REVERT_CHECKPOINT,
|
|
45
|
+
COMMIT_ALL_CHECKPOINTS,
|
|
46
|
+
REVERT_ALL_CHECKPOINTS,
|
|
47
|
+
|
|
48
|
+
COPY_STORES,
|
|
43
49
|
|
|
44
50
|
CLOSE = 999,
|
|
45
51
|
}
|
|
@@ -50,9 +56,9 @@ interface WithTreeId {
|
|
|
50
56
|
|
|
51
57
|
export interface WorldStateStatusSummary {
|
|
52
58
|
/** Last block number that can still be unwound. */
|
|
53
|
-
|
|
54
|
-
/** Last block number that is
|
|
55
|
-
|
|
59
|
+
unfinalizedBlockNumber: bigint;
|
|
60
|
+
/** Last block number that is finalized and cannot be unwound. */
|
|
61
|
+
finalizedBlockNumber: bigint;
|
|
56
62
|
/** Oldest block still available for historical queries and forks. */
|
|
57
63
|
oldestHistoricalBlock: bigint;
|
|
58
64
|
/** Whether the trees are in sync with each other */
|
|
@@ -76,10 +82,10 @@ export interface TreeMeta {
|
|
|
76
82
|
initialRoot: Fr;
|
|
77
83
|
/** The current oldest historical block number of the tree */
|
|
78
84
|
oldestHistoricBlock: bigint;
|
|
79
|
-
/** The current
|
|
80
|
-
|
|
81
|
-
/** The current
|
|
82
|
-
|
|
85
|
+
/** The current unfinalized block number of the tree */
|
|
86
|
+
unfinalizedBlockHeight: bigint;
|
|
87
|
+
/** The current finalized block number of the tree */
|
|
88
|
+
finalizedBlockHeight: bigint;
|
|
83
89
|
}
|
|
84
90
|
|
|
85
91
|
export interface DBStats {
|
|
@@ -94,6 +100,8 @@ export interface DBStats {
|
|
|
94
100
|
export interface TreeDBStats {
|
|
95
101
|
/** The configured max size of the DB mapping file (effectively the max possible size of the DB) */
|
|
96
102
|
mapSize: bigint;
|
|
103
|
+
/** The physical file size of the database on disk */
|
|
104
|
+
physicalFileSize: bigint;
|
|
97
105
|
/** Stats for the 'blocks' DB */
|
|
98
106
|
blocksDBStats: DBStats;
|
|
99
107
|
/** Stats for the 'nodes' DB */
|
|
@@ -149,6 +157,7 @@ export function buildEmptyDBStats() {
|
|
|
149
157
|
export function buildEmptyTreeDBStats() {
|
|
150
158
|
return {
|
|
151
159
|
mapSize: 0n,
|
|
160
|
+
physicalFileSize: 0n,
|
|
152
161
|
blocksDBStats: buildEmptyDBStats(),
|
|
153
162
|
nodesDBStats: buildEmptyDBStats(),
|
|
154
163
|
leafIndicesDBStats: buildEmptyDBStats(),
|
|
@@ -164,8 +173,8 @@ export function buildEmptyTreeMeta() {
|
|
|
164
173
|
depth: 0,
|
|
165
174
|
size: 0n,
|
|
166
175
|
committedSize: 0n,
|
|
167
|
-
|
|
168
|
-
|
|
176
|
+
unfinalizedBlockHeight: 0n,
|
|
177
|
+
finalizedBlockHeight: 0n,
|
|
169
178
|
oldestHistoricBlock: 0n,
|
|
170
179
|
root: Fr.ZERO,
|
|
171
180
|
initialRoot: Fr.ZERO,
|
|
@@ -195,8 +204,8 @@ export function buildEmptyWorldStateDBStats() {
|
|
|
195
204
|
|
|
196
205
|
export function buildEmptyWorldStateSummary() {
|
|
197
206
|
return {
|
|
198
|
-
|
|
199
|
-
|
|
207
|
+
unfinalizedBlockNumber: 0n,
|
|
208
|
+
finalizedBlockNumber: 0n,
|
|
200
209
|
oldestHistoricalBlock: 0n,
|
|
201
210
|
treesAreSynched: true,
|
|
202
211
|
} as WorldStateStatusSummary;
|
|
@@ -210,61 +219,62 @@ export function buildEmptyWorldStateStatusFull() {
|
|
|
210
219
|
} as WorldStateStatusFull;
|
|
211
220
|
}
|
|
212
221
|
|
|
213
|
-
export function
|
|
214
|
-
summary.
|
|
215
|
-
summary.
|
|
222
|
+
export function sanitizeSummary(summary: WorldStateStatusSummary) {
|
|
223
|
+
summary.finalizedBlockNumber = BigInt(summary.finalizedBlockNumber);
|
|
224
|
+
summary.unfinalizedBlockNumber = BigInt(summary.unfinalizedBlockNumber);
|
|
216
225
|
summary.oldestHistoricalBlock = BigInt(summary.oldestHistoricalBlock);
|
|
217
226
|
return summary;
|
|
218
227
|
}
|
|
219
228
|
|
|
220
|
-
export function
|
|
229
|
+
export function sanitizeDBStats(stats: DBStats) {
|
|
221
230
|
stats.numDataItems = BigInt(stats.numDataItems);
|
|
222
231
|
stats.totalUsedSize = BigInt(stats.totalUsedSize);
|
|
223
232
|
return stats;
|
|
224
233
|
}
|
|
225
234
|
|
|
226
|
-
export function
|
|
235
|
+
export function sanitizeMeta(meta: TreeMeta) {
|
|
227
236
|
meta.committedSize = BigInt(meta.committedSize);
|
|
228
|
-
meta.
|
|
237
|
+
meta.finalizedBlockHeight = BigInt(meta.finalizedBlockHeight);
|
|
229
238
|
meta.initialSize = BigInt(meta.initialSize);
|
|
230
239
|
meta.oldestHistoricBlock = BigInt(meta.oldestHistoricBlock);
|
|
231
240
|
meta.size = BigInt(meta.size);
|
|
232
|
-
meta.
|
|
241
|
+
meta.unfinalizedBlockHeight = BigInt(meta.unfinalizedBlockHeight);
|
|
233
242
|
return meta;
|
|
234
243
|
}
|
|
235
244
|
|
|
236
|
-
export function
|
|
237
|
-
stats.blocksDBStats =
|
|
238
|
-
stats.leafIndicesDBStats =
|
|
239
|
-
stats.leafPreimagesDBStats =
|
|
240
|
-
stats.blockIndicesDBStats =
|
|
241
|
-
stats.nodesDBStats =
|
|
245
|
+
export function sanitizeTreeDBStats(stats: TreeDBStats) {
|
|
246
|
+
stats.blocksDBStats = sanitizeDBStats(stats.blocksDBStats);
|
|
247
|
+
stats.leafIndicesDBStats = sanitizeDBStats(stats.leafIndicesDBStats);
|
|
248
|
+
stats.leafPreimagesDBStats = sanitizeDBStats(stats.leafPreimagesDBStats);
|
|
249
|
+
stats.blockIndicesDBStats = sanitizeDBStats(stats.blockIndicesDBStats);
|
|
250
|
+
stats.nodesDBStats = sanitizeDBStats(stats.nodesDBStats);
|
|
242
251
|
stats.mapSize = BigInt(stats.mapSize);
|
|
252
|
+
stats.physicalFileSize = BigInt(stats.physicalFileSize);
|
|
243
253
|
return stats;
|
|
244
254
|
}
|
|
245
255
|
|
|
246
|
-
export function
|
|
247
|
-
stats.archiveTreeStats =
|
|
248
|
-
stats.messageTreeStats =
|
|
249
|
-
stats.noteHashTreeStats =
|
|
250
|
-
stats.nullifierTreeStats =
|
|
251
|
-
stats.publicDataTreeStats =
|
|
256
|
+
export function sanitizeWorldStateDBStats(stats: WorldStateDBStats) {
|
|
257
|
+
stats.archiveTreeStats = sanitizeTreeDBStats(stats.archiveTreeStats);
|
|
258
|
+
stats.messageTreeStats = sanitizeTreeDBStats(stats.messageTreeStats);
|
|
259
|
+
stats.noteHashTreeStats = sanitizeTreeDBStats(stats.noteHashTreeStats);
|
|
260
|
+
stats.nullifierTreeStats = sanitizeTreeDBStats(stats.nullifierTreeStats);
|
|
261
|
+
stats.publicDataTreeStats = sanitizeTreeDBStats(stats.publicDataTreeStats);
|
|
252
262
|
return stats;
|
|
253
263
|
}
|
|
254
264
|
|
|
255
|
-
export function
|
|
256
|
-
meta.archiveTreeMeta =
|
|
257
|
-
meta.messageTreeMeta =
|
|
258
|
-
meta.noteHashTreeMeta =
|
|
259
|
-
meta.nullifierTreeMeta =
|
|
260
|
-
meta.publicDataTreeMeta =
|
|
265
|
+
export function sanitizeWorldStateTreeMeta(meta: WorldStateMeta) {
|
|
266
|
+
meta.archiveTreeMeta = sanitizeMeta(meta.archiveTreeMeta);
|
|
267
|
+
meta.messageTreeMeta = sanitizeMeta(meta.messageTreeMeta);
|
|
268
|
+
meta.noteHashTreeMeta = sanitizeMeta(meta.noteHashTreeMeta);
|
|
269
|
+
meta.nullifierTreeMeta = sanitizeMeta(meta.nullifierTreeMeta);
|
|
270
|
+
meta.publicDataTreeMeta = sanitizeMeta(meta.publicDataTreeMeta);
|
|
261
271
|
return meta;
|
|
262
272
|
}
|
|
263
273
|
|
|
264
|
-
export function
|
|
265
|
-
status.dbStats =
|
|
266
|
-
status.summary =
|
|
267
|
-
status.meta =
|
|
274
|
+
export function sanitizeFullStatus(status: WorldStateStatusFull) {
|
|
275
|
+
status.dbStats = sanitizeWorldStateDBStats(status.dbStats);
|
|
276
|
+
status.summary = sanitizeSummary(status.summary);
|
|
277
|
+
status.meta = sanitizeWorldStateTreeMeta(status.meta);
|
|
268
278
|
return status;
|
|
269
279
|
}
|
|
270
280
|
|
|
@@ -286,13 +296,13 @@ interface WithLeafIndex {
|
|
|
286
296
|
|
|
287
297
|
export type SerializedLeafValue =
|
|
288
298
|
| Buffer // Fr
|
|
289
|
-
| {
|
|
299
|
+
| { nullifier: Buffer } // NullifierLeaf
|
|
290
300
|
| { value: Buffer; slot: Buffer }; // PublicDataTreeLeaf
|
|
291
301
|
|
|
292
302
|
export type SerializedIndexedLeaf = {
|
|
293
|
-
|
|
303
|
+
leaf: Exclude<SerializedLeafValue, Buffer>;
|
|
294
304
|
nextIndex: bigint | number;
|
|
295
|
-
|
|
305
|
+
nextKey: Buffer; // Fr
|
|
296
306
|
};
|
|
297
307
|
|
|
298
308
|
interface WithLeafValues {
|
|
@@ -344,6 +354,16 @@ interface FindLeafIndicesResponse {
|
|
|
344
354
|
indices: bigint[];
|
|
345
355
|
}
|
|
346
356
|
|
|
357
|
+
interface FindSiblingPathsRequest extends WithTreeId, WithLeafValues, WithWorldStateRevision {}
|
|
358
|
+
|
|
359
|
+
interface SiblingPathAndIndex {
|
|
360
|
+
index: bigint;
|
|
361
|
+
path: Buffer[];
|
|
362
|
+
}
|
|
363
|
+
interface FindSiblingPathsResponse {
|
|
364
|
+
paths: (SiblingPathAndIndex | undefined)[];
|
|
365
|
+
}
|
|
366
|
+
|
|
347
367
|
interface FindLowLeafRequest extends WithTreeId, WithWorldStateRevision {
|
|
348
368
|
key: Fr;
|
|
349
369
|
}
|
|
@@ -409,6 +429,11 @@ interface CreateForkResponse {
|
|
|
409
429
|
|
|
410
430
|
interface DeleteForkRequest extends WithForkId {}
|
|
411
431
|
|
|
432
|
+
interface CopyStoresRequest extends WithCanonicalForkId {
|
|
433
|
+
dstPath: string;
|
|
434
|
+
compact: boolean;
|
|
435
|
+
}
|
|
436
|
+
|
|
412
437
|
export type WorldStateRequestCategories = WithForkId | WithWorldStateRevision | WithCanonicalForkId;
|
|
413
438
|
|
|
414
439
|
export function isWithForkId(body: WorldStateRequestCategories): body is WithForkId {
|
|
@@ -435,6 +460,7 @@ export type WorldStateRequest = {
|
|
|
435
460
|
|
|
436
461
|
[WorldStateMessageType.FIND_LEAF_INDICES]: FindLeafIndicesRequest;
|
|
437
462
|
[WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafRequest;
|
|
463
|
+
[WorldStateMessageType.FIND_SIBLING_PATHS]: FindSiblingPathsRequest;
|
|
438
464
|
|
|
439
465
|
[WorldStateMessageType.APPEND_LEAVES]: AppendLeavesRequest;
|
|
440
466
|
[WorldStateMessageType.BATCH_INSERT]: BatchInsertRequest;
|
|
@@ -452,13 +478,17 @@ export type WorldStateRequest = {
|
|
|
452
478
|
|
|
453
479
|
[WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS]: BlockShiftRequest;
|
|
454
480
|
[WorldStateMessageType.UNWIND_BLOCKS]: BlockShiftRequest;
|
|
455
|
-
[WorldStateMessageType.
|
|
481
|
+
[WorldStateMessageType.FINALIZE_BLOCKS]: BlockShiftRequest;
|
|
456
482
|
|
|
457
483
|
[WorldStateMessageType.GET_STATUS]: WithCanonicalForkId;
|
|
458
484
|
|
|
459
485
|
[WorldStateMessageType.CREATE_CHECKPOINT]: WithForkId;
|
|
460
486
|
[WorldStateMessageType.COMMIT_CHECKPOINT]: WithForkId;
|
|
461
487
|
[WorldStateMessageType.REVERT_CHECKPOINT]: WithForkId;
|
|
488
|
+
[WorldStateMessageType.COMMIT_ALL_CHECKPOINTS]: WithForkId;
|
|
489
|
+
[WorldStateMessageType.REVERT_ALL_CHECKPOINTS]: WithForkId;
|
|
490
|
+
|
|
491
|
+
[WorldStateMessageType.COPY_STORES]: CopyStoresRequest;
|
|
462
492
|
|
|
463
493
|
[WorldStateMessageType.CLOSE]: WithCanonicalForkId;
|
|
464
494
|
};
|
|
@@ -475,6 +505,7 @@ export type WorldStateResponse = {
|
|
|
475
505
|
|
|
476
506
|
[WorldStateMessageType.FIND_LEAF_INDICES]: FindLeafIndicesResponse;
|
|
477
507
|
[WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafResponse;
|
|
508
|
+
[WorldStateMessageType.FIND_SIBLING_PATHS]: FindSiblingPathsResponse;
|
|
478
509
|
|
|
479
510
|
[WorldStateMessageType.APPEND_LEAVES]: void;
|
|
480
511
|
[WorldStateMessageType.BATCH_INSERT]: BatchInsertResponse;
|
|
@@ -492,33 +523,20 @@ export type WorldStateResponse = {
|
|
|
492
523
|
|
|
493
524
|
[WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS]: WorldStateStatusFull;
|
|
494
525
|
[WorldStateMessageType.UNWIND_BLOCKS]: WorldStateStatusFull;
|
|
495
|
-
[WorldStateMessageType.
|
|
526
|
+
[WorldStateMessageType.FINALIZE_BLOCKS]: WorldStateStatusSummary;
|
|
496
527
|
|
|
497
528
|
[WorldStateMessageType.GET_STATUS]: WorldStateStatusSummary;
|
|
498
529
|
|
|
499
530
|
[WorldStateMessageType.CREATE_CHECKPOINT]: void;
|
|
500
531
|
[WorldStateMessageType.COMMIT_CHECKPOINT]: void;
|
|
501
532
|
[WorldStateMessageType.REVERT_CHECKPOINT]: void;
|
|
533
|
+
[WorldStateMessageType.COMMIT_ALL_CHECKPOINTS]: void;
|
|
534
|
+
[WorldStateMessageType.REVERT_ALL_CHECKPOINTS]: void;
|
|
502
535
|
|
|
503
|
-
[WorldStateMessageType.
|
|
504
|
-
};
|
|
536
|
+
[WorldStateMessageType.COPY_STORES]: void;
|
|
505
537
|
|
|
506
|
-
|
|
507
|
-
forkId: number;
|
|
508
|
-
blockNumber: number;
|
|
509
|
-
includeUncommitted: boolean;
|
|
538
|
+
[WorldStateMessageType.CLOSE]: void;
|
|
510
539
|
};
|
|
511
|
-
export function worldStateRevision(
|
|
512
|
-
includeUncommitted: boolean,
|
|
513
|
-
forkId: number | undefined,
|
|
514
|
-
blockNumber: number | undefined,
|
|
515
|
-
): WorldStateRevision {
|
|
516
|
-
return {
|
|
517
|
-
forkId: forkId ?? 0,
|
|
518
|
-
blockNumber: blockNumber ?? 0,
|
|
519
|
-
includeUncommitted,
|
|
520
|
-
};
|
|
521
|
-
}
|
|
522
540
|
|
|
523
541
|
type TreeStateReference = readonly [Buffer, number | bigint];
|
|
524
542
|
type BlockStateReference = Map<Exclude<MerkleTreeId, MerkleTreeId.ARCHIVE>, TreeStateReference>;
|
|
@@ -1,17 +1,20 @@
|
|
|
1
1
|
import { MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIERS_PER_TX, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
|
|
2
|
-
import { padArrayEnd } from '@aztec/foundation/collection';
|
|
2
|
+
import { fromEntries, padArrayEnd } from '@aztec/foundation/collection';
|
|
3
3
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
4
4
|
import { Fr } from '@aztec/foundation/fields';
|
|
5
|
-
import {
|
|
6
|
-
import type
|
|
5
|
+
import { tryRmDir } from '@aztec/foundation/fs';
|
|
6
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
7
|
+
import type { L2Block, L2BlockNew } from '@aztec/stdlib/block';
|
|
7
8
|
import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
|
|
8
9
|
import type {
|
|
9
10
|
IndexedTreeId,
|
|
10
11
|
MerkleTreeReadOperations,
|
|
11
12
|
MerkleTreeWriteOperations,
|
|
12
13
|
} from '@aztec/stdlib/interfaces/server';
|
|
14
|
+
import type { SnapshotDataKeys } from '@aztec/stdlib/snapshots';
|
|
13
15
|
import { MerkleTreeId, NullifierLeaf, type NullifierLeafPreimage, PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
14
16
|
import { BlockHeader, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
|
|
17
|
+
import { WorldStateRevision } from '@aztec/stdlib/world-state';
|
|
15
18
|
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
16
19
|
|
|
17
20
|
import assert from 'assert/strict';
|
|
@@ -20,6 +23,7 @@ import { tmpdir } from 'os';
|
|
|
20
23
|
import { join } from 'path';
|
|
21
24
|
|
|
22
25
|
import { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
|
|
26
|
+
import type { WorldStateTreeMapSizes } from '../synchronizer/factory.js';
|
|
23
27
|
import type { MerkleTreeAdminDatabase as MerkleTreeDatabase } from '../world-state-db/merkle_tree_db.js';
|
|
24
28
|
import { MerkleTreesFacade, MerkleTreesForkFacade, serializeLeaf } from './merkle_trees_facade.js';
|
|
25
29
|
import {
|
|
@@ -27,16 +31,17 @@ import {
|
|
|
27
31
|
type WorldStateStatusFull,
|
|
28
32
|
type WorldStateStatusSummary,
|
|
29
33
|
blockStateReference,
|
|
30
|
-
|
|
31
|
-
|
|
34
|
+
sanitizeFullStatus,
|
|
35
|
+
sanitizeSummary,
|
|
32
36
|
treeStateReferenceToSnapshot,
|
|
33
|
-
worldStateRevision,
|
|
34
37
|
} from './message.js';
|
|
35
38
|
import { NativeWorldState } from './native_world_state_instance.js';
|
|
36
39
|
|
|
37
40
|
// The current version of the world state database schema
|
|
38
41
|
// Increment this when making incompatible changes to the database schema
|
|
39
|
-
export const WORLD_STATE_DB_VERSION =
|
|
42
|
+
export const WORLD_STATE_DB_VERSION = 2; // The initial version
|
|
43
|
+
|
|
44
|
+
export const WORLD_STATE_DIR = 'world_state';
|
|
40
45
|
|
|
41
46
|
export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
42
47
|
protected initialHeader: BlockHeader | undefined;
|
|
@@ -44,38 +49,38 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
44
49
|
private cachedStatusSummary: WorldStateStatusSummary | undefined;
|
|
45
50
|
|
|
46
51
|
protected constructor(
|
|
47
|
-
protected
|
|
52
|
+
protected instance: NativeWorldState,
|
|
48
53
|
protected readonly worldStateInstrumentation: WorldStateInstrumentation,
|
|
49
|
-
protected readonly log = createLogger('world-state:database'),
|
|
54
|
+
protected readonly log: Logger = createLogger('world-state:database'),
|
|
50
55
|
private readonly cleanup = () => Promise.resolve(),
|
|
51
56
|
) {}
|
|
52
57
|
|
|
53
58
|
static async new(
|
|
54
59
|
rollupAddress: EthAddress,
|
|
55
60
|
dataDir: string,
|
|
56
|
-
|
|
61
|
+
wsTreeMapSizes: WorldStateTreeMapSizes,
|
|
57
62
|
prefilledPublicData: PublicDataTreeLeaf[] = [],
|
|
58
63
|
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
59
64
|
log = createLogger('world-state:database'),
|
|
60
65
|
cleanup = () => Promise.resolve(),
|
|
61
66
|
): Promise<NativeWorldStateService> {
|
|
62
|
-
const worldStateDirectory = join(dataDir,
|
|
67
|
+
const worldStateDirectory = join(dataDir, WORLD_STATE_DIR);
|
|
63
68
|
// Create a version manager to handle versioning
|
|
64
|
-
const versionManager = new DatabaseVersionManager(
|
|
65
|
-
WORLD_STATE_DB_VERSION,
|
|
69
|
+
const versionManager = new DatabaseVersionManager({
|
|
70
|
+
schemaVersion: WORLD_STATE_DB_VERSION,
|
|
66
71
|
rollupAddress,
|
|
67
|
-
worldStateDirectory,
|
|
68
|
-
(dir: string) => {
|
|
69
|
-
return Promise.resolve(new NativeWorldState(dir,
|
|
72
|
+
dataDirectory: worldStateDirectory,
|
|
73
|
+
onOpen: (dir: string) => {
|
|
74
|
+
return Promise.resolve(new NativeWorldState(dir, wsTreeMapSizes, prefilledPublicData, instrumentation));
|
|
70
75
|
},
|
|
71
|
-
);
|
|
76
|
+
});
|
|
72
77
|
|
|
73
78
|
const [instance] = await versionManager.open();
|
|
74
79
|
const worldState = new this(instance, instrumentation, log, cleanup);
|
|
75
80
|
try {
|
|
76
81
|
await worldState.init();
|
|
77
82
|
} catch (e) {
|
|
78
|
-
log.error(`Error
|
|
83
|
+
log.error(`Error initializing world state: ${e}`);
|
|
79
84
|
throw e;
|
|
80
85
|
}
|
|
81
86
|
|
|
@@ -91,7 +96,14 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
91
96
|
const log = createLogger('world-state:database');
|
|
92
97
|
const dataDir = await mkdtemp(join(tmpdir(), 'aztec-world-state-'));
|
|
93
98
|
const dbMapSizeKb = 10 * 1024 * 1024;
|
|
94
|
-
|
|
99
|
+
const worldStateTreeMapSizes: WorldStateTreeMapSizes = {
|
|
100
|
+
archiveTreeMapSizeKb: dbMapSizeKb,
|
|
101
|
+
nullifierTreeMapSizeKb: dbMapSizeKb,
|
|
102
|
+
noteHashTreeMapSizeKb: dbMapSizeKb,
|
|
103
|
+
messageTreeMapSizeKb: dbMapSizeKb,
|
|
104
|
+
publicDataTreeMapSizeKb: dbMapSizeKb,
|
|
105
|
+
};
|
|
106
|
+
log.debug(`Created temporary world state database at: ${dataDir} with tree map size: ${dbMapSizeKb}`);
|
|
95
107
|
|
|
96
108
|
// pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
|
|
97
109
|
const cleanup = async () => {
|
|
@@ -103,7 +115,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
103
115
|
}
|
|
104
116
|
};
|
|
105
117
|
|
|
106
|
-
return this.new(rollupAddress, dataDir,
|
|
118
|
+
return this.new(rollupAddress, dataDir, worldStateTreeMapSizes, prefilledPublicData, instrumentation, log, cleanup);
|
|
107
119
|
}
|
|
108
120
|
|
|
109
121
|
protected async init() {
|
|
@@ -127,12 +139,23 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
127
139
|
assert.strictEqual(initialHeaderIndex, 0n, 'Invalid initial archive state');
|
|
128
140
|
}
|
|
129
141
|
|
|
142
|
+
public async clear() {
|
|
143
|
+
await this.instance.close();
|
|
144
|
+
this.cachedStatusSummary = undefined;
|
|
145
|
+
await tryRmDir(this.instance.getDataDir(), this.log);
|
|
146
|
+
this.instance = this.instance.clone();
|
|
147
|
+
}
|
|
148
|
+
|
|
130
149
|
public getCommitted(): MerkleTreeReadOperations {
|
|
131
|
-
return new MerkleTreesFacade(this.instance, this.initialHeader!,
|
|
150
|
+
return new MerkleTreesFacade(this.instance, this.initialHeader!, WorldStateRevision.empty());
|
|
132
151
|
}
|
|
133
152
|
|
|
134
153
|
public getSnapshot(blockNumber: number): MerkleTreeReadOperations {
|
|
135
|
-
return new MerkleTreesFacade(
|
|
154
|
+
return new MerkleTreesFacade(
|
|
155
|
+
this.instance,
|
|
156
|
+
this.initialHeader!,
|
|
157
|
+
new WorldStateRevision(/*forkId=*/ 0, /* blockNumber=*/ blockNumber, /* includeUncommitted=*/ false),
|
|
158
|
+
);
|
|
136
159
|
}
|
|
137
160
|
|
|
138
161
|
public async fork(blockNumber?: number): Promise<MerkleTreeWriteOperations> {
|
|
@@ -141,19 +164,31 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
141
164
|
blockNumber: blockNumber ?? 0,
|
|
142
165
|
canonical: true,
|
|
143
166
|
});
|
|
144
|
-
return new MerkleTreesForkFacade(
|
|
167
|
+
return new MerkleTreesForkFacade(
|
|
168
|
+
this.instance,
|
|
169
|
+
this.initialHeader!,
|
|
170
|
+
new WorldStateRevision(/*forkId=*/ resp.forkId, /* blockNumber=*/ 0, /* includeUncommitted=*/ true),
|
|
171
|
+
);
|
|
145
172
|
}
|
|
146
173
|
|
|
147
174
|
public getInitialHeader(): BlockHeader {
|
|
148
175
|
return this.initialHeader!;
|
|
149
176
|
}
|
|
150
177
|
|
|
151
|
-
public async handleL2BlockAndMessages(
|
|
178
|
+
public async handleL2BlockAndMessages(
|
|
179
|
+
l2Block: L2Block | L2BlockNew,
|
|
180
|
+
l1ToL2Messages: Fr[],
|
|
181
|
+
// TODO(#17027)
|
|
182
|
+
// Temporary hack to only insert l1 to l2 messages for the first block in a checkpoint.
|
|
183
|
+
isFirstBlock = true,
|
|
184
|
+
): Promise<WorldStateStatusFull> {
|
|
152
185
|
// We have to pad both the values within tx effects because that's how the trees are built by circuits.
|
|
153
186
|
const paddedNoteHashes = l2Block.body.txEffects.flatMap(txEffect =>
|
|
154
187
|
padArrayEnd(txEffect.noteHashes, Fr.ZERO, MAX_NOTE_HASHES_PER_TX),
|
|
155
188
|
);
|
|
156
|
-
const paddedL1ToL2Messages =
|
|
189
|
+
const paddedL1ToL2Messages = isFirstBlock
|
|
190
|
+
? padArrayEnd<Fr, number>(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP)
|
|
191
|
+
: [];
|
|
157
192
|
|
|
158
193
|
const paddedNullifiers = l2Block.body.txEffects
|
|
159
194
|
.flatMap(txEffect => padArrayEnd(txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX))
|
|
@@ -173,7 +208,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
173
208
|
WorldStateMessageType.SYNC_BLOCK,
|
|
174
209
|
{
|
|
175
210
|
blockNumber: l2Block.number,
|
|
176
|
-
blockHeaderHash: await l2Block.
|
|
211
|
+
blockHeaderHash: await l2Block.hash(),
|
|
177
212
|
paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf),
|
|
178
213
|
paddedNoteHashes: paddedNoteHashes.map(serializeLeaf),
|
|
179
214
|
paddedNullifiers: paddedNullifiers.map(serializeLeaf),
|
|
@@ -181,7 +216,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
181
216
|
blockStateRef: blockStateReference(l2Block.header.state),
|
|
182
217
|
canonical: true,
|
|
183
218
|
},
|
|
184
|
-
this.
|
|
219
|
+
this.sanitizeAndCacheSummaryFromFull.bind(this),
|
|
185
220
|
this.deleteCachedSummary.bind(this),
|
|
186
221
|
);
|
|
187
222
|
} catch (err) {
|
|
@@ -200,16 +235,16 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
200
235
|
return BlockHeader.empty({ state });
|
|
201
236
|
}
|
|
202
237
|
|
|
203
|
-
private
|
|
204
|
-
const
|
|
205
|
-
this.cachedStatusSummary = { ...
|
|
206
|
-
return
|
|
238
|
+
private sanitizeAndCacheSummaryFromFull(response: WorldStateStatusFull) {
|
|
239
|
+
const sanitized = sanitizeFullStatus(response);
|
|
240
|
+
this.cachedStatusSummary = { ...sanitized.summary };
|
|
241
|
+
return sanitized;
|
|
207
242
|
}
|
|
208
243
|
|
|
209
|
-
private
|
|
210
|
-
const
|
|
211
|
-
this.cachedStatusSummary = { ...
|
|
212
|
-
return
|
|
244
|
+
private sanitizeAndCacheSummary(response: WorldStateStatusSummary) {
|
|
245
|
+
const sanitized = sanitizeSummary(response);
|
|
246
|
+
this.cachedStatusSummary = { ...sanitized };
|
|
247
|
+
return sanitized;
|
|
213
248
|
}
|
|
214
249
|
|
|
215
250
|
private deleteCachedSummary(_: string) {
|
|
@@ -217,19 +252,19 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
217
252
|
}
|
|
218
253
|
|
|
219
254
|
/**
|
|
220
|
-
* Advances the
|
|
221
|
-
* @param toBlockNumber The block number that is now the tip of the
|
|
255
|
+
* Advances the finalized block number to be the number provided
|
|
256
|
+
* @param toBlockNumber The block number that is now the tip of the finalized chain
|
|
222
257
|
* @returns The new WorldStateStatus
|
|
223
258
|
*/
|
|
224
|
-
public async
|
|
259
|
+
public async setFinalized(toBlockNumber: bigint) {
|
|
225
260
|
try {
|
|
226
261
|
await this.instance.call(
|
|
227
|
-
WorldStateMessageType.
|
|
262
|
+
WorldStateMessageType.FINALIZE_BLOCKS,
|
|
228
263
|
{
|
|
229
264
|
toBlockNumber,
|
|
230
265
|
canonical: true,
|
|
231
266
|
},
|
|
232
|
-
this.
|
|
267
|
+
this.sanitizeAndCacheSummary.bind(this),
|
|
233
268
|
this.deleteCachedSummary.bind(this),
|
|
234
269
|
);
|
|
235
270
|
} catch (err) {
|
|
@@ -252,7 +287,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
252
287
|
toBlockNumber,
|
|
253
288
|
canonical: true,
|
|
254
289
|
},
|
|
255
|
-
this.
|
|
290
|
+
this.sanitizeAndCacheSummaryFromFull.bind(this),
|
|
256
291
|
this.deleteCachedSummary.bind(this),
|
|
257
292
|
);
|
|
258
293
|
} catch (err) {
|
|
@@ -274,7 +309,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
274
309
|
toBlockNumber,
|
|
275
310
|
canonical: true,
|
|
276
311
|
},
|
|
277
|
-
this.
|
|
312
|
+
this.sanitizeAndCacheSummaryFromFull.bind(this),
|
|
278
313
|
this.deleteCachedSummary.bind(this),
|
|
279
314
|
);
|
|
280
315
|
} catch (err) {
|
|
@@ -290,7 +325,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
290
325
|
return await this.instance.call(
|
|
291
326
|
WorldStateMessageType.GET_STATUS,
|
|
292
327
|
{ canonical: true },
|
|
293
|
-
this.
|
|
328
|
+
this.sanitizeAndCacheSummary.bind(this),
|
|
294
329
|
);
|
|
295
330
|
}
|
|
296
331
|
|
|
@@ -314,4 +349,25 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
314
349
|
),
|
|
315
350
|
);
|
|
316
351
|
}
|
|
352
|
+
|
|
353
|
+
public async backupTo(
|
|
354
|
+
dstPath: string,
|
|
355
|
+
compact: boolean = true,
|
|
356
|
+
): Promise<Record<Exclude<SnapshotDataKeys, 'archiver'>, string>> {
|
|
357
|
+
await this.instance.call(WorldStateMessageType.COPY_STORES, {
|
|
358
|
+
dstPath,
|
|
359
|
+
compact,
|
|
360
|
+
canonical: true,
|
|
361
|
+
});
|
|
362
|
+
return fromEntries(NATIVE_WORLD_STATE_DBS.map(([name, dir]) => [name, join(dstPath, dir, 'data.mdb')] as const));
|
|
363
|
+
}
|
|
317
364
|
}
|
|
365
|
+
|
|
366
|
+
// The following paths are defined in cpp-land
|
|
367
|
+
export const NATIVE_WORLD_STATE_DBS = [
|
|
368
|
+
['l1-to-l2-message-tree', 'L1ToL2MessageTree'],
|
|
369
|
+
['archive-tree', 'ArchiveTree'],
|
|
370
|
+
['public-data-tree', 'PublicDataTree'],
|
|
371
|
+
['note-hash-tree', 'NoteHashTree'],
|
|
372
|
+
['nullifier-tree', 'NullifierTree'],
|
|
373
|
+
] as const;
|