@aztec/world-state 0.0.1-commit.2ed92850 → 0.0.1-commit.3100065
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/instrumentation/instrumentation.d.ts +1 -1
- package/dest/instrumentation/instrumentation.d.ts.map +1 -1
- package/dest/instrumentation/instrumentation.js +9 -2
- package/dest/native/fork_checkpoint.d.ts +7 -1
- package/dest/native/fork_checkpoint.d.ts.map +1 -1
- package/dest/native/fork_checkpoint.js +15 -3
- package/dest/native/ipc_world_state_instance.d.ts +50 -0
- package/dest/native/ipc_world_state_instance.d.ts.map +1 -0
- package/dest/native/ipc_world_state_instance.js +635 -0
- package/dest/native/merkle_trees_facade.d.ts +9 -6
- package/dest/native/merkle_trees_facade.d.ts.map +1 -1
- package/dest/native/merkle_trees_facade.js +33 -13
- package/dest/native/message.d.ts +17 -5
- package/dest/native/message.d.ts.map +1 -1
- package/dest/native/native_world_state.d.ts +32 -6
- package/dest/native/native_world_state.d.ts.map +1 -1
- package/dest/native/native_world_state.js +85 -28
- package/dest/native/native_world_state_instance.d.ts +6 -5
- package/dest/native/native_world_state_instance.d.ts.map +1 -1
- package/dest/native/native_world_state_instance.js +46 -26
- package/dest/native/world_state_ops_queue.js +5 -5
- package/dest/synchronizer/config.d.ts +3 -3
- package/dest/synchronizer/config.d.ts.map +1 -1
- package/dest/synchronizer/config.js +15 -13
- package/dest/synchronizer/errors.d.ts +8 -1
- package/dest/synchronizer/errors.d.ts.map +1 -1
- package/dest/synchronizer/errors.js +8 -1
- package/dest/synchronizer/factory.d.ts +6 -5
- package/dest/synchronizer/factory.d.ts.map +1 -1
- package/dest/synchronizer/factory.js +7 -6
- package/dest/synchronizer/index.d.ts +2 -1
- package/dest/synchronizer/index.d.ts.map +1 -1
- package/dest/synchronizer/index.js +1 -0
- package/dest/synchronizer/server_world_state_synchronizer.d.ts +13 -8
- package/dest/synchronizer/server_world_state_synchronizer.d.ts.map +1 -1
- package/dest/synchronizer/server_world_state_synchronizer.js +133 -47
- package/dest/test/utils.d.ts +1 -1
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +6 -1
- package/dest/testing.d.ts +4 -3
- package/dest/testing.d.ts.map +1 -1
- package/dest/testing.js +22 -7
- package/dest/world-state-db/merkle_tree_db.d.ts +1 -10
- package/dest/world-state-db/merkle_tree_db.d.ts.map +1 -1
- package/package.json +12 -11
- package/src/instrumentation/instrumentation.ts +9 -1
- package/src/native/fork_checkpoint.ts +19 -3
- package/src/native/ipc_world_state_instance.ts +725 -0
- package/src/native/merkle_trees_facade.ts +41 -16
- package/src/native/message.ts +18 -4
- package/src/native/native_world_state.ts +110 -33
- package/src/native/native_world_state_instance.ts +61 -32
- package/src/native/world_state_ops_queue.ts +5 -5
- package/src/synchronizer/config.ts +21 -15
- package/src/synchronizer/errors.ts +8 -0
- package/src/synchronizer/factory.ts +19 -11
- package/src/synchronizer/index.ts +1 -0
- package/src/synchronizer/server_world_state_synchronizer.ts +152 -42
- package/src/test/utils.ts +10 -1
- package/src/testing.ts +19 -10
- package/src/world-state-db/merkle_tree_db.ts +0 -10
|
@@ -4,6 +4,7 @@ import { createLogger } from '@aztec/foundation/log';
|
|
|
4
4
|
import { serializeToBuffer } from '@aztec/foundation/serialize';
|
|
5
5
|
import { sleep } from '@aztec/foundation/sleep';
|
|
6
6
|
import { type IndexedTreeLeafPreimage, SiblingPath } from '@aztec/foundation/trees';
|
|
7
|
+
import { BlockHash } from '@aztec/stdlib/block';
|
|
7
8
|
import type {
|
|
8
9
|
BatchInsertionResult,
|
|
9
10
|
IndexedTreeId,
|
|
@@ -118,7 +119,7 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations {
|
|
|
118
119
|
|
|
119
120
|
const leaf = deserializeLeafValue(resp);
|
|
120
121
|
if (leaf instanceof Fr) {
|
|
121
|
-
return leaf as any;
|
|
122
|
+
return treeId === MerkleTreeId.ARCHIVE ? (new BlockHash(leaf) as any) : (leaf as any);
|
|
122
123
|
} else {
|
|
123
124
|
return leaf.toBuffer() as any;
|
|
124
125
|
}
|
|
@@ -207,6 +208,7 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations {
|
|
|
207
208
|
|
|
208
209
|
export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTreeWriteOperations {
|
|
209
210
|
private log = createLogger('world-state:merkle-trees-fork-facade');
|
|
211
|
+
private closePromise: Promise<void> | undefined;
|
|
210
212
|
|
|
211
213
|
constructor(
|
|
212
214
|
instance: NativeWorldStateInstance,
|
|
@@ -228,7 +230,7 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
|
|
|
228
230
|
|
|
229
231
|
async appendLeaves<ID extends MerkleTreeId>(treeId: ID, leaves: MerkleTreeLeafType<ID>[]): Promise<void> {
|
|
230
232
|
await this.instance.call(WorldStateMessageType.APPEND_LEAVES, {
|
|
231
|
-
leaves: leaves.map(leaf => leaf as any),
|
|
233
|
+
leaves: leaves.map(leaf => serializeLeaf(hydrateLeaf(treeId, leaf as any))),
|
|
232
234
|
forkId: this.revision.forkId,
|
|
233
235
|
treeId,
|
|
234
236
|
});
|
|
@@ -290,8 +292,17 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
|
|
|
290
292
|
};
|
|
291
293
|
}
|
|
292
294
|
|
|
293
|
-
public
|
|
295
|
+
public close(): Promise<void> {
|
|
294
296
|
assert.notEqual(this.revision.forkId, 0, 'Fork ID must be set');
|
|
297
|
+
// Share the in-flight close promise across duplicate dispose calls so DELETE_FORK is sent at most once.
|
|
298
|
+
if (this.closePromise) {
|
|
299
|
+
return this.closePromise;
|
|
300
|
+
}
|
|
301
|
+
this.closePromise = this.doClose();
|
|
302
|
+
return this.closePromise;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
private async doClose(): Promise<void> {
|
|
295
306
|
try {
|
|
296
307
|
await this.instance.call(WorldStateMessageType.DELETE_FORK, { forkId: this.revision.forkId });
|
|
297
308
|
} catch (err: any) {
|
|
@@ -300,18 +311,21 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
|
|
|
300
311
|
if (err?.message === 'Native instance is closed') {
|
|
301
312
|
return;
|
|
302
313
|
}
|
|
314
|
+
// Ignore "Fork not found": the native fork was already destroyed by a pending-chain unwind or a
|
|
315
|
+
// historical prune (both call C++ remove_forks_for_block). Fork IDs are monotonic and never reused,
|
|
316
|
+
// so swallowing this on close cannot mask a deletion of a different fork.
|
|
317
|
+
if (err?.message === 'Fork not found') {
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
303
320
|
throw err;
|
|
304
321
|
}
|
|
305
322
|
}
|
|
306
323
|
|
|
307
|
-
async [Symbol.
|
|
324
|
+
async [Symbol.asyncDispose](): Promise<void> {
|
|
308
325
|
if (this.opts.closeDelayMs) {
|
|
309
326
|
void sleep(this.opts.closeDelayMs)
|
|
310
327
|
.then(() => this.close())
|
|
311
328
|
.catch(err => {
|
|
312
|
-
if (err && 'message' in err && err.message === 'Native instance is closed') {
|
|
313
|
-
return; // Ignore errors due to native instance being closed
|
|
314
|
-
}
|
|
315
329
|
this.log.warn('Error closing MerkleTreesForkFacade after delay', { err });
|
|
316
330
|
});
|
|
317
331
|
} else {
|
|
@@ -319,9 +333,10 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
|
|
|
319
333
|
}
|
|
320
334
|
}
|
|
321
335
|
|
|
322
|
-
public async createCheckpoint(): Promise<
|
|
336
|
+
public async createCheckpoint(): Promise<number> {
|
|
323
337
|
assert.notEqual(this.revision.forkId, 0, 'Fork ID must be set');
|
|
324
|
-
await this.instance.call(WorldStateMessageType.CREATE_CHECKPOINT, { forkId: this.revision.forkId });
|
|
338
|
+
const resp = await this.instance.call(WorldStateMessageType.CREATE_CHECKPOINT, { forkId: this.revision.forkId });
|
|
339
|
+
return resp.depth;
|
|
325
340
|
}
|
|
326
341
|
|
|
327
342
|
public async commitCheckpoint(): Promise<void> {
|
|
@@ -334,20 +349,28 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
|
|
|
334
349
|
await this.instance.call(WorldStateMessageType.REVERT_CHECKPOINT, { forkId: this.revision.forkId });
|
|
335
350
|
}
|
|
336
351
|
|
|
337
|
-
public async
|
|
352
|
+
public async commitAllCheckpointsTo(depth: number): Promise<void> {
|
|
338
353
|
assert.notEqual(this.revision.forkId, 0, 'Fork ID must be set');
|
|
339
|
-
await this.instance.call(WorldStateMessageType.COMMIT_ALL_CHECKPOINTS, {
|
|
354
|
+
await this.instance.call(WorldStateMessageType.COMMIT_ALL_CHECKPOINTS, {
|
|
355
|
+
forkId: this.revision.forkId,
|
|
356
|
+
depth,
|
|
357
|
+
});
|
|
340
358
|
}
|
|
341
359
|
|
|
342
|
-
public async
|
|
360
|
+
public async revertAllCheckpointsTo(depth: number): Promise<void> {
|
|
343
361
|
assert.notEqual(this.revision.forkId, 0, 'Fork ID must be set');
|
|
344
|
-
await this.instance.call(WorldStateMessageType.REVERT_ALL_CHECKPOINTS, {
|
|
362
|
+
await this.instance.call(WorldStateMessageType.REVERT_ALL_CHECKPOINTS, {
|
|
363
|
+
forkId: this.revision.forkId,
|
|
364
|
+
depth,
|
|
365
|
+
});
|
|
345
366
|
}
|
|
346
367
|
}
|
|
347
368
|
|
|
348
|
-
function hydrateLeaf
|
|
369
|
+
function hydrateLeaf(treeId: MerkleTreeId, leaf: Fr | BlockHash | Buffer) {
|
|
349
370
|
if (leaf instanceof Fr) {
|
|
350
371
|
return leaf;
|
|
372
|
+
} else if (leaf instanceof BlockHash) {
|
|
373
|
+
return leaf.toFr();
|
|
351
374
|
} else if (treeId === MerkleTreeId.NULLIFIER_TREE) {
|
|
352
375
|
return NullifierLeaf.fromBuffer(leaf);
|
|
353
376
|
} else if (treeId === MerkleTreeId.PUBLIC_DATA_TREE) {
|
|
@@ -357,8 +380,10 @@ function hydrateLeaf<ID extends MerkleTreeId>(treeId: ID, leaf: Fr | Buffer) {
|
|
|
357
380
|
}
|
|
358
381
|
}
|
|
359
382
|
|
|
360
|
-
export function serializeLeaf(leaf: Fr | NullifierLeaf | PublicDataTreeLeaf): SerializedLeafValue {
|
|
361
|
-
if (leaf instanceof
|
|
383
|
+
export function serializeLeaf(leaf: Fr | BlockHash | NullifierLeaf | PublicDataTreeLeaf): SerializedLeafValue {
|
|
384
|
+
if (leaf instanceof BlockHash) {
|
|
385
|
+
return leaf.toBuffer();
|
|
386
|
+
} else if (leaf instanceof Fr) {
|
|
362
387
|
return leaf.toBuffer();
|
|
363
388
|
} else if (leaf instanceof NullifierLeaf) {
|
|
364
389
|
return { nullifier: leaf.nullifier.toBuffer() };
|
package/src/native/message.ts
CHANGED
|
@@ -283,6 +283,16 @@ interface WithForkId {
|
|
|
283
283
|
forkId: number;
|
|
284
284
|
}
|
|
285
285
|
|
|
286
|
+
interface CreateCheckpointResponse {
|
|
287
|
+
depth: number;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/** Request to commit/revert all checkpoints down to a target depth. The resulting depth after the operation equals the given depth. */
|
|
291
|
+
interface CheckpointDepthRequest extends WithForkId {
|
|
292
|
+
/** The target depth after the operation. All checkpoints above this depth are committed/reverted. */
|
|
293
|
+
depth: number;
|
|
294
|
+
}
|
|
295
|
+
|
|
286
296
|
interface WithWorldStateRevision {
|
|
287
297
|
revision: WorldStateRevision;
|
|
288
298
|
}
|
|
@@ -412,7 +422,11 @@ interface UpdateArchiveRequest extends WithForkId {
|
|
|
412
422
|
interface SyncBlockRequest extends WithCanonicalForkId {
|
|
413
423
|
blockNumber: BlockNumber;
|
|
414
424
|
blockStateRef: BlockStateReference;
|
|
415
|
-
blockHeaderHash:
|
|
425
|
+
blockHeaderHash: Buffer;
|
|
426
|
+
/** Canonical archive root after this block; world state rejects the sync if its computed root differs. */
|
|
427
|
+
expectedArchiveRoot: Buffer;
|
|
428
|
+
/** Canonical archive root before this block (the block's lastArchive); verified against the local root. */
|
|
429
|
+
expectedPreviousArchiveRoot: Buffer;
|
|
416
430
|
paddedNoteHashes: readonly SerializedLeafValue[];
|
|
417
431
|
paddedL1ToL2Messages: readonly SerializedLeafValue[];
|
|
418
432
|
paddedNullifiers: readonly SerializedLeafValue[];
|
|
@@ -486,8 +500,8 @@ export type WorldStateRequest = {
|
|
|
486
500
|
[WorldStateMessageType.CREATE_CHECKPOINT]: WithForkId;
|
|
487
501
|
[WorldStateMessageType.COMMIT_CHECKPOINT]: WithForkId;
|
|
488
502
|
[WorldStateMessageType.REVERT_CHECKPOINT]: WithForkId;
|
|
489
|
-
[WorldStateMessageType.COMMIT_ALL_CHECKPOINTS]:
|
|
490
|
-
[WorldStateMessageType.REVERT_ALL_CHECKPOINTS]:
|
|
503
|
+
[WorldStateMessageType.COMMIT_ALL_CHECKPOINTS]: CheckpointDepthRequest;
|
|
504
|
+
[WorldStateMessageType.REVERT_ALL_CHECKPOINTS]: CheckpointDepthRequest;
|
|
491
505
|
|
|
492
506
|
[WorldStateMessageType.COPY_STORES]: CopyStoresRequest;
|
|
493
507
|
|
|
@@ -528,7 +542,7 @@ export type WorldStateResponse = {
|
|
|
528
542
|
|
|
529
543
|
[WorldStateMessageType.GET_STATUS]: WorldStateStatusSummary;
|
|
530
544
|
|
|
531
|
-
[WorldStateMessageType.CREATE_CHECKPOINT]:
|
|
545
|
+
[WorldStateMessageType.CREATE_CHECKPOINT]: CreateCheckpointResponse;
|
|
532
546
|
[WorldStateMessageType.COMMIT_CHECKPOINT]: void;
|
|
533
547
|
[WorldStateMessageType.REVERT_CHECKPOINT]: void;
|
|
534
548
|
[WorldStateMessageType.COMMIT_ALL_CHECKPOINTS]: void;
|
|
@@ -4,9 +4,9 @@ import { fromEntries, padArrayEnd } from '@aztec/foundation/collection';
|
|
|
4
4
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
5
5
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
6
6
|
import { tryRmDir } from '@aztec/foundation/fs';
|
|
7
|
-
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
7
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
8
8
|
import type { L2Block } from '@aztec/stdlib/block';
|
|
9
|
-
import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
|
|
9
|
+
import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
|
|
10
10
|
import type {
|
|
11
11
|
IndexedTreeId,
|
|
12
12
|
MerkleTreeReadOperations,
|
|
@@ -14,8 +14,8 @@ import type {
|
|
|
14
14
|
} from '@aztec/stdlib/interfaces/server';
|
|
15
15
|
import type { SnapshotDataKeys } from '@aztec/stdlib/snapshots';
|
|
16
16
|
import { MerkleTreeId, NullifierLeaf, type NullifierLeafPreimage, PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
17
|
-
import { BlockHeader, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
|
|
18
|
-
import { WorldStateRevision } from '@aztec/stdlib/world-state';
|
|
17
|
+
import { BlockHeader, GlobalVariables, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
|
|
18
|
+
import { EMPTY_GENESIS_DATA, type GenesisData, WorldStateRevision } from '@aztec/stdlib/world-state';
|
|
19
19
|
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
20
20
|
|
|
21
21
|
import assert from 'assert/strict';
|
|
@@ -44,6 +44,29 @@ export const WORLD_STATE_DB_VERSION = 2; // The initial version
|
|
|
44
44
|
|
|
45
45
|
export const WORLD_STATE_DIR = 'world_state';
|
|
46
46
|
|
|
47
|
+
const DEFAULT_TMP_TREE_MAP_SIZE_KB = 10 * 1024 * 1024;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Sets up a fresh `mkdtemp` directory + default `WorldStateTreeMapSizes` shared by both
|
|
51
|
+
* the `.tmp` (fsync-on) and `.ephemeral` (fsync-off) factories. Returns the raw tmpdir,
|
|
52
|
+
* the tree map sizes, and the package logger.
|
|
53
|
+
*/
|
|
54
|
+
async function createTmpWorldStateDir(
|
|
55
|
+
bindings?: LoggerBindings,
|
|
56
|
+
): Promise<{ dataDir: string; wsTreeMapSizes: WorldStateTreeMapSizes; log: Logger }> {
|
|
57
|
+
const log = createLogger('world-state:database', bindings);
|
|
58
|
+
const dataDir = await mkdtemp(join(tmpdir(), 'aztec-world-state-'));
|
|
59
|
+
const wsTreeMapSizes: WorldStateTreeMapSizes = {
|
|
60
|
+
archiveTreeMapSizeKb: DEFAULT_TMP_TREE_MAP_SIZE_KB,
|
|
61
|
+
nullifierTreeMapSizeKb: DEFAULT_TMP_TREE_MAP_SIZE_KB,
|
|
62
|
+
noteHashTreeMapSizeKb: DEFAULT_TMP_TREE_MAP_SIZE_KB,
|
|
63
|
+
messageTreeMapSizeKb: DEFAULT_TMP_TREE_MAP_SIZE_KB,
|
|
64
|
+
publicDataTreeMapSizeKb: DEFAULT_TMP_TREE_MAP_SIZE_KB,
|
|
65
|
+
};
|
|
66
|
+
log.debug(`Created temporary world state database at: ${dataDir} (map size ${DEFAULT_TMP_TREE_MAP_SIZE_KB} KB)`);
|
|
67
|
+
return { dataDir, wsTreeMapSizes, log };
|
|
68
|
+
}
|
|
69
|
+
|
|
47
70
|
export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
48
71
|
protected initialHeader: BlockHeader | undefined;
|
|
49
72
|
// This is read heavily and only changes when data is persisted, so we cache it
|
|
@@ -52,32 +75,47 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
52
75
|
protected constructor(
|
|
53
76
|
protected instance: NativeWorldState,
|
|
54
77
|
protected readonly worldStateInstrumentation: WorldStateInstrumentation,
|
|
55
|
-
protected readonly log: Logger
|
|
78
|
+
protected readonly log: Logger,
|
|
79
|
+
private readonly genesis: GenesisData = EMPTY_GENESIS_DATA,
|
|
56
80
|
private readonly cleanup = () => Promise.resolve(),
|
|
57
81
|
) {}
|
|
58
82
|
|
|
83
|
+
/**
|
|
84
|
+
* Opens a persistent world state at `dataDir`. Goes through `DatabaseVersionManager` so the
|
|
85
|
+
* caller's rollup address is bound to the on-disk schema and incompatible versions surface
|
|
86
|
+
* loudly. The LMDB envs commit with full fsync.
|
|
87
|
+
*/
|
|
59
88
|
static async new(
|
|
60
89
|
rollupAddress: EthAddress,
|
|
61
90
|
dataDir: string,
|
|
62
91
|
wsTreeMapSizes: WorldStateTreeMapSizes,
|
|
63
|
-
|
|
92
|
+
genesis: GenesisData = EMPTY_GENESIS_DATA,
|
|
64
93
|
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
65
|
-
|
|
94
|
+
bindings?: LoggerBindings,
|
|
66
95
|
cleanup = () => Promise.resolve(),
|
|
67
96
|
): Promise<NativeWorldStateService> {
|
|
97
|
+
const log = createLogger('world-state:database', bindings);
|
|
68
98
|
const worldStateDirectory = join(dataDir, WORLD_STATE_DIR);
|
|
69
|
-
// Create a version manager to handle versioning
|
|
70
99
|
const versionManager = new DatabaseVersionManager({
|
|
71
100
|
schemaVersion: WORLD_STATE_DB_VERSION,
|
|
72
101
|
rollupAddress,
|
|
73
102
|
dataDirectory: worldStateDirectory,
|
|
74
|
-
onOpen: (dir: string) =>
|
|
75
|
-
|
|
76
|
-
|
|
103
|
+
onOpen: (dir: string) =>
|
|
104
|
+
Promise.resolve(
|
|
105
|
+
new NativeWorldState(
|
|
106
|
+
dir,
|
|
107
|
+
wsTreeMapSizes,
|
|
108
|
+
genesis,
|
|
109
|
+
instrumentation,
|
|
110
|
+
bindings,
|
|
111
|
+
undefined,
|
|
112
|
+
/*ephemeral=*/ false,
|
|
113
|
+
),
|
|
114
|
+
),
|
|
77
115
|
});
|
|
78
116
|
|
|
79
117
|
const [instance] = await versionManager.open();
|
|
80
|
-
const worldState = new this(instance, instrumentation, log, cleanup);
|
|
118
|
+
const worldState = new this(instance, instrumentation, log, genesis, cleanup);
|
|
81
119
|
try {
|
|
82
120
|
await worldState.init();
|
|
83
121
|
} catch (e) {
|
|
@@ -88,25 +126,23 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
88
126
|
return worldState;
|
|
89
127
|
}
|
|
90
128
|
|
|
129
|
+
/**
|
|
130
|
+
* Opens a world state in a fresh tmpdir with full fsync semantics. Use when you need the
|
|
131
|
+
* on-disk file to remain crash-recoverable (e.g. for snapshot/backup tests) but don't
|
|
132
|
+
* want a persistent dataDir. Pass `cleanupTmpDir=false` to keep the directory after
|
|
133
|
+
* close for inspection.
|
|
134
|
+
*
|
|
135
|
+
* If you don't care about crash-recoverability — i.e. you just want a fast scratch
|
|
136
|
+
* database for tests — use {@link ephemeral} instead.
|
|
137
|
+
*/
|
|
91
138
|
static async tmp(
|
|
92
139
|
rollupAddress = EthAddress.ZERO,
|
|
93
140
|
cleanupTmpDir = true,
|
|
94
|
-
|
|
141
|
+
genesis: GenesisData = EMPTY_GENESIS_DATA,
|
|
95
142
|
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
143
|
+
bindings?: LoggerBindings,
|
|
96
144
|
): Promise<NativeWorldStateService> {
|
|
97
|
-
const log =
|
|
98
|
-
const dataDir = await mkdtemp(join(tmpdir(), 'aztec-world-state-'));
|
|
99
|
-
const dbMapSizeKb = 10 * 1024 * 1024;
|
|
100
|
-
const worldStateTreeMapSizes: WorldStateTreeMapSizes = {
|
|
101
|
-
archiveTreeMapSizeKb: dbMapSizeKb,
|
|
102
|
-
nullifierTreeMapSizeKb: dbMapSizeKb,
|
|
103
|
-
noteHashTreeMapSizeKb: dbMapSizeKb,
|
|
104
|
-
messageTreeMapSizeKb: dbMapSizeKb,
|
|
105
|
-
publicDataTreeMapSizeKb: dbMapSizeKb,
|
|
106
|
-
};
|
|
107
|
-
log.debug(`Created temporary world state database at: ${dataDir} with tree map size: ${dbMapSizeKb}`);
|
|
108
|
-
|
|
109
|
-
// pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
|
|
145
|
+
const { dataDir, wsTreeMapSizes, log } = await createTmpWorldStateDir(bindings);
|
|
110
146
|
const cleanup = async () => {
|
|
111
147
|
if (cleanupTmpDir) {
|
|
112
148
|
await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
|
|
@@ -115,8 +151,45 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
115
151
|
log.debug(`Leaving temporary world state database: ${dataDir}`);
|
|
116
152
|
}
|
|
117
153
|
};
|
|
154
|
+
return this.new(rollupAddress, dataDir, wsTreeMapSizes, genesis, instrumentation, bindings, cleanup);
|
|
155
|
+
}
|
|
118
156
|
|
|
119
|
-
|
|
157
|
+
/**
|
|
158
|
+
* Opens a fully-ephemeral world state. The directory is created in `os.tmpdir()`, the LMDB
|
|
159
|
+
* envs open with `MDB_NOSYNC | MDB_NOMETASYNC` so commits never block on fsync, and the
|
|
160
|
+
* directory is removed on dispose. A crash mid-write leaves the env unrecoverable.
|
|
161
|
+
*
|
|
162
|
+
* For unit tests and other isolated runs. Use {@link tmp} when you need fsync semantics in a
|
|
163
|
+
* tmp dir, and {@link new} for a persistent store. Skips {@link DatabaseVersionManager} —
|
|
164
|
+
* there is no on-disk schema to bind to and no rollup address is taken.
|
|
165
|
+
*/
|
|
166
|
+
static async ephemeral(
|
|
167
|
+
genesis: GenesisData = EMPTY_GENESIS_DATA,
|
|
168
|
+
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
169
|
+
bindings?: LoggerBindings,
|
|
170
|
+
): Promise<NativeWorldStateService> {
|
|
171
|
+
const { dataDir, wsTreeMapSizes, log } = await createTmpWorldStateDir(bindings);
|
|
172
|
+
const cleanup = async () => {
|
|
173
|
+
await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
|
|
174
|
+
log.debug(`Deleted ephemeral world state database: ${dataDir}`);
|
|
175
|
+
};
|
|
176
|
+
const instance = new NativeWorldState(
|
|
177
|
+
join(dataDir, WORLD_STATE_DIR),
|
|
178
|
+
wsTreeMapSizes,
|
|
179
|
+
genesis,
|
|
180
|
+
instrumentation,
|
|
181
|
+
bindings,
|
|
182
|
+
undefined,
|
|
183
|
+
/*ephemeral=*/ true,
|
|
184
|
+
);
|
|
185
|
+
const worldState = new this(instance, instrumentation, log, genesis, cleanup);
|
|
186
|
+
try {
|
|
187
|
+
await worldState.init();
|
|
188
|
+
} catch (e) {
|
|
189
|
+
log.error(`Error initializing ephemeral world state: ${e}`);
|
|
190
|
+
throw e;
|
|
191
|
+
}
|
|
192
|
+
return worldState;
|
|
120
193
|
}
|
|
121
194
|
|
|
122
195
|
protected async init() {
|
|
@@ -135,9 +208,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
135
208
|
|
|
136
209
|
// the initial header _must_ be the first element in the archive tree
|
|
137
210
|
// if this assertion fails, check that the hashing done in Header in yarn-project matches the initial header hash done in world_state.cpp
|
|
138
|
-
const indices = await committed.findLeafIndices(MerkleTreeId.ARCHIVE, [
|
|
139
|
-
(await this.initialHeader.hash()).toField(),
|
|
140
|
-
]);
|
|
211
|
+
const indices = await committed.findLeafIndices(MerkleTreeId.ARCHIVE, [await this.initialHeader.hash()]);
|
|
141
212
|
const initialHeaderIndex = indices[0];
|
|
142
213
|
assert.strictEqual(initialHeaderIndex, 0n, 'Invalid initial archive state');
|
|
143
214
|
}
|
|
@@ -175,7 +246,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
175
246
|
this.initialHeader!,
|
|
176
247
|
new WorldStateRevision(
|
|
177
248
|
/*forkId=*/ resp.forkId,
|
|
178
|
-
/* blockNumber=*/
|
|
249
|
+
/* blockNumber=*/ WorldStateRevision.LATEST,
|
|
179
250
|
/* includeUncommitted=*/ true,
|
|
180
251
|
),
|
|
181
252
|
opts,
|
|
@@ -222,7 +293,10 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
222
293
|
WorldStateMessageType.SYNC_BLOCK,
|
|
223
294
|
{
|
|
224
295
|
blockNumber: l2Block.number,
|
|
225
|
-
blockHeaderHash: await l2Block.hash(),
|
|
296
|
+
blockHeaderHash: (await l2Block.hash()).toBuffer(),
|
|
297
|
+
// Forwarded so the native sync verifies the archive root against canonical and rejects a divergent tree.
|
|
298
|
+
expectedArchiveRoot: l2Block.archive.root.toBuffer(),
|
|
299
|
+
expectedPreviousArchiveRoot: l2Block.header.lastArchive.root.toBuffer(),
|
|
226
300
|
paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf),
|
|
227
301
|
paddedNoteHashes: paddedNoteHashes.map(serializeLeaf),
|
|
228
302
|
paddedNullifiers: paddedNullifiers.map(serializeLeaf),
|
|
@@ -246,7 +320,10 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
246
320
|
|
|
247
321
|
private async buildInitialHeader(): Promise<BlockHeader> {
|
|
248
322
|
const state = await this.getInitialStateReference();
|
|
249
|
-
return BlockHeader.empty({
|
|
323
|
+
return BlockHeader.empty({
|
|
324
|
+
state,
|
|
325
|
+
globalVariables: GlobalVariables.empty({ timestamp: this.genesis.genesisTimestamp }),
|
|
326
|
+
});
|
|
250
327
|
}
|
|
251
328
|
|
|
252
329
|
private sanitizeAndCacheSummaryFromFull(response: WorldStateStatusFull) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
ARCHIVE_HEIGHT,
|
|
3
|
-
|
|
3
|
+
DomainSeparator,
|
|
4
4
|
L1_TO_L2_MSG_TREE_HEIGHT,
|
|
5
5
|
MAX_NULLIFIERS_PER_TX,
|
|
6
6
|
MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
|
|
@@ -8,10 +8,10 @@ import {
|
|
|
8
8
|
NULLIFIER_TREE_HEIGHT,
|
|
9
9
|
PUBLIC_DATA_TREE_HEIGHT,
|
|
10
10
|
} from '@aztec/constants';
|
|
11
|
-
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
11
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
12
12
|
import { NativeWorldState as BaseNativeWorldState, MsgpackChannel } from '@aztec/native';
|
|
13
13
|
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
14
|
-
import type
|
|
14
|
+
import { EMPTY_GENESIS_DATA, type GenesisData } from '@aztec/stdlib/world-state';
|
|
15
15
|
|
|
16
16
|
import assert from 'assert';
|
|
17
17
|
import { cpus } from 'os';
|
|
@@ -55,17 +55,34 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
55
55
|
constructor(
|
|
56
56
|
private readonly dataDir: string,
|
|
57
57
|
private readonly wsTreeMapSizes: WorldStateTreeMapSizes,
|
|
58
|
-
private readonly
|
|
58
|
+
private readonly genesis: GenesisData = EMPTY_GENESIS_DATA,
|
|
59
59
|
private readonly instrumentation: WorldStateInstrumentation,
|
|
60
|
-
|
|
60
|
+
bindings?: LoggerBindings,
|
|
61
|
+
private readonly log: Logger = createLogger('world-state:database', bindings),
|
|
62
|
+
private readonly ephemeral: boolean = false,
|
|
61
63
|
) {
|
|
62
64
|
const threads = Math.min(cpus().length, MAX_WORLD_STATE_THREADS);
|
|
63
65
|
log.info(
|
|
64
66
|
`Creating world state data store at directory ${dataDir} with map sizes ${JSON.stringify(
|
|
65
67
|
wsTreeMapSizes,
|
|
66
|
-
)} and ${threads} threads.`,
|
|
68
|
+
)} and ${threads} threads (ephemeral=${ephemeral}).`,
|
|
67
69
|
);
|
|
68
|
-
const prefilledPublicDataBufferArray = prefilledPublicData.map(d => [
|
|
70
|
+
const prefilledPublicDataBufferArray = genesis.prefilledPublicData.map(d => [
|
|
71
|
+
d.slot.toBuffer(),
|
|
72
|
+
d.value.toBuffer(),
|
|
73
|
+
]);
|
|
74
|
+
// Nullifiers to pre-insert into the genesis nullifier tree (empty by default, so production genesis roots are
|
|
75
|
+
// unchanged). The native indexed nullifier tree requires its prefilled leaves to be unique and strictly
|
|
76
|
+
// increasing, so we enforce that here before handing them over rather than failing deep inside the C++ tree
|
|
77
|
+
// construction.
|
|
78
|
+
const prefilledNullifiers = genesis.prefilledNullifiers ?? [];
|
|
79
|
+
for (let i = 1; i < prefilledNullifiers.length; i++) {
|
|
80
|
+
assert(
|
|
81
|
+
prefilledNullifiers[i].toBigInt() > prefilledNullifiers[i - 1].toBigInt(),
|
|
82
|
+
'Prefilled genesis nullifiers must be unique and strictly increasing',
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
const prefilledNullifiersBufferArray = prefilledNullifiers.map(n => n.toBuffer());
|
|
69
86
|
const ws = new BaseNativeWorldState(
|
|
70
87
|
dataDir,
|
|
71
88
|
{
|
|
@@ -80,7 +97,9 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
80
97
|
[MerkleTreeId.PUBLIC_DATA_TREE]: 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
|
|
81
98
|
},
|
|
82
99
|
prefilledPublicDataBufferArray,
|
|
83
|
-
|
|
100
|
+
prefilledNullifiersBufferArray,
|
|
101
|
+
DomainSeparator.BLOCK_HEADER_HASH,
|
|
102
|
+
Number(genesis.genesisTimestamp),
|
|
84
103
|
{
|
|
85
104
|
[MerkleTreeId.NULLIFIER_TREE]: wsTreeMapSizes.nullifierTreeMapSizeKb,
|
|
86
105
|
[MerkleTreeId.NOTE_HASH_TREE]: wsTreeMapSizes.noteHashTreeMapSizeKb,
|
|
@@ -89,6 +108,7 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
89
108
|
[MerkleTreeId.ARCHIVE]: wsTreeMapSizes.archiveTreeMapSizeKb,
|
|
90
109
|
},
|
|
91
110
|
threads,
|
|
111
|
+
ephemeral,
|
|
92
112
|
);
|
|
93
113
|
this.instance = new MsgpackChannel(ws);
|
|
94
114
|
// Manually create the queue for the canonical fork
|
|
@@ -103,9 +123,11 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
103
123
|
return new NativeWorldState(
|
|
104
124
|
this.dataDir,
|
|
105
125
|
this.wsTreeMapSizes,
|
|
106
|
-
this.
|
|
126
|
+
this.genesis,
|
|
107
127
|
this.instrumentation,
|
|
128
|
+
this.log.getBindings(),
|
|
108
129
|
this.log,
|
|
130
|
+
this.ephemeral,
|
|
109
131
|
);
|
|
110
132
|
}
|
|
111
133
|
|
|
@@ -178,30 +200,37 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
178
200
|
this.queues.set(forkId, requestQueue);
|
|
179
201
|
}
|
|
180
202
|
|
|
181
|
-
// Enqueue the request and wait for the response
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
+
// Enqueue the request and wait for the response. The per-fork queue is cleaned up in `finally` even on
|
|
204
|
+
// error, so the JS-side queues map cannot outlive the native fork (e.g. when the native fork was already
|
|
205
|
+
// destroyed by an unwind/historical-prune and DELETE_FORK rejects with "Fork not found").
|
|
206
|
+
let shouldDeleteForkQueue = false;
|
|
207
|
+
try {
|
|
208
|
+
const response = await requestQueue.execute(
|
|
209
|
+
async () => {
|
|
210
|
+
assert.notEqual(messageType, WorldStateMessageType.CLOSE, 'Use close() to close the native instance');
|
|
211
|
+
assert.equal(this.open, true, 'Native instance is closed');
|
|
212
|
+
let response: WorldStateResponse[T];
|
|
213
|
+
try {
|
|
214
|
+
response = await this._sendMessage(messageType, body);
|
|
215
|
+
} catch (error: any) {
|
|
216
|
+
errorHandler(error.message);
|
|
217
|
+
throw error;
|
|
218
|
+
}
|
|
219
|
+
return responseHandler(response);
|
|
220
|
+
},
|
|
221
|
+
messageType,
|
|
222
|
+
committedOnly,
|
|
223
|
+
);
|
|
224
|
+
return response;
|
|
225
|
+
} catch (err: any) {
|
|
226
|
+
shouldDeleteForkQueue = forkId !== 0 && err?.message === 'Fork not found';
|
|
227
|
+
throw err;
|
|
228
|
+
} finally {
|
|
229
|
+
if (messageType === WorldStateMessageType.DELETE_FORK || shouldDeleteForkQueue) {
|
|
230
|
+
await requestQueue.stop();
|
|
231
|
+
this.queues.delete(forkId);
|
|
232
|
+
}
|
|
203
233
|
}
|
|
204
|
-
return response;
|
|
205
234
|
}
|
|
206
235
|
|
|
207
236
|
/**
|
|
@@ -96,7 +96,7 @@ export class WorldStateOpsQueue {
|
|
|
96
96
|
// then send the request immediately
|
|
97
97
|
// If a mutating request is in flight then we must wait
|
|
98
98
|
// If a mutating request is not in flight but something is queued then it must be a mutating request
|
|
99
|
-
if (this.inFlightMutatingCount
|
|
99
|
+
if (this.inFlightMutatingCount === 0 && this.requests.length === 0) {
|
|
100
100
|
this.sendEnqueuedRequest(op);
|
|
101
101
|
} else {
|
|
102
102
|
this.requests.push(op);
|
|
@@ -122,7 +122,7 @@ export class WorldStateOpsQueue {
|
|
|
122
122
|
--this.inFlightCount;
|
|
123
123
|
|
|
124
124
|
// If there are still requests in flight then do nothing further
|
|
125
|
-
if (this.inFlightCount
|
|
125
|
+
if (this.inFlightCount !== 0) {
|
|
126
126
|
return;
|
|
127
127
|
}
|
|
128
128
|
|
|
@@ -134,7 +134,7 @@ export class WorldStateOpsQueue {
|
|
|
134
134
|
while (this.requests.length > 0) {
|
|
135
135
|
const next = this.requests[0];
|
|
136
136
|
if (next.mutating) {
|
|
137
|
-
if (this.inFlightCount
|
|
137
|
+
if (this.inFlightCount === 0) {
|
|
138
138
|
// send the mutating request
|
|
139
139
|
this.requests.shift();
|
|
140
140
|
this.sendEnqueuedRequest(next);
|
|
@@ -149,7 +149,7 @@ export class WorldStateOpsQueue {
|
|
|
149
149
|
}
|
|
150
150
|
|
|
151
151
|
// If the queue is empty, there is nothing in flight and we have been told to stop, then resolve the stop promise
|
|
152
|
-
if (this.inFlightCount
|
|
152
|
+
if (this.inFlightCount === 0 && this.stopResolve !== undefined) {
|
|
153
153
|
this.stopResolve();
|
|
154
154
|
}
|
|
155
155
|
}
|
|
@@ -182,7 +182,7 @@ export class WorldStateOpsQueue {
|
|
|
182
182
|
});
|
|
183
183
|
|
|
184
184
|
// If no outstanding requests then immediately resolve the promise
|
|
185
|
-
if (this.requests.length
|
|
185
|
+
if (this.requests.length === 0 && this.inFlightCount === 0 && this.stopResolve !== undefined) {
|
|
186
186
|
this.stopResolve();
|
|
187
187
|
}
|
|
188
188
|
return this.stopPromise;
|