@aztec/world-state 0.0.1-commit.24de95ac → 0.0.1-commit.2606882
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/index.d.ts +1 -1
- package/dest/instrumentation/instrumentation.d.ts +1 -1
- package/dest/instrumentation/instrumentation.d.ts.map +1 -1
- package/dest/instrumentation/instrumentation.js +17 -41
- package/dest/native/bench_metrics.d.ts +1 -1
- package/dest/native/bench_metrics.d.ts.map +1 -1
- package/dest/native/fork_checkpoint.d.ts +7 -1
- package/dest/native/fork_checkpoint.d.ts.map +1 -1
- package/dest/native/fork_checkpoint.js +15 -3
- package/dest/native/index.d.ts +1 -1
- package/dest/native/merkle_trees_facade.d.ts +19 -10
- package/dest/native/merkle_trees_facade.d.ts.map +1 -1
- package/dest/native/merkle_trees_facade.js +68 -17
- package/dest/native/message.d.ts +24 -15
- package/dest/native/message.d.ts.map +1 -1
- package/dest/native/message.js +14 -13
- package/dest/native/native_world_state.d.ts +42 -13
- package/dest/native/native_world_state.d.ts.map +1 -1
- package/dest/native/native_world_state.js +93 -35
- package/dest/native/native_world_state_instance.d.ts +15 -5
- package/dest/native/native_world_state_instance.d.ts.map +1 -1
- package/dest/native/native_world_state_instance.js +54 -26
- package/dest/native/world_state_ops_queue.d.ts +1 -1
- package/dest/native/world_state_ops_queue.d.ts.map +1 -1
- package/dest/native/world_state_ops_queue.js +5 -5
- package/dest/synchronizer/config.d.ts +3 -5
- package/dest/synchronizer/config.d.ts.map +1 -1
- package/dest/synchronizer/config.js +15 -18
- package/dest/synchronizer/errors.d.ts +1 -1
- package/dest/synchronizer/errors.d.ts.map +1 -1
- package/dest/synchronizer/factory.d.ts +6 -5
- package/dest/synchronizer/factory.d.ts.map +1 -1
- package/dest/synchronizer/factory.js +7 -6
- package/dest/synchronizer/index.d.ts +1 -1
- package/dest/synchronizer/server_world_state_synchronizer.d.ts +11 -28
- package/dest/synchronizer/server_world_state_synchronizer.d.ts.map +1 -1
- package/dest/synchronizer/server_world_state_synchronizer.js +165 -81
- package/dest/test/index.d.ts +1 -1
- package/dest/test/utils.d.ts +12 -5
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +54 -50
- package/dest/testing.d.ts +5 -4
- package/dest/testing.d.ts.map +1 -1
- package/dest/testing.js +11 -7
- package/dest/world-state-db/index.d.ts +1 -1
- package/dest/world-state-db/merkle_tree_db.d.ts +9 -19
- package/dest/world-state-db/merkle_tree_db.d.ts.map +1 -1
- package/package.json +15 -13
- package/src/instrumentation/instrumentation.ts +17 -41
- package/src/native/fork_checkpoint.ts +19 -3
- package/src/native/merkle_trees_facade.ts +79 -20
- package/src/native/message.ts +37 -26
- package/src/native/native_world_state.ts +136 -48
- package/src/native/native_world_state_instance.ts +72 -32
- package/src/native/world_state_ops_queue.ts +5 -5
- package/src/synchronizer/config.ts +16 -23
- package/src/synchronizer/factory.ts +19 -11
- package/src/synchronizer/server_world_state_synchronizer.ts +181 -106
- package/src/test/utils.ts +87 -92
- package/src/testing.ts +9 -10
- package/src/world-state-db/merkle_tree_db.ts +12 -19
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIERS_PER_TX, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
|
|
2
|
+
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
2
3
|
import { fromEntries, padArrayEnd } from '@aztec/foundation/collection';
|
|
4
|
+
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
3
5
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
4
|
-
import { Fr } from '@aztec/foundation/fields';
|
|
5
6
|
import { tryRmDir } from '@aztec/foundation/fs';
|
|
6
|
-
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
7
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
7
8
|
import type { L2Block } from '@aztec/stdlib/block';
|
|
8
|
-
import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
|
|
9
|
+
import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
|
|
9
10
|
import type {
|
|
10
11
|
IndexedTreeId,
|
|
11
12
|
MerkleTreeReadOperations,
|
|
@@ -13,8 +14,8 @@ import type {
|
|
|
13
14
|
} from '@aztec/stdlib/interfaces/server';
|
|
14
15
|
import type { SnapshotDataKeys } from '@aztec/stdlib/snapshots';
|
|
15
16
|
import { MerkleTreeId, NullifierLeaf, type NullifierLeafPreimage, PublicDataTreeLeaf } from '@aztec/stdlib/trees';
|
|
16
|
-
import { BlockHeader, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
|
|
17
|
-
import { WorldStateRevision } from '@aztec/stdlib/world-state';
|
|
17
|
+
import { BlockHeader, GlobalVariables, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
|
|
18
|
+
import { EMPTY_GENESIS_DATA, type GenesisData, WorldStateRevision } from '@aztec/stdlib/world-state';
|
|
18
19
|
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
19
20
|
|
|
20
21
|
import assert from 'assert/strict';
|
|
@@ -43,6 +44,29 @@ export const WORLD_STATE_DB_VERSION = 2; // The initial version
|
|
|
43
44
|
|
|
44
45
|
export const WORLD_STATE_DIR = 'world_state';
|
|
45
46
|
|
|
47
|
+
const DEFAULT_TMP_TREE_MAP_SIZE_KB = 10 * 1024 * 1024;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Sets up a fresh `mkdtemp` directory + default `WorldStateTreeMapSizes` shared by both
|
|
51
|
+
* the `.tmp` (fsync-on) and `.ephemeral` (fsync-off) factories. Returns the raw tmpdir,
|
|
52
|
+
* the tree map sizes, and the package logger.
|
|
53
|
+
*/
|
|
54
|
+
async function createTmpWorldStateDir(
|
|
55
|
+
bindings?: LoggerBindings,
|
|
56
|
+
): Promise<{ dataDir: string; wsTreeMapSizes: WorldStateTreeMapSizes; log: Logger }> {
|
|
57
|
+
const log = createLogger('world-state:database', bindings);
|
|
58
|
+
const dataDir = await mkdtemp(join(tmpdir(), 'aztec-world-state-'));
|
|
59
|
+
const wsTreeMapSizes: WorldStateTreeMapSizes = {
|
|
60
|
+
archiveTreeMapSizeKb: DEFAULT_TMP_TREE_MAP_SIZE_KB,
|
|
61
|
+
nullifierTreeMapSizeKb: DEFAULT_TMP_TREE_MAP_SIZE_KB,
|
|
62
|
+
noteHashTreeMapSizeKb: DEFAULT_TMP_TREE_MAP_SIZE_KB,
|
|
63
|
+
messageTreeMapSizeKb: DEFAULT_TMP_TREE_MAP_SIZE_KB,
|
|
64
|
+
publicDataTreeMapSizeKb: DEFAULT_TMP_TREE_MAP_SIZE_KB,
|
|
65
|
+
};
|
|
66
|
+
log.debug(`Created temporary world state database at: ${dataDir} (map size ${DEFAULT_TMP_TREE_MAP_SIZE_KB} KB)`);
|
|
67
|
+
return { dataDir, wsTreeMapSizes, log };
|
|
68
|
+
}
|
|
69
|
+
|
|
46
70
|
export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
47
71
|
protected initialHeader: BlockHeader | undefined;
|
|
48
72
|
// This is read heavily and only changes when data is persisted, so we cache it
|
|
@@ -51,32 +75,47 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
51
75
|
protected constructor(
|
|
52
76
|
protected instance: NativeWorldState,
|
|
53
77
|
protected readonly worldStateInstrumentation: WorldStateInstrumentation,
|
|
54
|
-
protected readonly log: Logger
|
|
78
|
+
protected readonly log: Logger,
|
|
79
|
+
private readonly genesis: GenesisData = EMPTY_GENESIS_DATA,
|
|
55
80
|
private readonly cleanup = () => Promise.resolve(),
|
|
56
81
|
) {}
|
|
57
82
|
|
|
83
|
+
/**
|
|
84
|
+
* Opens a persistent world state at `dataDir`. Goes through `DatabaseVersionManager` so the
|
|
85
|
+
* caller's rollup address is bound to the on-disk schema and incompatible versions surface
|
|
86
|
+
* loudly. The LMDB envs commit with full fsync.
|
|
87
|
+
*/
|
|
58
88
|
static async new(
|
|
59
89
|
rollupAddress: EthAddress,
|
|
60
90
|
dataDir: string,
|
|
61
91
|
wsTreeMapSizes: WorldStateTreeMapSizes,
|
|
62
|
-
|
|
92
|
+
genesis: GenesisData = EMPTY_GENESIS_DATA,
|
|
63
93
|
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
64
|
-
|
|
94
|
+
bindings?: LoggerBindings,
|
|
65
95
|
cleanup = () => Promise.resolve(),
|
|
66
96
|
): Promise<NativeWorldStateService> {
|
|
97
|
+
const log = createLogger('world-state:database', bindings);
|
|
67
98
|
const worldStateDirectory = join(dataDir, WORLD_STATE_DIR);
|
|
68
|
-
// Create a version manager to handle versioning
|
|
69
99
|
const versionManager = new DatabaseVersionManager({
|
|
70
100
|
schemaVersion: WORLD_STATE_DB_VERSION,
|
|
71
101
|
rollupAddress,
|
|
72
102
|
dataDirectory: worldStateDirectory,
|
|
73
|
-
onOpen: (dir: string) =>
|
|
74
|
-
|
|
75
|
-
|
|
103
|
+
onOpen: (dir: string) =>
|
|
104
|
+
Promise.resolve(
|
|
105
|
+
new NativeWorldState(
|
|
106
|
+
dir,
|
|
107
|
+
wsTreeMapSizes,
|
|
108
|
+
genesis,
|
|
109
|
+
instrumentation,
|
|
110
|
+
bindings,
|
|
111
|
+
undefined,
|
|
112
|
+
/*ephemeral=*/ false,
|
|
113
|
+
),
|
|
114
|
+
),
|
|
76
115
|
});
|
|
77
116
|
|
|
78
117
|
const [instance] = await versionManager.open();
|
|
79
|
-
const worldState = new this(instance, instrumentation, log, cleanup);
|
|
118
|
+
const worldState = new this(instance, instrumentation, log, genesis, cleanup);
|
|
80
119
|
try {
|
|
81
120
|
await worldState.init();
|
|
82
121
|
} catch (e) {
|
|
@@ -87,25 +126,23 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
87
126
|
return worldState;
|
|
88
127
|
}
|
|
89
128
|
|
|
129
|
+
/**
|
|
130
|
+
* Opens a world state in a fresh tmpdir with full fsync semantics. Use when you need the
|
|
131
|
+
* on-disk file to remain crash-recoverable (e.g. for snapshot/backup tests) but don't
|
|
132
|
+
* want a persistent dataDir. Pass `cleanupTmpDir=false` to keep the directory after
|
|
133
|
+
* close for inspection.
|
|
134
|
+
*
|
|
135
|
+
* If you don't care about crash-recoverability — i.e. you just want a fast scratch
|
|
136
|
+
* database for tests — use {@link ephemeral} instead.
|
|
137
|
+
*/
|
|
90
138
|
static async tmp(
|
|
91
139
|
rollupAddress = EthAddress.ZERO,
|
|
92
140
|
cleanupTmpDir = true,
|
|
93
|
-
|
|
141
|
+
genesis: GenesisData = EMPTY_GENESIS_DATA,
|
|
94
142
|
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
143
|
+
bindings?: LoggerBindings,
|
|
95
144
|
): Promise<NativeWorldStateService> {
|
|
96
|
-
const log =
|
|
97
|
-
const dataDir = await mkdtemp(join(tmpdir(), 'aztec-world-state-'));
|
|
98
|
-
const dbMapSizeKb = 10 * 1024 * 1024;
|
|
99
|
-
const worldStateTreeMapSizes: WorldStateTreeMapSizes = {
|
|
100
|
-
archiveTreeMapSizeKb: dbMapSizeKb,
|
|
101
|
-
nullifierTreeMapSizeKb: dbMapSizeKb,
|
|
102
|
-
noteHashTreeMapSizeKb: dbMapSizeKb,
|
|
103
|
-
messageTreeMapSizeKb: dbMapSizeKb,
|
|
104
|
-
publicDataTreeMapSizeKb: dbMapSizeKb,
|
|
105
|
-
};
|
|
106
|
-
log.debug(`Created temporary world state database at: ${dataDir} with tree map size: ${dbMapSizeKb}`);
|
|
107
|
-
|
|
108
|
-
// pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
|
|
145
|
+
const { dataDir, wsTreeMapSizes, log } = await createTmpWorldStateDir(bindings);
|
|
109
146
|
const cleanup = async () => {
|
|
110
147
|
if (cleanupTmpDir) {
|
|
111
148
|
await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
|
|
@@ -114,8 +151,45 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
114
151
|
log.debug(`Leaving temporary world state database: ${dataDir}`);
|
|
115
152
|
}
|
|
116
153
|
};
|
|
154
|
+
return this.new(rollupAddress, dataDir, wsTreeMapSizes, genesis, instrumentation, bindings, cleanup);
|
|
155
|
+
}
|
|
117
156
|
|
|
118
|
-
|
|
157
|
+
/**
|
|
158
|
+
* Opens a fully-ephemeral world state. The directory is created in `os.tmpdir()`, the LMDB
|
|
159
|
+
* envs open with `MDB_NOSYNC | MDB_NOMETASYNC` so commits never block on fsync, and the
|
|
160
|
+
* directory is removed on dispose. A crash mid-write leaves the env unrecoverable.
|
|
161
|
+
*
|
|
162
|
+
* For unit tests and other isolated runs. Use {@link tmp} when you need fsync semantics in a
|
|
163
|
+
* tmp dir, and {@link new} for a persistent store. Skips {@link DatabaseVersionManager} —
|
|
164
|
+
* there is no on-disk schema to bind to and no rollup address is taken.
|
|
165
|
+
*/
|
|
166
|
+
static async ephemeral(
|
|
167
|
+
genesis: GenesisData = EMPTY_GENESIS_DATA,
|
|
168
|
+
instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
|
|
169
|
+
bindings?: LoggerBindings,
|
|
170
|
+
): Promise<NativeWorldStateService> {
|
|
171
|
+
const { dataDir, wsTreeMapSizes, log } = await createTmpWorldStateDir(bindings);
|
|
172
|
+
const cleanup = async () => {
|
|
173
|
+
await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
|
|
174
|
+
log.debug(`Deleted ephemeral world state database: ${dataDir}`);
|
|
175
|
+
};
|
|
176
|
+
const instance = new NativeWorldState(
|
|
177
|
+
join(dataDir, WORLD_STATE_DIR),
|
|
178
|
+
wsTreeMapSizes,
|
|
179
|
+
genesis,
|
|
180
|
+
instrumentation,
|
|
181
|
+
bindings,
|
|
182
|
+
undefined,
|
|
183
|
+
/*ephemeral=*/ true,
|
|
184
|
+
);
|
|
185
|
+
const worldState = new this(instance, instrumentation, log, genesis, cleanup);
|
|
186
|
+
try {
|
|
187
|
+
await worldState.init();
|
|
188
|
+
} catch (e) {
|
|
189
|
+
log.error(`Error initializing ephemeral world state: ${e}`);
|
|
190
|
+
throw e;
|
|
191
|
+
}
|
|
192
|
+
return worldState;
|
|
119
193
|
}
|
|
120
194
|
|
|
121
195
|
protected async init() {
|
|
@@ -150,7 +224,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
150
224
|
return new MerkleTreesFacade(this.instance, this.initialHeader!, WorldStateRevision.empty());
|
|
151
225
|
}
|
|
152
226
|
|
|
153
|
-
public getSnapshot(blockNumber:
|
|
227
|
+
public getSnapshot(blockNumber: BlockNumber): MerkleTreeReadOperations {
|
|
154
228
|
return new MerkleTreesFacade(
|
|
155
229
|
this.instance,
|
|
156
230
|
this.initialHeader!,
|
|
@@ -158,16 +232,24 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
158
232
|
);
|
|
159
233
|
}
|
|
160
234
|
|
|
161
|
-
public async fork(
|
|
235
|
+
public async fork(
|
|
236
|
+
blockNumber?: BlockNumber,
|
|
237
|
+
opts: { closeDelayMs?: number } = {},
|
|
238
|
+
): Promise<MerkleTreeWriteOperations> {
|
|
162
239
|
const resp = await this.instance.call(WorldStateMessageType.CREATE_FORK, {
|
|
163
240
|
latest: blockNumber === undefined,
|
|
164
|
-
blockNumber: blockNumber ??
|
|
241
|
+
blockNumber: blockNumber ?? BlockNumber.ZERO,
|
|
165
242
|
canonical: true,
|
|
166
243
|
});
|
|
167
244
|
return new MerkleTreesForkFacade(
|
|
168
245
|
this.instance,
|
|
169
246
|
this.initialHeader!,
|
|
170
|
-
new WorldStateRevision(
|
|
247
|
+
new WorldStateRevision(
|
|
248
|
+
/*forkId=*/ resp.forkId,
|
|
249
|
+
/* blockNumber=*/ WorldStateRevision.LATEST,
|
|
250
|
+
/* includeUncommitted=*/ true,
|
|
251
|
+
),
|
|
252
|
+
opts,
|
|
171
253
|
);
|
|
172
254
|
}
|
|
173
255
|
|
|
@@ -175,21 +257,24 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
175
257
|
return this.initialHeader!;
|
|
176
258
|
}
|
|
177
259
|
|
|
178
|
-
public async handleL2BlockAndMessages(
|
|
179
|
-
l2Block
|
|
180
|
-
l1ToL2Messages
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
);
|
|
260
|
+
public async handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise<WorldStateStatusFull> {
|
|
261
|
+
const isFirstBlock = l2Block.indexWithinCheckpoint === 0;
|
|
262
|
+
if (!isFirstBlock && l1ToL2Messages.length > 0) {
|
|
263
|
+
throw new Error(
|
|
264
|
+
`L1 to L2 messages must be empty for non-first blocks, but got ${l1ToL2Messages.length} messages for block ${l2Block.number}.`,
|
|
265
|
+
);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// We have to pad the given l1 to l2 messages, and the note hashes and nullifiers within tx effects, because that's
|
|
269
|
+
// how the trees are built by circuits.
|
|
189
270
|
const paddedL1ToL2Messages = isFirstBlock
|
|
190
271
|
? padArrayEnd<Fr, number>(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP)
|
|
191
272
|
: [];
|
|
192
273
|
|
|
274
|
+
const paddedNoteHashes = l2Block.body.txEffects.flatMap(txEffect =>
|
|
275
|
+
padArrayEnd(txEffect.noteHashes, Fr.ZERO, MAX_NOTE_HASHES_PER_TX),
|
|
276
|
+
);
|
|
277
|
+
|
|
193
278
|
const paddedNullifiers = l2Block.body.txEffects
|
|
194
279
|
.flatMap(txEffect => padArrayEnd(txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX))
|
|
195
280
|
.map(nullifier => new NullifierLeaf(nullifier));
|
|
@@ -208,7 +293,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
208
293
|
WorldStateMessageType.SYNC_BLOCK,
|
|
209
294
|
{
|
|
210
295
|
blockNumber: l2Block.number,
|
|
211
|
-
blockHeaderHash: await l2Block.
|
|
296
|
+
blockHeaderHash: (await l2Block.hash()).toBuffer(),
|
|
212
297
|
paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf),
|
|
213
298
|
paddedNoteHashes: paddedNoteHashes.map(serializeLeaf),
|
|
214
299
|
paddedNullifiers: paddedNullifiers.map(serializeLeaf),
|
|
@@ -232,7 +317,10 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
232
317
|
|
|
233
318
|
private async buildInitialHeader(): Promise<BlockHeader> {
|
|
234
319
|
const state = await this.getInitialStateReference();
|
|
235
|
-
return BlockHeader.empty({
|
|
320
|
+
return BlockHeader.empty({
|
|
321
|
+
state,
|
|
322
|
+
globalVariables: GlobalVariables.empty({ timestamp: this.genesis.genesisTimestamp }),
|
|
323
|
+
});
|
|
236
324
|
}
|
|
237
325
|
|
|
238
326
|
private sanitizeAndCacheSummaryFromFull(response: WorldStateStatusFull) {
|
|
@@ -256,7 +344,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
256
344
|
* @param toBlockNumber The block number that is now the tip of the finalized chain
|
|
257
345
|
* @returns The new WorldStateStatus
|
|
258
346
|
*/
|
|
259
|
-
public async setFinalized(toBlockNumber:
|
|
347
|
+
public async setFinalized(toBlockNumber: BlockNumber) {
|
|
260
348
|
try {
|
|
261
349
|
await this.instance.call(
|
|
262
350
|
WorldStateMessageType.FINALIZE_BLOCKS,
|
|
@@ -279,7 +367,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
279
367
|
* @param toBlockNumber The block number of the new oldest historical block
|
|
280
368
|
* @returns The new WorldStateStatus
|
|
281
369
|
*/
|
|
282
|
-
public async removeHistoricalBlocks(toBlockNumber:
|
|
370
|
+
public async removeHistoricalBlocks(toBlockNumber: BlockNumber) {
|
|
283
371
|
try {
|
|
284
372
|
return await this.instance.call(
|
|
285
373
|
WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS,
|
|
@@ -301,7 +389,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
|
|
|
301
389
|
* @param toBlockNumber The block number of the new tip of the pending chain,
|
|
302
390
|
* @returns The new WorldStateStatus
|
|
303
391
|
*/
|
|
304
|
-
public async unwindBlocks(toBlockNumber:
|
|
392
|
+
public async unwindBlocks(toBlockNumber: BlockNumber) {
|
|
305
393
|
try {
|
|
306
394
|
return await this.instance.call(
|
|
307
395
|
WorldStateMessageType.UNWIND_BLOCKS,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
ARCHIVE_HEIGHT,
|
|
3
|
-
|
|
3
|
+
DomainSeparator,
|
|
4
4
|
L1_TO_L2_MSG_TREE_HEIGHT,
|
|
5
5
|
MAX_NULLIFIERS_PER_TX,
|
|
6
6
|
MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
|
|
@@ -8,10 +8,10 @@ import {
|
|
|
8
8
|
NULLIFIER_TREE_HEIGHT,
|
|
9
9
|
PUBLIC_DATA_TREE_HEIGHT,
|
|
10
10
|
} from '@aztec/constants';
|
|
11
|
-
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
11
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
12
12
|
import { NativeWorldState as BaseNativeWorldState, MsgpackChannel } from '@aztec/native';
|
|
13
13
|
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
14
|
-
import type
|
|
14
|
+
import { EMPTY_GENESIS_DATA, type GenesisData } from '@aztec/stdlib/world-state';
|
|
15
15
|
|
|
16
16
|
import assert from 'assert';
|
|
17
17
|
import { cpus } from 'os';
|
|
@@ -36,6 +36,8 @@ export interface NativeWorldStateInstance {
|
|
|
36
36
|
messageType: T,
|
|
37
37
|
body: WorldStateRequest[T] & WorldStateRequestCategories,
|
|
38
38
|
): Promise<WorldStateResponse[T]>;
|
|
39
|
+
// TODO(dbanks12): this returns any type, but we should strongly type it
|
|
40
|
+
getHandle(): any;
|
|
39
41
|
}
|
|
40
42
|
|
|
41
43
|
/**
|
|
@@ -53,17 +55,22 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
53
55
|
constructor(
|
|
54
56
|
private readonly dataDir: string,
|
|
55
57
|
private readonly wsTreeMapSizes: WorldStateTreeMapSizes,
|
|
56
|
-
private readonly
|
|
58
|
+
private readonly genesis: GenesisData = EMPTY_GENESIS_DATA,
|
|
57
59
|
private readonly instrumentation: WorldStateInstrumentation,
|
|
58
|
-
|
|
60
|
+
bindings?: LoggerBindings,
|
|
61
|
+
private readonly log: Logger = createLogger('world-state:database', bindings),
|
|
62
|
+
private readonly ephemeral: boolean = false,
|
|
59
63
|
) {
|
|
60
64
|
const threads = Math.min(cpus().length, MAX_WORLD_STATE_THREADS);
|
|
61
65
|
log.info(
|
|
62
66
|
`Creating world state data store at directory ${dataDir} with map sizes ${JSON.stringify(
|
|
63
67
|
wsTreeMapSizes,
|
|
64
|
-
)} and ${threads} threads.`,
|
|
68
|
+
)} and ${threads} threads (ephemeral=${ephemeral}).`,
|
|
65
69
|
);
|
|
66
|
-
const prefilledPublicDataBufferArray = prefilledPublicData.map(d => [
|
|
70
|
+
const prefilledPublicDataBufferArray = genesis.prefilledPublicData.map(d => [
|
|
71
|
+
d.slot.toBuffer(),
|
|
72
|
+
d.value.toBuffer(),
|
|
73
|
+
]);
|
|
67
74
|
const ws = new BaseNativeWorldState(
|
|
68
75
|
dataDir,
|
|
69
76
|
{
|
|
@@ -78,7 +85,8 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
78
85
|
[MerkleTreeId.PUBLIC_DATA_TREE]: 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
|
|
79
86
|
},
|
|
80
87
|
prefilledPublicDataBufferArray,
|
|
81
|
-
|
|
88
|
+
DomainSeparator.BLOCK_HEADER_HASH,
|
|
89
|
+
Number(genesis.genesisTimestamp),
|
|
82
90
|
{
|
|
83
91
|
[MerkleTreeId.NULLIFIER_TREE]: wsTreeMapSizes.nullifierTreeMapSizeKb,
|
|
84
92
|
[MerkleTreeId.NOTE_HASH_TREE]: wsTreeMapSizes.noteHashTreeMapSizeKb,
|
|
@@ -87,6 +95,7 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
87
95
|
[MerkleTreeId.ARCHIVE]: wsTreeMapSizes.archiveTreeMapSizeKb,
|
|
88
96
|
},
|
|
89
97
|
threads,
|
|
98
|
+
ephemeral,
|
|
90
99
|
);
|
|
91
100
|
this.instance = new MsgpackChannel(ws);
|
|
92
101
|
// Manually create the queue for the canonical fork
|
|
@@ -101,12 +110,40 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
101
110
|
return new NativeWorldState(
|
|
102
111
|
this.dataDir,
|
|
103
112
|
this.wsTreeMapSizes,
|
|
104
|
-
this.
|
|
113
|
+
this.genesis,
|
|
105
114
|
this.instrumentation,
|
|
115
|
+
this.log.getBindings(),
|
|
106
116
|
this.log,
|
|
117
|
+
this.ephemeral,
|
|
107
118
|
);
|
|
108
119
|
}
|
|
109
120
|
|
|
121
|
+
/**
|
|
122
|
+
* Gets the native WorldState handle from the underlying native instance.
|
|
123
|
+
* We call the getHandle() method on the native WorldState to get a NAPI External
|
|
124
|
+
* that wraps the underlying C++ WorldState pointer.
|
|
125
|
+
* @returns The NAPI External handle to the native WorldState instance,since
|
|
126
|
+
* the NAPI external type is opaque, we return any (we could also use an opaque symbol type)
|
|
127
|
+
*/
|
|
128
|
+
public getHandle(): any {
|
|
129
|
+
const worldStateWrapper = (this.instance as any).dest;
|
|
130
|
+
|
|
131
|
+
if (!worldStateWrapper) {
|
|
132
|
+
throw new Error('No WorldStateWrapper found');
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (typeof worldStateWrapper.getHandle !== 'function') {
|
|
136
|
+
throw new Error('WorldStateWrapper does not have getHandle method');
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Call getHandle() to get the NAPI External
|
|
140
|
+
try {
|
|
141
|
+
return worldStateWrapper.getHandle();
|
|
142
|
+
} catch (error) {
|
|
143
|
+
this.log.error('Failed to get native WorldState handle', error);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
110
147
|
/**
|
|
111
148
|
* Sends a message to the native instance and returns the response.
|
|
112
149
|
* @param messageType - The type of message to send
|
|
@@ -150,30 +187,33 @@ export class NativeWorldState implements NativeWorldStateInstance {
|
|
|
150
187
|
this.queues.set(forkId, requestQueue);
|
|
151
188
|
}
|
|
152
189
|
|
|
153
|
-
// Enqueue the request and wait for the response
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
190
|
+
// Enqueue the request and wait for the response. The per-fork queue is cleaned up in `finally` even on
|
|
191
|
+
// error, so the JS-side queues map cannot outlive the native fork (e.g. when the native fork was already
|
|
192
|
+
// destroyed by an unwind/historical-prune and DELETE_FORK rejects with "Fork not found").
|
|
193
|
+
try {
|
|
194
|
+
const response = await requestQueue.execute(
|
|
195
|
+
async () => {
|
|
196
|
+
assert.notEqual(messageType, WorldStateMessageType.CLOSE, 'Use close() to close the native instance');
|
|
197
|
+
assert.equal(this.open, true, 'Native instance is closed');
|
|
198
|
+
let response: WorldStateResponse[T];
|
|
199
|
+
try {
|
|
200
|
+
response = await this._sendMessage(messageType, body);
|
|
201
|
+
} catch (error: any) {
|
|
202
|
+
errorHandler(error.message);
|
|
203
|
+
throw error;
|
|
204
|
+
}
|
|
205
|
+
return responseHandler(response);
|
|
206
|
+
},
|
|
207
|
+
messageType,
|
|
208
|
+
committedOnly,
|
|
209
|
+
);
|
|
210
|
+
return response;
|
|
211
|
+
} finally {
|
|
212
|
+
if (messageType === WorldStateMessageType.DELETE_FORK) {
|
|
213
|
+
await requestQueue.stop();
|
|
214
|
+
this.queues.delete(forkId);
|
|
215
|
+
}
|
|
175
216
|
}
|
|
176
|
-
return response;
|
|
177
217
|
}
|
|
178
218
|
|
|
179
219
|
/**
|
|
@@ -96,7 +96,7 @@ export class WorldStateOpsQueue {
|
|
|
96
96
|
// then send the request immediately
|
|
97
97
|
// If a mutating request is in flight then we must wait
|
|
98
98
|
// If a mutating request is not in flight but something is queued then it must be a mutating request
|
|
99
|
-
if (this.inFlightMutatingCount
|
|
99
|
+
if (this.inFlightMutatingCount === 0 && this.requests.length === 0) {
|
|
100
100
|
this.sendEnqueuedRequest(op);
|
|
101
101
|
} else {
|
|
102
102
|
this.requests.push(op);
|
|
@@ -122,7 +122,7 @@ export class WorldStateOpsQueue {
|
|
|
122
122
|
--this.inFlightCount;
|
|
123
123
|
|
|
124
124
|
// If there are still requests in flight then do nothing further
|
|
125
|
-
if (this.inFlightCount
|
|
125
|
+
if (this.inFlightCount !== 0) {
|
|
126
126
|
return;
|
|
127
127
|
}
|
|
128
128
|
|
|
@@ -134,7 +134,7 @@ export class WorldStateOpsQueue {
|
|
|
134
134
|
while (this.requests.length > 0) {
|
|
135
135
|
const next = this.requests[0];
|
|
136
136
|
if (next.mutating) {
|
|
137
|
-
if (this.inFlightCount
|
|
137
|
+
if (this.inFlightCount === 0) {
|
|
138
138
|
// send the mutating request
|
|
139
139
|
this.requests.shift();
|
|
140
140
|
this.sendEnqueuedRequest(next);
|
|
@@ -149,7 +149,7 @@ export class WorldStateOpsQueue {
|
|
|
149
149
|
}
|
|
150
150
|
|
|
151
151
|
// If the queue is empty, there is nothing in flight and we have been told to stop, then resolve the stop promise
|
|
152
|
-
if (this.inFlightCount
|
|
152
|
+
if (this.inFlightCount === 0 && this.stopResolve !== undefined) {
|
|
153
153
|
this.stopResolve();
|
|
154
154
|
}
|
|
155
155
|
}
|
|
@@ -182,7 +182,7 @@ export class WorldStateOpsQueue {
|
|
|
182
182
|
});
|
|
183
183
|
|
|
184
184
|
// If no outstanding requests then immediately resolve the promise
|
|
185
|
-
if (this.requests.length
|
|
185
|
+
if (this.requests.length === 0 && this.inFlightCount === 0 && this.stopResolve !== undefined) {
|
|
186
186
|
this.stopResolve();
|
|
187
187
|
}
|
|
188
188
|
return this.stopPromise;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
type ConfigMappingsType,
|
|
3
|
-
booleanConfigHelper,
|
|
4
3
|
getConfigFromMappings,
|
|
5
4
|
numberConfigHelper,
|
|
5
|
+
optionalNumberConfigHelper,
|
|
6
6
|
} from '@aztec/foundation/config';
|
|
7
7
|
|
|
8
8
|
/** World State synchronizer configuration values. */
|
|
@@ -10,9 +10,6 @@ export interface WorldStateConfig {
|
|
|
10
10
|
/** The frequency in which to check. */
|
|
11
11
|
worldStateBlockCheckIntervalMS: number;
|
|
12
12
|
|
|
13
|
-
/** Whether to follow only the proven chain. */
|
|
14
|
-
worldStateProvenBlocksOnly: boolean;
|
|
15
|
-
|
|
16
13
|
/** Size of the batch for each get-blocks request from the synchronizer to the archiver. */
|
|
17
14
|
worldStateBlockRequestBatchSize?: number;
|
|
18
15
|
|
|
@@ -37,59 +34,53 @@ export interface WorldStateConfig {
|
|
|
37
34
|
/** Optional directory for the world state DB, if unspecified will default to the general data directory */
|
|
38
35
|
worldStateDataDirectory?: string;
|
|
39
36
|
|
|
40
|
-
/** The number of historic blocks to maintain */
|
|
41
|
-
|
|
37
|
+
/** The number of historic checkpoints worth of blocks to maintain */
|
|
38
|
+
worldStateCheckpointHistory: number;
|
|
42
39
|
}
|
|
43
40
|
|
|
44
41
|
export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
|
|
45
42
|
worldStateBlockCheckIntervalMS: {
|
|
46
43
|
env: 'WS_BLOCK_CHECK_INTERVAL_MS',
|
|
47
|
-
|
|
48
|
-
defaultValue: 100,
|
|
44
|
+
...numberConfigHelper(100),
|
|
49
45
|
description: 'The frequency in which to check.',
|
|
50
46
|
},
|
|
51
|
-
worldStateProvenBlocksOnly: {
|
|
52
|
-
env: 'WS_PROVEN_BLOCKS_ONLY',
|
|
53
|
-
description: 'Whether to follow only the proven chain.',
|
|
54
|
-
...booleanConfigHelper(),
|
|
55
|
-
},
|
|
56
47
|
worldStateBlockRequestBatchSize: {
|
|
57
48
|
env: 'WS_BLOCK_REQUEST_BATCH_SIZE',
|
|
58
|
-
|
|
49
|
+
...optionalNumberConfigHelper(),
|
|
59
50
|
description: 'Size of the batch for each get-blocks request from the synchronizer to the archiver.',
|
|
60
51
|
},
|
|
61
52
|
worldStateDbMapSizeKb: {
|
|
62
53
|
env: 'WS_DB_MAP_SIZE_KB',
|
|
63
|
-
|
|
54
|
+
...optionalNumberConfigHelper(),
|
|
64
55
|
description: 'The maximum possible size of the world state DB in KB. Overwrites the general dataStoreMapSizeKb.',
|
|
65
56
|
},
|
|
66
57
|
archiveTreeMapSizeKb: {
|
|
67
58
|
env: 'ARCHIVE_TREE_MAP_SIZE_KB',
|
|
68
|
-
|
|
59
|
+
...optionalNumberConfigHelper(),
|
|
69
60
|
description:
|
|
70
61
|
'The maximum possible size of the world state archive tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
71
62
|
},
|
|
72
63
|
nullifierTreeMapSizeKb: {
|
|
73
64
|
env: 'NULLIFIER_TREE_MAP_SIZE_KB',
|
|
74
|
-
|
|
65
|
+
...optionalNumberConfigHelper(),
|
|
75
66
|
description:
|
|
76
67
|
'The maximum possible size of the world state nullifier tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
77
68
|
},
|
|
78
69
|
noteHashTreeMapSizeKb: {
|
|
79
70
|
env: 'NOTE_HASH_TREE_MAP_SIZE_KB',
|
|
80
|
-
|
|
71
|
+
...optionalNumberConfigHelper(),
|
|
81
72
|
description:
|
|
82
73
|
'The maximum possible size of the world state note hash tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
83
74
|
},
|
|
84
75
|
messageTreeMapSizeKb: {
|
|
85
76
|
env: 'MESSAGE_TREE_MAP_SIZE_KB',
|
|
86
|
-
|
|
77
|
+
...optionalNumberConfigHelper(),
|
|
87
78
|
description:
|
|
88
79
|
'The maximum possible size of the world state message tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
89
80
|
},
|
|
90
81
|
publicDataTreeMapSizeKb: {
|
|
91
82
|
env: 'PUBLIC_DATA_TREE_MAP_SIZE_KB',
|
|
92
|
-
|
|
83
|
+
...optionalNumberConfigHelper(),
|
|
93
84
|
description:
|
|
94
85
|
'The maximum possible size of the world state public data tree in KB. Overwrites the general worldStateDbMapSizeKb.',
|
|
95
86
|
},
|
|
@@ -97,9 +88,11 @@ export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
|
|
|
97
88
|
env: 'WS_DATA_DIRECTORY',
|
|
98
89
|
description: 'Optional directory for the world state database',
|
|
99
90
|
},
|
|
100
|
-
|
|
101
|
-
env: '
|
|
102
|
-
description:
|
|
91
|
+
worldStateCheckpointHistory: {
|
|
92
|
+
env: 'WS_NUM_HISTORIC_CHECKPOINTS',
|
|
93
|
+
description:
|
|
94
|
+
'The number of historic checkpoints worth of blocks to maintain. Values less than 1 mean all history is maintained',
|
|
95
|
+
fallback: ['WS_NUM_HISTORIC_BLOCKS'],
|
|
103
96
|
...numberConfigHelper(64),
|
|
104
97
|
},
|
|
105
98
|
};
|